From e0004e7b0a58fba7239a3379c9b6ede7c2a50bec Mon Sep 17 00:00:00 2001 From: Lakshman Patel Date: Mon, 24 Aug 2026 07:43:46 +0530 Subject: [PATCH] feat(client): OpenAI-compatible image generation and audio transcription Adds provider-agnostic backends for the two well-defined public APIs that give hawk's pluggable MediaEngine / Transcriber seams a concrete default: - ImageClient: POST /v1/images/generations (b64_json or url results), returns raw bytes + provider URLs. - AudioClient: POST /v1/audio/transcriptions (multipart), returns the transcript text. Both are endpoint+credentials configurable so another provider (e.g. xAI image-gen) can replace the default without touching callers. Testable against an httptest server. Tests: 5 new cases (b64 decode, url results, 401 error, multipart form fields + transcript parse, 400 error). Full client suite green. --- client/media.go | 194 +++++++++++++++++++++++++++++++++++++++++++ client/media_test.go | 110 ++++++++++++++++++++++++ 2 files changed, 304 insertions(+) create mode 100644 client/media.go create mode 100644 client/media_test.go diff --git a/client/media.go b/client/media.go new file mode 100644 index 0000000..8fb0882 --- /dev/null +++ b/client/media.go @@ -0,0 +1,194 @@ +package client + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "mime/multipart" + "net/http" + "strings" + "time" +) + +// OpenAI-compatible image generation and audio transcription clients. +// +// These are provider-agnostic backends for the two well-defined, broadly +// supported public APIs (OpenAI Images: POST /v1/images/generations; OpenAI +// Audio: POST /v1/audio/transcriptions). They give hawk's pluggable +// MediaEngine / Transcriber seams a concrete default backend while staying +// testable against an httptest server. A future provider (xAI image-gen, +// etc.) can replace the endpoint/credentials without touching callers. + +// ImageGenRequest is the body for POST /v1/images/generations. +type ImageGenRequest struct { + Model string `json:"model"` + Prompt string `json:"prompt"` + N int `json:"n,omitempty"` + Size string `json:"size,omitempty"` + ResponseFormat string `json:"response_format,omitempty"` // "url" | "b64_json" +} + +// ImageGenResult is one generated image. +type ImageGenResult struct { + URL string `json:"url,omitempty"` + B64JSON string `json:"b64_json,omitempty"` + RevisedPrompt string `json:"revised_prompt,omitempty"` +} + +// ImageGenResponse is the top-level response. +type ImageGenResponse struct { + Created int64 `json:"created"` + Data []ImageGenResult `json:"data"` +} + +// ImageClient generates images via an OpenAI-compatible endpoint. +type ImageClient struct { + apiKey string + baseURL string + httpClient *http.Client +} + +// NewImageClient creates an image client. baseURL defaults to +// https://api.openai.com; set it to an OpenAI-compatible endpoint for others. +func NewImageClient(apiKey, baseURL string) *ImageClient { + if baseURL == "" { + baseURL = "https://api.openai.com" + } + return &ImageClient{apiKey: apiKey, baseURL: baseURL, httpClient: NewPooledHTTPClient(2 * time.Minute)} +} + +// Generate creates n images for prompt. Returns each as bytes (b64 decoded) +// plus the provider URL when present. size is e.g. "1024x1024". +func (c *ImageClient) Generate(ctx context.Context, prompt, model, size string, n int) ([][]byte, []string, error) { + if n <= 0 { + n = 1 + } + body, err := json.Marshal(ImageGenRequest{ + Model: model, Prompt: prompt, N: n, Size: size, + ResponseFormat: "b64_json", + }) + if err != nil { + return nil, nil, err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+"/v1/images/generations", bytes.NewReader(body)) // #nosec G107 -- configured base URL + if err != nil { + return nil, nil, err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+c.apiKey) + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, nil, fmt.Errorf("eyrie: image generate: %w", err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + errBody, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return nil, nil, fmt.Errorf("eyrie: image API %d: %s", resp.StatusCode, strings.TrimSpace(string(errBody))) + } + var out ImageGenResponse + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return nil, nil, fmt.Errorf("eyrie: image decode: %w", err) + } + imgs := make([][]byte, 0, len(out.Data)) + urls := make([]string, 0, len(out.Data)) + for _, d := range out.Data { + if d.B64JSON != "" { + b, derr := base64.StdEncoding.DecodeString(d.B64JSON) + if derr != nil { + return nil, nil, fmt.Errorf("eyrie: image b64 decode: %w", derr) + } + imgs = append(imgs, b) + } + if d.URL != "" { + urls = append(urls, d.URL) + } + } + return imgs, urls, nil +} + +// TranscriptionRequest is the multipart body for /v1/audio/transcriptions. +// audio is the raw bytes; model is the transcription model. +type TranscriptionRequest struct { + Model string + File []byte + FileName string + Language string // optional ISO-639-1 + Prompt string // optional context/hint +} + +// Transcript is the response. +type Transcript struct { + Text string `json:"text"` +} + +// AudioClient transcribes audio via an OpenAI-compatible endpoint. +type AudioClient struct { + apiKey string + baseURL string + httpClient *http.Client +} + +// NewAudioClient creates a transcription client. +func NewAudioClient(apiKey, baseURL string) *AudioClient { + if baseURL == "" { + baseURL = "https://api.openai.com" + } + return &AudioClient{apiKey: apiKey, baseURL: baseURL, httpClient: NewPooledHTTPClient(2 * time.Minute)} +} + +// Transcribe sends the audio file and returns the transcript text. +func (c *AudioClient) Transcribe(ctx context.Context, r TranscriptionRequest) (string, error) { + var buf bytes.Buffer + writer := multipart.NewWriter(&buf) + if err := writer.WriteField("model", r.Model); err != nil { + return "", err + } + if r.Language != "" { + if err := writer.WriteField("language", r.Language); err != nil { + return "", err + } + } + if r.Prompt != "" { + if err := writer.WriteField("prompt", r.Prompt); err != nil { + return "", err + } + } + name := r.FileName + if name == "" { + name = "audio.wav" + } + fw, err := writer.CreateFormFile("file", name) + if err != nil { + return "", err + } + if _, err := fw.Write(r.File); err != nil { + return "", err + } + if err := writer.Close(); err != nil { + return "", err + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+"/v1/audio/transcriptions", &buf) // #nosec G107 -- configured base URL + if err != nil { + return "", err + } + req.Header.Set("Content-Type", writer.FormDataContentType()) + req.Header.Set("Authorization", "Bearer "+c.apiKey) + resp, err := c.httpClient.Do(req) + if err != nil { + return "", fmt.Errorf("eyrie: transcribe: %w", err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + errBody, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return "", fmt.Errorf("eyrie: audio API %d: %s", resp.StatusCode, strings.TrimSpace(string(errBody))) + } + var tr Transcript + if err := json.NewDecoder(resp.Body).Decode(&tr); err != nil { + return "", fmt.Errorf("eyrie: transcript decode: %w", err) + } + return tr.Text, nil +} diff --git a/client/media_test.go b/client/media_test.go new file mode 100644 index 0000000..1d06407 --- /dev/null +++ b/client/media_test.go @@ -0,0 +1,110 @@ +package client + +import ( + "context" + "encoding/base64" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestImageClientGenerateB64(t *testing.T) { + pngB64 := base64.StdEncoding.EncodeToString([]byte("fakepng")) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/images/generations" { + t.Fatalf("path = %s", r.URL.Path) + } + fmt.Fprintf(w, `{"created":1,"data":[{"b64_json":%q},{"b64_json":%q}]}`, pngB64, pngB64) + })) + defer srv.Close() + c := NewImageClient("k", srv.URL) + imgs, urls, err := c.Generate(context.Background(), "a cat", "dall-e-3", "1024x1024", 2) + if err != nil { + t.Fatal(err) + } + if len(imgs) != 2 { + t.Fatalf("imgs = %d", len(imgs)) + } + if string(imgs[0]) != "fakepng" { + t.Fatalf("img0 = %q", imgs[0]) + } + if len(urls) != 0 { + t.Fatalf("unexpected urls: %v", urls) + } +} + +func TestImageClientURLResults(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + fmt.Fprint(w, `{"created":1,"data":[{"url":"https://example.com/i.png"}]}`) + })) + defer srv.Close() + c := NewImageClient("k", srv.URL) + imgs, urls, err := c.Generate(context.Background(), "p", "", "", 1) + if err != nil { + t.Fatal(err) + } + if len(imgs) != 0 || len(urls) != 1 || urls[0] != "https://example.com/i.png" { + t.Fatalf("imgs=%d urls=%v", len(imgs), urls) + } +} + +func TestImageClientError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusUnauthorized) + fmt.Fprint(w, `{"error":{"message":"bad key"}}`) + })) + defer srv.Close() + c := NewImageClient("k", srv.URL) + if _, _, err := c.Generate(context.Background(), "p", "", "", 1); err == nil || !strings.Contains(err.Error(), "401") { + t.Fatalf("err = %v", err) + } +} + +func TestAudioClientTranscribe(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/audio/transcriptions" { + t.Fatalf("path = %s", r.URL.Path) + } + // Verify multipart with a file part. + if !strings.Contains(r.Header.Get("Content-Type"), "multipart/form-data") { + t.Fatal("expected multipart") + } + if err := r.ParseMultipartForm(1 << 20); err != nil { + t.Fatalf("parse form: %v", err) + } + if r.FormValue("model") != "whisper-1" { + t.Fatalf("model = %q", r.FormValue("model")) + } + file, _, err := r.FormFile("file") + if err != nil { + t.Fatalf("form file: %v", err) + } + file.Close() + fmt.Fprint(w, `{"text":"hello world"}`) + })) + defer srv.Close() + c := NewAudioClient("k", srv.URL) + text, err := c.Transcribe(context.Background(), TranscriptionRequest{ + Model: "whisper-1", File: []byte("audiobytes"), FileName: "voice.ogg", + }) + if err != nil { + t.Fatal(err) + } + if text != "hello world" { + t.Fatalf("text = %q", text) + } +} + +func TestAudioClientError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusBadRequest) + fmt.Fprint(w, `{"error":{"message":"bad audio"}}`) + })) + defer srv.Close() + c := NewAudioClient("k", srv.URL) + if _, err := c.Transcribe(context.Background(), TranscriptionRequest{Model: "whisper-1", File: []byte("x")}); err == nil || !strings.Contains(err.Error(), "400") { + t.Fatalf("err = %v", err) + } +}