为异步生图任务增加 S3 兼容对象存储支持,任务结果不再把大图内联存进 Redis: - 新增可插拔接口 service.ImageStorage(Save -> url),适配别的厂商只需实现它 - S3 实现 S3ImageStorage(AWS S3 / R2 / 阿里云 OSS / MinIO),与备份共用 S3 客户端构造 - 新增 image_storage 配置(config.yaml + IMAGE_STORAGE_* 环境变量),默认关闭 - enabled 同时作为总开关:关闭或未配置对象存储时,异步生图接口返回 404 且不写 Redis,从根上避免几 MB 的 b64_json 结果撑爆 Redis - 完成时把图片上传对象存储并把结果改写为短链接(公开直链或 presigned), 上传失败则任务标记为失败 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SM1tf3CFVRzC7guuhBXvMd
158 lines
5.6 KiB
Go
158 lines
5.6 KiB
Go
package service
|
|
|
|
import (
|
|
"context"
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"errors"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
// pngBytes is a minimal payload whose signature makes http.DetectContentType
|
|
// report image/png.
|
|
var pngBytes = []byte("\x89PNG\r\n\x1a\nfake-png-payload")
|
|
|
|
type savedImage struct {
|
|
key string
|
|
contentType string
|
|
data []byte
|
|
}
|
|
|
|
type fakeImageStorage struct {
|
|
saved []savedImage
|
|
url string
|
|
err error
|
|
}
|
|
|
|
func (f *fakeImageStorage) Save(_ context.Context, key, contentType string, data []byte) (string, error) {
|
|
if f.err != nil {
|
|
return "", f.err
|
|
}
|
|
f.saved = append(f.saved, savedImage{key: key, contentType: contentType, data: append([]byte(nil), data...)})
|
|
if f.url != "" {
|
|
return f.url, nil
|
|
}
|
|
return "https://cdn.test/" + key, nil
|
|
}
|
|
|
|
func TestImageResultUploaderRewritesB64JSON(t *testing.T) {
|
|
storage := &fakeImageStorage{}
|
|
uploader := NewImageResultUploader(storage, "images/", 0, nil)
|
|
|
|
b64 := base64.StdEncoding.EncodeToString(pngBytes)
|
|
result := json.RawMessage(`{"created":1,"data":[{"b64_json":"` + b64 + `","revised_prompt":"a cat"}]}`)
|
|
|
|
out, err := uploader.Rewrite(context.Background(), "imgtask_abc", result)
|
|
require.NoError(t, err)
|
|
|
|
require.Len(t, storage.saved, 1)
|
|
require.Equal(t, "images/imgtask_abc-0.png", storage.saved[0].key)
|
|
require.Equal(t, "image/png", storage.saved[0].contentType)
|
|
require.Equal(t, pngBytes, storage.saved[0].data)
|
|
|
|
var parsed struct {
|
|
Data []map[string]json.RawMessage `json:"data"`
|
|
}
|
|
require.NoError(t, json.Unmarshal(out, &parsed))
|
|
require.Len(t, parsed.Data, 1)
|
|
require.JSONEq(t, `"https://cdn.test/images/imgtask_abc-0.png"`, string(parsed.Data[0]["url"]))
|
|
_, hasB64 := parsed.Data[0]["b64_json"]
|
|
require.False(t, hasB64, "b64_json must be stripped after offload")
|
|
require.JSONEq(t, `"a cat"`, string(parsed.Data[0]["revised_prompt"]), "unrelated fields preserved")
|
|
}
|
|
|
|
func TestImageResultUploaderRewritesURL(t *testing.T) {
|
|
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
w.Header().Set("Content-Type", "image/png")
|
|
_, _ = w.Write(pngBytes)
|
|
}))
|
|
defer upstream.Close()
|
|
|
|
storage := &fakeImageStorage{}
|
|
uploader := NewImageResultUploader(storage, "images/", 0, nil)
|
|
|
|
result := json.RawMessage(`{"created":1,"data":[{"url":"` + upstream.URL + `/pic.png"}]}`)
|
|
out, err := uploader.Rewrite(context.Background(), "imgtask_xyz", result)
|
|
require.NoError(t, err)
|
|
|
|
require.Len(t, storage.saved, 1)
|
|
require.Equal(t, pngBytes, storage.saved[0].data)
|
|
require.Equal(t, "image/png", storage.saved[0].contentType)
|
|
|
|
var parsed struct {
|
|
Data []map[string]json.RawMessage `json:"data"`
|
|
}
|
|
require.NoError(t, json.Unmarshal(out, &parsed))
|
|
require.JSONEq(t, `"https://cdn.test/images/imgtask_xyz-0.png"`, string(parsed.Data[0]["url"]))
|
|
}
|
|
|
|
func TestImageResultUploaderPropagatesStorageError(t *testing.T) {
|
|
storage := &fakeImageStorage{err: errors.New("bucket unreachable")}
|
|
uploader := NewImageResultUploader(storage, "images/", 0, nil)
|
|
|
|
b64 := base64.StdEncoding.EncodeToString(pngBytes)
|
|
result := json.RawMessage(`{"data":[{"b64_json":"` + b64 + `"}]}`)
|
|
|
|
_, err := uploader.Rewrite(context.Background(), "imgtask_err", result)
|
|
require.Error(t, err)
|
|
require.Contains(t, err.Error(), "bucket unreachable")
|
|
}
|
|
|
|
func TestImageResultUploaderNilStoragePassthrough(t *testing.T) {
|
|
var uploader *ImageResultUploader
|
|
result := json.RawMessage(`{"data":[{"url":"https://example.test/x.png"}]}`)
|
|
out, err := uploader.Rewrite(context.Background(), "imgtask_nil", result)
|
|
require.NoError(t, err)
|
|
require.JSONEq(t, string(result), string(out))
|
|
}
|
|
|
|
func TestImageTaskServiceCompleteOffloadsToStorage(t *testing.T) {
|
|
store := &imageTaskMemoryStore{}
|
|
storage := &fakeImageStorage{}
|
|
uploader := NewImageResultUploader(storage, "images/", 0, nil)
|
|
svc := NewImageTaskServiceWithUploader(store, uploader, time.Hour, time.Minute)
|
|
require.True(t, svc.Enabled())
|
|
|
|
owner := ImageTaskOwner{UserID: 1, APIKeyID: 2}
|
|
created, err := svc.Create(context.Background(), owner)
|
|
require.NoError(t, err)
|
|
|
|
b64 := base64.StdEncoding.EncodeToString(pngBytes)
|
|
result := json.RawMessage(`{"created":1,"data":[{"b64_json":"` + b64 + `"}]}`)
|
|
require.NoError(t, svc.Complete(context.Background(), created.ID, http.StatusOK, result))
|
|
|
|
got, err := svc.Get(context.Background(), owner, created.ID)
|
|
require.NoError(t, err)
|
|
require.Equal(t, ImageTaskStatusCompleted, got.Status)
|
|
require.Equal(t, "https://cdn.test/images/"+created.ID+"-0.png", got.ImageURL)
|
|
require.NotContains(t, string(got.Result), "b64_json", "large base64 must not be persisted to Redis")
|
|
require.Len(t, storage.saved, 1)
|
|
}
|
|
|
|
func TestImageTaskServiceCompleteOffloadFailureMarksFailed(t *testing.T) {
|
|
store := &imageTaskMemoryStore{}
|
|
storage := &fakeImageStorage{err: errors.New("bucket unreachable")}
|
|
uploader := NewImageResultUploader(storage, "images/", 0, nil)
|
|
svc := NewImageTaskServiceWithUploader(store, uploader, time.Hour, time.Minute)
|
|
|
|
owner := ImageTaskOwner{UserID: 1, APIKeyID: 2}
|
|
created, err := svc.Create(context.Background(), owner)
|
|
require.NoError(t, err)
|
|
|
|
b64 := base64.StdEncoding.EncodeToString(pngBytes)
|
|
result := json.RawMessage(`{"data":[{"b64_json":"` + b64 + `"}]}`)
|
|
require.NoError(t, svc.Complete(context.Background(), created.ID, http.StatusOK, result))
|
|
|
|
got, err := svc.Get(context.Background(), owner, created.ID)
|
|
require.NoError(t, err)
|
|
require.Equal(t, ImageTaskStatusFailed, got.Status)
|
|
require.Equal(t, http.StatusBadGateway, got.HTTPStatus)
|
|
require.Contains(t, string(got.Error), "object storage")
|
|
require.NotContains(t, string(got.Result), "b64_json", "failed offload must not persist base64 to Redis")
|
|
}
|