From 4dc0aedbfeaecceb716012f36cbe423cb9ac1dee Mon Sep 17 00:00:00 2001 From: wucm667 Date: Fri, 24 Jul 2026 02:42:23 +0800 Subject: [PATCH 1/3] fix(gemini): preserve image output in chat completions --- .../gemini_chat_completions_compat_service.go | 2 +- ...ni_chat_completions_compat_service_test.go | 157 ++++++++++++++++++ .../service/gemini_messages_compat_service.go | 34 +++- 3 files changed, 189 insertions(+), 4 deletions(-) create mode 100644 backend/internal/service/gemini_chat_completions_compat_service_test.go diff --git a/backend/internal/service/gemini_chat_completions_compat_service.go b/backend/internal/service/gemini_chat_completions_compat_service.go index de996649b..5f868a5d6 100644 --- a/backend/internal/service/gemini_chat_completions_compat_service.go +++ b/backend/internal/service/gemini_chat_completions_compat_service.go @@ -473,7 +473,7 @@ func geminiResponseToChatCompletions( rawData []byte, usageOverride *ClaudeUsage, ) (*apicompat.ChatCompletionsResponse, *ClaudeUsage, error) { - claudeRespMap, usage := convertGeminiToClaudeMessage(geminiResp, originalModel, rawData) + claudeRespMap, usage := convertGeminiToClaudeMessage(geminiResp, originalModel, rawData, true) if usageOverride != nil && (usageOverride.InputTokens > 0 || usageOverride.OutputTokens > 0 || usageOverride.CacheReadInputTokens > 0) { usage = usageOverride if usageMap, ok := claudeRespMap["usage"].(map[string]any); ok { diff --git a/backend/internal/service/gemini_chat_completions_compat_service_test.go b/backend/internal/service/gemini_chat_completions_compat_service_test.go new file mode 100644 index 000000000..98e867560 --- /dev/null +++ b/backend/internal/service/gemini_chat_completions_compat_service_test.go @@ -0,0 +1,157 @@ +package service + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestGeminiResponseToChatCompletionsPreservesInlineData(t *testing.T) { + tests := []struct { + name string + parts []any + want string + }{ + { + name: "image only", + parts: []any{ + map[string]any{"inlineData": map[string]any{"mimeType": "image/png", "data": "aW1hZ2U="}}, + }, + want: "![image](data:image/png;base64,aW1hZ2U=)", + }, + { + name: "text and image", + parts: []any{ + map[string]any{"text": "rendered image:\n"}, + map[string]any{"inlineData": map[string]any{"mimeType": "image/webp", "data": "d2VicA=="}}, + }, + want: "rendered image:\n![image](data:image/webp;base64,d2VicA==)", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + geminiResp := map[string]any{ + "candidates": []any{map[string]any{ + "content": map[string]any{"parts": tt.parts}, + "finishReason": "STOP", + }}, + } + rawData, err := json.Marshal(geminiResp) + require.NoError(t, err) + + got, _, err := geminiResponseToChatCompletions(geminiResp, "gemini-test", rawData, nil) + require.NoError(t, err) + require.Len(t, got.Choices, 1) + + var content string + require.NoError(t, json.Unmarshal(got.Choices[0].Message.Content, &content)) + require.Equal(t, tt.want, content) + require.Equal(t, "stop", got.Choices[0].FinishReason) + }) + } +} + +func TestGeminiResponseToChatCompletionsOmitsInvalidInlineData(t *testing.T) { + tests := []struct { + name string + inlineData map[string]any + }{ + { + name: "unsupported MIME type", + inlineData: map[string]any{"mimeType": "image/svg+xml", "data": "PHN2Zz48L3N2Zz4="}, + }, + { + name: "malformed MIME type", + inlineData: map[string]any{"mimeType": "image/png; charset=utf-8", "data": "aW1hZ2U="}, + }, + { + name: "malformed base64", + inlineData: map[string]any{"mimeType": "image/png", "data": "not-valid-base64!!!"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + geminiResp := map[string]any{ + "candidates": []any{map[string]any{ + "content": map[string]any{"parts": []any{map[string]any{"text": "before"}, map[string]any{"inlineData": tt.inlineData}, map[string]any{"text": "after"}}}, + "finishReason": "STOP", + }}, + } + rawData, err := json.Marshal(geminiResp) + require.NoError(t, err) + + got, _, err := geminiResponseToChatCompletions(geminiResp, "gemini-test", rawData, nil) + require.NoError(t, err) + + var content string + require.NoError(t, json.Unmarshal(got.Choices[0].Message.Content, &content)) + require.Equal(t, "beforeafter", content) + }) + } +} + +func TestConvertGeminiToClaudeMessageOmitsInlineDataForAnthropicMessages(t *testing.T) { + geminiResp := map[string]any{ + "candidates": []any{map[string]any{ + "content": map[string]any{"parts": []any{ + map[string]any{"text": "before"}, + map[string]any{"inlineData": map[string]any{"mimeType": "image/png", "data": "aW1hZ2U="}}, + map[string]any{"functionCall": map[string]any{"name": "get_weather", "args": map[string]any{"city": "Paris"}}}, + map[string]any{"text": "after"}, + }}, + "finishReason": "STOP", + }}, + } + rawData, err := json.Marshal(geminiResp) + require.NoError(t, err) + + withInlineData, _ := convertGeminiToClaudeMessage(geminiResp, "gemini-test", rawData, true) + contentWithInlineData := withInlineData["content"].([]any) + require.Len(t, contentWithInlineData, 4) + require.Equal(t, map[string]any{"type": "text", "text": "before"}, contentWithInlineData[0]) + require.Equal(t, map[string]any{"type": "text", "text": "![image](data:image/png;base64,aW1hZ2U=)"}, contentWithInlineData[1]) + require.Equal(t, "tool_use", contentWithInlineData[2].(map[string]any)["type"]) + require.Equal(t, "get_weather", contentWithInlineData[2].(map[string]any)["name"]) + require.Equal(t, map[string]any{"type": "text", "text": "after"}, contentWithInlineData[3]) + + withoutInlineData, _ := convertGeminiToClaudeMessage(geminiResp, "gemini-test", rawData, false) + contentWithoutInlineData := withoutInlineData["content"].([]any) + require.Len(t, contentWithoutInlineData, 3) + require.Equal(t, map[string]any{"type": "text", "text": "before"}, contentWithoutInlineData[0]) + require.Equal(t, "tool_use", contentWithoutInlineData[1].(map[string]any)["type"]) + require.Equal(t, "get_weather", contentWithoutInlineData[1].(map[string]any)["name"]) + require.Equal(t, map[string]any{"type": "text", "text": "after"}, contentWithoutInlineData[2]) +} + +func TestGeminiResponseToChatCompletionsRetainsTextAndToolBehavior(t *testing.T) { + geminiResp := map[string]any{ + "candidates": []any{map[string]any{ + "content": map[string]any{"parts": []any{ + map[string]any{"text": "checking"}, + map[string]any{"functionCall": map[string]any{ + "name": "get_weather", + "args": map[string]any{"city": "Paris"}, + }}, + }}, + "finishReason": "STOP", + }}, + } + rawData, err := json.Marshal(geminiResp) + require.NoError(t, err) + + got, _, err := geminiResponseToChatCompletions(geminiResp, "gemini-test", rawData, nil) + require.NoError(t, err) + require.Len(t, got.Choices, 1) + + choice := got.Choices[0] + var content string + require.NoError(t, json.Unmarshal(choice.Message.Content, &content)) + require.Equal(t, "checking", content) + require.Equal(t, "tool_calls", choice.FinishReason) + require.Len(t, choice.Message.ToolCalls, 1) + require.Equal(t, "get_weather", choice.Message.ToolCalls[0].Function.Name) + require.JSONEq(t, `{"city":"Paris"}`, choice.Message.ToolCalls[0].Function.Arguments) +} diff --git a/backend/internal/service/gemini_messages_compat_service.go b/backend/internal/service/gemini_messages_compat_service.go index 2aaba767e..1a3821847 100644 --- a/backend/internal/service/gemini_messages_compat_service.go +++ b/backend/internal/service/gemini_messages_compat_service.go @@ -5,6 +5,7 @@ import ( "bytes" "context" "crypto/rand" + "encoding/base64" "encoding/hex" "encoding/json" "errors" @@ -1068,7 +1069,7 @@ func (s *GeminiMessagesCompatService) Forward(ctx context.Context, c *gin.Contex return nil, s.writeClaudeError(c, http.StatusBadGateway, "upstream_error", "Failed to read upstream stream") } collectedBytes, _ := json.Marshal(collected) - claudeResp, usageObj2 := convertGeminiToClaudeMessage(collected, originalModel, collectedBytes) + claudeResp, usageObj2 := convertGeminiToClaudeMessage(collected, originalModel, collectedBytes, false) c.JSON(http.StatusOK, claudeResp) usage = usageObj2 if usageObj != nil && (usageObj.InputTokens > 0 || usageObj.OutputTokens > 0) { @@ -1965,7 +1966,7 @@ func (s *GeminiMessagesCompatService) handleNonStreamingResponse(c *gin.Context, return nil, s.writeClaudeError(c, http.StatusBadGateway, "upstream_error", "Failed to parse upstream response") } - claudeResp, usage := convertGeminiToClaudeMessage(geminiResp, originalModel, unwrappedBody) + claudeResp, usage := convertGeminiToClaudeMessage(geminiResp, originalModel, unwrappedBody, false) c.JSON(http.StatusOK, claudeResp) return usage, nil @@ -2717,7 +2718,7 @@ func unwrapGeminiResponse(raw []byte) ([]byte, error) { return raw, nil } -func convertGeminiToClaudeMessage(geminiResp map[string]any, originalModel string, rawData []byte) (map[string]any, *ClaudeUsage) { +func convertGeminiToClaudeMessage(geminiResp map[string]any, originalModel string, rawData []byte, includeInlineData bool) (map[string]any, *ClaudeUsage) { usage := extractGeminiUsage(rawData) if usage == nil { usage = &ClaudeUsage{} @@ -2740,6 +2741,16 @@ func convertGeminiToClaudeMessage(geminiResp map[string]any, originalModel strin "text": text, }) } + if inlineData, ok := pm["inlineData"].(map[string]any); includeInlineData && ok { + mimeType, _ := inlineData["mimeType"].(string) + data, _ := inlineData["data"].(string) + if isGeminiInlineImageMIMEType(mimeType) && isValidBase64(data) { + contentBlocks = append(contentBlocks, map[string]any{ + "type": "text", + "text": fmt.Sprintf("![image](data:%s;base64,%s)", mimeType, data), + }) + } + } if fc, ok := pm["functionCall"].(map[string]any); ok { name, _ := fc["name"].(string) if strings.TrimSpace(name) == "" { @@ -2782,6 +2793,23 @@ func convertGeminiToClaudeMessage(geminiResp map[string]any, originalModel strin return resp, usage } +func isGeminiInlineImageMIMEType(mimeType string) bool { + switch mimeType { + case "image/gif", "image/jpeg", "image/png", "image/webp": + return true + default: + return false + } +} + +func isValidBase64(data string) bool { + if data == "" { + return false + } + _, err := base64.StdEncoding.DecodeString(data) + return err == nil +} + func extractGeminiUsage(data []byte) *ClaudeUsage { usage := gjson.GetBytes(data, "usageMetadata") if !usage.Exists() { From 8276fdf6058d2b07ed51a89e3649c8cef2472aa7 Mon Sep 17 00:00:00 2001 From: wucm667 Date: Fri, 24 Jul 2026 03:00:26 +0800 Subject: [PATCH 2/3] fix: resolve Gemini compat test lint --- ...ini_chat_completions_compat_service_test.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/backend/internal/service/gemini_chat_completions_compat_service_test.go b/backend/internal/service/gemini_chat_completions_compat_service_test.go index 98e867560..e4ebf7a83 100644 --- a/backend/internal/service/gemini_chat_completions_compat_service_test.go +++ b/backend/internal/service/gemini_chat_completions_compat_service_test.go @@ -109,20 +109,26 @@ func TestConvertGeminiToClaudeMessageOmitsInlineDataForAnthropicMessages(t *test require.NoError(t, err) withInlineData, _ := convertGeminiToClaudeMessage(geminiResp, "gemini-test", rawData, true) - contentWithInlineData := withInlineData["content"].([]any) + contentWithInlineData, ok := withInlineData["content"].([]any) + require.True(t, ok) require.Len(t, contentWithInlineData, 4) require.Equal(t, map[string]any{"type": "text", "text": "before"}, contentWithInlineData[0]) require.Equal(t, map[string]any{"type": "text", "text": "![image](data:image/png;base64,aW1hZ2U=)"}, contentWithInlineData[1]) - require.Equal(t, "tool_use", contentWithInlineData[2].(map[string]any)["type"]) - require.Equal(t, "get_weather", contentWithInlineData[2].(map[string]any)["name"]) + toolUse, ok := contentWithInlineData[2].(map[string]any) + require.True(t, ok) + require.Equal(t, "tool_use", toolUse["type"]) + require.Equal(t, "get_weather", toolUse["name"]) require.Equal(t, map[string]any{"type": "text", "text": "after"}, contentWithInlineData[3]) withoutInlineData, _ := convertGeminiToClaudeMessage(geminiResp, "gemini-test", rawData, false) - contentWithoutInlineData := withoutInlineData["content"].([]any) + contentWithoutInlineData, ok := withoutInlineData["content"].([]any) + require.True(t, ok) require.Len(t, contentWithoutInlineData, 3) require.Equal(t, map[string]any{"type": "text", "text": "before"}, contentWithoutInlineData[0]) - require.Equal(t, "tool_use", contentWithoutInlineData[1].(map[string]any)["type"]) - require.Equal(t, "get_weather", contentWithoutInlineData[1].(map[string]any)["name"]) + toolUseWithoutInlineData, ok := contentWithoutInlineData[1].(map[string]any) + require.True(t, ok) + require.Equal(t, "tool_use", toolUseWithoutInlineData["type"]) + require.Equal(t, "get_weather", toolUseWithoutInlineData["name"]) require.Equal(t, map[string]any{"type": "text", "text": "after"}, contentWithoutInlineData[2]) } From b5979050f3f94887c4220ff42156958d95163115 Mon Sep 17 00:00:00 2001 From: wucm667 Date: Fri, 24 Jul 2026 08:33:58 +0800 Subject: [PATCH 3/3] fix(frontend): upgrade postcss for security audit --- frontend/package.json | 2 +- frontend/pnpm-lock.yaml | 46 ++++++++++++++++++++--------------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/frontend/package.json b/frontend/package.json index b3c3450a3..c92599031 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -51,7 +51,7 @@ "eslint": "^8.57.0", "eslint-plugin-vue": "^9.25.0", "jsdom": "^24.1.3", - "postcss": "^8.4.32", + "postcss": "^8.5.12", "tailwindcss": "^3.4.0", "typescript": "~5.6.0", "vite": "^5.0.10", diff --git a/frontend/pnpm-lock.yaml b/frontend/pnpm-lock.yaml index 4bc3d2fee..b98ec68f3 100644 --- a/frontend/pnpm-lock.yaml +++ b/frontend/pnpm-lock.yaml @@ -105,7 +105,7 @@ importers: version: 2.4.6 autoprefixer: specifier: ^10.4.16 - version: 10.4.23(postcss@8.5.6) + version: 10.4.23(postcss@8.5.12) eslint: specifier: ^8.57.0 version: 8.57.1 @@ -116,8 +116,8 @@ importers: specifier: ^24.1.3 version: 24.1.3 postcss: - specifier: ^8.4.32 - version: 8.5.6 + specifier: ^8.5.12 + version: 8.5.12 tailwindcss: specifier: ^3.4.0 version: 3.4.19 @@ -3550,8 +3550,8 @@ packages: postcss-value-parser@4.2.0: resolution: {integrity: sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==} - postcss@8.5.6: - resolution: {integrity: sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg==} + postcss@8.5.12: + resolution: {integrity: sha512-W62t/Se6rA0Az3DfCL0AqJwXuKwBeYg6nOaIgzP+xZ7N5BFCI7DYi1qs6ygUYT6rvfi6t9k65UMLJC+PHZpDAA==} engines: {node: ^10 || ^12 || >=14} prelude-ls@1.2.1: @@ -6284,7 +6284,7 @@ snapshots: '@vue/shared': 3.5.26 estree-walker: 2.0.2 magic-string: 0.30.21 - postcss: 8.5.6 + postcss: 8.5.12 source-map-js: 1.2.1 '@vue/compiler-ssr@3.5.26': @@ -6515,13 +6515,13 @@ snapshots: attr-accept@2.2.5: {} - autoprefixer@10.4.23(postcss@8.5.6): + autoprefixer@10.4.23(postcss@8.5.12): dependencies: browserslist: 4.28.1 caniuse-lite: 1.0.30001761 fraction.js: 5.3.4 picocolors: 1.1.1 - postcss: 8.5.6 + postcss: 8.5.12 postcss-value-parser: 4.2.0 axios@1.18.1: @@ -8607,28 +8607,28 @@ snapshots: dependencies: '@babel/runtime': 7.28.4 - postcss-import@15.1.0(postcss@8.5.6): + postcss-import@15.1.0(postcss@8.5.12): dependencies: - postcss: 8.5.6 + postcss: 8.5.12 postcss-value-parser: 4.2.0 read-cache: 1.0.0 resolve: 1.22.11 - postcss-js@4.1.0(postcss@8.5.6): + postcss-js@4.1.0(postcss@8.5.12): dependencies: camelcase-css: 2.0.1 - postcss: 8.5.6 + postcss: 8.5.12 - postcss-load-config@6.0.1(jiti@1.21.7)(postcss@8.5.6): + postcss-load-config@6.0.1(jiti@1.21.7)(postcss@8.5.12): dependencies: lilconfig: 3.1.3 optionalDependencies: jiti: 1.21.7 - postcss: 8.5.6 + postcss: 8.5.12 - postcss-nested@6.2.0(postcss@8.5.6): + postcss-nested@6.2.0(postcss@8.5.12): dependencies: - postcss: 8.5.6 + postcss: 8.5.12 postcss-selector-parser: 6.1.2 postcss-selector-parser@6.1.2: @@ -8638,7 +8638,7 @@ snapshots: postcss-value-parser@4.2.0: {} - postcss@8.5.6: + postcss@8.5.12: dependencies: nanoid: 3.3.11 picocolors: 1.1.1 @@ -9253,11 +9253,11 @@ snapshots: normalize-path: 3.0.0 object-hash: 3.0.0 picocolors: 1.1.1 - postcss: 8.5.6 - postcss-import: 15.1.0(postcss@8.5.6) - postcss-js: 4.1.0(postcss@8.5.6) - postcss-load-config: 6.0.1(jiti@1.21.7)(postcss@8.5.6) - postcss-nested: 6.2.0(postcss@8.5.6) + postcss: 8.5.12 + postcss-import: 15.1.0(postcss@8.5.12) + postcss-js: 4.1.0(postcss@8.5.12) + postcss-load-config: 6.0.1(jiti@1.21.7)(postcss@8.5.12) + postcss-nested: 6.2.0(postcss@8.5.12) postcss-selector-parser: 6.1.2 resolve: 1.22.11 sucrase: 3.35.1 @@ -9489,7 +9489,7 @@ snapshots: vite@5.4.21(@types/node@20.19.27): dependencies: esbuild: 0.21.5 - postcss: 8.5.6 + postcss: 8.5.12 rollup: 4.54.0 optionalDependencies: '@types/node': 20.19.27