fix(gateway): 客户端断开后 failover 静默终止,不再误报 502 账号耗尽

上游请求经 detachUpstreamContext(WithoutCancel) 有意脱离客户端取消(保
计费),但 failover 循环仍用原始 c.Request.Context() 重新选号:客户端
断开后上游返回 520 等可 failover 错误时,重新选号必然得到 context
canceled,被误判为账号耗尽,记录并返回通用 502。

修复:客户端已断开 ⇒ failover 静默终止。

- 新增 failoverClientGone(c):请求 ctx 已取消时先停 compact 心跳
  (建立 happens-before,对齐其它终结路径),响应未提交则标 499
  (statusClientClosedRequest,与并发槽取消路径同惯例)
- 7 个 OpenAI 内联 failover 循环(Responses/Messages/chat_completions/
  embeddings/images/grok_media/alpha_search)加双 guard:换号前 +
  选号失败分支入口;guard 位于 ReportOpenAIAccountScheduleResult(false)
  之后、RecordOpenAIAccountSwitch/池模式重试之前,账号健康副作用
  (service 层 detached ctx)不受影响
- FailoverState.HandleFailoverError/HandleSelectionExhausted 入口加
  ctx.Err() 检查返回 FailoverCanceled,取消不再改动 failover 状态;
  全部 10 个 FailoverCanceled 分支统一调用 failoverClientGone 归类 499
- 上游 detach 与计费设计不变;真实上游 520 事件仍完整落 ops
  (面板显示码 COALESCE(upstream_status_code,status_code)=520,
  错误率/告警口径不变)

测试:新增 openai_responses_failover_cancel_test.go 复现 issue 场景
(520+取消 ⇒ 不切号、499、无 502 终态)+ 在线客户端对照(正常切换、
耗尽 502);failover_loop_test.go 补入口取消用例并修正取消语义断言。

Fixes #4257
This commit is contained in:
shaw
2026-07-15 09:29:05 +08:00
parent accb918eca
commit a0593b0bf8
13 changed files with 406 additions and 11 deletions
+97 -3
View File
@@ -3,11 +3,13 @@ package handler
import (
"context"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/require"
)
@@ -489,7 +491,28 @@ func TestHandleFailoverError_ContextCanceled(t *testing.T) {
err := newTestFailoverErr(400, true, false)
ctx, cancel := context.WithCancel(context.Background())
cancel() // 立即取消
go func() {
time.Sleep(30 * time.Millisecond)
cancel() // 通过入口检查后、sleep 期间取消
}()
start := time.Now()
action := fs.HandleFailoverError(ctx, mock, 100, "openai", maxSameAccountRetries, err)
elapsed := time.Since(start)
require.Equal(t, FailoverCanceled, action)
require.Less(t, elapsed, 400*time.Millisecond, "sleep 应被取消打断")
// 进入重试分支后才取消:重试计数已递增
require.Equal(t, 1, fs.SameAccountRetryCount[100])
})
t.Run("入口即已取消_不改动任何failover状态", func(t *testing.T) {
mock := &mockTempUnscheduler{}
fs := NewFailoverState(3, false)
err := newTestFailoverErr(520, false, false)
ctx, cancel := context.WithCancel(context.Background())
cancel() // 调用前客户端已断开
start := time.Now()
action := fs.HandleFailoverError(ctx, mock, 100, "openai", maxSameAccountRetries, err)
@@ -497,8 +520,12 @@ func TestHandleFailoverError_ContextCanceled(t *testing.T) {
require.Equal(t, FailoverCanceled, action)
require.Less(t, elapsed, 100*time.Millisecond, "应立即返回")
// 入口已取消时不得改变任何重试状态。
require.Zero(t, fs.SameAccountRetryCount[100])
// 入口已取消时不得改变任何 failover 状态。
require.Equal(t, 0, fs.SwitchCount, "取消的请求不应计入切换")
require.Equal(t, 0, fs.SameAccountRetryCount[100], "取消的请求不应改动重试计数")
require.NotContains(t, fs.FailedAccountIDs, int64(100))
require.Nil(t, fs.LastFailoverErr)
require.Empty(t, mock.calls, "不应触发 TempUnschedule")
})
t.Run("Antigravity延迟期间context取消", func(t *testing.T) {
@@ -800,6 +827,29 @@ func TestHandleSelectionExhausted(t *testing.T) {
require.Less(t, elapsed, 100*time.Millisecond, "应立即返回")
})
t.Run("context已取消_非503也返回Canceled而非Exhausted", func(t *testing.T) {
// #4257 核心场景:客户端断开后选号失败源于 context canceled,
// 不应被当成账号耗尽转成 502。
fs := NewFailoverState(3, false)
fs.LastFailoverErr = newTestFailoverErr(520, false, false)
ctx, cancel := context.WithCancel(context.Background())
cancel()
action := fs.HandleSelectionExhausted(ctx)
require.Equal(t, FailoverCanceled, action)
})
t.Run("context已取消_无LastFailoverErr也返回Canceled", func(t *testing.T) {
fs := NewFailoverState(3, false)
ctx, cancel := context.WithCancel(context.Background())
cancel()
action := fs.HandleSelectionExhausted(ctx)
require.Equal(t, FailoverCanceled, action)
})
t.Run("503且SwitchCount等于MaxSwitches_仍可重试", func(t *testing.T) {
fs := NewFailoverState(2, false)
fs.LastFailoverErr = newTestFailoverErr(503, false, false)
@@ -809,3 +859,47 @@ func TestHandleSelectionExhausted(t *testing.T) {
require.Equal(t, FailoverContinue, action)
})
}
// ---------------------------------------------------------------------------
// failoverClientGone 测试
// ---------------------------------------------------------------------------
func TestFailoverClientGone(t *testing.T) {
gin.SetMode(gin.TestMode)
t.Run("活跃请求返回false", func(t *testing.T) {
rec := httptest.NewRecorder()
c, _ := gin.CreateTestContext(rec)
c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil)
require.False(t, failoverClientGone(c))
require.Equal(t, http.StatusOK, c.Writer.Status(), "不应改动状态码")
})
t.Run("客户端已断开_返回true并标记499", func(t *testing.T) {
rec := httptest.NewRecorder()
c, _ := gin.CreateTestContext(rec)
ctx, cancel := context.WithCancel(context.Background())
cancel()
c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil).WithContext(ctx)
require.True(t, failoverClientGone(c))
require.Equal(t, statusClientClosedRequest, c.Writer.Status())
})
t.Run("响应已提交_不改状态码", func(t *testing.T) {
rec := httptest.NewRecorder()
c, _ := gin.CreateTestContext(rec)
ctx, cancel := context.WithCancel(context.Background())
cancel()
c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil).WithContext(ctx)
c.String(http.StatusOK, "partial")
require.True(t, failoverClientGone(c))
require.Equal(t, http.StatusOK, c.Writer.Status(), "已提交的状态码不应被覆盖")
})
t.Run("nil安全", func(t *testing.T) {
require.False(t, failoverClientGone(nil))
})
}