fix(gateway): 客户端断开后 failover 静默终止,不再误报 502 账号耗尽
上游请求经 detachUpstreamContext(WithoutCancel) 有意脱离客户端取消(保 计费),但 failover 循环仍用原始 c.Request.Context() 重新选号:客户端 断开后上游返回 520 等可 failover 错误时,重新选号必然得到 context canceled,被误判为账号耗尽,记录并返回通用 502。 修复:客户端已断开 ⇒ failover 静默终止。 - 新增 failoverClientGone(c):请求 ctx 已取消时先停 compact 心跳 (建立 happens-before,对齐其它终结路径),响应未提交则标 499 (statusClientClosedRequest,与并发槽取消路径同惯例) - 7 个 OpenAI 内联 failover 循环(Responses/Messages/chat_completions/ embeddings/images/grok_media/alpha_search)加双 guard:换号前 + 选号失败分支入口;guard 位于 ReportOpenAIAccountScheduleResult(false) 之后、RecordOpenAIAccountSwitch/池模式重试之前,账号健康副作用 (service 层 detached ctx)不受影响 - FailoverState.HandleFailoverError/HandleSelectionExhausted 入口加 ctx.Err() 检查返回 FailoverCanceled,取消不再改动 failover 状态; 全部 10 个 FailoverCanceled 分支统一调用 failoverClientGone 归类 499 - 上游 detach 与计费设计不变;真实上游 520 事件仍完整落 ops (面板显示码 COALESCE(upstream_status_code,status_code)=520, 错误率/告警口径不变) 测试:新增 openai_responses_failover_cancel_test.go 复现 issue 场景 (520+取消 ⇒ 不切号、499、无 502 终态)+ 在线客户端对照(正常切换、 耗尽 502);failover_loop_test.go 补入口取消用例并修正取消语义断言。 Fixes #4257
This commit is contained in:
@@ -3,11 +3,13 @@ package handler
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
@@ -489,7 +491,28 @@ func TestHandleFailoverError_ContextCanceled(t *testing.T) {
|
||||
err := newTestFailoverErr(400, true, false)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel() // 立即取消
|
||||
go func() {
|
||||
time.Sleep(30 * time.Millisecond)
|
||||
cancel() // 通过入口检查后、sleep 期间取消
|
||||
}()
|
||||
|
||||
start := time.Now()
|
||||
action := fs.HandleFailoverError(ctx, mock, 100, "openai", maxSameAccountRetries, err)
|
||||
elapsed := time.Since(start)
|
||||
|
||||
require.Equal(t, FailoverCanceled, action)
|
||||
require.Less(t, elapsed, 400*time.Millisecond, "sleep 应被取消打断")
|
||||
// 进入重试分支后才取消:重试计数已递增
|
||||
require.Equal(t, 1, fs.SameAccountRetryCount[100])
|
||||
})
|
||||
|
||||
t.Run("入口即已取消_不改动任何failover状态", func(t *testing.T) {
|
||||
mock := &mockTempUnscheduler{}
|
||||
fs := NewFailoverState(3, false)
|
||||
err := newTestFailoverErr(520, false, false)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel() // 调用前客户端已断开
|
||||
|
||||
start := time.Now()
|
||||
action := fs.HandleFailoverError(ctx, mock, 100, "openai", maxSameAccountRetries, err)
|
||||
@@ -497,8 +520,12 @@ func TestHandleFailoverError_ContextCanceled(t *testing.T) {
|
||||
|
||||
require.Equal(t, FailoverCanceled, action)
|
||||
require.Less(t, elapsed, 100*time.Millisecond, "应立即返回")
|
||||
// 入口已取消时不得改变任何重试状态。
|
||||
require.Zero(t, fs.SameAccountRetryCount[100])
|
||||
// 入口已取消时不得改变任何 failover 状态。
|
||||
require.Equal(t, 0, fs.SwitchCount, "取消的请求不应计入切换")
|
||||
require.Equal(t, 0, fs.SameAccountRetryCount[100], "取消的请求不应改动重试计数")
|
||||
require.NotContains(t, fs.FailedAccountIDs, int64(100))
|
||||
require.Nil(t, fs.LastFailoverErr)
|
||||
require.Empty(t, mock.calls, "不应触发 TempUnschedule")
|
||||
})
|
||||
|
||||
t.Run("Antigravity延迟期间context取消", func(t *testing.T) {
|
||||
@@ -800,6 +827,29 @@ func TestHandleSelectionExhausted(t *testing.T) {
|
||||
require.Less(t, elapsed, 100*time.Millisecond, "应立即返回")
|
||||
})
|
||||
|
||||
t.Run("context已取消_非503也返回Canceled而非Exhausted", func(t *testing.T) {
|
||||
// #4257 核心场景:客户端断开后选号失败源于 context canceled,
|
||||
// 不应被当成账号耗尽转成 502。
|
||||
fs := NewFailoverState(3, false)
|
||||
fs.LastFailoverErr = newTestFailoverErr(520, false, false)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
action := fs.HandleSelectionExhausted(ctx)
|
||||
require.Equal(t, FailoverCanceled, action)
|
||||
})
|
||||
|
||||
t.Run("context已取消_无LastFailoverErr也返回Canceled", func(t *testing.T) {
|
||||
fs := NewFailoverState(3, false)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
action := fs.HandleSelectionExhausted(ctx)
|
||||
require.Equal(t, FailoverCanceled, action)
|
||||
})
|
||||
|
||||
t.Run("503且SwitchCount等于MaxSwitches_仍可重试", func(t *testing.T) {
|
||||
fs := NewFailoverState(2, false)
|
||||
fs.LastFailoverErr = newTestFailoverErr(503, false, false)
|
||||
@@ -809,3 +859,47 @@ func TestHandleSelectionExhausted(t *testing.T) {
|
||||
require.Equal(t, FailoverContinue, action)
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// failoverClientGone 测试
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func TestFailoverClientGone(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
|
||||
t.Run("活跃请求返回false", func(t *testing.T) {
|
||||
rec := httptest.NewRecorder()
|
||||
c, _ := gin.CreateTestContext(rec)
|
||||
c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil)
|
||||
|
||||
require.False(t, failoverClientGone(c))
|
||||
require.Equal(t, http.StatusOK, c.Writer.Status(), "不应改动状态码")
|
||||
})
|
||||
|
||||
t.Run("客户端已断开_返回true并标记499", func(t *testing.T) {
|
||||
rec := httptest.NewRecorder()
|
||||
c, _ := gin.CreateTestContext(rec)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil).WithContext(ctx)
|
||||
|
||||
require.True(t, failoverClientGone(c))
|
||||
require.Equal(t, statusClientClosedRequest, c.Writer.Status())
|
||||
})
|
||||
|
||||
t.Run("响应已提交_不改状态码", func(t *testing.T) {
|
||||
rec := httptest.NewRecorder()
|
||||
c, _ := gin.CreateTestContext(rec)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil).WithContext(ctx)
|
||||
c.String(http.StatusOK, "partial")
|
||||
|
||||
require.True(t, failoverClientGone(c))
|
||||
require.Equal(t, http.StatusOK, c.Writer.Status(), "已提交的状态码不应被覆盖")
|
||||
})
|
||||
|
||||
t.Run("nil安全", func(t *testing.T) {
|
||||
require.False(t, failoverClientGone(nil))
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user