Files
sub2api/backend/internal/service/ratelimit_service_openai_test.go
T
bdf7ead157 feat(spark-shadow): OpenAI Spark 链接型影子账号
背景:gpt-5.3-codex-spark 使用独立于 codex 全局(5h/7d)的配额窗口(数据源是
/wham/usage 响应体的 codex_bengalfox,而非 codex 全局用的 x-codex-* 响应头),且
只能挂在已完成 OAuth 授权的 OpenAI 账号下复用其登录态,不能作为独立账号单独接入。
为此新增“链接型影子账号”(spark shadow account):影子账号本身不持有任何凭据,
通过 parent_account_id 指向母账号,凭据/token/代理透传自母账号并共享母账号的刷新
周期,仅在配额维度(quota_dimension=spark)和用量窗口上与母账号完全独立调度、互不
连坐。

实现:
- 数据模型:migration 154(+154a)给 accounts 表加 parent_account_id /
  quota_dimension 列 + 4 条约束(维度合法 / parent⟺非 global 维度一致 / 禁自指 /
  FK)+ 2 个 CONCURRENTLY 索引(母账号索引 + 每母账号至多一个影子的唯一索引)。
- 创建:POST /api/v1/admin/accounts/:id/shadow(CreateShadow)—— 一母一影(唯一
  索引兜底并发竞态),继承母账号 proxy/分组/并发/优先级(显式传参可覆盖),默认
  model_mapping 恒等映射到 spark(拒绝非 spark 模型),母账号必须是真实的 OpenAI
  OAuth 账号(非影子)。
- 凭据透传:resolveCredentialAccount 把影子解析回母账号,GetAccessToken / 请求头
  / WS 三条路径统一走此函数;影子自身 Credentials 恒为空(仅允许写 model_mapping),
  凭据写入的汇聚点 persistAccountCredentials 对影子早返 no-op,防止误写。
- 调度:parentHealthyForShadow 只看母账号是否仍是 OpenAI OAuth + 凭据/传输是否
  可用(active、token 未过期、未处于 401/刷新失败/传输故障导致的临时不可调度冷却),
  刻意不看母账号的 global 限流窗口——两条 429 道互不连坐。
- 用量:影子的 codex_5h/7d 走 OpenAIQuotaService.QueryUsage(/wham/usage 的
  codex_bengalfox),与母账号走的 WSv2 探测(/responses 头)完全独立的数据源、
  刷新节流与 staleness 判定。
- 备份:ExportData 显式排除影子账号(影子不持凭据,通用凭据型导入强制
  credentials 非空、无法表达父子链接),按 skipped_shadows 计数提示前端。
- 前端:账号操作菜单新增“创建 Spark 影子”入口,影子行展示回填的母账号信息
  (邮箱 / plan / 隐私模式 / 订阅到期 / chatgpt_account_id),批量操作自动跳过
  影子账号。

说明:migrations 目录用完整文件名(而非纯数字前缀)标识迁移,故本次新增的
154_account_spark_shadow.sql / 154a_..._notx.sql 与已有的
154_add_ops_system_logs_api_key_id.sql 按序号共存,与目录里 145/151 已有的
先例一致。

测试:新增约 20 个测试文件,覆盖 handler(CreateShadow 校验 / 母账号信息回填)、
repository(影子 round-trip / 一母一影唯一索引 / 迁移 schema)、service(凭据
透传三路径 / 调度母健康门 / 用量窗口来源与刷新节流 / CRS 母账号不变量 / 各类
早返与 fail-closed 场景)及前端组件(账号列表 / 操作菜单 / 用量重置)。

验证(镜像 CI;golangci-lint 首次全量分析耗时过长被跳过,其余全部实测):
- gofmt -l:干净
- go build ./... / go vet ./...:通过
- go test ./... -count=1:全绿(全部包 ok,含 internal/service、
  internal/repository、migrations)
- go test -tags integration ./internal/repository/... ./internal/service/...
  (真实 Postgres,testcontainers):全绿,含迁移幂等性
  (TestMigrationsRunner_IsIdempotent_AndSchemaIsUpToDate)与影子相关全部用例
- pnpm lint:check / pnpm typecheck / pnpm build(真实 vite 构建)/
  pnpm vitest run:全绿(124 文件 760 用例)

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-07-01 12:21:45 +08:00

528 lines
18 KiB
Go

//go:build unit
package service
import (
"context"
"net/http"
"testing"
"time"
"github.com/Wei-Shaw/sub2api/internal/config"
"github.com/stretchr/testify/require"
)
func TestCalculateOpenAI429ResetTime_7dExhausted(t *testing.T) {
svc := &RateLimitService{}
// Simulate headers when 7d limit is exhausted (100% used)
// Primary = 7d (10080 minutes), Secondary = 5h (300 minutes)
headers := http.Header{}
headers.Set("x-codex-primary-used-percent", "100")
headers.Set("x-codex-primary-reset-after-seconds", "384607") // ~4.5 days
headers.Set("x-codex-primary-window-minutes", "10080") // 7 days
headers.Set("x-codex-secondary-used-percent", "3")
headers.Set("x-codex-secondary-reset-after-seconds", "17369") // ~4.8 hours
headers.Set("x-codex-secondary-window-minutes", "300") // 5 hours
before := time.Now()
resetAt := svc.calculateOpenAI429ResetTime(headers)
after := time.Now()
if resetAt == nil {
t.Fatal("expected non-nil resetAt")
}
// Should be approximately 384607 seconds from now
expectedDuration := 384607 * time.Second
minExpected := before.Add(expectedDuration)
maxExpected := after.Add(expectedDuration)
if resetAt.Before(minExpected) || resetAt.After(maxExpected) {
t.Errorf("resetAt %v not in expected range [%v, %v]", resetAt, minExpected, maxExpected)
}
}
func TestCalculateOpenAI429ResetTime_5hExhausted(t *testing.T) {
svc := &RateLimitService{}
// Simulate headers when 5h limit is exhausted (100% used)
headers := http.Header{}
headers.Set("x-codex-primary-used-percent", "50")
headers.Set("x-codex-primary-reset-after-seconds", "500000")
headers.Set("x-codex-primary-window-minutes", "10080") // 7 days
headers.Set("x-codex-secondary-used-percent", "100")
headers.Set("x-codex-secondary-reset-after-seconds", "3600") // 1 hour
headers.Set("x-codex-secondary-window-minutes", "300") // 5 hours
before := time.Now()
resetAt := svc.calculateOpenAI429ResetTime(headers)
after := time.Now()
if resetAt == nil {
t.Fatal("expected non-nil resetAt")
}
// Should be approximately 3600 seconds from now
expectedDuration := 3600 * time.Second
minExpected := before.Add(expectedDuration)
maxExpected := after.Add(expectedDuration)
if resetAt.Before(minExpected) || resetAt.After(maxExpected) {
t.Errorf("resetAt %v not in expected range [%v, %v]", resetAt, minExpected, maxExpected)
}
}
func TestCalculateOpenAI429ResetTime_NeitherExhausted_UsesMax(t *testing.T) {
svc := &RateLimitService{}
// Neither limit at 100%, should use the longer reset time
headers := http.Header{}
headers.Set("x-codex-primary-used-percent", "80")
headers.Set("x-codex-primary-reset-after-seconds", "100000")
headers.Set("x-codex-primary-window-minutes", "10080")
headers.Set("x-codex-secondary-used-percent", "90")
headers.Set("x-codex-secondary-reset-after-seconds", "5000")
headers.Set("x-codex-secondary-window-minutes", "300")
before := time.Now()
resetAt := svc.calculateOpenAI429ResetTime(headers)
after := time.Now()
if resetAt == nil {
t.Fatal("expected non-nil resetAt")
}
// Should use the max (100000 seconds from 7d window)
expectedDuration := 100000 * time.Second
minExpected := before.Add(expectedDuration)
maxExpected := after.Add(expectedDuration)
if resetAt.Before(minExpected) || resetAt.After(maxExpected) {
t.Errorf("resetAt %v not in expected range [%v, %v]", resetAt, minExpected, maxExpected)
}
}
func TestCalculateOpenAI429ResetTime_NoCodexHeaders(t *testing.T) {
svc := &RateLimitService{}
// No codex headers at all
headers := http.Header{}
headers.Set("content-type", "application/json")
resetAt := svc.calculateOpenAI429ResetTime(headers)
if resetAt != nil {
t.Errorf("expected nil resetAt when no codex headers, got %v", resetAt)
}
}
func TestCalculateOpenAI429ResetTime_ReversedWindowOrder(t *testing.T) {
svc := &RateLimitService{}
// Test when OpenAI sends primary as 5h and secondary as 7d (reversed)
headers := http.Header{}
headers.Set("x-codex-primary-used-percent", "100") // This is 5h
headers.Set("x-codex-primary-reset-after-seconds", "3600") // 1 hour
headers.Set("x-codex-primary-window-minutes", "300") // 5 hours - smaller!
headers.Set("x-codex-secondary-used-percent", "50")
headers.Set("x-codex-secondary-reset-after-seconds", "500000")
headers.Set("x-codex-secondary-window-minutes", "10080") // 7 days - larger!
before := time.Now()
resetAt := svc.calculateOpenAI429ResetTime(headers)
after := time.Now()
if resetAt == nil {
t.Fatal("expected non-nil resetAt")
}
// Should correctly identify that primary is 5h (smaller window) and use its reset time
expectedDuration := 3600 * time.Second
minExpected := before.Add(expectedDuration)
maxExpected := after.Add(expectedDuration)
if resetAt.Before(minExpected) || resetAt.After(maxExpected) {
t.Errorf("resetAt %v not in expected range [%v, %v]", resetAt, minExpected, maxExpected)
}
}
type openAI429SnapshotRepo struct {
mockAccountRepoForGemini
rateLimitedID int64
updatedExtra map[string]any
bulkUpdatedIDs []int64
bulkUpdatedPayload AccountBulkUpdate
}
func (r *openAI429SnapshotRepo) SetRateLimited(_ context.Context, id int64, _ time.Time) error {
r.rateLimitedID = id
return nil
}
func (r *openAI429SnapshotRepo) UpdateExtra(_ context.Context, _ int64, updates map[string]any) error {
r.updatedExtra = updates
return nil
}
func (r *openAI429SnapshotRepo) BulkUpdate(_ context.Context, ids []int64, updates AccountBulkUpdate) (int64, error) {
r.bulkUpdatedIDs = append([]int64(nil), ids...)
r.bulkUpdatedPayload = updates
return int64(len(ids)), nil
}
func TestHandle429_OpenAIPersistsCodexSnapshotImmediately(t *testing.T) {
repo := &openAI429SnapshotRepo{}
svc := NewRateLimitService(repo, nil, nil, nil, nil)
account := &Account{ID: 123, Platform: PlatformOpenAI, Type: AccountTypeOAuth}
headers := http.Header{}
headers.Set("x-codex-primary-used-percent", "100")
headers.Set("x-codex-primary-reset-after-seconds", "604800")
headers.Set("x-codex-primary-window-minutes", "10080")
headers.Set("x-codex-secondary-used-percent", "100")
headers.Set("x-codex-secondary-reset-after-seconds", "18000")
headers.Set("x-codex-secondary-window-minutes", "300")
svc.handle429(context.Background(), account, headers, nil)
if repo.rateLimitedID != account.ID {
t.Fatalf("rateLimitedID = %d, want %d", repo.rateLimitedID, account.ID)
}
if len(repo.updatedExtra) == 0 {
t.Fatal("expected codex snapshot to be persisted on 429")
}
if got := repo.updatedExtra["codex_5h_used_percent"]; got != 100.0 {
t.Fatalf("codex_5h_used_percent = %v, want 100", got)
}
if got := repo.updatedExtra["codex_7d_used_percent"]; got != 100.0 {
t.Fatalf("codex_7d_used_percent = %v, want 100", got)
}
}
func TestHandle429_OpenAISyncsObservedPlanType(t *testing.T) {
repo := &openAI429SnapshotRepo{}
svc := NewRateLimitService(repo, nil, nil, nil, nil)
account := &Account{
ID: 124,
Platform: PlatformOpenAI,
Type: AccountTypeOAuth,
Credentials: map[string]any{"plan_type": "plus"},
}
body := []byte(`{"error":{"type":"usage_limit_reached","message":"limit reached","plan_type":"free","resets_at":1777283883}}`)
svc.handle429(context.Background(), account, http.Header{}, body)
require.Equal(t, []int64{account.ID}, repo.bulkUpdatedIDs)
require.Equal(t, "free", repo.bulkUpdatedPayload.Credentials["plan_type"])
require.Equal(t, "free", account.Credentials["plan_type"])
require.Equal(t, account.ID, repo.rateLimitedID)
}
// TestHandle429_SkipsSparkShadow 外审第8轮 P1:spark 影子的限流状态只由 QueryUsage(/wham/usage
// codex_bengalfox)维护;/responses 429 携带的 global x-codex-* 不得对影子做任何 DB 限流写入,
// 否则会把 spark 误耦合到 global codex 窗口、冷却到 global reset。
func TestHandle429_SkipsSparkShadow(t *testing.T) {
headers := http.Header{}
headers.Set("x-codex-primary-used-percent", "100")
headers.Set("x-codex-primary-reset-after-seconds", "604800")
headers.Set("x-codex-primary-window-minutes", "10080")
headers.Set("x-codex-secondary-used-percent", "100")
headers.Set("x-codex-secondary-reset-after-seconds", "18000")
headers.Set("x-codex-secondary-window-minutes", "300")
parentID := int64(900)
shadowRepo := &openAI429SnapshotRepo{}
shadowSvc := NewRateLimitService(shadowRepo, nil, nil, nil, nil)
shadow := &Account{
ID: 901,
Platform: PlatformOpenAI,
Type: AccountTypeOAuth,
ParentAccountID: &parentID,
QuotaDimension: QuotaDimensionSpark,
}
shadowSvc.handle429(context.Background(), shadow, headers, nil)
require.Zero(t, shadowRepo.rateLimitedID, "spark shadow must not be SetRateLimited from /responses global 429")
require.Empty(t, shadowRepo.updatedExtra, "spark shadow must not get a codex snapshot from /responses 429")
// 反向对照:普通 OpenAI OAuth 账号仍按 global 429 限流。
normalRepo := &openAI429SnapshotRepo{}
normalSvc := NewRateLimitService(normalRepo, nil, nil, nil, nil)
normal := &Account{ID: 902, Platform: PlatformOpenAI, Type: AccountTypeOAuth}
normalSvc.handle429(context.Background(), normal, headers, nil)
require.Equal(t, normal.ID, normalRepo.rateLimitedID, "normal OpenAI OAuth account should still be rate limited")
}
func TestNormalizedCodexLimits(t *testing.T) {
// Test the Normalize() method directly
pUsed := 100.0
pReset := 384607
pWindow := 10080
sUsed := 3.0
sReset := 17369
sWindow := 300
snapshot := &OpenAICodexUsageSnapshot{
PrimaryUsedPercent: &pUsed,
PrimaryResetAfterSeconds: &pReset,
PrimaryWindowMinutes: &pWindow,
SecondaryUsedPercent: &sUsed,
SecondaryResetAfterSeconds: &sReset,
SecondaryWindowMinutes: &sWindow,
}
normalized := snapshot.Normalize()
if normalized == nil {
t.Fatal("expected non-nil normalized")
}
// Primary has larger window (10080 > 300), so primary should be 7d
if normalized.Used7dPercent == nil || *normalized.Used7dPercent != 100.0 {
t.Errorf("expected Used7dPercent=100, got %v", normalized.Used7dPercent)
}
if normalized.Reset7dSeconds == nil || *normalized.Reset7dSeconds != 384607 {
t.Errorf("expected Reset7dSeconds=384607, got %v", normalized.Reset7dSeconds)
}
if normalized.Used5hPercent == nil || *normalized.Used5hPercent != 3.0 {
t.Errorf("expected Used5hPercent=3, got %v", normalized.Used5hPercent)
}
if normalized.Reset5hSeconds == nil || *normalized.Reset5hSeconds != 17369 {
t.Errorf("expected Reset5hSeconds=17369, got %v", normalized.Reset5hSeconds)
}
}
func TestNormalizedCodexLimits_OnlyPrimaryData(t *testing.T) {
// Test when only primary has data, no window_minutes
pUsed := 80.0
pReset := 50000
snapshot := &OpenAICodexUsageSnapshot{
PrimaryUsedPercent: &pUsed,
PrimaryResetAfterSeconds: &pReset,
// No window_minutes, no secondary data
}
normalized := snapshot.Normalize()
if normalized == nil {
t.Fatal("expected non-nil normalized")
}
// Legacy assumption: primary=7d, secondary=5h
if normalized.Used7dPercent == nil || *normalized.Used7dPercent != 80.0 {
t.Errorf("expected Used7dPercent=80, got %v", normalized.Used7dPercent)
}
if normalized.Reset7dSeconds == nil || *normalized.Reset7dSeconds != 50000 {
t.Errorf("expected Reset7dSeconds=50000, got %v", normalized.Reset7dSeconds)
}
// Secondary (5h) should be nil
if normalized.Used5hPercent != nil {
t.Errorf("expected Used5hPercent=nil, got %v", *normalized.Used5hPercent)
}
if normalized.Reset5hSeconds != nil {
t.Errorf("expected Reset5hSeconds=nil, got %v", *normalized.Reset5hSeconds)
}
}
func TestRateLimitService_HandleUpstreamError_403PreservesOriginalUpstreamMessage(t *testing.T) {
repo := &rateLimitAccountRepoStub{}
service := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
account := &Account{
ID: 201,
Platform: PlatformOpenAI,
Type: AccountTypeOAuth,
}
shouldDisable := service.HandleUpstreamError(
context.Background(),
account,
403,
http.Header{},
[]byte(`{"error":{"message":"workspace forbidden by policy","type":"invalid_request_error"}}`),
)
require.True(t, shouldDisable)
require.Equal(t, 1, repo.setErrorCalls)
require.Contains(t, repo.lastErrorMsg, "workspace forbidden by policy")
require.NotContains(t, repo.lastErrorMsg, "account may be suspended or lack permissions")
}
func TestRateLimitService_HandleUpstreamError_403FallsBackToRawBody(t *testing.T) {
repo := &rateLimitAccountRepoStub{}
service := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
account := &Account{
ID: 202,
Platform: PlatformOpenAI,
Type: AccountTypeOAuth,
}
shouldDisable := service.HandleUpstreamError(
context.Background(),
account,
403,
http.Header{},
[]byte(`{"error":{"type":"access_denied","details":{"reason":"ip_blocked"}}}`),
)
require.True(t, shouldDisable)
require.Equal(t, 1, repo.setErrorCalls)
require.Contains(t, repo.lastErrorMsg, `"access_denied"`)
require.Contains(t, repo.lastErrorMsg, `"ip_blocked"`)
require.NotContains(t, repo.lastErrorMsg, "account may be suspended or lack permissions")
}
func TestNormalizedCodexLimits_OnlySecondaryData(t *testing.T) {
// Test when only secondary has data, no window_minutes
sUsed := 60.0
sReset := 3000
snapshot := &OpenAICodexUsageSnapshot{
SecondaryUsedPercent: &sUsed,
SecondaryResetAfterSeconds: &sReset,
// No window_minutes, no primary data
}
normalized := snapshot.Normalize()
if normalized == nil {
t.Fatal("expected non-nil normalized")
}
// Legacy assumption: primary=7d, secondary=5h
// So secondary goes to 5h
if normalized.Used5hPercent == nil || *normalized.Used5hPercent != 60.0 {
t.Errorf("expected Used5hPercent=60, got %v", normalized.Used5hPercent)
}
if normalized.Reset5hSeconds == nil || *normalized.Reset5hSeconds != 3000 {
t.Errorf("expected Reset5hSeconds=3000, got %v", normalized.Reset5hSeconds)
}
// Primary (7d) should be nil
if normalized.Used7dPercent != nil {
t.Errorf("expected Used7dPercent=nil, got %v", *normalized.Used7dPercent)
}
}
func TestNormalizedCodexLimits_BothDataNoWindowMinutes(t *testing.T) {
// Test when both have data but no window_minutes
pUsed := 100.0
pReset := 400000
sUsed := 50.0
sReset := 10000
snapshot := &OpenAICodexUsageSnapshot{
PrimaryUsedPercent: &pUsed,
PrimaryResetAfterSeconds: &pReset,
SecondaryUsedPercent: &sUsed,
SecondaryResetAfterSeconds: &sReset,
// No window_minutes
}
normalized := snapshot.Normalize()
if normalized == nil {
t.Fatal("expected non-nil normalized")
}
// Legacy assumption: primary=7d, secondary=5h
if normalized.Used7dPercent == nil || *normalized.Used7dPercent != 100.0 {
t.Errorf("expected Used7dPercent=100, got %v", normalized.Used7dPercent)
}
if normalized.Reset7dSeconds == nil || *normalized.Reset7dSeconds != 400000 {
t.Errorf("expected Reset7dSeconds=400000, got %v", normalized.Reset7dSeconds)
}
if normalized.Used5hPercent == nil || *normalized.Used5hPercent != 50.0 {
t.Errorf("expected Used5hPercent=50, got %v", normalized.Used5hPercent)
}
if normalized.Reset5hSeconds == nil || *normalized.Reset5hSeconds != 10000 {
t.Errorf("expected Reset5hSeconds=10000, got %v", normalized.Reset5hSeconds)
}
}
func TestHandle429_AnthropicPlatformUnaffected(t *testing.T) {
// Verify that Anthropic platform accounts still use the original logic
// This test ensures we don't break existing Claude account rate limiting
svc := &RateLimitService{}
// Simulate Anthropic 429 headers
headers := http.Header{}
headers.Set("anthropic-ratelimit-unified-reset", "1737820800") // A future Unix timestamp
// For Anthropic platform, calculateOpenAI429ResetTime should return nil
// because it only handles OpenAI platform
resetAt := svc.calculateOpenAI429ResetTime(headers)
// Should return nil since there are no x-codex-* headers
if resetAt != nil {
t.Errorf("expected nil for Anthropic headers, got %v", resetAt)
}
}
func TestCalculateOpenAI429ResetTime_UserProvidedScenario(t *testing.T) {
// This is the exact scenario from the user:
// codex_7d_used_percent: 100
// codex_7d_reset_after_seconds: 384607 (约4.5天后重置)
// codex_5h_used_percent: 3
// codex_5h_reset_after_seconds: 17369 (约4.8小时后重置)
svc := &RateLimitService{}
// Simulate headers matching user's data
// Note: We need to map the canonical 5h/7d back to primary/secondary
// Based on typical OpenAI behavior: primary=7d (larger window), secondary=5h (smaller window)
headers := http.Header{}
headers.Set("x-codex-primary-used-percent", "100")
headers.Set("x-codex-primary-reset-after-seconds", "384607")
headers.Set("x-codex-primary-window-minutes", "10080") // 7 days = 10080 minutes
headers.Set("x-codex-secondary-used-percent", "3")
headers.Set("x-codex-secondary-reset-after-seconds", "17369")
headers.Set("x-codex-secondary-window-minutes", "300") // 5 hours = 300 minutes
before := time.Now()
resetAt := svc.calculateOpenAI429ResetTime(headers)
after := time.Now()
if resetAt == nil {
t.Fatal("expected non-nil resetAt for user scenario")
}
// Should use the 7d reset time (384607 seconds) since 7d limit is exhausted (100%)
expectedDuration := 384607 * time.Second
minExpected := before.Add(expectedDuration)
maxExpected := after.Add(expectedDuration)
if resetAt.Before(minExpected) || resetAt.After(maxExpected) {
t.Errorf("resetAt %v not in expected range [%v, %v]", resetAt, minExpected, maxExpected)
}
// Verify it's approximately 4.45 days (384607 seconds)
duration := resetAt.Sub(before)
actualDays := duration.Hours() / 24.0
// 384607 / 86400 = ~4.45 days
if actualDays < 4.4 || actualDays > 4.5 {
t.Errorf("expected ~4.45 days, got %.2f days", actualDays)
}
t.Logf("User scenario: reset_at=%v, duration=%.2f days", resetAt, actualDays)
}
func TestCalculateOpenAI429ResetTime_5MinFallbackWhenNoReset(t *testing.T) {
// Test that we return nil when there's used_percent but no reset_after_seconds
// This should cause the caller to use the default 5-minute fallback
svc := &RateLimitService{}
headers := http.Header{}
headers.Set("x-codex-primary-used-percent", "100")
// No reset_after_seconds!
resetAt := svc.calculateOpenAI429ResetTime(headers)
// Should return nil since there's no reset time available
if resetAt != nil {
t.Errorf("expected nil when no reset_after_seconds, got %v", resetAt)
}
}