visa2andClaude Opus 4.8
362f9e77bf
Three periodic background jobs ran on every instance with no cross-instance coordination, multiplying their cost (and side effects) by the replica count: - DashboardAggregationService.runScheduledAggregation: N× heavy GROUP BY aggregation queries every minute plus watermark write races. - PaymentOrderExpiryService.runOnce: N× upstream payment-provider reconcile/ expiry API calls per pending order. - SubscriptionExpiryService.sendExpiryReminders: N× full active-subscription scans every minute and potential duplicate reminder emails. Add a LeaderLockCache abstraction so only one instance runs each job per cycle: - The interface lives in the service layer; the Redis-backed implementation (SetNX + compare-and-delete release) lives in the repository layer, so the service package keeps its depguard "must not import redis" boundary intact. - tryAcquireSingletonLeaderLock prefers the cache and falls back to a Postgres advisory lock when Redis errors, mirroring the Ops background services. When neither backend is configured the job runs ungated, preserving single-instance and test behavior (no self-lockout: the lock is released every cycle). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
80 lines
2.5 KiB
Go
80 lines
2.5 KiB
Go
//go:build unit
|
|
|
|
package repository
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/alicebob/miniredis/v2"
|
|
"github.com/redis/go-redis/v9"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func newLeaderLockTestCache(t *testing.T) (*leaderLockCache, *miniredis.Miniredis) {
|
|
t.Helper()
|
|
mr := miniredis.RunT(t)
|
|
rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
|
|
t.Cleanup(func() { _ = rdb.Close() })
|
|
return &leaderLockCache{rdb: rdb}, mr
|
|
}
|
|
|
|
func TestLeaderLockCache_AcquireContendedRelease(t *testing.T) {
|
|
cache, _ := newLeaderLockTestCache(t)
|
|
ctx := context.Background()
|
|
const key = "dashboard:aggregation:leader"
|
|
|
|
ok, err := cache.TryAcquireLeaderLock(ctx, key, "A", time.Minute)
|
|
require.NoError(t, err)
|
|
require.True(t, ok, "first owner should acquire")
|
|
|
|
ok, err = cache.TryAcquireLeaderLock(ctx, key, "B", time.Minute)
|
|
require.NoError(t, err)
|
|
require.False(t, ok, "peer must be locked out while held")
|
|
|
|
require.NoError(t, cache.ReleaseLeaderLock(ctx, key, "A"))
|
|
|
|
ok, err = cache.TryAcquireLeaderLock(ctx, key, "B", time.Minute)
|
|
require.NoError(t, err)
|
|
require.True(t, ok, "peer should acquire after release")
|
|
}
|
|
|
|
// A stale owner whose lock expired and was re-acquired by a peer must not delete
|
|
// the peer's lock when its late release fires (compare-and-delete by owner).
|
|
func TestLeaderLockCache_ReleaseIsCompareAndDelete(t *testing.T) {
|
|
cache, _ := newLeaderLockTestCache(t)
|
|
ctx := context.Background()
|
|
const key = "payment:order:expiry:leader"
|
|
|
|
ok, err := cache.TryAcquireLeaderLock(ctx, key, "A", time.Minute)
|
|
require.NoError(t, err)
|
|
require.True(t, ok)
|
|
|
|
// Simulate A's lock expiring and peer B taking ownership.
|
|
require.NoError(t, cache.rdb.Set(ctx, leaderLockKeyPrefix+key, "B", time.Minute).Err())
|
|
|
|
// A's stale release must be a no-op against B's lock.
|
|
require.NoError(t, cache.ReleaseLeaderLock(ctx, key, "A"))
|
|
|
|
val, err := cache.rdb.Get(ctx, leaderLockKeyPrefix+key).Result()
|
|
require.NoError(t, err)
|
|
require.Equal(t, "B", val, "stale owner must not delete the new owner's lock")
|
|
}
|
|
|
|
func TestLeaderLockCache_TTLExpires(t *testing.T) {
|
|
cache, mr := newLeaderLockTestCache(t)
|
|
ctx := context.Background()
|
|
const key = "subscription:expiry:reminder:leader"
|
|
|
|
ok, err := cache.TryAcquireLeaderLock(ctx, key, "A", time.Minute)
|
|
require.NoError(t, err)
|
|
require.True(t, ok)
|
|
|
|
mr.FastForward(2 * time.Minute)
|
|
|
|
ok, err = cache.TryAcquireLeaderLock(ctx, key, "B", time.Minute)
|
|
require.NoError(t, err)
|
|
require.True(t, ok, "lock should be re-acquirable after the TTL expires")
|
|
}
|