57_Wolve
12bf35caf8
Greenfield Go multi-tenant IPFS Pinning Service wire-compatible with the
IPFS Pinning Services API spec. Paired 1:1 with Kubo over localhost RPC,
clustered via embedded NATS JetStream, Postgres source-of-truth with
RLS-enforced tenancy, Fiber + huma v2 for the HTTP surface, Authentik
OIDC for session login with kid-rotated HS256 JWT API tokens.
Feature-complete against the 22-milestone build plan, including the
ship-it v1.0 gap items:
* admin CLIs: drain/uncordon, maintenance, mint-token, rotate-key,
prune-denylist, rebalance --dry-run, cache-stats, cluster-presences
* TTL leader election via NATS KV, fence tokens, JetStream dedup
* rebalancer (plan/apply split), reconciler, requeue sweeper
* ristretto caches with NATS-backed cross-node invalidation
(placements live-nodes + token denylist)
* maintenance watchdog for stuck cluster-pause flag
* Prometheus /metrics with CIDR ACL, HTTP/pin/scheduler/cache gauges
* rate limiting: session (10/min) + anonymous global (120/min)
* integration tests: rebalance, refcount multi-org, RLS belt
* goreleaser (tar + deb/rpm/apk + Alpine Docker) targeting Gitea
Stack: Cobra/Viper, Fiber v2 + huma v2, embedded NATS JetStream,
pgx/sqlc/golang-migrate, ristretto, TypeID, prometheus/client_golang,
testcontainers-go.
112 lines
2.5 KiB
Go
112 lines
2.5 KiB
Go
package leader_test
|
|
|
|
import (
|
|
"context"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"anchorage/internal/pkg/leader"
|
|
embeddednats "anchorage/internal/pkg/nats"
|
|
)
|
|
|
|
// startEmbedded spins up an in-process NATS server for this test.
|
|
func startEmbedded(t *testing.T) *embeddednats.Server {
|
|
t.Helper()
|
|
srv, err := embeddednats.Start(context.Background(), embeddednats.ServerConfig{
|
|
ServerName: "leader-test",
|
|
DataDir: t.TempDir(),
|
|
ClientHost: "127.0.0.1",
|
|
ClientPort: -1,
|
|
JSReplicas: 1,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("embeddednats.Start: %v", err)
|
|
}
|
|
t.Cleanup(srv.Close)
|
|
return srv
|
|
}
|
|
|
|
func TestSingleCandidatePromotes(t *testing.T) {
|
|
srv := startEmbedded(t)
|
|
|
|
var promoted, demoted atomic.Int32
|
|
|
|
el, err := leader.New(context.Background(), srv.JS, leader.Options{
|
|
NodeID: "nod_solo",
|
|
TTL: time.Second,
|
|
RenewInterval: 200 * time.Millisecond,
|
|
ClaimInterval: 200 * time.Millisecond,
|
|
OnPromote: func(context.Context) { promoted.Add(1) },
|
|
OnDemote: func(context.Context) { demoted.Add(1) },
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("leader.New: %v", err)
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
|
defer cancel()
|
|
|
|
go func() { _ = el.Run(ctx) }()
|
|
|
|
// Wait up to 2s for promotion.
|
|
deadline := time.Now().Add(2 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
if el.IsLeader() {
|
|
break
|
|
}
|
|
time.Sleep(50 * time.Millisecond)
|
|
}
|
|
if !el.IsLeader() {
|
|
t.Fatalf("candidate never promoted within 2s")
|
|
}
|
|
if promoted.Load() != 1 {
|
|
t.Errorf("OnPromote fired %d times, want 1", promoted.Load())
|
|
}
|
|
|
|
cancel()
|
|
time.Sleep(300 * time.Millisecond)
|
|
|
|
if demoted.Load() < 1 {
|
|
t.Errorf("OnDemote did not fire on ctx cancel")
|
|
}
|
|
}
|
|
|
|
func TestTwoCandidatesExactlyOneWins(t *testing.T) {
|
|
srv := startEmbedded(t)
|
|
|
|
mk := func(id string) *leader.Elector {
|
|
el, err := leader.New(context.Background(), srv.JS, leader.Options{
|
|
NodeID: id,
|
|
TTL: time.Second,
|
|
RenewInterval: 200 * time.Millisecond,
|
|
ClaimInterval: 200 * time.Millisecond,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("leader.New: %v", err)
|
|
}
|
|
return el
|
|
}
|
|
|
|
a := mk("nod_a")
|
|
b := mk("nod_b")
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
|
|
go func() { _ = a.Run(ctx) }()
|
|
go func() { _ = b.Run(ctx) }()
|
|
|
|
deadline := time.Now().Add(3 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
if a.IsLeader() || b.IsLeader() {
|
|
break
|
|
}
|
|
time.Sleep(50 * time.Millisecond)
|
|
}
|
|
|
|
if a.IsLeader() == b.IsLeader() {
|
|
t.Fatalf("both candidates report leader=%v (expected exactly one true)", a.IsLeader())
|
|
}
|
|
}
|