Greenfield Go multi-tenant IPFS Pinning Service wire-compatible with the
IPFS Pinning Services API spec. Paired 1:1 with Kubo over localhost RPC,
clustered via embedded NATS JetStream, Postgres source-of-truth with
RLS-enforced tenancy, Fiber + huma v2 for the HTTP surface, Authentik
OIDC for session login with kid-rotated HS256 JWT API tokens.
Feature-complete against the 22-milestone build plan, including the
ship-it v1.0 gap items:
* admin CLIs: drain/uncordon, maintenance, mint-token, rotate-key,
prune-denylist, rebalance --dry-run, cache-stats, cluster-presences
* TTL leader election via NATS KV, fence tokens, JetStream dedup
* rebalancer (plan/apply split), reconciler, requeue sweeper
* ristretto caches with NATS-backed cross-node invalidation
(placements live-nodes + token denylist)
* maintenance watchdog for stuck cluster-pause flag
* Prometheus /metrics with CIDR ACL, HTTP/pin/scheduler/cache gauges
* rate limiting: session (10/min) + anonymous global (120/min)
* integration tests: rebalance, refcount multi-org, RLS belt
* goreleaser (tar + deb/rpm/apk + Alpine Docker) targeting Gitea
Stack: Cobra/Viper, Fiber v2 + huma v2, embedded NATS JetStream,
pgx/sqlc/golang-migrate, ristretto, TypeID, prometheus/client_golang,
testcontainers-go.
104 lines
2.5 KiB
Go
104 lines
2.5 KiB
Go
package scheduler
|
|
|
|
import (
|
|
"context"
|
|
"log/slog"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"anchorage/internal/pkg/ids"
|
|
"anchorage/internal/pkg/ipfs"
|
|
"anchorage/internal/pkg/store"
|
|
)
|
|
|
|
// Reconciler diffs this node's Postgres placements against its Kubo
|
|
// daemon's actual pinset every Interval. Drift (CIDs anchorage believes
|
|
// are pinned but Kubo doesn't have, or vice versa) is counted and
|
|
// exposed via the DriftCount atomic so /v1/ready can surface it.
|
|
type Reconciler struct {
|
|
NodeID ids.NodeID
|
|
Store store.Store
|
|
Backend ipfs.Backend
|
|
Interval time.Duration
|
|
AutoRepair bool
|
|
// DriftCount is incremented on every drifted CID observed during a
|
|
// tick. Read atomically from any goroutine.
|
|
DriftCount atomic.Int64
|
|
}
|
|
|
|
// Run ticks every Interval until ctx is cancelled. Interval <= 0
|
|
// disables the loop — the reconciler returns immediately (ctx.Err()
|
|
// when caller cancels) so the composition root can opt out without
|
|
// branching.
|
|
func (r *Reconciler) Run(ctx context.Context) error {
|
|
if r.Interval <= 0 {
|
|
slog.Info("reconciler: disabled (Interval <= 0)", "node_id", r.NodeID)
|
|
<-ctx.Done()
|
|
return ctx.Err()
|
|
}
|
|
ticker := time.NewTicker(r.Interval)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case <-ticker.C:
|
|
if err := r.tick(ctx); err != nil {
|
|
slog.Warn("reconciler: tick", "err", err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (r *Reconciler) tick(ctx context.Context) error {
|
|
pgPlacements, err := r.Store.Pins().ListPlacementsForNode(ctx, r.NodeID, store.PinStatusPinned)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
kuboCIDs, err := r.Backend.List(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
pgSet := map[string]bool{}
|
|
for _, pl := range pgPlacements {
|
|
// Placements carry the request_id but we need the CID. For large
|
|
// clusters this should be replaced with a JOIN query; the memstore
|
|
// walks its map which is fine for dev and tests.
|
|
pinRow, err := r.Store.Pins().GetByRequestID(ctx, pl.RequestID)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
pgSet[pinRow.CID] = true
|
|
}
|
|
kuboSet := map[string]bool{}
|
|
for _, c := range kuboCIDs {
|
|
kuboSet[c] = true
|
|
}
|
|
|
|
drift := int64(0)
|
|
for cid := range pgSet {
|
|
if !kuboSet[cid] {
|
|
drift++
|
|
if r.AutoRepair {
|
|
if err := r.Backend.Pin(ctx, cid, nil); err != nil {
|
|
slog.Warn("reconciler: repin failed", "cid", cid, "err", err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
for cid := range kuboSet {
|
|
if !pgSet[cid] {
|
|
drift++
|
|
if r.AutoRepair {
|
|
_ = r.Backend.Unpin(ctx, cid)
|
|
}
|
|
}
|
|
}
|
|
r.DriftCount.Store(drift)
|
|
if drift > 0 {
|
|
slog.Info("reconciler: drift detected", "count", drift)
|
|
}
|
|
return nil
|
|
}
|