57_Wolve
12bf35caf8
Greenfield Go multi-tenant IPFS Pinning Service wire-compatible with the
IPFS Pinning Services API spec. Paired 1:1 with Kubo over localhost RPC,
clustered via embedded NATS JetStream, Postgres source-of-truth with
RLS-enforced tenancy, Fiber + huma v2 for the HTTP surface, Authentik
OIDC for session login with kid-rotated HS256 JWT API tokens.
Feature-complete against the 22-milestone build plan, including the
ship-it v1.0 gap items:
* admin CLIs: drain/uncordon, maintenance, mint-token, rotate-key,
prune-denylist, rebalance --dry-run, cache-stats, cluster-presences
* TTL leader election via NATS KV, fence tokens, JetStream dedup
* rebalancer (plan/apply split), reconciler, requeue sweeper
* ristretto caches with NATS-backed cross-node invalidation
(placements live-nodes + token denylist)
* maintenance watchdog for stuck cluster-pause flag
* Prometheus /metrics with CIDR ACL, HTTP/pin/scheduler/cache gauges
* rate limiting: session (10/min) + anonymous global (120/min)
* integration tests: rebalance, refcount multi-org, RLS belt
* goreleaser (tar + deb/rpm/apk + Alpine Docker) targeting Gitea
Stack: Cobra/Viper, Fiber v2 + huma v2, embedded NATS JetStream,
pgx/sqlc/golang-migrate, ristretto, TypeID, prometheus/client_golang,
testcontainers-go.
87 lines
2.2 KiB
Go
87 lines
2.2 KiB
Go
package scheduler
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"time"
|
|
|
|
"github.com/nats-io/nats.go"
|
|
|
|
"anchorage/internal/pkg/pin"
|
|
"anchorage/internal/pkg/store"
|
|
)
|
|
|
|
// RequeueSweeper republishes pin jobs whose placements have been stuck
|
|
// in 'queued' past a threshold — covers the rare "Postgres committed
|
|
// but NATS publish lost" case. Leader-gated.
|
|
type RequeueSweeper struct {
|
|
Store store.Store
|
|
NC *nats.Conn
|
|
Interval time.Duration
|
|
StuckAfter time.Duration
|
|
// MaintenanceGate pauses the sweeper while cluster maintenance is on.
|
|
MaintenanceGate func(context.Context) bool
|
|
}
|
|
|
|
// Run ticks every Interval, republishing stuck placements as pin.jobs.<nodeID>
|
|
// with the placement's current fence value (so the publisher dedup
|
|
// absorbs any already-in-flight duplicate).
|
|
func (s *RequeueSweeper) Run(ctx context.Context) error {
|
|
ticker := time.NewTicker(s.Interval)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case <-ticker.C:
|
|
if s.MaintenanceGate != nil && s.MaintenanceGate(ctx) {
|
|
continue
|
|
}
|
|
if err := s.tick(ctx); err != nil {
|
|
slog.Warn("sweeper: tick", "err", err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s *RequeueSweeper) tick(ctx context.Context) error {
|
|
stuck, err := s.Store.Pins().StuckPlacements(ctx, s.StuckAfter)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, pl := range stuck {
|
|
pinRow, err := s.Store.Pins().GetByRequestID(ctx, pl.RequestID)
|
|
if err != nil {
|
|
if !errors.Is(err, store.ErrNotFound) {
|
|
slog.Warn("sweeper: get pin", "request_id", pl.RequestID, "err", err)
|
|
}
|
|
continue
|
|
}
|
|
job := pin.Job{
|
|
RequestID: pl.RequestID.String(),
|
|
OrgID: pinRow.OrgID.String(),
|
|
CID: pinRow.CID,
|
|
Origins: pinRow.Origins,
|
|
Fence: pl.Fence,
|
|
Action: "pin",
|
|
}
|
|
body, _ := json.Marshal(job)
|
|
msg := &nats.Msg{
|
|
Subject: fmt.Sprintf("%s.%s", pin.JobsSubjectPrefix, pl.NodeID.String()),
|
|
Data: body,
|
|
Header: nats.Header{},
|
|
}
|
|
msg.Header.Set("Nats-Msg-Id", fmt.Sprintf("%s:%s:%d", pl.RequestID, pl.NodeID, pl.Fence))
|
|
if err := s.NC.PublishMsg(msg); err != nil {
|
|
slog.Warn("sweeper: publish", "err", err)
|
|
continue
|
|
}
|
|
slog.Info("sweeper: requeued",
|
|
"request_id", pl.RequestID, "node_id", pl.NodeID, "fence", pl.Fence)
|
|
}
|
|
return nil
|
|
}
|