anchorage/internal/pkg/pin/service.go

// Package pin owns anchorage's pin-creation and placement logic.
//
// Service implements the commit-point rules from the plan:
//
//  1. One Postgres transaction per POST /v1/pins: upserts the pins row,
//     computes placements via rendezvous hashing, inserts pin_placements
//     rows with fence=1, bumps pin_refcount for each (node, cid), writes
//     an audit row.
//  2. On commit, publishes one pin.jobs.<nodeID> message per placement
//     with Nats-Msg-Id = <rid>:<nodeID>:<fence> for JetStream dedup.
//
// The NATS publisher is optional: if nil (tests, standalone mode), the
// transaction still commits but placements stay in 'queued' forever
// until someone wakes them up.
package pin

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"log/slog"

	"github.com/nats-io/nats.go"

	"anchorage/internal/pkg/ids"
	"anchorage/internal/pkg/metrics"
	"anchorage/internal/pkg/placement"
	"anchorage/internal/pkg/store"
)

// JobsSubjectPrefix is the NATS subject prefix for per-node pin work.
const JobsSubjectPrefix = "pin.jobs"

// EventsSubjectPrefix is the NATS subject prefix for pin status fan-out.
const EventsSubjectPrefix = "pin.events"

// Job is the body of a pin.jobs.<nodeID> message.
type Job struct {
	RequestID string   `json:"request_id"`
	OrgID     string   `json:"org_id"`
	CID       string   `json:"cid"`
	Origins   []string `json:"origins,omitempty"`
	Fence     int64    `json:"fence"`
	// Action = "pin" or "unpin". anchorage uses the same subject for
	// both so the consumer has one code path.
	Action string `json:"action"`
}

// Event is the body of a pin.events.<orgID>.<requestID> message.
type Event struct {
	RequestID string `json:"request_id"`
	OrgID     string `json:"org_id"`
	Status    string `json:"status"`
	NodeID    string `json:"node_id,omitempty"`
	Reason    string `json:"reason,omitempty"`
}

// Options configures the Service.
type Options struct {
	MinReplicas int
	NC          *nats.Conn
	// LiveNodes overrides how live-node discovery happens. nil falls back
	// to a pass-through store lookup on every POST /v1/pins; production
	// passes a *CachedLiveNodes so the hot path doesn't hit Postgres.
	LiveNodes LiveNodesSource
}

// Service is the pin-creation engine.
type Service struct {
	opts  Options
	store store.Store
	live  LiveNodesSource
}

// NewService constructs a Service. opts.MinReplicas must be >= 1.
func NewService(s store.Store, opts Options) (*Service, error) {
	if opts.MinReplicas < 1 {
		return nil, errors.New("pin: MinReplicas must be >= 1")
	}
	live := opts.LiveNodes
	if live == nil {
		live = storeLiveNodes{s: s}
	}
	return &Service{opts: opts, store: s, live: live}, nil
}

// CreateInput mirrors the spec's PinCreate body plus org context.
type CreateInput struct {
	OrgID   ids.OrgID
	CID     string
	Name    *string
	Meta    map[string]any
	Origins []string
}

// Create is the atomic pin-creation path. Returns the created Pin and
// the list of placements so the caller can report them back to the user.
//
// Idempotency: if a live pin already exists for (orgID, cid), Create
// returns that existing record with store.ErrConflict so the caller
// can map it to the spec's requirement of returning the existing
// requestid on duplicates. ErrConflict is not counted as an error in
// the PinOps metric — it's a valid idempotent-read outcome.
func (s *Service) Create(ctx context.Context, in CreateInput) (created *store.Pin, placements []*store.Placement, err error) {
	defer observePinErr("create", &err)
	if in.CID == "" {
		return nil, nil, errors.New("pin: CID is required")
	}
	if existing, err := s.store.Pins().GetLiveByCID(ctx, in.OrgID, in.CID); err == nil {
		return existing, nil, store.ErrConflict
	} else if !errors.Is(err, store.ErrNotFound) {
		return nil, nil, err
	}

	liveNodes, err := s.live.Live(ctx)
	if err != nil {
		return nil, nil, fmt.Errorf("pin: list live nodes: %w", err)
	}
	if len(liveNodes) == 0 {
		return nil, nil, errors.New("pin: no live nodes available for placement")
	}

	candidates := make([]placement.Node, 0, len(liveNodes))
	for _, n := range liveNodes {
		candidates = append(candidates, placement.Node{ID: n.ID.String(), Status: n.Status})
	}
	want := s.opts.MinReplicas
	if want > len(candidates) {
		want = len(candidates)
	}
	selected := placement.Compute(in.OrgID.String(), in.CID, want, candidates)

	pinID, err := ids.NewPin()
	if err != nil {
		return nil, nil, err
	}
	newPin := &store.Pin{
		RequestID: pinID,
		OrgID:     in.OrgID,
		CID:       in.CID,
		Name:      in.Name,
		Meta:      in.Meta,
		Origins:   in.Origins,
		Status:    store.PinStatusQueued,
	}

	err = s.store.Tx(ctx, func(txCtx context.Context, tx store.Store) error {
		if err := tx.Pins().Create(txCtx, newPin); err != nil {
			return err
		}
		for _, nodeStr := range selected {
			nodeID, err := ids.ParseNode(nodeStr)
			if err != nil {
				return err
			}
			pl, err := tx.Pins().InsertPlacement(txCtx, pinID, nodeID, 1)
			if err != nil {
				return err
			}
			if err := tx.Pins().IncRefcount(txCtx, nodeID, in.CID); err != nil {
				return err
			}
			placements = append(placements, pl)
		}
		return tx.Audit().Insert(txCtx, &store.AuditEntry{
			OrgID:  &in.OrgID,
			Action: "pin.create",
			Target: pinID.String(),
			Result: "ok",
			Detail: map[string]any{"cid": in.CID, "placements": selected},
		})
	})
	if err != nil {
		return nil, nil, err
	}

	// Publish per-placement jobs only after commit succeeds.
	s.publishJobs(newPin, placements)

	metrics.PinOps.WithLabelValues("create", "ok").Inc()
	return newPin, placements, nil
}

// Replace swaps a pin's CID + related metadata atomically, reshuffling
// placements and refcounts inside a single Postgres transaction. Unpin
// jobs for the old CID and pin jobs for the new CID are published
// after commit.
//
// Spec: POST /v1/pins/{requestid}.
func (s *Service) Replace(ctx context.Context, orgID ids.OrgID, rid ids.PinID, in CreateInput) (replaced *store.Pin, placements []*store.Placement, err error) {
	defer observePinErr("replace", &err)
	if in.CID == "" {
		return nil, nil, errors.New("pin: CID is required")
	}

	// Fetch old placements first — Delete-in-tx + cascade would lose
	// them before we get a chance to issue unpin jobs.
	oldPlacements, err := s.store.Pins().ListPlacements(ctx, rid)
	if err != nil {
		return nil, nil, err
	}

	// Compute new placements from the current live-node set.
	liveNodes, err := s.live.Live(ctx)
	if err != nil {
		return nil, nil, fmt.Errorf("pin: list live nodes: %w", err)
	}
	if len(liveNodes) == 0 {
		return nil, nil, errors.New("pin: no live nodes available for placement")
	}
	candidates := make([]placement.Node, 0, len(liveNodes))
	for _, n := range liveNodes {
		candidates = append(candidates, placement.Node{ID: n.ID.String(), Status: n.Status})
	}
	want := s.opts.MinReplicas
	if want > len(candidates) {
		want = len(candidates)
	}
	selected := placement.Compute(orgID.String(), in.CID, want, candidates)

	var (
		newPin        *store.Pin
		newPlacements []*store.Placement
		oldCID        string
	)
	err = s.store.Tx(ctx, func(txCtx context.Context, tx store.Store) error {
		existing, err := tx.Pins().Get(txCtx, orgID, rid)
		if err != nil {
			return err
		}
		oldCID = existing.CID

		// Decrement + GC old refcounts so a replace that keeps the pin
		// on the same nodes doesn't double-count.
		for _, pl := range oldPlacements {
			if _, err := tx.Pins().DecRefcount(txCtx, pl.NodeID, existing.CID); err != nil && !errors.Is(err, store.ErrNotFound) {
				return err
			}
			_ = tx.Pins().DeleteRefcountIfZero(txCtx, pl.NodeID, existing.CID)
		}

		replaced, err := tx.Pins().Replace(txCtx, orgID, rid, in.CID, in.Name, in.Meta, in.Origins)
		if err != nil {
			return err
		}
		newPin = replaced

		// Drop old placement rows that are NOT reused, insert placement
		// rows for the new set. Fence monotonically bumps via
		// ReplacePlacement so any in-flight worker on the old fence
		// no-ops its completion.
		keep := map[ids.NodeID]bool{}
		for _, nodeStr := range selected {
			nodeID, perr := ids.ParseNode(nodeStr)
			if perr != nil {
				return perr
			}
			keep[nodeID] = true
		}
		for _, pl := range oldPlacements {
			if keep[pl.NodeID] {
				continue
			}
			// Non-kept old placement: the CASCADE on Delete below will
			// drop it. Nothing to do here.
			_ = pl
		}
		for _, nodeStr := range selected {
			nodeID, _ := ids.ParseNode(nodeStr)
			// If placement already exists, ReplacePlacement bumps the
			// fence; otherwise insert fresh.
			if _, err := tx.Pins().GetPlacement(txCtx, rid, nodeID); err == nil {
				bumped, err := tx.Pins().ReplacePlacement(txCtx, rid, nodeID, nodeID)
				if err != nil {
					return err
				}
				newPlacements = append(newPlacements, bumped)
			} else {
				pl, err := tx.Pins().InsertPlacement(txCtx, rid, nodeID, 1)
				if err != nil {
					return err
				}
				newPlacements = append(newPlacements, pl)
			}
			if err := tx.Pins().IncRefcount(txCtx, nodeID, in.CID); err != nil {
				return err
			}
		}
		return tx.Audit().Insert(txCtx, &store.AuditEntry{
			OrgID:  &orgID,
			Action: "pin.replace",
			Target: rid.String(),
			Result: "ok",
			Detail: map[string]any{"from_cid": existing.CID, "to_cid": in.CID, "placements": selected},
		})
	})
	if err != nil {
		return nil, nil, err
	}

	// Unpin jobs for the old placements that are no longer kept.
	newNodes := map[ids.NodeID]bool{}
	for _, pl := range newPlacements {
		newNodes[pl.NodeID] = true
	}
	for _, pl := range oldPlacements {
		if newNodes[pl.NodeID] {
			continue
		}
		s.publishJob(Job{
			RequestID: rid.String(),
			OrgID:     orgID.String(),
			CID:       oldCID,
			Fence:     pl.Fence,
			Action:    "unpin",
		}, pl.NodeID.String())
	}
	// Pin jobs for the new placements.
	s.publishJobs(newPin, newPlacements)
	metrics.PinOps.WithLabelValues("replace", "ok").Inc()
	return newPin, newPlacements, nil
}

// Delete removes the pin and publishes unpin jobs for each placement.
func (s *Service) Delete(ctx context.Context, orgID ids.OrgID, rid ids.PinID) (err error) {
	defer observePinErr("delete", &err)
	pin, err := s.store.Pins().Get(ctx, orgID, rid)
	if err != nil {
		return err
	}
	placements, err := s.store.Pins().ListPlacements(ctx, rid)
	if err != nil {
		return err
	}
	err = s.store.Tx(ctx, func(txCtx context.Context, tx store.Store) error {
		if err := tx.Pins().Delete(txCtx, orgID, rid); err != nil {
			return err
		}
		return tx.Audit().Insert(txCtx, &store.AuditEntry{
			OrgID:  &orgID,
			Action: "pin.delete",
			Target: rid.String(),
			Result: "ok",
		})
	})
	if err != nil {
		return err
	}
	for _, pl := range placements {
		s.publishJob(Job{
			RequestID: rid.String(),
			OrgID:     orgID.String(),
			CID:       pin.CID,
			Fence:     pl.Fence,
			Action:    "unpin",
		}, pl.NodeID.String())
	}
	metrics.PinOps.WithLabelValues("delete", "ok").Inc()
	return nil
}

func (s *Service) publishJobs(pin *store.Pin, placements []*store.Placement) {
	for _, pl := range placements {
		s.publishJob(Job{
			RequestID: pin.RequestID.String(),
			OrgID:     pin.OrgID.String(),
			CID:       pin.CID,
			Origins:   pin.Origins,
			Fence:     pl.Fence,
			Action:    "pin",
		}, pl.NodeID.String())
	}
}

func (s *Service) publishJob(j Job, nodeID string) {
	if s.opts.NC == nil {
		slog.Debug("pin: NC nil, job not published", "request_id", j.RequestID, "node_id", nodeID)
		return
	}
	body, err := json.Marshal(j)
	if err != nil {
		slog.Error("pin: marshal job", "err", err)
		return
	}
	msg := &nats.Msg{
		Subject: fmt.Sprintf("%s.%s", JobsSubjectPrefix, nodeID),
		Data:    body,
		Header:  nats.Header{},
	}
	// JetStream publisher-side dedup: a retry within the dedup window is
	// absorbed by the broker.
	msg.Header.Set("Nats-Msg-Id", fmt.Sprintf("%s:%s:%d", j.RequestID, nodeID, j.Fence))
	if err := s.opts.NC.PublishMsg(msg); err != nil {
		slog.Error("pin: publish job", "err", err, "subject", msg.Subject)
	}
}

// observePinErr is deferred by every public Service method; when the
// named err return is non-nil (and not ErrConflict, which is a valid
// idempotent outcome), it increments the PinOps error counter. Success
// paths stay explicit at the end of each method.
func observePinErr(op string, errPtr *error) {
	if errPtr == nil || *errPtr == nil {
		return
	}
	if errors.Is(*errPtr, store.ErrConflict) {
		return
	}
	metrics.PinOps.WithLabelValues(op, "err").Inc()
}

// PublishEvent fans a pin status change out to WebSocket subscribers.
func PublishEvent(nc *nats.Conn, e Event) {
	if nc == nil {
		return
	}
	body, err := json.Marshal(e)
	if err != nil {
		slog.Error("pin: marshal event", "err", err)
		return
	}
	subj := fmt.Sprintf("%s.%s.%s", EventsSubjectPrefix, e.OrgID, e.RequestID)
	if err := nc.Publish(subj, body); err != nil {
		slog.Warn("pin: publish event", "err", err, "subject", subj)
	}
}