at-container-registry/pkg/auth/hold_remote.go

package auth

import (
	"context"
	"database/sql"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log/slog"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"sync"
	"time"

	"atcr.io/pkg/atproto"
)

// RemoteHoldAuthorizer queries a hold's PDS via XRPC endpoints
// Used by AppView to authorize access to remote holds
// Implements caching for captain records to reduce XRPC calls
type RemoteHoldAuthorizer struct {
	db                 *sql.DB
	httpClient         *http.Client
	cacheTTL           time.Duration   // TTL for captain record cache
	recentDenials      sync.Map        // In-memory cache for first denials
	stopCleanup        chan struct{}   // Signal to stop cleanup goroutine
	firstDenialBackoff time.Duration   // Backoff duration for first denial (default: 10s)
	cleanupInterval    time.Duration   // Cleanup goroutine interval (default: 10s)
	cleanupGracePeriod time.Duration   // Grace period before cleanup (default: 5s)
	dbBackoffDurations []time.Duration // Backoff durations for DB denials (default: [1m, 5m, 15m, 1h])
}

// denialEntry stores timestamp for in-memory first denials
type denialEntry struct {
	timestamp time.Time
}

// NewRemoteHoldAuthorizer creates a new remote authorizer for AppView with production defaults
func NewRemoteHoldAuthorizer(db *sql.DB) HoldAuthorizer {
	return NewRemoteHoldAuthorizerWithBackoffs(db,
		10*time.Second, // firstDenialBackoff
		10*time.Second, // cleanupInterval
		5*time.Second,  // cleanupGracePeriod
		[]time.Duration{ // dbBackoffDurations
			1 * time.Minute,
			5 * time.Minute,
			15 * time.Minute,
			60 * time.Minute,
		},
	)
}

// NewRemoteHoldAuthorizerWithBackoffs creates a new remote authorizer with custom backoff durations
// Used for testing to avoid long sleeps
func NewRemoteHoldAuthorizerWithBackoffs(db *sql.DB, firstDenialBackoff, cleanupInterval, cleanupGracePeriod time.Duration, dbBackoffDurations []time.Duration) HoldAuthorizer {
	a := &RemoteHoldAuthorizer{
		db: db,
		httpClient: &http.Client{
			Timeout: 10 * time.Second,
		},
		cacheTTL:           1 * time.Hour, // 1 hour cache TTL
		stopCleanup:        make(chan struct{}),
		firstDenialBackoff: firstDenialBackoff,
		cleanupInterval:    cleanupInterval,
		cleanupGracePeriod: cleanupGracePeriod,
		dbBackoffDurations: dbBackoffDurations,
	}

	// Start cleanup goroutine for in-memory denials
	go a.cleanupRecentDenials()

	return a
}

// cleanupRecentDenials runs periodically to remove expired first-denial entries
func (a *RemoteHoldAuthorizer) cleanupRecentDenials() {
	ticker := time.NewTicker(a.cleanupInterval)
	defer ticker.Stop()

	for {
		select {
		case <-ticker.C:
			now := time.Now()
			a.recentDenials.Range(func(key, value any) bool {
				entry := value.(denialEntry)
				// Remove entries older than backoff + grace period
				if now.Sub(entry.timestamp) > a.firstDenialBackoff+a.cleanupGracePeriod {
					a.recentDenials.Delete(key)
				}
				return true
			})
		case <-a.stopCleanup:
			return
		}
	}
}

// GetCaptainRecord retrieves a captain record with caching
// 1. Check database cache
// 2. If cache miss or expired, query hold's XRPC endpoint
// 3. Update cache
func (a *RemoteHoldAuthorizer) GetCaptainRecord(ctx context.Context, holdDID string) (*atproto.CaptainRecord, error) {
	// Try cache first
	var staleCached *captainRecordWithMeta
	if a.db != nil {
		cached, err := a.getCachedCaptainRecord(holdDID)
		if err == nil && cached != nil {
			// Cache hit - check if still valid
			if time.Since(cached.UpdatedAt) < a.cacheTTL {
				return cached.CaptainRecord, nil
			}
			// Cache expired - keep as fallback in case XRPC fetch fails
			staleCached = cached
		}
	}

	// Cache miss or expired - query XRPC endpoint
	record, err := a.fetchCaptainRecordFromXRPC(ctx, holdDID)
	if err != nil {
		// If the hold is unreachable but we have stale cache, use it.
		// Successor fields don't change once set, so stale data is safe.
		if staleCached != nil {
			slog.Warn("Captain record fetch failed, using stale cache",
				"holdDID", holdDID,
				"cache_age", time.Since(staleCached.UpdatedAt),
				"error", err)
			return staleCached.CaptainRecord, nil
		}
		// A hold DID that can never resolve is a property of stored user data
		// (a stale or hand-edited sailor profile defaultHold), not a fault on
		// our side, and nothing is cached on this path — so it re-logs on every
		// single request for that user. Two such accounts once produced 99.9%
		// of this service's ERROR lines. Keep ERROR for failures an operator
		// could actually act on.
		if errors.Is(err, atproto.ErrHoldDIDPermanent) {
			slog.Debug("Captain record fetch skipped, hold DID cannot be resolved",
				"holdDID", holdDID,
				"denial_reason", "captain_record_fetch_failed",
				"error", err)
		} else {
			slog.Error("Captain record fetch failed",
				"holdDID", holdDID,
				"denial_reason", "captain_record_fetch_failed",
				"error", err)
		}
		return nil, fmt.Errorf("failed to get captain record for %s: %w", holdDID, err)
	}

	// Cache only records published by a DID that actually runs a hold.
	//
	// Any account can put an io.atcr.hold.captain record in its own repo, and
	// ResolveHoldURL falls back to the DID's PDS endpoint when there is no
	// atcr_hold service — so a user who points their sailor profile at their own
	// DID reaches this path with a record they wrote themselves. Cached, it
	// becomes a row in hold_captain_records, and GetAvailableHolds offers every
	// row with allow_all_crew=1 to every user's hold picker.
	//
	// 69307c0 closed this for the two Jetstream writers and left this third one
	// open. Gate the CACHE only, not the authorization decision: an unresolvable
	// DID during a PLC blip would otherwise fail a legitimate push, and the
	// freshly-fetched record is no less trustworthy than it was before.
	if a.db != nil && a.cacheableHold(ctx, holdDID) {
		if err := a.setCachedCaptainRecord(holdDID, record); err != nil {
			// Log error but don't fail - caching is best-effort
			slog.Warn("Failed to cache captain record", "error", err, "holdDID", holdDID)
		}
	}

	return record, nil
}

// hasHoldService reports whether a DID advertises an atcr_hold service. It is a
// package var so tests can exercise the negative case: the real implementation
// trusts any did:web in test mode, which is exactly the shape most tests use.
var hasHoldService = atproto.HasHoldService

// captainRecordWithMeta includes UpdatedAt for cache management
type captainRecordWithMeta struct {
	*atproto.CaptainRecord
	UpdatedAt time.Time
}

// getCachedCaptainRecord retrieves a captain record from database cache
func (a *RemoteHoldAuthorizer) getCachedCaptainRecord(holdDID string) (*captainRecordWithMeta, error) {
	query := `
		SELECT owner_did, public, allow_all_crew, deployed_at, region, successor, updated_at
		FROM hold_captain_records
		WHERE hold_did = ?
	`

	var record atproto.CaptainRecord
	var deployedAt, region, successor sql.NullString
	var updatedAt time.Time

	err := a.db.QueryRow(query, holdDID).Scan(
		&record.Owner,
		&record.Public,
		&record.AllowAllCrew,
		&deployedAt,
		&region,
		&successor,
		&updatedAt,
	)

	if err == sql.ErrNoRows {
		return nil, nil // Cache miss
	}

	if err != nil {
		return nil, fmt.Errorf("cache query failed: %w", err)
	}

	// Handle nullable fields
	if deployedAt.Valid {
		record.DeployedAt = deployedAt.String
	}
	if region.Valid {
		record.Region = region.String
	}
	if successor.Valid {
		record.Successor = successor.String
	}

	return &captainRecordWithMeta{
		CaptainRecord: &record,
		UpdatedAt:     updatedAt,
	}, nil
}

// cacheableHold reports whether a captain record from holdDID may be cached.
// Errors are treated as "do not cache" rather than "deny": the record is still
// returned to the caller, it just does not become a durable row.
func (a *RemoteHoldAuthorizer) cacheableHold(ctx context.Context, holdDID string) bool {
	isHold, err := hasHoldService(ctx, holdDID)
	if err != nil {
		slog.Warn("Not caching captain record; hold DID unresolvable",
			"holdDID", holdDID, "error", err)
		return false
	}
	if !isHold {
		slog.Info("Not caching captain record from non-hold DID", "holdDID", holdDID)
		return false
	}
	return true
}

// setCachedCaptainRecord stores a captain record in database cache
func (a *RemoteHoldAuthorizer) setCachedCaptainRecord(holdDID string, record *atproto.CaptainRecord) error {
	query := `
		INSERT INTO hold_captain_records (
			hold_did, owner_did, public, allow_all_crew,
			deployed_at, region, successor, updated_at
		) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
		ON CONFLICT(hold_did) DO UPDATE SET
			owner_did = excluded.owner_did,
			public = excluded.public,
			allow_all_crew = excluded.allow_all_crew,
			deployed_at = excluded.deployed_at,
			region = excluded.region,
			successor = excluded.successor,
			updated_at = excluded.updated_at
	`

	_, err := a.db.Exec(query,
		holdDID,
		record.Owner,
		record.Public,
		record.AllowAllCrew,
		nullString(record.DeployedAt),
		nullString(record.Region),
		nullString(record.Successor),
		time.Now(),
	)

	return err
}

// fetchCaptainRecordFromXRPC queries the hold's XRPC endpoint for captain record
func (a *RemoteHoldAuthorizer) fetchCaptainRecordFromXRPC(ctx context.Context, holdDID string) (*atproto.CaptainRecord, error) {
	// Resolve DID to URL
	holdURL, err := atproto.ResolveHoldURL(ctx, holdDID)
	if err != nil {
		return nil, fmt.Errorf("failed to resolve hold URL: %w", err)
	}

	// Build XRPC request URL
	// GET /xrpc/com.atproto.repo.getRecord?repo={did}&collection=io.atcr.hold.captain&rkey=self
	xrpcURL := fmt.Sprintf("%s%s?repo=%s&collection=%s&rkey=self",
		holdURL, atproto.RepoGetRecord, url.QueryEscape(holdDID), url.QueryEscape(atproto.CaptainCollection))

	req, err := http.NewRequestWithContext(ctx, "GET", xrpcURL, nil)
	if err != nil {
		return nil, err
	}

	resp, err := a.httpClient.Do(req)
	if err != nil {
		return nil, fmt.Errorf("XRPC request failed: %w", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		body, _ := io.ReadAll(resp.Body)
		return nil, fmt.Errorf("XRPC request failed: status %d: %s", resp.StatusCode, string(body))
	}

	// Parse response
	var xrpcResp struct {
		URI   string `json:"uri"`
		CID   string `json:"cid"`
		Value struct {
			Type         string `json:"$type"`
			Owner        string `json:"owner"`
			Public       bool   `json:"public"`
			AllowAllCrew bool   `json:"allowAllCrew"`
			DeployedAt   string `json:"deployedAt"`
			Region       string `json:"region,omitempty"`
			Successor    string `json:"successor,omitempty"`
		} `json:"value"`
	}

	if err := json.NewDecoder(resp.Body).Decode(&xrpcResp); err != nil {
		return nil, fmt.Errorf("failed to decode XRPC response: %w", err)
	}

	// Convert to our type
	record := &atproto.CaptainRecord{
		Type:         atproto.CaptainCollection,
		Owner:        xrpcResp.Value.Owner,
		Public:       xrpcResp.Value.Public,
		AllowAllCrew: xrpcResp.Value.AllowAllCrew,
		DeployedAt:   xrpcResp.Value.DeployedAt,
		Region:       xrpcResp.Value.Region,
		Successor:    xrpcResp.Value.Successor,
	}

	return record, nil
}

// IsCrewMember checks if userDID is a crew member with caching
// 1. Check approval cache (15min TTL)
// 2. Check denial cache with exponential backoff
// 3. If cache miss, query XRPC endpoint and update cache
func (a *RemoteHoldAuthorizer) IsCrewMember(ctx context.Context, holdDID, userDID string) (bool, error) {
	// Skip caching if no database
	if a.db == nil {
		return a.isCrewMemberNoCache(ctx, holdDID, userDID)
	}

	// Check approval cache first (15min TTL)
	if approved, err := a.getCachedApproval(holdDID, userDID); err == nil && approved {
		slog.Debug("Using cached crew approval", "holdDID", holdDID, "userDID", userDID)
		return true, nil
	}

	// Check denial cache with backoff
	if blocked, err := a.isBlockedByDenialBackoff(holdDID, userDID); err == nil && blocked {
		// Still in backoff period - don't query again
		// Detailed logging already emitted by isBlockedByDenialBackoff
		return false, nil
	}

	// Cache miss or expired - query XRPC endpoint
	slog.Debug("Crew membership cache miss, querying hold", "holdDID", holdDID, "userDID", userDID)
	isCrew, err := a.isCrewMemberNoCache(ctx, holdDID, userDID)
	if err != nil {
		slog.Warn("Crew membership query error", "error", err, "holdDID", holdDID, "userDID", userDID)
		return false, err
	}

	// Update cache based on result
	if isCrew {
		// Cache approval for 15 minutes
		slog.Debug("Crew membership approved, caching for 15min", "holdDID", holdDID, "userDID", userDID)
		_ = a.cacheApproval(holdDID, userDID, 15*time.Minute)
	} else {
		// Cache denial with exponential backoff
		slog.Debug("Crew membership denied, caching with backoff", "holdDID", holdDID, "userDID", userDID)
		_ = a.cacheDenial(holdDID, userDID)
	}

	return isCrew, nil
}

// isCrewMemberNoCache queries XRPC without caching (internal helper)
// Uses O(1) lookup via getRecord with hash-based rkey instead of pagination
func (a *RemoteHoldAuthorizer) isCrewMemberNoCache(ctx context.Context, holdDID, userDID string) (bool, error) {
	// Resolve DID to URL
	holdURL, err := atproto.ResolveHoldURL(ctx, holdDID)
	if err != nil {
		return false, fmt.Errorf("failed to resolve hold URL: %w", err)
	}

	// Generate deterministic rkey from member DID (hash-based)
	rkey := atproto.CrewRecordKey(userDID)

	// Build XRPC request URL for direct record lookup
	// GET /xrpc/com.atproto.repo.getRecord?repo={did}&collection=io.atcr.hold.crew&rkey={hash}
	xrpcURL := fmt.Sprintf("%s%s?repo=%s&collection=%s&rkey=%s",
		holdURL, atproto.RepoGetRecord, url.QueryEscape(holdDID), url.QueryEscape(atproto.CrewCollection), url.QueryEscape(rkey))

	req, err := http.NewRequestWithContext(ctx, "GET", xrpcURL, nil)
	if err != nil {
		return false, err
	}

	resp, err := a.httpClient.Do(req)
	if err != nil {
		return false, fmt.Errorf("XRPC request failed: %w", err)
	}
	defer resp.Body.Close()

	// 404 means not a crew member (record doesn't exist)
	if resp.StatusCode == http.StatusNotFound {
		return false, nil
	}

	if resp.StatusCode != http.StatusOK {
		body, _ := io.ReadAll(resp.Body)
		return false, fmt.Errorf("XRPC request failed: status %d: %s", resp.StatusCode, string(body))
	}

	// Parse response to verify the member DID matches
	var xrpcResp struct {
		URI   string `json:"uri"`
		CID   string `json:"cid"`
		Value struct {
			Type        string   `json:"$type"`
			Member      string   `json:"member"`
			Role        string   `json:"role"`
			Permissions []string `json:"permissions"`
			AddedAt     string   `json:"addedAt"`
		} `json:"value"`
	}

	if err := json.NewDecoder(resp.Body).Decode(&xrpcResp); err != nil {
		return false, fmt.Errorf("failed to decode XRPC response: %w", err)
	}

	// Verify the member DID matches (sanity check)
	if xrpcResp.Value.Member == userDID {
		return true, nil
	}

	// Hash collision or invalid record - treat as not a member
	return false, nil
}

// IsCachedCrewMember returns true if there is a non-expired approval row.
// Never makes network calls. Cache miss or no DB returns (false, nil).
func (a *RemoteHoldAuthorizer) IsCachedCrewMember(ctx context.Context, holdDID, userDID string) (bool, error) {
	if a.db == nil {
		return false, nil
	}
	return a.getCachedApproval(holdDID, userDID)
}

// RecordCrewApproval writes an approval to the cache with the standard 15-min TTL.
// No-op if there is no DB.
func (a *RemoteHoldAuthorizer) RecordCrewApproval(ctx context.Context, holdDID, userDID string) error {
	if a.db == nil {
		return nil
	}
	return a.cacheApproval(holdDID, userDID, 15*time.Minute)
}

// CheckReadAccess implements read authorization using shared logic
func (a *RemoteHoldAuthorizer) CheckReadAccess(ctx context.Context, holdDID, userDID string) (bool, error) {
	captain, err := a.GetCaptainRecord(ctx, holdDID)
	if err != nil {
		return false, err
	}

	// Only a private hold needs the crew lookup, and only for a caller who
	// could be crew. Public holds and anonymous callers are decided by the
	// captain record alone, which keeps the common pull path free of the
	// crew query (a cached XRPC round trip on this implementation).
	isCrew := false
	if !captain.Public && userDID != "" && userDID != captain.Owner {
		isCrew, err = a.IsCrewMember(ctx, holdDID, userDID)
		if err != nil {
			return false, err
		}
	}

	return CheckReadAccessWithCaptain(captain, userDID, isCrew), nil
}

// CheckWriteAccess implements write authorization using shared logic
func (a *RemoteHoldAuthorizer) CheckWriteAccess(ctx context.Context, holdDID, userDID string) (bool, error) {
	captain, err := a.GetCaptainRecord(ctx, holdDID)
	if err != nil {
		return false, err
	}

	isCrew, err := a.IsCrewMember(ctx, holdDID, userDID)
	if err != nil {
		return false, err
	}

	return CheckWriteAccessWithCaptain(captain, userDID, isCrew), nil
}

// nullString converts a string to sql.NullString
func nullString(s string) sql.NullString {
	if s == "" {
		return sql.NullString{Valid: false}
	}
	return sql.NullString{String: s, Valid: true}
}

// getCachedApproval checks if user has a cached crew approval
func (a *RemoteHoldAuthorizer) getCachedApproval(holdDID, userDID string) (bool, error) {
	query := `
		SELECT expires_at
		FROM hold_crew_approvals
		WHERE hold_did = ? AND user_did = ?
	`

	var expiresAt time.Time
	err := a.db.QueryRow(query, holdDID, userDID).Scan(&expiresAt)

	if err == sql.ErrNoRows {
		return false, nil // Cache miss
	}

	if err != nil {
		return false, err
	}

	// Check if approval has expired
	if time.Now().After(expiresAt) {
		// Expired - clean up
		_ = a.deleteCachedApproval(holdDID, userDID)
		return false, nil
	}

	return true, nil
}

// cacheApproval stores a crew approval with TTL
func (a *RemoteHoldAuthorizer) cacheApproval(holdDID, userDID string, ttl time.Duration) error {
	query := `
		INSERT INTO hold_crew_approvals (hold_did, user_did, approved_at, expires_at)
		VALUES (?, ?, ?, ?)
		ON CONFLICT(hold_did, user_did) DO UPDATE SET
			approved_at = excluded.approved_at,
			expires_at = excluded.expires_at
	`

	now := time.Now()
	expiresAt := now.Add(ttl)

	_, err := a.db.Exec(query, holdDID, userDID, now, expiresAt)
	return err
}

// deleteCachedApproval removes an expired approval
func (a *RemoteHoldAuthorizer) deleteCachedApproval(holdDID, userDID string) error {
	query := `DELETE FROM hold_crew_approvals WHERE hold_did = ? AND user_did = ?`
	_, err := a.db.Exec(query, holdDID, userDID)
	return err
}

// isBlockedByDenialBackoff checks if user is in denial backoff period
// Checks in-memory cache first (for 10s first denials), then DB (for longer backoffs)
func (a *RemoteHoldAuthorizer) isBlockedByDenialBackoff(holdDID, userDID string) (bool, error) {
	// Check in-memory cache first (first denials with configurable backoff)
	key := fmt.Sprintf("%s:%s", holdDID, userDID)
	if val, ok := a.recentDenials.Load(key); ok {
		entry := val.(denialEntry)
		// Check if still within first denial backoff period
		if time.Since(entry.timestamp) < a.firstDenialBackoff {
			slog.Debug("Write access blocked by in-memory denial cache",
				"holdDID", holdDID,
				"userDID", userDID,
				"denial_reason", "first_denial_backoff",
				"backoff_type", "in_memory",
				"backoff_duration", a.firstDenialBackoff,
				"denied_at", entry.timestamp,
				"retry_after", entry.timestamp.Add(a.firstDenialBackoff))
			return true, nil // Still blocked by in-memory first denial
		}
	}

	// Check database for longer backoffs (second+ denials)
	query := `
		SELECT next_retry_at
		FROM hold_crew_denials
		WHERE hold_did = ? AND user_did = ?
	`

	var nextRetryAt time.Time
	err := a.db.QueryRow(query, holdDID, userDID).Scan(&nextRetryAt)

	if err == sql.ErrNoRows {
		return false, nil // No denial record
	}

	if err != nil {
		return false, err
	}

	// Check if still in backoff period
	if time.Now().Before(nextRetryAt) {
		slog.Debug("Write access blocked by database denial cache",
			"holdDID", holdDID,
			"userDID", userDID,
			"denial_reason", "exponential_backoff",
			"backoff_type", "database",
			"next_retry_at", nextRetryAt,
			"retry_in", time.Until(nextRetryAt).Round(time.Second))
		return true, nil // Still blocked
	}

	// Backoff period expired - can retry
	return false, nil
}

// sqliteNow is SQLite's clock rendered as RFC 3339 with milliseconds, matching
// what the driver writes for a Go time.Time closely enough that both parse back
// into time.Time correctly.
//
// datetime() is not usable here because it truncates to whole seconds, and the
// backoff durations can be sub-second in tests.
const sqliteNow = `strftime('%Y-%m-%dT%H:%M:%fZ','now')`

// sqliteNowPlus returns the same, offset by a SQLite time modifier expression.
func sqliteNowPlus(modifier string) string {
	return `strftime('%Y-%m-%dT%H:%M:%fZ','now', ` + modifier + `)`
}

// backoffSecondsCaseSQL builds a CASE expression mapping the NEW denial count to
// its backoff in seconds, mirroring getBackoffDuration.
//
// getBackoffDuration indexes the ladder at newCount-1, clamped to the last
// entry, so entry i applies when newCount == i+1 and the final entry covers
// everything beyond.
//
// The values are durations from configuration, formatted as numbers, so there is
// no injection surface here: nothing from a request reaches this string.
func (a *RemoteHoldAuthorizer) backoffSecondsCaseSQL() string {
	backoffs := a.dbBackoffDurations
	if len(backoffs) == 0 {
		return "0"
	}

	secs := func(d time.Duration) string {
		return strconv.FormatFloat(d.Seconds(), 'f', 3, 64)
	}

	// A single-rung ladder applies to every count, and "CASE ELSE x END" with no
	// WHEN arm is a syntax error, so emit the bare number.
	if len(backoffs) == 1 {
		return secs(backoffs[0])
	}

	var sb strings.Builder
	sb.WriteString("CASE")
	for i := range len(backoffs) - 1 {
		fmt.Fprintf(&sb, " WHEN hold_crew_denials.denial_count + 1 <= %d THEN %s", i+1, secs(backoffs[i]))
	}
	fmt.Fprintf(&sb, " ELSE %s END", secs(backoffs[len(backoffs)-1]))
	return sb.String()
}

// cacheDenial stores or updates a denial with exponential backoff.
// First denial: in-memory only (configurable backoff, default 10s)
// Second+ denial: database with exponential backoff (configurable, default 1m/5m/15m/1h)
//
// The database write is a single atomic statement. It used to be a SELECT of
// denial_count, an increment in Go, and an upsert of the computed value, which
// loses increments when two requests for the same (hold, user) overlap. That was
// already possible across goroutines and becomes routine once more than one
// AppView instance serves traffic, and its effect is that the backoff escalates
// more slowly than configured, so a denied client keeps hammering the hold.
//
// next_retry_at is therefore computed in SQL too, from the count the same
// statement is producing, rather than in Go from a count that may already be
// stale by the time the write lands.
func (a *RemoteHoldAuthorizer) cacheDenial(holdDID, userDID string) error {
	key := fmt.Sprintf("%s:%s", holdDID, userDID)

	// Check if this is a first denial (not in memory, not in DB)
	_, inMemory := a.recentDenials.Load(key)

	if !inMemory {
		var existing int
		err := a.db.QueryRow(
			`SELECT denial_count FROM hold_crew_denials WHERE hold_did = ? AND user_did = ?`,
			holdDID, userDID,
		).Scan(&existing)
		if err != nil && !errors.Is(err, sql.ErrNoRows) {
			return err
		}
		if errors.Is(err, sql.ErrNoRows) {
			// First denial: store only in memory with configurable backoff.
			// This read only picks a branch; it is no longer the source of the
			// increment, so a stale answer costs at most one skipped or one
			// extra database write, never a lost count.
			now := time.Now()
			a.recentDenials.Store(key, denialEntry{timestamp: now})
			slog.Info("Cached first crew denial (in-memory)",
				"holdDID", holdDID,
				"userDID", userDID,
				"denial_count", 1,
				"backoff_type", "in_memory",
				"backoff_duration", a.firstDenialBackoff,
				"retry_after", now.Add(a.firstDenialBackoff))
			return nil
		}
	}

	// Second+ denial: one statement, so concurrent denials for the same
	// (hold, user) each add exactly one.
	upsertQuery := `
		INSERT INTO hold_crew_denials (hold_did, user_did, denial_count, next_retry_at, last_denied_at)
		VALUES (?, ?, 1, ` + sqliteNowPlus(`'+' || `+firstBackoffSeconds(a)+` || ' seconds'`) + `, ` + sqliteNow + `)
		ON CONFLICT(hold_did, user_did) DO UPDATE SET
			denial_count = hold_crew_denials.denial_count + 1,
			next_retry_at = ` + sqliteNowPlus(`'+' || (`+a.backoffSecondsCaseSQL()+`) || ' seconds'`) + `,
			last_denied_at = ` + sqliteNow + `
	`

	if _, err := a.db.Exec(upsertQuery, holdDID, userDID); err != nil {
		return err
	}

	// Remove from in-memory cache since we're now tracking in DB
	a.recentDenials.Delete(key)

	// Read back purely for the log line. Denial diagnostics are the reason this
	// logging exists, so it is worth a cheap extra read, but a failure here must
	// not fail the denial itself.
	var denialCount int
	var nextRetry time.Time
	if err := a.db.QueryRow(
		`SELECT denial_count, next_retry_at FROM hold_crew_denials WHERE hold_did = ? AND user_did = ?`,
		holdDID, userDID,
	).Scan(&denialCount, &nextRetry); err != nil {
		slog.Debug("Could not read back denial state for logging", "error", err)
		return nil
	}

	slog.Info("Cached crew denial with exponential backoff",
		"holdDID", holdDID,
		"userDID", userDID,
		"denial_count", denialCount,
		"backoff_type", "database",
		"backoff_duration", a.getBackoffDuration(denialCount),
		"next_retry_at", nextRetry)

	return nil
}

// firstBackoffSeconds renders the backoff for a freshly inserted denial row
// (count 1), which getBackoffDuration maps to the first entry in the ladder.
func firstBackoffSeconds(a *RemoteHoldAuthorizer) string {
	if len(a.dbBackoffDurations) == 0 {
		return "0"
	}
	return strconv.FormatFloat(a.dbBackoffDurations[0].Seconds(), 'f', 3, 64)
}

// getBackoffDuration returns the backoff duration based on denial count
// Note: First denial is in-memory only and not tracked by this function
// This function handles second+ denials using configurable durations
func (a *RemoteHoldAuthorizer) getBackoffDuration(denialCount int) time.Duration {
	backoffs := a.dbBackoffDurations

	idx := denialCount - 1
	if idx >= len(backoffs) {
		idx = len(backoffs) - 1
	}

	return backoffs[idx]
}

// ClearCrewDenial removes crew denial from both in-memory and database caches
// This allows immediate access after a user becomes a crew member
func (a *RemoteHoldAuthorizer) ClearCrewDenial(ctx context.Context, holdDID, userDID string) error {
	// Clear in-memory cache
	key := fmt.Sprintf("%s:%s", holdDID, userDID)
	a.recentDenials.Delete(key)

	// Clear database cache
	if a.db != nil {
		query := `DELETE FROM hold_crew_denials WHERE hold_did = ? AND user_did = ?`
		_, err := a.db.ExecContext(ctx, query, holdDID, userDID)
		if err != nil {
			return fmt.Errorf("failed to clear denial cache: %w", err)
		}
	}

	slog.Debug("Cleared crew denial cache", "holdDID", holdDID, "userDID", userDID)
	return nil
}

// ClearAllDenials removes all crew denials from both in-memory and database
// caches, giving users sitting on a long backoff an immediate retry after a
// deploy.
//
// Called by the AppView's cleanup worker when it acquires the cleanup lease, NOT
// on every boot. The database half is a table-wide DELETE shared by every
// instance, so calling it unconditionally at startup meant a rolling deploy
// wiped the backoffs once per instance and every scale-out event wiped them
// again. Keep it behind the lease.
func (a *RemoteHoldAuthorizer) ClearAllDenials() error {
	// Clear all in-memory denials
	a.recentDenials.Range(func(key, value any) bool {
		a.recentDenials.Delete(key)
		return true
	})

	// Clear all database denials
	if a.db != nil {
		_, err := a.db.Exec("DELETE FROM hold_crew_denials")
		if err != nil {
			return fmt.Errorf("failed to clear all denial caches: %w", err)
		}
	}

	slog.Info("Cleared all crew denial caches on startup")
	return nil
}