at-container-registry/pkg/appview/holdhealth/worker.go

package holdhealth

import (
	"context"
	"database/sql"
	"fmt"
	"log/slog"
	"sync"
	"time"

	"atcr.io/pkg/atproto"
)

// DBQuerier interface for database queries (allows mocking in tests)
type DBQuerier interface {
	GetUniqueHoldEndpoints() ([]string, error)
}

// Worker runs background health checks for hold endpoints
type Worker struct {
	checker       *Checker
	db            DBQuerier
	refreshTicker *time.Ticker
	cleanupTicker *time.Ticker
	stopChan      chan struct{}
	wg            sync.WaitGroup
	startupDelay  time.Duration
}

// NewWorker creates a new background worker
func NewWorker(checker *Checker, db DBQuerier, refreshInterval time.Duration) *Worker {
	return &Worker{
		checker:       checker,
		db:            db,
		refreshTicker: time.NewTicker(refreshInterval),
		cleanupTicker: time.NewTicker(30 * time.Minute), // Cleanup every 30 minutes
		stopChan:      make(chan struct{}),
		startupDelay:  0, // No delay by default for backward compatibility
	}
}

// NewWorkerWithStartupDelay creates a new background worker with a startup delay
func NewWorkerWithStartupDelay(checker *Checker, db DBQuerier, refreshInterval, startupDelay time.Duration) *Worker {
	return &Worker{
		checker:       checker,
		db:            db,
		refreshTicker: time.NewTicker(refreshInterval),
		cleanupTicker: time.NewTicker(30 * time.Minute), // Cleanup every 30 minutes
		stopChan:      make(chan struct{}),
		startupDelay:  startupDelay,
	}
}

// Start begins the background worker
func (w *Worker) Start(ctx context.Context) {
	w.wg.Go(func() {
		slog.Info("Hold health worker starting background health checks")

		// Wait for services to be ready (Docker startup race condition)
		if w.startupDelay > 0 {
			slog.Info("Hold health worker waiting for services to be ready", "delay", w.startupDelay)
			select {
			case <-time.After(w.startupDelay):
				// Continue with initial check
			case <-ctx.Done():
				slog.Info("Hold health worker context cancelled during startup delay")
				return
			}
		}

		// Perform initial check
		w.refreshAllHolds(ctx)

		for {
			select {
			case <-ctx.Done():
				slog.Info("Hold health worker context cancelled, stopping")
				return
			case <-w.stopChan:
				slog.Info("Hold health worker stop signal received")
				return
			case <-w.refreshTicker.C:
				w.refreshAllHolds(ctx)
			case <-w.cleanupTicker.C:
				slog.Info("Hold health worker running cache cleanup")
				w.checker.Cleanup()
			}
		}
	})
}

// Stop gracefully stops the worker
func (w *Worker) Stop() {
	close(w.stopChan)
	w.refreshTicker.Stop()
	w.cleanupTicker.Stop()
	w.wg.Wait()
	slog.Info("Hold health worker stopped")
}

// refreshAllHolds queries the database for unique hold endpoints and refreshes their health status
func (w *Worker) refreshAllHolds(ctx context.Context) {
	slog.Info("Hold health worker starting refresh cycle")

	// Get unique hold endpoints from database
	endpoints, err := w.db.GetUniqueHoldEndpoints()
	if err != nil {
		slog.Error("Hold health worker failed to fetch hold endpoints", "error", err)
		return
	}

	if len(endpoints) == 0 {
		slog.Info("Hold health worker no hold endpoints to check")
		return
	}

	slog.Info("Hold health worker fetched hold endpoint entries from database", "count", len(endpoints))

	// Deduplicate endpoints by normalizing to canonical DID format
	// This handles cases where the same hold is stored with different representations:
	// - http://172.28.0.3:8080 (internal IP)
	// - http://hold01.atcr.io (external hostname)
	// - did:web:hold01.atcr.io (DID format)
	// All normalize to the same DID: did:web:hold01.atcr.io (or did:web:172.28.0.3:8080)
	seen := make(map[string]bool)
	uniqueEndpoints := make([]string, 0, len(endpoints))

	for _, endpoint := range endpoints {
		// Normalize to canonical DID format
		normalizedDID, err := atproto.ResolveHoldDID(ctx, endpoint)
		if err != nil {
			slog.Debug("Failed to resolve hold DID during health check", "endpoint", endpoint, "error", err)
			continue
		}

		// Skip if we've already seen this normalized DID
		if seen[normalizedDID] {
			continue
		}

		seen[normalizedDID] = true
		// Use the normalized DID for health checks
		uniqueEndpoints = append(uniqueEndpoints, normalizedDID)
	}

	slog.Info("Hold health worker checking unique hold endpoints", "unique_count", len(uniqueEndpoints), "total_count", len(endpoints))

	// Check health concurrently with rate limiting
	// Use a semaphore to limit concurrent requests (max 10 at a time)
	sem := make(chan struct{}, 10)
	var wg sync.WaitGroup

	reachable := 0
	unreachable := 0
	var statsMu sync.Mutex

	for _, endpoint := range uniqueEndpoints {
		wg.Go(func() {
			// Acquire semaphore
			sem <- struct{}{}
			defer func() { <-sem }()

			// Check health
			isReachable, err := w.checker.CheckHealth(ctx, endpoint)

			// Update cache
			w.checker.SetStatus(endpoint, isReachable, err)

			// Update stats
			statsMu.Lock()
			if isReachable {
				reachable++
			} else {
				unreachable++
				slog.Warn("Hold health worker hold unreachable", "endpoint", endpoint, "error", err)
			}
			statsMu.Unlock()
		})
	}

	// Wait for all checks to complete
	wg.Wait()

	slog.Info("Hold health worker refresh complete", "reachable", reachable, "unreachable", unreachable)
}

// DBAdapter wraps sql.DB to implement DBQuerier interface
type DBAdapter struct {
	db *sql.DB
}

// NewDBAdapter creates a new database adapter
func NewDBAdapter(db *sql.DB) *DBAdapter {
	return &DBAdapter{db: db}
}

// GetUniqueHoldEndpoints queries the database for unique hold endpoints
func (a *DBAdapter) GetUniqueHoldEndpoints() ([]string, error) {
	rows, err := a.db.Query(`SELECT DISTINCT hold_endpoint FROM manifests WHERE hold_endpoint != ''`)
	if err != nil {
		return nil, fmt.Errorf("failed to query hold endpoints: %w", err)
	}
	defer rows.Close()

	var endpoints []string
	for rows.Next() {
		var endpoint string
		if err := rows.Scan(&endpoint); err != nil {
			return nil, fmt.Errorf("failed to scan endpoint: %w", err)
		}
		endpoints = append(endpoints, endpoint)
	}

	if err := rows.Err(); err != nil {
		return nil, fmt.Errorf("error iterating rows: %w", err)
	}

	return endpoints, nil
}