at-container-registry/pkg/hold/admin/jobs.go

package admin

import (
	"context"
	"net/http"
	"sync"
	"time"

	"github.com/go-chi/chi/v5"
)

// Long-running admin operations (bulk crew tier remaps, crew imports,
// scan-record backfills) must NOT run as a synchronous loop bound to the HTTP
// request context. A reverse proxy times out such a request around the 10s mark
// and cancels r.Context(), which aborts the work mid-flight ("context canceled"
// from the blockstore). Instead they run here as a detached background job: the
// kickoff handler returns a progress fragment immediately, the work runs under
// its own context.Background() timeout, and the page polls
// /admin/api/jobs/{key}/status until it finishes.
//
// This is the same shape hand-rolled in gc.startBackground; the admin package
// keeps its own copy because gc must not import admin.

// jobProgress is the live progress of a running job. It is a value type copied
// under the job's lock — never hand a pointer the job loop mutates to a template.
type jobProgress struct {
	Done    int
	Total   int // 0 = indeterminate (spinner only, no progress bar)
	Message string
}

// jobResult is the generic success payload rendered by partials/job_result.html.
// Jobs with richer output (crew import, scan backfill) register their own result
// template and return a different struct instead.
type jobResult struct {
	Message      string // summary line
	Failed       int    // >0 renders the alert as a warning rather than success
	ReloadURL    string // optional htmx affordance to reload a tab after the job
	ReloadTarget string // CSS selector the ReloadURL swaps into
}

// jobState is the state machine for one background job, keyed by a stable string
// (e.g. "crew-remap-tier"). Only one run per key at a time.
type jobState struct {
	mu             sync.Mutex
	title          string
	resultTemplate string
	started        bool
	running        bool
	startedAt      time.Time
	progress       jobProgress
	result         any
	err            string
}

// jobRegistry holds one jobState per key. The zero value is ready to use.
type jobRegistry struct {
	mu   sync.Mutex
	jobs map[string]*jobState
}

// get returns the jobState for key, creating an empty one on first access.
func (r *jobRegistry) get(key string) *jobState {
	r.mu.Lock()
	defer r.mu.Unlock()
	if r.jobs == nil {
		r.jobs = make(map[string]*jobState)
	}
	st, ok := r.jobs[key]
	if !ok {
		st = &jobState{}
		r.jobs[key] = st
	}
	return st
}

// jobSnapshot is the read-only view rendered to templates.
type jobSnapshot struct {
	Key            string
	Title          string
	ResultTemplate string
	Started        bool
	Running        bool
	StartedAt      time.Time
	Progress       jobProgress
	Result         any
	Error          string
}

// startJob launches fn in a detached goroutine under its own timeout and returns
// true. If a job with this key is already running it returns false and leaves the
// in-flight run untouched (the caller should just render the current snapshot).
//
// fn publishes progress via the passed callback and returns a result value
// (rendered by resultTemplate) or an error (rendered by partials/gc_error.html).
// fn must use the ctx it is given — that ctx carries the detached timeout, not
// the request deadline.
func (ui *AdminUI) startJob(key, title, resultTemplate string, timeout time.Duration,
	fn func(ctx context.Context, progress func(jobProgress)) (any, error)) bool {
	st := ui.jobs.get(key)

	st.mu.Lock()
	if st.running {
		st.mu.Unlock()
		return false
	}
	st.running = true
	st.started = true
	st.startedAt = time.Now()
	st.title = title
	st.resultTemplate = resultTemplate
	st.progress = jobProgress{}
	st.result = nil
	st.err = ""
	st.mu.Unlock()

	publish := func(p jobProgress) {
		st.mu.Lock()
		st.progress = p
		st.mu.Unlock()
	}

	go func() {
		defer func() {
			st.mu.Lock()
			st.running = false
			st.mu.Unlock()
		}()

		ctx, cancel := context.WithTimeout(context.Background(), timeout)
		defer cancel()

		res, err := fn(ctx, publish)

		st.mu.Lock()
		if err != nil {
			st.err = err.Error()
		} else {
			st.result = res
		}
		st.mu.Unlock()
	}()

	return true
}

// jobSnapshot returns a copy of the current state for key — safe to render
// without holding the lock.
func (ui *AdminUI) jobSnapshot(key string) jobSnapshot {
	st := ui.jobs.get(key)
	st.mu.Lock()
	defer st.mu.Unlock()
	return jobSnapshot{
		Key:            key,
		Title:          st.title,
		ResultTemplate: st.resultTemplate,
		Started:        st.started,
		Running:        st.running,
		StartedAt:      st.startedAt,
		Progress:       st.progress,
		Result:         st.result,
		Error:          st.err,
	}
}

// handleJobStatus is polled by the progress fragment. It renders the progress
// fragment while running, the error fragment on failure, or the job's registered
// result template on success. A never-started key renders an empty body.
func (ui *AdminUI) handleJobStatus(w http.ResponseWriter, r *http.Request) {
	key := chi.URLParam(r, "key")
	snap := ui.jobSnapshot(key)

	switch {
	case !snap.Started:
		_, _ = w.Write([]byte(""))
	case snap.Running:
		ui.renderTemplate(w, "partials/job_progress.html", snap)
	case snap.Error != "":
		ui.renderTemplate(w, "partials/gc_error.html", struct{ Error string }{snap.Error})
	default:
		ui.renderTemplate(w, snap.ResultTemplate, snap.Result)
	}
}