remove distribution from hold, add vulnerability scanning in appview.

1. Removing distribution/distribution from the Hold Service (biggest change)
  The hold service previously used distribution's StorageDriver interface for all blob operations. This replaces it with direct AWS SDK v2 calls through ATCR's own pkg/s3.S3Service:
  - New S3Service methods: Stat(), PutBytes(), Move(), Delete(), WalkBlobs(), ListPrefix() added to pkg/s3/types.go
  - Pull zone fix: Presigned URLs are now generated against the real S3 endpoint, then the host is swapped to the CDN URL post-signing (previously the CDN URL was set as the endpoint, which
  broke SigV4 signatures)
  - All hold subsystems migrated: GC, OCI uploads, XRPC handlers, profile uploads, scan broadcaster, manifest posts — all now use *s3.S3Service instead of storagedriver.StorageDriver
  - Config simplified: Removed configuration.Storage type and buildStorageConfigFromFields(); replaced with a simple S3Params() method
  - Mock expanded: MockS3Client gains an in-memory object store + 5 new methods, replacing duplicate mockStorageDriver implementations in tests (~160 lines deleted from each test file)
2. Vulnerability Scan UI in AppView (new feature)
  Displays scan results from the hold's PDS on the repository page:
  - New lexicon: io/atcr/hold/scan.json with vulnReportBlob field for storing full Grype reports
  - Two new HTMX endpoints: /api/scan-result (badge) and /api/vuln-details (modal with CVE table)
  - New templates: vuln-badge.html (severity count chips) and vuln-details.html (full CVE table with NVD/GHSA links)
  - Repository page: Lazy-loads scan badges per manifest via HTMX
  - Tests: ~590 lines of test coverage for both handlers
3. S3 Diagnostic Tool
  New cmd/s3-test/main.go (418 lines) — tests S3 connectivity with both SDK v1 and v2, including presigned URL generation, pull zone host swapping, and verbose signing debug output.
4. Deployment Tooling
  - New syncServiceUnit() for comparing/updating systemd units on servers
  - Update command now syncs config keys (adds missing keys from template) and service units with daemon-reload
5. DB Migration
  0011_fix_captain_successor_column.yaml — rebuilds hold_captain_records to add the successor column that was missed in a previous migration.
6. Documentation
  - APPVIEW-UI-FUTURE.md rewritten as a status-tracked feature inventory
  - DISTRIBUTION.md renamed to CREDENTIAL_HELPER.md
  - New REMOVING_DISTRIBUTION.md — 480-line analysis of fully removing distribution from the appview side
7. go.mod
  aws-sdk-go v1 moved from indirect to direct (needed by cmd/s3-test).

pkg/s3/mock.go

@@ -1,13 +1,17 @@
 package s3
 
 import (
+	"bytes"
 	"context"
 	"fmt"
+	"io"
+	"strings"
 	"sync"
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	awss3 "github.com/aws/aws-sdk-go-v2/service/s3"
+	s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/google/uuid"
 )
 
@@ -22,6 +26,9 @@ type MockS3Client struct {
 	// If empty, a UUID is generated.
 	UploadID string
 
+	// Objects stores in-memory blobs for PutObject/HeadObject/DeleteObject/CopyObject/ListObjectsV2.
+	Objects map[string][]byte
+
 	// Track calls for verification in tests
 	mu                   sync.Mutex
 	CreateMultipartCalls []CreateMultipartCall
@@ -36,6 +43,8 @@ type MockS3Client struct {
 	CreateMultipartError error
 	CompleteError        error
 	AbortError           error
+	HeadObjectError      error
+	CopyObjectError      error
 }
 
 // CreateMultipartCall records a CreateMultipartUpload call
@@ -89,6 +98,7 @@ type PutObjectCall struct {
 func NewMockS3Client(testServerURL string) *MockS3Client {
 	return &MockS3Client{
 		TestServerURL:        testServerURL,
+		Objects:              make(map[string][]byte),
 		CreateMultipartCalls: []CreateMultipartCall{},
 		CompleteCalls:        []CompleteCall{},
 		AbortCalls:           []AbortCall{},
@@ -144,6 +154,14 @@ func (m *MockS3Client) CompleteMultipartUpload(ctx context.Context, input *awss3
 		return nil, m.CompleteError
 	}
 
+	// Store a placeholder object at the key so Stat/HeadObject works after complete
+	key := aws.ToString(input.Key)
+	if m.Objects != nil {
+		if _, exists := m.Objects[key]; !exists {
+			m.Objects[key] = []byte("completed-multipart")
+		}
+	}
+
 	// Return a mock ETag
 	etag := "\"mock-etag-" + uuid.New().String() + "\""
 	return &awss3.CompleteMultipartUploadOutput{
@@ -169,6 +187,141 @@ func (m *MockS3Client) AbortMultipartUpload(ctx context.Context, input *awss3.Ab
 	return &awss3.AbortMultipartUploadOutput{}, nil
 }
 
+// HeadObject implements S3Client
+func (m *MockS3Client) HeadObject(ctx context.Context, input *awss3.HeadObjectInput, opts ...func(*awss3.Options)) (*awss3.HeadObjectOutput, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	if m.HeadObjectError != nil {
+		return nil, m.HeadObjectError
+	}
+
+	key := aws.ToString(input.Key)
+	data, ok := m.Objects[key]
+	if !ok {
+		return nil, fmt.Errorf("NoSuchKey: object %s not found", key)
+	}
+
+	size := int64(len(data))
+	return &awss3.HeadObjectOutput{
+		ContentLength: &size,
+	}, nil
+}
+
+// PutObject implements S3Client
+func (m *MockS3Client) PutObject(ctx context.Context, input *awss3.PutObjectInput, opts ...func(*awss3.Options)) (*awss3.PutObjectOutput, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	key := aws.ToString(input.Key)
+	m.PutObjectCalls = append(m.PutObjectCalls, PutObjectCall{
+		Bucket: aws.ToString(input.Bucket),
+		Key:    key,
+	})
+
+	if input.Body != nil {
+		data, err := io.ReadAll(input.Body)
+		if err != nil {
+			return nil, err
+		}
+		m.Objects[key] = data
+	} else {
+		m.Objects[key] = []byte{}
+	}
+
+	return &awss3.PutObjectOutput{}, nil
+}
+
+// CopyObject implements S3Client
+func (m *MockS3Client) CopyObject(ctx context.Context, input *awss3.CopyObjectInput, opts ...func(*awss3.Options)) (*awss3.CopyObjectOutput, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	if m.CopyObjectError != nil {
+		return nil, m.CopyObjectError
+	}
+
+	// CopySource is "bucket/key"
+	copySource := aws.ToString(input.CopySource)
+	// Strip bucket prefix to get key
+	parts := strings.SplitN(copySource, "/", 2)
+	srcKey := copySource
+	if len(parts) == 2 {
+		srcKey = parts[1]
+	}
+
+	data, ok := m.Objects[srcKey]
+	if !ok {
+		return nil, fmt.Errorf("NoSuchKey: source object %s not found", srcKey)
+	}
+
+	dstKey := aws.ToString(input.Key)
+	m.Objects[dstKey] = append([]byte{}, data...)
+
+	return &awss3.CopyObjectOutput{}, nil
+}
+
+// DeleteObject implements S3Client
+func (m *MockS3Client) DeleteObject(ctx context.Context, input *awss3.DeleteObjectInput, opts ...func(*awss3.Options)) (*awss3.DeleteObjectOutput, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	key := aws.ToString(input.Key)
+	delete(m.Objects, key)
+
+	return &awss3.DeleteObjectOutput{}, nil
+}
+
+// ListObjectsV2 implements S3Client
+func (m *MockS3Client) ListObjectsV2(ctx context.Context, input *awss3.ListObjectsV2Input, opts ...func(*awss3.Options)) (*awss3.ListObjectsV2Output, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	prefix := aws.ToString(input.Prefix)
+	delimiter := aws.ToString(input.Delimiter)
+
+	var contents []s3types.Object
+	commonPrefixes := map[string]bool{}
+
+	for key, data := range m.Objects {
+		if !strings.HasPrefix(key, prefix) {
+			continue
+		}
+
+		if delimiter != "" {
+			// Check if there's a delimiter after the prefix
+			rest := strings.TrimPrefix(key, prefix)
+			idx := strings.Index(rest, delimiter)
+			if idx >= 0 {
+				// Has delimiter — this is a common prefix, not a content object
+				cp := prefix + rest[:idx+len(delimiter)]
+				commonPrefixes[cp] = true
+				continue
+			}
+		}
+
+		size := int64(len(data))
+		k := key
+		contents = append(contents, s3types.Object{
+			Key:  &k,
+			Size: &size,
+		})
+	}
+
+	var cps []s3types.CommonPrefix
+	for cp := range commonPrefixes {
+		p := cp
+		cps = append(cps, s3types.CommonPrefix{Prefix: &p})
+	}
+
+	falseVal := false
+	return &awss3.ListObjectsV2Output{
+		Contents:       contents,
+		CommonPrefixes: cps,
+		IsTruncated:    &falseVal,
+	}, nil
+}
+
 // PresignUploadPart implements S3Client
 // Returns a mock presigned URL for test server
 func (m *MockS3Client) PresignUploadPart(ctx context.Context, input *awss3.UploadPartInput, expires time.Duration) (string, error) {
@@ -229,6 +382,31 @@ func (m *MockS3Client) PresignPutObject(ctx context.Context, input *awss3.PutObj
 		Key:    aws.ToString(input.Key),
 	})
 
+	// Also store the body if provided (for PresignPutObject used in tests that also check objects)
+	if input.Body != nil {
+		key := aws.ToString(input.Key)
+		data, _ := io.ReadAll(input.Body)
+		m.Objects[key] = data
+	}
+
 	url := fmt.Sprintf("%s/put/%s", m.TestServerURL, aws.ToString(input.Key))
 	return url, nil
 }
+
+// SetObject is a test helper to pre-populate an object in the mock store.
+func (m *MockS3Client) SetObject(key string, data []byte) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.Objects[key] = append([]byte{}, data...)
+}
+
+// GetObject is a test helper to read an object from the mock store (nil if not found).
+func (m *MockS3Client) GetObject(key string) []byte {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	data, ok := m.Objects[key]
+	if !ok {
+		return nil
+	}
+	return bytes.Clone(data)
+}

pkg/s3/types.go

@@ -3,9 +3,11 @@
 package s3
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"log/slog"
+	"net/url"
 	"strings"
 	"time"
 
@@ -24,6 +26,13 @@ type S3Client interface {
 	CompleteMultipartUpload(ctx context.Context, input *awss3.CompleteMultipartUploadInput, opts ...func(*awss3.Options)) (*awss3.CompleteMultipartUploadOutput, error)
 	AbortMultipartUpload(ctx context.Context, input *awss3.AbortMultipartUploadInput, opts ...func(*awss3.Options)) (*awss3.AbortMultipartUploadOutput, error)
 
+	// Direct object operations
+	HeadObject(ctx context.Context, input *awss3.HeadObjectInput, opts ...func(*awss3.Options)) (*awss3.HeadObjectOutput, error)
+	PutObject(ctx context.Context, input *awss3.PutObjectInput, opts ...func(*awss3.Options)) (*awss3.PutObjectOutput, error)
+	CopyObject(ctx context.Context, input *awss3.CopyObjectInput, opts ...func(*awss3.Options)) (*awss3.CopyObjectOutput, error)
+	DeleteObject(ctx context.Context, input *awss3.DeleteObjectInput, opts ...func(*awss3.Options)) (*awss3.DeleteObjectOutput, error)
+	ListObjectsV2(ctx context.Context, input *awss3.ListObjectsV2Input, opts ...func(*awss3.Options)) (*awss3.ListObjectsV2Output, error)
+
 	// Presigned URL operations - return URL string directly
 	PresignGetObject(ctx context.Context, input *awss3.GetObjectInput, expires time.Duration) (string, error)
 	PresignHeadObject(ctx context.Context, input *awss3.HeadObjectInput, expires time.Duration) (string, error)
@@ -61,31 +70,49 @@ func (r *RealS3Client) AbortMultipartUpload(ctx context.Context, input *awss3.Ab
 	return r.client.AbortMultipartUpload(ctx, input, opts...)
 }
 
+// HeadObject implements S3Client
+func (r *RealS3Client) HeadObject(ctx context.Context, input *awss3.HeadObjectInput, opts ...func(*awss3.Options)) (*awss3.HeadObjectOutput, error) {
+	return r.client.HeadObject(ctx, input, opts...)
+}
+
+// PutObject implements S3Client
+func (r *RealS3Client) PutObject(ctx context.Context, input *awss3.PutObjectInput, opts ...func(*awss3.Options)) (*awss3.PutObjectOutput, error) {
+	return r.client.PutObject(ctx, input, opts...)
+}
+
+// CopyObject implements S3Client
+func (r *RealS3Client) CopyObject(ctx context.Context, input *awss3.CopyObjectInput, opts ...func(*awss3.Options)) (*awss3.CopyObjectOutput, error) {
+	return r.client.CopyObject(ctx, input, opts...)
+}
+
+// DeleteObject implements S3Client
+func (r *RealS3Client) DeleteObject(ctx context.Context, input *awss3.DeleteObjectInput, opts ...func(*awss3.Options)) (*awss3.DeleteObjectOutput, error) {
+	return r.client.DeleteObject(ctx, input, opts...)
+}
+
+// ListObjectsV2 implements S3Client
+func (r *RealS3Client) ListObjectsV2(ctx context.Context, input *awss3.ListObjectsV2Input, opts ...func(*awss3.Options)) (*awss3.ListObjectsV2Output, error) {
+	return r.client.ListObjectsV2(ctx, input, opts...)
+}
+
 // PresignGetObject implements S3Client
 func (r *RealS3Client) PresignGetObject(ctx context.Context, input *awss3.GetObjectInput, expires time.Duration) (string, error) {
 	result, err := r.presign.PresignGetObject(ctx, input, func(opts *awss3.PresignOptions) {
 		opts.Expires = expires
-		if r.pullZone != "" {
-			opts.ClientOptions = append(opts.ClientOptions, func(o *awss3.Options) {
-				o.BaseEndpoint = aws.String(r.pullZone)
-			})
-		}
 	})
 	if err != nil {
 		return "", err
 	}
-	return result.URL, nil
+	return r.applyPullZone(result.URL), nil
 }
 
 // PresignHeadObject implements S3Client
+// Note: pull zone is intentionally NOT applied to HEAD requests. CDNs like
+// Bunny may convert HEAD to GET when proxying, which breaks the SigV4
+// signature. HEAD responses have no body, so CDN caching provides no benefit.
 func (r *RealS3Client) PresignHeadObject(ctx context.Context, input *awss3.HeadObjectInput, expires time.Duration) (string, error) {
 	result, err := r.presign.PresignHeadObject(ctx, input, func(opts *awss3.PresignOptions) {
 		opts.Expires = expires
-		if r.pullZone != "" {
-			opts.ClientOptions = append(opts.ClientOptions, func(o *awss3.Options) {
-				o.BaseEndpoint = aws.String(r.pullZone)
-			})
-		}
 	})
 	if err != nil {
 		return "", err
@@ -193,6 +220,190 @@ func NewS3Service(params map[string]any) (*S3Service, error) {
 	}, nil
 }
 
+// s3Key converts a blob path (with leading /) to an S3 key with prefix.
+func (s *S3Service) s3Key(blobPath string) string {
+	key := strings.TrimPrefix(blobPath, "/")
+	if s.PathPrefix != "" {
+		key = s.PathPrefix + "/" + key
+	}
+	return key
+}
+
+// Stat returns the size of an object at blobPath, or an error if it doesn't exist.
+func (s *S3Service) Stat(ctx context.Context, blobPath string) (int64, error) {
+	key := s.s3Key(blobPath)
+	out, err := s.Client.HeadObject(ctx, &awss3.HeadObjectInput{
+		Bucket: &s.Bucket,
+		Key:    &key,
+	})
+	if err != nil {
+		return 0, err
+	}
+	if out.ContentLength != nil {
+		return *out.ContentLength, nil
+	}
+	return 0, nil
+}
+
+// PutBytes uploads data to blobPath with the given content type.
+func (s *S3Service) PutBytes(ctx context.Context, blobPath string, data []byte, contentType string) error {
+	key := s.s3Key(blobPath)
+	_, err := s.Client.PutObject(ctx, &awss3.PutObjectInput{
+		Bucket:      &s.Bucket,
+		Key:         &key,
+		Body:        bytes.NewReader(data),
+		ContentType: &contentType,
+	})
+	return err
+}
+
+// Move copies srcPath to dstPath then deletes srcPath.
+func (s *S3Service) Move(ctx context.Context, srcPath, dstPath string) error {
+	srcKey := s.s3Key(srcPath)
+	dstKey := s.s3Key(dstPath)
+	copySource := s.Bucket + "/" + srcKey
+
+	_, err := s.Client.CopyObject(ctx, &awss3.CopyObjectInput{
+		Bucket:     &s.Bucket,
+		Key:        &dstKey,
+		CopySource: &copySource,
+	})
+	if err != nil {
+		return fmt.Errorf("copy %s -> %s: %w", srcPath, dstPath, err)
+	}
+
+	_, err = s.Client.DeleteObject(ctx, &awss3.DeleteObjectInput{
+		Bucket: &s.Bucket,
+		Key:    &srcKey,
+	})
+	if err != nil {
+		return fmt.Errorf("delete source %s after copy: %w", srcPath, err)
+	}
+
+	return nil
+}
+
+// Delete removes the object at blobPath.
+func (s *S3Service) Delete(ctx context.Context, blobPath string) error {
+	key := s.s3Key(blobPath)
+	_, err := s.Client.DeleteObject(ctx, &awss3.DeleteObjectInput{
+		Bucket: &s.Bucket,
+		Key:    &key,
+	})
+	return err
+}
+
+// WalkBlobs paginates ListObjectsV2 under prefix and calls fn for each object.
+// Keys passed to fn have the PathPrefix stripped (same format as BlobPath output).
+func (s *S3Service) WalkBlobs(ctx context.Context, prefix string, fn func(key string, size int64) error) error {
+	s3Prefix := s.s3Key(prefix)
+	if !strings.HasSuffix(s3Prefix, "/") {
+		s3Prefix += "/"
+	}
+
+	var continuationToken *string
+	for {
+		out, err := s.Client.ListObjectsV2(ctx, &awss3.ListObjectsV2Input{
+			Bucket:            &s.Bucket,
+			Prefix:            &s3Prefix,
+			ContinuationToken: continuationToken,
+		})
+		if err != nil {
+			return fmt.Errorf("list objects under %s: %w", prefix, err)
+		}
+
+		for _, obj := range out.Contents {
+			if obj.Key == nil {
+				continue
+			}
+			// Strip PathPrefix to get the logical path
+			key := *obj.Key
+			if s.PathPrefix != "" {
+				key = strings.TrimPrefix(key, s.PathPrefix+"/")
+			}
+			// Restore leading / for consistency with BlobPath
+			key = "/" + key
+
+			var size int64
+			if obj.Size != nil {
+				size = *obj.Size
+			}
+			if err := fn(key, size); err != nil {
+				return err
+			}
+		}
+
+		if out.IsTruncated == nil || !*out.IsTruncated {
+			break
+		}
+		continuationToken = out.NextContinuationToken
+	}
+	return nil
+}
+
+// ListPrefix returns immediate children (common prefixes) under blobPath using Delimiter="/".
+func (s *S3Service) ListPrefix(ctx context.Context, blobPath string) ([]string, error) {
+	s3Prefix := s.s3Key(blobPath)
+	if !strings.HasSuffix(s3Prefix, "/") {
+		s3Prefix += "/"
+	}
+	delimiter := "/"
+
+	var results []string
+	var continuationToken *string
+	for {
+		out, err := s.Client.ListObjectsV2(ctx, &awss3.ListObjectsV2Input{
+			Bucket:            &s.Bucket,
+			Prefix:            &s3Prefix,
+			Delimiter:         &delimiter,
+			ContinuationToken: continuationToken,
+		})
+		if err != nil {
+			return nil, fmt.Errorf("list prefix %s: %w", blobPath, err)
+		}
+
+		for _, cp := range out.CommonPrefixes {
+			if cp.Prefix == nil {
+				continue
+			}
+			// Strip the PathPrefix and reconstruct logical path
+			p := *cp.Prefix
+			if s.PathPrefix != "" {
+				p = strings.TrimPrefix(p, s.PathPrefix+"/")
+			}
+			p = "/" + strings.TrimSuffix(p, "/")
+			results = append(results, p)
+		}
+
+		if out.IsTruncated == nil || !*out.IsTruncated {
+			break
+		}
+		continuationToken = out.NextContinuationToken
+	}
+
+	return results, nil
+}
+
+// applyPullZone replaces the host in a presigned URL with the pull zone host.
+// The signature is computed against the real S3 endpoint so that the origin
+// can validate it; the CDN just proxies the request transparently.
+func (r *RealS3Client) applyPullZone(presignedURL string) string {
+	if r.pullZone == "" {
+		return presignedURL
+	}
+	parsed, err := url.Parse(presignedURL)
+	if err != nil {
+		return presignedURL
+	}
+	pz, err := url.Parse(r.pullZone)
+	if err != nil {
+		return presignedURL
+	}
+	parsed.Scheme = pz.Scheme
+	parsed.Host = pz.Host
+	return parsed.String()
+}
+
 // BlobPath converts a digest (e.g., "sha256:abc123...") or temp path to a storage path
 // Distribution stores blobs as: /docker/registry/v2/blobs/{algorithm}/{xx}/{hash}/data
 // where xx is the first 2 characters of the hash for directory sharding