remove distribution from hold, add vulnerability scanning in appview.

1. Removing distribution/distribution from the Hold Service (biggest change)
  The hold service previously used distribution's StorageDriver interface for all blob operations. This replaces it with direct AWS SDK v2 calls through ATCR's own pkg/s3.S3Service:
  - New S3Service methods: Stat(), PutBytes(), Move(), Delete(), WalkBlobs(), ListPrefix() added to pkg/s3/types.go
  - Pull zone fix: Presigned URLs are now generated against the real S3 endpoint, then the host is swapped to the CDN URL post-signing (previously the CDN URL was set as the endpoint, which
  broke SigV4 signatures)
  - All hold subsystems migrated: GC, OCI uploads, XRPC handlers, profile uploads, scan broadcaster, manifest posts — all now use *s3.S3Service instead of storagedriver.StorageDriver
  - Config simplified: Removed configuration.Storage type and buildStorageConfigFromFields(); replaced with a simple S3Params() method
  - Mock expanded: MockS3Client gains an in-memory object store + 5 new methods, replacing duplicate mockStorageDriver implementations in tests (~160 lines deleted from each test file)
2. Vulnerability Scan UI in AppView (new feature)
  Displays scan results from the hold's PDS on the repository page:
  - New lexicon: io/atcr/hold/scan.json with vulnReportBlob field for storing full Grype reports
  - Two new HTMX endpoints: /api/scan-result (badge) and /api/vuln-details (modal with CVE table)
  - New templates: vuln-badge.html (severity count chips) and vuln-details.html (full CVE table with NVD/GHSA links)
  - Repository page: Lazy-loads scan badges per manifest via HTMX
  - Tests: ~590 lines of test coverage for both handlers
3. S3 Diagnostic Tool
  New cmd/s3-test/main.go (418 lines) — tests S3 connectivity with both SDK v1 and v2, including presigned URL generation, pull zone host swapping, and verbose signing debug output.
4. Deployment Tooling
  - New syncServiceUnit() for comparing/updating systemd units on servers
  - Update command now syncs config keys (adds missing keys from template) and service units with daemon-reload
5. DB Migration
  0011_fix_captain_successor_column.yaml — rebuilds hold_captain_records to add the successor column that was missed in a previous migration.
6. Documentation
  - APPVIEW-UI-FUTURE.md rewritten as a status-tracked feature inventory
  - DISTRIBUTION.md renamed to CREDENTIAL_HELPER.md
  - New REMOVING_DISTRIBUTION.md — 480-line analysis of fully removing distribution from the appview side
7. go.mod
  aws-sdk-go v1 moved from indirect to direct (needed by cmd/s3-test).
This commit is contained in:
Evan Jarrett
2026-02-13 15:26:24 -06:00
parent 434a5f1eee
commit de02e1f046
38 changed files with 3134 additions and 962 deletions
+178
View File
@@ -1,13 +1,17 @@
package s3
import (
"bytes"
"context"
"fmt"
"io"
"strings"
"sync"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
awss3 "github.com/aws/aws-sdk-go-v2/service/s3"
s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/google/uuid"
)
@@ -22,6 +26,9 @@ type MockS3Client struct {
// If empty, a UUID is generated.
UploadID string
// Objects stores in-memory blobs for PutObject/HeadObject/DeleteObject/CopyObject/ListObjectsV2.
Objects map[string][]byte
// Track calls for verification in tests
mu sync.Mutex
CreateMultipartCalls []CreateMultipartCall
@@ -36,6 +43,8 @@ type MockS3Client struct {
CreateMultipartError error
CompleteError error
AbortError error
HeadObjectError error
CopyObjectError error
}
// CreateMultipartCall records a CreateMultipartUpload call
@@ -89,6 +98,7 @@ type PutObjectCall struct {
func NewMockS3Client(testServerURL string) *MockS3Client {
return &MockS3Client{
TestServerURL: testServerURL,
Objects: make(map[string][]byte),
CreateMultipartCalls: []CreateMultipartCall{},
CompleteCalls: []CompleteCall{},
AbortCalls: []AbortCall{},
@@ -144,6 +154,14 @@ func (m *MockS3Client) CompleteMultipartUpload(ctx context.Context, input *awss3
return nil, m.CompleteError
}
// Store a placeholder object at the key so Stat/HeadObject works after complete
key := aws.ToString(input.Key)
if m.Objects != nil {
if _, exists := m.Objects[key]; !exists {
m.Objects[key] = []byte("completed-multipart")
}
}
// Return a mock ETag
etag := "\"mock-etag-" + uuid.New().String() + "\""
return &awss3.CompleteMultipartUploadOutput{
@@ -169,6 +187,141 @@ func (m *MockS3Client) AbortMultipartUpload(ctx context.Context, input *awss3.Ab
return &awss3.AbortMultipartUploadOutput{}, nil
}
// HeadObject implements S3Client
func (m *MockS3Client) HeadObject(ctx context.Context, input *awss3.HeadObjectInput, opts ...func(*awss3.Options)) (*awss3.HeadObjectOutput, error) {
m.mu.Lock()
defer m.mu.Unlock()
if m.HeadObjectError != nil {
return nil, m.HeadObjectError
}
key := aws.ToString(input.Key)
data, ok := m.Objects[key]
if !ok {
return nil, fmt.Errorf("NoSuchKey: object %s not found", key)
}
size := int64(len(data))
return &awss3.HeadObjectOutput{
ContentLength: &size,
}, nil
}
// PutObject implements S3Client
func (m *MockS3Client) PutObject(ctx context.Context, input *awss3.PutObjectInput, opts ...func(*awss3.Options)) (*awss3.PutObjectOutput, error) {
m.mu.Lock()
defer m.mu.Unlock()
key := aws.ToString(input.Key)
m.PutObjectCalls = append(m.PutObjectCalls, PutObjectCall{
Bucket: aws.ToString(input.Bucket),
Key: key,
})
if input.Body != nil {
data, err := io.ReadAll(input.Body)
if err != nil {
return nil, err
}
m.Objects[key] = data
} else {
m.Objects[key] = []byte{}
}
return &awss3.PutObjectOutput{}, nil
}
// CopyObject implements S3Client
func (m *MockS3Client) CopyObject(ctx context.Context, input *awss3.CopyObjectInput, opts ...func(*awss3.Options)) (*awss3.CopyObjectOutput, error) {
m.mu.Lock()
defer m.mu.Unlock()
if m.CopyObjectError != nil {
return nil, m.CopyObjectError
}
// CopySource is "bucket/key"
copySource := aws.ToString(input.CopySource)
// Strip bucket prefix to get key
parts := strings.SplitN(copySource, "/", 2)
srcKey := copySource
if len(parts) == 2 {
srcKey = parts[1]
}
data, ok := m.Objects[srcKey]
if !ok {
return nil, fmt.Errorf("NoSuchKey: source object %s not found", srcKey)
}
dstKey := aws.ToString(input.Key)
m.Objects[dstKey] = append([]byte{}, data...)
return &awss3.CopyObjectOutput{}, nil
}
// DeleteObject implements S3Client
func (m *MockS3Client) DeleteObject(ctx context.Context, input *awss3.DeleteObjectInput, opts ...func(*awss3.Options)) (*awss3.DeleteObjectOutput, error) {
m.mu.Lock()
defer m.mu.Unlock()
key := aws.ToString(input.Key)
delete(m.Objects, key)
return &awss3.DeleteObjectOutput{}, nil
}
// ListObjectsV2 implements S3Client
func (m *MockS3Client) ListObjectsV2(ctx context.Context, input *awss3.ListObjectsV2Input, opts ...func(*awss3.Options)) (*awss3.ListObjectsV2Output, error) {
m.mu.Lock()
defer m.mu.Unlock()
prefix := aws.ToString(input.Prefix)
delimiter := aws.ToString(input.Delimiter)
var contents []s3types.Object
commonPrefixes := map[string]bool{}
for key, data := range m.Objects {
if !strings.HasPrefix(key, prefix) {
continue
}
if delimiter != "" {
// Check if there's a delimiter after the prefix
rest := strings.TrimPrefix(key, prefix)
idx := strings.Index(rest, delimiter)
if idx >= 0 {
// Has delimiter — this is a common prefix, not a content object
cp := prefix + rest[:idx+len(delimiter)]
commonPrefixes[cp] = true
continue
}
}
size := int64(len(data))
k := key
contents = append(contents, s3types.Object{
Key: &k,
Size: &size,
})
}
var cps []s3types.CommonPrefix
for cp := range commonPrefixes {
p := cp
cps = append(cps, s3types.CommonPrefix{Prefix: &p})
}
falseVal := false
return &awss3.ListObjectsV2Output{
Contents: contents,
CommonPrefixes: cps,
IsTruncated: &falseVal,
}, nil
}
// PresignUploadPart implements S3Client
// Returns a mock presigned URL for test server
func (m *MockS3Client) PresignUploadPart(ctx context.Context, input *awss3.UploadPartInput, expires time.Duration) (string, error) {
@@ -229,6 +382,31 @@ func (m *MockS3Client) PresignPutObject(ctx context.Context, input *awss3.PutObj
Key: aws.ToString(input.Key),
})
// Also store the body if provided (for PresignPutObject used in tests that also check objects)
if input.Body != nil {
key := aws.ToString(input.Key)
data, _ := io.ReadAll(input.Body)
m.Objects[key] = data
}
url := fmt.Sprintf("%s/put/%s", m.TestServerURL, aws.ToString(input.Key))
return url, nil
}
// SetObject is a test helper to pre-populate an object in the mock store.
func (m *MockS3Client) SetObject(key string, data []byte) {
m.mu.Lock()
defer m.mu.Unlock()
m.Objects[key] = append([]byte{}, data...)
}
// GetObject is a test helper to read an object from the mock store (nil if not found).
func (m *MockS3Client) GetObject(key string) []byte {
m.mu.Lock()
defer m.mu.Unlock()
data, ok := m.Objects[key]
if !ok {
return nil
}
return bytes.Clone(data)
}