mirror of
https://tangled.org/evan.jarrett.net/at-container-registry
synced 2026-09-25 11:44:16 +00:00
remove distribution from hold, add vulnerability scanning in appview.
1. Removing distribution/distribution from the Hold Service (biggest change) The hold service previously used distribution's StorageDriver interface for all blob operations. This replaces it with direct AWS SDK v2 calls through ATCR's own pkg/s3.S3Service: - New S3Service methods: Stat(), PutBytes(), Move(), Delete(), WalkBlobs(), ListPrefix() added to pkg/s3/types.go - Pull zone fix: Presigned URLs are now generated against the real S3 endpoint, then the host is swapped to the CDN URL post-signing (previously the CDN URL was set as the endpoint, which broke SigV4 signatures) - All hold subsystems migrated: GC, OCI uploads, XRPC handlers, profile uploads, scan broadcaster, manifest posts — all now use *s3.S3Service instead of storagedriver.StorageDriver - Config simplified: Removed configuration.Storage type and buildStorageConfigFromFields(); replaced with a simple S3Params() method - Mock expanded: MockS3Client gains an in-memory object store + 5 new methods, replacing duplicate mockStorageDriver implementations in tests (~160 lines deleted from each test file) 2. Vulnerability Scan UI in AppView (new feature) Displays scan results from the hold's PDS on the repository page: - New lexicon: io/atcr/hold/scan.json with vulnReportBlob field for storing full Grype reports - Two new HTMX endpoints: /api/scan-result (badge) and /api/vuln-details (modal with CVE table) - New templates: vuln-badge.html (severity count chips) and vuln-details.html (full CVE table with NVD/GHSA links) - Repository page: Lazy-loads scan badges per manifest via HTMX - Tests: ~590 lines of test coverage for both handlers 3. S3 Diagnostic Tool New cmd/s3-test/main.go (418 lines) — tests S3 connectivity with both SDK v1 and v2, including presigned URL generation, pull zone host swapping, and verbose signing debug output. 4. Deployment Tooling - New syncServiceUnit() for comparing/updating systemd units on servers - Update command now syncs config keys (adds missing keys from template) and service units with daemon-reload 5. DB Migration 0011_fix_captain_successor_column.yaml — rebuilds hold_captain_records to add the successor column that was missed in a previous migration. 6. Documentation - APPVIEW-UI-FUTURE.md rewritten as a status-tracked feature inventory - DISTRIBUTION.md renamed to CREDENTIAL_HELPER.md - New REMOVING_DISTRIBUTION.md — 480-line analysis of fully removing distribution from the appview side 7. go.mod aws-sdk-go v1 moved from indirect to direct (needed by cmd/s3-test).
This commit is contained in:
+178
@@ -1,13 +1,17 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
awss3 "github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
@@ -22,6 +26,9 @@ type MockS3Client struct {
|
||||
// If empty, a UUID is generated.
|
||||
UploadID string
|
||||
|
||||
// Objects stores in-memory blobs for PutObject/HeadObject/DeleteObject/CopyObject/ListObjectsV2.
|
||||
Objects map[string][]byte
|
||||
|
||||
// Track calls for verification in tests
|
||||
mu sync.Mutex
|
||||
CreateMultipartCalls []CreateMultipartCall
|
||||
@@ -36,6 +43,8 @@ type MockS3Client struct {
|
||||
CreateMultipartError error
|
||||
CompleteError error
|
||||
AbortError error
|
||||
HeadObjectError error
|
||||
CopyObjectError error
|
||||
}
|
||||
|
||||
// CreateMultipartCall records a CreateMultipartUpload call
|
||||
@@ -89,6 +98,7 @@ type PutObjectCall struct {
|
||||
func NewMockS3Client(testServerURL string) *MockS3Client {
|
||||
return &MockS3Client{
|
||||
TestServerURL: testServerURL,
|
||||
Objects: make(map[string][]byte),
|
||||
CreateMultipartCalls: []CreateMultipartCall{},
|
||||
CompleteCalls: []CompleteCall{},
|
||||
AbortCalls: []AbortCall{},
|
||||
@@ -144,6 +154,14 @@ func (m *MockS3Client) CompleteMultipartUpload(ctx context.Context, input *awss3
|
||||
return nil, m.CompleteError
|
||||
}
|
||||
|
||||
// Store a placeholder object at the key so Stat/HeadObject works after complete
|
||||
key := aws.ToString(input.Key)
|
||||
if m.Objects != nil {
|
||||
if _, exists := m.Objects[key]; !exists {
|
||||
m.Objects[key] = []byte("completed-multipart")
|
||||
}
|
||||
}
|
||||
|
||||
// Return a mock ETag
|
||||
etag := "\"mock-etag-" + uuid.New().String() + "\""
|
||||
return &awss3.CompleteMultipartUploadOutput{
|
||||
@@ -169,6 +187,141 @@ func (m *MockS3Client) AbortMultipartUpload(ctx context.Context, input *awss3.Ab
|
||||
return &awss3.AbortMultipartUploadOutput{}, nil
|
||||
}
|
||||
|
||||
// HeadObject implements S3Client
|
||||
func (m *MockS3Client) HeadObject(ctx context.Context, input *awss3.HeadObjectInput, opts ...func(*awss3.Options)) (*awss3.HeadObjectOutput, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
if m.HeadObjectError != nil {
|
||||
return nil, m.HeadObjectError
|
||||
}
|
||||
|
||||
key := aws.ToString(input.Key)
|
||||
data, ok := m.Objects[key]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("NoSuchKey: object %s not found", key)
|
||||
}
|
||||
|
||||
size := int64(len(data))
|
||||
return &awss3.HeadObjectOutput{
|
||||
ContentLength: &size,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// PutObject implements S3Client
|
||||
func (m *MockS3Client) PutObject(ctx context.Context, input *awss3.PutObjectInput, opts ...func(*awss3.Options)) (*awss3.PutObjectOutput, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
key := aws.ToString(input.Key)
|
||||
m.PutObjectCalls = append(m.PutObjectCalls, PutObjectCall{
|
||||
Bucket: aws.ToString(input.Bucket),
|
||||
Key: key,
|
||||
})
|
||||
|
||||
if input.Body != nil {
|
||||
data, err := io.ReadAll(input.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m.Objects[key] = data
|
||||
} else {
|
||||
m.Objects[key] = []byte{}
|
||||
}
|
||||
|
||||
return &awss3.PutObjectOutput{}, nil
|
||||
}
|
||||
|
||||
// CopyObject implements S3Client
|
||||
func (m *MockS3Client) CopyObject(ctx context.Context, input *awss3.CopyObjectInput, opts ...func(*awss3.Options)) (*awss3.CopyObjectOutput, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
if m.CopyObjectError != nil {
|
||||
return nil, m.CopyObjectError
|
||||
}
|
||||
|
||||
// CopySource is "bucket/key"
|
||||
copySource := aws.ToString(input.CopySource)
|
||||
// Strip bucket prefix to get key
|
||||
parts := strings.SplitN(copySource, "/", 2)
|
||||
srcKey := copySource
|
||||
if len(parts) == 2 {
|
||||
srcKey = parts[1]
|
||||
}
|
||||
|
||||
data, ok := m.Objects[srcKey]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("NoSuchKey: source object %s not found", srcKey)
|
||||
}
|
||||
|
||||
dstKey := aws.ToString(input.Key)
|
||||
m.Objects[dstKey] = append([]byte{}, data...)
|
||||
|
||||
return &awss3.CopyObjectOutput{}, nil
|
||||
}
|
||||
|
||||
// DeleteObject implements S3Client
|
||||
func (m *MockS3Client) DeleteObject(ctx context.Context, input *awss3.DeleteObjectInput, opts ...func(*awss3.Options)) (*awss3.DeleteObjectOutput, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
key := aws.ToString(input.Key)
|
||||
delete(m.Objects, key)
|
||||
|
||||
return &awss3.DeleteObjectOutput{}, nil
|
||||
}
|
||||
|
||||
// ListObjectsV2 implements S3Client
|
||||
func (m *MockS3Client) ListObjectsV2(ctx context.Context, input *awss3.ListObjectsV2Input, opts ...func(*awss3.Options)) (*awss3.ListObjectsV2Output, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
prefix := aws.ToString(input.Prefix)
|
||||
delimiter := aws.ToString(input.Delimiter)
|
||||
|
||||
var contents []s3types.Object
|
||||
commonPrefixes := map[string]bool{}
|
||||
|
||||
for key, data := range m.Objects {
|
||||
if !strings.HasPrefix(key, prefix) {
|
||||
continue
|
||||
}
|
||||
|
||||
if delimiter != "" {
|
||||
// Check if there's a delimiter after the prefix
|
||||
rest := strings.TrimPrefix(key, prefix)
|
||||
idx := strings.Index(rest, delimiter)
|
||||
if idx >= 0 {
|
||||
// Has delimiter — this is a common prefix, not a content object
|
||||
cp := prefix + rest[:idx+len(delimiter)]
|
||||
commonPrefixes[cp] = true
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
size := int64(len(data))
|
||||
k := key
|
||||
contents = append(contents, s3types.Object{
|
||||
Key: &k,
|
||||
Size: &size,
|
||||
})
|
||||
}
|
||||
|
||||
var cps []s3types.CommonPrefix
|
||||
for cp := range commonPrefixes {
|
||||
p := cp
|
||||
cps = append(cps, s3types.CommonPrefix{Prefix: &p})
|
||||
}
|
||||
|
||||
falseVal := false
|
||||
return &awss3.ListObjectsV2Output{
|
||||
Contents: contents,
|
||||
CommonPrefixes: cps,
|
||||
IsTruncated: &falseVal,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// PresignUploadPart implements S3Client
|
||||
// Returns a mock presigned URL for test server
|
||||
func (m *MockS3Client) PresignUploadPart(ctx context.Context, input *awss3.UploadPartInput, expires time.Duration) (string, error) {
|
||||
@@ -229,6 +382,31 @@ func (m *MockS3Client) PresignPutObject(ctx context.Context, input *awss3.PutObj
|
||||
Key: aws.ToString(input.Key),
|
||||
})
|
||||
|
||||
// Also store the body if provided (for PresignPutObject used in tests that also check objects)
|
||||
if input.Body != nil {
|
||||
key := aws.ToString(input.Key)
|
||||
data, _ := io.ReadAll(input.Body)
|
||||
m.Objects[key] = data
|
||||
}
|
||||
|
||||
url := fmt.Sprintf("%s/put/%s", m.TestServerURL, aws.ToString(input.Key))
|
||||
return url, nil
|
||||
}
|
||||
|
||||
// SetObject is a test helper to pre-populate an object in the mock store.
|
||||
func (m *MockS3Client) SetObject(key string, data []byte) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.Objects[key] = append([]byte{}, data...)
|
||||
}
|
||||
|
||||
// GetObject is a test helper to read an object from the mock store (nil if not found).
|
||||
func (m *MockS3Client) GetObject(key string) []byte {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
data, ok := m.Objects[key]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return bytes.Clone(data)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user