mirrors/at-container-registry
1
0
Fork 0
mirror of https://tangled.org/evan.jarrett.net/at-container-registry synced 2026-09-25 03:34:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

use presigned urls for s3 to avoid hold bandwidth

Browse Source
This commit is contained in:
Evan Jarrett
2025-10-10 09:43:26 -05:00
parent b9619ad762
commit 3fad473910
8 changed files with 186 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.hold.example
+6
View File
@@ -16,6 +16,12 @@ HOLD_PUBLIC_URL=http://127.0.0.1:8080
# Storage driver type (s3, filesystem)
# Default: s3
#
# S3 Presigned URLs:
# When using S3 storage, presigned URLs are automatically enabled for direct
# client ↔ S3 transfers. This eliminates the hold service as a bandwidth
# bottleneck, reducing hold bandwidth by ~99% for push/pull operations.
# Falls back to proxy mode automatically for non-S3 drivers.
STORAGE_DRIVER=filesystem
# For S3/Storj/Minio:
Dockerfile.appview
+10 -31
View File
@@ -1,46 +1,31 @@
# ==========================================
# Stage 1: Build stage with Debian (glibc)
# ==========================================
FROM golang:1.25.2-trixie AS builder
# Install SQLite development libraries (for CGO compilation)
RUN apt-get update && \
apt-get install -y --no-install-recommends sqlite3 libsqlite3-dev && \
rm -rf /var/lib/apt/lists/*
# Set working directory
WORKDIR /build
# Copy go mod files and download dependencies (cached layer)
COPY go.mod go.sum ./
RUN go mod download
# Copy source code
COPY . .
# Build optimized binary:
# - CGO_ENABLED=1: Required for SQLite (mattn/go-sqlite3)
# - -ldflags="-s -w": Strip debug symbols (~30% size reduction)
# - -tags sqlite_omit_load_extension: Remove SQLite extension loading (~100KB savings)
# - -trimpath: Remove build paths (reproducible builds)
# SQLite is statically embedded in the binary (no runtime .so needed)
RUN CGO_ENABLED=1 go build \
-ldflags="-s -w" \
-tags sqlite_omit_load_extension \
-trimpath \
-o atcr-appview ./cmd/appview
# Collect minimal runtime dependencies based on ldd output
RUN mkdir -p /runtime-deps/lib/x86_64-linux-gnu /runtime-deps/lib64 && \
# Core glibc library (only one the binary links to)
cp -L /lib/x86_64-linux-gnu/libc.so.6 /runtime-deps/lib/x86_64-linux-gnu/ && \
# Dynamic linker
cp -L /lib64/ld-linux-x86-64.so.2 /runtime-deps/lib64/ && \
# NSS modules for DNS resolution (loaded via dlopen at runtime, not shown in ldd)
cp -L /lib/x86_64-linux-gnu/libnss_dns.so.2 /runtime-deps/lib/x86_64-linux-gnu/ && \
cp -L /lib/x86_64-linux-gnu/libnss_files.so.2 /runtime-deps/lib/x86_64-linux-gnu/ && \
# NSS modules depend on libresolv
cp -L /lib/x86_64-linux-gnu/libresolv.so.2 /runtime-deps/lib/x86_64-linux-gnu/ && \
# Collect minimal runtime dependencies
RUN mkdir -p /runtime-deps/lib64 /runtime-deps/lib/x86_64-linux-gnu && \
# Core glibc libraries (from ldd output)
cp /lib/x86_64-linux-gnu/libc.so.6 /runtime-deps/lib64/ && \
cp /lib/x86_64-linux-gnu/libresolv.so.2 /runtime-deps/lib64/ && \
cp /lib64/ld-linux-x86-64.so.2 /runtime-deps/lib64/ && \
# NSS (Name Service Switch) modules for DNS resolution
cp /lib/x86_64-linux-gnu/libnss_dns.so.2 /runtime-deps/lib/x86_64-linux-gnu/ && \
cp /lib/x86_64-linux-gnu/libnss_files.so.2 /runtime-deps/lib/x86_64-linux-gnu/ && \
# Create NSS config (tells glibc to check /etc/hosts then DNS)
echo "hosts: files dns" > /tmp/nsswitch.conf
@@ -51,20 +36,16 @@ FROM scratch
# Copy minimal glibc runtime dependencies
COPY --from=builder /runtime-deps /
# Copy NSS configuration for DNS resolution
COPY --from=builder /tmp/nsswitch.conf /etc/nsswitch.conf
# Copy CA certificates for HTTPS (PDS, Jetstream, relay connections)
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
# Copy timezone data for timestamp formatting
COPY --from=builder /usr/share/zoneinfo /usr/share/zoneinfo
# Copy optimized binary (SQLite embedded)
COPY --from=builder /build/atcr-appview /atcr-appview
# Expose port (main HTTP server)
# Expose ports
EXPOSE 5000
# OCI image annotations
@@ -77,7 +58,5 @@ LABEL org.opencontainers.image.title="ATCR AppView" \
org.opencontainers.image.version="0.1.0" \
io.atcr.icon="https://imgs.blue/evan.jarrett.net/1TpTNrRelfloN2emuWZDrWmPT0o93bAjEnozjD6UPgoVV9m4"
# Run the AppView (no config file - uses environment variables)
# Creates /var/lib/atcr directories on first run via Go code
ENTRYPOINT ["/atcr-appview"]
CMD ["serve"]
cmd/appview/config.go
+8 -8
View File
@@ -97,8 +97,8 @@ func buildStorageConfig() configuration.Storage {
storage["maintenance"] = configuration.Parameters{
"uploadpurging": map[interface{}]interface{}{
"enabled": false,
"age": 7 * 24 * time.Hour, // 168h
"interval": 24 * time.Hour, // 24h
"age": 7 * 24 * time.Hour, // 168h
"interval": 24 * time.Hour, // 24h
"dryrun": false,
},
}
@@ -141,12 +141,12 @@ func buildAuthConfig(baseURL string) (configuration.Auth, error) {
return configuration.Auth{
"token": configuration.Parameters{
"realm": realm,
"service": serviceName,
"issuer": serviceName,
"rootcertbundle": certPath,
"privatekey": privateKeyPath,
"expiration": expiration,
"realm": realm,
"service": serviceName,
"issuer": serviceName,
"rootcertbundle": certPath,
"privatekey": privateKeyPath,
"expiration": expiration,
},
}, nil
}
cmd/appview/serve.go
+5 -5
View File
@@ -33,11 +33,11 @@ import (
// Define sensitive tables that should never be accessible from public queries
var sensitiveTables = map[string]bool{
"oauth_sessions": true, // OAuth tokens
"ui_sessions": true, // Session IDs
"oauth_auth_requests": true, // OAuth state
"devices": true, // Device secret hashes
"pending_device_auth": true, // Pending device secrets
"oauth_sessions": true, // OAuth tokens
"ui_sessions": true, // Session IDs
"oauth_auth_requests": true, // OAuth state
"devices": true, // Device secret hashes
"pending_device_auth": true, // Pending device secrets
}
// readOnlyAuthorizerCallback blocks access to sensitive tables
cmd/hold/main.go
+151 -10
View File
@@ -12,6 +12,10 @@ import (
"strings"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/distribution/distribution/v3/configuration"
storagedriver "github.com/distribution/distribution/v3/registry/storage/driver"
"github.com/distribution/distribution/v3/registry/storage/driver/factory"
@@ -70,8 +74,11 @@ type ServerConfig struct {
// HoldService provides presigned URLs for blob storage in a hold
type HoldService struct {
driver storagedriver.StorageDriver
config *Config
driver storagedriver.StorageDriver
config *Config
s3Client *s3.S3 // S3 client for presigned URLs (nil if not S3 storage)
bucket string // S3 bucket name
s3PathPrefix string // S3 path prefix (if any)
}
// NewHoldService creates a new hold service
@@ -83,10 +90,86 @@ func NewHoldService(cfg *Config) (*HoldService, error) {
return nil, fmt.Errorf("failed to create storage driver: %w", err)
}
return &HoldService{
service := &HoldService{
driver: driver,
config: cfg,
}, nil
}
// Initialize S3 client for presigned URLs (if using S3 storage)
if err := service.initS3Client(); err != nil {
log.Printf("WARNING: S3 presigned URLs disabled: %v", err)
}
return service, nil
}
// initS3Client initializes the S3 client for presigned URL generation
// Returns nil error if S3 client is successfully initialized
// Returns error if storage is not S3 or if initialization fails (service will fall back to proxy mode)
func (s *HoldService) initS3Client() error {
// Check if storage driver is S3
if s.config.Storage.Type() != "s3" {
log.Printf("Storage driver is %s (not S3), presigned URLs disabled", s.config.Storage.Type())
return nil // Not an error - just using different driver
}
// Extract S3 configuration from storage parameters
params, ok := s.config.Storage.Parameters()["s3"].(configuration.Parameters)
if !ok {
return fmt.Errorf("failed to get S3 parameters from storage config")
}
// Extract required S3 configuration
region, _ := params["region"].(string)
if region == "" {
region = "us-east-1" // Default region
}
accessKey, _ := params["accesskey"].(string)
secretKey, _ := params["secretkey"].(string)
bucket, _ := params["bucket"].(string)
if bucket == "" {
return fmt.Errorf("S3 bucket not configured")
}
// Build AWS config
awsConfig := &aws.Config{
Region: aws.String(region),
}
// Add credentials if provided (allow IAM role auth if not provided)
if accessKey != "" && secretKey != "" {
awsConfig.Credentials = credentials.NewStaticCredentials(accessKey, secretKey, "")
}
// Add custom endpoint for S3-compatible services (Storj, MinIO, R2, etc.)
if endpoint, ok := params["regionendpoint"].(string); ok && endpoint != "" {
awsConfig.Endpoint = aws.String(endpoint)
awsConfig.S3ForcePathStyle = aws.Bool(true) // Required for MinIO, Storj
}
// Create AWS session
sess, err := session.NewSession(awsConfig)
if err != nil {
return fmt.Errorf("failed to create AWS session: %w", err)
}
// Create S3 client
s.s3Client = s3.New(sess)
s.bucket = bucket
// Extract path prefix if configured (rootdirectory in S3 params)
if rootDir, ok := params["rootdirectory"].(string); ok && rootDir != "" {
s.s3PathPrefix = strings.TrimPrefix(rootDir, "/")
}
log.Printf("S3 presigned URLs enabled for bucket: %s", s.bucket)
if s.s3PathPrefix != "" {
log.Printf("S3 path prefix: %s", s.s3PathPrefix)
}
return nil
}
// GetPresignedURLRequest represents a request for a presigned download URL
@@ -491,17 +574,75 @@ func (s *HoldService) getDownloadURL(ctx context.Context, digest string, did str
return "", fmt.Errorf("blob not found: %w", err)
}
// For drivers that support presigned URLs (S3), use those
// For now, return a proxy URL through this service with DID for authorization
return fmt.Sprintf("%s/blobs/%s?did=%s", s.config.Server.PublicURL, digest, did), nil
// If S3 client available, generate presigned URL
if s.s3Client != nil {
// Build S3 key from blob path
// blobPath returns paths like: /docker/registry/v2/blobs/sha256/ab/abc123.../data
s3Key := strings.TrimPrefix(path, "/")
if s.s3PathPrefix != "" {
s3Key = s.s3PathPrefix + "/" + s3Key
}
// Generate presigned GET URL
req, _ := s.s3Client.GetObjectRequest(&s3.GetObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(s3Key),
})
url, err := req.Presign(15 * time.Minute)
if err != nil {
log.Printf("WARN: Presigned URL generation failed for %s, falling back to proxy: %v", digest, err)
return s.getProxyDownloadURL(digest, did), nil
}
log.Printf("Generated presigned download URL for %s (expires in 15min)", digest)
return url, nil
}
// Fallback: return proxy URL through this service
return s.getProxyDownloadURL(digest, did), nil
}
// getProxyDownloadURL returns a proxy URL for blob download (fallback when presigned URLs unavailable)
func (s *HoldService) getProxyDownloadURL(digest, did string) string {
return fmt.Sprintf("%s/blobs/%s?did=%s", s.config.Server.PublicURL, digest, did)
}
// getUploadURL generates an upload URL for a blob
// Note: This is called from HandlePutPresignedURL which has the DID in the request
func (s *HoldService) getUploadURL(ctx context.Context, digest string, size int64, did string) (string, error) {
// For drivers that support presigned URLs (S3), use those
// For now, return a proxy URL through this service with DID for authorization
return fmt.Sprintf("%s/blobs/%s?did=%s", s.config.Server.PublicURL, digest, did), nil
// If S3 client available, generate presigned URL
if s.s3Client != nil {
// Build S3 key from blob path
path := blobPath(digest)
s3Key := strings.TrimPrefix(path, "/")
if s.s3PathPrefix != "" {
s3Key = s.s3PathPrefix + "/" + s3Key
}
// Generate presigned PUT URL
req, _ := s.s3Client.PutObjectRequest(&s3.PutObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(s3Key),
})
url, err := req.Presign(15 * time.Minute)
if err != nil {
log.Printf("WARN: Presigned URL generation failed for %s, falling back to proxy: %v", digest, err)
return s.getProxyUploadURL(digest, did), nil
}
log.Printf("Generated presigned upload URL for %s (expires in 15min)", digest)
return url, nil
}
// Fallback: return proxy URL through this service
return s.getProxyUploadURL(digest, did), nil
}
// getProxyUploadURL returns a proxy URL for blob upload (fallback when presigned URLs unavailable)
func (s *HoldService) getProxyUploadURL(digest, did string) string {
return fmt.Sprintf("%s/blobs/%s?did=%s", s.config.Server.PublicURL, digest, did)
}
// RegisterRequest represents a request to register this hold in a user's PDS
go.mod
+1 -1
View File
@@ -3,6 +3,7 @@ module atcr.io
go 1.24.7
require (
github.com/aws/aws-sdk-go v1.55.5
github.com/bluesky-social/indigo v0.0.0-20251003000214-3259b215110e
github.com/distribution/distribution/v3 v3.0.0
github.com/distribution/reference v0.6.0
@@ -19,7 +20,6 @@ require (
)
require (
github.com/aws/aws-sdk-go v1.55.5 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bshuster-repo/logrus-logstash-hook v1.0.0 // indirect
github.com/carlmjohnson/versioninfo v0.22.5 // indirect
pkg/appview/db/schema.go
+4 -4
View File
@@ -196,10 +196,10 @@ func InitDB(path string) (*sql.DB, error) {
// Migration represents a database migration
type Migration struct {
Version int
Name string
Description string `yaml:"description"`
Query string `yaml:"query"`
Version int
Name string
Description string `yaml:"description"`
Query string `yaml:"query"`
}
// runMigrations applies any pending database migrations
pkg/appview/jetstream/worker.go
+1 -1
View File
@@ -46,7 +46,7 @@ type Worker struct {
pongMutex sync.Mutex
// In-memory cursor tracking for reconnects
lastCursor int64
lastCursor int64
cursorMutex sync.RWMutex
}