From 3fad4739101aafeec153027923252658e46d202b Mon Sep 17 00:00:00 2001 From: Evan Jarrett Date: Fri, 10 Oct 2025 09:43:26 -0500 Subject: [PATCH] use presigned urls for s3 to avoid hold bandwidth --- .env.hold.example | 6 ++ Dockerfile.appview | 41 ++------ cmd/appview/config.go | 16 ++-- cmd/appview/serve.go | 10 +- cmd/hold/main.go | 161 ++++++++++++++++++++++++++++++-- go.mod | 2 +- pkg/appview/db/schema.go | 8 +- pkg/appview/jetstream/worker.go | 2 +- 8 files changed, 186 insertions(+), 60 deletions(-) diff --git a/.env.hold.example b/.env.hold.example index 3f3c8f5..861baf9 100644 --- a/.env.hold.example +++ b/.env.hold.example @@ -16,6 +16,12 @@ HOLD_PUBLIC_URL=http://127.0.0.1:8080 # Storage driver type (s3, filesystem) # Default: s3 +# +# S3 Presigned URLs: +# When using S3 storage, presigned URLs are automatically enabled for direct +# client ↔ S3 transfers. This eliminates the hold service as a bandwidth +# bottleneck, reducing hold bandwidth by ~99% for push/pull operations. +# Falls back to proxy mode automatically for non-S3 drivers. STORAGE_DRIVER=filesystem # For S3/Storj/Minio: diff --git a/Dockerfile.appview b/Dockerfile.appview index bd35b77..4061d3b 100644 --- a/Dockerfile.appview +++ b/Dockerfile.appview @@ -1,46 +1,31 @@ -# ========================================== -# Stage 1: Build stage with Debian (glibc) -# ========================================== FROM golang:1.25.2-trixie AS builder -# Install SQLite development libraries (for CGO compilation) RUN apt-get update && \ apt-get install -y --no-install-recommends sqlite3 libsqlite3-dev && \ rm -rf /var/lib/apt/lists/* -# Set working directory WORKDIR /build -# Copy go mod files and download dependencies (cached layer) COPY go.mod go.sum ./ RUN go mod download -# Copy source code COPY . . -# Build optimized binary: -# - CGO_ENABLED=1: Required for SQLite (mattn/go-sqlite3) -# - -ldflags="-s -w": Strip debug symbols (~30% size reduction) -# - -tags sqlite_omit_load_extension: Remove SQLite extension loading (~100KB savings) -# - -trimpath: Remove build paths (reproducible builds) -# SQLite is statically embedded in the binary (no runtime .so needed) RUN CGO_ENABLED=1 go build \ -ldflags="-s -w" \ -tags sqlite_omit_load_extension \ -trimpath \ -o atcr-appview ./cmd/appview -# Collect minimal runtime dependencies based on ldd output -RUN mkdir -p /runtime-deps/lib/x86_64-linux-gnu /runtime-deps/lib64 && \ - # Core glibc library (only one the binary links to) - cp -L /lib/x86_64-linux-gnu/libc.so.6 /runtime-deps/lib/x86_64-linux-gnu/ && \ - # Dynamic linker - cp -L /lib64/ld-linux-x86-64.so.2 /runtime-deps/lib64/ && \ - # NSS modules for DNS resolution (loaded via dlopen at runtime, not shown in ldd) - cp -L /lib/x86_64-linux-gnu/libnss_dns.so.2 /runtime-deps/lib/x86_64-linux-gnu/ && \ - cp -L /lib/x86_64-linux-gnu/libnss_files.so.2 /runtime-deps/lib/x86_64-linux-gnu/ && \ - # NSS modules depend on libresolv - cp -L /lib/x86_64-linux-gnu/libresolv.so.2 /runtime-deps/lib/x86_64-linux-gnu/ && \ +# Collect minimal runtime dependencies +RUN mkdir -p /runtime-deps/lib64 /runtime-deps/lib/x86_64-linux-gnu && \ + # Core glibc libraries (from ldd output) + cp /lib/x86_64-linux-gnu/libc.so.6 /runtime-deps/lib64/ && \ + cp /lib/x86_64-linux-gnu/libresolv.so.2 /runtime-deps/lib64/ && \ + cp /lib64/ld-linux-x86-64.so.2 /runtime-deps/lib64/ && \ + # NSS (Name Service Switch) modules for DNS resolution + cp /lib/x86_64-linux-gnu/libnss_dns.so.2 /runtime-deps/lib/x86_64-linux-gnu/ && \ + cp /lib/x86_64-linux-gnu/libnss_files.so.2 /runtime-deps/lib/x86_64-linux-gnu/ && \ # Create NSS config (tells glibc to check /etc/hosts then DNS) echo "hosts: files dns" > /tmp/nsswitch.conf @@ -51,20 +36,16 @@ FROM scratch # Copy minimal glibc runtime dependencies COPY --from=builder /runtime-deps / - # Copy NSS configuration for DNS resolution COPY --from=builder /tmp/nsswitch.conf /etc/nsswitch.conf - # Copy CA certificates for HTTPS (PDS, Jetstream, relay connections) COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ - # Copy timezone data for timestamp formatting COPY --from=builder /usr/share/zoneinfo /usr/share/zoneinfo - # Copy optimized binary (SQLite embedded) COPY --from=builder /build/atcr-appview /atcr-appview -# Expose port (main HTTP server) +# Expose ports EXPOSE 5000 # OCI image annotations @@ -77,7 +58,5 @@ LABEL org.opencontainers.image.title="ATCR AppView" \ org.opencontainers.image.version="0.1.0" \ io.atcr.icon="https://imgs.blue/evan.jarrett.net/1TpTNrRelfloN2emuWZDrWmPT0o93bAjEnozjD6UPgoVV9m4" -# Run the AppView (no config file - uses environment variables) -# Creates /var/lib/atcr directories on first run via Go code ENTRYPOINT ["/atcr-appview"] CMD ["serve"] diff --git a/cmd/appview/config.go b/cmd/appview/config.go index 5be0f9d..0a9e7b3 100644 --- a/cmd/appview/config.go +++ b/cmd/appview/config.go @@ -97,8 +97,8 @@ func buildStorageConfig() configuration.Storage { storage["maintenance"] = configuration.Parameters{ "uploadpurging": map[interface{}]interface{}{ "enabled": false, - "age": 7 * 24 * time.Hour, // 168h - "interval": 24 * time.Hour, // 24h + "age": 7 * 24 * time.Hour, // 168h + "interval": 24 * time.Hour, // 24h "dryrun": false, }, } @@ -141,12 +141,12 @@ func buildAuthConfig(baseURL string) (configuration.Auth, error) { return configuration.Auth{ "token": configuration.Parameters{ - "realm": realm, - "service": serviceName, - "issuer": serviceName, - "rootcertbundle": certPath, - "privatekey": privateKeyPath, - "expiration": expiration, + "realm": realm, + "service": serviceName, + "issuer": serviceName, + "rootcertbundle": certPath, + "privatekey": privateKeyPath, + "expiration": expiration, }, }, nil } diff --git a/cmd/appview/serve.go b/cmd/appview/serve.go index dda4465..2057d56 100644 --- a/cmd/appview/serve.go +++ b/cmd/appview/serve.go @@ -33,11 +33,11 @@ import ( // Define sensitive tables that should never be accessible from public queries var sensitiveTables = map[string]bool{ - "oauth_sessions": true, // OAuth tokens - "ui_sessions": true, // Session IDs - "oauth_auth_requests": true, // OAuth state - "devices": true, // Device secret hashes - "pending_device_auth": true, // Pending device secrets + "oauth_sessions": true, // OAuth tokens + "ui_sessions": true, // Session IDs + "oauth_auth_requests": true, // OAuth state + "devices": true, // Device secret hashes + "pending_device_auth": true, // Pending device secrets } // readOnlyAuthorizerCallback blocks access to sensitive tables diff --git a/cmd/hold/main.go b/cmd/hold/main.go index 040718d..ba8eb3f 100644 --- a/cmd/hold/main.go +++ b/cmd/hold/main.go @@ -12,6 +12,10 @@ import ( "strings" "time" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" "github.com/distribution/distribution/v3/configuration" storagedriver "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/distribution/distribution/v3/registry/storage/driver/factory" @@ -70,8 +74,11 @@ type ServerConfig struct { // HoldService provides presigned URLs for blob storage in a hold type HoldService struct { - driver storagedriver.StorageDriver - config *Config + driver storagedriver.StorageDriver + config *Config + s3Client *s3.S3 // S3 client for presigned URLs (nil if not S3 storage) + bucket string // S3 bucket name + s3PathPrefix string // S3 path prefix (if any) } // NewHoldService creates a new hold service @@ -83,10 +90,86 @@ func NewHoldService(cfg *Config) (*HoldService, error) { return nil, fmt.Errorf("failed to create storage driver: %w", err) } - return &HoldService{ + service := &HoldService{ driver: driver, config: cfg, - }, nil + } + + // Initialize S3 client for presigned URLs (if using S3 storage) + if err := service.initS3Client(); err != nil { + log.Printf("WARNING: S3 presigned URLs disabled: %v", err) + } + + return service, nil +} + +// initS3Client initializes the S3 client for presigned URL generation +// Returns nil error if S3 client is successfully initialized +// Returns error if storage is not S3 or if initialization fails (service will fall back to proxy mode) +func (s *HoldService) initS3Client() error { + // Check if storage driver is S3 + if s.config.Storage.Type() != "s3" { + log.Printf("Storage driver is %s (not S3), presigned URLs disabled", s.config.Storage.Type()) + return nil // Not an error - just using different driver + } + + // Extract S3 configuration from storage parameters + params, ok := s.config.Storage.Parameters()["s3"].(configuration.Parameters) + if !ok { + return fmt.Errorf("failed to get S3 parameters from storage config") + } + + // Extract required S3 configuration + region, _ := params["region"].(string) + if region == "" { + region = "us-east-1" // Default region + } + + accessKey, _ := params["accesskey"].(string) + secretKey, _ := params["secretkey"].(string) + bucket, _ := params["bucket"].(string) + + if bucket == "" { + return fmt.Errorf("S3 bucket not configured") + } + + // Build AWS config + awsConfig := &aws.Config{ + Region: aws.String(region), + } + + // Add credentials if provided (allow IAM role auth if not provided) + if accessKey != "" && secretKey != "" { + awsConfig.Credentials = credentials.NewStaticCredentials(accessKey, secretKey, "") + } + + // Add custom endpoint for S3-compatible services (Storj, MinIO, R2, etc.) + if endpoint, ok := params["regionendpoint"].(string); ok && endpoint != "" { + awsConfig.Endpoint = aws.String(endpoint) + awsConfig.S3ForcePathStyle = aws.Bool(true) // Required for MinIO, Storj + } + + // Create AWS session + sess, err := session.NewSession(awsConfig) + if err != nil { + return fmt.Errorf("failed to create AWS session: %w", err) + } + + // Create S3 client + s.s3Client = s3.New(sess) + s.bucket = bucket + + // Extract path prefix if configured (rootdirectory in S3 params) + if rootDir, ok := params["rootdirectory"].(string); ok && rootDir != "" { + s.s3PathPrefix = strings.TrimPrefix(rootDir, "/") + } + + log.Printf("S3 presigned URLs enabled for bucket: %s", s.bucket) + if s.s3PathPrefix != "" { + log.Printf("S3 path prefix: %s", s.s3PathPrefix) + } + + return nil } // GetPresignedURLRequest represents a request for a presigned download URL @@ -491,17 +574,75 @@ func (s *HoldService) getDownloadURL(ctx context.Context, digest string, did str return "", fmt.Errorf("blob not found: %w", err) } - // For drivers that support presigned URLs (S3), use those - // For now, return a proxy URL through this service with DID for authorization - return fmt.Sprintf("%s/blobs/%s?did=%s", s.config.Server.PublicURL, digest, did), nil + // If S3 client available, generate presigned URL + if s.s3Client != nil { + // Build S3 key from blob path + // blobPath returns paths like: /docker/registry/v2/blobs/sha256/ab/abc123.../data + s3Key := strings.TrimPrefix(path, "/") + if s.s3PathPrefix != "" { + s3Key = s.s3PathPrefix + "/" + s3Key + } + + // Generate presigned GET URL + req, _ := s.s3Client.GetObjectRequest(&s3.GetObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(s3Key), + }) + + url, err := req.Presign(15 * time.Minute) + if err != nil { + log.Printf("WARN: Presigned URL generation failed for %s, falling back to proxy: %v", digest, err) + return s.getProxyDownloadURL(digest, did), nil + } + + log.Printf("Generated presigned download URL for %s (expires in 15min)", digest) + return url, nil + } + + // Fallback: return proxy URL through this service + return s.getProxyDownloadURL(digest, did), nil +} + +// getProxyDownloadURL returns a proxy URL for blob download (fallback when presigned URLs unavailable) +func (s *HoldService) getProxyDownloadURL(digest, did string) string { + return fmt.Sprintf("%s/blobs/%s?did=%s", s.config.Server.PublicURL, digest, did) } // getUploadURL generates an upload URL for a blob // Note: This is called from HandlePutPresignedURL which has the DID in the request func (s *HoldService) getUploadURL(ctx context.Context, digest string, size int64, did string) (string, error) { - // For drivers that support presigned URLs (S3), use those - // For now, return a proxy URL through this service with DID for authorization - return fmt.Sprintf("%s/blobs/%s?did=%s", s.config.Server.PublicURL, digest, did), nil + // If S3 client available, generate presigned URL + if s.s3Client != nil { + // Build S3 key from blob path + path := blobPath(digest) + s3Key := strings.TrimPrefix(path, "/") + if s.s3PathPrefix != "" { + s3Key = s.s3PathPrefix + "/" + s3Key + } + + // Generate presigned PUT URL + req, _ := s.s3Client.PutObjectRequest(&s3.PutObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(s3Key), + }) + + url, err := req.Presign(15 * time.Minute) + if err != nil { + log.Printf("WARN: Presigned URL generation failed for %s, falling back to proxy: %v", digest, err) + return s.getProxyUploadURL(digest, did), nil + } + + log.Printf("Generated presigned upload URL for %s (expires in 15min)", digest) + return url, nil + } + + // Fallback: return proxy URL through this service + return s.getProxyUploadURL(digest, did), nil +} + +// getProxyUploadURL returns a proxy URL for blob upload (fallback when presigned URLs unavailable) +func (s *HoldService) getProxyUploadURL(digest, did string) string { + return fmt.Sprintf("%s/blobs/%s?did=%s", s.config.Server.PublicURL, digest, did) } // RegisterRequest represents a request to register this hold in a user's PDS diff --git a/go.mod b/go.mod index 0f9d76d..5964105 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module atcr.io go 1.24.7 require ( + github.com/aws/aws-sdk-go v1.55.5 github.com/bluesky-social/indigo v0.0.0-20251003000214-3259b215110e github.com/distribution/distribution/v3 v3.0.0 github.com/distribution/reference v0.6.0 @@ -19,7 +20,6 @@ require ( ) require ( - github.com/aws/aws-sdk-go v1.55.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bshuster-repo/logrus-logstash-hook v1.0.0 // indirect github.com/carlmjohnson/versioninfo v0.22.5 // indirect diff --git a/pkg/appview/db/schema.go b/pkg/appview/db/schema.go index 172b32c..68f4a84 100644 --- a/pkg/appview/db/schema.go +++ b/pkg/appview/db/schema.go @@ -196,10 +196,10 @@ func InitDB(path string) (*sql.DB, error) { // Migration represents a database migration type Migration struct { - Version int - Name string - Description string `yaml:"description"` - Query string `yaml:"query"` + Version int + Name string + Description string `yaml:"description"` + Query string `yaml:"query"` } // runMigrations applies any pending database migrations diff --git a/pkg/appview/jetstream/worker.go b/pkg/appview/jetstream/worker.go index 54c95ab..155c0bd 100644 --- a/pkg/appview/jetstream/worker.go +++ b/pkg/appview/jetstream/worker.go @@ -46,7 +46,7 @@ type Worker struct { pongMutex sync.Mutex // In-memory cursor tracking for reconnects - lastCursor int64 + lastCursor int64 cursorMutex sync.RWMutex }