mirror of
https://tangled.org/evan.jarrett.net/at-container-registry
synced 2026-09-20 17:24:16 +00:00
1. Removing distribution/distribution from the Hold Service (biggest change) The hold service previously used distribution's StorageDriver interface for all blob operations. This replaces it with direct AWS SDK v2 calls through ATCR's own pkg/s3.S3Service: - New S3Service methods: Stat(), PutBytes(), Move(), Delete(), WalkBlobs(), ListPrefix() added to pkg/s3/types.go - Pull zone fix: Presigned URLs are now generated against the real S3 endpoint, then the host is swapped to the CDN URL post-signing (previously the CDN URL was set as the endpoint, which broke SigV4 signatures) - All hold subsystems migrated: GC, OCI uploads, XRPC handlers, profile uploads, scan broadcaster, manifest posts — all now use *s3.S3Service instead of storagedriver.StorageDriver - Config simplified: Removed configuration.Storage type and buildStorageConfigFromFields(); replaced with a simple S3Params() method - Mock expanded: MockS3Client gains an in-memory object store + 5 new methods, replacing duplicate mockStorageDriver implementations in tests (~160 lines deleted from each test file) 2. Vulnerability Scan UI in AppView (new feature) Displays scan results from the hold's PDS on the repository page: - New lexicon: io/atcr/hold/scan.json with vulnReportBlob field for storing full Grype reports - Two new HTMX endpoints: /api/scan-result (badge) and /api/vuln-details (modal with CVE table) - New templates: vuln-badge.html (severity count chips) and vuln-details.html (full CVE table with NVD/GHSA links) - Repository page: Lazy-loads scan badges per manifest via HTMX - Tests: ~590 lines of test coverage for both handlers 3. S3 Diagnostic Tool New cmd/s3-test/main.go (418 lines) — tests S3 connectivity with both SDK v1 and v2, including presigned URL generation, pull zone host swapping, and verbose signing debug output. 4. Deployment Tooling - New syncServiceUnit() for comparing/updating systemd units on servers - Update command now syncs config keys (adds missing keys from template) and service units with daemon-reload 5. DB Migration 0011_fix_captain_successor_column.yaml — rebuilds hold_captain_records to add the successor column that was missed in a previous migration. 6. Documentation - APPVIEW-UI-FUTURE.md rewritten as a status-tracked feature inventory - DISTRIBUTION.md renamed to CREDENTIAL_HELPER.md - New REMOVING_DISTRIBUTION.md — 480-line analysis of fully removing distribution from the appview side 7. go.mod aws-sdk-go v1 moved from indirect to direct (needed by cmd/s3-test).
289 lines
11 KiB
Go
289 lines
11 KiB
Go
// Package hold implements the ATCR hold service, which provides BYOS
|
|
// (Bring Your Own Storage) functionality. It includes an embedded PDS for
|
|
// storing captain and crew records, generates presigned URLs for blob storage,
|
|
// and handles authorization based on crew membership. Configuration is loaded
|
|
// via Viper with YAML file support and environment variable overrides.
|
|
package hold
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log/slog"
|
|
"path/filepath"
|
|
"time"
|
|
|
|
"github.com/spf13/viper"
|
|
|
|
"atcr.io/pkg/config"
|
|
"atcr.io/pkg/hold/gc"
|
|
"atcr.io/pkg/hold/quota"
|
|
)
|
|
|
|
// Config represents the hold service configuration
|
|
type Config struct {
|
|
Version string `yaml:"version" comment:"Configuration format version."`
|
|
LogLevel string `yaml:"log_level" comment:"Log level: debug, info, warn, error."`
|
|
LogShipper config.LogShipperConfig `yaml:"log_shipper" comment:"Remote log shipping settings."`
|
|
Storage StorageConfig `yaml:"storage" comment:"S3-compatible blob storage settings."`
|
|
Server ServerConfig `yaml:"server" comment:"HTTP server and identity settings."`
|
|
Registration RegistrationConfig `yaml:"registration" comment:"Auto-registration and bootstrap settings."`
|
|
Database DatabaseConfig `yaml:"database" comment:"Embedded PDS database settings."`
|
|
Admin AdminConfig `yaml:"admin" comment:"Admin panel settings."`
|
|
GC gc.Config `yaml:"gc" comment:"Garbage collection settings."`
|
|
Quota quota.Config `yaml:"quota" comment:"Storage quota tiers. Empty disables quota enforcement."`
|
|
Scanner ScannerConfig `yaml:"scanner" comment:"Vulnerability scanner settings. Empty disables scanning."`
|
|
configPath string `yaml:"-"` // internal: path to YAML file for subsystem config loading
|
|
}
|
|
|
|
// ConfigPath returns the path to the YAML configuration file used to load this config.
|
|
// Subsystems (e.g. billing) use this to re-read the same file for extended fields.
|
|
func (c *Config) ConfigPath() string { return c.configPath }
|
|
|
|
// AdminConfig defines admin panel settings
|
|
type AdminConfig struct {
|
|
// Enable the web-based admin panel.
|
|
Enabled bool `yaml:"enabled" comment:"Enable the web-based admin panel for crew and storage management."`
|
|
}
|
|
|
|
// RegistrationConfig defines auto-registration settings
|
|
type RegistrationConfig struct {
|
|
// DID of the hold captain.
|
|
OwnerDID string `yaml:"owner_did" comment:"DID of the hold captain. If set, auto-creates captain and profile records on startup."`
|
|
|
|
// Allow any authenticated user to join as crew.
|
|
AllowAllCrew bool `yaml:"allow_all_crew" comment:"Create a wildcard crew record allowing any authenticated user to join."`
|
|
|
|
// URL to fetch avatar image from during bootstrap.
|
|
ProfileAvatarURL string `yaml:"profile_avatar_url" comment:"URL to fetch avatar image from during bootstrap."`
|
|
|
|
// Post to Bluesky when users push images.
|
|
EnableBlueskyPosts bool `yaml:"enable_bluesky_posts" comment:"Post to Bluesky when users push images. Synced to captain record on startup."`
|
|
|
|
// Deployment region, auto-detected from cloud metadata or S3 config.
|
|
Region string `yaml:"region" comment:"Deployment region, auto-detected from cloud metadata or S3 config."`
|
|
}
|
|
|
|
// StorageConfig holds S3 storage credentials.
|
|
type StorageConfig struct {
|
|
// S3-compatible access key.
|
|
AccessKey string `yaml:"access_key" comment:"S3-compatible access key (AWS, Storj, Minio, UpCloud)."`
|
|
|
|
// S3-compatible secret key.
|
|
SecretKey string `yaml:"secret_key" comment:"S3-compatible secret key."`
|
|
|
|
// S3 region.
|
|
Region string `yaml:"region" comment:"S3 region, e.g. \"us-east-1\". Used for request signing."`
|
|
|
|
// S3 bucket name.
|
|
Bucket string `yaml:"bucket" comment:"S3 bucket for blob storage (REQUIRED). Must already exist."`
|
|
|
|
// Custom S3 endpoint for non-AWS providers.
|
|
Endpoint string `yaml:"endpoint" comment:"Custom S3 endpoint for non-AWS providers (e.g. \"https://gateway.storjshare.io\")."`
|
|
|
|
// CDN pull zone URL for presigned download URLs.
|
|
PullZone string `yaml:"pull_zone" comment:"CDN pull zone URL for downloads. When set, presigned GET/HEAD URLs use this host instead of the S3 endpoint. Uploads and API calls still use the S3 endpoint."`
|
|
}
|
|
|
|
// S3Params returns a params map suitable for s3.NewS3Service.
|
|
func (s StorageConfig) S3Params() map[string]any {
|
|
params := map[string]any{
|
|
"accesskey": s.AccessKey,
|
|
"secretkey": s.SecretKey,
|
|
"region": s.Region,
|
|
"bucket": s.Bucket,
|
|
}
|
|
if s.Endpoint != "" {
|
|
params["regionendpoint"] = s.Endpoint
|
|
params["forcepathstyle"] = true
|
|
}
|
|
if s.PullZone != "" {
|
|
params["pullzone"] = s.PullZone
|
|
}
|
|
return params
|
|
}
|
|
|
|
// ServerConfig defines server settings
|
|
type ServerConfig struct {
|
|
// Listen address for the HTTP server.
|
|
Addr string `yaml:"addr" comment:"Listen address, e.g. \":8080\" or \"0.0.0.0:8080\"."`
|
|
|
|
// Externally reachable URL used for did:web identity.
|
|
PublicURL string `yaml:"public_url" comment:"Externally reachable URL used for did:web identity (REQUIRED), e.g. \"https://hold.example.com\"."`
|
|
|
|
// Allow unauthenticated blob reads.
|
|
Public bool `yaml:"public" comment:"Allow unauthenticated blob reads. If false, readers need crew membership."`
|
|
|
|
// DID of successor hold for migration.
|
|
Successor string `yaml:"successor" comment:"DID of successor hold for migration. Appview redirects all requests to the successor."`
|
|
|
|
// Use localhost for OAuth redirects during development.
|
|
TestMode bool `yaml:"test_mode" comment:"Use localhost for OAuth redirects during development."`
|
|
|
|
// Request crawl from this relay on startup.
|
|
RelayEndpoint string `yaml:"relay_endpoint" comment:"Request crawl from this relay on startup to make the embedded PDS discoverable."`
|
|
|
|
// ReadTimeout for HTTP requests.
|
|
ReadTimeout time.Duration `yaml:"read_timeout" comment:"Read timeout for HTTP requests."`
|
|
|
|
// WriteTimeout for HTTP requests.
|
|
WriteTimeout time.Duration `yaml:"write_timeout" comment:"Write timeout for HTTP requests."`
|
|
}
|
|
|
|
// ScannerConfig defines vulnerability scanner settings
|
|
type ScannerConfig struct {
|
|
// Shared secret for scanner WebSocket authentication. Empty disables scanning.
|
|
Secret string `yaml:"secret" comment:"Shared secret for scanner WebSocket auth. Empty disables scanning."`
|
|
}
|
|
|
|
// DatabaseConfig defines embedded PDS database settings
|
|
type DatabaseConfig struct {
|
|
// Directory for the embedded PDS database.
|
|
Path string `yaml:"path" comment:"Directory for the embedded PDS database (carstore + SQLite)."`
|
|
|
|
// PDS signing key path.
|
|
KeyPath string `yaml:"key_path" comment:"PDS signing key path. Defaults to {database.path}/signing.key."`
|
|
|
|
// libSQL sync URL for embedded replica mode.
|
|
LibsqlSyncURL string `yaml:"libsql_sync_url" comment:"libSQL sync URL (libsql://...). Works with Turso cloud, Bunny DB, or self-hosted libsql-server. Leave empty for local-only SQLite."`
|
|
|
|
// Auth token for libSQL sync.
|
|
LibsqlAuthToken string `yaml:"libsql_auth_token" comment:"Auth token for libSQL sync. Required if libsql_sync_url is set."`
|
|
|
|
// How often to sync with remote libSQL server.
|
|
LibsqlSyncInterval time.Duration `yaml:"libsql_sync_interval" comment:"How often to sync with remote libSQL server. Default: 60s."`
|
|
}
|
|
|
|
// setHoldDefaults registers all default values on the given Viper instance.
|
|
func setHoldDefaults(v *viper.Viper) {
|
|
v.SetDefault("version", "0.1")
|
|
v.SetDefault("log_level", "info")
|
|
|
|
// Server defaults
|
|
v.SetDefault("server.addr", ":8080")
|
|
v.SetDefault("server.public_url", "")
|
|
v.SetDefault("server.public", false)
|
|
v.SetDefault("server.successor", "")
|
|
v.SetDefault("server.test_mode", false)
|
|
v.SetDefault("server.relay_endpoint", "")
|
|
v.SetDefault("server.read_timeout", "5m")
|
|
v.SetDefault("server.write_timeout", "5m")
|
|
|
|
// Registration defaults
|
|
v.SetDefault("registration.owner_did", "")
|
|
v.SetDefault("registration.allow_all_crew", false)
|
|
v.SetDefault("registration.profile_avatar_url", "https://atcr.io/web-app-manifest-192x192.png")
|
|
v.SetDefault("registration.enable_bluesky_posts", false)
|
|
|
|
// Database defaults
|
|
v.SetDefault("database.path", "/var/lib/atcr-hold")
|
|
v.SetDefault("database.key_path", "")
|
|
v.SetDefault("database.libsql_sync_url", "")
|
|
v.SetDefault("database.libsql_auth_token", "")
|
|
v.SetDefault("database.libsql_sync_interval", "60s")
|
|
|
|
// Admin defaults
|
|
v.SetDefault("admin.enabled", true)
|
|
|
|
// Storage defaults
|
|
v.SetDefault("storage.access_key", "")
|
|
v.SetDefault("storage.secret_key", "")
|
|
v.SetDefault("storage.region", "us-east-1")
|
|
v.SetDefault("storage.bucket", "")
|
|
v.SetDefault("storage.endpoint", "")
|
|
v.SetDefault("storage.pull_zone", "")
|
|
|
|
// GC defaults
|
|
v.SetDefault("gc.enabled", false)
|
|
// Scanner defaults
|
|
v.SetDefault("scanner.secret", "")
|
|
|
|
// Log shipper defaults
|
|
v.SetDefault("log_shipper.batch_size", 100)
|
|
v.SetDefault("log_shipper.flush_interval", "5s")
|
|
}
|
|
|
|
// DefaultConfig returns a Config populated with all default values (no validation).
|
|
func DefaultConfig() *Config {
|
|
v := config.NewViper("HOLD", "")
|
|
setHoldDefaults(v)
|
|
|
|
cfg := &Config{}
|
|
_ = v.Unmarshal(cfg, config.UnmarshalOption())
|
|
return cfg
|
|
}
|
|
|
|
// ExampleYAML returns a fully-commented YAML configuration with default values.
|
|
// Includes example quota tiers for documentation (defaults have quotas disabled).
|
|
func ExampleYAML() ([]byte, error) {
|
|
cfg := DefaultConfig()
|
|
|
|
// Populate example quota tiers so operators see the structure
|
|
cfg.Quota = quota.Config{
|
|
Tiers: map[string]quota.TierConfig{
|
|
"deckhand": {Quota: "5GB"},
|
|
"bosun": {Quota: "50GB"},
|
|
"quartermaster": {Quota: "100GB"},
|
|
},
|
|
Defaults: quota.DefaultsConfig{
|
|
NewCrewTier: "deckhand",
|
|
},
|
|
}
|
|
|
|
return config.MarshalCommentedYAML("ATCR Hold Service Configuration", cfg)
|
|
}
|
|
|
|
// LoadConfig builds a complete configuration using Viper layered loading:
|
|
// defaults -> YAML file -> environment variables.
|
|
// yamlPath is optional; empty string means env-only (backward compatible).
|
|
func LoadConfig(yamlPath string) (*Config, error) {
|
|
v := config.NewViper("HOLD", yamlPath)
|
|
|
|
// Set defaults
|
|
setHoldDefaults(v)
|
|
|
|
// Bind standard AWS/S3 env vars to storage fields
|
|
_ = v.BindEnv("storage.access_key", "AWS_ACCESS_KEY_ID")
|
|
_ = v.BindEnv("storage.secret_key", "AWS_SECRET_ACCESS_KEY")
|
|
_ = v.BindEnv("storage.region", "AWS_REGION")
|
|
_ = v.BindEnv("storage.bucket", "S3_BUCKET")
|
|
_ = v.BindEnv("storage.endpoint", "S3_ENDPOINT")
|
|
_ = v.BindEnv("storage.pull_zone", "S3_PULL_ZONE")
|
|
|
|
// Bind legacy GC env vars (backward compat)
|
|
_ = v.BindEnv("gc.enabled", "GC_ENABLED")
|
|
|
|
// Unmarshal into config struct
|
|
cfg := &Config{}
|
|
if err := v.Unmarshal(cfg, config.UnmarshalOption()); err != nil {
|
|
return nil, fmt.Errorf("failed to unmarshal config: %w", err)
|
|
}
|
|
|
|
// Validation
|
|
if cfg.Server.PublicURL == "" {
|
|
return nil, fmt.Errorf("server.public_url is required (env: HOLD_SERVER_PUBLIC_URL)")
|
|
}
|
|
|
|
if cfg.Storage.Bucket == "" {
|
|
return nil, fmt.Errorf("storage.bucket is required (env: S3_BUCKET) - S3 is the only supported storage backend")
|
|
}
|
|
|
|
// Post-load: derive key path from database path if not set
|
|
if cfg.Database.KeyPath == "" && cfg.Database.Path != "" {
|
|
cfg.Database.KeyPath = filepath.Join(cfg.Database.Path, "signing.key")
|
|
}
|
|
|
|
// Store config path for subsystem config loading (e.g. billing)
|
|
cfg.configPath = yamlPath
|
|
|
|
// Detect region from cloud metadata or S3 config
|
|
if meta, err := DetectCloudMetadata(context.Background()); err == nil && meta != nil {
|
|
cfg.Registration.Region = meta.Region
|
|
slog.Info("Detected cloud metadata", "region", meta.Region)
|
|
} else {
|
|
cfg.Registration.Region = cfg.Storage.Region
|
|
slog.Info("Using S3 region", "region", cfg.Registration.Region)
|
|
}
|
|
|
|
return cfg, nil
|
|
}
|