From aa965c5a0857a7c969468b9f2fcc5c402b7ae5d1 Mon Sep 17 00:00:00 2001 From: miyuko Date: Wed, 15 Oct 2025 19:12:19 +0100 Subject: [PATCH] Use `s3:GetObject` instead of `s3:ListObjects` for `CheckDomain`. --- src/backend.go | 18 +++++++ src/backend_fs.go | 39 ++++++++++++++ src/backend_s3.go | 131 ++++++++++++++++++++++++++++++++++++++++------ src/main.go | 11 ++++ src/migrate.go | 49 +++++++++++++++++ src/observe.go | 28 ++++++++++ src/update.go | 5 ++ 7 files changed, 265 insertions(+), 16 deletions(-) create mode 100644 src/migrate.go diff --git a/src/backend.go b/src/backend.go index a62d841..038a68e 100644 --- a/src/backend.go +++ b/src/backend.go @@ -21,11 +21,23 @@ func splitBlobName(name string) []string { } } +type BackendFeature string + +const ( + FeatureCheckDomainMarker BackendFeature = "check-domain-marker" +) + type GetManifestOptions struct { BypassCache bool } type Backend interface { + // Returns true if the feature has been enabled for this store, false otherwise. + HasFeature(ctx context.Context, feature BackendFeature) bool + + // Enables the feature for this store. + EnableFeature(ctx context.Context, feature BackendFeature) error + // Retrieve a blob. Returns `reader, size, mtime, err`. GetBlob(ctx context.Context, name string) (reader io.ReadSeeker, size uint64, mtime time.Time, err error) @@ -52,8 +64,14 @@ type Backend interface { // Delete a manifest. DeleteManifest(ctx context.Context, name string) error + // List all manifests. + ListManifests(ctx context.Context) (manifests []string, err error) + // Check whether a domain has any deployments. CheckDomain(ctx context.Context, domain string) (found bool, err error) + + // Creates a domain. This allows us to start serving content for the domain. + CreateDomain(ctx context.Context, domain string) error } var backend Backend diff --git a/src/backend_fs.go b/src/backend_fs.go index 3d94219..ebabd99 100644 --- a/src/backend_fs.go +++ b/src/backend_fs.go @@ -6,8 +6,10 @@ import ( "errors" "fmt" "io" + "io/fs" "os" "path/filepath" + "strings" "time" ) @@ -68,6 +70,24 @@ func (fs *FSBackend) Backend() Backend { return fs } +func (fs *FSBackend) HasFeature(ctx context.Context, feature BackendFeature) bool { + switch feature { + case FeatureCheckDomainMarker: + return true + default: + return false + } +} + +func (fs *FSBackend) EnableFeature(ctx context.Context, feature BackendFeature) error { + switch feature { + case FeatureCheckDomainMarker: + return nil + default: + return fmt.Errorf("not implemented") + } +} + func (fs *FSBackend) GetBlob( ctx context.Context, name string, @@ -133,6 +153,21 @@ func (fs *FSBackend) DeleteBlob(ctx context.Context, name string) error { return fs.blobRoot.Remove(blobPath) } +func (b *FSBackend) ListManifests(ctx context.Context) (manifests []string, err error) { + err = fs.WalkDir(b.siteRoot.FS(), ".", func(path string, d fs.DirEntry, err error) error { + if strings.Count(path, "/") > 1 { + return fs.SkipDir + } + _, project, _ := strings.Cut(path, "/") + if project == "" || strings.HasPrefix(project, ".") && project != ".index" { + return nil + } + manifests = append(manifests, path) + return nil + }) + return +} + func (fs *FSBackend) GetManifest(ctx context.Context, name string, opts GetManifestOptions) (*Manifest, error) { data, err := fs.siteRoot.ReadFile(name) if errors.Is(err, os.ErrNotExist) { @@ -201,3 +236,7 @@ func (fs *FSBackend) CheckDomain(ctx context.Context, domain string) (bool, erro return false, err } } + +func (fs *FSBackend) CreateDomain(ctx context.Context, domain string) error { + return nil // no-op +} diff --git a/src/backend_s3.go b/src/backend_s3.go index 04eb1c5..4c91593 100644 --- a/src/backend_s3.go +++ b/src/backend_s3.go @@ -9,6 +9,7 @@ import ( "log" "net/http" "path" + "strings" "time" "github.com/c2h5oh/datasize" @@ -118,10 +119,11 @@ type CachedManifest struct { func (c *CachedManifest) Weight() uint32 { return c.weight } type S3Backend struct { - client *minio.Client - bucket string - blobCache *observedCache[string, *CachedBlob] - siteCache *observedCache[string, *CachedManifest] + client *minio.Client + bucket string + blobCache *observedCache[string, *CachedBlob] + siteCache *observedCache[string, *CachedManifest] + featureCache *otter.Cache[BackendFeature, bool] } var _ Backend = (*S3Backend)(nil) @@ -200,7 +202,14 @@ func NewS3Backend(ctx context.Context, config *S3Config) (*S3Backend, error) { return nil, err } - return &S3Backend{client, bucket, blobCache, siteCache}, nil + featureCache, err := otter.New(&otter.Options[BackendFeature, bool]{ + RefreshCalculator: otter.RefreshWriting[BackendFeature, bool](10 * time.Minute), + }) + if err != nil { + return nil, err + } + + return &S3Backend{client, bucket, blobCache, siteCache, featureCache}, nil } func (s3 *S3Backend) Backend() Backend { @@ -211,6 +220,42 @@ func blobObjectName(name string) string { return fmt.Sprintf("blob/%s", path.Join(splitBlobName(name)...)) } +func storeFeatureObjectName(feature BackendFeature) string { + return fmt.Sprintf("meta/feature/%s", feature) +} + +func (s3 *S3Backend) HasFeature(ctx context.Context, feature BackendFeature) bool { + loader := func(ctx context.Context, feature BackendFeature) (bool, error) { + _, err := s3.client.StatObject(ctx, s3.bucket, storeFeatureObjectName(feature), + minio.StatObjectOptions{}) + if err != nil { + if errResp := minio.ToErrorResponse(err); errResp.Code == "NoSuchKey" { + log.Printf("s3 feature %q: disabled", feature) + return false, nil + } else { + return false, err + } + } + log.Printf("s3 feature %q: enabled", feature) + return true, nil + } + + isOn, err := s3.featureCache.Get(ctx, feature, otter.LoaderFunc[BackendFeature, bool](loader)) + if err != nil { + err = fmt.Errorf("getting s3 backend feature %q: %w", feature, err) + ObserveError(err) + log.Print(err) + return false + } + return isOn +} + +func (s3 *S3Backend) EnableFeature(ctx context.Context, feature BackendFeature) error { + _, err := s3.client.PutObject(ctx, s3.bucket, storeFeatureObjectName(feature), + &bytes.Reader{}, 0, minio.PutObjectOptions{}) + return err +} + func (s3 *S3Backend) GetBlob( ctx context.Context, name string, @@ -307,6 +352,34 @@ func stagedManifestObjectName(manifestData []byte) string { return fmt.Sprintf("dirty/%x", sha256.Sum256(manifestData)) } +func (s3 *S3Backend) ListManifests(ctx context.Context) (manifests []string, err error) { + log.Print("s3: list manifests") + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + prefix := manifestObjectName("") + for object := range s3.client.ListObjectsIter(ctx, s3.bucket, minio.ListObjectsOptions{ + Prefix: prefix, + Recursive: true, + }) { + if object.Err != nil { + return nil, object.Err + } + key := strings.TrimRight(strings.TrimPrefix(object.Key, prefix), "/") + if strings.Count(key, "/") > 1 { + continue + } + _, project, _ := strings.Cut(key, "/") + if project == "" || strings.HasPrefix(project, ".") && project != ".index" { + continue + } + manifests = append(manifests, key) + } + + return +} + type s3ManifestLoader struct { s3 *S3Backend } @@ -426,19 +499,45 @@ func (s3 *S3Backend) DeleteManifest(ctx context.Context, name string) error { return err } -func (s3 *S3Backend) CheckDomain(ctx context.Context, domain string) (bool, error) { +func domainCheckObjectName(domain string) string { + return manifestObjectName(fmt.Sprintf("%s/.exists", domain)) +} + +func (s3 *S3Backend) CheckDomain(ctx context.Context, domain string) (exists bool, err error) { log.Printf("s3: check domain %s\n", domain) - ctx, cancel := context.WithCancel(ctx) - defer cancel() - - for object := range s3.client.ListObjectsIter(ctx, s3.bucket, minio.ListObjectsOptions{ - Prefix: manifestObjectName(fmt.Sprintf("%s/", domain)), - }) { - if object.Err != nil { - return false, object.Err + _, err = s3.client.StatObject(ctx, s3.bucket, domainCheckObjectName(domain), + minio.StatObjectOptions{}) + if err != nil { + if errResp := minio.ToErrorResponse(err); errResp.Code == "NoSuchKey" { + exists, err = false, nil } - return true, nil + } else { + exists = true } - return false, nil + + if !exists && !s3.HasFeature(ctx, FeatureCheckDomainMarker) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + for object := range s3.client.ListObjectsIter(ctx, s3.bucket, minio.ListObjectsOptions{ + Prefix: manifestObjectName(fmt.Sprintf("%s/", domain)), + }) { + if object.Err != nil { + return false, object.Err + } + return true, nil + } + return false, nil + } + + return +} + +func (s3 *S3Backend) CreateDomain(ctx context.Context, domain string) error { + log.Printf("s3: create domain %s\n", domain) + + _, err := s3.client.PutObject(ctx, s3.bucket, domainCheckObjectName(domain), + &bytes.Reader{}, 0, minio.PutObjectOptions{}) + return err } diff --git a/src/main.go b/src/main.go index 596bbd1..7efa3f0 100644 --- a/src/main.go +++ b/src/main.go @@ -78,6 +78,8 @@ func Main() { "load configuration from `filename`") noConfig := flag.Bool("no-config", false, "run without configuration file (configure via environment variables)") + runMigration := flag.String("run-migration", "", + "run a specific store migration (available: \"create-domain-markers\")") getManifest := flag.String("get-manifest", "", "write manifest for `webroot` (either 'domain.tld' or 'domain.tld/dir') to stdout as ProtoJSON") getBlob := flag.String("get-blob", "", @@ -139,6 +141,15 @@ func Main() { } switch { + case *runMigration != "": + if err := ConfigureBackend(&config.Storage); err != nil { + log.Fatalln(err) + } + + if err := RunMigration(context.Background(), *runMigration); err != nil { + log.Fatalln(err) + } + case *getManifest != "": if err := ConfigureBackend(&config.Storage); err != nil { log.Fatalln(err) diff --git a/src/migrate.go b/src/migrate.go new file mode 100644 index 0000000..27374e2 --- /dev/null +++ b/src/migrate.go @@ -0,0 +1,49 @@ +package git_pages + +import ( + "context" + "fmt" + "log" + "slices" + "strings" +) + +func RunMigration(ctx context.Context, name string) error { + switch name { + case "create-domain-markers": + return createDomainMarkers(ctx) + default: + return fmt.Errorf("unknown migration name (expected one of \"create-domain-markers\")") + } +} + +func createDomainMarkers(ctx context.Context) error { + if backend.HasFeature(ctx, FeatureCheckDomainMarker) { + log.Print("store already has domain markers") + return nil + } + + var manifests, domains []string + manifests, err := backend.ListManifests(ctx) + if err != nil { + return fmt.Errorf("list manifests: %w", err) + } + slices.Sort(manifests) + for _, manifest := range manifests { + domain, _, _ := strings.Cut(manifest, "/") + if len(domains) == 0 || domains[len(domains)-1] != domain { + domains = append(domains, domain) + } + } + for idx, domain := range domains { + log.Printf("(%d / %d) creating domain %s", idx+1, len(domains), domain) + if err := backend.CreateDomain(ctx, domain); err != nil { + return fmt.Errorf("creating domain %s: %w", domain, err) + } + } + if err := backend.EnableFeature(ctx, FeatureCheckDomainMarker); err != nil { + return err + } + log.Printf("created markers for %d domains", len(domains)) + return nil +} diff --git a/src/observe.go b/src/observe.go index 4904f50..3c1b233 100644 --- a/src/observe.go +++ b/src/observe.go @@ -266,6 +266,20 @@ func NewObservedBackend(backend Backend) Backend { return &observedBackend{inner: backend} } +func (backend *observedBackend) HasFeature(ctx context.Context, feature BackendFeature) (isOn bool) { + span, ctx := ObserveFunction(ctx, "HasFeature") + isOn = backend.inner.HasFeature(ctx, feature) + span.Finish() + return +} + +func (backend *observedBackend) EnableFeature(ctx context.Context, feature BackendFeature) (err error) { + span, ctx := ObserveFunction(ctx, "EnableFeature") + err = backend.inner.EnableFeature(ctx, feature) + span.Finish() + return +} + func (backend *observedBackend) GetBlob( ctx context.Context, name string, @@ -302,6 +316,13 @@ func (backend *observedBackend) DeleteBlob(ctx context.Context, name string) (er return } +func (backend *observedBackend) ListManifests(ctx context.Context) (manifests []string, err error) { + span, ctx := ObserveFunction(ctx, "ListManifests") + manifests, err = backend.inner.ListManifests(ctx) + span.Finish() + return +} + func (backend *observedBackend) GetManifest( ctx context.Context, name string, @@ -345,3 +366,10 @@ func (backend *observedBackend) CheckDomain(ctx context.Context, domain string) span.Finish() return } + +func (backend *observedBackend) CreateDomain(ctx context.Context, domain string) (err error) { + span, ctx := ObserveFunction(ctx, "CreateDomain", "manifest.domain", domain) + err = backend.inner.CreateDomain(ctx, domain) + span.Finish() + return +} diff --git a/src/update.go b/src/update.go index cac76aa..9aa1ae7 100644 --- a/src/update.go +++ b/src/update.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "log" + "strings" ) type UpdateOutcome int @@ -42,6 +43,10 @@ func Update(ctx context.Context, webRoot string, manifest *Manifest) UpdateResul } } else if err = PrepareManifest(ctx, manifest); err == nil { newManifest, err = StoreManifest(ctx, webRoot, manifest) + if err == nil { + domain, _, _ := strings.Cut(webRoot, "/") + err = backend.CreateDomain(ctx, domain) + } if err == nil { if oldManifest == nil { outcome = UpdateCreated