From ed2d853cbe06f4af7fb07f394cdb0c27671691d6 Mon Sep 17 00:00:00 2001 From: Catherine Date: Sat, 6 Dec 2025 00:09:10 +0000 Subject: [PATCH] Add `EnumerateManifests` API and `-list-manifests` option. The new API replaces the `ListManifests` API. This also adds `Name` and `Size` to manifest metadata. --- src/backend.go | 7 ++++-- src/backend_fs.go | 47 +++++++++++++++++++++++------------ src/backend_s3.go | 63 ++++++++++++++++++++++++++--------------------- src/main.go | 17 ++++++++++++- src/migrate.go | 11 ++++++--- src/observe.go | 19 ++++++++------ 6 files changed, 106 insertions(+), 58 deletions(-) diff --git a/src/backend.go b/src/backend.go index 6e90f6e..280a54f 100644 --- a/src/backend.go +++ b/src/backend.go @@ -46,6 +46,8 @@ type GetManifestOptions struct { } type ManifestMetadata struct { + Name string + Size int64 LastModified time.Time ETag string } @@ -122,8 +124,9 @@ type Backend interface { // Delete a manifest. DeleteManifest(ctx context.Context, name string, opts ModifyManifestOptions) error - // List all manifests. - ListManifests(ctx context.Context) (manifests []string, err error) + // Iterate through all manifests. Whether manifests that are newly added during iteration + // will appear in the results is unspecified. + EnumerateManifests(ctx context.Context) iter.Seq2[ManifestMetadata, error] // Check whether a domain has any deployments. CheckDomain(ctx context.Context, domain string) (found bool, err error) diff --git a/src/backend_fs.go b/src/backend_fs.go index 6d0e01b..965cacf 100644 --- a/src/backend_fs.go +++ b/src/backend_fs.go @@ -207,22 +207,6 @@ func (fs *FSBackend) EnumerateBlobs(ctx context.Context) iter.Seq2[BlobMetadata, } } -func (fs *FSBackend) ListManifests(ctx context.Context) (manifests []string, err error) { - err = iofs.WalkDir(fs.siteRoot.FS(), ".", - func(path string, entry iofs.DirEntry, err error) error { - if strings.Count(path, "/") > 1 { - return iofs.SkipDir - } - _, project, _ := strings.Cut(path, "/") - if project == "" || strings.HasPrefix(project, ".") && project != ".index" { - return nil - } - manifests = append(manifests, path) - return nil - }) - return -} - func (fs *FSBackend) GetManifest( ctx context.Context, name string, opts GetManifestOptions, ) ( @@ -412,6 +396,37 @@ func (fs *FSBackend) DeleteManifest( } } +func (fs *FSBackend) EnumerateManifests(ctx context.Context) iter.Seq2[ManifestMetadata, error] { + return func(yield func(ManifestMetadata, error) bool) { + iofs.WalkDir(fs.siteRoot.FS(), ".", + func(path string, entry iofs.DirEntry, err error) error { + _, project, _ := strings.Cut(path, "/") + var metadata ManifestMetadata + if err != nil { + // report error + } else if entry.IsDir() { + // skip directory + return nil + } else if project == "" || strings.HasPrefix(project, ".") && project != ".index" { + // skip internal + return nil + } else if info, err := entry.Info(); err != nil { + // report error + } else { + // report blob + metadata.Name = path + metadata.Size = info.Size() + metadata.LastModified = info.ModTime() + // not setting metadata.ETag since it is too costly + } + if !yield(metadata, err) { + return iofs.SkipAll + } + return nil + }) + } +} + func (fs *FSBackend) CheckDomain(ctx context.Context, domain string) (bool, error) { _, err := fs.siteRoot.Stat(domain) if errors.Is(err, os.ErrNotExist) { diff --git a/src/backend_s3.go b/src/backend_s3.go index aa99460..d5f209a 100644 --- a/src/backend_s3.go +++ b/src/backend_s3.go @@ -397,34 +397,6 @@ func stagedManifestObjectName(manifestData []byte) string { return fmt.Sprintf("dirty/%x", sha256.Sum256(manifestData)) } -func (s3 *S3Backend) ListManifests(ctx context.Context) (manifests []string, err error) { - logc.Print(ctx, "s3: list manifests") - - ctx, cancel := context.WithCancel(ctx) - defer cancel() - - prefix := manifestObjectName("") - for object := range s3.client.ListObjectsIter(ctx, s3.bucket, minio.ListObjectsOptions{ - Prefix: prefix, - Recursive: true, - }) { - if object.Err != nil { - return nil, object.Err - } - key := strings.TrimRight(strings.TrimPrefix(object.Key, prefix), "/") - if strings.Count(key, "/") > 1 { - continue - } - _, project, _ := strings.Cut(key, "/") - if project == "" || strings.HasPrefix(project, ".") && project != ".index" { - continue - } - manifests = append(manifests, key) - } - - return -} - type s3ManifestLoader struct { s3 *S3Backend } @@ -668,6 +640,41 @@ func (s3 *S3Backend) DeleteManifest( return err } +func (s3 *S3Backend) EnumerateManifests(ctx context.Context) iter.Seq2[ManifestMetadata, error] { + return func(yield func(ManifestMetadata, error) bool) { + logc.Print(ctx, "s3: enumerate manifests") + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + prefix := "site/" + for object := range s3.client.ListObjectsIter(ctx, s3.bucket, minio.ListObjectsOptions{ + Prefix: prefix, + Recursive: true, + }) { + var metadata ManifestMetadata + var err error + if err = object.Err; err == nil { + key := strings.TrimPrefix(object.Key, prefix) + _, project, _ := strings.Cut(key, "/") + if strings.HasSuffix(key, "/") { + continue // directory; skip + } else if project == "" || strings.HasPrefix(project, ".") && project != ".index" { + continue // internal; skip + } else { + metadata.Name = key + metadata.Size = object.Size + metadata.LastModified = object.LastModified + metadata.ETag = object.ETag + } + } + if !yield(metadata, err) { + break + } + } + } +} + func domainCheckObjectName(domain string) string { return manifestObjectName(fmt.Sprintf("%s/.exists", domain)) } diff --git a/src/main.go b/src/main.go index ea302c6..c0cf511 100644 --- a/src/main.go +++ b/src/main.go @@ -171,7 +171,7 @@ func usage() { fmt.Fprintf(os.Stderr, "(server) "+ "git-pages [-config |-no-config]\n") fmt.Fprintf(os.Stderr, "(debug) "+ - "git-pages {-list-blobs}\n") + "git-pages {-list-blobs|-list-manifests}\n") fmt.Fprintf(os.Stderr, "(debug) "+ "git-pages {-get-blob|-get-manifest|-get-archive|-update-site} [file]\n") fmt.Fprintf(os.Stderr, "(admin) "+ @@ -203,6 +203,8 @@ func Main() { "enumerate every blob with its metadata") getManifest := flag.String("get-manifest", "", "write manifest for `site` (either 'domain.tld' or 'domain.tld/dir') as ProtoJSON") + listManifests := flag.Bool("list-manifests", false, + "enumerate every manifest with its metadata") getArchive := flag.String("get-archive", "", "write archive for `site` (either 'domain.tld' or 'domain.tld/dir') in tar format") updateSite := flag.String("update-site", "", @@ -225,6 +227,7 @@ func Main() { *getBlob != "", *listBlobs, *getManifest != "", + *listManifests, *getArchive != "", *updateSite != "", *freezeDomain != "", @@ -317,6 +320,18 @@ func Main() { } fmt.Fprintln(fileOutputArg(), string(ManifestJSON(manifest))) + case *listManifests: + for metadata, err := range backend.EnumerateManifests(ctx) { + if err != nil { + logc.Fatalln(ctx, err) + } + fmt.Fprintf(color.Output, "%s %s %s\n", + metadata.Name, + color.HiWhiteString(metadata.LastModified.UTC().Format(time.RFC3339)), + color.HiGreenString(fmt.Sprint(metadata.Size)), + ) + } + case *getArchive != "": webRoot := webRootArg(*getArchive) manifest, metadata, err := diff --git a/src/migrate.go b/src/migrate.go index 812acb6..d3ef153 100644 --- a/src/migrate.go +++ b/src/migrate.go @@ -22,12 +22,15 @@ func createDomainMarkers(ctx context.Context) error { return nil } - var manifests, domains []string - manifests, err := backend.ListManifests(ctx) - if err != nil { - return fmt.Errorf("list manifests: %w", err) + var manifests []string + for metadata, err := range backend.EnumerateManifests(ctx) { + if err != nil { + return fmt.Errorf("enum manifests: %w", err) + } + manifests = append(manifests, metadata.Name) } slices.Sort(manifests) + var domains []string for _, manifest := range manifests { domain, _, _ := strings.Cut(manifest, "/") if len(domains) == 0 || domains[len(domains)-1] != domain { diff --git a/src/observe.go b/src/observe.go index e8cfddb..c8b0d43 100644 --- a/src/observe.go +++ b/src/observe.go @@ -385,13 +385,6 @@ func (backend *observedBackend) EnumerateBlobs(ctx context.Context) iter.Seq2[Bl } } -func (backend *observedBackend) ListManifests(ctx context.Context) (manifests []string, err error) { - span, ctx := ObserveFunction(ctx, "ListManifests") - manifests, err = backend.inner.ListManifests(ctx) - span.Finish() - return -} - func (backend *observedBackend) GetManifest( ctx context.Context, name string, opts GetManifestOptions, ) ( @@ -433,6 +426,18 @@ func (backend *observedBackend) DeleteManifest(ctx context.Context, name string, return } +func (backend *observedBackend) EnumerateManifests(ctx context.Context) iter.Seq2[ManifestMetadata, error] { + return func(yield func(ManifestMetadata, error) bool) { + span, ctx := ObserveFunction(ctx, "EnumerateManifests") + for metadata, err := range backend.inner.EnumerateManifests(ctx) { + if !yield(metadata, err) { + break + } + } + span.Finish() + } +} + func (backend *observedBackend) CheckDomain(ctx context.Context, domain string) (found bool, err error) { span, ctx := ObserveFunction(ctx, "CheckDomain", "domain.name", domain) found, err = backend.inner.CheckDomain(ctx, domain)