Add EnumerateManifests API and -list-manifests option.

The new API replaces the `ListManifests` API.

This also adds `Name` and `Size` to manifest metadata.

src/backend.go

@@ -46,6 +46,8 @@ type GetManifestOptions struct {
 }
 
 type ManifestMetadata struct {
+	Name         string
+	Size         int64
 	LastModified time.Time
 	ETag         string
 }
@@ -122,8 +124,9 @@ type Backend interface {
 	// Delete a manifest.
 	DeleteManifest(ctx context.Context, name string, opts ModifyManifestOptions) error
 
-	// List all manifests.
-	ListManifests(ctx context.Context) (manifests []string, err error)
+	// Iterate through all manifests. Whether manifests that are newly added during iteration
+	// will appear in the results is unspecified.
+	EnumerateManifests(ctx context.Context) iter.Seq2[ManifestMetadata, error]
 
 	// Check whether a domain has any deployments.
 	CheckDomain(ctx context.Context, domain string) (found bool, err error)

src/backend_fs.go

@@ -207,22 +207,6 @@ func (fs *FSBackend) EnumerateBlobs(ctx context.Context) iter.Seq2[BlobMetadata,
 	}
 }
 
-func (fs *FSBackend) ListManifests(ctx context.Context) (manifests []string, err error) {
-	err = iofs.WalkDir(fs.siteRoot.FS(), ".",
-		func(path string, entry iofs.DirEntry, err error) error {
-			if strings.Count(path, "/") > 1 {
-				return iofs.SkipDir
-			}
-			_, project, _ := strings.Cut(path, "/")
-			if project == "" || strings.HasPrefix(project, ".") && project != ".index" {
-				return nil
-			}
-			manifests = append(manifests, path)
-			return nil
-		})
-	return
-}
-
 func (fs *FSBackend) GetManifest(
 	ctx context.Context, name string, opts GetManifestOptions,
 ) (
@@ -412,6 +396,37 @@ func (fs *FSBackend) DeleteManifest(
 	}
 }
 
+func (fs *FSBackend) EnumerateManifests(ctx context.Context) iter.Seq2[ManifestMetadata, error] {
+	return func(yield func(ManifestMetadata, error) bool) {
+		iofs.WalkDir(fs.siteRoot.FS(), ".",
+			func(path string, entry iofs.DirEntry, err error) error {
+				_, project, _ := strings.Cut(path, "/")
+				var metadata ManifestMetadata
+				if err != nil {
+					// report error
+				} else if entry.IsDir() {
+					// skip directory
+					return nil
+				} else if project == "" || strings.HasPrefix(project, ".") && project != ".index" {
+					// skip internal
+					return nil
+				} else if info, err := entry.Info(); err != nil {
+					// report error
+				} else {
+					// report blob
+					metadata.Name = path
+					metadata.Size = info.Size()
+					metadata.LastModified = info.ModTime()
+					// not setting metadata.ETag since it is too costly
+				}
+				if !yield(metadata, err) {
+					return iofs.SkipAll
+				}
+				return nil
+			})
+	}
+}
+
 func (fs *FSBackend) CheckDomain(ctx context.Context, domain string) (bool, error) {
 	_, err := fs.siteRoot.Stat(domain)
 	if errors.Is(err, os.ErrNotExist) {

src/backend_s3.go

@@ -397,34 +397,6 @@ func stagedManifestObjectName(manifestData []byte) string {
 	return fmt.Sprintf("dirty/%x", sha256.Sum256(manifestData))
 }
 
-func (s3 *S3Backend) ListManifests(ctx context.Context) (manifests []string, err error) {
-	logc.Print(ctx, "s3: list manifests")
-
-	ctx, cancel := context.WithCancel(ctx)
-	defer cancel()
-
-	prefix := manifestObjectName("")
-	for object := range s3.client.ListObjectsIter(ctx, s3.bucket, minio.ListObjectsOptions{
-		Prefix:    prefix,
-		Recursive: true,
-	}) {
-		if object.Err != nil {
-			return nil, object.Err
-		}
-		key := strings.TrimRight(strings.TrimPrefix(object.Key, prefix), "/")
-		if strings.Count(key, "/") > 1 {
-			continue
-		}
-		_, project, _ := strings.Cut(key, "/")
-		if project == "" || strings.HasPrefix(project, ".") && project != ".index" {
-			continue
-		}
-		manifests = append(manifests, key)
-	}
-
-	return
-}
-
 type s3ManifestLoader struct {
 	s3 *S3Backend
 }
@@ -668,6 +640,41 @@ func (s3 *S3Backend) DeleteManifest(
 	return err
 }
 
+func (s3 *S3Backend) EnumerateManifests(ctx context.Context) iter.Seq2[ManifestMetadata, error] {
+	return func(yield func(ManifestMetadata, error) bool) {
+		logc.Print(ctx, "s3: enumerate manifests")
+
+		ctx, cancel := context.WithCancel(ctx)
+		defer cancel()
+
+		prefix := "site/"
+		for object := range s3.client.ListObjectsIter(ctx, s3.bucket, minio.ListObjectsOptions{
+			Prefix:    prefix,
+			Recursive: true,
+		}) {
+			var metadata ManifestMetadata
+			var err error
+			if err = object.Err; err == nil {
+				key := strings.TrimPrefix(object.Key, prefix)
+				_, project, _ := strings.Cut(key, "/")
+				if strings.HasSuffix(key, "/") {
+					continue // directory; skip
+				} else if project == "" || strings.HasPrefix(project, ".") && project != ".index" {
+					continue // internal; skip
+				} else {
+					metadata.Name = key
+					metadata.Size = object.Size
+					metadata.LastModified = object.LastModified
+					metadata.ETag = object.ETag
+				}
+			}
+			if !yield(metadata, err) {
+				break
+			}
+		}
+	}
+}
+
 func domainCheckObjectName(domain string) string {
 	return manifestObjectName(fmt.Sprintf("%s/.exists", domain))
 }

src/main.go

@@ -171,7 +171,7 @@ func usage() {
 	fmt.Fprintf(os.Stderr, "(server) "+
 		"git-pages [-config <file>|-no-config]\n")
 	fmt.Fprintf(os.Stderr, "(debug)  "+
-		"git-pages {-list-blobs}\n")
+		"git-pages {-list-blobs|-list-manifests}\n")
 	fmt.Fprintf(os.Stderr, "(debug)  "+
 		"git-pages {-get-blob|-get-manifest|-get-archive|-update-site} <ref> [file]\n")
 	fmt.Fprintf(os.Stderr, "(admin)  "+
@@ -203,6 +203,8 @@ func Main() {
 		"enumerate every blob with its metadata")
 	getManifest := flag.String("get-manifest", "",
 		"write manifest for `site` (either 'domain.tld' or 'domain.tld/dir') as ProtoJSON")
+	listManifests := flag.Bool("list-manifests", false,
+		"enumerate every manifest with its metadata")
 	getArchive := flag.String("get-archive", "",
 		"write archive for `site` (either 'domain.tld' or 'domain.tld/dir') in tar format")
 	updateSite := flag.String("update-site", "",
@@ -225,6 +227,7 @@ func Main() {
 		*getBlob != "",
 		*listBlobs,
 		*getManifest != "",
+		*listManifests,
 		*getArchive != "",
 		*updateSite != "",
 		*freezeDomain != "",
@@ -317,6 +320,18 @@ func Main() {
 		}
 		fmt.Fprintln(fileOutputArg(), string(ManifestJSON(manifest)))
 
+	case *listManifests:
+		for metadata, err := range backend.EnumerateManifests(ctx) {
+			if err != nil {
+				logc.Fatalln(ctx, err)
+			}
+			fmt.Fprintf(color.Output, "%s %s %s\n",
+				metadata.Name,
+				color.HiWhiteString(metadata.LastModified.UTC().Format(time.RFC3339)),
+				color.HiGreenString(fmt.Sprint(metadata.Size)),
+			)
+		}
+
 	case *getArchive != "":
 		webRoot := webRootArg(*getArchive)
 		manifest, metadata, err :=

src/migrate.go

@@ -22,12 +22,15 @@ func createDomainMarkers(ctx context.Context) error {
 		return nil
 	}
 
-	var manifests, domains []string
-	manifests, err := backend.ListManifests(ctx)
-	if err != nil {
-		return fmt.Errorf("list manifests: %w", err)
+	var manifests []string
+	for metadata, err := range backend.EnumerateManifests(ctx) {
+		if err != nil {
+			return fmt.Errorf("enum manifests: %w", err)
+		}
+		manifests = append(manifests, metadata.Name)
 	}
 	slices.Sort(manifests)
+	var domains []string
 	for _, manifest := range manifests {
 		domain, _, _ := strings.Cut(manifest, "/")
 		if len(domains) == 0 || domains[len(domains)-1] != domain {

src/observe.go

@@ -385,13 +385,6 @@ func (backend *observedBackend) EnumerateBlobs(ctx context.Context) iter.Seq2[Bl
 	}
 }
 
-func (backend *observedBackend) ListManifests(ctx context.Context) (manifests []string, err error) {
-	span, ctx := ObserveFunction(ctx, "ListManifests")
-	manifests, err = backend.inner.ListManifests(ctx)
-	span.Finish()
-	return
-}
-
 func (backend *observedBackend) GetManifest(
 	ctx context.Context, name string, opts GetManifestOptions,
 ) (
@@ -433,6 +426,18 @@ func (backend *observedBackend) DeleteManifest(ctx context.Context, name string,
 	return
 }
 
+func (backend *observedBackend) EnumerateManifests(ctx context.Context) iter.Seq2[ManifestMetadata, error] {
+	return func(yield func(ManifestMetadata, error) bool) {
+		span, ctx := ObserveFunction(ctx, "EnumerateManifests")
+		for metadata, err := range backend.inner.EnumerateManifests(ctx) {
+			if !yield(metadata, err) {
+				break
+			}
+		}
+		span.Finish()
+	}
+}
+
 func (backend *observedBackend) CheckDomain(ctx context.Context, domain string) (found bool, err error) {
 	span, ctx := ObserveFunction(ctx, "CheckDomain", "domain.name", domain)
 	found, err = backend.inner.CheckDomain(ctx, domain)