Add EnumerateBlobs API and -list-blobs option.

This also adds `Name` to blob metadata.

src/backend.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"io"
 	"iter"
-	"slices"
 	"strings"
 	"time"
 )
@@ -17,14 +16,17 @@ var ErrWriteConflict = errors.New("write conflict")
 var ErrDomainFrozen = errors.New("domain administratively frozen")
 
 func splitBlobName(name string) []string {
-	algo, hash, found := strings.Cut(name, "-")
-	if found {
-		return slices.Concat([]string{algo}, splitBlobName(hash))
+	if algo, hash, found := strings.Cut(name, "-"); found {
+		return []string{algo, hash[0:2], hash[2:4], hash[4:]}
 	} else {
-		return []string{name[0:2], name[2:4], name[4:]}
+		panic("malformed blob name")
 	}
 }
 
+func joinBlobName(parts []string) string {
+	return fmt.Sprintf("%s-%s", parts[0], strings.Join(parts[1:], ""))
+}
+
 type BackendFeature string
 
 const (
@@ -32,6 +34,7 @@ const (
 )
 
 type BlobMetadata struct {
+	Name         string
 	Size         int64
 	LastModified time.Time
 }
@@ -93,6 +96,10 @@ type Backend interface {
 	// Delete a blob. This is an unconditional operation that can break integrity of manifests.
 	DeleteBlob(ctx context.Context, name string) error
 
+	// Iterate through all blobs. Whether blobs that are newly added during iteration will appear
+	// in the results is unspecified.
+	EnumerateBlobs(ctx context.Context) iter.Seq2[BlobMetadata, error]
+
 	// Retrieve a manifest.
 	GetManifest(ctx context.Context, name string, opts GetManifestOptions) (
 		manifest *Manifest, metadata ManifestMetadata, err error,

src/backend_fs.go

@@ -133,7 +133,7 @@ func (fs *FSBackend) GetBlob(
 		err = fmt.Errorf("open: %w", err)
 		return
 	}
-	return file, BlobMetadata{int64(stat.Size()), stat.ModTime()}, nil
+	return file, BlobMetadata{name, int64(stat.Size()), stat.ModTime()}, nil
 }
 
 func (fs *FSBackend) PutBlob(ctx context.Context, name string, data []byte) error {
@@ -181,6 +181,32 @@ func (fs *FSBackend) DeleteBlob(ctx context.Context, name string) error {
 	return fs.blobRoot.Remove(blobPath)
 }
 
+func (fs *FSBackend) EnumerateBlobs(ctx context.Context) iter.Seq2[BlobMetadata, error] {
+	return func(yield func(BlobMetadata, error) bool) {
+		iofs.WalkDir(fs.blobRoot.FS(), ".",
+			func(path string, entry iofs.DirEntry, err error) error {
+				var metadata BlobMetadata
+				if err != nil {
+					// report error
+				} else if entry.IsDir() {
+					// skip directory
+					return nil
+				} else if info, err := entry.Info(); err != nil {
+					// report error
+				} else {
+					// report blob
+					metadata.Name = joinBlobName(strings.Split(path, "/"))
+					metadata.Size = info.Size()
+					metadata.LastModified = info.ModTime()
+				}
+				if !yield(metadata, err) {
+					return iofs.SkipAll
+				}
+				return nil
+			})
+	}
+}
+
 func (fs *FSBackend) ListManifests(ctx context.Context) (manifests []string, err error) {
 	err = iofs.WalkDir(fs.siteRoot.FS(), ".",
 		func(path string, entry iofs.DirEntry, err error) error {

src/backend_s3.go

@@ -316,6 +316,7 @@ func (s3 *S3Backend) GetBlob(
 		}
 	} else {
 		reader = bytes.NewReader(cached.blob)
+		metadata.Name = name
 		metadata.Size = int64(len(cached.blob))
 		metadata.LastModified = cached.mtime
 	}
@@ -357,6 +358,37 @@ func (s3 *S3Backend) DeleteBlob(ctx context.Context, name string) error {
 		minio.RemoveObjectOptions{})
 }
 
+func (s3 *S3Backend) EnumerateBlobs(ctx context.Context) iter.Seq2[BlobMetadata, error] {
+	return func(yield func(BlobMetadata, error) bool) {
+		logc.Print(ctx, "s3: enumerate blobs")
+
+		ctx, cancel := context.WithCancel(ctx)
+		defer cancel()
+
+		prefix := "blob/"
+		for object := range s3.client.ListObjectsIter(ctx, s3.bucket, minio.ListObjectsOptions{
+			Prefix:    prefix,
+			Recursive: true,
+		}) {
+			var metadata BlobMetadata
+			var err error
+			if err = object.Err; err == nil {
+				key := strings.TrimPrefix(object.Key, prefix)
+				if strings.HasSuffix(key, "/") {
+					continue // directory; skip
+				} else {
+					metadata.Name = joinBlobName(strings.Split(key, "/"))
+					metadata.Size = object.Size
+					metadata.LastModified = object.LastModified
+				}
+			}
+			if !yield(metadata, err) {
+				break
+			}
+		}
+	}
+}
+
 func manifestObjectName(name string) string {
 	return fmt.Sprintf("site/%s", name)
 }

src/main.go

@@ -170,14 +170,16 @@ func usage() {
 	fmt.Fprintf(os.Stderr, "Usage:\n")
 	fmt.Fprintf(os.Stderr, "(server) "+
 		"git-pages [-config <file>|-no-config]\n")
+	fmt.Fprintf(os.Stderr, "(debug)  "+
+		"git-pages {-list-blobs}\n")
+	fmt.Fprintf(os.Stderr, "(debug)  "+
+		"git-pages {-get-blob|-get-manifest|-get-archive|-update-site} <ref> [file]\n")
 	fmt.Fprintf(os.Stderr, "(admin)  "+
 		"git-pages {-run-migration <name>|-freeze-domain <domain>|-unfreeze-domain <domain>}\n")
 	fmt.Fprintf(os.Stderr, "(audit)  "+
 		"git-pages {-audit-log|-audit-read <id>|-audit-server <endpoint> <program> [args...]}\n")
 	fmt.Fprintf(os.Stderr, "(info)   "+
 		"git-pages {-print-config-env-vars|-print-config}\n")
-	fmt.Fprintf(os.Stderr, "(cli)    "+
-		"git-pages {-get-blob|-get-manifest|-get-archive|-update-site} <ref> [file]\n")
 	flag.PrintDefaults()
 }
 
@@ -197,6 +199,8 @@ func Main() {
 		"run a store `migration` (one of: create-domain-markers)")
 	getBlob := flag.String("get-blob", "",
 		"write contents of `blob` ('sha256-xxxxxxx...xxx')")
+	listBlobs := flag.Bool("list-blobs", false,
+		"enumerate every blob with its metadata")
 	getManifest := flag.String("get-manifest", "",
 		"write manifest for `site` (either 'domain.tld' or 'domain.tld/dir') as ProtoJSON")
 	getArchive := flag.String("get-archive", "",
@@ -219,6 +223,7 @@ func Main() {
 	for _, selected := range []bool{
 		*runMigration != "",
 		*getBlob != "",
+		*listBlobs,
 		*getManifest != "",
 		*getArchive != "",
 		*updateSite != "",
@@ -272,32 +277,39 @@ func Main() {
 		logc.Fatalln(ctx, err)
 	}
 
-	switch {
-	case *runMigration != "":
+	// The server has its own logic for creating the backend.
+	if cliOperations > 0 {
 		if backend, err = CreateBackend(ctx, &config.Storage); err != nil {
 			logc.Fatalln(ctx, err)
 		}
+	}
 
+	switch {
+	case *runMigration != "":
 		if err := RunMigration(ctx, *runMigration); err != nil {
 			logc.Fatalln(ctx, err)
 		}
 
 	case *getBlob != "":
-		if backend, err = CreateBackend(ctx, &config.Storage); err != nil {
-			logc.Fatalln(ctx, err)
-		}
-
 		reader, _, err := backend.GetBlob(ctx, *getBlob)
 		if err != nil {
 			logc.Fatalln(ctx, err)
 		}
 		io.Copy(fileOutputArg(), reader)
 
-	case *getManifest != "":
-		if backend, err = CreateBackend(ctx, &config.Storage); err != nil {
-			logc.Fatalln(ctx, err)
+	case *listBlobs:
+		for metadata, err := range backend.EnumerateBlobs(ctx) {
+			if err != nil {
+				logc.Fatalln(ctx, err)
+			}
+			fmt.Fprintf(color.Output, "%s %s %s\n",
+				metadata.Name,
+				color.HiWhiteString(metadata.LastModified.UTC().Format(time.RFC3339)),
+				color.HiGreenString(fmt.Sprint(metadata.Size)),
+			)
 		}
 
+	case *getManifest != "":
 		webRoot := webRootArg(*getManifest)
 		manifest, _, err := backend.GetManifest(ctx, webRoot, GetManifestOptions{})
 		if err != nil {
@@ -306,10 +318,6 @@ func Main() {
 		fmt.Fprintln(fileOutputArg(), string(ManifestJSON(manifest)))
 
 	case *getArchive != "":
-		if backend, err = CreateBackend(ctx, &config.Storage); err != nil {
-			logc.Fatalln(ctx, err)
-		}
-
 		webRoot := webRootArg(*getArchive)
 		manifest, metadata, err :=
 			backend.GetManifest(ctx, webRoot, GetManifestOptions{})
@@ -324,10 +332,6 @@ func Main() {
 		ctx = WithPrincipal(ctx)
 		GetPrincipal(ctx).CliAdmin = proto.Bool(true)
 
-		if backend, err = CreateBackend(ctx, &config.Storage); err != nil {
-			logc.Fatalln(ctx, err)
-		}
-
 		if flag.NArg() != 1 {
 			logc.Fatalln(ctx, "update source must be provided as the argument")
 		}
@@ -402,10 +406,6 @@ func Main() {
 			freeze = false
 		}
 
-		if backend, err = CreateBackend(ctx, &config.Storage); err != nil {
-			logc.Fatalln(ctx, err)
-		}
-
 		if err = backend.FreezeDomain(ctx, domain, freeze); err != nil {
 			logc.Fatalln(ctx, err)
 		}
@@ -416,10 +416,6 @@ func Main() {
 		}
 
 	case *auditLog:
-		if backend, err = CreateBackend(ctx, &config.Storage); err != nil {
-			logc.Fatalln(ctx, err)
-		}
-
 		ch := make(chan *AuditRecord)
 		ids := []AuditID{}
 		for id, err := range backend.SearchAuditLog(ctx, SearchAuditLogOptions{}) {
@@ -454,10 +450,6 @@ func Main() {
 		}
 
 	case *auditRead != "":
-		if backend, err = CreateBackend(ctx, &config.Storage); err != nil {
-			logc.Fatalln(ctx, err)
-		}
-
 		id, err := ParseAuditID(*auditRead)
 		if err != nil {
 			logc.Fatalln(ctx, err)
@@ -473,10 +465,6 @@ func Main() {
 		}
 
 	case *auditServer != "":
-		if backend, err = CreateBackend(ctx, &config.Storage); err != nil {
-			logc.Fatalln(ctx, err)
-		}
-
 		if flag.NArg() < 1 {
 			logc.Fatalln(ctx, "handler path not provided")
 		}

src/observe.go

@@ -373,6 +373,18 @@ func (backend *observedBackend) DeleteBlob(ctx context.Context, name string) (er
 	return
 }
 
+func (backend *observedBackend) EnumerateBlobs(ctx context.Context) iter.Seq2[BlobMetadata, error] {
+	return func(yield func(BlobMetadata, error) bool) {
+		span, ctx := ObserveFunction(ctx, "EnumerateBlobs")
+		for metadata, err := range backend.inner.EnumerateBlobs(ctx) {
+			if !yield(metadata, err) {
+				break
+			}
+		}
+		span.Finish()
+	}
+}
+
 func (backend *observedBackend) ListManifests(ctx context.Context) (manifests []string, err error) {
 	span, ctx := ObserveFunction(ctx, "ListManifests")
 	manifests, err = backend.inner.ListManifests(ctx)