diff --git a/src/backend.go b/src/backend.go index f00b434..6e90f6e 100644 --- a/src/backend.go +++ b/src/backend.go @@ -6,7 +6,6 @@ import ( "fmt" "io" "iter" - "slices" "strings" "time" ) @@ -17,14 +16,17 @@ var ErrWriteConflict = errors.New("write conflict") var ErrDomainFrozen = errors.New("domain administratively frozen") func splitBlobName(name string) []string { - algo, hash, found := strings.Cut(name, "-") - if found { - return slices.Concat([]string{algo}, splitBlobName(hash)) + if algo, hash, found := strings.Cut(name, "-"); found { + return []string{algo, hash[0:2], hash[2:4], hash[4:]} } else { - return []string{name[0:2], name[2:4], name[4:]} + panic("malformed blob name") } } +func joinBlobName(parts []string) string { + return fmt.Sprintf("%s-%s", parts[0], strings.Join(parts[1:], "")) +} + type BackendFeature string const ( @@ -32,6 +34,7 @@ const ( ) type BlobMetadata struct { + Name string Size int64 LastModified time.Time } @@ -93,6 +96,10 @@ type Backend interface { // Delete a blob. This is an unconditional operation that can break integrity of manifests. DeleteBlob(ctx context.Context, name string) error + // Iterate through all blobs. Whether blobs that are newly added during iteration will appear + // in the results is unspecified. + EnumerateBlobs(ctx context.Context) iter.Seq2[BlobMetadata, error] + // Retrieve a manifest. GetManifest(ctx context.Context, name string, opts GetManifestOptions) ( manifest *Manifest, metadata ManifestMetadata, err error, diff --git a/src/backend_fs.go b/src/backend_fs.go index b7efe58..6d0e01b 100644 --- a/src/backend_fs.go +++ b/src/backend_fs.go @@ -133,7 +133,7 @@ func (fs *FSBackend) GetBlob( err = fmt.Errorf("open: %w", err) return } - return file, BlobMetadata{int64(stat.Size()), stat.ModTime()}, nil + return file, BlobMetadata{name, int64(stat.Size()), stat.ModTime()}, nil } func (fs *FSBackend) PutBlob(ctx context.Context, name string, data []byte) error { @@ -181,6 +181,32 @@ func (fs *FSBackend) DeleteBlob(ctx context.Context, name string) error { return fs.blobRoot.Remove(blobPath) } +func (fs *FSBackend) EnumerateBlobs(ctx context.Context) iter.Seq2[BlobMetadata, error] { + return func(yield func(BlobMetadata, error) bool) { + iofs.WalkDir(fs.blobRoot.FS(), ".", + func(path string, entry iofs.DirEntry, err error) error { + var metadata BlobMetadata + if err != nil { + // report error + } else if entry.IsDir() { + // skip directory + return nil + } else if info, err := entry.Info(); err != nil { + // report error + } else { + // report blob + metadata.Name = joinBlobName(strings.Split(path, "/")) + metadata.Size = info.Size() + metadata.LastModified = info.ModTime() + } + if !yield(metadata, err) { + return iofs.SkipAll + } + return nil + }) + } +} + func (fs *FSBackend) ListManifests(ctx context.Context) (manifests []string, err error) { err = iofs.WalkDir(fs.siteRoot.FS(), ".", func(path string, entry iofs.DirEntry, err error) error { diff --git a/src/backend_s3.go b/src/backend_s3.go index 1def010..aa99460 100644 --- a/src/backend_s3.go +++ b/src/backend_s3.go @@ -316,6 +316,7 @@ func (s3 *S3Backend) GetBlob( } } else { reader = bytes.NewReader(cached.blob) + metadata.Name = name metadata.Size = int64(len(cached.blob)) metadata.LastModified = cached.mtime } @@ -357,6 +358,37 @@ func (s3 *S3Backend) DeleteBlob(ctx context.Context, name string) error { minio.RemoveObjectOptions{}) } +func (s3 *S3Backend) EnumerateBlobs(ctx context.Context) iter.Seq2[BlobMetadata, error] { + return func(yield func(BlobMetadata, error) bool) { + logc.Print(ctx, "s3: enumerate blobs") + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + prefix := "blob/" + for object := range s3.client.ListObjectsIter(ctx, s3.bucket, minio.ListObjectsOptions{ + Prefix: prefix, + Recursive: true, + }) { + var metadata BlobMetadata + var err error + if err = object.Err; err == nil { + key := strings.TrimPrefix(object.Key, prefix) + if strings.HasSuffix(key, "/") { + continue // directory; skip + } else { + metadata.Name = joinBlobName(strings.Split(key, "/")) + metadata.Size = object.Size + metadata.LastModified = object.LastModified + } + } + if !yield(metadata, err) { + break + } + } + } +} + func manifestObjectName(name string) string { return fmt.Sprintf("site/%s", name) } diff --git a/src/main.go b/src/main.go index 70bb478..ea302c6 100644 --- a/src/main.go +++ b/src/main.go @@ -170,14 +170,16 @@ func usage() { fmt.Fprintf(os.Stderr, "Usage:\n") fmt.Fprintf(os.Stderr, "(server) "+ "git-pages [-config |-no-config]\n") + fmt.Fprintf(os.Stderr, "(debug) "+ + "git-pages {-list-blobs}\n") + fmt.Fprintf(os.Stderr, "(debug) "+ + "git-pages {-get-blob|-get-manifest|-get-archive|-update-site} [file]\n") fmt.Fprintf(os.Stderr, "(admin) "+ "git-pages {-run-migration |-freeze-domain |-unfreeze-domain }\n") fmt.Fprintf(os.Stderr, "(audit) "+ "git-pages {-audit-log|-audit-read |-audit-server [args...]}\n") fmt.Fprintf(os.Stderr, "(info) "+ "git-pages {-print-config-env-vars|-print-config}\n") - fmt.Fprintf(os.Stderr, "(cli) "+ - "git-pages {-get-blob|-get-manifest|-get-archive|-update-site} [file]\n") flag.PrintDefaults() } @@ -197,6 +199,8 @@ func Main() { "run a store `migration` (one of: create-domain-markers)") getBlob := flag.String("get-blob", "", "write contents of `blob` ('sha256-xxxxxxx...xxx')") + listBlobs := flag.Bool("list-blobs", false, + "enumerate every blob with its metadata") getManifest := flag.String("get-manifest", "", "write manifest for `site` (either 'domain.tld' or 'domain.tld/dir') as ProtoJSON") getArchive := flag.String("get-archive", "", @@ -219,6 +223,7 @@ func Main() { for _, selected := range []bool{ *runMigration != "", *getBlob != "", + *listBlobs, *getManifest != "", *getArchive != "", *updateSite != "", @@ -272,32 +277,39 @@ func Main() { logc.Fatalln(ctx, err) } - switch { - case *runMigration != "": + // The server has its own logic for creating the backend. + if cliOperations > 0 { if backend, err = CreateBackend(ctx, &config.Storage); err != nil { logc.Fatalln(ctx, err) } + } + switch { + case *runMigration != "": if err := RunMigration(ctx, *runMigration); err != nil { logc.Fatalln(ctx, err) } case *getBlob != "": - if backend, err = CreateBackend(ctx, &config.Storage); err != nil { - logc.Fatalln(ctx, err) - } - reader, _, err := backend.GetBlob(ctx, *getBlob) if err != nil { logc.Fatalln(ctx, err) } io.Copy(fileOutputArg(), reader) - case *getManifest != "": - if backend, err = CreateBackend(ctx, &config.Storage); err != nil { - logc.Fatalln(ctx, err) + case *listBlobs: + for metadata, err := range backend.EnumerateBlobs(ctx) { + if err != nil { + logc.Fatalln(ctx, err) + } + fmt.Fprintf(color.Output, "%s %s %s\n", + metadata.Name, + color.HiWhiteString(metadata.LastModified.UTC().Format(time.RFC3339)), + color.HiGreenString(fmt.Sprint(metadata.Size)), + ) } + case *getManifest != "": webRoot := webRootArg(*getManifest) manifest, _, err := backend.GetManifest(ctx, webRoot, GetManifestOptions{}) if err != nil { @@ -306,10 +318,6 @@ func Main() { fmt.Fprintln(fileOutputArg(), string(ManifestJSON(manifest))) case *getArchive != "": - if backend, err = CreateBackend(ctx, &config.Storage); err != nil { - logc.Fatalln(ctx, err) - } - webRoot := webRootArg(*getArchive) manifest, metadata, err := backend.GetManifest(ctx, webRoot, GetManifestOptions{}) @@ -324,10 +332,6 @@ func Main() { ctx = WithPrincipal(ctx) GetPrincipal(ctx).CliAdmin = proto.Bool(true) - if backend, err = CreateBackend(ctx, &config.Storage); err != nil { - logc.Fatalln(ctx, err) - } - if flag.NArg() != 1 { logc.Fatalln(ctx, "update source must be provided as the argument") } @@ -402,10 +406,6 @@ func Main() { freeze = false } - if backend, err = CreateBackend(ctx, &config.Storage); err != nil { - logc.Fatalln(ctx, err) - } - if err = backend.FreezeDomain(ctx, domain, freeze); err != nil { logc.Fatalln(ctx, err) } @@ -416,10 +416,6 @@ func Main() { } case *auditLog: - if backend, err = CreateBackend(ctx, &config.Storage); err != nil { - logc.Fatalln(ctx, err) - } - ch := make(chan *AuditRecord) ids := []AuditID{} for id, err := range backend.SearchAuditLog(ctx, SearchAuditLogOptions{}) { @@ -454,10 +450,6 @@ func Main() { } case *auditRead != "": - if backend, err = CreateBackend(ctx, &config.Storage); err != nil { - logc.Fatalln(ctx, err) - } - id, err := ParseAuditID(*auditRead) if err != nil { logc.Fatalln(ctx, err) @@ -473,10 +465,6 @@ func Main() { } case *auditServer != "": - if backend, err = CreateBackend(ctx, &config.Storage); err != nil { - logc.Fatalln(ctx, err) - } - if flag.NArg() < 1 { logc.Fatalln(ctx, "handler path not provided") } diff --git a/src/observe.go b/src/observe.go index 352e643..e8cfddb 100644 --- a/src/observe.go +++ b/src/observe.go @@ -373,6 +373,18 @@ func (backend *observedBackend) DeleteBlob(ctx context.Context, name string) (er return } +func (backend *observedBackend) EnumerateBlobs(ctx context.Context) iter.Seq2[BlobMetadata, error] { + return func(yield func(BlobMetadata, error) bool) { + span, ctx := ObserveFunction(ctx, "EnumerateBlobs") + for metadata, err := range backend.inner.EnumerateBlobs(ctx) { + if !yield(metadata, err) { + break + } + } + span.Finish() + } +} + func (backend *observedBackend) ListManifests(ctx context.Context) (manifests []string, err error) { span, ctx := ObserveFunction(ctx, "ListManifests") manifests, err = backend.inner.ListManifests(ctx)