From a2bee712a2bcff47e2608c9baeb0c30682ef16ca Mon Sep 17 00:00:00 2001 From: Catherine Date: Wed, 7 Oct 2026 19:27:38 +0000 Subject: [PATCH] Rename `-repair-storage` to `-scrub-storage`. --- src/main.go | 22 +++++------ src/{repair.go => scrub.go} | 78 +++++++++++++++++-------------------- 2 files changed, 47 insertions(+), 53 deletions(-) rename src/{repair.go => scrub.go} (69%) diff --git a/src/main.go b/src/main.go index 5d99c8b..ee5b3bf 100644 --- a/src/main.go +++ b/src/main.go @@ -220,7 +220,7 @@ func usage() { fmt.Fprintf(os.Stderr, "(audit) "+ "git-pages -audit-server [args...]\n") fmt.Fprintf(os.Stderr, "(maint) "+ - "git-pages {-expire-sites|-repair-storage} [-dry-run]\n") + "git-pages {-expire-sites|-scrub-storage} [-dry-run]\n") fmt.Fprintf(os.Stderr, "(maint) "+ "git-pages {-run-migration |-trace-garbage|-analyze-storage {text|json}}\n") flag.PrintDefaults() @@ -274,8 +274,8 @@ func Main(versionInfo string) { "listen for notifications on `endpoint` and spawn a process for each audit event") expireSites := flag.Bool("expire-sites", false, "expire sites according to their manifest") - repairStorage := flag.Bool("repair-storage", false, - "repair data storage by replacing corrupt manifests with placeholders") + scrubStorage := flag.Bool("scrub-storage", false, + "verify well-formedness of all data (CAUTION: will take offline sites with integrity issues)") runMigration := flag.String("run-migration", "", "run a store `migration` (one of: create-domain-markers)") analyzeStorage := flag.String("analyze-storage", "", @@ -314,7 +314,7 @@ func Main(versionInfo string) { *auditDetach != "", *auditServer != "", *expireSites, - *repairStorage, + *scrubStorage, *runMigration != "", *analyzeStorage != "", *traceGarbage, @@ -327,10 +327,10 @@ func Main(versionInfo string) { logc.Fatalln(ctx, "-list-blobs, -list-manifests, -get-blob, -get-manifest, "+ "-get-archive, -update-site, -delete-site, -freeze-domain, -unfreeze-domain, "+ "-purge-domain, -audit-log, -audit-read, -audit-rollback, -audit-expire, "+ - "-audit-detach, -audit-server, -expire-sites, -repair-storage, -run-migration, "+ + "-audit-detach, -audit-server, -expire-sites, -scrub-storage, -run-migration, "+ "-analyze-storage, and -trace-garbage are mutually exclusive") } - if *dryRun && !(*expireSites || *repairStorage) { + if *dryRun && !(*expireSites || *scrubStorage) { logc.Fatalln(ctx, "-dry-run is not applicable in this context") } @@ -787,18 +787,18 @@ func Main(versionInfo string) { countExpired, countTransient) } - case *repairStorage: + case *scrubStorage: if *GetReason(ctx) == "" { - *GetReason(ctx) = "-repair-storage" + *GetReason(ctx) = "-scrub-storage" } - allGood, err := RepairStorage(ctx, *dryRun) + allGood, err := ScrubStorage(ctx, *dryRun) if err != nil { logc.Fatalln(ctx, err) } if !allGood { - // Exit code 1 indicates an error during the repair process; exit code 2 indicates - // that repair was successful, or that there is data to repair (for -dry-run). + // Exit code 1 indicates an error during the scrubbing process; exit code 2 indicates + // either that repair was successful, or that there is data to repair (if -dry-run). os.Exit(2) } diff --git a/src/repair.go b/src/scrub.go similarity index 69% rename from src/repair.go rename to src/scrub.go index 8b4a77a..30ac526 100644 --- a/src/repair.go +++ b/src/scrub.go @@ -68,7 +68,7 @@ func createBlobIndex() *blobIndex { } func (index *blobIndex) IsFresh(lastModified time.Time) bool { - return lastModified.Add(clockSkewTolerance).Before(index.refTime) + return lastModified.After(index.refTime.Add(-clockSkewTolerance)) } func (index *blobIndex) Add(metadata BlobMetadata) { @@ -80,27 +80,27 @@ func (index *blobIndex) Has(name string) bool { return found } -type repairObject struct { +type scrubObject struct { Manifest *Manifest LastModified time.Time ObjectName string } func hasDanglingBlobReferences( - ctx context.Context, blobIndex *blobIndex, repairManifest *repairObject, + ctx context.Context, blobIndex *blobIndex, scrubManifest *scrubObject, ) bool { - if !blobIndex.IsFresh(repairManifest.LastModified) { + if blobIndex.IsFresh(scrubManifest.LastModified) { // We can't evaluate this manifest, it is too recent. return false } hasDangling := false - for _, entry := range repairManifest.Manifest.GetContents() { + for _, entry := range scrubManifest.Manifest.GetContents() { if entry.GetType() == Type_ExternalFile { blobName := string(entry.GetData()) if !blobIndex.Has(blobName) { - logc.Printf(ctx, "repair fix: %s: dangling reference %s", - repairManifest.ObjectName, blobName) + logc.Printf(ctx, "scrub fix: %s: dangling reference %s\n", + scrubManifest.ObjectName, blobName) hasDangling = true // Continue printing dangling reference names. } @@ -109,7 +109,7 @@ func hasDanglingBlobReferences( return hasDangling } -func RepairStorage(ctx context.Context, dryRun bool) (bool, error) { +func ScrubStorage(ctx context.Context, dryRun bool) (bool, error) { dryRunSuffix := "" if dryRun { dryRunSuffix = " (dry run)" @@ -118,17 +118,17 @@ func RepairStorage(ctx context.Context, dryRun bool) (bool, error) { var corruptedBlobs, corruptedManifests, corruptedAuditRecords uint blobIndex := createBlobIndex() // captures `time.Now()` as reference time - logc.Println(ctx, "repair: checking blobs"+dryRunSuffix) + logc.Println(ctx, "scrub: checking blobs"+dryRunSuffix) for metadata, err := range backend.EnumerateBlobs(ctx) { if err != nil { - // Error fetching metadata, can't repair that. - return false, fmt.Errorf("repair err: %w", err) + // Error fetching metadata, can't scrub that. + return false, fmt.Errorf("scrub err: %w", err) } blobReader, _, err := backend.GetBlob(ctx, metadata.Name) if err != nil { - // Error fetching data, can't repair that. - return false, fmt.Errorf("repair err: %w", err) + // Error fetching data, can't scrub that. + return false, fmt.Errorf("scrub err: %w", err) } blobData, err := io.ReadAll(blobReader) @@ -137,40 +137,40 @@ func RepairStorage(ctx context.Context, dryRun bool) (bool, error) { } if err != nil { // As above. - return false, fmt.Errorf("repair err: %w", err) + return false, fmt.Errorf("scrub err: %w", err) } if blobHash, ok := strings.CutPrefix(metadata.Name, "sha256-"); ok { if blobHash != fmt.Sprintf("%x", sha256.Sum256(blobData)) { - logc.Printf(ctx, "repair fix: blob hash mismatch: %s\n", metadata.Name) + logc.Printf(ctx, "scrub fix: blob hash mismatch: %s\n", metadata.Name) corruptedBlobs += 1 if !dryRun { err = backend.DeleteBlob(ctx, metadata.Name) if err != nil { - return false, fmt.Errorf("repair err: delete blob: %w", err) + return false, fmt.Errorf("scrub err: delete blob: %w", err) } } } else { blobIndex.Add(metadata) } } else { - return false, fmt.Errorf("repair err: invalid blob name: %s", metadata.Name) + return false, fmt.Errorf("scrub err: invalid blob name: %s", metadata.Name) } } - logc.Println(ctx, "repair: checking manifests"+dryRunSuffix) + logc.Println(ctx, "scrub: checking manifests"+dryRunSuffix) for metadata, err := range backend.EnumerateManifests(ctx) { if err != nil { - // Error fetching metadata, can't repair that. - return false, fmt.Errorf("repair err: %w", err) + // Error fetching metadata, can't scrub that. + return false, fmt.Errorf("scrub err: %w", err) } manifest, _, err := backend.GetManifest(ctx, metadata.Name, GetManifestOptions{}) if errors.Is(err, proto.Error) { - logc.Printf(ctx, "repair fix: %s\n", err) + logc.Printf(ctx, "scrub fix: %s\n", err) corruptedManifests += 1 - } else if hasDanglingBlobReferences(ctx, blobIndex, &repairObject{ + } else if hasDanglingBlobReferences(ctx, blobIndex, &scrubObject{ Manifest: manifest, LastModified: metadata.LastModified, ObjectName: fmt.Sprintf("site/%s", metadata.Name), @@ -184,41 +184,41 @@ func RepairStorage(ctx context.Context, dryRun bool) (bool, error) { manifest := createPlaceholderManifest(ctx, metadata.Name) err = backend.StageManifest(ctx, manifest) if err != nil { - return false, fmt.Errorf("repair err: stage manifest: %w", err) + return false, fmt.Errorf("scrub err: stage manifest: %w", err) } err = backend.CommitManifest(ctx, metadata.Name, manifest, ModifyManifestOptions{}) if err != nil { - return false, fmt.Errorf("repair err: commit manifest: %w", err) + return false, fmt.Errorf("scrub err: commit manifest: %w", err) } } } // Enumerate blobs live via audit records. - logc.Println(ctx, "repair: checking audit records"+dryRunSuffix) + logc.Println(ctx, "scrub: checking audit records"+dryRunSuffix) for auditID, err := range backend.SearchAuditLog(ctx, SearchAuditLogOptions{}) { if err != nil { - // Error fetching metadata, can't repair that. - return false, fmt.Errorf("repair err: %w", err) + // Error fetching metadata, can't scrub that. + return false, fmt.Errorf("scrub err: %w", err) } auditRecord, err := backend.QueryAuditLog(ctx, auditID) if errors.Is(err, proto.Error) { - logc.Printf(ctx, "repair fix: %s\n", err) + logc.Printf(ctx, "scrub fix: %s\n", err) corruptedAuditRecords += 1 if !dryRun { - // In general, you're not supposed to do this, but repair is an exception. + // In general, you're not supposed to do this, but scrub is an exception. auditRecord := createPlaceholderAuditRecord(ctx, auditID) err = backend.ExpireAuditRecord(ctx, auditID) if err != nil { - return false, fmt.Errorf("repair err: expire audit: %w", err) + return false, fmt.Errorf("scrub err: expire audit: %w", err) } err = backend.AppendAuditLog(ctx, auditID, auditRecord) if err != nil { - return false, fmt.Errorf("repair err: append audit: %w", err) + return false, fmt.Errorf("scrub err: append audit: %w", err) } } - } else if hasDanglingBlobReferences(ctx, blobIndex, &repairObject{ + } else if hasDanglingBlobReferences(ctx, blobIndex, &scrubObject{ Manifest: auditRecord.GetManifest(), LastModified: auditRecord.GetTimestamp().AsTime(), ObjectName: fmt.Sprintf("audit/%s", auditID), @@ -228,21 +228,15 @@ func RepairStorage(ctx context.Context, dryRun bool) (bool, error) { if !dryRun { err = backend.DetachAuditRecord(ctx, auditID) if err != nil { - return false, fmt.Errorf("repair err: detach audit: %w", err) + return false, fmt.Errorf("scrub err: detach audit: %w", err) } } } } - if dryRun { - logc.Printf(ctx, "repair: %d blobs corrupt", corruptedManifests) - logc.Printf(ctx, "repair: %d manifests corrupt", corruptedManifests) - logc.Printf(ctx, "repair: %d audit records corrupt", corruptedAuditRecords) - } else { - logc.Printf(ctx, "repair: %d blobs repaired", corruptedBlobs) - logc.Printf(ctx, "repair: %d manifests repaired", corruptedManifests) - logc.Printf(ctx, "repair: %d audit records repaired", corruptedAuditRecords) - } + logc.Printf(ctx, "scrub: %d blobs corrupt%s\n", corruptedBlobs, dryRunSuffix) + logc.Printf(ctx, "scrub: %d manifests corrupt%s\n", corruptedManifests, dryRunSuffix) + logc.Printf(ctx, "scrub: %d audit records corrupt%s\n", corruptedAuditRecords, dryRunSuffix) allGood := corruptedManifests == 0 && corruptedAuditRecords == 0 return allGood, nil