Add -repair-storage command.

This command replaces corrupted manifests and audit records with placeholders indicating the corrupt status of the object in question. As a result, other operations (many of which require that all objects can be parsed) continue to work as expected. The contents of a corrupted site manifest is replaced with a human readable index page explaining the situation, and every URL is rewritten to point to that index page.
2026-10-22 15:32:34 +00:00 · 2026-10-05 00:27:29 +00:00
parent e52dc6ed86
commit a8c7bbb316
4 changed files with 180 additions and 11 deletions
@@ -220,7 +220,7 @@ func usage() {
 	fmt.Fprintf(os.Stderr, "(audit)  "+
 		"git-pages  -audit-server <endpoint> <program> [args...]\n")
 	fmt.Fprintf(os.Stderr, "(maint)  "+
-		"git-pages  -expire-sites [-dry-run]\n")
+		"git-pages {-expire-sites|-repair-storage} [-dry-run]\n")
 	fmt.Fprintf(os.Stderr, "(maint)  "+
 		"git-pages {-run-migration <name>|-trace-garbage|-analyze-storage {text|json}}\n")
 	flag.PrintDefaults()
@@ -274,6 +274,8 @@ func Main(versionInfo string) {
 		"listen for notifications on `endpoint` and spawn a process for each audit event")
 	expireSites := flag.Bool("expire-sites", false,
 		"expire sites according to their manifest")
+	repairStorage := flag.Bool("repair-storage", false,
+		"repair data storage by replacing corrupt manifests with placeholders")
 	runMigration := flag.String("run-migration", "",
 		"run a store `migration` (one of: create-domain-markers)")
 	analyzeStorage := flag.String("analyze-storage", "",
@@ -312,6 +314,7 @@ func Main(versionInfo string) {
 		*auditDetach != "",
 		*auditServer != "",
 		*expireSites,
+		*repairStorage,
 		*runMigration != "",
 		*analyzeStorage != "",
 		*traceGarbage,
@@ -324,10 +327,10 @@ func Main(versionInfo string) {
 		logc.Fatalln(ctx, "-list-blobs, -list-manifests, -get-blob, -get-manifest, "+
 			"-get-archive, -update-site, -delete-site, -freeze-domain, -unfreeze-domain, "+
 			"-purge-domain, -audit-log, -audit-read, -audit-rollback, -audit-expire, "+
-			"-audit-detach, -audit-server, -expire-sites, -run-migration, -analyze-storage, "+
-			"and -trace-garbage are mutually exclusive")
+			"-audit-detach, -audit-server, -expire-sites, -repair-storage, -run-migration, "+
+			"-analyze-storage, and -trace-garbage are mutually exclusive")
 	}
-	if *dryRun && !(*expireSites) {
+	if *dryRun && !(*expireSites || *repairStorage) {
 		logc.Fatalln(ctx, "-dry-run is not applicable in this context")
 	}

@@ -429,6 +432,11 @@ func Main(versionInfo string) {
 				color.HiWhiteString(metadata.LastModified.UTC().Format(time.RFC3339)),
 				color.HiGreenString(fmt.Sprint(metadata.Size)),
 			)
+			if manifest.GetCorrupted() {
+				parts = append(parts,
+					color.HiRedString("(corrupted)"),
+				)
+			}
 			fmt.Fprintln(color.Output, strings.Join(parts, " "))
 		}

@@ -588,7 +596,15 @@ func Main(versionInfo string) {
 			parts := []string{
 				record.GetAuditID().String(),
 				color.HiWhiteString("%s", record.GetTimestamp().AsTime().UTC().Format(time.RFC3339)),
-				fmt.Sprint(record.GetEvent()),
+			}
+			if record.GetEvent() == AuditEvent_CorruptedEvent {
+				parts = append(parts,
+					color.HiRedString("%s", fmt.Sprint(record.GetEvent())),
+				)
+			} else {
+				parts = append(parts,
+					fmt.Sprint(record.GetEvent()),
+				)
 			}
 			if record.Manifest != nil && record.Manifest.ExpiresAt != nil {
 				parts = append(parts,
@@ -768,6 +784,21 @@ func Main(versionInfo string) {
 				countExpired, countTransient)
 		}

+	case *repairStorage:
+		if *GetReason(ctx) == "" {
+			*GetReason(ctx) = "-repair-storage"
+		}
+
+		allGood, err := RepairStorage(ctx, *dryRun)
+		if err != nil {
+			logc.Fatalln(ctx, err)
+		}
+		if !allGood {
+			// Exit code 1 indicates an error during the repair process; exit code 2 indicates
+			// that repair was successful, or that there is data to repair (for -dry-run).
+			os.Exit(2)
+		}
+
 	case *runMigration != "":
 		if err = RunMigration(ctx, *runMigration); err != nil {
 			logc.Fatalln(ctx, err)
@@ -0,0 +1,118 @@
+package git_pages
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+
+	"google.golang.org/protobuf/proto"
+	timestamppb "google.golang.org/protobuf/types/known/timestamppb"
+)
+
+func createPlaceholderManifest(ctx context.Context, name string) *Manifest {
+	manifest := NewManifest()
+	manifest.Corrupted = proto.Bool(true)
+	AddFile(manifest, "index.html",
+		[]byte("<!DOCTYPE html>"+
+			"<h1>Data corruption</h1>"+
+			"<p>The site <code>"+name+"</code> has suffered data corruption and must be "+
+			"re-uploaded. Contact the server administrator for more information.</p>"),
+	)
+	manifest.Redirects = []*RedirectRule{
+		{
+			From: proto.String("/*"),
+			To:   proto.String("/index.html"),
+			// Status 500 cannot be used in a `_redirects` file, but we can create such a redirect
+			// rule manually.
+			Status: proto.Uint32(http.StatusInternalServerError),
+			Force:  proto.Bool(true),
+		},
+	}
+	return manifest
+}
+
+func createPlaceholderAuditRecord(ctx context.Context, id AuditID) *AuditRecord {
+	record := &AuditRecord{Event: AuditEvent_CorruptedEvent.Enum()}
+	record.Id = proto.Int64(int64(id))
+	record.Timestamp = timestamppb.Now()
+	record.Principal = GetPrincipal(ctx)
+	if reason := GetReason(ctx); reason != nil && *reason != "" {
+		record.Reason = GetReason(ctx)
+	}
+	return record
+}
+
+func RepairStorage(ctx context.Context, dryRun bool) (bool, error) {
+	var corruptedManifests, corruptedAuditRecords uint
+
+	dryRunSuffix := ""
+	if dryRun {
+		dryRunSuffix = " (dry run)"
+	}
+
+	logc.Println(ctx, "repair: checking manifests"+dryRunSuffix)
+	for metadata, err := range backend.EnumerateManifests(ctx) {
+		if err != nil {
+			// Error fetching metadata, can't repair that.
+			return false, fmt.Errorf("repair err: %w", err)
+		}
+
+		_, _, err = backend.GetManifest(ctx, metadata.Name, GetManifestOptions{})
+		if errors.Is(err, proto.Error) {
+			logc.Printf(ctx, "repair fix: %s\n", err)
+			corruptedManifests += 1
+
+			if !dryRun {
+				manifest := createPlaceholderManifest(ctx, metadata.Name)
+				err = backend.StageManifest(ctx, manifest)
+				if err != nil {
+					return false, fmt.Errorf("repair err: stage manifest: %w", err)
+				}
+				err = backend.CommitManifest(ctx, metadata.Name, manifest, ModifyManifestOptions{})
+				if err != nil {
+					return false, fmt.Errorf("repair err: commit manifest: %w", err)
+				}
+			}
+		}
+	}
+
+	// Enumerate blobs live via audit records.
+	logc.Println(ctx, "repair: checking audit records"+dryRunSuffix)
+	for auditID, err := range backend.SearchAuditLog(ctx, SearchAuditLogOptions{}) {
+		if err != nil {
+			// Error fetching metadata, can't repair that.
+			return false, fmt.Errorf("repair err: %w", err)
+		}
+
+		_, err = backend.QueryAuditLog(ctx, auditID)
+		if errors.Is(err, proto.Error) {
+			logc.Printf(ctx, "repair fix: %s\n", err)
+			corruptedAuditRecords += 1
+
+			if !dryRun {
+				// In general, you're not supposed to do this, but repair is an exception.
+				auditRecord := createPlaceholderAuditRecord(ctx, auditID)
+				err = backend.ExpireAuditRecord(ctx, auditID)
+				if err != nil {
+					return false, fmt.Errorf("repair err: expire audit: %w", err)
+				}
+				err = backend.AppendAuditLog(ctx, auditID, auditRecord)
+				if err != nil {
+					return false, fmt.Errorf("repair err: append audit: %w", err)
+				}
+			}
+		}
+	}
+
+	if dryRun {
+		logc.Printf(ctx, "repair: %d manifests corrupt", corruptedManifests)
+		logc.Printf(ctx, "repair: %d audit records corrupt", corruptedAuditRecords)
+	} else {
+		logc.Printf(ctx, "repair: %d manifests repaired", corruptedManifests)
+		logc.Printf(ctx, "repair: %d audit records repaired", corruptedAuditRecords)
+	}
+
+	allGood := corruptedManifests == 0 && corruptedAuditRecords == 0
+	return allGood, nil
+}
@@ -1,6 +1,6 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
-// 	protoc-gen-go v1.36.9
+// 	protoc-gen-go v1.36.10
 // 	protoc        v6.30.2
 // source: schema.proto

@@ -146,6 +146,8 @@ const (
 	AuditEvent_FreezeDomain AuditEvent = 3
 	// A domain was thawed.
 	AuditEvent_UnfreezeDomain AuditEvent = 4
+	// Event lost due to data corruption.
+	AuditEvent_CorruptedEvent AuditEvent = 6
 )

 // Enum value maps for AuditEvent.
@@ -157,6 +159,7 @@ var (
 		5: "ExpireManifest",
 		3: "FreezeDomain",
 		4: "UnfreezeDomain",
+		6: "CorruptedEvent",
 	}
 	AuditEvent_value = map[string]int32{
 		"InvalidEvent":   0,
@@ -165,6 +168,7 @@ var (
 		"ExpireManifest": 5,
 		"FreezeDomain":   3,
 		"UnfreezeDomain": 4,
+		"CorruptedEvent": 6,
 	}
 )

@@ -657,7 +661,9 @@ type Manifest struct {
 	// Site expiration.
 	ExpiresAt *timestamppb.Timestamp `protobuf:"bytes,12,opt,name=expires_at,json=expiresAt" json:"expires_at,omitempty"`
 	// Diagnostics for non-fatal errors.
-	Problems      []*Problem `protobuf:"bytes,7,rep,name=problems" json:"problems,omitempty"`
+	Problems []*Problem `protobuf:"bytes,7,rep,name=problems" json:"problems,omitempty"`
+	// Indicator of data corruption.
+	Corrupted     *bool `protobuf:"varint,13,opt,name=corrupted" json:"corrupted,omitempty"`
 	unknownFields protoimpl.UnknownFields
 	sizeCache     protoimpl.SizeCache
 }
@@ -776,6 +782,13 @@ func (x *Manifest) GetProblems() []*Problem {
 	return nil
 }

+func (x *Manifest) GetCorrupted() bool {
+	if x != nil && x.Corrupted != nil {
+		return *x.Corrupted
+	}
+	return false
+}
+
 type AuditRecord struct {
 	state protoimpl.MessageState `protogen:"open.v1"`
 	// Audit event metadata.
@@ -1042,7 +1055,7 @@ const file_schema_proto_rawDesc = "" +
 	"\vcredentials\x18\x02 \x03(\v2\x10.BasicCredentialR\vcredentials\"3\n" +
 	"\aProblem\x12\x12\n" +
 	"\x04path\x18\x01 \x01(\tR\x04path\x12\x14\n" +
-	"\x05cause\x18\x02 \x01(\tR\x05cause\"\xa2\x04\n" +
+	"\x05cause\x18\x02 \x01(\tR\x05cause\"\xc0\x04\n" +
 	"\bManifest\x12\x19\n" +
 	"\brepo_url\x18\x01 \x01(\tR\arepoUrl\x12\x16\n" +
 	"\x06branch\x18\x02 \x01(\tR\x06branch\x12\x16\n" +
@@ -1059,7 +1072,8 @@ const file_schema_proto_rawDesc = "" +
 	"basic_auth\x18\v \x03(\v2\x0e.BasicAuthRuleR\tbasicAuth\x129\n" +
 	"\n" +
 	"expires_at\x18\f \x01(\v2\x1a.google.protobuf.TimestampR\texpiresAt\x12$\n" +
-	"\bproblems\x18\a \x03(\v2\b.ProblemR\bproblems\x1aC\n" +
+	"\bproblems\x18\a \x03(\v2\b.ProblemR\bproblems\x12\x1c\n" +
+	"\tcorrupted\x18\r \x01(\bR\tcorrupted\x1aC\n" +
 	"\rContentsEntry\x12\x10\n" +
 	"\x03key\x18\x01 \x01(\tR\x03key\x12\x1c\n" +
 	"\x05value\x18\x02 \x01(\v2\x06.EntryR\x05value:\x028\x01\"\x95\x02\n" +
@@ -1095,7 +1109,7 @@ const file_schema_proto_rawDesc = "" +
 	"\aSymlink\x10\x04*#\n" +
 	"\tTransform\x12\f\n" +
 	"\bIdentity\x10\x00\x12\b\n" +
-	"\x04Zstd\x10\x01*\x80\x01\n" +
+	"\x04Zstd\x10\x01*\x94\x01\n" +
 	"\n" +
 	"AuditEvent\x12\x10\n" +
 	"\fInvalidEvent\x10\x00\x12\x12\n" +
@@ -1103,7 +1117,8 @@ const file_schema_proto_rawDesc = "" +
 	"\x0eDeleteManifest\x10\x02\x12\x12\n" +
 	"\x0eExpireManifest\x10\x05\x12\x10\n" +
 	"\fFreezeDomain\x10\x03\x12\x12\n" +
-	"\x0eUnfreezeDomain\x10\x04B,Z*codeberg.org/git-pages/git-pages/git_pagesb\beditionsp\xe8\a"
+	"\x0eUnfreezeDomain\x10\x04\x12\x12\n" +
+	"\x0eCorruptedEvent\x10\x06B,Z*codeberg.org/git-pages/git-pages/git_pagesb\beditionsp\xe8\a"

 var (
 	file_schema_proto_rawDescOnce sync.Once
@@ -113,6 +113,9 @@ message Manifest {

 	// Diagnostics for non-fatal errors.
 	repeated Problem problems = 7;
+
+	// Indicator of data corruption.
+	bool corrupted = 13;
 }

 enum AuditEvent {
@@ -128,6 +131,8 @@ enum AuditEvent {
 	FreezeDomain = 3;
 	// A domain was thawed.
 	UnfreezeDomain = 4;
+	// Event lost due to data corruption.
+	CorruptedEvent = 6;
 }

 message AuditRecord {