diff --git a/src/main.go b/src/main.go index c278d18..51c7fd6 100644 --- a/src/main.go +++ b/src/main.go @@ -220,7 +220,7 @@ func usage() { fmt.Fprintf(os.Stderr, "(audit) "+ "git-pages -audit-server [args...]\n") fmt.Fprintf(os.Stderr, "(maint) "+ - "git-pages -expire-sites [-dry-run]\n") + "git-pages {-expire-sites|-repair-storage} [-dry-run]\n") fmt.Fprintf(os.Stderr, "(maint) "+ "git-pages {-run-migration |-trace-garbage|-analyze-storage {text|json}}\n") flag.PrintDefaults() @@ -274,6 +274,8 @@ func Main(versionInfo string) { "listen for notifications on `endpoint` and spawn a process for each audit event") expireSites := flag.Bool("expire-sites", false, "expire sites according to their manifest") + repairStorage := flag.Bool("repair-storage", false, + "repair data storage by replacing corrupt manifests with placeholders") runMigration := flag.String("run-migration", "", "run a store `migration` (one of: create-domain-markers)") analyzeStorage := flag.String("analyze-storage", "", @@ -312,6 +314,7 @@ func Main(versionInfo string) { *auditDetach != "", *auditServer != "", *expireSites, + *repairStorage, *runMigration != "", *analyzeStorage != "", *traceGarbage, @@ -324,10 +327,10 @@ func Main(versionInfo string) { logc.Fatalln(ctx, "-list-blobs, -list-manifests, -get-blob, -get-manifest, "+ "-get-archive, -update-site, -delete-site, -freeze-domain, -unfreeze-domain, "+ "-purge-domain, -audit-log, -audit-read, -audit-rollback, -audit-expire, "+ - "-audit-detach, -audit-server, -expire-sites, -run-migration, -analyze-storage, "+ - "and -trace-garbage are mutually exclusive") + "-audit-detach, -audit-server, -expire-sites, -repair-storage, -run-migration, "+ + "-analyze-storage, and -trace-garbage are mutually exclusive") } - if *dryRun && !(*expireSites) { + if *dryRun && !(*expireSites || *repairStorage) { logc.Fatalln(ctx, "-dry-run is not applicable in this context") } @@ -429,6 +432,11 @@ func Main(versionInfo string) { color.HiWhiteString(metadata.LastModified.UTC().Format(time.RFC3339)), color.HiGreenString(fmt.Sprint(metadata.Size)), ) + if manifest.GetCorrupted() { + parts = append(parts, + color.HiRedString("(corrupted)"), + ) + } fmt.Fprintln(color.Output, strings.Join(parts, " ")) } @@ -588,7 +596,15 @@ func Main(versionInfo string) { parts := []string{ record.GetAuditID().String(), color.HiWhiteString("%s", record.GetTimestamp().AsTime().UTC().Format(time.RFC3339)), - fmt.Sprint(record.GetEvent()), + } + if record.GetEvent() == AuditEvent_CorruptedEvent { + parts = append(parts, + color.HiRedString("%s", fmt.Sprint(record.GetEvent())), + ) + } else { + parts = append(parts, + fmt.Sprint(record.GetEvent()), + ) } if record.Manifest != nil && record.Manifest.ExpiresAt != nil { parts = append(parts, @@ -768,6 +784,21 @@ func Main(versionInfo string) { countExpired, countTransient) } + case *repairStorage: + if *GetReason(ctx) == "" { + *GetReason(ctx) = "-repair-storage" + } + + allGood, err := RepairStorage(ctx, *dryRun) + if err != nil { + logc.Fatalln(ctx, err) + } + if !allGood { + // Exit code 1 indicates an error during the repair process; exit code 2 indicates + // that repair was successful, or that there is data to repair (for -dry-run). + os.Exit(2) + } + case *runMigration != "": if err = RunMigration(ctx, *runMigration); err != nil { logc.Fatalln(ctx, err) diff --git a/src/repair.go b/src/repair.go new file mode 100644 index 0000000..d8def0b --- /dev/null +++ b/src/repair.go @@ -0,0 +1,118 @@ +package git_pages + +import ( + "context" + "errors" + "fmt" + "net/http" + + "google.golang.org/protobuf/proto" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" +) + +func createPlaceholderManifest(ctx context.Context, name string) *Manifest { + manifest := NewManifest() + manifest.Corrupted = proto.Bool(true) + AddFile(manifest, "index.html", + []byte(""+ + "

Data corruption

"+ + "

The site "+name+" has suffered data corruption and must be "+ + "re-uploaded. Contact the server administrator for more information.

"), + ) + manifest.Redirects = []*RedirectRule{ + { + From: proto.String("/*"), + To: proto.String("/index.html"), + // Status 500 cannot be used in a `_redirects` file, but we can create such a redirect + // rule manually. + Status: proto.Uint32(http.StatusInternalServerError), + Force: proto.Bool(true), + }, + } + return manifest +} + +func createPlaceholderAuditRecord(ctx context.Context, id AuditID) *AuditRecord { + record := &AuditRecord{Event: AuditEvent_CorruptedEvent.Enum()} + record.Id = proto.Int64(int64(id)) + record.Timestamp = timestamppb.Now() + record.Principal = GetPrincipal(ctx) + if reason := GetReason(ctx); reason != nil && *reason != "" { + record.Reason = GetReason(ctx) + } + return record +} + +func RepairStorage(ctx context.Context, dryRun bool) (bool, error) { + var corruptedManifests, corruptedAuditRecords uint + + dryRunSuffix := "" + if dryRun { + dryRunSuffix = " (dry run)" + } + + logc.Println(ctx, "repair: checking manifests"+dryRunSuffix) + for metadata, err := range backend.EnumerateManifests(ctx) { + if err != nil { + // Error fetching metadata, can't repair that. + return false, fmt.Errorf("repair err: %w", err) + } + + _, _, err = backend.GetManifest(ctx, metadata.Name, GetManifestOptions{}) + if errors.Is(err, proto.Error) { + logc.Printf(ctx, "repair fix: %s\n", err) + corruptedManifests += 1 + + if !dryRun { + manifest := createPlaceholderManifest(ctx, metadata.Name) + err = backend.StageManifest(ctx, manifest) + if err != nil { + return false, fmt.Errorf("repair err: stage manifest: %w", err) + } + err = backend.CommitManifest(ctx, metadata.Name, manifest, ModifyManifestOptions{}) + if err != nil { + return false, fmt.Errorf("repair err: commit manifest: %w", err) + } + } + } + } + + // Enumerate blobs live via audit records. + logc.Println(ctx, "repair: checking audit records"+dryRunSuffix) + for auditID, err := range backend.SearchAuditLog(ctx, SearchAuditLogOptions{}) { + if err != nil { + // Error fetching metadata, can't repair that. + return false, fmt.Errorf("repair err: %w", err) + } + + _, err = backend.QueryAuditLog(ctx, auditID) + if errors.Is(err, proto.Error) { + logc.Printf(ctx, "repair fix: %s\n", err) + corruptedAuditRecords += 1 + + if !dryRun { + // In general, you're not supposed to do this, but repair is an exception. + auditRecord := createPlaceholderAuditRecord(ctx, auditID) + err = backend.ExpireAuditRecord(ctx, auditID) + if err != nil { + return false, fmt.Errorf("repair err: expire audit: %w", err) + } + err = backend.AppendAuditLog(ctx, auditID, auditRecord) + if err != nil { + return false, fmt.Errorf("repair err: append audit: %w", err) + } + } + } + } + + if dryRun { + logc.Printf(ctx, "repair: %d manifests corrupt", corruptedManifests) + logc.Printf(ctx, "repair: %d audit records corrupt", corruptedAuditRecords) + } else { + logc.Printf(ctx, "repair: %d manifests repaired", corruptedManifests) + logc.Printf(ctx, "repair: %d audit records repaired", corruptedAuditRecords) + } + + allGood := corruptedManifests == 0 && corruptedAuditRecords == 0 + return allGood, nil +} diff --git a/src/schema.pb.go b/src/schema.pb.go index d245b07..89f0325 100644 --- a/src/schema.pb.go +++ b/src/schema.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.9 +// protoc-gen-go v1.36.10 // protoc v6.30.2 // source: schema.proto @@ -146,6 +146,8 @@ const ( AuditEvent_FreezeDomain AuditEvent = 3 // A domain was thawed. AuditEvent_UnfreezeDomain AuditEvent = 4 + // Event lost due to data corruption. + AuditEvent_CorruptedEvent AuditEvent = 6 ) // Enum value maps for AuditEvent. @@ -157,6 +159,7 @@ var ( 5: "ExpireManifest", 3: "FreezeDomain", 4: "UnfreezeDomain", + 6: "CorruptedEvent", } AuditEvent_value = map[string]int32{ "InvalidEvent": 0, @@ -165,6 +168,7 @@ var ( "ExpireManifest": 5, "FreezeDomain": 3, "UnfreezeDomain": 4, + "CorruptedEvent": 6, } ) @@ -657,7 +661,9 @@ type Manifest struct { // Site expiration. ExpiresAt *timestamppb.Timestamp `protobuf:"bytes,12,opt,name=expires_at,json=expiresAt" json:"expires_at,omitempty"` // Diagnostics for non-fatal errors. - Problems []*Problem `protobuf:"bytes,7,rep,name=problems" json:"problems,omitempty"` + Problems []*Problem `protobuf:"bytes,7,rep,name=problems" json:"problems,omitempty"` + // Indicator of data corruption. + Corrupted *bool `protobuf:"varint,13,opt,name=corrupted" json:"corrupted,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -776,6 +782,13 @@ func (x *Manifest) GetProblems() []*Problem { return nil } +func (x *Manifest) GetCorrupted() bool { + if x != nil && x.Corrupted != nil { + return *x.Corrupted + } + return false +} + type AuditRecord struct { state protoimpl.MessageState `protogen:"open.v1"` // Audit event metadata. @@ -1042,7 +1055,7 @@ const file_schema_proto_rawDesc = "" + "\vcredentials\x18\x02 \x03(\v2\x10.BasicCredentialR\vcredentials\"3\n" + "\aProblem\x12\x12\n" + "\x04path\x18\x01 \x01(\tR\x04path\x12\x14\n" + - "\x05cause\x18\x02 \x01(\tR\x05cause\"\xa2\x04\n" + + "\x05cause\x18\x02 \x01(\tR\x05cause\"\xc0\x04\n" + "\bManifest\x12\x19\n" + "\brepo_url\x18\x01 \x01(\tR\arepoUrl\x12\x16\n" + "\x06branch\x18\x02 \x01(\tR\x06branch\x12\x16\n" + @@ -1059,7 +1072,8 @@ const file_schema_proto_rawDesc = "" + "basic_auth\x18\v \x03(\v2\x0e.BasicAuthRuleR\tbasicAuth\x129\n" + "\n" + "expires_at\x18\f \x01(\v2\x1a.google.protobuf.TimestampR\texpiresAt\x12$\n" + - "\bproblems\x18\a \x03(\v2\b.ProblemR\bproblems\x1aC\n" + + "\bproblems\x18\a \x03(\v2\b.ProblemR\bproblems\x12\x1c\n" + + "\tcorrupted\x18\r \x01(\bR\tcorrupted\x1aC\n" + "\rContentsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x1c\n" + "\x05value\x18\x02 \x01(\v2\x06.EntryR\x05value:\x028\x01\"\x95\x02\n" + @@ -1095,7 +1109,7 @@ const file_schema_proto_rawDesc = "" + "\aSymlink\x10\x04*#\n" + "\tTransform\x12\f\n" + "\bIdentity\x10\x00\x12\b\n" + - "\x04Zstd\x10\x01*\x80\x01\n" + + "\x04Zstd\x10\x01*\x94\x01\n" + "\n" + "AuditEvent\x12\x10\n" + "\fInvalidEvent\x10\x00\x12\x12\n" + @@ -1103,7 +1117,8 @@ const file_schema_proto_rawDesc = "" + "\x0eDeleteManifest\x10\x02\x12\x12\n" + "\x0eExpireManifest\x10\x05\x12\x10\n" + "\fFreezeDomain\x10\x03\x12\x12\n" + - "\x0eUnfreezeDomain\x10\x04B,Z*codeberg.org/git-pages/git-pages/git_pagesb\beditionsp\xe8\a" + "\x0eUnfreezeDomain\x10\x04\x12\x12\n" + + "\x0eCorruptedEvent\x10\x06B,Z*codeberg.org/git-pages/git-pages/git_pagesb\beditionsp\xe8\a" var ( file_schema_proto_rawDescOnce sync.Once diff --git a/src/schema.proto b/src/schema.proto index 0b909d9..ae1b4f3 100644 --- a/src/schema.proto +++ b/src/schema.proto @@ -113,6 +113,9 @@ message Manifest { // Diagnostics for non-fatal errors. repeated Problem problems = 7; + + // Indicator of data corruption. + bool corrupted = 13; } enum AuditEvent { @@ -128,6 +131,8 @@ enum AuditEvent { FreezeDomain = 3; // A domain was thawed. UnfreezeDomain = 4; + // Event lost due to data corruption. + CorruptedEvent = 6; } message AuditRecord {