Allow detaching audit records from their blobs for garbage collection.

Resolves: https://codeberg.org/git-pages/git-pages/issues/148

src/backend.go

@@ -160,6 +160,9 @@ type Backend interface {
 	// Retrieve audit record contents for given IDs.
 	GetAuditLogRecords(ctx context.Context, ids iter.Seq2[AuditID, error]) iter.Seq2[*AuditRecord, error]
 
+	// Detach an audit record from its blobs.
+	DetachAuditRecord(ctx context.Context, id AuditID) error
+
 	// Delete an audit record with a given ID.
 	ExpireAuditRecord(ctx context.Context, id AuditID) error
 }

src/backend_fs.go

@@ -484,6 +484,10 @@ func (fs *FSBackend) HaveDomainsChanged(ctx context.Context, since time.Time) (b
 	return true, nil // not implemented
 }
 
+func auditDetachedName(id AuditID) string {
+	return fmt.Sprintf("%s.detached", id)
+}
+
 func (fs *FSBackend) AppendAuditLog(ctx context.Context, id AuditID, record *AuditRecord) error {
 	if _, err := fs.auditRoot.Stat(id.String()); err == nil {
 		panic(fmt.Errorf("audit ID collision: %s", id))
@@ -498,6 +502,11 @@ func (fs *FSBackend) QueryAuditLog(ctx context.Context, id AuditID) (*AuditRecor
 	} else if record, err := DecodeAuditRecord(data); err != nil {
 		return nil, fmt.Errorf("decode: %w", err)
 	} else {
+		if _, err := fs.auditRoot.Stat(auditDetachedName(id)); err == nil {
+			record.Manifest = nil
+		} else if !errors.Is(err, os.ErrNotExist) {
+			return nil, fmt.Errorf("stat detached marker: %w", err)
+		}
 		return record, nil
 	}
 }
@@ -514,6 +523,8 @@ func (fs *FSBackend) SearchAuditLog(
 				var id AuditID
 				if err != nil {
 					// report error
+				} else if strings.Contains(path, ".") {
+					return nil // skip
 				} else if id, err = ParseAuditID(path); err != nil {
 					// report error
 				} else if !opts.Since.IsZero() && id.CompareTime(opts.Since) < 0 {
@@ -546,6 +557,10 @@ func (fs *FSBackend) GetAuditLogRecords(
 	}
 }
 
+func (fs *FSBackend) DetachAuditRecord(ctx context.Context, id AuditID) error {
+	return fs.auditRoot.WriteFile(auditDetachedName(id), []byte{}, 0o644)
+}
+
 func (fs *FSBackend) ExpireAuditRecord(ctx context.Context, id AuditID) error {
 	return fs.auditRoot.Remove(id.String())
 }

src/backend_s3.go

@@ -827,6 +827,10 @@ func auditObjectName(id AuditID) string {
 	return fmt.Sprintf("audit/%s", id)
 }
 
+func auditDetachedObjectName(id AuditID) string {
+	return fmt.Sprintf("audit/%s.detached", id)
+}
+
 func (s3 *S3Backend) AppendAuditLog(ctx context.Context, id AuditID, record *AuditRecord) error {
 	logc.Printf(ctx, "s3: append audit %s\n", id)
 
@@ -858,7 +862,20 @@ func (s3 *S3Backend) QueryAuditLog(ctx context.Context, id AuditID) (*AuditRecor
 		return nil, err
 	}
 
-	return DecodeAuditRecord(data)
+	record, err := DecodeAuditRecord(data)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = s3.client.StatObject(ctx, s3.bucket, auditDetachedObjectName(id),
+		minio.StatObjectOptions{})
+	if err == nil {
+		record.Manifest = nil
+	} else if errResp := minio.ToErrorResponse(err); err != nil && errResp.Code != "NoSuchKey" {
+		return nil, err
+	}
+
+	return record, nil
 }
 
 func (s3 *S3Backend) SearchAuditLog(
@@ -878,6 +895,8 @@ func (s3 *S3Backend) SearchAuditLog(
 			var err error
 			if object.Err != nil {
 				err = object.Err
+			} else if strings.Contains(object.Key, ".") {
+				continue
 			} else if id, err = ParseAuditID(strings.TrimPrefix(object.Key, prefix)); err != nil {
 				// report error
 			} else if !opts.Since.IsZero() && id.CompareTime(opts.Since) < 0 {
@@ -929,6 +948,14 @@ func (s3 *S3Backend) GetAuditLogRecords(
 	}
 }
 
+func (s3 *S3Backend) DetachAuditRecord(ctx context.Context, id AuditID) error {
+	logc.Printf(ctx, "s3: detach audit record %s\n", id)
+
+	_, err := s3.client.PutObject(ctx, s3.bucket, auditDetachedObjectName(id),
+		&bytes.Reader{}, 0, minio.PutObjectOptions{})
+	return err
+}
+
 func (s3 *S3Backend) ExpireAuditRecord(ctx context.Context, id AuditID) error {
 	logc.Printf(ctx, "s3: expire audit record %s\n", id)
 

src/main.go

@@ -193,7 +193,9 @@ func usage() {
 	fmt.Fprintf(os.Stderr, "(admin)  "+
 		"git-pages {-freeze-domain <domain>|-unfreeze-domain <domain>}\n")
 	fmt.Fprintf(os.Stderr, "(audit)  "+
-		"git-pages {-audit-log|-audit-read <id>|-audit-server <endpoint> <program> [args...]}\n")
+		"git-pages {-audit-log|-audit-server <endpoint> <program> [args...]}\n")
+	fmt.Fprintf(os.Stderr, "(audit)  "+
+		"git-pages {-audit-read|-audit-rollback|-audit-detach} <id>\n")
 	fmt.Fprintf(os.Stderr, "(audit)  "+
 		"git-pages {-audit-expire <days>}\n")
 	fmt.Fprintf(os.Stderr, "(maint)  "+
@@ -237,6 +239,8 @@ func Main(versionInfo string) {
 		"extract contents of audit record `id` to files '<id>-*'")
 	auditRollback := flag.String("audit-rollback", "",
 		"restore site from contents of audit record `id`")
+	auditDetach := flag.String("audit-detach", "",
+		"detach all blobs of audit record `id`")
 	auditServer := flag.String("audit-server", "",
 		"listen for notifications on `endpoint` and spawn a process for each audit event")
 	auditExpire := flag.String("audit-expire", "",
@@ -269,6 +273,7 @@ func Main(versionInfo string) {
 		*auditLog,
 		*auditRead != "",
 		*auditRollback != "",
+		*auditDetach != "",
 		*auditServer != "",
 		*auditExpire != "",
 		*runMigration != "",
@@ -282,8 +287,8 @@ func Main(versionInfo string) {
 	if cliOperations > 1 {
 		logc.Fatalln(ctx, "-list-blobs, -list-manifests, -get-blob, -get-manifest, -get-archive, "+
 			"-update-site, -freeze-domain, -unfreeze-domain, -audit-log, -audit-read, "+
-			"-audit-rollback, -audit-server, -run-migration, -size-histogram, "+
-			"and -trace-garbage are mutually exclusive")
+			"-audit-rollback, -audit-detach, -audit-server, -audit-expire, -run-migration, "+
+			"-size-histogram, and -trace-garbage are mutually exclusive")
 	}
 
 	if *configTomlPath != "" && *noConfig {
@@ -334,15 +339,15 @@ func Main(versionInfo string) {
 		logc.Fatalln(ctx, err)
 	}
 
-	if domainCache, err = CreateDomainCache(ctx); err != nil {
-		logc.Fatalln(ctx, err)
-	}
-
 	// The server has its own logic for creating the backend.
 	if cliOperations > 0 {
 		if backend, err = CreateBackend(ctx, &config.Storage); err != nil {
 			logc.Fatalln(ctx, err)
 		}
+
+		if domainCache, err = CreateDomainCache(ctx); err != nil {
+			logc.Fatalln(ctx, err)
+		}
 	}
 
 	switch {
@@ -553,6 +558,17 @@ func Main(versionInfo string) {
 			logc.Fatalln(ctx, err)
 		}
 
+	case *auditDetach != "":
+		id, err := ParseAuditID(*auditDetach)
+		if err != nil {
+			logc.Fatalln(ctx, err)
+		}
+
+		err = backend.DetachAuditRecord(ctx, id)
+		if err != nil {
+			logc.Fatalln(ctx, err)
+		}
+
 	case *auditServer != "":
 		if flag.NArg() < 1 {
 			logc.Fatalln(ctx, "handler path not provided")
@@ -692,6 +708,10 @@ func Main(versionInfo string) {
 		}
 		backend = NewObservedBackend(backend)
 
+		if domainCache, err = CreateDomainCache(ctx); err != nil {
+			logc.Fatalln(ctx, err)
+		}
+
 		middleware := chainHTTPMiddleware(
 			panicHandler,
 			remoteAddrMiddleware,

src/observe.go

@@ -398,6 +398,13 @@ func (backend *observedBackend) GetAuditLogRecords(
 	}
 }
 
+func (backend *observedBackend) DetachAuditRecord(ctx context.Context, id AuditID) (err error) {
+	span, ctx := ObserveFunction(ctx, "DetachAuditRecord", "audit.id", id)
+	err = backend.inner.DetachAuditRecord(ctx, id)
+	span.Finish()
+	return
+}
+
 func (backend *observedBackend) ExpireAuditRecord(ctx context.Context, id AuditID) (err error) {
 	span, ctx := ObserveFunction(ctx, "ExpireAuditRecord", "audit.id", id)
 	err = backend.inner.ExpireAuditRecord(ctx, id)