git-pages/src/garbage.go

package git_pages

import (
	"context"
	"fmt"

	"github.com/c2h5oh/datasize"
	"github.com/dghubble/trie"
)

func trieReduce(data trie.Trier) (items, total int64) {
	data.Walk(func(key string, value any) error {
		items += 1
		total += *value.(*int64)
		return nil
	})
	return
}

func TraceGarbage(ctx context.Context) error {
	allBlobs := trie.NewRuneTrie()
	liveBlobs := trie.NewRuneTrie()

	traceManifest := func(manifestName string, manifest *Manifest) error {
		for _, entry := range manifest.GetContents() {
			if entry.GetType() == Type_ExternalFile {
				blobName := string(entry.Data)
				if size := allBlobs.Get(blobName); size == nil {
					return fmt.Errorf("%s: dangling reference %s", manifestName, blobName)
				} else {
					liveBlobs.Put(blobName, size)
				}
			}
		}
		return nil
	}

	// Enumerate all blobs.
	for metadata, err := range backend.EnumerateBlobs(ctx) {
		if err != nil {
			return fmt.Errorf("trace blobs err: %w", err)
		}
		allBlobs.Put(metadata.Name, &metadata.Size)
	}

	// Enumerate blobs live via site manifests.
	for item, err := range backend.GetAllManifests(ctx) {
		metadata, manifest := item.Splat()
		if err != nil {
			return fmt.Errorf("trace sites err: %w", err)
		}
		err = traceManifest(metadata.Name, manifest)
		if err != nil {
			return fmt.Errorf("trace sites err: %w", err)
		}
	}

	// Enumerate blobs live via audit records.

	auditIDs := backend.SearchAuditLog(ctx, SearchAuditLogOptions{})
	for record, err := range backend.GetAuditLogRecords(ctx, auditIDs) {
		if err != nil {
			logc.Fatalln(ctx, err)
		}
		if record.Manifest != nil {
			err = traceManifest(record.GetAuditID().String(), record.Manifest)
			if err != nil {
				return fmt.Errorf("trace audit err: %w", err)
			}
		}
	}

	allBlobsCount, allBlobsSize := trieReduce(allBlobs)
	liveBlobsCount, liveBlobsSize := trieReduce(liveBlobs)
	logc.Printf(ctx, "trace all: %d blobs, %s",
		allBlobsCount, datasize.ByteSize(allBlobsSize).HR())
	logc.Printf(ctx, "trace live: %d blobs, %s",
		liveBlobsCount, datasize.ByteSize(liveBlobsSize).HR())
	logc.Printf(ctx, "trace dead: %d blobs, %s",
		allBlobsCount-liveBlobsCount, datasize.ByteSize(allBlobsSize-liveBlobsSize).HR())

	return nil
}