Significantly improve efficiency of tracing.

I thought I was being smart by using a trie to record blob existence
and sizes. I was not. The trie approach had at least ~5 times less
throughput and consumed entirely unreasonable amounts of RAM.

A hashmap works just fine here.
This commit is contained in:
Catherine
2026-05-05 01:57:40 +00:00
parent dd7268a657
commit 73e47cd8d5
4 changed files with 24 additions and 29 deletions

View File

@@ -5,30 +5,29 @@ import (
"fmt"
"github.com/c2h5oh/datasize"
"github.com/dghubble/trie"
)
func trieReduce(data trie.Trier) (items, total int64) {
data.Walk(func(key string, value any) error {
items += 1
total += *value.(*int64)
return nil
})
return
}
func TraceGarbage(ctx context.Context) error {
allBlobs := trie.NewRuneTrie()
liveBlobs := trie.NewRuneTrie()
allBlobs := map[string]int64{}
liveBlobs := map[string]int64{}
traceManifest := func(manifestName string, manifest *Manifest) error {
reduceBlobs := func(data map[string]int64) (items, total int64) {
for _, value := range data {
items += 1
total += value
}
return
}
traceManifest := func(manifestKind string, manifestName string, manifest *Manifest) error {
for _, entry := range manifest.GetContents() {
if entry.GetType() == Type_ExternalFile {
blobName := string(entry.Data)
if size := allBlobs.Get(blobName); size == nil {
return fmt.Errorf("%s: dangling reference %s", manifestName, blobName)
if size, ok := allBlobs[blobName]; ok {
liveBlobs[blobName] = size
} else {
liveBlobs.Put(blobName, size)
logc.Printf(ctx, "trace manifest: %s/%s: dangling reference %s",
manifestKind, manifestName, blobName)
}
}
}
@@ -36,42 +35,44 @@ func TraceGarbage(ctx context.Context) error {
}
// Enumerate all blobs.
logc.Printf(ctx, "trace: enumerating blobs")
for metadata, err := range backend.EnumerateBlobs(ctx) {
if err != nil {
return fmt.Errorf("trace blobs err: %w", err)
}
allBlobs.Put(metadata.Name, &metadata.Size)
allBlobs[metadata.Name] = metadata.Size
}
// Enumerate blobs live via site manifests.
logc.Printf(ctx, "trace: enumerating manifests")
for item, err := range backend.GetAllManifests(ctx) {
metadata, manifest := item.Splat()
if err != nil {
return fmt.Errorf("trace sites err: %w", err)
}
err = traceManifest(metadata.Name, manifest)
err = traceManifest("site", metadata.Name, manifest)
if err != nil {
return fmt.Errorf("trace sites err: %w", err)
}
}
// Enumerate blobs live via audit records.
logc.Printf(ctx, "trace: enumerating audit records")
auditIDs := backend.SearchAuditLog(ctx, SearchAuditLogOptions{})
for record, err := range backend.GetAuditLogRecords(ctx, auditIDs) {
if err != nil {
logc.Fatalln(ctx, err)
return fmt.Errorf("trace audit err: %w", err)
}
if record.Manifest != nil {
err = traceManifest(record.GetAuditID().String(), record.Manifest)
err = traceManifest("audit", record.GetAuditID().String(), record.Manifest)
if err != nil {
return fmt.Errorf("trace audit err: %w", err)
}
}
}
allBlobsCount, allBlobsSize := trieReduce(allBlobs)
liveBlobsCount, liveBlobsSize := trieReduce(liveBlobs)
allBlobsCount, allBlobsSize := reduceBlobs(allBlobs)
liveBlobsCount, liveBlobsSize := reduceBlobs(liveBlobs)
logc.Printf(ctx, "trace all: %d blobs, %s",
allBlobsCount, datasize.ByteSize(allBlobsSize).HR())
logc.Printf(ctx, "trace live: %d blobs, %s",