mirror of
https://codeberg.org/git-pages/git-pages.git
synced 2026-05-25 08:32:05 +00:00
Significantly improve efficiency of tracing.
I thought I was being smart by using a trie to record blob existence and sizes. I was not. The trie approach had at least ~5 times less throughput and consumed entirely unreasonable amounts of RAM. A hashmap works just fine here.
This commit is contained in:
@@ -5,30 +5,29 @@ import (
|
||||
"fmt"
|
||||
|
||||
"github.com/c2h5oh/datasize"
|
||||
"github.com/dghubble/trie"
|
||||
)
|
||||
|
||||
func trieReduce(data trie.Trier) (items, total int64) {
|
||||
data.Walk(func(key string, value any) error {
|
||||
items += 1
|
||||
total += *value.(*int64)
|
||||
return nil
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
func TraceGarbage(ctx context.Context) error {
|
||||
allBlobs := trie.NewRuneTrie()
|
||||
liveBlobs := trie.NewRuneTrie()
|
||||
allBlobs := map[string]int64{}
|
||||
liveBlobs := map[string]int64{}
|
||||
|
||||
traceManifest := func(manifestName string, manifest *Manifest) error {
|
||||
reduceBlobs := func(data map[string]int64) (items, total int64) {
|
||||
for _, value := range data {
|
||||
items += 1
|
||||
total += value
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
traceManifest := func(manifestKind string, manifestName string, manifest *Manifest) error {
|
||||
for _, entry := range manifest.GetContents() {
|
||||
if entry.GetType() == Type_ExternalFile {
|
||||
blobName := string(entry.Data)
|
||||
if size := allBlobs.Get(blobName); size == nil {
|
||||
return fmt.Errorf("%s: dangling reference %s", manifestName, blobName)
|
||||
if size, ok := allBlobs[blobName]; ok {
|
||||
liveBlobs[blobName] = size
|
||||
} else {
|
||||
liveBlobs.Put(blobName, size)
|
||||
logc.Printf(ctx, "trace manifest: %s/%s: dangling reference %s",
|
||||
manifestKind, manifestName, blobName)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -36,42 +35,44 @@ func TraceGarbage(ctx context.Context) error {
|
||||
}
|
||||
|
||||
// Enumerate all blobs.
|
||||
logc.Printf(ctx, "trace: enumerating blobs")
|
||||
for metadata, err := range backend.EnumerateBlobs(ctx) {
|
||||
if err != nil {
|
||||
return fmt.Errorf("trace blobs err: %w", err)
|
||||
}
|
||||
allBlobs.Put(metadata.Name, &metadata.Size)
|
||||
allBlobs[metadata.Name] = metadata.Size
|
||||
}
|
||||
|
||||
// Enumerate blobs live via site manifests.
|
||||
logc.Printf(ctx, "trace: enumerating manifests")
|
||||
for item, err := range backend.GetAllManifests(ctx) {
|
||||
metadata, manifest := item.Splat()
|
||||
if err != nil {
|
||||
return fmt.Errorf("trace sites err: %w", err)
|
||||
}
|
||||
err = traceManifest(metadata.Name, manifest)
|
||||
err = traceManifest("site", metadata.Name, manifest)
|
||||
if err != nil {
|
||||
return fmt.Errorf("trace sites err: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Enumerate blobs live via audit records.
|
||||
|
||||
logc.Printf(ctx, "trace: enumerating audit records")
|
||||
auditIDs := backend.SearchAuditLog(ctx, SearchAuditLogOptions{})
|
||||
for record, err := range backend.GetAuditLogRecords(ctx, auditIDs) {
|
||||
if err != nil {
|
||||
logc.Fatalln(ctx, err)
|
||||
return fmt.Errorf("trace audit err: %w", err)
|
||||
}
|
||||
if record.Manifest != nil {
|
||||
err = traceManifest(record.GetAuditID().String(), record.Manifest)
|
||||
err = traceManifest("audit", record.GetAuditID().String(), record.Manifest)
|
||||
if err != nil {
|
||||
return fmt.Errorf("trace audit err: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
allBlobsCount, allBlobsSize := trieReduce(allBlobs)
|
||||
liveBlobsCount, liveBlobsSize := trieReduce(liveBlobs)
|
||||
allBlobsCount, allBlobsSize := reduceBlobs(allBlobs)
|
||||
liveBlobsCount, liveBlobsSize := reduceBlobs(liveBlobs)
|
||||
logc.Printf(ctx, "trace all: %d blobs, %s",
|
||||
allBlobsCount, datasize.ByteSize(allBlobsSize).HR())
|
||||
logc.Printf(ctx, "trace live: %d blobs, %s",
|
||||
|
||||
Reference in New Issue
Block a user