diff --git a/src/analysis.go b/src/analysis.go new file mode 100644 index 0000000..5ed82e3 --- /dev/null +++ b/src/analysis.go @@ -0,0 +1,106 @@ +package git_pages + +import ( + "context" + "fmt" + "strings" +) + +type StorageSize struct { + Domain string `json:"domain"` + // Size of live storage for the current versions of sites. + CurrentSize int64 `json:"currentSize"` + // Size of live blobs (only) for the current version of sites. + CurrentBlobSize int64 `json:"-"` + // Size of live storage for the non-current versions of sites in audit records. + NonCurrentSize int64 `json:"nonCurrentSize"` + // Total size of live storage for this domain. + TotalSize int64 `json:"totalSize"` +} + +func AnalyzeStorage(ctx context.Context) ([]*StorageSize, error) { + type storageData struct { + siteManifests int64 + siteBlobs map[string]int64 + auditRecords int64 + auditBlobs map[string]int64 + } + + thickStats := map[string]*storageData{} + thinStats := []*StorageSize{} + + getStats := func(domain string) *storageData { + if _, found := thickStats[domain]; !found { + thickStats[domain] = &storageData{ + siteBlobs: map[string]int64{}, + auditBlobs: map[string]int64{}, + } + } + return thickStats[domain] + } + + totalStats := getStats("*") + + logc.Printf(ctx, "analyze: enumerating manifests") + for item, err := range backend.GetAllManifests(ctx) { + metadata, manifest := item.Splat() + if err != nil { + return nil, fmt.Errorf("analyze err: %w", err) + } + domain, _, _ := strings.Cut(metadata.Name, "/") + stats := getStats(domain) + stats.siteManifests += metadata.Size + totalStats.siteManifests += metadata.Size + for _, entry := range manifest.GetContents() { + if entry.GetType() == Type_ExternalFile { + blobName, blobSize := string(entry.Data), entry.GetCompressedSize() + stats.siteBlobs[blobName] = blobSize + totalStats.siteBlobs[blobName] = blobSize + } + } + } + + logc.Printf(ctx, "analyze: enumerating audit records") + auditIDs := backend.SearchAuditLog(ctx, SearchAuditLogOptions{}) + for record, err := range backend.GetAuditLogRecords(ctx, auditIDs) { + if err != nil { + return nil, fmt.Errorf("analyze err: %w", err) + } + domain := record.GetDomain() + stats := getStats(domain) + recordSize := int64(len(EncodeAuditRecord(record))) + stats.auditRecords += recordSize + totalStats.auditRecords += recordSize + if record.Manifest == nil || record.IsDetached() { + continue + } + for _, entry := range record.Manifest.GetContents() { + if entry.GetType() == Type_ExternalFile { + blobName, blobSize := string(entry.Data), entry.GetCompressedSize() + if _, found := stats.siteBlobs[blobName]; found { + continue // already accounted for + } + stats.auditBlobs[blobName] = entry.GetCompressedSize() + totalStats.auditBlobs[blobName] = blobSize + } + } + } + + // Now aggregate the information. + for domain, stats := range thickStats { + sizes := StorageSize{Domain: domain} + sizes.CurrentSize += stats.siteManifests + for _, size := range stats.siteBlobs { + sizes.CurrentSize += size + sizes.CurrentBlobSize += size + } + sizes.NonCurrentSize += stats.auditRecords + for _, size := range stats.auditBlobs { + sizes.NonCurrentSize += size + } + sizes.TotalSize = sizes.CurrentSize + sizes.NonCurrentSize + thinStats = append(thinStats, &sizes) + } + + return thinStats, nil +} diff --git a/src/histogram.go b/src/histogram.go deleted file mode 100644 index cf8c97f..0000000 --- a/src/histogram.go +++ /dev/null @@ -1,35 +0,0 @@ -package git_pages - -import ( - "context" - "fmt" - "maps" - "slices" - "strings" -) - -type DomainStatistics struct { - Domain string - OriginalSize int64 - CompressedSize int64 - StoredSize int64 -} - -func SizeHistogram(ctx context.Context) ([]*DomainStatistics, error) { - statisticsMap := map[string]*DomainStatistics{} - for item, err := range backend.GetAllManifests(ctx) { - metadata, manifest := item.Splat() - if err != nil { - return nil, fmt.Errorf("size histogram err: %w", err) - } - domain, _, _ := strings.Cut(metadata.Name, "/") - if _, found := statisticsMap[domain]; !found { - statisticsMap[domain] = &DomainStatistics{Domain: domain} - } - statistics := statisticsMap[domain] - statistics.OriginalSize += metadata.Size + manifest.GetOriginalSize() - statistics.CompressedSize += metadata.Size + manifest.GetCompressedSize() - statistics.StoredSize += metadata.Size + manifest.GetStoredSize() - } - return slices.Collect(maps.Values(statisticsMap)), nil -} diff --git a/src/main.go b/src/main.go index 5d5d054..61380bc 100644 --- a/src/main.go +++ b/src/main.go @@ -4,6 +4,7 @@ import ( "cmp" "context" "crypto/tls" + "encoding/json" "errors" "flag" "fmt" @@ -219,7 +220,7 @@ func usage() { fmt.Fprintf(os.Stderr, "(maint) "+ "git-pages -site-expire [-dry-run]\n") fmt.Fprintf(os.Stderr, "(maint) "+ - "git-pages {-run-migration |-trace-garbage|-size-histogram {original|stored}}\n") + "git-pages {-run-migration |-trace-garbage|-analyze-storage}\n") flag.PrintDefaults() } @@ -269,8 +270,8 @@ func Main(versionInfo string) { "expire sites according to their manifest") runMigration := flag.String("run-migration", "", "run a store `migration` (one of: create-domain-markers)") - sizeHistogram := flag.String("size-histogram", "", - "display histogram of `size-type` (original or stored) per domain") + analyzeStorage := flag.String("analyze-storage", "", + "display aggregate storage used per domain") traceGarbage := flag.Bool("trace-garbage", false, "estimate total size of unreachable blobs") dryRun := flag.Bool("dry-run", false, @@ -302,7 +303,7 @@ func Main(versionInfo string) { *auditServer != "", *siteExpire, *runMigration != "", - *sizeHistogram != "", + *analyzeStorage != "", *traceGarbage, } { if selected { @@ -313,7 +314,7 @@ func Main(versionInfo string) { logc.Fatalln(ctx, "-list-blobs, -list-manifests, -get-blob, -get-manifest, -get-archive, "+ "-update-site, -freeze-domain, -unfreeze-domain, -audit-log, -audit-read, "+ "-audit-rollback, -audit-expire, -audit-detach, -audit-server, -site-expire, "+ - "-run-migration, -size-histogram, and -trace-garbage are mutually exclusive") + "-run-migration, -analyze-storage, and -trace-garbage are mutually exclusive") } if *dryRun && !(*siteExpire) { logc.Fatalln(ctx, "-dry-run is not applicable in this context") @@ -536,10 +537,10 @@ func Main(versionInfo string) { for _, record := range records { parts := []string{ record.GetAuditID().String(), - color.HiWhiteString(record.GetTimestamp().AsTime().UTC().Format(time.RFC3339)), + color.HiWhiteString("%s", record.GetTimestamp().AsTime().UTC().Format(time.RFC3339)), fmt.Sprint(record.GetEvent()), - color.HiGreenString(record.DescribeResource()), - color.HiMagentaString(record.DescribePrincipal()), + color.HiMagentaString("%s", record.DescribeResource()), + color.HiGreenString("%s", record.DescribePrincipal()), } if record.IsDetached() { parts = append(parts, @@ -713,47 +714,73 @@ func Main(versionInfo string) { logc.Fatalln(ctx, err) } - case *sizeHistogram != "": - extractSize := func(s *DomainStatistics) int64 { return 0 } - switch *sizeHistogram { - case "original": - // Displays a size histogram using the `manifest.OriginalSize`, which is useful to see - // which site is the closest to hitting the size limit (checked against apparent size). - // This apparent size does not have any direct relationship with used storage. - extractSize = func(s *DomainStatistics) int64 { return s.OriginalSize } - case "stored": - // Displays a size histogram using the `manifest.StoredSize`, which is useful to see - // which site consumes the most resources. The site is keeping at least this many - // bytes worth of blobs alive, but removing it may not free any space because - // deduplication is global. - extractSize = func(s *DomainStatistics) int64 { return s.StoredSize } - default: - logc.Fatalln(ctx, "unknown histogram type") + case *analyzeStorage == "text": + // datasize.ByteSize.HR() is a little too wide for the 8-char column. + formatSize := func(b datasize.ByteSize) string { + switch { + case b > datasize.GB: + return fmt.Sprintf("%.1fG", b.GBytes()) + case b > datasize.MB: + return fmt.Sprintf("%.1fM", b.MBytes()) + case b > datasize.KB: + return fmt.Sprintf("%.1fK", b.KBytes()) + default: + return fmt.Sprintf("%dB", b) + } } - histogram, err := SizeHistogram(ctx) + analysis, err := AnalyzeStorage(ctx) if err != nil { logc.Fatalln(ctx, err) } - slices.SortFunc(histogram, func(a *DomainStatistics, b *DomainStatistics) int { - return cmp.Compare(extractSize(a), extractSize(b)) + slices.SortFunc(analysis, func(a *StorageSize, b *StorageSize) int { + return cmp.Compare(a.TotalSize, b.TotalSize) }) - if len(histogram) > 0 { - fullScaleSize := max(extractSize(histogram[len(histogram)-1]), 1) - fullScaleWidth := int64(40) - for _, statistics := range histogram { - size := extractSize(statistics) - barWidth := size * fullScaleWidth / fullScaleSize - spaceWidth := fullScaleWidth - barWidth - bar := strings.Repeat("*", int(barWidth)) + strings.Repeat(" ", int(spaceWidth)) - fmt.Fprintf(color.Output, "%s %s %s\n", - color.HiBlackString(fmt.Sprint("|", bar, "|")), - statistics.Domain, - color.HiGreenString(datasize.ByteSize(extractSize(statistics)).HR()), + for _, sizes := range analysis { + var colorize func(string, ...interface{}) string + fractionSize := + float32(sizes.CurrentBlobSize) / float32(config.Limits.MaxSiteSize.Bytes()) + switch { + case fractionSize > 0.9: + colorize = color.HiRedString + case fractionSize > 0.7: + colorize = color.HiYellowString + case fractionSize > 0.1: + colorize = color.HiGreenString + default: + colorize = color.HiWhiteString + } + if sizes.Domain != "*" { + fmt.Fprintf(color.Output, "%s\t%s\t%s\t%s\t%s\n", + colorize("%.0f%%", fractionSize*100.0), + colorize("%s", formatSize(datasize.ByteSize(sizes.CurrentSize))), + formatSize(datasize.ByteSize(sizes.NonCurrentSize)), + formatSize(datasize.ByteSize(sizes.TotalSize)), + color.HiMagentaString("%s", sizes.Domain), + ) + } else { + fmt.Fprintf(color.Output, "---\t%s\t%s\t%s\t%s\n", + color.HiCyanString("%s", formatSize(datasize.ByteSize(sizes.CurrentSize))), + formatSize(datasize.ByteSize(sizes.NonCurrentSize)), + formatSize(datasize.ByteSize(sizes.TotalSize)), + color.HiMagentaString("%s", sizes.Domain), ) } } + fmt.Fprintf(color.Output, "Quota%%\tCurrent\tNonCurr\tTotal\tDomain\n") + + case *analyzeStorage == "json": + analysis, err := AnalyzeStorage(ctx) + if err != nil { + logc.Fatalln(ctx, err) + } + + encoder := json.NewEncoder(os.Stdout) + encoder.Encode(analysis) + + case *analyzeStorage != "": + logc.Fatalf(ctx, "unsupported -analyze-storage mode") case *traceGarbage: if err = TraceGarbage(ctx); err != nil {