git-pages/src/analysis.go

package git_pages

import (
	"context"
	"fmt"
	"strings"
)

type StorageSize struct {
	Domain string `json:"domain"`
	// Size of live storage for the current versions of sites.
	CurrentSize int64 `json:"currentSize"`
	// Size of live blobs (only) for the current version of sites.
	CurrentBlobSize int64 `json:"-"`
	// Size of live storage for the non-current versions of sites in audit records.
	NonCurrentSize int64 `json:"nonCurrentSize"`
	// Total size of live storage for this domain.
	TotalSize int64 `json:"totalSize"`
}

func AnalyzeStorage(ctx context.Context) ([]*StorageSize, error) {
	type storageData struct {
		siteManifests int64
		siteBlobs     map[string]int64
		auditRecords  int64
		auditBlobs    map[string]int64
	}

	thickStats := map[string]*storageData{}
	thinStats := []*StorageSize{}

	getStats := func(domain string) *storageData {
		if _, found := thickStats[domain]; !found {
			thickStats[domain] = &storageData{
				siteBlobs:  map[string]int64{},
				auditBlobs: map[string]int64{},
			}
		}
		return thickStats[domain]
	}

	totalStats := getStats("*")

	logc.Printf(ctx, "analyze: enumerating manifests")
	for item, err := range backend.GetAllManifests(ctx) {
		metadata, manifest := item.Splat()
		if err != nil {
			return nil, fmt.Errorf("analyze err: %w", err)
		}
		domain, _, _ := strings.Cut(metadata.Name, "/")
		stats := getStats(domain)
		stats.siteManifests += metadata.Size
		totalStats.siteManifests += metadata.Size
		for _, entry := range manifest.GetContents() {
			if entry.GetType() == Type_ExternalFile {
				blobName, blobSize := string(entry.Data), entry.GetCompressedSize()
				stats.siteBlobs[blobName] = blobSize
				totalStats.siteBlobs[blobName] = blobSize
			}
		}
	}

	logc.Printf(ctx, "analyze: enumerating audit records")
	auditIDs := backend.SearchAuditLog(ctx, SearchAuditLogOptions{})
	for record, err := range backend.GetAuditLogRecords(ctx, auditIDs) {
		if err != nil {
			return nil, fmt.Errorf("analyze err: %w", err)
		}
		domain := record.GetDomain()
		stats := getStats(domain)
		recordSize := int64(len(EncodeAuditRecord(record)))
		stats.auditRecords += recordSize
		totalStats.auditRecords += recordSize
		if record.Manifest == nil || record.IsDetached() {
			continue
		}
		for _, entry := range record.Manifest.GetContents() {
			if entry.GetType() == Type_ExternalFile {
				blobName, blobSize := string(entry.Data), entry.GetCompressedSize()
				if _, found := stats.siteBlobs[blobName]; found {
					continue // already accounted for
				}
				stats.auditBlobs[blobName] = entry.GetCompressedSize()
				totalStats.auditBlobs[blobName] = blobSize
			}
		}
	}

	// Now aggregate the information.
	for domain, stats := range thickStats {
		sizes := StorageSize{Domain: domain}
		sizes.CurrentSize += stats.siteManifests
		for _, size := range stats.siteBlobs {
			sizes.CurrentSize += size
			sizes.CurrentBlobSize += size
		}
		sizes.NonCurrentSize += stats.auditRecords
		for _, size := range stats.auditBlobs {
			sizes.NonCurrentSize += size
		}
		sizes.TotalSize = sizes.CurrentSize + sizes.NonCurrentSize
		thinStats = append(thinStats, &sizes)
	}

	return thinStats, nil
}