git-pages/src/existence.go

// The existence cache allows fast rejection of requests for nonexistent domains or sites.
// This is principally important for floods of crawler requests which probe either random
// domains (with A/AAAA records not pointing to the git-pages host), or random sites
// (typically as a result of probing for vulnerable URLs). With the S3 backend this can
// result in severe congestion of the backend channel and high CPU use.

package git_pages

import (
	"context"
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/bits-and-blooms/bloom/v3"
)

type Existence int

const (
	ExistenceImpossible Existence = iota
	ExistencePossible
)

func (value Existence) IsImpossible() bool {
	return value == ExistenceImpossible
}

func (value Existence) IsPossible() bool {
	return value == ExistencePossible
}

func (value Existence) String() string {
	switch value {
	case ExistenceImpossible:
		return "impossible"
	case ExistencePossible:
		return "possible"
	default:
		return "(invalid)"
	}
}

type ExistenceCache interface {
	// Check if we might be serving the site.
	CheckSite(ctx context.Context, site string) (result Existence)

	// Check if we might be serving the domain.
	CheckDomain(ctx context.Context, domain string) (result Existence)

	// Add the site to the cache.
	AddSite(ctx context.Context, site string)
}

func CreateExistenceCache(ctx context.Context) (ExistenceCache, error) {
	switch config.Storage.Type {
	case "s3":
		maxAge := time.Duration(config.Storage.S3.SiteCache.MaxAge)
		return createBloomExistenceCache(ctx, maxAge)
	default:
		// not needed
	}

	return &dummyExistenceCache{}, nil
}

type bloomExistenceCache struct {
	sites    *bloom.BloomFilter
	domains  *bloom.BloomFilter
	filterMu sync.Mutex

	accessCh    chan struct{}
	refreshMu   sync.Mutex
	lastRefresh time.Time
	maxAge      time.Duration
}

func createBloomExistenceCache(ctx context.Context, maxAge time.Duration) (ExistenceCache, error) {
	cache := bloomExistenceCache{
		accessCh: make(chan struct{}),
		maxAge:   maxAge,
	}

	if err := cache.refresh(ctx); err != nil {
		return nil, err
	}

	go cache.handleFilterUpdates(ctx)
	return &cache, nil
}

func (c *bloomExistenceCache) handleFilterUpdates(ctx context.Context) {
	for range c.accessCh {
		if time.Since(c.lastRefresh) > c.maxAge {
			logc.Println(ctx, "existence: refreshing")
			if err := c.refresh(ctx); err != nil {
				logc.Printf(ctx, "existence: refresh error: %v", err)
			}
		}
	}
}

func (c *bloomExistenceCache) refresh(ctx context.Context) error {
	c.refreshMu.Lock()
	defer c.refreshMu.Unlock()

	if changed, err := backend.HasSiteListChanged(ctx, c.lastRefresh); err != nil {
		return err
	} else if !changed {
		logc.Println(ctx, "existence: unchanged")
		c.lastRefresh = time.Now()
		return nil
	}

	// Create two 256 KiB Bloom filters that will fit ~150K entries each with 0.1% false positive rate.
	sites := bloom.New(256*1024*8, 10)
	domains := bloom.New(256*1024*8, 10)

	logc.Printf(ctx, "existence: refreshing")
	siteCount := 0
	for metadata, err := range backend.EnumerateManifests(ctx) {
		if err != nil {
			return fmt.Errorf("enum manifests: %w", err)
		}
		site := metadata.Name
		domain, _, _ := strings.Cut(site, "/")
		sites.AddString(site)
		domains.AddString(domain)
		siteCount++
	}

	c.filterMu.Lock()
	c.sites = sites
	c.domains = domains
	c.filterMu.Unlock()

	logc.Printf(ctx, "existence: refreshed with %d sites", siteCount)
	c.lastRefresh = time.Now()
	return nil
}

func (c *bloomExistenceCache) CheckSite(ctx context.Context, site string) (result Existence) {
	select {
	case c.accessCh <- struct{}{}:
	default:
	}

	c.filterMu.Lock()
	if c.sites.TestString(site) {
		result = ExistencePossible
	} else {
		result = ExistenceImpossible
	}
	c.filterMu.Unlock()

	logc.Debugf(ctx, "existence: site %s: %s", site, result)
	return
}

func (c *bloomExistenceCache) CheckDomain(ctx context.Context, domain string) (result Existence) {
	select {
	case c.accessCh <- struct{}{}:
	default:
	}

	c.filterMu.Lock()
	if c.domains.TestString(domain) {
		result = ExistencePossible
	} else {
		result = ExistenceImpossible
	}
	c.filterMu.Unlock()

	logc.Debugf(ctx, "existence: domain %s: %s", domain, result)
	return
}

func (c *bloomExistenceCache) AddSite(ctx context.Context, site string) {
	c.refreshMu.Lock()
	defer c.refreshMu.Unlock()

	domain, _, _ := strings.Cut(site, "/")

	c.filterMu.Lock()
	c.sites.AddString(site)
	c.domains.AddString(domain)
	c.filterMu.Unlock()

	logc.Printf(ctx, "existence: added site %s", site)
}

type dummyExistenceCache struct{}

func (d dummyExistenceCache) CheckSite(context.Context, string) Existence {
	return ExistencePossible
}

func (d dummyExistenceCache) CheckDomain(context.Context, string) Existence {
	return ExistencePossible
}

func (d dummyExistenceCache) AddSite(context.Context, string) {}