Add site-level granularity to the domain existence cache.

src/backend.go

@@ -132,6 +132,9 @@ type Backend interface {
 	// `EnumerateManifests`.
 	GetAllManifests(ctx context.Context) iter.Seq2[tuple[*ManifestMetadata, *Manifest], error]
 
+	// Check whether the set of sites we serve has changed since the time passed to this method.
+	HasSiteListChanged(ctx context.Context, since time.Time) (changed bool, err error)
+
 	// Check whether a domain has any deployments.
 	CheckDomain(ctx context.Context, domain string) (found bool, err error)
 
@@ -145,9 +148,6 @@ type Backend interface {
 	// Thaw a domain. This removes the previously placed administrative lock (if any).
 	UnfreezeDomain(ctx context.Context, domain string) error
 
-	// Check whether the set of domains we serve has changed since the time passed to this method.
-	HaveDomainsChanged(ctx context.Context, since time.Time) (changed bool, err error)
-
 	// Append a record to the audit log.
 	AppendAuditLog(ctx context.Context, id AuditID, record *AuditRecord) error
 

src/backend_fs.go

@@ -480,7 +480,7 @@ func (fs *FSBackend) UnfreezeDomain(ctx context.Context, domain string) error {
 	}
 }
 
-func (fs *FSBackend) HaveDomainsChanged(ctx context.Context, since time.Time) (bool, error) {
+func (fs *FSBackend) HasSiteListChanged(ctx context.Context, since time.Time) (bool, error) {
 	return true, nil // not implemented
 }
 

src/backend_s3.go

@@ -563,25 +563,28 @@ func (s3 *S3Backend) HasAtomicCAS(ctx context.Context) bool {
 
 func (s3 *S3Backend) checkManifestPrecondition(
 	ctx context.Context, name string, opts ModifyManifestOptions,
-) error {
-	if opts.IfUnmodifiedSince.IsZero() && opts.IfMatch == "" {
-		return nil
-	}
-
+) (exists bool, err error) {
 	stat, err := s3.client.StatObject(ctx, s3.bucket, manifestObjectName(name),
 		minio.GetObjectOptions{})
 	if err != nil {
-		return err
+		errResp := minio.ToErrorResponse(err)
+		if opts.IfUnmodifiedSince.IsZero() && opts.IfMatch == "" && errResp.Code == "NoSuchKey" {
+			exists = false
+		} else {
+			return false, err
+		}
+	} else {
+		exists = true
 	}
 
 	if !opts.IfUnmodifiedSince.IsZero() && stat.LastModified.Compare(opts.IfUnmodifiedSince) > 0 {
-		return fmt.Errorf("%w: If-Unmodified-Since", ErrPreconditionFailed)
+		return exists, fmt.Errorf("%w: If-Unmodified-Since", ErrPreconditionFailed)
 	}
 	if opts.IfMatch != "" && stat.ETag != opts.IfMatch {
-		return fmt.Errorf("%w: If-Match", ErrPreconditionFailed)
+		return exists, fmt.Errorf("%w: If-Match", ErrPreconditionFailed)
 	}
 
-	return nil
+	return exists, nil
 }
 
 func (s3 *S3Backend) CommitManifest(
@@ -595,7 +598,8 @@ func (s3 *S3Backend) CommitManifest(
 		return err
 	}
 
-	if err := s3.checkManifestPrecondition(ctx, name, opts); err != nil {
+	existed, err := s3.checkManifestPrecondition(ctx, name, opts)
+	if err != nil {
 		return err
 	}
 
@@ -622,9 +626,15 @@ func (s3 *S3Backend) CommitManifest(
 		}
 	} else if removeErr != nil {
 		return removeErr
-	} else {
-		return nil
 	}
+
+	if !existed {
+		if err := s3.bumpLastSiteUpdateTimestamp(ctx); err != nil {
+			return err
+		}
+	}
+
+	return nil
 }
 
 func (s3 *S3Backend) DeleteManifest(
@@ -637,17 +647,25 @@ func (s3 *S3Backend) DeleteManifest(
 		return err
 	}
 
-	if err := s3.checkManifestPrecondition(ctx, name, opts); err != nil {
+	existed, err := s3.checkManifestPrecondition(ctx, name, opts)
+	if err != nil {
 		return err
 	}
 
-	err := s3.client.RemoveObject(ctx, s3.bucket, manifestObjectName(name),
+	err = s3.client.RemoveObject(ctx, s3.bucket, manifestObjectName(name),
 		minio.RemoveObjectOptions{})
 	if err != nil {
 		return err
 	}
 	s3.siteCache.Cache.Invalidate(name)
-	return s3.bumpLastDomainUpdateTimestamp(ctx)
+
+	if existed {
+		if err := s3.bumpLastSiteUpdateTimestamp(ctx); err != nil {
+			return err
+		}
+	}
+
+	return nil
 }
 
 func (s3 *S3Backend) EnumerateManifests(ctx context.Context) iter.Seq2[*ManifestMetadata, error] {
@@ -767,19 +785,8 @@ func (s3 *S3Backend) CheckDomain(ctx context.Context, domain string) (exists boo
 func (s3 *S3Backend) CreateDomain(ctx context.Context, domain string) error {
 	logc.Printf(ctx, "s3: create domain %s\n", domain)
 
-	exists, err := s3.CheckDomain(ctx, domain)
-	if err != nil {
-		return err
-	}
-
-	_, err = s3.client.PutObject(ctx, s3.bucket, domainCheckObjectName(domain),
+	_, err := s3.client.PutObject(ctx, s3.bucket, domainCheckObjectName(domain),
 		&bytes.Reader{}, 0, minio.PutObjectOptions{})
-	if err != nil {
-		return err
-	}
-	if !exists {
-		err = s3.bumpLastDomainUpdateTimestamp(ctx)
-	}
 	return err
 }
 
@@ -789,7 +796,6 @@ func (s3 *S3Backend) FreezeDomain(ctx context.Context, domain string) error {
 	_, err := s3.client.PutObject(ctx, s3.bucket, domainFrozenObjectName(domain),
 		&bytes.Reader{}, 0, minio.PutObjectOptions{})
 	return err
-
 }
 
 func (s3 *S3Backend) UnfreezeDomain(ctx context.Context, domain string) error {
@@ -804,21 +810,24 @@ func (s3 *S3Backend) UnfreezeDomain(ctx context.Context, domain string) error {
 	}
 }
 
-const lastDomainUpdateObjectName = "meta/last-domain-update"
+const lastSiteUpdateObjectName = "meta/last-site-update"
 
-func (s3 *S3Backend) HaveDomainsChanged(ctx context.Context, since time.Time) (bool, error) {
-	info, err := s3.client.StatObject(ctx, s3.bucket, lastDomainUpdateObjectName,
+func (s3 *S3Backend) HasSiteListChanged(ctx context.Context, since time.Time) (bool, error) {
+	info, err := s3.client.StatObject(ctx, s3.bucket, lastSiteUpdateObjectName,
 		minio.GetObjectOptions{})
 	if err != nil {
+		if errResp := minio.ToErrorResponse(err); errResp.Code == "NoSuchKey" {
+			return true, nil
+		}
 		return false, err
 	}
 
 	return info.LastModified.After(since), nil
 }
 
-func (s3 *S3Backend) bumpLastDomainUpdateTimestamp(ctx context.Context) error {
-	logc.Print(ctx, "s3: bumping last domain update timestamp")
-	_, err := s3.client.PutObject(ctx, s3.bucket, lastDomainUpdateObjectName,
+func (s3 *S3Backend) bumpLastSiteUpdateTimestamp(ctx context.Context) error {
+	logc.Print(ctx, "s3: bumping last site update timestamp")
+	_, err := s3.client.PutObject(ctx, s3.bucket, lastSiteUpdateObjectName,
 		&bytes.Reader{}, 0, minio.PutObjectOptions{})
 	return err
 }

src/caddy.go

@@ -30,7 +30,7 @@ func ServeCaddy(w http.ResponseWriter, r *http.Request) {
 	domain = strings.ToLower(domain)
 
 	// Run a cheap check as to whether we might be serving the domain.
-	var found = domainCache.CheckDomain(r.Context(), domain)
+	var found = siteExistenceCache.CheckDomain(r.Context(), domain)
 
 	if found {
 		// Run an expensive check as to whether we are actually serving the domain.

src/domain_cache.go

@@ -1,132 +0,0 @@
-package git_pages
-
-import (
-	"context"
-	"fmt"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/bits-and-blooms/bloom/v3"
-)
-
-type DomainCache interface {
-	// Check if we might be serving the domain.
-	CheckDomain(ctx context.Context, domain string) (found bool)
-
-	// Add the domain to the cache.
-	AddDomain(ctx context.Context, domain string)
-}
-
-func CreateDomainCache(ctx context.Context) (DomainCache, error) {
-	if !config.Feature("domain-existence-cache") {
-		return &dummyDomainCache{}, nil
-	}
-	return createBloomDomainCache(ctx)
-}
-
-type bloomDomainCache struct {
-	filter   *bloom.BloomFilter
-	filterMu sync.Mutex
-
-	accessCh    chan struct{}
-	refreshMu   sync.Mutex
-	lastRefresh time.Time
-	maxAge      time.Duration
-}
-
-func createBloomDomainCache(ctx context.Context) (DomainCache, error) {
-	cache := bloomDomainCache{
-		accessCh: make(chan struct{}),
-	}
-
-	switch config.Storage.Type {
-	case "fs":
-		// the FS backend has no cache
-	case "s3":
-		cache.maxAge = time.Duration(config.Storage.S3.SiteCache.MaxAge)
-	default:
-		panic(fmt.Errorf("unknown backend: %s", config.Storage.Type))
-	}
-
-	if err := cache.refresh(ctx); err != nil {
-		return nil, err
-	}
-
-	go cache.handleFilterUpdates(ctx)
-
-	return &cache, nil
-}
-
-func (c *bloomDomainCache) handleFilterUpdates(ctx context.Context) {
-	for range c.accessCh {
-		if time.Since(c.lastRefresh) > c.maxAge {
-			logc.Print(ctx, "domain cache: refreshing")
-			if err := c.refresh(ctx); err != nil {
-				logc.Printf(ctx, "domain cache: refresh error: %v", err)
-			}
-		}
-	}
-}
-
-func (c *bloomDomainCache) refresh(ctx context.Context) error {
-	c.refreshMu.Lock()
-	defer c.refreshMu.Unlock()
-
-	if changed, err := backend.HaveDomainsChanged(ctx, c.lastRefresh); err != nil {
-		return err
-	} else if !changed {
-		logc.Print(ctx, "domain cache: unchanged")
-		c.lastRefresh = time.Now()
-		return nil
-	}
-
-	// Create a 256 KiB Bloom filter that will fit ~150K entries with 0.1% false positive rate.
-	filter := bloom.New(256*1024, 10)
-	for metadata, err := range backend.EnumerateManifests(ctx) {
-		if err != nil {
-			return fmt.Errorf("enum manifests: %w", err)
-		}
-		domain, _, _ := strings.Cut(metadata.Name, "/")
-		filter.AddString(domain)
-	}
-
-	c.filterMu.Lock()
-	c.filter = filter
-	c.filterMu.Unlock()
-
-	logc.Printf(ctx, "domain cache: refreshed with approx. %d domains", filter.ApproximatedSize())
-	c.lastRefresh = time.Now()
-	return nil
-}
-
-func (c *bloomDomainCache) CheckDomain(ctx context.Context, domain string) (found bool) {
-	select {
-	case c.accessCh <- struct{}{}:
-	default:
-	}
-
-	c.filterMu.Lock()
-	found = c.filter.TestString(domain)
-	c.filterMu.Unlock()
-
-	logc.Printf(ctx, "domain cache: bloom filter returns %v for %q", found, domain)
-	return
-}
-
-func (c *bloomDomainCache) AddDomain(ctx context.Context, domain string) {
-	c.refreshMu.Lock()
-	defer c.refreshMu.Unlock()
-
-	c.filterMu.Lock()
-	c.filter.AddString(domain)
-	c.filterMu.Unlock()
-
-	logc.Printf(ctx, "domain cache: added %q", domain)
-}
-
-type dummyDomainCache struct{}
-
-func (d dummyDomainCache) CheckDomain(context.Context, string) bool { return true }
-
-func (d dummyDomainCache) AddDomain(context.Context, string) {}

src/existence_cache.go

@@ -0,0 +1,161 @@
+package git_pages
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/bits-and-blooms/bloom/v3"
+)
+
+type SiteExistenceCache interface {
+	// Check if we might be serving the site.
+	CheckSite(ctx context.Context, site string) (found bool)
+
+	// Check if we might be serving the domain.
+	CheckDomain(ctx context.Context, domain string) (found bool)
+
+	// Add the site to the cache.
+	AddSite(ctx context.Context, site string)
+}
+
+func CreateSiteExistenceCache(ctx context.Context) (SiteExistenceCache, error) {
+	if !config.Feature("site-existence-cache") {
+		return &dummySiteExistenceCache{}, nil
+	}
+	return createBloomSiteExistenceCache(ctx)
+}
+
+type bloomSiteExistenceCache struct {
+	sites    *bloom.BloomFilter
+	domains  *bloom.BloomFilter
+	filterMu sync.Mutex
+
+	accessCh    chan struct{}
+	refreshMu   sync.Mutex
+	lastRefresh time.Time
+	maxAge      time.Duration
+}
+
+func createBloomSiteExistenceCache(ctx context.Context) (SiteExistenceCache, error) {
+	cache := bloomSiteExistenceCache{
+		accessCh: make(chan struct{}),
+	}
+
+	switch config.Storage.Type {
+	case "fs":
+		// the FS backend has no cache
+	case "s3":
+		cache.maxAge = time.Duration(config.Storage.S3.SiteCache.MaxAge)
+	default:
+		panic(fmt.Errorf("unknown backend: %s", config.Storage.Type))
+	}
+
+	if err := cache.refresh(ctx); err != nil {
+		return nil, err
+	}
+
+	go cache.handleFilterUpdates(ctx)
+
+	return &cache, nil
+}
+
+func (c *bloomSiteExistenceCache) handleFilterUpdates(ctx context.Context) {
+	for range c.accessCh {
+		if time.Since(c.lastRefresh) > c.maxAge {
+			logc.Print(ctx, "site existence cache: refreshing")
+			if err := c.refresh(ctx); err != nil {
+				logc.Printf(ctx, "site existence cache: refresh error: %v", err)
+			}
+		}
+	}
+}
+
+func (c *bloomSiteExistenceCache) refresh(ctx context.Context) error {
+	c.refreshMu.Lock()
+	defer c.refreshMu.Unlock()
+
+	if changed, err := backend.HasSiteListChanged(ctx, c.lastRefresh); err != nil {
+		return err
+	} else if !changed {
+		logc.Print(ctx, "site existence cache: unchanged")
+		c.lastRefresh = time.Now()
+		return nil
+	}
+
+	var siteCount int
+	// Create two 256 KiB Bloom filters that will fit ~150K entries each with 0.1% false positive rate.
+	sites := bloom.New(256*1024, 10)
+	domains := bloom.New(256*1024, 10)
+	for metadata, err := range backend.EnumerateManifests(ctx) {
+		if err != nil {
+			return fmt.Errorf("enum manifests: %w", err)
+		}
+		site := metadata.Name
+		domain, _, _ := strings.Cut(site, "/")
+		sites.AddString(site)
+		domains.AddString(domain)
+		siteCount++
+	}
+
+	c.filterMu.Lock()
+	c.sites = sites
+	c.domains = domains
+	c.filterMu.Unlock()
+
+	logc.Printf(ctx, "site existence cache: refreshed with %d sites", siteCount)
+	c.lastRefresh = time.Now()
+	return nil
+}
+
+func (c *bloomSiteExistenceCache) CheckSite(ctx context.Context, site string) (found bool) {
+	select {
+	case c.accessCh <- struct{}{}:
+	default:
+	}
+
+	c.filterMu.Lock()
+	found = c.sites.TestString(site)
+	c.filterMu.Unlock()
+
+	logc.Printf(ctx, "site existence cache: bloom filter returns %v for site %q", found, site)
+	return
+}
+
+func (c *bloomSiteExistenceCache) CheckDomain(ctx context.Context, domain string) (found bool) {
+	select {
+	case c.accessCh <- struct{}{}:
+	default:
+	}
+
+	c.filterMu.Lock()
+	found = c.domains.TestString(domain)
+	c.filterMu.Unlock()
+
+	logc.Printf(ctx, "site existence cache: bloom filter returns %v for domain %q", found, domain)
+	return
+}
+
+func (c *bloomSiteExistenceCache) AddSite(ctx context.Context, site string) {
+	c.refreshMu.Lock()
+	defer c.refreshMu.Unlock()
+
+	domain, _, _ := strings.Cut(site, "/")
+
+	c.filterMu.Lock()
+	c.sites.AddString(site)
+	c.domains.AddString(domain)
+	c.filterMu.Unlock()
+
+	logc.Printf(ctx, "site existence cache: added site %q", site)
+}
+
+type dummySiteExistenceCache struct{}
+
+func (d dummySiteExistenceCache) CheckSite(context.Context, string) bool { return true }
+
+func (d dummySiteExistenceCache) CheckDomain(context.Context, string) bool { return true }
+
+func (d dummySiteExistenceCache) AddSite(context.Context, string) {}

src/main.go

@@ -34,7 +34,7 @@ var config *Config
 var wildcards []*WildcardPattern
 var fallback http.Handler
 var backend Backend
-var domainCache DomainCache
+var siteExistenceCache SiteExistenceCache
 
 func configureFeatures(ctx context.Context) (err error) {
 	if len(config.Features) > 0 {
@@ -345,7 +345,7 @@ func Main(versionInfo string) {
 			logc.Fatalln(ctx, err)
 		}
 
-		if domainCache, err = CreateDomainCache(ctx); err != nil {
+		if siteExistenceCache, err = CreateSiteExistenceCache(ctx); err != nil {
 			logc.Fatalln(ctx, err)
 		}
 	}
@@ -742,7 +742,7 @@ func Main(versionInfo string) {
 		}
 		backend = NewObservedBackend(backend)
 
-		if domainCache, err = CreateDomainCache(ctx); err != nil {
+		if siteExistenceCache, err = CreateSiteExistenceCache(ctx); err != nil {
 			logc.Fatalln(ctx, err)
 		}
 

src/observe.go

@@ -346,9 +346,9 @@ func (backend *observedBackend) UnfreezeDomain(ctx context.Context, domain strin
 	return
 }
 
-func (backend *observedBackend) HaveDomainsChanged(ctx context.Context, since time.Time) (changed bool, err error) {
-	span, ctx := ObserveFunction(ctx, "HaveDomainsChanged", "since", since)
-	changed, err = backend.inner.HaveDomainsChanged(ctx, since)
+func (backend *observedBackend) HasSiteListChanged(ctx context.Context, since time.Time) (changed bool, err error) {
+	span, ctx := ObserveFunction(ctx, "HasSiteListChanged", "since", since)
+	changed, err = backend.inner.HasSiteListChanged(ctx, since)
 	span.Finish()
 	return
 }

src/pages.go

@@ -129,13 +129,7 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
 	if err != nil {
 		return err
 	}
-
 	host = normalizeHost(host)
-	if !domainCache.CheckDomain(r.Context(), host) {
-		w.WriteHeader(http.StatusNotFound)
-		fmt.Fprintf(w, "site not found\n")
-		return nil
-	}
 
 	type indexManifestResult struct {
 		manifest *Manifest
@@ -144,8 +138,13 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
 	}
 	indexManifestCh := make(chan indexManifestResult, 1)
 	go func() {
+		webRoot := makeWebRoot(host, ".index")
+		if !siteExistenceCache.CheckSite(r.Context(), webRoot) {
+			close(indexManifestCh)
+			return
+		}
 		manifest, metadata, err := backend.GetManifest(
-			r.Context(), makeWebRoot(host, ".index"),
+			r.Context(), webRoot,
 			GetManifestOptions{BypassCache: bypassCache},
 		)
 		indexManifestCh <- (indexManifestResult{manifest, metadata, err})
@@ -154,11 +153,12 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
 	err = nil
 	sitePath = strings.TrimPrefix(r.URL.Path, "/")
 	if projectName, projectPath, hasProjectSlash := strings.Cut(sitePath, "/"); projectName != "" {
-		if ValidateProjectName(projectName) == nil {
+		webRoot := makeWebRoot(host, projectName)
+		if ValidateProjectName(projectName) == nil && siteExistenceCache.CheckSite(r.Context(), webRoot) {
 			var projectManifest *Manifest
 			var projectMetadata ManifestMetadata
 			projectManifest, projectMetadata, err = backend.GetManifest(
-				r.Context(), makeWebRoot(host, projectName),
+				r.Context(), webRoot,
 				GetManifestOptions{BypassCache: bypassCache},
 			)
 			if err == nil {
@@ -173,7 +173,7 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
 	if manifest == nil && (err == nil || errors.Is(err, ErrObjectNotFound)) {
 		result := <-indexManifestCh
 		manifest, metadata, err = result.manifest, result.metadata, result.err
-		if manifest == nil && errors.Is(err, ErrObjectNotFound) {
+		if manifest == nil && (err == nil || errors.Is(err, ErrObjectNotFound)) {
 			if fallback != nil {
 				logc.Printf(r.Context(), "fallback: %s via %s", host, config.Fallback.ProxyTo)
 				fallback.ServeHTTP(w, r)

src/update.go

@@ -59,7 +59,7 @@ func Update(
 		if err == nil {
 			domain, _, _ := strings.Cut(webRoot, "/")
 			err = backend.CreateDomain(ctx, domain)
-			domainCache.AddDomain(ctx, domain)
+			siteExistenceCache.AddSite(ctx, webRoot)
 		}
 		if err == nil {
 			if oldManifest == nil {