diff --git a/README.md b/README.md index d6b6897..6cbd3b5 100644 --- a/README.md +++ b/README.md @@ -71,13 +71,14 @@ An object store (filesystem, S3, ...) is used as the sole mechanism for state st - The `site/` prefix contains site manifests organized by domain and project name (e.g. `site/example.org/myproject` or `site/example.org/.index`). - The manifest is a Protobuf object containing a flat mapping of paths to entries. An entry is comprised of type (file, directory, symlink, etc) and data, which may be stored inline or refer to a blob. - A small amount of internal metadata within a manifest allows attributing deployments to their source and computing quotas. - - The S3 backend caches manifests in memory. Since a manifest is necessary and sufficient to return `304 Not Modified` responses for a matching `ETag`, this drastically reduces navigation latency. - Additionally, the object store contains *staged manifests*, representing an in-progress update operation. - An update first creates a staged manifest, then uploads blobs, then replaces the deployed manifest with the staged one. This avoids TOCTTOU race conditions during garbage collection. - Stable marshalling allows addressing staged manifests by the hash of their contents. This approach, unlike the v1 one, cannot be easily introspected with normal Unix commands, but is very friendly to S3-style object storage services, as it does not rely on operations these services cannot support (subtree rename, directory stat, symlink/readlink). +The S3 backend, intended for (relatively) high latency connections, caches both manifests and blobs in memory. Since a manifest is necessary and sufficient to return `304 Not Modified` responses for a matching `ETag`, this drastically reduces navigation latency. Blobs are content-addressed and are an obvious target for a last level cache. + Architecture (v1) ----------------- diff --git a/src/backend.go b/src/backend.go index bfadf96..7157bfe 100644 --- a/src/backend.go +++ b/src/backend.go @@ -24,7 +24,7 @@ import ( type Backend interface { // Retrieve a blob. Returns `reader, mtime, err`. - GetBlob(name string) (io.ReadSeekCloser, time.Time, error) + GetBlob(name string) (io.ReadSeeker, time.Time, error) // Store a blob. If a blob called `name` already exists, this function returns `nil` without // regards to the old or new contents. It is expected that blobs are content-addressed, i.e. @@ -114,7 +114,7 @@ func splitBlobName(name string) []string { } } -func (fs *FSBackend) GetBlob(name string) (io.ReadSeekCloser, time.Time, error) { +func (fs *FSBackend) GetBlob(name string) (io.ReadSeeker, time.Time, error) { blobPath := filepath.Join(splitBlobName(name)...) stat, err := fs.blobRoot.Stat(blobPath) if err != nil { @@ -207,16 +207,50 @@ func (fs *FSBackend) DeleteManifest(name string) error { return fs.siteRoot.Remove(name) } +type CachedBlob struct { + blob []byte + mtime time.Time +} + type CachedManifest struct { manifest *Manifest weight uint32 } type S3Backend struct { - ctx context.Context - client *minio.Client - bucket string - cache *otter.Cache[string, *CachedManifest] + ctx context.Context + client *minio.Client + bucket string + blobCache *otter.Cache[string, *CachedBlob] + siteCache *otter.Cache[string, *CachedManifest] +} + +func defaultCacheConfig[K comparable, V any]( + config CacheConfig, + maxAge time.Duration, + maxSize uint64, + weigher func(K, V) uint32, +) (*otter.Options[K, V], error) { + var err error + if config.MaxAge != "" { + maxAge, err = time.ParseDuration(config.MaxAge) + if err != nil { + return nil, fmt.Errorf("max-age: %s", err) + } + } + if config.MaxSize != 0 { + maxSize = config.MaxSize + } + + options := &otter.Options[K, V]{} + if maxSize != 0 { + options.MaximumWeight = maxSize + options.Weigher = weigher + } + if maxAge != 0 { + options.ExpiryCalculator = otter.ExpiryCreating[K, V](maxAge) + } + return options, nil } func NewS3Backend( @@ -252,27 +286,33 @@ func NewS3Backend( } } - cacheConfig := config.Backend.S3.Cache - var maxWeight uint64 = 134217728 - if cacheConfig.MaxSize != 0 { - maxWeight = cacheConfig.MaxSize + blobCacheOptions, err := defaultCacheConfig[string, *CachedBlob]( + config.Backend.S3.BlobCache, + 0, 256*1048576, + func(key string, value *CachedBlob) uint32 { return uint32(len(value.blob)) }) + if err != nil { + return nil, err } - var maxAge time.Duration = 5 * time.Second - if cacheConfig.MaxAge != "" { - maxAge, err = time.ParseDuration(cacheConfig.MaxAge) - if err != nil { - return nil, fmt.Errorf("max-age: %s", err) - } - } - cache := otter.Must[string, *CachedManifest](&otter.Options[string, *CachedManifest]{ - MaximumWeight: maxWeight, - Weigher: func(key string, value *CachedManifest) uint32 { - return value.weight - }, - ExpiryCalculator: otter.ExpiryCreating[string, *CachedManifest](maxAge), - }) - return &S3Backend{ctx, client, bucket, cache}, nil + blobCache, err := otter.New(blobCacheOptions) + if err != nil { + return nil, err + } + + siteCacheOptions, err := defaultCacheConfig[string, *CachedManifest]( + config.Backend.S3.SiteCache, + 5*time.Second, 16*1048576, + func(key string, value *CachedManifest) uint32 { return value.weight }) + if err != nil { + return nil, err + } + + siteCache, err := otter.New(siteCacheOptions) + if err != nil { + return nil, err + } + + return &S3Backend{ctx, client, bucket, blobCache, siteCache}, nil } func (s3 *S3Backend) Backend() Backend { @@ -283,21 +323,36 @@ func blobObjectName(name string) string { return fmt.Sprintf("blob/%s", name) } -func (s3 *S3Backend) GetBlob(name string) (io.ReadSeekCloser, time.Time, error) { - log.Printf("s3: get blob %s\n", name) +func (s3 *S3Backend) GetBlob(name string) (io.ReadSeeker, time.Time, error) { + loader := func(ctx context.Context, name string) (*CachedBlob, error) { + log.Printf("s3: get blob %s\n", name) - object, err := s3.client.GetObject(s3.ctx, s3.bucket, blobObjectName(name), - minio.GetObjectOptions{}) + object, err := s3.client.GetObject(s3.ctx, s3.bucket, blobObjectName(name), + minio.GetObjectOptions{}) + if err != nil { + return nil, err + } + defer object.Close() + + stat, err := object.Stat() + if err != nil { + return nil, err + } + + data, err := io.ReadAll(object) + if err != nil { + return nil, err + } + + return &CachedBlob{data, stat.LastModified}, nil + } + + cached, err := s3.blobCache.Get(s3.ctx, name, otter.LoaderFunc[string, *CachedBlob](loader)) if err != nil { return nil, time.Time{}, err } - stat, err := object.Stat() - if err != nil { - return nil, time.Time{}, err - } - - return object, stat.LastModified, nil + return bytes.NewReader(cached.blob), cached.mtime, err } func (s3 *S3Backend) PutBlob(name string, data []byte) error { @@ -312,12 +367,17 @@ func (s3 *S3Backend) PutBlob(name string, data []byte) error { bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{}) if err != nil { return err + } else { + log.Printf("s3: put blob %s (created)\n", name) + return nil } } else { return err } + } else { + log.Printf("s3: put blob %s (exists)\n", name) + return nil } - return nil // already exists or was created } func (s3 *S3Backend) DeleteBlob(name string) error { @@ -344,6 +404,7 @@ func (s3 *S3Backend) GetManifest(name string) (*Manifest, error) { if err != nil { return nil, err } + defer object.Close() data, err := io.ReadAll(object) if err != nil { @@ -358,7 +419,7 @@ func (s3 *S3Backend) GetManifest(name string) (*Manifest, error) { return &CachedManifest{manifest, uint32(len(data))}, nil } - cached, err := s3.cache.Get(s3.ctx, name, otter.LoaderFunc[string, *CachedManifest](loader)) + cached, err := s3.siteCache.Get(s3.ctx, name, otter.LoaderFunc[string, *CachedManifest](loader)) if err != nil { return nil, err } diff --git a/src/config.go b/src/config.go index 53bbcac..bc186c6 100644 --- a/src/config.go +++ b/src/config.go @@ -11,6 +11,11 @@ type ListenConfig struct { Address string `toml:"address"` } +type CacheConfig struct { + MaxSize uint64 `toml:"max-size"` // in bytes + MaxAge string `toml:"max-age"` +} + type Config struct { Pages ListenConfig `toml:"pages"` Caddy ListenConfig `toml:"caddy"` @@ -25,16 +30,14 @@ type Config struct { Root string `toml:"root"` } `toml:"fs"` S3 struct { - Endpoint string `toml:"endpoint"` - Insecure bool `toml:"insecure"` - AccessKeyID string `toml:"access-key-id"` - SecretAccessKey string `toml:"secret-access-key"` - Region string `toml:"region"` - Bucket string `toml:"bucket"` - Cache struct { - MaxSize uint64 `toml:"max-size"` // in bytes - MaxAge string `toml:"max-age"` - } `toml:"cache"` + Endpoint string `toml:"endpoint"` + Insecure bool `toml:"insecure"` + AccessKeyID string `toml:"access-key-id"` + SecretAccessKey string `toml:"secret-access-key"` + Region string `toml:"region"` + Bucket string `toml:"bucket"` + BlobCache CacheConfig `toml:"blob-cache"` + SiteCache CacheConfig `toml:"site-cache"` } } `toml:"backend"` } diff --git a/src/pages.go b/src/pages.go index 980815b..c5888c6 100644 --- a/src/pages.go +++ b/src/pages.go @@ -75,16 +75,12 @@ func getPage(w http.ResponseWriter, r *http.Request) error { w.WriteHeader(http.StatusNotModified) return nil } else { - var blob io.ReadSeekCloser - blob, mtime, err = backend.GetBlob(string(entry.Data)) + reader, mtime, err = backend.GetBlob(string(entry.Data)) if err != nil { w.WriteHeader(http.StatusInternalServerError) fmt.Fprintf(w, "internal server error\n") return err } - defer blob.Close() - - reader = blob w.Header().Set("ETag", etag) } } else if entry.Type == Type_Directory {