mirrors/git-pages
1
0
Fork 0
mirror of https://codeberg.org/git-pages/git-pages.git synced 2026-08-17 09:26:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use s3:GetObject instead of s3:ListObjects for CheckDomain.

Browse Source
This commit is contained in:
miyuko
2025-10-22 13:45:15 +01:00
parent 34db13e603
commit aa965c5a08
7 changed files with 265 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/backend.go
+18
View File
@@ -21,11 +21,23 @@ func splitBlobName(name string) []string {
}
}
type BackendFeature string
const (
FeatureCheckDomainMarker BackendFeature = "check-domain-marker"
)
type GetManifestOptions struct {
BypassCache bool
}
type Backend interface {
// Returns true if the feature has been enabled for this store, false otherwise.
HasFeature(ctx context.Context, feature BackendFeature) bool
// Enables the feature for this store.
EnableFeature(ctx context.Context, feature BackendFeature) error
// Retrieve a blob. Returns `reader, size, mtime, err`.
GetBlob(ctx context.Context, name string) (reader io.ReadSeeker, size uint64, mtime time.Time, err error)
@@ -52,8 +64,14 @@ type Backend interface {
// Delete a manifest.
DeleteManifest(ctx context.Context, name string) error
// List all manifests.
ListManifests(ctx context.Context) (manifests []string, err error)
// Check whether a domain has any deployments.
CheckDomain(ctx context.Context, domain string) (found bool, err error)
// Creates a domain. This allows us to start serving content for the domain.
CreateDomain(ctx context.Context, domain string) error
}
var backend Backend
src/backend_fs.go
+39
View File
@@ -6,8 +6,10 @@ import (
"errors"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"strings"
"time"
)
@@ -68,6 +70,24 @@ func (fs *FSBackend) Backend() Backend {
return fs
}
func (fs *FSBackend) HasFeature(ctx context.Context, feature BackendFeature) bool {
switch feature {
case FeatureCheckDomainMarker:
return true
default:
return false
}
}
func (fs *FSBackend) EnableFeature(ctx context.Context, feature BackendFeature) error {
switch feature {
case FeatureCheckDomainMarker:
return nil
default:
return fmt.Errorf("not implemented")
}
}
func (fs *FSBackend) GetBlob(
ctx context.Context,
name string,
@@ -133,6 +153,21 @@ func (fs *FSBackend) DeleteBlob(ctx context.Context, name string) error {
return fs.blobRoot.Remove(blobPath)
}
func (b *FSBackend) ListManifests(ctx context.Context) (manifests []string, err error) {
err = fs.WalkDir(b.siteRoot.FS(), ".", func(path string, d fs.DirEntry, err error) error {
if strings.Count(path, "/") > 1 {
return fs.SkipDir
}
_, project, _ := strings.Cut(path, "/")
if project == "" || strings.HasPrefix(project, ".") && project != ".index" {
return nil
}
manifests = append(manifests, path)
return nil
})
return
}
func (fs *FSBackend) GetManifest(ctx context.Context, name string, opts GetManifestOptions) (*Manifest, error) {
data, err := fs.siteRoot.ReadFile(name)
if errors.Is(err, os.ErrNotExist) {
@@ -201,3 +236,7 @@ func (fs *FSBackend) CheckDomain(ctx context.Context, domain string) (bool, erro
return false, err
}
}
func (fs *FSBackend) CreateDomain(ctx context.Context, domain string) error {
return nil // no-op
}
src/backend_s3.go
+115 -16
View File
@@ -9,6 +9,7 @@ import (
"log"
"net/http"
"path"
"strings"
"time"
"github.com/c2h5oh/datasize"
@@ -118,10 +119,11 @@ type CachedManifest struct {
func (c *CachedManifest) Weight() uint32 { return c.weight }
type S3Backend struct {
client *minio.Client
bucket string
blobCache *observedCache[string, *CachedBlob]
siteCache *observedCache[string, *CachedManifest]
client *minio.Client
bucket string
blobCache *observedCache[string, *CachedBlob]
siteCache *observedCache[string, *CachedManifest]
featureCache *otter.Cache[BackendFeature, bool]
}
var _ Backend = (*S3Backend)(nil)
@@ -200,7 +202,14 @@ func NewS3Backend(ctx context.Context, config *S3Config) (*S3Backend, error) {
return nil, err
}
return &S3Backend{client, bucket, blobCache, siteCache}, nil
featureCache, err := otter.New(&otter.Options[BackendFeature, bool]{
RefreshCalculator: otter.RefreshWriting[BackendFeature, bool](10 * time.Minute),
})
if err != nil {
return nil, err
}
return &S3Backend{client, bucket, blobCache, siteCache, featureCache}, nil
}
func (s3 *S3Backend) Backend() Backend {
@@ -211,6 +220,42 @@ func blobObjectName(name string) string {
return fmt.Sprintf("blob/%s", path.Join(splitBlobName(name)...))
}
func storeFeatureObjectName(feature BackendFeature) string {
return fmt.Sprintf("meta/feature/%s", feature)
}
func (s3 *S3Backend) HasFeature(ctx context.Context, feature BackendFeature) bool {
loader := func(ctx context.Context, feature BackendFeature) (bool, error) {
_, err := s3.client.StatObject(ctx, s3.bucket, storeFeatureObjectName(feature),
minio.StatObjectOptions{})
if err != nil {
if errResp := minio.ToErrorResponse(err); errResp.Code == "NoSuchKey" {
log.Printf("s3 feature %q: disabled", feature)
return false, nil
} else {
return false, err
}
}
log.Printf("s3 feature %q: enabled", feature)
return true, nil
}
isOn, err := s3.featureCache.Get(ctx, feature, otter.LoaderFunc[BackendFeature, bool](loader))
if err != nil {
err = fmt.Errorf("getting s3 backend feature %q: %w", feature, err)
ObserveError(err)
log.Print(err)
return false
}
return isOn
}
func (s3 *S3Backend) EnableFeature(ctx context.Context, feature BackendFeature) error {
_, err := s3.client.PutObject(ctx, s3.bucket, storeFeatureObjectName(feature),
&bytes.Reader{}, 0, minio.PutObjectOptions{})
return err
}
func (s3 *S3Backend) GetBlob(
ctx context.Context,
name string,
@@ -307,6 +352,34 @@ func stagedManifestObjectName(manifestData []byte) string {
return fmt.Sprintf("dirty/%x", sha256.Sum256(manifestData))
}
func (s3 *S3Backend) ListManifests(ctx context.Context) (manifests []string, err error) {
log.Print("s3: list manifests")
ctx, cancel := context.WithCancel(ctx)
defer cancel()
prefix := manifestObjectName("")
for object := range s3.client.ListObjectsIter(ctx, s3.bucket, minio.ListObjectsOptions{
Prefix: prefix,
Recursive: true,
}) {
if object.Err != nil {
return nil, object.Err
}
key := strings.TrimRight(strings.TrimPrefix(object.Key, prefix), "/")
if strings.Count(key, "/") > 1 {
continue
}
_, project, _ := strings.Cut(key, "/")
if project == "" || strings.HasPrefix(project, ".") && project != ".index" {
continue
}
manifests = append(manifests, key)
}
return
}
type s3ManifestLoader struct {
s3 *S3Backend
}
@@ -426,19 +499,45 @@ func (s3 *S3Backend) DeleteManifest(ctx context.Context, name string) error {
return err
}
func (s3 *S3Backend) CheckDomain(ctx context.Context, domain string) (bool, error) {
func domainCheckObjectName(domain string) string {
return manifestObjectName(fmt.Sprintf("%s/.exists", domain))
}
func (s3 *S3Backend) CheckDomain(ctx context.Context, domain string) (exists bool, err error) {
log.Printf("s3: check domain %s\n", domain)
ctx, cancel := context.WithCancel(ctx)
defer cancel()
for object := range s3.client.ListObjectsIter(ctx, s3.bucket, minio.ListObjectsOptions{
Prefix: manifestObjectName(fmt.Sprintf("%s/", domain)),
}) {
if object.Err != nil {
return false, object.Err
_, err = s3.client.StatObject(ctx, s3.bucket, domainCheckObjectName(domain),
minio.StatObjectOptions{})
if err != nil {
if errResp := minio.ToErrorResponse(err); errResp.Code == "NoSuchKey" {
exists, err = false, nil
}
return true, nil
} else {
exists = true
}
return false, nil
if !exists && !s3.HasFeature(ctx, FeatureCheckDomainMarker) {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
for object := range s3.client.ListObjectsIter(ctx, s3.bucket, minio.ListObjectsOptions{
Prefix: manifestObjectName(fmt.Sprintf("%s/", domain)),
}) {
if object.Err != nil {
return false, object.Err
}
return true, nil
}
return false, nil
}
return
}
func (s3 *S3Backend) CreateDomain(ctx context.Context, domain string) error {
log.Printf("s3: create domain %s\n", domain)
_, err := s3.client.PutObject(ctx, s3.bucket, domainCheckObjectName(domain),
&bytes.Reader{}, 0, minio.PutObjectOptions{})
return err
}
src/main.go
+11
View File
@@ -78,6 +78,8 @@ func Main() {
"load configuration from `filename`")
noConfig := flag.Bool("no-config", false,
"run without configuration file (configure via environment variables)")
runMigration := flag.String("run-migration", "",
"run a specific store migration (available: \"create-domain-markers\")")
getManifest := flag.String("get-manifest", "",
"write manifest for `webroot` (either 'domain.tld' or 'domain.tld/dir') to stdout as ProtoJSON")
getBlob := flag.String("get-blob", "",
@@ -139,6 +141,15 @@ func Main() {
}
switch {
case *runMigration != "":
if err := ConfigureBackend(&config.Storage); err != nil {
log.Fatalln(err)
}
if err := RunMigration(context.Background(), *runMigration); err != nil {
log.Fatalln(err)
}
case *getManifest != "":
if err := ConfigureBackend(&config.Storage); err != nil {
log.Fatalln(err)
src/migrate.go
+49
View File
@@ -0,0 +1,49 @@
package git_pages
import (
"context"
"fmt"
"log"
"slices"
"strings"
)
func RunMigration(ctx context.Context, name string) error {
switch name {
case "create-domain-markers":
return createDomainMarkers(ctx)
default:
return fmt.Errorf("unknown migration name (expected one of \"create-domain-markers\")")
}
}
func createDomainMarkers(ctx context.Context) error {
if backend.HasFeature(ctx, FeatureCheckDomainMarker) {
log.Print("store already has domain markers")
return nil
}
var manifests, domains []string
manifests, err := backend.ListManifests(ctx)
if err != nil {
return fmt.Errorf("list manifests: %w", err)
}
slices.Sort(manifests)
for _, manifest := range manifests {
domain, _, _ := strings.Cut(manifest, "/")
if len(domains) == 0 || domains[len(domains)-1] != domain {
domains = append(domains, domain)
}
}
for idx, domain := range domains {
log.Printf("(%d / %d) creating domain %s", idx+1, len(domains), domain)
if err := backend.CreateDomain(ctx, domain); err != nil {
return fmt.Errorf("creating domain %s: %w", domain, err)
}
}
if err := backend.EnableFeature(ctx, FeatureCheckDomainMarker); err != nil {
return err
}
log.Printf("created markers for %d domains", len(domains))
return nil
}
src/observe.go
+28
View File
@@ -266,6 +266,20 @@ func NewObservedBackend(backend Backend) Backend {
return &observedBackend{inner: backend}
}
func (backend *observedBackend) HasFeature(ctx context.Context, feature BackendFeature) (isOn bool) {
span, ctx := ObserveFunction(ctx, "HasFeature")
isOn = backend.inner.HasFeature(ctx, feature)
span.Finish()
return
}
func (backend *observedBackend) EnableFeature(ctx context.Context, feature BackendFeature) (err error) {
span, ctx := ObserveFunction(ctx, "EnableFeature")
err = backend.inner.EnableFeature(ctx, feature)
span.Finish()
return
}
func (backend *observedBackend) GetBlob(
ctx context.Context,
name string,
@@ -302,6 +316,13 @@ func (backend *observedBackend) DeleteBlob(ctx context.Context, name string) (er
return
}
func (backend *observedBackend) ListManifests(ctx context.Context) (manifests []string, err error) {
span, ctx := ObserveFunction(ctx, "ListManifests")
manifests, err = backend.inner.ListManifests(ctx)
span.Finish()
return
}
func (backend *observedBackend) GetManifest(
ctx context.Context,
name string,
@@ -345,3 +366,10 @@ func (backend *observedBackend) CheckDomain(ctx context.Context, domain string)
span.Finish()
return
}
func (backend *observedBackend) CreateDomain(ctx context.Context, domain string) (err error) {
span, ctx := ObserveFunction(ctx, "CreateDomain", "manifest.domain", domain)
err = backend.inner.CreateDomain(ctx, domain)
span.Finish()
return
}
src/update.go
+5
View File
@@ -6,6 +6,7 @@ import (
"fmt"
"io"
"log"
"strings"
)
type UpdateOutcome int
@@ -42,6 +43,10 @@ func Update(ctx context.Context, webRoot string, manifest *Manifest) UpdateResul
}
} else if err = PrepareManifest(ctx, manifest); err == nil {
newManifest, err = StoreManifest(ctx, webRoot, manifest)
if err == nil {
domain, _, _ := strings.Cut(webRoot, "/")
err = backend.CreateDomain(ctx, domain)
}
if err == nil {
if oldManifest == nil {
outcome = UpdateCreated