From 0e342b11f63f047f34b6997fb1a5a6eb157cc42e Mon Sep 17 00:00:00 2001 From: Catherine Date: Wed, 19 Nov 2025 22:37:06 +0000 Subject: [PATCH] Add `Last-Modified:` header to `/.git-pages/` metadata responses. --- src/backend.go | 8 ++++++-- src/backend_fs.go | 34 ++++++++++++++++++++++------------ src/backend_s3.go | 19 ++++++++----------- src/main.go | 2 +- src/observe.go | 17 +++++------------ src/pages.go | 31 ++++++++++++++++++++++--------- src/update.go | 2 +- 7 files changed, 65 insertions(+), 48 deletions(-) diff --git a/src/backend.go b/src/backend.go index a9635be..a9028b0 100644 --- a/src/backend.go +++ b/src/backend.go @@ -39,7 +39,9 @@ type Backend interface { EnableFeature(ctx context.Context, feature BackendFeature) error // Retrieve a blob. Returns `reader, size, mtime, err`. - GetBlob(ctx context.Context, name string) (reader io.ReadSeeker, size uint64, mtime time.Time, err error) + GetBlob(ctx context.Context, name string) ( + reader io.ReadSeeker, size uint64, mtime time.Time, err error, + ) // Store a blob. If a blob called `name` already exists, this function returns `nil` without // regards to the old or new contents. It is expected that blobs are content-addressed, i.e. @@ -50,7 +52,9 @@ type Backend interface { DeleteBlob(ctx context.Context, name string) error // Retrieve a manifest. - GetManifest(ctx context.Context, name string, opts GetManifestOptions) (*Manifest, error) + GetManifest(ctx context.Context, name string, opts GetManifestOptions) ( + manifest *Manifest, mtime time.Time, err error, + ) // Stage a manifest. This operation stores a new version of a manifest, locking any blobs // referenced from it in place (for garbage collection purposes) but without any other side diff --git a/src/backend_fs.go b/src/backend_fs.go index c821642..20e82e0 100644 --- a/src/backend_fs.go +++ b/src/backend_fs.go @@ -89,13 +89,9 @@ func (fs *FSBackend) EnableFeature(ctx context.Context, feature BackendFeature) } func (fs *FSBackend) GetBlob( - ctx context.Context, - name string, + ctx context.Context, name string, ) ( - reader io.ReadSeeker, - size uint64, - mtime time.Time, - err error, + reader io.ReadSeeker, size uint64, mtime time.Time, err error, ) { blobPath := filepath.Join(splitBlobName(name)...) stat, err := fs.blobRoot.Stat(blobPath) @@ -168,15 +164,29 @@ func (b *FSBackend) ListManifests(ctx context.Context) (manifests []string, err return } -func (fs *FSBackend) GetManifest(ctx context.Context, name string, opts GetManifestOptions) (*Manifest, error) { - data, err := fs.siteRoot.ReadFile(name) +func (fs *FSBackend) GetManifest( + ctx context.Context, name string, opts GetManifestOptions, +) ( + manifest *Manifest, mtime time.Time, err error, +) { + stat, err := fs.siteRoot.Stat(name) if errors.Is(err, os.ErrNotExist) { - return nil, fmt.Errorf("%w: %s", ErrObjectNotFound, err.(*os.PathError).Path) + err = fmt.Errorf("%w: %s", ErrObjectNotFound, err.(*os.PathError).Path) + return } else if err != nil { - return nil, err + err = fmt.Errorf("stat: %w", err) + return } - - return DecodeManifest(data) + data, err := fs.siteRoot.ReadFile(name) + if err != nil { + err = fmt.Errorf("read: %w", err) + return + } + manifest, err = DecodeManifest(data) + if err != nil { + return + } + return manifest, stat.ModTime(), nil } func stagedManifestName(manifestData []byte) string { diff --git a/src/backend_s3.go b/src/backend_s3.go index 0767929..355058e 100644 --- a/src/backend_s3.go +++ b/src/backend_s3.go @@ -117,6 +117,7 @@ func (c *CachedBlob) Weight() uint32 { return uint32(len(c.blob)) } type CachedManifest struct { manifest *Manifest weight uint32 + mtime time.Time etag string err error } @@ -262,13 +263,9 @@ func (s3 *S3Backend) EnableFeature(ctx context.Context, feature BackendFeature) } func (s3 *S3Backend) GetBlob( - ctx context.Context, - name string, + ctx context.Context, name string, ) ( - reader io.ReadSeeker, - size uint64, - mtime time.Time, - err error, + reader io.ReadSeeker, size uint64, mtime time.Time, err error, ) { loader := func(ctx context.Context, name string) (*CachedBlob, error) { log.Printf("s3: get blob %s\n", name) @@ -434,7 +431,7 @@ func (l s3ManifestLoader) load(ctx context.Context, name string, oldManifest *Ca With(prometheus.Labels{"kind": "manifest"}). Observe(time.Since(startTime).Seconds()) - return &CachedManifest{manifest, uint32(len(data)), stat.ETag, nil}, nil + return &CachedManifest{manifest, uint32(len(data)), stat.LastModified, stat.ETag, nil}, nil } var cached *CachedManifest @@ -443,7 +440,7 @@ func (l s3ManifestLoader) load(ctx context.Context, name string, oldManifest *Ca if errResp := minio.ToErrorResponse(err); errResp.Code == "NoSuchKey" { s3GetObjectErrorsCount.With(prometheus.Labels{"object_kind": "manifest"}).Inc() err = fmt.Errorf("%w: %s", ErrObjectNotFound, errResp.Key) - return &CachedManifest{nil, 1, "", err}, nil + return &CachedManifest{nil, 1, time.Time{}, "", err}, nil } else if errResp.StatusCode == http.StatusNotModified && oldManifest != nil { return oldManifest, nil } else { @@ -457,7 +454,7 @@ func (l s3ManifestLoader) load(ctx context.Context, name string, oldManifest *Ca func (s3 *S3Backend) GetManifest( ctx context.Context, name string, opts GetManifestOptions, ) ( - manifest *Manifest, err error, + manifest *Manifest, mtime time.Time, err error, ) { if opts.BypassCache { entry, found := s3.siteCache.Cache.GetEntry(name) @@ -471,8 +468,8 @@ func (s3 *S3Backend) GetManifest( if err != nil { return } else { - // This could be `manifest, nil` or `nil, ErrObjectNotFound`. - manifest, err = cached.manifest, cached.err + // This could be `manifest, mtime, nil` or `nil, time.Time{}, ErrObjectNotFound`. + manifest, mtime, err = cached.manifest, cached.mtime, cached.err return } } diff --git a/src/main.go b/src/main.go index 7efa3f0..0eb06e5 100644 --- a/src/main.go +++ b/src/main.go @@ -160,7 +160,7 @@ func Main() { webRoot += "/.index" } - manifest, err := backend.GetManifest(context.Background(), webRoot, GetManifestOptions{}) + manifest, _, err := backend.GetManifest(context.Background(), webRoot, GetManifestOptions{}) if err != nil { log.Fatalln(err) } diff --git a/src/observe.go b/src/observe.go index 8f05cb9..222b8a1 100644 --- a/src/observe.go +++ b/src/observe.go @@ -288,13 +288,9 @@ func (backend *observedBackend) EnableFeature(ctx context.Context, feature Backe } func (backend *observedBackend) GetBlob( - ctx context.Context, - name string, + ctx context.Context, name string, ) ( - reader io.ReadSeeker, - size uint64, - mtime time.Time, - err error, + reader io.ReadSeeker, size uint64, mtime time.Time, err error, ) { span, ctx := ObserveFunction(ctx, "GetBlob", "blob.name", name) if reader, size, mtime, err = backend.inner.GetBlob(ctx, name); err == nil { @@ -331,18 +327,15 @@ func (backend *observedBackend) ListManifests(ctx context.Context) (manifests [] } func (backend *observedBackend) GetManifest( - ctx context.Context, - name string, - opts GetManifestOptions, + ctx context.Context, name string, opts GetManifestOptions, ) ( - manifest *Manifest, - err error, + manifest *Manifest, mtime time.Time, err error, ) { span, ctx := ObserveFunction(ctx, "GetManifest", "manifest.name", name, "manifest.bypass_cache", opts.BypassCache, ) - if manifest, err = backend.inner.GetManifest(ctx, name, opts); err == nil { + if manifest, mtime, err = backend.inner.GetManifest(ctx, name, opts); err == nil { manifestsRetrievedCount.Inc() } span.Finish() diff --git a/src/pages.go b/src/pages.go index 8991d2c..95f47ce 100644 --- a/src/pages.go +++ b/src/pages.go @@ -77,6 +77,7 @@ func getPage(w http.ResponseWriter, r *http.Request) error { var err error var sitePath string var manifest *Manifest + var manifestMtime time.Time cacheControl, err := cacheobject.ParseRequestCacheControl(r.Header.Get("Cache-Control")) if err != nil { @@ -95,33 +96,39 @@ func getPage(w http.ResponseWriter, r *http.Request) error { } type indexManifestResult struct { - manifest *Manifest - err error + manifest *Manifest + manifestMtime time.Time + err error } indexManifestCh := make(chan indexManifestResult, 1) go func() { - manifest, err := backend.GetManifest(r.Context(), makeWebRoot(host, ".index"), - GetManifestOptions{BypassCache: bypassCache}) - indexManifestCh <- (indexManifestResult{manifest, err}) + manifest, mtime, err := backend.GetManifest( + r.Context(), makeWebRoot(host, ".index"), + GetManifestOptions{BypassCache: bypassCache}, + ) + indexManifestCh <- (indexManifestResult{manifest, mtime, err}) }() err = nil sitePath = strings.TrimPrefix(r.URL.Path, "/") if projectName, projectPath, hasProjectSlash := strings.Cut(sitePath, "/"); projectName != "" { var projectManifest *Manifest - projectManifest, err = backend.GetManifest(r.Context(), makeWebRoot(host, projectName), - GetManifestOptions{BypassCache: bypassCache}) + var projectManifestMtime time.Time + projectManifest, projectManifestMtime, err = backend.GetManifest( + r.Context(), makeWebRoot(host, projectName), + GetManifestOptions{BypassCache: bypassCache}, + ) if err == nil { if !hasProjectSlash { writeRedirect(w, http.StatusFound, r.URL.Path+"/") return nil } - sitePath, manifest = projectPath, projectManifest + sitePath, manifest, manifestMtime = projectPath, projectManifest, projectManifestMtime } } if manifest == nil && (err == nil || errors.Is(err, ErrObjectNotFound)) { result := <-indexManifestCh - manifest, err = result.manifest, result.err + manifest, manifestMtime, err = result.manifest, result.manifestMtime, result.err if manifest == nil && errors.Is(err, ErrObjectNotFound) { if found, fallbackErr := HandleWildcardFallback(w, r); found { return fallbackErr @@ -151,10 +158,13 @@ func getPage(w http.ResponseWriter, r *http.Request) error { return nil } if metadataPath, found := strings.CutPrefix(sitePath, ".git-pages/"); found { + lastModified := manifestMtime.UTC().Format(http.TimeFormat) switch metadataPath { case "health": + w.Header().Add("Last-Modified", lastModified) w.WriteHeader(http.StatusOK) fmt.Fprintf(w, "ok\n") + case "manifest.json": // metadata requests require authorization to avoid making pushes from private // repositories enumerable @@ -162,9 +172,12 @@ func getPage(w http.ResponseWriter, r *http.Request) error { if err != nil { return err } + w.Header().Add("Content-Type", "application/json; charset=utf-8") + w.Header().Add("Last-Modified", lastModified) w.WriteHeader(http.StatusOK) w.Write([]byte(ManifestDebugJSON(manifest))) + default: w.WriteHeader(http.StatusNotFound) fmt.Fprintf(w, "not found\n") diff --git a/src/update.go b/src/update.go index 9aa1ae7..9a61024 100644 --- a/src/update.go +++ b/src/update.go @@ -31,7 +31,7 @@ func Update(ctx context.Context, webRoot string, manifest *Manifest) UpdateResul var err error outcome := UpdateError - oldManifest, _ = backend.GetManifest(ctx, webRoot, GetManifestOptions{}) + oldManifest, _, _ = backend.GetManifest(ctx, webRoot, GetManifestOptions{}) if IsManifestEmpty(manifest) { newManifest, err = manifest, backend.DeleteManifest(ctx, webRoot) if err == nil {