From a7063e00ef9b49d82d16c60c61e42fbbe546ae55 Mon Sep 17 00:00:00 2001 From: Catherine Date: Thu, 28 May 2026 09:54:29 +0000 Subject: [PATCH] Implement site expiration. Requires `feature = ["expiration"]`. --- README.md | 1 + conf/config.default.toml | 1 + conf/config.example.toml | 2 ++ src/audit.go | 14 ++++++++++ src/backend.go | 5 +++- src/backend_fs.go | 6 ++++ src/backend_s3.go | 6 ++++ src/config.go | 4 +++ src/main.go | 59 +++++++++++++++++++++++++++++++++++---- src/manifest.go | 9 +++++- src/observe.go | 7 +++++ src/pages.go | 39 +++++++++++++++++++++++--- src/schema.pb.go | 49 +++++++++++++++++++++----------- src/schema.proto | 5 ++++ src/update.go | 60 ++++++++++++++++++++++++++++++---------- 15 files changed, 225 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 5d76fab..b6a2f9a 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,7 @@ Features - If the site has no contents after the update is applied, performs the same action as `DELETE`. * In response to a `DELETE` request, the server unpublishes a site. The URL of the request must be the root URL of the site that is being unpublished. Site data remains stored for an indeterminate period of time, but becomes completely inaccessible. * If a `Dry-Run: yes` header is provided with a `PUT`, `PATCH`, `DELETE`, or `POST` request, only the authorization checks are run; no destructive updates are made. +* If a `Expires: ` header is provided with a `PUT` or `PATCH` request, and the `[limits].allow-expiration` configuration option is enabled, and the site with that name does not exist or is already scheduled to expire enabled, the site is then scheduled to expire at `` (in the HTTP date format, e.g. `Mon, 02 Jan 2006 15:04:05 GMT`). Expired sites are removed by the `git-pages -site-expire` command, which must be scheduled to periorically run for this feature to work. * All updates to site content are atomic (subject to consistency guarantees of the storage backend). That is, there is an instantaneous moment during an update before which the server will return the old content and after which it will return the new content. * Files with a certain name, when placed in the root of a site, have special functions: - [Netlify `_redirects`][_redirects] file can be used to specify HTTP redirect and rewrite rules. The _git-pages_ implementation currently does not support placeholders, query parameters, or conditions, and may differ from Netlify in other minor ways. If you find that a supported `_redirects` file feature does not work the same as on Netlify, please file an issue. (Note that _git-pages_ does not perform URL normalization; `/foo` and `/foo/` are *not* the same, unlike with Netlify.) diff --git a/conf/config.default.toml b/conf/config.default.toml index 239e647..5c8e671 100644 --- a/conf/config.default.toml +++ b/conf/config.default.toml @@ -27,6 +27,7 @@ forbidden-domains = [] allowed-repository-url-prefixes = [] allowed-custom-headers = ['X-Clacks-Overhead'] allow-basic-auth = false +allow-expiration = false [audit] node-id = 0 diff --git a/conf/config.example.toml b/conf/config.example.toml index bf8340e..b3fba93 100644 --- a/conf/config.example.toml +++ b/conf/config.example.toml @@ -11,6 +11,7 @@ metrics = "tcp/localhost:3002" [[wildcard]] # non-default section domain = "codeberg.page" +preview-domain = "preview.codeberg.page" clone-url = "https://codeberg.org//.git" index-repo = "pages" index-repo-branch = "main" @@ -54,6 +55,7 @@ forbidden-domains = [] allowed-repository-url-prefixes = [] allowed-custom-headers = ["X-Clacks-Overhead"] allow-basic-auth = false +allow-expiration = false [audit] node-id = 0 diff --git a/src/audit.go b/src/audit.go index 3b76df9..630027e 100644 --- a/src/audit.go +++ b/src/audit.go @@ -394,6 +394,20 @@ func (audited *auditedBackend) DeleteManifest( return audited.Backend.DeleteManifest(ctx, name, opts) } +func (audited *auditedBackend) ExpireManifest(ctx context.Context, name string) (err error) { + domain, project, ok := strings.Cut(name, "/") + if !ok { + panic("malformed manifest name") + } + audited.appendNewAuditRecord(ctx, &AuditRecord{ + Event: AuditEvent_ExpireManifest.Enum(), + Domain: proto.String(domain), + Project: proto.String(project), + }) + + return audited.Backend.ExpireManifest(ctx, name) +} + func (audited *auditedBackend) FreezeDomain(ctx context.Context, domain string) (err error) { audited.appendNewAuditRecord(ctx, &AuditRecord{ Event: AuditEvent_FreezeDomain.Enum(), diff --git a/src/backend.go b/src/backend.go index 86ab6b6..8c1ca61 100644 --- a/src/backend.go +++ b/src/backend.go @@ -121,9 +121,12 @@ type Backend interface { // the old version or the new version of the manifest, never anything else. CommitManifest(ctx context.Context, name string, manifest *Manifest, opts ModifyManifestOptions) error - // Delete a manifest. + // Delete a manifest. This operation is initiated via the API. DeleteManifest(ctx context.Context, name string, opts ModifyManifestOptions) error + // Expire a manifest. This operation is initiated via a scheduled job. + ExpireManifest(ctx context.Context, name string) error + // Iterate through metadata of all manifests. Whether manifests that are newly added during // iteration will appear in the results is unspecified. EnumerateManifests(ctx context.Context) iter.Seq2[*ManifestMetadata, error] diff --git a/src/backend_fs.go b/src/backend_fs.go index 3bd44ac..378cb4f 100644 --- a/src/backend_fs.go +++ b/src/backend_fs.go @@ -405,6 +405,12 @@ func (fs *FSBackend) DeleteManifest( } } +func (fs *FSBackend) ExpireManifest( + ctx context.Context, name string, +) error { + return fs.DeleteManifest(ctx, name, ModifyManifestOptions{}) +} + func (fs *FSBackend) EnumerateManifests(ctx context.Context) iter.Seq2[*ManifestMetadata, error] { return func(yield func(*ManifestMetadata, error) bool) { iofs.WalkDir(fs.siteRoot.FS(), ".", diff --git a/src/backend_s3.go b/src/backend_s3.go index 51d18a0..3f3bae5 100644 --- a/src/backend_s3.go +++ b/src/backend_s3.go @@ -668,6 +668,12 @@ func (s3 *S3Backend) DeleteManifest( return nil } +func (s3 *S3Backend) ExpireManifest( + ctx context.Context, name string, +) error { + return s3.DeleteManifest(ctx, name, ModifyManifestOptions{}) +} + func (s3 *S3Backend) EnumerateManifests(ctx context.Context) iter.Seq2[*ManifestMetadata, error] { return func(yield func(*ManifestMetadata, error) bool) { logc.Println(ctx, "s3: enumerate manifests") diff --git a/src/config.go b/src/config.go index 5f209a2..0985fa6 100644 --- a/src/config.go +++ b/src/config.go @@ -150,6 +150,10 @@ type LimitsConfig struct { // Whether to allow Netlify-style credentials specified in a `Basic-Auth:` pseudo-header. // These credentials are plaintext. AllowBasicAuth bool `toml:"allow-basic-auth" default:"false"` + // Whether to allow an expiration date to be specified for uploaded sites. If enabled, you + // must also configure a scheduled job calling `git-pages -site-expire` regularly (perhaps + // hourly or daily). + AllowExpiration bool `toml:"allow-expiration" default:"false"` } type AuditConfig struct { diff --git a/src/main.go b/src/main.go index 20250a8..5d5d054 100644 --- a/src/main.go +++ b/src/main.go @@ -44,7 +44,7 @@ func configureFeatures(ctx context.Context) (err error) { for _, feature := range config.Features { switch feature { // Work-in-progress features: - case "preview": + case "preview", "expiration": // Permanently unstable features: case "codeberg-pages-compat", "relaxed-idna": // Stabilized features: @@ -216,6 +216,8 @@ func usage() { "git-pages {-audit-expire |-audit-detach /}\n") fmt.Fprintf(os.Stderr, "(audit) "+ "git-pages -audit-server [args...]\n") + fmt.Fprintf(os.Stderr, "(maint) "+ + "git-pages -site-expire [-dry-run]\n") fmt.Fprintf(os.Stderr, "(maint) "+ "git-pages {-run-migration |-trace-garbage|-size-histogram {original|stored}}\n") flag.PrintDefaults() @@ -263,12 +265,16 @@ func Main(versionInfo string) { "detach all blobs of audit records for a single `site` (or the entire domain with 'domain.tld/*')") auditServer := flag.String("audit-server", "", "listen for notifications on `endpoint` and spawn a process for each audit event") + siteExpire := flag.Bool("site-expire", false, + "expire sites according to their manifest") runMigration := flag.String("run-migration", "", "run a store `migration` (one of: create-domain-markers)") sizeHistogram := flag.String("size-histogram", "", "display histogram of `size-type` (original or stored) per domain") traceGarbage := flag.Bool("trace-garbage", false, "estimate total size of unreachable blobs") + dryRun := flag.Bool("dry-run", false, + "print what would be performed instead of executing it") version := flag.Bool("version", false, "display version") flag.Parse() @@ -294,6 +300,7 @@ func Main(versionInfo string) { *auditExpire != "", *auditDetach != "", *auditServer != "", + *siteExpire, *runMigration != "", *sizeHistogram != "", *traceGarbage, @@ -305,8 +312,11 @@ func Main(versionInfo string) { if cliOperations > 1 { logc.Fatalln(ctx, "-list-blobs, -list-manifests, -get-blob, -get-manifest, -get-archive, "+ "-update-site, -freeze-domain, -unfreeze-domain, -audit-log, -audit-read, "+ - "-audit-rollback, -audit-expire, -audit-detach, -audit-server, -run-migration, "+ - "-size-histogram, and -trace-garbage are mutually exclusive") + "-audit-rollback, -audit-expire, -audit-detach, -audit-server, -site-expire, "+ + "-run-migration, -size-histogram, and -trace-garbage are mutually exclusive") + } + if *dryRun && !(*siteExpire) { + logc.Fatalln(ctx, "-dry-run is not applicable in this context") } if *configTomlPath != "" && *noConfig { @@ -455,7 +465,7 @@ func Main(versionInfo string) { } webRoot := webRootArg(*updateSite) - result = UpdateFromArchive(ctx, webRoot, "", contentType, file) + result = UpdateFromArchive(ctx, webRoot, "", contentType, file, UpdateOptions{}) } else { branch := "pages" if sourceURL.Fragment != "" { @@ -463,7 +473,7 @@ func Main(versionInfo string) { } webRoot := webRootArg(*updateSite) - result = UpdateFromRepository(ctx, webRoot, sourceURL.String(), branch) + result = UpdateFromRepository(ctx, webRoot, sourceURL.String(), branch, UpdateOptions{}) } switch result.outcome { @@ -647,7 +657,6 @@ func Main(versionInfo string) { for id, err := range ids { if err != nil { logc.Fatalln(ctx, err) - continue } err = backend.ExpireAuditRecord(ctx, id) @@ -661,6 +670,44 @@ func Main(versionInfo string) { logc.Printf(ctx, "audit: expired %d records\n", count) + case *siteExpire: + ctx = WithPrincipal(ctx) + GetPrincipal(ctx).CliAdmin = proto.Bool(true) + + if !config.Feature("expiration") { + logc.Fatalf(ctx, "expire: feature disabled") + } + + countExpired, countTransient := 0, 0 + for item, err := range backend.GetAllManifests(ctx) { + metadata, manifest := item.Splat() + if err != nil { + logc.Fatalln(ctx, err) + } + if manifest.ExpiresAt != nil { + countTransient += 1 + if manifest.ExpiresAt.AsTime().Before(time.Now()) { + if !*dryRun { + err = backend.ExpireManifest(ctx, metadata.Name) + if err != nil { + logc.Fatalln(ctx, err) + } + } + logc.Printf(ctx, "expire: site %s expired at %s", + metadata.Name, manifest.ExpiresAt.AsTime()) + countExpired += 1 + } + } + } + + if *dryRun { + logc.Printf(ctx, "expire: would expire %d out of %d transient sites (dry run)\n", + countExpired, countTransient) + } else { + logc.Printf(ctx, "expire: expired %d out of %d transient sites\n", + countExpired, countTransient) + } + case *runMigration != "": if err = RunMigration(ctx, *runMigration); err != nil { logc.Fatalln(ctx, err) diff --git a/src/manifest.go b/src/manifest.go index 9aa4e48..03f9343 100644 --- a/src/manifest.go +++ b/src/manifest.go @@ -59,8 +59,12 @@ func IsManifestEmpty(manifest *Manifest) bool { panic(fmt.Errorf("malformed manifest %v", manifest)) } -// Returns `true` if `left` and `right` contain the same files with the same types and data. +// Returns `true` if `left` and `right` contain the same files with the same types and data, +// and use the same options. func CompareManifest(left *Manifest, right *Manifest) bool { + if left.ExpiresAt.AsTime() != right.ExpiresAt.AsTime() { + return false + } if len(left.Contents) != len(right.Contents) { return false } @@ -80,6 +84,9 @@ func CompareManifest(left *Manifest, right *Manifest) bool { } func EncodeManifest(manifest *Manifest) (data []byte) { + if manifest.ExpiresAt != nil && manifest.ExpiresAt.AsTime().IsZero() { + panic("invalid expires_at value") // ambiguous and shouldn't happen + } data, err := proto.MarshalOptions{Deterministic: true}.Marshal(manifest) if err != nil { panic(err) diff --git a/src/observe.go b/src/observe.go index 9bcf26d..a7210ed 100644 --- a/src/observe.go +++ b/src/observe.go @@ -294,6 +294,13 @@ func (backend *observedBackend) DeleteManifest(ctx context.Context, name string, return } +func (backend *observedBackend) ExpireManifest(ctx context.Context, name string) (err error) { + span, ctx := ObserveFunction(ctx, "ExpireManifest", "manifest.name", name) + err = backend.inner.ExpireManifest(ctx, name) + span.Finish() + return +} + func (backend *observedBackend) EnumerateManifests(ctx context.Context) iter.Seq2[*ManifestMetadata, error] { return func(yield func(*ManifestMetadata, error) bool) { span, ctx := ObserveFunction(ctx, "EnumerateManifests") diff --git a/src/pages.go b/src/pages.go index 0dfa963..da55153 100644 --- a/src/pages.go +++ b/src/pages.go @@ -492,6 +492,27 @@ func checkDryRun(w http.ResponseWriter, r *http.Request) bool { return false } +func getUpdateOptions(w http.ResponseWriter, r *http.Request) (opts UpdateOptions, ok bool) { + var err error + + if config.Feature("expiration") { + if expires := r.Header.Get("Expires"); expires != "" { + opts.expiresAt, err = time.Parse(http.TimeFormat, expires) + if err != nil { + http.Error(w, "malformed Expires: header", http.StatusBadRequest) + return + } + if !config.Limits.AllowExpiration { + http.Error(w, "expiration forbidden by policy", http.StatusBadRequest) + return + } + } + } + + ok = true + return +} + func putPage(w http.ResponseWriter, r *http.Request) error { var result UpdateResult @@ -509,6 +530,11 @@ func putPage(w http.ResponseWriter, r *http.Request) error { return err } + opts, ok := getUpdateOptions(w, r) + if !ok { + return nil + } + ctx, cancel := context.WithTimeout(r.Context(), time.Duration(config.Limits.UpdateTimeout)) defer cancel() @@ -543,7 +569,7 @@ func putPage(w http.ResponseWriter, r *http.Request) error { return nil } - result = UpdateFromRepository(ctx, webRoot, repoURL, branch) + result = UpdateFromRepository(ctx, webRoot, repoURL, branch, opts) default: auth, err := AuthorizeUpdateFromArchive(r) @@ -562,7 +588,7 @@ func putPage(w http.ResponseWriter, r *http.Request) error { // request body contains archive reader := http.MaxBytesReader(w, r.Body, int64(config.Limits.MaxSiteSize.Bytes())) - result = UpdateFromArchive(ctx, webRoot, repoURL, contentType, reader) + result = UpdateFromArchive(ctx, webRoot, repoURL, contentType, reader, opts) } return reportUpdateResult(w, r, result) @@ -583,6 +609,11 @@ func patchPage(w http.ResponseWriter, r *http.Request) error { return err } + opts, ok := getUpdateOptions(w, r) + if !ok { + return nil + } + auth, err := AuthorizeUpdateFromArchive(r) if err != nil { return err @@ -631,7 +662,7 @@ func patchPage(w http.ResponseWriter, r *http.Request) error { contentType := getMediaType(r.Header.Get("Content-Type")) reader := http.MaxBytesReader(w, r.Body, int64(config.Limits.MaxSiteSize.Bytes())) - result := PartialUpdateFromArchive(ctx, webRoot, contentType, reader, parents) + result := PartialUpdateFromArchive(ctx, webRoot, contentType, reader, parents, opts) return reportUpdateResult(w, r, result) } @@ -830,7 +861,7 @@ func postPage(w http.ResponseWriter, r *http.Request) error { ctx, cancel := context.WithTimeout(ctx, time.Duration(config.Limits.UpdateTimeout)) defer cancel() - result := UpdateFromRepository(ctx, webRoot, repoURL, auth.branch) + result := UpdateFromRepository(ctx, webRoot, repoURL, auth.branch, UpdateOptions{}) resultChan <- result observeSiteUpdate("webhook", &result) }(context.WithoutCancel(r.Context())) diff --git a/src/schema.pb.go b/src/schema.pb.go index 788cd28..5b79494 100644 --- a/src/schema.pb.go +++ b/src/schema.pb.go @@ -140,6 +140,8 @@ const ( AuditEvent_CommitManifest AuditEvent = 1 // A manifest was deleted (a site was deleted). AuditEvent_DeleteManifest AuditEvent = 2 + // A manifest was deleted (a site has expired). + AuditEvent_ExpireManifest AuditEvent = 5 // A domain was frozen. AuditEvent_FreezeDomain AuditEvent = 3 // A domain was thawed. @@ -152,6 +154,7 @@ var ( 0: "InvalidEvent", 1: "CommitManifest", 2: "DeleteManifest", + 5: "ExpireManifest", 3: "FreezeDomain", 4: "UnfreezeDomain", } @@ -159,6 +162,7 @@ var ( "InvalidEvent": 0, "CommitManifest": 1, "DeleteManifest": 2, + "ExpireManifest": 5, "FreezeDomain": 3, "UnfreezeDomain": 4, } @@ -650,6 +654,8 @@ type Manifest struct { Redirects []*RedirectRule `protobuf:"bytes,6,rep,name=redirects" json:"redirects,omitempty"` Headers []*HeaderRule `protobuf:"bytes,9,rep,name=headers" json:"headers,omitempty"` BasicAuth []*BasicAuthRule `protobuf:"bytes,11,rep,name=basic_auth,json=basicAuth" json:"basic_auth,omitempty"` + // Site expiration. + ExpiresAt *timestamppb.Timestamp `protobuf:"bytes,12,opt,name=expires_at,json=expiresAt" json:"expires_at,omitempty"` // Diagnostics for non-fatal errors. Problems []*Problem `protobuf:"bytes,7,rep,name=problems" json:"problems,omitempty"` unknownFields protoimpl.UnknownFields @@ -756,6 +762,13 @@ func (x *Manifest) GetBasicAuth() []*BasicAuthRule { return nil } +func (x *Manifest) GetExpiresAt() *timestamppb.Timestamp { + if x != nil { + return x.ExpiresAt + } + return nil +} + func (x *Manifest) GetProblems() []*Problem { if x != nil { return x.Problems @@ -1021,7 +1034,7 @@ const file_schema_proto_rawDesc = "" + "\vcredentials\x18\x02 \x03(\v2\x10.BasicCredentialR\vcredentials\"3\n" + "\aProblem\x12\x12\n" + "\x04path\x18\x01 \x01(\tR\x04path\x12\x14\n" + - "\x05cause\x18\x02 \x01(\tR\x05cause\"\xe7\x03\n" + + "\x05cause\x18\x02 \x01(\tR\x05cause\"\xa2\x04\n" + "\bManifest\x12\x19\n" + "\brepo_url\x18\x01 \x01(\tR\arepoUrl\x12\x16\n" + "\x06branch\x18\x02 \x01(\tR\x06branch\x12\x16\n" + @@ -1035,7 +1048,9 @@ const file_schema_proto_rawDesc = "" + "\tredirects\x18\x06 \x03(\v2\r.RedirectRuleR\tredirects\x12%\n" + "\aheaders\x18\t \x03(\v2\v.HeaderRuleR\aheaders\x12-\n" + "\n" + - "basic_auth\x18\v \x03(\v2\x0e.BasicAuthRuleR\tbasicAuth\x12$\n" + + "basic_auth\x18\v \x03(\v2\x0e.BasicAuthRuleR\tbasicAuth\x129\n" + + "\n" + + "expires_at\x18\f \x01(\v2\x1a.google.protobuf.TimestampR\texpiresAt\x12$\n" + "\bproblems\x18\a \x03(\v2\b.ProblemR\bproblems\x1aC\n" + "\rContentsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x1c\n" + @@ -1071,12 +1086,13 @@ const file_schema_proto_rawDesc = "" + "\aSymlink\x10\x04*#\n" + "\tTransform\x12\f\n" + "\bIdentity\x10\x00\x12\b\n" + - "\x04Zstd\x10\x01*l\n" + + "\x04Zstd\x10\x01*\x80\x01\n" + "\n" + "AuditEvent\x12\x10\n" + "\fInvalidEvent\x10\x00\x12\x12\n" + "\x0eCommitManifest\x10\x01\x12\x12\n" + - "\x0eDeleteManifest\x10\x02\x12\x10\n" + + "\x0eDeleteManifest\x10\x02\x12\x12\n" + + "\x0eExpireManifest\x10\x05\x12\x10\n" + "\fFreezeDomain\x10\x03\x12\x12\n" + "\x0eUnfreezeDomain\x10\x04B,Z*codeberg.org/git-pages/git-pages/git_pagesb\beditionsp\xe8\a" @@ -1121,18 +1137,19 @@ var file_schema_proto_depIdxs = []int32{ 4, // 5: Manifest.redirects:type_name -> RedirectRule 6, // 6: Manifest.headers:type_name -> HeaderRule 8, // 7: Manifest.basic_auth:type_name -> BasicAuthRule - 9, // 8: Manifest.problems:type_name -> Problem - 15, // 9: AuditRecord.timestamp:type_name -> google.protobuf.Timestamp - 2, // 10: AuditRecord.event:type_name -> AuditEvent - 12, // 11: AuditRecord.principal:type_name -> Principal - 10, // 12: AuditRecord.manifest:type_name -> Manifest - 13, // 13: Principal.forge_user:type_name -> ForgeUser - 3, // 14: Manifest.ContentsEntry.value:type_name -> Entry - 15, // [15:15] is the sub-list for method output_type - 15, // [15:15] is the sub-list for method input_type - 15, // [15:15] is the sub-list for extension type_name - 15, // [15:15] is the sub-list for extension extendee - 0, // [0:15] is the sub-list for field type_name + 15, // 8: Manifest.expires_at:type_name -> google.protobuf.Timestamp + 9, // 9: Manifest.problems:type_name -> Problem + 15, // 10: AuditRecord.timestamp:type_name -> google.protobuf.Timestamp + 2, // 11: AuditRecord.event:type_name -> AuditEvent + 12, // 12: AuditRecord.principal:type_name -> Principal + 10, // 13: AuditRecord.manifest:type_name -> Manifest + 13, // 14: Principal.forge_user:type_name -> ForgeUser + 3, // 15: Manifest.ContentsEntry.value:type_name -> Entry + 16, // [16:16] is the sub-list for method output_type + 16, // [16:16] is the sub-list for method input_type + 16, // [16:16] is the sub-list for extension type_name + 16, // [16:16] is the sub-list for extension extendee + 0, // [0:16] is the sub-list for field type_name } func init() { file_schema_proto_init() } diff --git a/src/schema.proto b/src/schema.proto index 99fca45..379b4b3 100644 --- a/src/schema.proto +++ b/src/schema.proto @@ -108,6 +108,9 @@ message Manifest { repeated HeaderRule headers = 9; repeated BasicAuthRule basic_auth = 11; + // Site expiration. + google.protobuf.Timestamp expires_at = 12; + // Diagnostics for non-fatal errors. repeated Problem problems = 7; } @@ -119,6 +122,8 @@ enum AuditEvent { CommitManifest = 1; // A manifest was deleted (a site was deleted). DeleteManifest = 2; + // A manifest was deleted (a site has expired). + ExpireManifest = 5; // A domain was frozen. FreezeDomain = 3; // A domain was thawed. diff --git a/src/update.go b/src/update.go index 1b8b2cc..5867d17 100644 --- a/src/update.go +++ b/src/update.go @@ -6,8 +6,10 @@ import ( "fmt" "io" "strings" + "time" "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" ) const BlobReferencePrefix = "/git/blobs/" @@ -20,6 +22,16 @@ func (err UnresolvedRefError) Error() string { return fmt.Sprintf("%d unresolved blob references", len(err.missing)) } +type UpdateOptions struct { + expiresAt time.Time +} + +func (opts *UpdateOptions) Apply(manifest *Manifest) { + if !opts.expiresAt.IsZero() { + manifest.ExpiresAt = timestamppb.New(opts.expiresAt) + } +} + type UpdateOutcome int const ( @@ -37,6 +49,8 @@ type UpdateResult struct { err error } +var errExpireExistingSite = fmt.Errorf("cannot expire an existing site") + func Update( ctx context.Context, webRoot string, oldManifest, newManifest *Manifest, opts ModifyManifestOptions, @@ -45,7 +59,9 @@ func Update( var storedManifest *Manifest outcome := UpdateError - if IsManifestEmpty(newManifest) { + if oldManifest != nil && oldManifest.ExpiresAt == nil && newManifest.ExpiresAt != nil { + err = errExpireExistingSite + } else if IsManifestEmpty(newManifest) { storedManifest, err = newManifest, backend.DeleteManifest(ctx, webRoot, opts) if err == nil { if oldManifest == nil { @@ -101,14 +117,20 @@ func UpdateFromRepository( webRoot string, repoURL string, branch string, + opts UpdateOptions, ) (result UpdateResult) { span, ctx := ObserveFunction(ctx, "UpdateFromRepository", "repo.url", repoURL) defer span.Finish() + defer observeUpdateResult(result) logc.Printf(ctx, "update %s: %s %s\n", webRoot, repoURL, branch) - // Ignore errors; worst case we have to re-fetch all of the blobs. - oldManifest, _, _ := backend.GetManifest(ctx, webRoot, GetManifestOptions{}) + oldManifest, _, err := backend.GetManifest(ctx, webRoot, GetManifestOptions{}) + if err != nil && !errors.Is(err, ErrObjectNotFound) { + logc.Printf(ctx, "update %s err: %s", webRoot, err) + result = UpdateResult{UpdateError, nil, err} + return + } newManifest, err := FetchRepository(ctx, repoURL, branch, oldManifest) if errors.Is(err, context.DeadlineExceeded) { @@ -116,11 +138,11 @@ func UpdateFromRepository( } else if err != nil { result = UpdateResult{UpdateError, nil, err} } else { + opts.Apply(newManifest) result = Update(ctx, webRoot, oldManifest, newManifest, ModifyManifestOptions{}) } - observeUpdateResult(result) - return result + return } var errArchiveFormat = errors.New("unsupported archive format") @@ -131,11 +153,19 @@ func UpdateFromArchive( repoURL string, contentType string, reader io.Reader, + opts UpdateOptions, ) (result UpdateResult) { - var err error + span, ctx := ObserveFunction(ctx, "UpdateFromArchive", + "repo.url", repoURL, "archive.type", contentType) + defer span.Finish() + defer observeUpdateResult(result) - // Ignore errors; worst case we have to re-fetch all of the blobs. - oldManifest, _, _ := backend.GetManifest(ctx, webRoot, GetManifestOptions{}) + oldManifest, _, err := backend.GetManifest(ctx, webRoot, GetManifestOptions{}) + if err != nil && !errors.Is(err, ErrObjectNotFound) { + logc.Printf(ctx, "update %s err: %s", webRoot, err) + result = UpdateResult{UpdateError, nil, err} + return + } extractTar := func(ctx context.Context, reader io.Reader) (*Manifest, error) { return ExtractTar(ctx, reader, oldManifest) @@ -166,11 +196,9 @@ func UpdateFromArchive( if repoURL != "" { newManifest.RepoUrl = &repoURL } - + opts.Apply(newManifest) result = Update(ctx, webRoot, oldManifest, newManifest, ModifyManifestOptions{}) } - - observeUpdateResult(result) return } @@ -180,8 +208,11 @@ func PartialUpdateFromArchive( contentType string, reader io.Reader, parents CreateParentsMode, + opts UpdateOptions, ) (result UpdateResult) { - var err error + span, ctx := ObserveFunction(ctx, "PartialUpdateFromArchive", "archive.type", contentType) + defer span.Finish() + defer observeUpdateResult(result) // Here the old manifest is used both as a substrate to which a patch is applied, as well // as a "load linked" operation for a future "store conditional" update which, taken together, @@ -190,7 +221,8 @@ func PartialUpdateFromArchive( GetManifestOptions{BypassCache: true}) if err != nil { logc.Printf(ctx, "patch %s err: %s", webRoot, err) - return UpdateResult{UpdateError, nil, err} + result = UpdateResult{UpdateError, nil, err} + return } applyTarPatch := func(ctx context.Context, reader io.Reader) (*Manifest, error) { @@ -227,6 +259,7 @@ func PartialUpdateFromArchive( logc.Printf(ctx, "patch %s err: %s", webRoot, err) result = UpdateResult{UpdateError, nil, err} } else { + opts.Apply(newManifest) result = Update(ctx, webRoot, oldManifest, newManifest, ModifyManifestOptions{ IfUnmodifiedSince: oldMetadata.LastModified, @@ -240,7 +273,6 @@ func PartialUpdateFromArchive( } } - observeUpdateResult(result) return }