Implement site expiration.

Requires `feature = ["expiration"]`.

README.md

@@ -90,6 +90,7 @@ Features
     - If the site has no contents after the update is applied, performs the same action as `DELETE`.
 * In response to a `DELETE` request, the server unpublishes a site. The URL of the request must be the root URL of the site that is being unpublished. Site data remains stored for an indeterminate period of time, but becomes completely inaccessible.
 * If a `Dry-Run: yes` header is provided with a `PUT`, `PATCH`, `DELETE`, or `POST` request, only the authorization checks are run; no destructive updates are made.
+* If a `Expires: <timestamp>` header is provided with a `PUT` or `PATCH` request, and the `[limits].allow-expiration` configuration option is enabled, and the site with that name does not exist or is already scheduled to expire enabled, the site is then scheduled to expire at `<timestamp>` (in the HTTP date format, e.g. `Mon, 02 Jan 2006 15:04:05 GMT`). Expired sites are removed by the `git-pages -site-expire` command, which must be scheduled to periorically run for this feature to work.
 * All updates to site content are atomic (subject to consistency guarantees of the storage backend). That is, there is an instantaneous moment during an update before which the server will return the old content and after which it will return the new content.
 * Files with a certain name, when placed in the root of a site, have special functions:
     - [Netlify `_redirects`][_redirects] file can be used to specify HTTP redirect and rewrite rules. The _git-pages_ implementation currently does not support placeholders, query parameters, or conditions, and may differ from Netlify in other minor ways. If you find that a supported `_redirects` file feature does not work the same as on Netlify, please file an issue. (Note that _git-pages_ does not perform URL normalization; `/foo` and `/foo/` are *not* the same, unlike with Netlify.)

conf/config.default.toml

@@ -27,6 +27,7 @@ forbidden-domains = []
 allowed-repository-url-prefixes = []
 allowed-custom-headers = ['X-Clacks-Overhead']
 allow-basic-auth = false
+allow-expiration = false
 
 [audit]
 node-id = 0

conf/config.example.toml

@@ -11,6 +11,7 @@ metrics = "tcp/localhost:3002"
 
 [[wildcard]] # non-default section
 domain = "codeberg.page"
+preview-domain = "preview.codeberg.page"
 clone-url = "https://codeberg.org/<user>/<project>.git"
 index-repo = "pages"
 index-repo-branch = "main"
@@ -54,6 +55,7 @@ forbidden-domains = []
 allowed-repository-url-prefixes = []
 allowed-custom-headers = ["X-Clacks-Overhead"]
 allow-basic-auth = false
+allow-expiration = false
 
 [audit]
 node-id = 0

src/audit.go

@@ -394,6 +394,20 @@ func (audited *auditedBackend) DeleteManifest(
 	return audited.Backend.DeleteManifest(ctx, name, opts)
 }
 
+func (audited *auditedBackend) ExpireManifest(ctx context.Context, name string) (err error) {
+	domain, project, ok := strings.Cut(name, "/")
+	if !ok {
+		panic("malformed manifest name")
+	}
+	audited.appendNewAuditRecord(ctx, &AuditRecord{
+		Event:   AuditEvent_ExpireManifest.Enum(),
+		Domain:  proto.String(domain),
+		Project: proto.String(project),
+	})
+
+	return audited.Backend.ExpireManifest(ctx, name)
+}
+
 func (audited *auditedBackend) FreezeDomain(ctx context.Context, domain string) (err error) {
 	audited.appendNewAuditRecord(ctx, &AuditRecord{
 		Event:  AuditEvent_FreezeDomain.Enum(),

src/backend.go

@@ -121,9 +121,12 @@ type Backend interface {
 	// the old version or the new version of the manifest, never anything else.
 	CommitManifest(ctx context.Context, name string, manifest *Manifest, opts ModifyManifestOptions) error
 
-	// Delete a manifest.
+	// Delete a manifest. This operation is initiated via the API.
 	DeleteManifest(ctx context.Context, name string, opts ModifyManifestOptions) error
 
+	// Expire a manifest. This operation is initiated via a scheduled job.
+	ExpireManifest(ctx context.Context, name string) error
+
 	// Iterate through metadata of all manifests. Whether manifests that are newly added during
 	// iteration will appear in the results is unspecified.
 	EnumerateManifests(ctx context.Context) iter.Seq2[*ManifestMetadata, error]

src/backend_fs.go

@@ -405,6 +405,12 @@ func (fs *FSBackend) DeleteManifest(
 	}
 }
 
+func (fs *FSBackend) ExpireManifest(
+	ctx context.Context, name string,
+) error {
+	return fs.DeleteManifest(ctx, name, ModifyManifestOptions{})
+}
+
 func (fs *FSBackend) EnumerateManifests(ctx context.Context) iter.Seq2[*ManifestMetadata, error] {
 	return func(yield func(*ManifestMetadata, error) bool) {
 		iofs.WalkDir(fs.siteRoot.FS(), ".",

src/backend_s3.go

@@ -668,6 +668,12 @@ func (s3 *S3Backend) DeleteManifest(
 	return nil
 }
 
+func (s3 *S3Backend) ExpireManifest(
+	ctx context.Context, name string,
+) error {
+	return s3.DeleteManifest(ctx, name, ModifyManifestOptions{})
+}
+
 func (s3 *S3Backend) EnumerateManifests(ctx context.Context) iter.Seq2[*ManifestMetadata, error] {
 	return func(yield func(*ManifestMetadata, error) bool) {
 		logc.Println(ctx, "s3: enumerate manifests")

src/config.go

@@ -150,6 +150,10 @@ type LimitsConfig struct {
 	// Whether to allow Netlify-style credentials specified in a `Basic-Auth:` pseudo-header.
 	// These credentials are plaintext.
 	AllowBasicAuth bool `toml:"allow-basic-auth" default:"false"`
+	// Whether to allow an expiration date to be specified for uploaded sites. If enabled, you
+	// must also configure a scheduled job calling `git-pages -site-expire` regularly (perhaps
+	// hourly or daily).
+	AllowExpiration bool `toml:"allow-expiration" default:"false"`
 }
 
 type AuditConfig struct {

src/main.go

@@ -44,7 +44,7 @@ func configureFeatures(ctx context.Context) (err error) {
 	for _, feature := range config.Features {
 		switch feature {
 		// Work-in-progress features:
-		case "preview":
+		case "preview", "expiration":
 		// Permanently unstable features:
 		case "codeberg-pages-compat", "relaxed-idna":
 		// Stabilized features:
@@ -216,6 +216,8 @@ func usage() {
 		"git-pages {-audit-expire <days>|-audit-detach <domain>/<project>}\n")
 	fmt.Fprintf(os.Stderr, "(audit)  "+
 		"git-pages  -audit-server <endpoint> <program> [args...]\n")
+	fmt.Fprintf(os.Stderr, "(maint)  "+
+		"git-pages  -site-expire [-dry-run]\n")
 	fmt.Fprintf(os.Stderr, "(maint)  "+
 		"git-pages {-run-migration <name>|-trace-garbage|-size-histogram {original|stored}}\n")
 	flag.PrintDefaults()
@@ -263,12 +265,16 @@ func Main(versionInfo string) {
 		"detach all blobs of audit records for a single `site` (or the entire domain with 'domain.tld/*')")
 	auditServer := flag.String("audit-server", "",
 		"listen for notifications on `endpoint` and spawn a process for each audit event")
+	siteExpire := flag.Bool("site-expire", false,
+		"expire sites according to their manifest")
 	runMigration := flag.String("run-migration", "",
 		"run a store `migration` (one of: create-domain-markers)")
 	sizeHistogram := flag.String("size-histogram", "",
 		"display histogram of `size-type` (original or stored) per domain")
 	traceGarbage := flag.Bool("trace-garbage", false,
 		"estimate total size of unreachable blobs")
+	dryRun := flag.Bool("dry-run", false,
+		"print what would be performed instead of executing it")
 	version := flag.Bool("version", false,
 		"display version")
 	flag.Parse()
@@ -294,6 +300,7 @@ func Main(versionInfo string) {
 		*auditExpire != "",
 		*auditDetach != "",
 		*auditServer != "",
+		*siteExpire,
 		*runMigration != "",
 		*sizeHistogram != "",
 		*traceGarbage,
@@ -305,8 +312,11 @@ func Main(versionInfo string) {
 	if cliOperations > 1 {
 		logc.Fatalln(ctx, "-list-blobs, -list-manifests, -get-blob, -get-manifest, -get-archive, "+
 			"-update-site, -freeze-domain, -unfreeze-domain, -audit-log, -audit-read, "+
-			"-audit-rollback, -audit-expire, -audit-detach, -audit-server, -run-migration, "+
-			"-size-histogram, and -trace-garbage are mutually exclusive")
+			"-audit-rollback, -audit-expire, -audit-detach, -audit-server, -site-expire, "+
+			"-run-migration, -size-histogram, and -trace-garbage are mutually exclusive")
+	}
+	if *dryRun && !(*siteExpire) {
+		logc.Fatalln(ctx, "-dry-run is not applicable in this context")
 	}
 
 	if *configTomlPath != "" && *noConfig {
@@ -455,7 +465,7 @@ func Main(versionInfo string) {
 			}
 
 			webRoot := webRootArg(*updateSite)
-			result = UpdateFromArchive(ctx, webRoot, "", contentType, file)
+			result = UpdateFromArchive(ctx, webRoot, "", contentType, file, UpdateOptions{})
 		} else {
 			branch := "pages"
 			if sourceURL.Fragment != "" {
@@ -463,7 +473,7 @@ func Main(versionInfo string) {
 			}
 
 			webRoot := webRootArg(*updateSite)
-			result = UpdateFromRepository(ctx, webRoot, sourceURL.String(), branch)
+			result = UpdateFromRepository(ctx, webRoot, sourceURL.String(), branch, UpdateOptions{})
 		}
 
 		switch result.outcome {
@@ -647,7 +657,6 @@ func Main(versionInfo string) {
 		for id, err := range ids {
 			if err != nil {
 				logc.Fatalln(ctx, err)
-				continue
 			}
 
 			err = backend.ExpireAuditRecord(ctx, id)
@@ -661,6 +670,44 @@ func Main(versionInfo string) {
 
 		logc.Printf(ctx, "audit: expired %d records\n", count)
 
+	case *siteExpire:
+		ctx = WithPrincipal(ctx)
+		GetPrincipal(ctx).CliAdmin = proto.Bool(true)
+
+		if !config.Feature("expiration") {
+			logc.Fatalf(ctx, "expire: feature disabled")
+		}
+
+		countExpired, countTransient := 0, 0
+		for item, err := range backend.GetAllManifests(ctx) {
+			metadata, manifest := item.Splat()
+			if err != nil {
+				logc.Fatalln(ctx, err)
+			}
+			if manifest.ExpiresAt != nil {
+				countTransient += 1
+				if manifest.ExpiresAt.AsTime().Before(time.Now()) {
+					if !*dryRun {
+						err = backend.ExpireManifest(ctx, metadata.Name)
+						if err != nil {
+							logc.Fatalln(ctx, err)
+						}
+					}
+					logc.Printf(ctx, "expire: site %s expired at %s",
+						metadata.Name, manifest.ExpiresAt.AsTime())
+					countExpired += 1
+				}
+			}
+		}
+
+		if *dryRun {
+			logc.Printf(ctx, "expire: would expire %d out of %d transient sites (dry run)\n",
+				countExpired, countTransient)
+		} else {
+			logc.Printf(ctx, "expire: expired %d out of %d transient sites\n",
+				countExpired, countTransient)
+		}
+
 	case *runMigration != "":
 		if err = RunMigration(ctx, *runMigration); err != nil {
 			logc.Fatalln(ctx, err)

src/manifest.go

@@ -59,8 +59,12 @@ func IsManifestEmpty(manifest *Manifest) bool {
 	panic(fmt.Errorf("malformed manifest %v", manifest))
 }
 
-// Returns `true` if `left` and `right` contain the same files with the same types and data.
+// Returns `true` if `left` and `right` contain the same files with the same types and data,
+// and use the same options.
 func CompareManifest(left *Manifest, right *Manifest) bool {
+	if left.ExpiresAt.AsTime() != right.ExpiresAt.AsTime() {
+		return false
+	}
 	if len(left.Contents) != len(right.Contents) {
 		return false
 	}
@@ -80,6 +84,9 @@ func CompareManifest(left *Manifest, right *Manifest) bool {
 }
 
 func EncodeManifest(manifest *Manifest) (data []byte) {
+	if manifest.ExpiresAt != nil && manifest.ExpiresAt.AsTime().IsZero() {
+		panic("invalid expires_at value") // ambiguous and shouldn't happen
+	}
 	data, err := proto.MarshalOptions{Deterministic: true}.Marshal(manifest)
 	if err != nil {
 		panic(err)

src/observe.go

@@ -294,6 +294,13 @@ func (backend *observedBackend) DeleteManifest(ctx context.Context, name string,
 	return
 }
 
+func (backend *observedBackend) ExpireManifest(ctx context.Context, name string) (err error) {
+	span, ctx := ObserveFunction(ctx, "ExpireManifest", "manifest.name", name)
+	err = backend.inner.ExpireManifest(ctx, name)
+	span.Finish()
+	return
+}
+
 func (backend *observedBackend) EnumerateManifests(ctx context.Context) iter.Seq2[*ManifestMetadata, error] {
 	return func(yield func(*ManifestMetadata, error) bool) {
 		span, ctx := ObserveFunction(ctx, "EnumerateManifests")

src/pages.go

@@ -492,6 +492,27 @@ func checkDryRun(w http.ResponseWriter, r *http.Request) bool {
 	return false
 }
 
+func getUpdateOptions(w http.ResponseWriter, r *http.Request) (opts UpdateOptions, ok bool) {
+	var err error
+
+	if config.Feature("expiration") {
+		if expires := r.Header.Get("Expires"); expires != "" {
+			opts.expiresAt, err = time.Parse(http.TimeFormat, expires)
+			if err != nil {
+				http.Error(w, "malformed Expires: header", http.StatusBadRequest)
+				return
+			}
+			if !config.Limits.AllowExpiration {
+				http.Error(w, "expiration forbidden by policy", http.StatusBadRequest)
+				return
+			}
+		}
+	}
+
+	ok = true
+	return
+}
+
 func putPage(w http.ResponseWriter, r *http.Request) error {
 	var result UpdateResult
 
@@ -509,6 +530,11 @@ func putPage(w http.ResponseWriter, r *http.Request) error {
 		return err
 	}
 
+	opts, ok := getUpdateOptions(w, r)
+	if !ok {
+		return nil
+	}
+
 	ctx, cancel := context.WithTimeout(r.Context(), time.Duration(config.Limits.UpdateTimeout))
 	defer cancel()
 
@@ -543,7 +569,7 @@ func putPage(w http.ResponseWriter, r *http.Request) error {
 			return nil
 		}
 
-		result = UpdateFromRepository(ctx, webRoot, repoURL, branch)
+		result = UpdateFromRepository(ctx, webRoot, repoURL, branch, opts)
 
 	default:
 		auth, err := AuthorizeUpdateFromArchive(r)
@@ -562,7 +588,7 @@ func putPage(w http.ResponseWriter, r *http.Request) error {
 
 		// request body contains archive
 		reader := http.MaxBytesReader(w, r.Body, int64(config.Limits.MaxSiteSize.Bytes()))
-		result = UpdateFromArchive(ctx, webRoot, repoURL, contentType, reader)
+		result = UpdateFromArchive(ctx, webRoot, repoURL, contentType, reader, opts)
 	}
 
 	return reportUpdateResult(w, r, result)
@@ -583,6 +609,11 @@ func patchPage(w http.ResponseWriter, r *http.Request) error {
 		return err
 	}
 
+	opts, ok := getUpdateOptions(w, r)
+	if !ok {
+		return nil
+	}
+
 	auth, err := AuthorizeUpdateFromArchive(r)
 	if err != nil {
 		return err
@@ -631,7 +662,7 @@ func patchPage(w http.ResponseWriter, r *http.Request) error {
 
 	contentType := getMediaType(r.Header.Get("Content-Type"))
 	reader := http.MaxBytesReader(w, r.Body, int64(config.Limits.MaxSiteSize.Bytes()))
-	result := PartialUpdateFromArchive(ctx, webRoot, contentType, reader, parents)
+	result := PartialUpdateFromArchive(ctx, webRoot, contentType, reader, parents, opts)
 	return reportUpdateResult(w, r, result)
 }
 
@@ -830,7 +861,7 @@ func postPage(w http.ResponseWriter, r *http.Request) error {
 		ctx, cancel := context.WithTimeout(ctx, time.Duration(config.Limits.UpdateTimeout))
 		defer cancel()
 
-		result := UpdateFromRepository(ctx, webRoot, repoURL, auth.branch)
+		result := UpdateFromRepository(ctx, webRoot, repoURL, auth.branch, UpdateOptions{})
 		resultChan <- result
 		observeSiteUpdate("webhook", &result)
 	}(context.WithoutCancel(r.Context()))

src/schema.pb.go

@@ -140,6 +140,8 @@ const (
 	AuditEvent_CommitManifest AuditEvent = 1
 	// A manifest was deleted (a site was deleted).
 	AuditEvent_DeleteManifest AuditEvent = 2
+	// A manifest was deleted (a site has expired).
+	AuditEvent_ExpireManifest AuditEvent = 5
 	// A domain was frozen.
 	AuditEvent_FreezeDomain AuditEvent = 3
 	// A domain was thawed.
@@ -152,6 +154,7 @@ var (
 		0: "InvalidEvent",
 		1: "CommitManifest",
 		2: "DeleteManifest",
+		5: "ExpireManifest",
 		3: "FreezeDomain",
 		4: "UnfreezeDomain",
 	}
@@ -159,6 +162,7 @@ var (
 		"InvalidEvent":   0,
 		"CommitManifest": 1,
 		"DeleteManifest": 2,
+		"ExpireManifest": 5,
 		"FreezeDomain":   3,
 		"UnfreezeDomain": 4,
 	}
@@ -650,6 +654,8 @@ type Manifest struct {
 	Redirects []*RedirectRule  `protobuf:"bytes,6,rep,name=redirects" json:"redirects,omitempty"`
 	Headers   []*HeaderRule    `protobuf:"bytes,9,rep,name=headers" json:"headers,omitempty"`
 	BasicAuth []*BasicAuthRule `protobuf:"bytes,11,rep,name=basic_auth,json=basicAuth" json:"basic_auth,omitempty"`
+	// Site expiration.
+	ExpiresAt *timestamppb.Timestamp `protobuf:"bytes,12,opt,name=expires_at,json=expiresAt" json:"expires_at,omitempty"`
 	// Diagnostics for non-fatal errors.
 	Problems      []*Problem `protobuf:"bytes,7,rep,name=problems" json:"problems,omitempty"`
 	unknownFields protoimpl.UnknownFields
@@ -756,6 +762,13 @@ func (x *Manifest) GetBasicAuth() []*BasicAuthRule {
 	return nil
 }
 
+func (x *Manifest) GetExpiresAt() *timestamppb.Timestamp {
+	if x != nil {
+		return x.ExpiresAt
+	}
+	return nil
+}
+
 func (x *Manifest) GetProblems() []*Problem {
 	if x != nil {
 		return x.Problems
@@ -1021,7 +1034,7 @@ const file_schema_proto_rawDesc = "" +
 	"\vcredentials\x18\x02 \x03(\v2\x10.BasicCredentialR\vcredentials\"3\n" +
 	"\aProblem\x12\x12\n" +
 	"\x04path\x18\x01 \x01(\tR\x04path\x12\x14\n" +
-	"\x05cause\x18\x02 \x01(\tR\x05cause\"\xe7\x03\n" +
+	"\x05cause\x18\x02 \x01(\tR\x05cause\"\xa2\x04\n" +
 	"\bManifest\x12\x19\n" +
 	"\brepo_url\x18\x01 \x01(\tR\arepoUrl\x12\x16\n" +
 	"\x06branch\x18\x02 \x01(\tR\x06branch\x12\x16\n" +
@@ -1035,7 +1048,9 @@ const file_schema_proto_rawDesc = "" +
 	"\tredirects\x18\x06 \x03(\v2\r.RedirectRuleR\tredirects\x12%\n" +
 	"\aheaders\x18\t \x03(\v2\v.HeaderRuleR\aheaders\x12-\n" +
 	"\n" +
-	"basic_auth\x18\v \x03(\v2\x0e.BasicAuthRuleR\tbasicAuth\x12$\n" +
+	"basic_auth\x18\v \x03(\v2\x0e.BasicAuthRuleR\tbasicAuth\x129\n" +
+	"\n" +
+	"expires_at\x18\f \x01(\v2\x1a.google.protobuf.TimestampR\texpiresAt\x12$\n" +
 	"\bproblems\x18\a \x03(\v2\b.ProblemR\bproblems\x1aC\n" +
 	"\rContentsEntry\x12\x10\n" +
 	"\x03key\x18\x01 \x01(\tR\x03key\x12\x1c\n" +
@@ -1071,12 +1086,13 @@ const file_schema_proto_rawDesc = "" +
 	"\aSymlink\x10\x04*#\n" +
 	"\tTransform\x12\f\n" +
 	"\bIdentity\x10\x00\x12\b\n" +
-	"\x04Zstd\x10\x01*l\n" +
+	"\x04Zstd\x10\x01*\x80\x01\n" +
 	"\n" +
 	"AuditEvent\x12\x10\n" +
 	"\fInvalidEvent\x10\x00\x12\x12\n" +
 	"\x0eCommitManifest\x10\x01\x12\x12\n" +
-	"\x0eDeleteManifest\x10\x02\x12\x10\n" +
+	"\x0eDeleteManifest\x10\x02\x12\x12\n" +
+	"\x0eExpireManifest\x10\x05\x12\x10\n" +
 	"\fFreezeDomain\x10\x03\x12\x12\n" +
 	"\x0eUnfreezeDomain\x10\x04B,Z*codeberg.org/git-pages/git-pages/git_pagesb\beditionsp\xe8\a"
 
@@ -1121,18 +1137,19 @@ var file_schema_proto_depIdxs = []int32{
 	4,  // 5: Manifest.redirects:type_name -> RedirectRule
 	6,  // 6: Manifest.headers:type_name -> HeaderRule
 	8,  // 7: Manifest.basic_auth:type_name -> BasicAuthRule
-	9,  // 8: Manifest.problems:type_name -> Problem
-	15, // 9: AuditRecord.timestamp:type_name -> google.protobuf.Timestamp
-	2,  // 10: AuditRecord.event:type_name -> AuditEvent
-	12, // 11: AuditRecord.principal:type_name -> Principal
-	10, // 12: AuditRecord.manifest:type_name -> Manifest
-	13, // 13: Principal.forge_user:type_name -> ForgeUser
-	3,  // 14: Manifest.ContentsEntry.value:type_name -> Entry
-	15, // [15:15] is the sub-list for method output_type
-	15, // [15:15] is the sub-list for method input_type
-	15, // [15:15] is the sub-list for extension type_name
-	15, // [15:15] is the sub-list for extension extendee
-	0,  // [0:15] is the sub-list for field type_name
+	15, // 8: Manifest.expires_at:type_name -> google.protobuf.Timestamp
+	9,  // 9: Manifest.problems:type_name -> Problem
+	15, // 10: AuditRecord.timestamp:type_name -> google.protobuf.Timestamp
+	2,  // 11: AuditRecord.event:type_name -> AuditEvent
+	12, // 12: AuditRecord.principal:type_name -> Principal
+	10, // 13: AuditRecord.manifest:type_name -> Manifest
+	13, // 14: Principal.forge_user:type_name -> ForgeUser
+	3,  // 15: Manifest.ContentsEntry.value:type_name -> Entry
+	16, // [16:16] is the sub-list for method output_type
+	16, // [16:16] is the sub-list for method input_type
+	16, // [16:16] is the sub-list for extension type_name
+	16, // [16:16] is the sub-list for extension extendee
+	0,  // [0:16] is the sub-list for field type_name
 }
 
 func init() { file_schema_proto_init() }

src/schema.proto

@@ -108,6 +108,9 @@ message Manifest {
 	repeated HeaderRule headers = 9;
 	repeated BasicAuthRule basic_auth = 11;
 
+	// Site expiration.
+	google.protobuf.Timestamp expires_at = 12;
+
 	// Diagnostics for non-fatal errors.
 	repeated Problem problems = 7;
 }
@@ -119,6 +122,8 @@ enum AuditEvent {
 	CommitManifest = 1;
 	// A manifest was deleted (a site was deleted).
 	DeleteManifest = 2;
+	// A manifest was deleted (a site has expired).
+	ExpireManifest = 5;
 	// A domain was frozen.
 	FreezeDomain = 3;
 	// A domain was thawed.

src/update.go

@@ -6,8 +6,10 @@ import (
 	"fmt"
 	"io"
 	"strings"
+	"time"
 
 	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
 const BlobReferencePrefix = "/git/blobs/"
@@ -20,6 +22,16 @@ func (err UnresolvedRefError) Error() string {
 	return fmt.Sprintf("%d unresolved blob references", len(err.missing))
 }
 
+type UpdateOptions struct {
+	expiresAt time.Time
+}
+
+func (opts *UpdateOptions) Apply(manifest *Manifest) {
+	if !opts.expiresAt.IsZero() {
+		manifest.ExpiresAt = timestamppb.New(opts.expiresAt)
+	}
+}
+
 type UpdateOutcome int
 
 const (
@@ -37,6 +49,8 @@ type UpdateResult struct {
 	err      error
 }
 
+var errExpireExistingSite = fmt.Errorf("cannot expire an existing site")
+
 func Update(
 	ctx context.Context, webRoot string, oldManifest, newManifest *Manifest,
 	opts ModifyManifestOptions,
@@ -45,7 +59,9 @@ func Update(
 	var storedManifest *Manifest
 
 	outcome := UpdateError
-	if IsManifestEmpty(newManifest) {
+	if oldManifest != nil && oldManifest.ExpiresAt == nil && newManifest.ExpiresAt != nil {
+		err = errExpireExistingSite
+	} else if IsManifestEmpty(newManifest) {
 		storedManifest, err = newManifest, backend.DeleteManifest(ctx, webRoot, opts)
 		if err == nil {
 			if oldManifest == nil {
@@ -101,14 +117,20 @@ func UpdateFromRepository(
 	webRoot string,
 	repoURL string,
 	branch string,
+	opts UpdateOptions,
 ) (result UpdateResult) {
 	span, ctx := ObserveFunction(ctx, "UpdateFromRepository", "repo.url", repoURL)
 	defer span.Finish()
+	defer observeUpdateResult(result)
 
 	logc.Printf(ctx, "update %s: %s %s\n", webRoot, repoURL, branch)
 
-	// Ignore errors; worst case we have to re-fetch all of the blobs.
-	oldManifest, _, _ := backend.GetManifest(ctx, webRoot, GetManifestOptions{})
+	oldManifest, _, err := backend.GetManifest(ctx, webRoot, GetManifestOptions{})
+	if err != nil && !errors.Is(err, ErrObjectNotFound) {
+		logc.Printf(ctx, "update %s err: %s", webRoot, err)
+		result = UpdateResult{UpdateError, nil, err}
+		return
+	}
 
 	newManifest, err := FetchRepository(ctx, repoURL, branch, oldManifest)
 	if errors.Is(err, context.DeadlineExceeded) {
@@ -116,11 +138,11 @@ func UpdateFromRepository(
 	} else if err != nil {
 		result = UpdateResult{UpdateError, nil, err}
 	} else {
+		opts.Apply(newManifest)
 		result = Update(ctx, webRoot, oldManifest, newManifest, ModifyManifestOptions{})
 	}
 
-	observeUpdateResult(result)
-	return result
+	return
 }
 
 var errArchiveFormat = errors.New("unsupported archive format")
@@ -131,11 +153,19 @@ func UpdateFromArchive(
 	repoURL string,
 	contentType string,
 	reader io.Reader,
+	opts UpdateOptions,
 ) (result UpdateResult) {
-	var err error
+	span, ctx := ObserveFunction(ctx, "UpdateFromArchive",
+		"repo.url", repoURL, "archive.type", contentType)
+	defer span.Finish()
+	defer observeUpdateResult(result)
 
-	// Ignore errors; worst case we have to re-fetch all of the blobs.
-	oldManifest, _, _ := backend.GetManifest(ctx, webRoot, GetManifestOptions{})
+	oldManifest, _, err := backend.GetManifest(ctx, webRoot, GetManifestOptions{})
+	if err != nil && !errors.Is(err, ErrObjectNotFound) {
+		logc.Printf(ctx, "update %s err: %s", webRoot, err)
+		result = UpdateResult{UpdateError, nil, err}
+		return
+	}
 
 	extractTar := func(ctx context.Context, reader io.Reader) (*Manifest, error) {
 		return ExtractTar(ctx, reader, oldManifest)
@@ -166,11 +196,9 @@ func UpdateFromArchive(
 		if repoURL != "" {
 			newManifest.RepoUrl = &repoURL
 		}
-
+		opts.Apply(newManifest)
 		result = Update(ctx, webRoot, oldManifest, newManifest, ModifyManifestOptions{})
 	}
-
-	observeUpdateResult(result)
 	return
 }
 
@@ -180,8 +208,11 @@ func PartialUpdateFromArchive(
 	contentType string,
 	reader io.Reader,
 	parents CreateParentsMode,
+	opts UpdateOptions,
 ) (result UpdateResult) {
-	var err error
+	span, ctx := ObserveFunction(ctx, "PartialUpdateFromArchive", "archive.type", contentType)
+	defer span.Finish()
+	defer observeUpdateResult(result)
 
 	// Here the old manifest is used both as a substrate to which a patch is applied, as well
 	// as a "load linked" operation for a future "store conditional" update which, taken together,
@@ -190,7 +221,8 @@ func PartialUpdateFromArchive(
 		GetManifestOptions{BypassCache: true})
 	if err != nil {
 		logc.Printf(ctx, "patch %s err: %s", webRoot, err)
-		return UpdateResult{UpdateError, nil, err}
+		result = UpdateResult{UpdateError, nil, err}
+		return
 	}
 
 	applyTarPatch := func(ctx context.Context, reader io.Reader) (*Manifest, error) {
@@ -227,6 +259,7 @@ func PartialUpdateFromArchive(
 		logc.Printf(ctx, "patch %s err: %s", webRoot, err)
 		result = UpdateResult{UpdateError, nil, err}
 	} else {
+		opts.Apply(newManifest)
 		result = Update(ctx, webRoot, oldManifest, newManifest,
 			ModifyManifestOptions{
 				IfUnmodifiedSince: oldMetadata.LastModified,
@@ -240,7 +273,6 @@ func PartialUpdateFromArchive(
 		}
 	}
 
-	observeUpdateResult(result)
 	return
 }