Parse _redirects file and store rules in manifest.

flake.nix

@@ -19,7 +19,7 @@
 
         git-pages = pkgs.buildGo125Module {
           pname = "git-pages";
-          version = "1.0.0";
+          version = "0";
 
           src = nix-filter {
             root = self;
@@ -42,7 +42,7 @@
             "-s -w"
           ];
 
-          vendorHash = "sha256-M4tTB0zXwLnehONGrtSl6B92gfpjm83g+qqM1HtNjis=";
+          vendorHash = "sha256-RCuX+z74m+G+Ptg/DF727VxnV0WEQWLaQPdN9+Ma9Do=";
 
           fixupPhase = ''
             # Apparently `go install` doesn't support renaming the binary, so country girls make do.

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/maypok86/otter/v2 v2.2.1
 	github.com/minio/minio-go/v7 v7.0.95
 	github.com/pelletier/go-toml/v2 v2.2.4
+	github.com/tj/go-redirects v0.0.0-20200911105812-fd1ba1020b37
 	github.com/valyala/fasttemplate v1.2.2
 	google.golang.org/protobuf v1.36.9
 )
@@ -34,9 +35,11 @@ require (
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
 	github.com/philhofer/fwd v1.2.0 // indirect
 	github.com/pjbgf/sha1cd v0.5.0 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	github.com/rs/xid v1.6.0 // indirect
 	github.com/sergi/go-diff v1.4.0 // indirect
 	github.com/tinylib/msgp v1.3.0 // indirect
+	github.com/tj/assert v0.0.3 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	golang.org/x/crypto v0.41.0 // indirect
 	golang.org/x/exp v0.0.0-20250531010427-b6e5de432a8b // indirect

go.sum

@@ -69,6 +69,8 @@ github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
 github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
 github.com/pjbgf/sha1cd v0.5.0 h1:a+UkboSi1znleCDUNT3M5YxjOnN1fz2FhN48FlwCxs0=
 github.com/pjbgf/sha1cd v0.5.0/go.mod h1:lhpGlyHLpQZoxMv8HcgXvZEhcGs0PG/vsZnEJ7H0iCM=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
@@ -77,10 +79,15 @@ github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw=
 github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
 github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
 github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww=
 github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0=
+github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk=
+github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk=
+github.com/tj/go-redirects v0.0.0-20200911105812-fd1ba1020b37 h1:K11tjwz8zTTSZkz4TUjfLN+y8uJWP38BbyPqZ2yB/Yk=
+github.com/tj/go-redirects v0.0.0-20200911105812-fd1ba1020b37/go.mod h1:E0E2H2gQA+uoi27VCSU+a/BULPtadQA78q3cpTjZbZw=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo=
@@ -103,5 +110,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

src/manifest.go

@@ -7,6 +7,7 @@ import (
 	"crypto/sha256"
 	"errors"
 	"fmt"
+	"log"
 	"path"
 	"strings"
 	"sync"
@@ -102,14 +103,31 @@ again:
 	}
 }
 
+// Apply post-processing steps to the manifest.
+// At the moment, there isn't a good way to report errors except to log them on the terminal.
+// (Perhaps in the future they could be exposed at `.git-pages/status.txt`?)
+func PrepareManifest(manifest *Manifest) error {
+	// Parse Netlify-style `_redirects`
+	if err := ProcessRedirects(manifest); err != nil {
+		log.Printf("redirects err: %s\n", err)
+	} else if len(manifest.Redirects) > 0 {
+		log.Printf("redirects ok: %d rules\n", len(manifest.Redirects))
+	}
+
+	return nil
+}
+
 const ExternalSizeMin uint32 = 256
 
+// Replaces inline file data over certain size with references to an external content-addressable
+// store, without performing any I/O. Returns an updated copy of the manifest.
 func ExternalizeFiles(manifest *Manifest) *Manifest {
 	newManifest := Manifest{
-		RepoUrl:  manifest.RepoUrl,
-		Branch:   manifest.Branch,
-		Commit:   manifest.Commit,
-		Contents: make(map[string]*Entry),
+		RepoUrl:   manifest.RepoUrl,
+		Branch:    manifest.Branch,
+		Commit:    manifest.Commit,
+		Contents:  make(map[string]*Entry),
+		Redirects: manifest.Redirects,
 	}
 	var totalSize uint32
 	for name, entry := range manifest.Contents {
@@ -130,8 +148,8 @@ func ExternalizeFiles(manifest *Manifest) *Manifest {
 
 const ManifestSizeMax int = 1048576
 
-// Accepts a manifest with inline files, returns a manifest with external files after writing
-// file contents and the manifest itself to the storage.
+// Uploads inline file data over certain size to the storage backend. Returns a copy of
+// the manifest updated to refer to an external content-addressable store.
 func StoreManifest(name string, manifest *Manifest) (*Manifest, error) {
 	extManifest := ExternalizeFiles(manifest)
 	extManifestData := EncodeManifest(extManifest)

src/redirects.go

@@ -0,0 +1,91 @@
+package main
+
+import (
+	"fmt"
+	"net/http"
+	"slices"
+	"strings"
+
+	"github.com/tj/go-redirects"
+	"google.golang.org/protobuf/proto"
+)
+
+const redirectsFile string = "_redirects"
+
+func unparseRule(rule redirects.Rule) string {
+	var statusPart string
+	if rule.Force {
+		statusPart = fmt.Sprintf("%d!", rule.Status)
+	} else {
+		statusPart = fmt.Sprintf("%d", rule.Status)
+	}
+	parts := []string{
+		rule.From,
+		rule.To,
+		statusPart,
+	}
+	for name, value := range rule.Params {
+		parts = append(parts, fmt.Sprintf("%s=%s", name, value))
+	}
+	return strings.Join(parts, " ")
+}
+
+var validRedirectHTTPCodes []uint = []uint{
+	http.StatusOK,
+	http.StatusMovedPermanently,
+	http.StatusFound,
+	http.StatusSeeOther,
+	http.StatusTemporaryRedirect,
+	http.StatusPermanentRedirect,
+	http.StatusForbidden,
+	http.StatusNotFound,
+	http.StatusGone,
+	http.StatusTeapot,
+	http.StatusUnavailableForLegalReasons,
+}
+
+func validateRule(rule redirects.Rule) error {
+	if len(rule.Params) > 0 {
+		return fmt.Errorf("rules with parameters are not supported")
+	}
+	if rule.IsProxy() {
+		return fmt.Errorf("proxy rules are not supported")
+	}
+	if !slices.Contains(validRedirectHTTPCodes, uint(rule.Status)) {
+		return fmt.Errorf("rule cannot use status %d: must be %v",
+			rule.Status, validRedirectHTTPCodes)
+	}
+	if strings.Contains(rule.From, "*") && !strings.HasSuffix(rule.From, "/*") {
+		return fmt.Errorf("splat * must be its own final segment of the path")
+	}
+	return nil
+}
+
+// Parses redirects file and injects rules into the manifest.
+func ProcessRedirects(manifest *Manifest) error {
+	redirectsEntry := manifest.Contents[redirectsFile]
+	delete(manifest.Contents, redirectsFile)
+	if redirectsEntry == nil {
+		return nil
+	} else if redirectsEntry.GetType() != Type_InlineFile {
+		return fmt.Errorf("%q is not a regular file", redirectsFile)
+	}
+
+	rules, err := redirects.ParseString(string(redirectsEntry.GetData()))
+	if err != nil {
+		return fmt.Errorf("syntax error: %w", err)
+	}
+
+	for index, rule := range rules {
+		if err := validateRule(rule); err != nil {
+			return fmt.Errorf("rule #%d: %w (in %q)", index+1, err, unparseRule(rule))
+		}
+		manifest.Redirects = append(manifest.Redirects, &Redirect{
+			From:   proto.String(rule.From),
+			To:     proto.String(rule.To),
+			Status: proto.Uint32(uint32(rule.Status)),
+			Force:  proto.Bool(rule.Force),
+		})
+	}
+	return nil
+}

src/schema.pb.go

@@ -141,6 +141,74 @@ func (x *Entry) GetData() []byte {
 	return nil
 }
 
+type Redirect struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	From          *string                `protobuf:"bytes,1,opt,name=from" json:"from,omitempty"`
+	To            *string                `protobuf:"bytes,2,opt,name=to" json:"to,omitempty"`
+	Status        *uint32                `protobuf:"varint,3,opt,name=status" json:"status,omitempty"`
+	Force         *bool                  `protobuf:"varint,4,opt,name=force" json:"force,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *Redirect) Reset() {
+	*x = Redirect{}
+	mi := &file_schema_proto_msgTypes[1]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *Redirect) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Redirect) ProtoMessage() {}
+
+func (x *Redirect) ProtoReflect() protoreflect.Message {
+	mi := &file_schema_proto_msgTypes[1]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use Redirect.ProtoReflect.Descriptor instead.
+func (*Redirect) Descriptor() ([]byte, []int) {
+	return file_schema_proto_rawDescGZIP(), []int{1}
+}
+
+func (x *Redirect) GetFrom() string {
+	if x != nil && x.From != nil {
+		return *x.From
+	}
+	return ""
+}
+
+func (x *Redirect) GetTo() string {
+	if x != nil && x.To != nil {
+		return *x.To
+	}
+	return ""
+}
+
+func (x *Redirect) GetStatus() uint32 {
+	if x != nil && x.Status != nil {
+		return *x.Status
+	}
+	return 0
+}
+
+func (x *Redirect) GetForce() bool {
+	if x != nil && x.Force != nil {
+		return *x.Force
+	}
+	return false
+}
+
 type Manifest struct {
 	state         protoimpl.MessageState `protogen:"open.v1"`
 	RepoUrl       *string                `protobuf:"bytes,1,opt,name=repo_url,json=repoUrl" json:"repo_url,omitempty"`
@@ -148,13 +216,14 @@ type Manifest struct {
 	Commit        *string                `protobuf:"bytes,3,opt,name=commit" json:"commit,omitempty"`
 	Contents      map[string]*Entry      `protobuf:"bytes,4,rep,name=contents" json:"contents,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
 	TotalSize     *uint32                `protobuf:"varint,5,opt,name=total_size,json=totalSize" json:"total_size,omitempty"`
+	Redirects     []*Redirect            `protobuf:"bytes,6,rep,name=redirects" json:"redirects,omitempty"`
 	unknownFields protoimpl.UnknownFields
 	sizeCache     protoimpl.SizeCache
 }
 
 func (x *Manifest) Reset() {
 	*x = Manifest{}
-	mi := &file_schema_proto_msgTypes[1]
+	mi := &file_schema_proto_msgTypes[2]
 	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 	ms.StoreMessageInfo(mi)
 }
@@ -166,7 +235,7 @@ func (x *Manifest) String() string {
 func (*Manifest) ProtoMessage() {}
 
 func (x *Manifest) ProtoReflect() protoreflect.Message {
-	mi := &file_schema_proto_msgTypes[1]
+	mi := &file_schema_proto_msgTypes[2]
 	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -179,7 +248,7 @@ func (x *Manifest) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use Manifest.ProtoReflect.Descriptor instead.
 func (*Manifest) Descriptor() ([]byte, []int) {
-	return file_schema_proto_rawDescGZIP(), []int{1}
+	return file_schema_proto_rawDescGZIP(), []int{2}
 }
 
 func (x *Manifest) GetRepoUrl() string {
@@ -217,6 +286,13 @@ func (x *Manifest) GetTotalSize() uint32 {
 	return 0
 }
 
+func (x *Manifest) GetRedirects() []*Redirect {
+	if x != nil {
+		return x.Redirects
+	}
+	return nil
+}
+
 var File_schema_proto protoreflect.FileDescriptor
 
 const file_schema_proto_rawDesc = "" +
@@ -225,14 +301,20 @@ const file_schema_proto_rawDesc = "" +
 	"\x05Entry\x12\x19\n" +
 	"\x04type\x18\x01 \x01(\x0e2\x05.TypeR\x04type\x12\x12\n" +
 	"\x04size\x18\x02 \x01(\rR\x04size\x12\x12\n" +
-	"\x04data\x18\x03 \x01(\fR\x04data\"\xee\x01\n" +
+	"\x04data\x18\x03 \x01(\fR\x04data\"\\\n" +
+	"\bRedirect\x12\x12\n" +
+	"\x04from\x18\x01 \x01(\tR\x04from\x12\x0e\n" +
+	"\x02to\x18\x02 \x01(\tR\x02to\x12\x16\n" +
+	"\x06status\x18\x03 \x01(\rR\x06status\x12\x14\n" +
+	"\x05force\x18\x04 \x01(\bR\x05force\"\x97\x02\n" +
 	"\bManifest\x12\x19\n" +
 	"\brepo_url\x18\x01 \x01(\tR\arepoUrl\x12\x16\n" +
 	"\x06branch\x18\x02 \x01(\tR\x06branch\x12\x16\n" +
 	"\x06commit\x18\x03 \x01(\tR\x06commit\x123\n" +
 	"\bcontents\x18\x04 \x03(\v2\x17.Manifest.ContentsEntryR\bcontents\x12\x1d\n" +
 	"\n" +
-	"total_size\x18\x05 \x01(\rR\ttotalSize\x1aC\n" +
+	"total_size\x18\x05 \x01(\rR\ttotalSize\x12'\n" +
+	"\tredirects\x18\x06 \x03(\v2\t.RedirectR\tredirects\x1aC\n" +
 	"\rContentsEntry\x12\x10\n" +
 	"\x03key\x18\x01 \x01(\tR\x03key\x12\x1c\n" +
 	"\x05value\x18\x02 \x01(\v2\x06.EntryR\x05value:\x028\x01*Q\n" +
@@ -257,22 +339,24 @@ func file_schema_proto_rawDescGZIP() []byte {
 }
 
 var file_schema_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
-var file_schema_proto_msgTypes = make([]protoimpl.MessageInfo, 3)
+var file_schema_proto_msgTypes = make([]protoimpl.MessageInfo, 4)
 var file_schema_proto_goTypes = []any{
 	(Type)(0),        // 0: Type
 	(*Entry)(nil),    // 1: Entry
-	(*Manifest)(nil), // 2: Manifest
-	nil,              // 3: Manifest.ContentsEntry
+	(*Redirect)(nil), // 2: Redirect
+	(*Manifest)(nil), // 3: Manifest
+	nil,              // 4: Manifest.ContentsEntry
 }
 var file_schema_proto_depIdxs = []int32{
 	0, // 0: Entry.type:type_name -> Type
-	3, // 1: Manifest.contents:type_name -> Manifest.ContentsEntry
-	1, // 2: Manifest.ContentsEntry.value:type_name -> Entry
-	3, // [3:3] is the sub-list for method output_type
-	3, // [3:3] is the sub-list for method input_type
-	3, // [3:3] is the sub-list for extension type_name
-	3, // [3:3] is the sub-list for extension extendee
-	0, // [0:3] is the sub-list for field type_name
+	4, // 1: Manifest.contents:type_name -> Manifest.ContentsEntry
+	2, // 2: Manifest.redirects:type_name -> Redirect
+	1, // 3: Manifest.ContentsEntry.value:type_name -> Entry
+	4, // [4:4] is the sub-list for method output_type
+	4, // [4:4] is the sub-list for method input_type
+	4, // [4:4] is the sub-list for extension type_name
+	4, // [4:4] is the sub-list for extension extendee
+	0, // [0:4] is the sub-list for field type_name
 }
 
 func init() { file_schema_proto_init() }
@@ -286,7 +370,7 @@ func file_schema_proto_init() {
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
 			RawDescriptor: unsafe.Slice(unsafe.StringData(file_schema_proto_rawDesc), len(file_schema_proto_rawDesc)),
 			NumEnums:      1,
-			NumMessages:   3,
+			NumMessages:   4,
 			NumExtensions: 0,
 			NumServices:   0,
 		},

src/schema.proto

@@ -21,10 +21,18 @@ message Entry {
 	bytes data = 3;
 }
 
+message Redirect {
+	string from = 1;
+	string to = 2;
+	uint32 status = 3;
+	bool force = 4;
+}
+
 message Manifest {
 	string repo_url = 1;
 	string branch = 2;
 	string commit = 3;
 	map<string, Entry> contents = 4;
 	uint32 total_size = 5;
+	repeated Redirect redirects = 6;
 }

src/update.go

@@ -51,7 +51,7 @@ func Update(
 					outcome = UpdateDeleted
 				}
 			}
-		} else {
+		} else if err = PrepareManifest(fetchManifest); err == nil {
 			newManifest, err = StoreManifest(webRoot, fetchManifest)
 			if err == nil {
 				if oldManifest == nil {