diff --git a/flake.nix b/flake.nix index 309b27a..655f73c 100644 --- a/flake.nix +++ b/flake.nix @@ -19,7 +19,7 @@ git-pages = pkgs.buildGo125Module { pname = "git-pages"; - version = "1.0.0"; + version = "0"; src = nix-filter { root = self; @@ -42,7 +42,7 @@ "-s -w" ]; - vendorHash = "sha256-M4tTB0zXwLnehONGrtSl6B92gfpjm83g+qqM1HtNjis="; + vendorHash = "sha256-RCuX+z74m+G+Ptg/DF727VxnV0WEQWLaQPdN9+Ma9Do="; fixupPhase = '' # Apparently `go install` doesn't support renaming the binary, so country girls make do. diff --git a/go.mod b/go.mod index 96a3044..22c7d4d 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/maypok86/otter/v2 v2.2.1 github.com/minio/minio-go/v7 v7.0.95 github.com/pelletier/go-toml/v2 v2.2.4 + github.com/tj/go-redirects v0.0.0-20200911105812-fd1ba1020b37 github.com/valyala/fasttemplate v1.2.2 google.golang.org/protobuf v1.36.9 ) @@ -34,9 +35,11 @@ require ( github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/philhofer/fwd v1.2.0 // indirect github.com/pjbgf/sha1cd v0.5.0 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/rs/xid v1.6.0 // indirect github.com/sergi/go-diff v1.4.0 // indirect github.com/tinylib/msgp v1.3.0 // indirect + github.com/tj/assert v0.0.3 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect golang.org/x/crypto v0.41.0 // indirect golang.org/x/exp v0.0.0-20250531010427-b6e5de432a8b // indirect diff --git a/go.sum b/go.sum index b9c023f..f4a00d7 100644 --- a/go.sum +++ b/go.sum @@ -69,6 +69,8 @@ github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pjbgf/sha1cd v0.5.0 h1:a+UkboSi1znleCDUNT3M5YxjOnN1fz2FhN48FlwCxs0= github.com/pjbgf/sha1cd v0.5.0/go.mod h1:lhpGlyHLpQZoxMv8HcgXvZEhcGs0PG/vsZnEJ7H0iCM= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= @@ -77,10 +79,15 @@ github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww= github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0= +github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= +github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= +github.com/tj/go-redirects v0.0.0-20200911105812-fd1ba1020b37 h1:K11tjwz8zTTSZkz4TUjfLN+y8uJWP38BbyPqZ2yB/Yk= +github.com/tj/go-redirects v0.0.0-20200911105812-fd1ba1020b37/go.mod h1:E0E2H2gQA+uoi27VCSU+a/BULPtadQA78q3cpTjZbZw= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= @@ -103,5 +110,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/src/manifest.go b/src/manifest.go index 5cdf3b5..a0346b8 100644 --- a/src/manifest.go +++ b/src/manifest.go @@ -7,6 +7,7 @@ import ( "crypto/sha256" "errors" "fmt" + "log" "path" "strings" "sync" @@ -102,14 +103,31 @@ again: } } +// Apply post-processing steps to the manifest. +// At the moment, there isn't a good way to report errors except to log them on the terminal. +// (Perhaps in the future they could be exposed at `.git-pages/status.txt`?) +func PrepareManifest(manifest *Manifest) error { + // Parse Netlify-style `_redirects` + if err := ProcessRedirects(manifest); err != nil { + log.Printf("redirects err: %s\n", err) + } else if len(manifest.Redirects) > 0 { + log.Printf("redirects ok: %d rules\n", len(manifest.Redirects)) + } + + return nil +} + const ExternalSizeMin uint32 = 256 +// Replaces inline file data over certain size with references to an external content-addressable +// store, without performing any I/O. Returns an updated copy of the manifest. func ExternalizeFiles(manifest *Manifest) *Manifest { newManifest := Manifest{ - RepoUrl: manifest.RepoUrl, - Branch: manifest.Branch, - Commit: manifest.Commit, - Contents: make(map[string]*Entry), + RepoUrl: manifest.RepoUrl, + Branch: manifest.Branch, + Commit: manifest.Commit, + Contents: make(map[string]*Entry), + Redirects: manifest.Redirects, } var totalSize uint32 for name, entry := range manifest.Contents { @@ -130,8 +148,8 @@ func ExternalizeFiles(manifest *Manifest) *Manifest { const ManifestSizeMax int = 1048576 -// Accepts a manifest with inline files, returns a manifest with external files after writing -// file contents and the manifest itself to the storage. +// Uploads inline file data over certain size to the storage backend. Returns a copy of +// the manifest updated to refer to an external content-addressable store. func StoreManifest(name string, manifest *Manifest) (*Manifest, error) { extManifest := ExternalizeFiles(manifest) extManifestData := EncodeManifest(extManifest) diff --git a/src/redirects.go b/src/redirects.go new file mode 100644 index 0000000..ce4c06f --- /dev/null +++ b/src/redirects.go @@ -0,0 +1,91 @@ +package main + +import ( + "fmt" + "net/http" + "slices" + "strings" + + "github.com/tj/go-redirects" + "google.golang.org/protobuf/proto" +) + +const redirectsFile string = "_redirects" + +func unparseRule(rule redirects.Rule) string { + var statusPart string + if rule.Force { + statusPart = fmt.Sprintf("%d!", rule.Status) + } else { + statusPart = fmt.Sprintf("%d", rule.Status) + } + parts := []string{ + rule.From, + rule.To, + statusPart, + } + for name, value := range rule.Params { + parts = append(parts, fmt.Sprintf("%s=%s", name, value)) + } + return strings.Join(parts, " ") +} + +var validRedirectHTTPCodes []uint = []uint{ + http.StatusOK, + http.StatusMovedPermanently, + http.StatusFound, + http.StatusSeeOther, + http.StatusTemporaryRedirect, + http.StatusPermanentRedirect, + http.StatusForbidden, + http.StatusNotFound, + http.StatusGone, + http.StatusTeapot, + http.StatusUnavailableForLegalReasons, +} + +func validateRule(rule redirects.Rule) error { + if len(rule.Params) > 0 { + return fmt.Errorf("rules with parameters are not supported") + } + if rule.IsProxy() { + return fmt.Errorf("proxy rules are not supported") + } + if !slices.Contains(validRedirectHTTPCodes, uint(rule.Status)) { + return fmt.Errorf("rule cannot use status %d: must be %v", + rule.Status, validRedirectHTTPCodes) + } + if strings.Contains(rule.From, "*") && !strings.HasSuffix(rule.From, "/*") { + return fmt.Errorf("splat * must be its own final segment of the path") + } + return nil +} + +// Parses redirects file and injects rules into the manifest. +func ProcessRedirects(manifest *Manifest) error { + redirectsEntry := manifest.Contents[redirectsFile] + delete(manifest.Contents, redirectsFile) + if redirectsEntry == nil { + return nil + } else if redirectsEntry.GetType() != Type_InlineFile { + return fmt.Errorf("%q is not a regular file", redirectsFile) + } + + rules, err := redirects.ParseString(string(redirectsEntry.GetData())) + if err != nil { + return fmt.Errorf("syntax error: %w", err) + } + + for index, rule := range rules { + if err := validateRule(rule); err != nil { + return fmt.Errorf("rule #%d: %w (in %q)", index+1, err, unparseRule(rule)) + } + manifest.Redirects = append(manifest.Redirects, &Redirect{ + From: proto.String(rule.From), + To: proto.String(rule.To), + Status: proto.Uint32(uint32(rule.Status)), + Force: proto.Bool(rule.Force), + }) + } + return nil +} diff --git a/src/schema.pb.go b/src/schema.pb.go index 4705a13..65b1932 100644 --- a/src/schema.pb.go +++ b/src/schema.pb.go @@ -141,6 +141,74 @@ func (x *Entry) GetData() []byte { return nil } +type Redirect struct { + state protoimpl.MessageState `protogen:"open.v1"` + From *string `protobuf:"bytes,1,opt,name=from" json:"from,omitempty"` + To *string `protobuf:"bytes,2,opt,name=to" json:"to,omitempty"` + Status *uint32 `protobuf:"varint,3,opt,name=status" json:"status,omitempty"` + Force *bool `protobuf:"varint,4,opt,name=force" json:"force,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Redirect) Reset() { + *x = Redirect{} + mi := &file_schema_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Redirect) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Redirect) ProtoMessage() {} + +func (x *Redirect) ProtoReflect() protoreflect.Message { + mi := &file_schema_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Redirect.ProtoReflect.Descriptor instead. +func (*Redirect) Descriptor() ([]byte, []int) { + return file_schema_proto_rawDescGZIP(), []int{1} +} + +func (x *Redirect) GetFrom() string { + if x != nil && x.From != nil { + return *x.From + } + return "" +} + +func (x *Redirect) GetTo() string { + if x != nil && x.To != nil { + return *x.To + } + return "" +} + +func (x *Redirect) GetStatus() uint32 { + if x != nil && x.Status != nil { + return *x.Status + } + return 0 +} + +func (x *Redirect) GetForce() bool { + if x != nil && x.Force != nil { + return *x.Force + } + return false +} + type Manifest struct { state protoimpl.MessageState `protogen:"open.v1"` RepoUrl *string `protobuf:"bytes,1,opt,name=repo_url,json=repoUrl" json:"repo_url,omitempty"` @@ -148,13 +216,14 @@ type Manifest struct { Commit *string `protobuf:"bytes,3,opt,name=commit" json:"commit,omitempty"` Contents map[string]*Entry `protobuf:"bytes,4,rep,name=contents" json:"contents,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` TotalSize *uint32 `protobuf:"varint,5,opt,name=total_size,json=totalSize" json:"total_size,omitempty"` + Redirects []*Redirect `protobuf:"bytes,6,rep,name=redirects" json:"redirects,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *Manifest) Reset() { *x = Manifest{} - mi := &file_schema_proto_msgTypes[1] + mi := &file_schema_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -166,7 +235,7 @@ func (x *Manifest) String() string { func (*Manifest) ProtoMessage() {} func (x *Manifest) ProtoReflect() protoreflect.Message { - mi := &file_schema_proto_msgTypes[1] + mi := &file_schema_proto_msgTypes[2] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -179,7 +248,7 @@ func (x *Manifest) ProtoReflect() protoreflect.Message { // Deprecated: Use Manifest.ProtoReflect.Descriptor instead. func (*Manifest) Descriptor() ([]byte, []int) { - return file_schema_proto_rawDescGZIP(), []int{1} + return file_schema_proto_rawDescGZIP(), []int{2} } func (x *Manifest) GetRepoUrl() string { @@ -217,6 +286,13 @@ func (x *Manifest) GetTotalSize() uint32 { return 0 } +func (x *Manifest) GetRedirects() []*Redirect { + if x != nil { + return x.Redirects + } + return nil +} + var File_schema_proto protoreflect.FileDescriptor const file_schema_proto_rawDesc = "" + @@ -225,14 +301,20 @@ const file_schema_proto_rawDesc = "" + "\x05Entry\x12\x19\n" + "\x04type\x18\x01 \x01(\x0e2\x05.TypeR\x04type\x12\x12\n" + "\x04size\x18\x02 \x01(\rR\x04size\x12\x12\n" + - "\x04data\x18\x03 \x01(\fR\x04data\"\xee\x01\n" + + "\x04data\x18\x03 \x01(\fR\x04data\"\\\n" + + "\bRedirect\x12\x12\n" + + "\x04from\x18\x01 \x01(\tR\x04from\x12\x0e\n" + + "\x02to\x18\x02 \x01(\tR\x02to\x12\x16\n" + + "\x06status\x18\x03 \x01(\rR\x06status\x12\x14\n" + + "\x05force\x18\x04 \x01(\bR\x05force\"\x97\x02\n" + "\bManifest\x12\x19\n" + "\brepo_url\x18\x01 \x01(\tR\arepoUrl\x12\x16\n" + "\x06branch\x18\x02 \x01(\tR\x06branch\x12\x16\n" + "\x06commit\x18\x03 \x01(\tR\x06commit\x123\n" + "\bcontents\x18\x04 \x03(\v2\x17.Manifest.ContentsEntryR\bcontents\x12\x1d\n" + "\n" + - "total_size\x18\x05 \x01(\rR\ttotalSize\x1aC\n" + + "total_size\x18\x05 \x01(\rR\ttotalSize\x12'\n" + + "\tredirects\x18\x06 \x03(\v2\t.RedirectR\tredirects\x1aC\n" + "\rContentsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x1c\n" + "\x05value\x18\x02 \x01(\v2\x06.EntryR\x05value:\x028\x01*Q\n" + @@ -257,22 +339,24 @@ func file_schema_proto_rawDescGZIP() []byte { } var file_schema_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_schema_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_schema_proto_msgTypes = make([]protoimpl.MessageInfo, 4) var file_schema_proto_goTypes = []any{ (Type)(0), // 0: Type (*Entry)(nil), // 1: Entry - (*Manifest)(nil), // 2: Manifest - nil, // 3: Manifest.ContentsEntry + (*Redirect)(nil), // 2: Redirect + (*Manifest)(nil), // 3: Manifest + nil, // 4: Manifest.ContentsEntry } var file_schema_proto_depIdxs = []int32{ 0, // 0: Entry.type:type_name -> Type - 3, // 1: Manifest.contents:type_name -> Manifest.ContentsEntry - 1, // 2: Manifest.ContentsEntry.value:type_name -> Entry - 3, // [3:3] is the sub-list for method output_type - 3, // [3:3] is the sub-list for method input_type - 3, // [3:3] is the sub-list for extension type_name - 3, // [3:3] is the sub-list for extension extendee - 0, // [0:3] is the sub-list for field type_name + 4, // 1: Manifest.contents:type_name -> Manifest.ContentsEntry + 2, // 2: Manifest.redirects:type_name -> Redirect + 1, // 3: Manifest.ContentsEntry.value:type_name -> Entry + 4, // [4:4] is the sub-list for method output_type + 4, // [4:4] is the sub-list for method input_type + 4, // [4:4] is the sub-list for extension type_name + 4, // [4:4] is the sub-list for extension extendee + 0, // [0:4] is the sub-list for field type_name } func init() { file_schema_proto_init() } @@ -286,7 +370,7 @@ func file_schema_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_schema_proto_rawDesc), len(file_schema_proto_rawDesc)), NumEnums: 1, - NumMessages: 3, + NumMessages: 4, NumExtensions: 0, NumServices: 0, }, diff --git a/src/schema.proto b/src/schema.proto index 088d0ba..98a91cc 100644 --- a/src/schema.proto +++ b/src/schema.proto @@ -21,10 +21,18 @@ message Entry { bytes data = 3; } +message Redirect { + string from = 1; + string to = 2; + uint32 status = 3; + bool force = 4; +} + message Manifest { string repo_url = 1; string branch = 2; string commit = 3; map contents = 4; uint32 total_size = 5; + repeated Redirect redirects = 6; } diff --git a/src/update.go b/src/update.go index b2d173a..407c99c 100644 --- a/src/update.go +++ b/src/update.go @@ -51,7 +51,7 @@ func Update( outcome = UpdateDeleted } } - } else { + } else if err = PrepareManifest(fetchManifest); err == nil { newManifest, err = StoreManifest(webRoot, fetchManifest) if err == nil { if oldManifest == nil {