diff --git a/src/fetch.go b/src/fetch.go index 167f4f0..c957a8c 100644 --- a/src/fetch.go +++ b/src/fetch.go @@ -13,6 +13,7 @@ import ( "github.com/go-git/go-git/v6/plumbing/filemode" "github.com/go-git/go-git/v6/plumbing/object" "github.com/go-git/go-git/v6/storage/filesystem" + "google.golang.org/protobuf/proto" ) const largeObjectThreshold int64 = 1048576 @@ -61,12 +62,16 @@ func FetchRepository(ctx context.Context, repoURL string, branch string) (*Manif defer walker.Close() manifest := Manifest{ - RepoURL: repoURL, - Branch: branch, - Commit: ref.Hash().String(), - Tree: make(map[string]*Entry), + RepoUrl: proto.String(repoURL), + Branch: proto.String(branch), + Commit: proto.String(ref.Hash().String()), + Files: make(map[string]*Entry), + } + manifest.Files[""] = &Entry{ + Type: Type_Directory.Enum(), + Size: proto.Uint64(0), + Data: []byte{}, } - manifest.Tree[""] = &Entry{Type: Type_Directory, Size: 0, Data: []byte{}} for { name, entry, err := walker.Next() if err == io.EOF { @@ -92,18 +97,18 @@ func FetchRepository(ctx context.Context, repoURL string, branch string) (*Manif } if entry.Mode == filemode.Symlink { - manifestEntry.Type = Type_Symlink + manifestEntry.Type = Type_Symlink.Enum() } else { - manifestEntry.Type = Type_InlineFile + manifestEntry.Type = Type_InlineFile.Enum() } - manifestEntry.Size = blob.Size + manifestEntry.Size = proto.Uint64(uint64(blob.Size)) manifestEntry.Data = data } else if entry.Mode == filemode.Dir { - manifestEntry.Type = Type_Directory + manifestEntry.Type = Type_Directory.Enum() } else { - manifestEntry.Type = Type_Invalid + manifestEntry.Type = Type_Invalid.Enum() } - manifest.Tree[name] = &manifestEntry + manifest.Files[name] = &manifestEntry } } return &manifest, nil diff --git a/src/manifest.go b/src/manifest.go index a19f1c4..5c61770 100644 --- a/src/manifest.go +++ b/src/manifest.go @@ -1,3 +1,5 @@ +//go:generate protoc --go_out=. --go_opt=paths=source_relative schema.proto + package main import ( @@ -13,11 +15,11 @@ import ( ) func IsManifestEmpty(manifest *Manifest) bool { - if len(manifest.Tree) > 1 { + if len(manifest.Files) > 1 { return false } - for name, entry := range manifest.Tree { - if name == "" && entry.Type == Type_Directory { + for name, entry := range manifest.Files { + if name == "" && entry.GetType() == Type_Directory { return true } } @@ -26,15 +28,15 @@ func IsManifestEmpty(manifest *Manifest) bool { // Returns `true` if `left` and `right` contain the same files with the same types and data. func CompareManifest(left *Manifest, right *Manifest) bool { - if len(left.Tree) != len(right.Tree) { + if len(left.Files) != len(right.Files) { return false } - for name, leftEntry := range left.Tree { - rightEntry := right.Tree[name] + for name, leftEntry := range left.Files { + rightEntry := right.Files[name] if rightEntry == nil { return false } - if leftEntry.Type != rightEntry.Type { + if leftEntry.GetType() != rightEntry.GetType() { return false } if bytes.Compare(leftEntry.Data, rightEntry.Data) != 0 { @@ -69,8 +71,8 @@ again: parts := strings.Split(inPath, "/") for i := 1; i <= len(parts); i++ { linkPath := path.Join(parts[:i]...) - entry := manifest.Tree[linkPath] - if entry != nil && entry.Type == Type_Symlink { + entry := manifest.Files[linkPath] + if entry != nil && entry.GetType() == Type_Symlink { inPath = path.Join( path.Dir(linkPath), string(entry.Data), @@ -88,24 +90,24 @@ again: } } -const ExternalSizeMin int64 = 256 +const ExternalSizeMin uint64 = 256 func ExternalizeFiles(manifest *Manifest) *Manifest { newManifest := Manifest{ - RepoURL: manifest.RepoURL, + RepoUrl: manifest.RepoUrl, Branch: manifest.Branch, Commit: manifest.Commit, - Tree: make(map[string]*Entry), + Files: make(map[string]*Entry), } - for name, entry := range manifest.Tree { - if entry.Type == Type_InlineFile && entry.Size > ExternalSizeMin { - newManifest.Tree[name] = &Entry{ - Type: Type_ExternalFile, + for name, entry := range manifest.Files { + if entry.GetType() == Type_InlineFile && entry.GetSize() > ExternalSizeMin { + newManifest.Files[name] = &Entry{ + Type: Type_ExternalFile.Enum(), Size: entry.Size, Data: fmt.Appendf(nil, "sha256-%x", sha256.Sum256(entry.Data)), } } else { - newManifest.Tree[name] = entry + newManifest.Files[name] = entry } } return &newManifest @@ -127,11 +129,11 @@ func StoreManifest(name string, manifest *Manifest) (*Manifest, error) { } wg := sync.WaitGroup{} - ch := make(chan error, len(extManifest.Tree)) - for name, entry := range extManifest.Tree { - if entry.Type == Type_ExternalFile { + ch := make(chan error, len(extManifest.Files)) + for name, entry := range extManifest.Files { + if entry.GetType() == Type_ExternalFile { wg.Go(func() { - err := backend.PutBlob(string(entry.Data), manifest.Tree[name].Data) + err := backend.PutBlob(string(entry.Data), manifest.Files[name].Data) if err != nil { ch <- fmt.Errorf("put blob %s: %w", name, err) } diff --git a/src/migrate.go b/src/migrate.go index ab0bdbb..b5df33f 100644 --- a/src/migrate.go +++ b/src/migrate.go @@ -6,11 +6,13 @@ import ( "log" "os" "path/filepath" + + "google.golang.org/protobuf/proto" ) func readToManifest(root *os.Root) (*Manifest, error) { manifest := Manifest{} - manifest.Tree = make(map[string]*Entry) + manifest.Files = make(map[string]*Entry) err := fs.WalkDir(root.FS(), ".", func(path string, dirEntry fs.DirEntry, err error) error { if err != nil { return err @@ -18,30 +20,30 @@ func readToManifest(root *os.Root) (*Manifest, error) { manifestEntry := Entry{} if dirEntry.IsDir() { - manifestEntry.Type = Type_Directory + manifestEntry.Type = Type_Directory.Enum() } else if dirEntry.Type().IsRegular() { data, err := root.ReadFile(path) if err != nil { return err } - manifestEntry.Type = Type_InlineFile - manifestEntry.Size = int64(len(data)) + manifestEntry.Type = Type_InlineFile.Enum() + manifestEntry.Size = proto.Uint64(uint64(len(data))) manifestEntry.Data = data } else if dirEntry.Type().Type() == fs.ModeSymlink { target, err := root.Readlink(path) if err != nil { return err } - manifestEntry.Type = Type_Symlink - manifestEntry.Size = int64(len(target)) + manifestEntry.Type = Type_Symlink.Enum() + manifestEntry.Size = proto.Uint64(uint64(len(target))) manifestEntry.Data = []byte(target) } else { - log.Println("migrate v1: illegal %s/%s", root.Name(), path) + log.Printf("migrate v1: illegal %s/%s\n", root.Name(), path) } if path == "." { path = "" } - manifest.Tree[path] = &manifestEntry + manifest.Files[path] = &manifestEntry return nil }) return &manifest, err diff --git a/src/pages.go b/src/pages.go index 424ed9d..0764549 100644 --- a/src/pages.go +++ b/src/pages.go @@ -64,17 +64,17 @@ func getPage(w http.ResponseWriter, r *http.Request) error { fmt.Fprintln(w, err) return err } - entry = manifest.Tree[entryPath] - if entry == nil || entry.Type == Type_Invalid { + entry = manifest.Files[entryPath] + if entry == nil || entry.GetType() == Type_Invalid { is404 = true if entryPath == notFoundPage { break } entryPath = notFoundPage continue - } else if entry.Type == Type_InlineFile { + } else if entry.GetType() == Type_InlineFile { reader = bytes.NewReader(entry.Data) - } else if entry.Type == Type_ExternalFile { + } else if entry.GetType() == Type_ExternalFile { etag := fmt.Sprintf(`"%s"`, entry.Data) if r.Header.Get("If-None-Match") == etag { w.WriteHeader(http.StatusNotModified) @@ -88,7 +88,7 @@ func getPage(w http.ResponseWriter, r *http.Request) error { } w.Header().Set("ETag", etag) } - } else if entry.Type == Type_Directory { + } else if entry.GetType() == Type_Directory { if strings.HasSuffix(r.URL.Path, "/") { entryPath = path.Join(entryPath, "index.html") continue @@ -101,7 +101,7 @@ func getPage(w http.ResponseWriter, r *http.Request) error { fmt.Fprintf(w, "see %s\n", newPath) return nil } - } else if entry.Type == Type_Symlink { + } else if entry.GetType() == Type_Symlink { return fmt.Errorf("unexpected symlink") } break @@ -207,7 +207,7 @@ func putPage(w http.ResponseWriter, r *http.Request) error { w.Header().Add("X-Pages-Outcome", "deleted") } if result.manifest != nil { - fmt.Fprintln(w, result.manifest.Commit) + fmt.Fprintln(w, *result.manifest.Commit) } else if result.err != nil { fmt.Fprintln(w, result.err) } else { diff --git a/src/schema.pb.go b/src/schema.pb.go index c0ab0e3..888c535 100644 --- a/src/schema.pb.go +++ b/src/schema.pb.go @@ -1,18 +1,17 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.36.9 -// protoc v3.21.12 -// source: src/schema.proto +// protoc v6.30.2 +// source: schema.proto package main import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" sync "sync" unsafe "unsafe" - - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" ) const ( @@ -66,11 +65,11 @@ func (x Type) String() string { } func (Type) Descriptor() protoreflect.EnumDescriptor { - return file_src_schema_proto_enumTypes[0].Descriptor() + return file_schema_proto_enumTypes[0].Descriptor() } func (Type) Type() protoreflect.EnumType { - return &file_src_schema_proto_enumTypes[0] + return &file_schema_proto_enumTypes[0] } func (x Type) Number() protoreflect.EnumNumber { @@ -79,21 +78,21 @@ func (x Type) Number() protoreflect.EnumNumber { // Deprecated: Use Type.Descriptor instead. func (Type) EnumDescriptor() ([]byte, []int) { - return file_src_schema_proto_rawDescGZIP(), []int{0} + return file_schema_proto_rawDescGZIP(), []int{0} } type Entry struct { state protoimpl.MessageState `protogen:"open.v1"` - Type Type `protobuf:"varint,1,opt,name=type,proto3,enum=main.Type" json:"type,omitempty"` - Size int64 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"` - Data []byte `protobuf:"bytes,3,opt,name=data,proto3" json:"data,omitempty"` + Type *Type `protobuf:"varint,1,opt,name=type,enum=Type" json:"type,omitempty"` + Size *uint64 `protobuf:"varint,2,opt,name=size" json:"size,omitempty"` + Data []byte `protobuf:"bytes,3,opt,name=data" json:"data,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *Entry) Reset() { *x = Entry{} - mi := &file_src_schema_proto_msgTypes[0] + mi := &file_schema_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -105,7 +104,7 @@ func (x *Entry) String() string { func (*Entry) ProtoMessage() {} func (x *Entry) ProtoReflect() protoreflect.Message { - mi := &file_src_schema_proto_msgTypes[0] + mi := &file_schema_proto_msgTypes[0] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -118,19 +117,19 @@ func (x *Entry) ProtoReflect() protoreflect.Message { // Deprecated: Use Entry.ProtoReflect.Descriptor instead. func (*Entry) Descriptor() ([]byte, []int) { - return file_src_schema_proto_rawDescGZIP(), []int{0} + return file_schema_proto_rawDescGZIP(), []int{0} } func (x *Entry) GetType() Type { - if x != nil { - return x.Type + if x != nil && x.Type != nil { + return *x.Type } return Type_Invalid } -func (x *Entry) GetSize() int64 { - if x != nil { - return x.Size +func (x *Entry) GetSize() uint64 { + if x != nil && x.Size != nil { + return *x.Size } return 0 } @@ -144,17 +143,17 @@ func (x *Entry) GetData() []byte { type Manifest struct { state protoimpl.MessageState `protogen:"open.v1"` - RepoURL string `protobuf:"bytes,1,opt,name=repoURL,proto3" json:"repoURL,omitempty"` - Branch string `protobuf:"bytes,2,opt,name=branch,proto3" json:"branch,omitempty"` - Commit string `protobuf:"bytes,3,opt,name=commit,proto3" json:"commit,omitempty"` - Tree map[string]*Entry `protobuf:"bytes,4,rep,name=tree,proto3" json:"tree,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + RepoUrl *string `protobuf:"bytes,1,opt,name=repo_url,json=repoUrl" json:"repo_url,omitempty"` + Branch *string `protobuf:"bytes,2,opt,name=branch" json:"branch,omitempty"` + Commit *string `protobuf:"bytes,3,opt,name=commit" json:"commit,omitempty"` + Files map[string]*Entry `protobuf:"bytes,4,rep,name=files" json:"files,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *Manifest) Reset() { *x = Manifest{} - mi := &file_src_schema_proto_msgTypes[1] + mi := &file_schema_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -166,7 +165,7 @@ func (x *Manifest) String() string { func (*Manifest) ProtoMessage() {} func (x *Manifest) ProtoReflect() protoreflect.Message { - mi := &file_src_schema_proto_msgTypes[1] + mi := &file_schema_proto_msgTypes[1] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -179,87 +178,87 @@ func (x *Manifest) ProtoReflect() protoreflect.Message { // Deprecated: Use Manifest.ProtoReflect.Descriptor instead. func (*Manifest) Descriptor() ([]byte, []int) { - return file_src_schema_proto_rawDescGZIP(), []int{1} + return file_schema_proto_rawDescGZIP(), []int{1} } -func (x *Manifest) GetRepoURL() string { - if x != nil { - return x.RepoURL +func (x *Manifest) GetRepoUrl() string { + if x != nil && x.RepoUrl != nil { + return *x.RepoUrl } return "" } func (x *Manifest) GetBranch() string { - if x != nil { - return x.Branch + if x != nil && x.Branch != nil { + return *x.Branch } return "" } func (x *Manifest) GetCommit() string { - if x != nil { - return x.Commit + if x != nil && x.Commit != nil { + return *x.Commit } return "" } -func (x *Manifest) GetTree() map[string]*Entry { +func (x *Manifest) GetFiles() map[string]*Entry { if x != nil { - return x.Tree + return x.Files } return nil } -var File_src_schema_proto protoreflect.FileDescriptor +var File_schema_proto protoreflect.FileDescriptor -const file_src_schema_proto_rawDesc = "" + +const file_schema_proto_rawDesc = "" + "\n" + - "\x10src/schema.proto\x12\x04main\"O\n" + - "\x05Entry\x12\x1e\n" + - "\x04type\x18\x01 \x01(\x0e2\n" + - ".main.TypeR\x04type\x12\x12\n" + - "\x04size\x18\x02 \x01(\x03R\x04size\x12\x12\n" + - "\x04data\x18\x03 \x01(\fR\x04data\"\xc8\x01\n" + - "\bManifest\x12\x18\n" + - "\arepoURL\x18\x01 \x01(\tR\arepoURL\x12\x16\n" + + "\fschema.proto\"J\n" + + "\x05Entry\x12\x19\n" + + "\x04type\x18\x01 \x01(\x0e2\x05.TypeR\x04type\x12\x12\n" + + "\x04size\x18\x02 \x01(\x04R\x04size\x12\x12\n" + + "\x04data\x18\x03 \x01(\fR\x04data\"\xc3\x01\n" + + "\bManifest\x12\x19\n" + + "\brepo_url\x18\x01 \x01(\tR\arepoUrl\x12\x16\n" + "\x06branch\x18\x02 \x01(\tR\x06branch\x12\x16\n" + - "\x06commit\x18\x03 \x01(\tR\x06commit\x12,\n" + - "\x04tree\x18\x04 \x03(\v2\x18.main.Manifest.TreeEntryR\x04tree\x1aD\n" + - "\tTreeEntry\x12\x10\n" + - "\x03key\x18\x01 \x01(\tR\x03key\x12!\n" + - "\x05value\x18\x02 \x01(\v2\v.main.EntryR\x05value:\x028\x01*Q\n" + + "\x06commit\x18\x03 \x01(\tR\x06commit\x12*\n" + + "\x05files\x18\x04 \x03(\v2\x14.Manifest.FilesEntryR\x05files\x1a@\n" + + "\n" + + "FilesEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x1c\n" + + "\x05value\x18\x02 \x01(\v2\x06.EntryR\x05value:\x028\x01*Q\n" + "\x04Type\x12\v\n" + "\aInvalid\x10\x00\x12\r\n" + "\tDirectory\x10\x01\x12\x0e\n" + "\n" + "InlineFile\x10\x02\x12\x10\n" + "\fExternalFile\x10\x03\x12\v\n" + - "\aSymlink\x10\x04b\x06proto3" + "\aSymlink\x10\x04B\x1fZ\x1dwhitequark.org/git-pages/mainb\beditionsp\xe8\a" var ( - file_src_schema_proto_rawDescOnce sync.Once - file_src_schema_proto_rawDescData []byte + file_schema_proto_rawDescOnce sync.Once + file_schema_proto_rawDescData []byte ) -func file_src_schema_proto_rawDescGZIP() []byte { - file_src_schema_proto_rawDescOnce.Do(func() { - file_src_schema_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_src_schema_proto_rawDesc), len(file_src_schema_proto_rawDesc))) +func file_schema_proto_rawDescGZIP() []byte { + file_schema_proto_rawDescOnce.Do(func() { + file_schema_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_schema_proto_rawDesc), len(file_schema_proto_rawDesc))) }) - return file_src_schema_proto_rawDescData + return file_schema_proto_rawDescData } -var file_src_schema_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_src_schema_proto_msgTypes = make([]protoimpl.MessageInfo, 3) -var file_src_schema_proto_goTypes = []any{ - (Type)(0), // 0: main.Type - (*Entry)(nil), // 1: main.Entry - (*Manifest)(nil), // 2: main.Manifest - nil, // 3: main.Manifest.TreeEntry +var file_schema_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_schema_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_schema_proto_goTypes = []any{ + (Type)(0), // 0: Type + (*Entry)(nil), // 1: Entry + (*Manifest)(nil), // 2: Manifest + nil, // 3: Manifest.FilesEntry } -var file_src_schema_proto_depIdxs = []int32{ - 0, // 0: main.Entry.type:type_name -> main.Type - 3, // 1: main.Manifest.tree:type_name -> main.Manifest.TreeEntry - 1, // 2: main.Manifest.TreeEntry.value:type_name -> main.Entry +var file_schema_proto_depIdxs = []int32{ + 0, // 0: Entry.type:type_name -> Type + 3, // 1: Manifest.files:type_name -> Manifest.FilesEntry + 1, // 2: Manifest.FilesEntry.value:type_name -> Entry 3, // [3:3] is the sub-list for method output_type 3, // [3:3] is the sub-list for method input_type 3, // [3:3] is the sub-list for extension type_name @@ -267,27 +266,27 @@ var file_src_schema_proto_depIdxs = []int32{ 0, // [0:3] is the sub-list for field type_name } -func init() { file_src_schema_proto_init() } -func file_src_schema_proto_init() { - if File_src_schema_proto != nil { +func init() { file_schema_proto_init() } +func file_schema_proto_init() { + if File_schema_proto != nil { return } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: unsafe.Slice(unsafe.StringData(file_src_schema_proto_rawDesc), len(file_src_schema_proto_rawDesc)), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_schema_proto_rawDesc), len(file_schema_proto_rawDesc)), NumEnums: 1, NumMessages: 3, NumExtensions: 0, NumServices: 0, }, - GoTypes: file_src_schema_proto_goTypes, - DependencyIndexes: file_src_schema_proto_depIdxs, - EnumInfos: file_src_schema_proto_enumTypes, - MessageInfos: file_src_schema_proto_msgTypes, + GoTypes: file_schema_proto_goTypes, + DependencyIndexes: file_schema_proto_depIdxs, + EnumInfos: file_schema_proto_enumTypes, + MessageInfos: file_schema_proto_msgTypes, }.Build() - File_src_schema_proto = out.File - file_src_schema_proto_goTypes = nil - file_src_schema_proto_depIdxs = nil + File_schema_proto = out.File + file_schema_proto_goTypes = nil + file_schema_proto_depIdxs = nil } diff --git a/src/schema.proto b/src/schema.proto index b214727..92596dc 100644 --- a/src/schema.proto +++ b/src/schema.proto @@ -1,6 +1,6 @@ -syntax = "proto3"; +edition = "2023"; -package main; +option go_package = "whitequark.org/git-pages/main"; enum Type { // Invalid entry. @@ -17,13 +17,13 @@ enum Type { message Entry { Type type = 1; - int64 size = 2; + uint64 size = 2; bytes data = 3; } message Manifest { - string repoURL = 1; + string repo_url = 1; string branch = 2; string commit = 3; - map tree = 4;; + map files = 4; } diff --git a/src/update.go b/src/update.go index 98384a9..b2d173a 100644 --- a/src/update.go +++ b/src/update.go @@ -77,7 +77,7 @@ func Update( case UpdateNoChange: status = "unchanged" } - log.Printf("update ok: %s %s %s", webRoot, newManifest.Commit, status) + log.Printf("update ok: %s %s %s", webRoot, *newManifest.Commit, status) } else { log.Printf("update err: %s %s", webRoot, err) }