Upgrade protobuf schema to edition 2023. NFCI

Also, some renames for consistency:
- `Manifest.repoURL`→`Manifest.repo_url`
- `Manifest.tree`→`Manifest.files`
This commit is contained in:
Catherine
2025-09-19 14:10:54 +00:00
parent 46d54503ee
commit d89f03e665
7 changed files with 140 additions and 132 deletions

View File

@@ -13,6 +13,7 @@ import (
"github.com/go-git/go-git/v6/plumbing/filemode"
"github.com/go-git/go-git/v6/plumbing/object"
"github.com/go-git/go-git/v6/storage/filesystem"
"google.golang.org/protobuf/proto"
)
const largeObjectThreshold int64 = 1048576
@@ -61,12 +62,16 @@ func FetchRepository(ctx context.Context, repoURL string, branch string) (*Manif
defer walker.Close()
manifest := Manifest{
RepoURL: repoURL,
Branch: branch,
Commit: ref.Hash().String(),
Tree: make(map[string]*Entry),
RepoUrl: proto.String(repoURL),
Branch: proto.String(branch),
Commit: proto.String(ref.Hash().String()),
Files: make(map[string]*Entry),
}
manifest.Files[""] = &Entry{
Type: Type_Directory.Enum(),
Size: proto.Uint64(0),
Data: []byte{},
}
manifest.Tree[""] = &Entry{Type: Type_Directory, Size: 0, Data: []byte{}}
for {
name, entry, err := walker.Next()
if err == io.EOF {
@@ -92,18 +97,18 @@ func FetchRepository(ctx context.Context, repoURL string, branch string) (*Manif
}
if entry.Mode == filemode.Symlink {
manifestEntry.Type = Type_Symlink
manifestEntry.Type = Type_Symlink.Enum()
} else {
manifestEntry.Type = Type_InlineFile
manifestEntry.Type = Type_InlineFile.Enum()
}
manifestEntry.Size = blob.Size
manifestEntry.Size = proto.Uint64(uint64(blob.Size))
manifestEntry.Data = data
} else if entry.Mode == filemode.Dir {
manifestEntry.Type = Type_Directory
manifestEntry.Type = Type_Directory.Enum()
} else {
manifestEntry.Type = Type_Invalid
manifestEntry.Type = Type_Invalid.Enum()
}
manifest.Tree[name] = &manifestEntry
manifest.Files[name] = &manifestEntry
}
}
return &manifest, nil

View File

@@ -1,3 +1,5 @@
//go:generate protoc --go_out=. --go_opt=paths=source_relative schema.proto
package main
import (
@@ -13,11 +15,11 @@ import (
)
func IsManifestEmpty(manifest *Manifest) bool {
if len(manifest.Tree) > 1 {
if len(manifest.Files) > 1 {
return false
}
for name, entry := range manifest.Tree {
if name == "" && entry.Type == Type_Directory {
for name, entry := range manifest.Files {
if name == "" && entry.GetType() == Type_Directory {
return true
}
}
@@ -26,15 +28,15 @@ func IsManifestEmpty(manifest *Manifest) bool {
// Returns `true` if `left` and `right` contain the same files with the same types and data.
func CompareManifest(left *Manifest, right *Manifest) bool {
if len(left.Tree) != len(right.Tree) {
if len(left.Files) != len(right.Files) {
return false
}
for name, leftEntry := range left.Tree {
rightEntry := right.Tree[name]
for name, leftEntry := range left.Files {
rightEntry := right.Files[name]
if rightEntry == nil {
return false
}
if leftEntry.Type != rightEntry.Type {
if leftEntry.GetType() != rightEntry.GetType() {
return false
}
if bytes.Compare(leftEntry.Data, rightEntry.Data) != 0 {
@@ -69,8 +71,8 @@ again:
parts := strings.Split(inPath, "/")
for i := 1; i <= len(parts); i++ {
linkPath := path.Join(parts[:i]...)
entry := manifest.Tree[linkPath]
if entry != nil && entry.Type == Type_Symlink {
entry := manifest.Files[linkPath]
if entry != nil && entry.GetType() == Type_Symlink {
inPath = path.Join(
path.Dir(linkPath),
string(entry.Data),
@@ -88,24 +90,24 @@ again:
}
}
const ExternalSizeMin int64 = 256
const ExternalSizeMin uint64 = 256
func ExternalizeFiles(manifest *Manifest) *Manifest {
newManifest := Manifest{
RepoURL: manifest.RepoURL,
RepoUrl: manifest.RepoUrl,
Branch: manifest.Branch,
Commit: manifest.Commit,
Tree: make(map[string]*Entry),
Files: make(map[string]*Entry),
}
for name, entry := range manifest.Tree {
if entry.Type == Type_InlineFile && entry.Size > ExternalSizeMin {
newManifest.Tree[name] = &Entry{
Type: Type_ExternalFile,
for name, entry := range manifest.Files {
if entry.GetType() == Type_InlineFile && entry.GetSize() > ExternalSizeMin {
newManifest.Files[name] = &Entry{
Type: Type_ExternalFile.Enum(),
Size: entry.Size,
Data: fmt.Appendf(nil, "sha256-%x", sha256.Sum256(entry.Data)),
}
} else {
newManifest.Tree[name] = entry
newManifest.Files[name] = entry
}
}
return &newManifest
@@ -127,11 +129,11 @@ func StoreManifest(name string, manifest *Manifest) (*Manifest, error) {
}
wg := sync.WaitGroup{}
ch := make(chan error, len(extManifest.Tree))
for name, entry := range extManifest.Tree {
if entry.Type == Type_ExternalFile {
ch := make(chan error, len(extManifest.Files))
for name, entry := range extManifest.Files {
if entry.GetType() == Type_ExternalFile {
wg.Go(func() {
err := backend.PutBlob(string(entry.Data), manifest.Tree[name].Data)
err := backend.PutBlob(string(entry.Data), manifest.Files[name].Data)
if err != nil {
ch <- fmt.Errorf("put blob %s: %w", name, err)
}

View File

@@ -6,11 +6,13 @@ import (
"log"
"os"
"path/filepath"
"google.golang.org/protobuf/proto"
)
func readToManifest(root *os.Root) (*Manifest, error) {
manifest := Manifest{}
manifest.Tree = make(map[string]*Entry)
manifest.Files = make(map[string]*Entry)
err := fs.WalkDir(root.FS(), ".", func(path string, dirEntry fs.DirEntry, err error) error {
if err != nil {
return err
@@ -18,30 +20,30 @@ func readToManifest(root *os.Root) (*Manifest, error) {
manifestEntry := Entry{}
if dirEntry.IsDir() {
manifestEntry.Type = Type_Directory
manifestEntry.Type = Type_Directory.Enum()
} else if dirEntry.Type().IsRegular() {
data, err := root.ReadFile(path)
if err != nil {
return err
}
manifestEntry.Type = Type_InlineFile
manifestEntry.Size = int64(len(data))
manifestEntry.Type = Type_InlineFile.Enum()
manifestEntry.Size = proto.Uint64(uint64(len(data)))
manifestEntry.Data = data
} else if dirEntry.Type().Type() == fs.ModeSymlink {
target, err := root.Readlink(path)
if err != nil {
return err
}
manifestEntry.Type = Type_Symlink
manifestEntry.Size = int64(len(target))
manifestEntry.Type = Type_Symlink.Enum()
manifestEntry.Size = proto.Uint64(uint64(len(target)))
manifestEntry.Data = []byte(target)
} else {
log.Println("migrate v1: illegal %s/%s", root.Name(), path)
log.Printf("migrate v1: illegal %s/%s\n", root.Name(), path)
}
if path == "." {
path = ""
}
manifest.Tree[path] = &manifestEntry
manifest.Files[path] = &manifestEntry
return nil
})
return &manifest, err

View File

@@ -64,17 +64,17 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
fmt.Fprintln(w, err)
return err
}
entry = manifest.Tree[entryPath]
if entry == nil || entry.Type == Type_Invalid {
entry = manifest.Files[entryPath]
if entry == nil || entry.GetType() == Type_Invalid {
is404 = true
if entryPath == notFoundPage {
break
}
entryPath = notFoundPage
continue
} else if entry.Type == Type_InlineFile {
} else if entry.GetType() == Type_InlineFile {
reader = bytes.NewReader(entry.Data)
} else if entry.Type == Type_ExternalFile {
} else if entry.GetType() == Type_ExternalFile {
etag := fmt.Sprintf(`"%s"`, entry.Data)
if r.Header.Get("If-None-Match") == etag {
w.WriteHeader(http.StatusNotModified)
@@ -88,7 +88,7 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
}
w.Header().Set("ETag", etag)
}
} else if entry.Type == Type_Directory {
} else if entry.GetType() == Type_Directory {
if strings.HasSuffix(r.URL.Path, "/") {
entryPath = path.Join(entryPath, "index.html")
continue
@@ -101,7 +101,7 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
fmt.Fprintf(w, "see %s\n", newPath)
return nil
}
} else if entry.Type == Type_Symlink {
} else if entry.GetType() == Type_Symlink {
return fmt.Errorf("unexpected symlink")
}
break
@@ -207,7 +207,7 @@ func putPage(w http.ResponseWriter, r *http.Request) error {
w.Header().Add("X-Pages-Outcome", "deleted")
}
if result.manifest != nil {
fmt.Fprintln(w, result.manifest.Commit)
fmt.Fprintln(w, *result.manifest.Commit)
} else if result.err != nil {
fmt.Fprintln(w, result.err)
} else {

View File

@@ -1,18 +1,17 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.36.9
// protoc v3.21.12
// source: src/schema.proto
// protoc v6.30.2
// source: schema.proto
package main
import (
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
reflect "reflect"
sync "sync"
unsafe "unsafe"
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
)
const (
@@ -66,11 +65,11 @@ func (x Type) String() string {
}
func (Type) Descriptor() protoreflect.EnumDescriptor {
return file_src_schema_proto_enumTypes[0].Descriptor()
return file_schema_proto_enumTypes[0].Descriptor()
}
func (Type) Type() protoreflect.EnumType {
return &file_src_schema_proto_enumTypes[0]
return &file_schema_proto_enumTypes[0]
}
func (x Type) Number() protoreflect.EnumNumber {
@@ -79,21 +78,21 @@ func (x Type) Number() protoreflect.EnumNumber {
// Deprecated: Use Type.Descriptor instead.
func (Type) EnumDescriptor() ([]byte, []int) {
return file_src_schema_proto_rawDescGZIP(), []int{0}
return file_schema_proto_rawDescGZIP(), []int{0}
}
type Entry struct {
state protoimpl.MessageState `protogen:"open.v1"`
Type Type `protobuf:"varint,1,opt,name=type,proto3,enum=main.Type" json:"type,omitempty"`
Size int64 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"`
Data []byte `protobuf:"bytes,3,opt,name=data,proto3" json:"data,omitempty"`
Type *Type `protobuf:"varint,1,opt,name=type,enum=Type" json:"type,omitempty"`
Size *uint64 `protobuf:"varint,2,opt,name=size" json:"size,omitempty"`
Data []byte `protobuf:"bytes,3,opt,name=data" json:"data,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *Entry) Reset() {
*x = Entry{}
mi := &file_src_schema_proto_msgTypes[0]
mi := &file_schema_proto_msgTypes[0]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -105,7 +104,7 @@ func (x *Entry) String() string {
func (*Entry) ProtoMessage() {}
func (x *Entry) ProtoReflect() protoreflect.Message {
mi := &file_src_schema_proto_msgTypes[0]
mi := &file_schema_proto_msgTypes[0]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -118,19 +117,19 @@ func (x *Entry) ProtoReflect() protoreflect.Message {
// Deprecated: Use Entry.ProtoReflect.Descriptor instead.
func (*Entry) Descriptor() ([]byte, []int) {
return file_src_schema_proto_rawDescGZIP(), []int{0}
return file_schema_proto_rawDescGZIP(), []int{0}
}
func (x *Entry) GetType() Type {
if x != nil {
return x.Type
if x != nil && x.Type != nil {
return *x.Type
}
return Type_Invalid
}
func (x *Entry) GetSize() int64 {
if x != nil {
return x.Size
func (x *Entry) GetSize() uint64 {
if x != nil && x.Size != nil {
return *x.Size
}
return 0
}
@@ -144,17 +143,17 @@ func (x *Entry) GetData() []byte {
type Manifest struct {
state protoimpl.MessageState `protogen:"open.v1"`
RepoURL string `protobuf:"bytes,1,opt,name=repoURL,proto3" json:"repoURL,omitempty"`
Branch string `protobuf:"bytes,2,opt,name=branch,proto3" json:"branch,omitempty"`
Commit string `protobuf:"bytes,3,opt,name=commit,proto3" json:"commit,omitempty"`
Tree map[string]*Entry `protobuf:"bytes,4,rep,name=tree,proto3" json:"tree,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
RepoUrl *string `protobuf:"bytes,1,opt,name=repo_url,json=repoUrl" json:"repo_url,omitempty"`
Branch *string `protobuf:"bytes,2,opt,name=branch" json:"branch,omitempty"`
Commit *string `protobuf:"bytes,3,opt,name=commit" json:"commit,omitempty"`
Files map[string]*Entry `protobuf:"bytes,4,rep,name=files" json:"files,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *Manifest) Reset() {
*x = Manifest{}
mi := &file_src_schema_proto_msgTypes[1]
mi := &file_schema_proto_msgTypes[1]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -166,7 +165,7 @@ func (x *Manifest) String() string {
func (*Manifest) ProtoMessage() {}
func (x *Manifest) ProtoReflect() protoreflect.Message {
mi := &file_src_schema_proto_msgTypes[1]
mi := &file_schema_proto_msgTypes[1]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -179,87 +178,87 @@ func (x *Manifest) ProtoReflect() protoreflect.Message {
// Deprecated: Use Manifest.ProtoReflect.Descriptor instead.
func (*Manifest) Descriptor() ([]byte, []int) {
return file_src_schema_proto_rawDescGZIP(), []int{1}
return file_schema_proto_rawDescGZIP(), []int{1}
}
func (x *Manifest) GetRepoURL() string {
if x != nil {
return x.RepoURL
func (x *Manifest) GetRepoUrl() string {
if x != nil && x.RepoUrl != nil {
return *x.RepoUrl
}
return ""
}
func (x *Manifest) GetBranch() string {
if x != nil {
return x.Branch
if x != nil && x.Branch != nil {
return *x.Branch
}
return ""
}
func (x *Manifest) GetCommit() string {
if x != nil {
return x.Commit
if x != nil && x.Commit != nil {
return *x.Commit
}
return ""
}
func (x *Manifest) GetTree() map[string]*Entry {
func (x *Manifest) GetFiles() map[string]*Entry {
if x != nil {
return x.Tree
return x.Files
}
return nil
}
var File_src_schema_proto protoreflect.FileDescriptor
var File_schema_proto protoreflect.FileDescriptor
const file_src_schema_proto_rawDesc = "" +
const file_schema_proto_rawDesc = "" +
"\n" +
"\x10src/schema.proto\x12\x04main\"O\n" +
"\x05Entry\x12\x1e\n" +
"\x04type\x18\x01 \x01(\x0e2\n" +
".main.TypeR\x04type\x12\x12\n" +
"\x04size\x18\x02 \x01(\x03R\x04size\x12\x12\n" +
"\x04data\x18\x03 \x01(\fR\x04data\"\xc8\x01\n" +
"\bManifest\x12\x18\n" +
"\arepoURL\x18\x01 \x01(\tR\arepoURL\x12\x16\n" +
"\fschema.proto\"J\n" +
"\x05Entry\x12\x19\n" +
"\x04type\x18\x01 \x01(\x0e2\x05.TypeR\x04type\x12\x12\n" +
"\x04size\x18\x02 \x01(\x04R\x04size\x12\x12\n" +
"\x04data\x18\x03 \x01(\fR\x04data\"\xc3\x01\n" +
"\bManifest\x12\x19\n" +
"\brepo_url\x18\x01 \x01(\tR\arepoUrl\x12\x16\n" +
"\x06branch\x18\x02 \x01(\tR\x06branch\x12\x16\n" +
"\x06commit\x18\x03 \x01(\tR\x06commit\x12,\n" +
"\x04tree\x18\x04 \x03(\v2\x18.main.Manifest.TreeEntryR\x04tree\x1aD\n" +
"\tTreeEntry\x12\x10\n" +
"\x03key\x18\x01 \x01(\tR\x03key\x12!\n" +
"\x05value\x18\x02 \x01(\v2\v.main.EntryR\x05value:\x028\x01*Q\n" +
"\x06commit\x18\x03 \x01(\tR\x06commit\x12*\n" +
"\x05files\x18\x04 \x03(\v2\x14.Manifest.FilesEntryR\x05files\x1a@\n" +
"\n" +
"FilesEntry\x12\x10\n" +
"\x03key\x18\x01 \x01(\tR\x03key\x12\x1c\n" +
"\x05value\x18\x02 \x01(\v2\x06.EntryR\x05value:\x028\x01*Q\n" +
"\x04Type\x12\v\n" +
"\aInvalid\x10\x00\x12\r\n" +
"\tDirectory\x10\x01\x12\x0e\n" +
"\n" +
"InlineFile\x10\x02\x12\x10\n" +
"\fExternalFile\x10\x03\x12\v\n" +
"\aSymlink\x10\x04b\x06proto3"
"\aSymlink\x10\x04B\x1fZ\x1dwhitequark.org/git-pages/mainb\beditionsp\xe8\a"
var (
file_src_schema_proto_rawDescOnce sync.Once
file_src_schema_proto_rawDescData []byte
file_schema_proto_rawDescOnce sync.Once
file_schema_proto_rawDescData []byte
)
func file_src_schema_proto_rawDescGZIP() []byte {
file_src_schema_proto_rawDescOnce.Do(func() {
file_src_schema_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_src_schema_proto_rawDesc), len(file_src_schema_proto_rawDesc)))
func file_schema_proto_rawDescGZIP() []byte {
file_schema_proto_rawDescOnce.Do(func() {
file_schema_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_schema_proto_rawDesc), len(file_schema_proto_rawDesc)))
})
return file_src_schema_proto_rawDescData
return file_schema_proto_rawDescData
}
var file_src_schema_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
var file_src_schema_proto_msgTypes = make([]protoimpl.MessageInfo, 3)
var file_src_schema_proto_goTypes = []any{
(Type)(0), // 0: main.Type
(*Entry)(nil), // 1: main.Entry
(*Manifest)(nil), // 2: main.Manifest
nil, // 3: main.Manifest.TreeEntry
var file_schema_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
var file_schema_proto_msgTypes = make([]protoimpl.MessageInfo, 3)
var file_schema_proto_goTypes = []any{
(Type)(0), // 0: Type
(*Entry)(nil), // 1: Entry
(*Manifest)(nil), // 2: Manifest
nil, // 3: Manifest.FilesEntry
}
var file_src_schema_proto_depIdxs = []int32{
0, // 0: main.Entry.type:type_name -> main.Type
3, // 1: main.Manifest.tree:type_name -> main.Manifest.TreeEntry
1, // 2: main.Manifest.TreeEntry.value:type_name -> main.Entry
var file_schema_proto_depIdxs = []int32{
0, // 0: Entry.type:type_name -> Type
3, // 1: Manifest.files:type_name -> Manifest.FilesEntry
1, // 2: Manifest.FilesEntry.value:type_name -> Entry
3, // [3:3] is the sub-list for method output_type
3, // [3:3] is the sub-list for method input_type
3, // [3:3] is the sub-list for extension type_name
@@ -267,27 +266,27 @@ var file_src_schema_proto_depIdxs = []int32{
0, // [0:3] is the sub-list for field type_name
}
func init() { file_src_schema_proto_init() }
func file_src_schema_proto_init() {
if File_src_schema_proto != nil {
func init() { file_schema_proto_init() }
func file_schema_proto_init() {
if File_schema_proto != nil {
return
}
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: unsafe.Slice(unsafe.StringData(file_src_schema_proto_rawDesc), len(file_src_schema_proto_rawDesc)),
RawDescriptor: unsafe.Slice(unsafe.StringData(file_schema_proto_rawDesc), len(file_schema_proto_rawDesc)),
NumEnums: 1,
NumMessages: 3,
NumExtensions: 0,
NumServices: 0,
},
GoTypes: file_src_schema_proto_goTypes,
DependencyIndexes: file_src_schema_proto_depIdxs,
EnumInfos: file_src_schema_proto_enumTypes,
MessageInfos: file_src_schema_proto_msgTypes,
GoTypes: file_schema_proto_goTypes,
DependencyIndexes: file_schema_proto_depIdxs,
EnumInfos: file_schema_proto_enumTypes,
MessageInfos: file_schema_proto_msgTypes,
}.Build()
File_src_schema_proto = out.File
file_src_schema_proto_goTypes = nil
file_src_schema_proto_depIdxs = nil
File_schema_proto = out.File
file_schema_proto_goTypes = nil
file_schema_proto_depIdxs = nil
}

View File

@@ -1,6 +1,6 @@
syntax = "proto3";
edition = "2023";
package main;
option go_package = "whitequark.org/git-pages/main";
enum Type {
// Invalid entry.
@@ -17,13 +17,13 @@ enum Type {
message Entry {
Type type = 1;
int64 size = 2;
uint64 size = 2;
bytes data = 3;
}
message Manifest {
string repoURL = 1;
string repo_url = 1;
string branch = 2;
string commit = 3;
map<string, Entry> tree = 4;;
map<string, Entry> files = 4;
}

View File

@@ -77,7 +77,7 @@ func Update(
case UpdateNoChange:
status = "unchanged"
}
log.Printf("update ok: %s %s %s", webRoot, newManifest.Commit, status)
log.Printf("update ok: %s %s %s", webRoot, *newManifest.Commit, status)
} else {
log.Printf("update err: %s %s", webRoot, err)
}