Refactor Protobuf schema.

This is to prepare for making manifest debug representation accessible.

- change `Entry.size` to `uint32` so that it's serialized as a number
  in protoJSON export
- rename `Manifest.files` to `Manifest.contents`
- leave size and data for the root directory empty, same as with
  non-root directories fetched from git
This commit is contained in:
Catherine
2025-09-19 15:19:42 +00:00
parent d89f03e665
commit dbfdd5d418
6 changed files with 44 additions and 49 deletions

View File

@@ -62,16 +62,12 @@ func FetchRepository(ctx context.Context, repoURL string, branch string) (*Manif
defer walker.Close()
manifest := Manifest{
RepoUrl: proto.String(repoURL),
Branch: proto.String(branch),
Commit: proto.String(ref.Hash().String()),
Files: make(map[string]*Entry),
}
manifest.Files[""] = &Entry{
Type: Type_Directory.Enum(),
Size: proto.Uint64(0),
Data: []byte{},
RepoUrl: proto.String(repoURL),
Branch: proto.String(branch),
Commit: proto.String(ref.Hash().String()),
Contents: make(map[string]*Entry),
}
manifest.Contents[""] = &Entry{Type: Type_Directory.Enum()}
for {
name, entry, err := walker.Next()
if err == io.EOF {
@@ -101,14 +97,14 @@ func FetchRepository(ctx context.Context, repoURL string, branch string) (*Manif
} else {
manifestEntry.Type = Type_InlineFile.Enum()
}
manifestEntry.Size = proto.Uint64(uint64(blob.Size))
manifestEntry.Size = proto.Uint32(uint32(blob.Size))
manifestEntry.Data = data
} else if entry.Mode == filemode.Dir {
manifestEntry.Type = Type_Directory.Enum()
} else {
manifestEntry.Type = Type_Invalid.Enum()
}
manifest.Files[name] = &manifestEntry
manifest.Contents[name] = &manifestEntry
}
}
return &manifest, nil

View File

@@ -15,10 +15,10 @@ import (
)
func IsManifestEmpty(manifest *Manifest) bool {
if len(manifest.Files) > 1 {
if len(manifest.Contents) > 1 {
return false
}
for name, entry := range manifest.Files {
for name, entry := range manifest.Contents {
if name == "" && entry.GetType() == Type_Directory {
return true
}
@@ -28,11 +28,11 @@ func IsManifestEmpty(manifest *Manifest) bool {
// Returns `true` if `left` and `right` contain the same files with the same types and data.
func CompareManifest(left *Manifest, right *Manifest) bool {
if len(left.Files) != len(right.Files) {
if len(left.Contents) != len(right.Contents) {
return false
}
for name, leftEntry := range left.Files {
rightEntry := right.Files[name]
for name, leftEntry := range left.Contents {
rightEntry := right.Contents[name]
if rightEntry == nil {
return false
}
@@ -71,7 +71,7 @@ again:
parts := strings.Split(inPath, "/")
for i := 1; i <= len(parts); i++ {
linkPath := path.Join(parts[:i]...)
entry := manifest.Files[linkPath]
entry := manifest.Contents[linkPath]
if entry != nil && entry.GetType() == Type_Symlink {
inPath = path.Join(
path.Dir(linkPath),
@@ -90,24 +90,24 @@ again:
}
}
const ExternalSizeMin uint64 = 256
const ExternalSizeMin uint32 = 256
func ExternalizeFiles(manifest *Manifest) *Manifest {
newManifest := Manifest{
RepoUrl: manifest.RepoUrl,
Branch: manifest.Branch,
Commit: manifest.Commit,
Files: make(map[string]*Entry),
RepoUrl: manifest.RepoUrl,
Branch: manifest.Branch,
Commit: manifest.Commit,
Contents: make(map[string]*Entry),
}
for name, entry := range manifest.Files {
for name, entry := range manifest.Contents {
if entry.GetType() == Type_InlineFile && entry.GetSize() > ExternalSizeMin {
newManifest.Files[name] = &Entry{
newManifest.Contents[name] = &Entry{
Type: Type_ExternalFile.Enum(),
Size: entry.Size,
Data: fmt.Appendf(nil, "sha256-%x", sha256.Sum256(entry.Data)),
}
} else {
newManifest.Files[name] = entry
newManifest.Contents[name] = entry
}
}
return &newManifest
@@ -129,11 +129,11 @@ func StoreManifest(name string, manifest *Manifest) (*Manifest, error) {
}
wg := sync.WaitGroup{}
ch := make(chan error, len(extManifest.Files))
for name, entry := range extManifest.Files {
ch := make(chan error, len(extManifest.Contents))
for name, entry := range extManifest.Contents {
if entry.GetType() == Type_ExternalFile {
wg.Go(func() {
err := backend.PutBlob(string(entry.Data), manifest.Files[name].Data)
err := backend.PutBlob(string(entry.Data), manifest.Contents[name].Data)
if err != nil {
ch <- fmt.Errorf("put blob %s: %w", name, err)
}

View File

@@ -12,7 +12,7 @@ import (
func readToManifest(root *os.Root) (*Manifest, error) {
manifest := Manifest{}
manifest.Files = make(map[string]*Entry)
manifest.Contents = make(map[string]*Entry)
err := fs.WalkDir(root.FS(), ".", func(path string, dirEntry fs.DirEntry, err error) error {
if err != nil {
return err
@@ -27,7 +27,7 @@ func readToManifest(root *os.Root) (*Manifest, error) {
return err
}
manifestEntry.Type = Type_InlineFile.Enum()
manifestEntry.Size = proto.Uint64(uint64(len(data)))
manifestEntry.Size = proto.Uint32(uint32(len(data)))
manifestEntry.Data = data
} else if dirEntry.Type().Type() == fs.ModeSymlink {
target, err := root.Readlink(path)
@@ -35,7 +35,7 @@ func readToManifest(root *os.Root) (*Manifest, error) {
return err
}
manifestEntry.Type = Type_Symlink.Enum()
manifestEntry.Size = proto.Uint64(uint64(len(target)))
manifestEntry.Size = proto.Uint32(uint32(len(target)))
manifestEntry.Data = []byte(target)
} else {
log.Printf("migrate v1: illegal %s/%s\n", root.Name(), path)
@@ -43,7 +43,7 @@ func readToManifest(root *os.Root) (*Manifest, error) {
if path == "." {
path = ""
}
manifest.Files[path] = &manifestEntry
manifest.Contents[path] = &manifestEntry
return nil
})
return &manifest, err

View File

@@ -64,7 +64,7 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
fmt.Fprintln(w, err)
return err
}
entry = manifest.Files[entryPath]
entry = manifest.Contents[entryPath]
if entry == nil || entry.GetType() == Type_Invalid {
is404 = true
if entryPath == notFoundPage {

View File

@@ -84,7 +84,7 @@ func (Type) EnumDescriptor() ([]byte, []int) {
type Entry struct {
state protoimpl.MessageState `protogen:"open.v1"`
Type *Type `protobuf:"varint,1,opt,name=type,enum=Type" json:"type,omitempty"`
Size *uint64 `protobuf:"varint,2,opt,name=size" json:"size,omitempty"`
Size *uint32 `protobuf:"varint,2,opt,name=size" json:"size,omitempty"`
Data []byte `protobuf:"bytes,3,opt,name=data" json:"data,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
@@ -127,7 +127,7 @@ func (x *Entry) GetType() Type {
return Type_Invalid
}
func (x *Entry) GetSize() uint64 {
func (x *Entry) GetSize() uint32 {
if x != nil && x.Size != nil {
return *x.Size
}
@@ -146,7 +146,7 @@ type Manifest struct {
RepoUrl *string `protobuf:"bytes,1,opt,name=repo_url,json=repoUrl" json:"repo_url,omitempty"`
Branch *string `protobuf:"bytes,2,opt,name=branch" json:"branch,omitempty"`
Commit *string `protobuf:"bytes,3,opt,name=commit" json:"commit,omitempty"`
Files map[string]*Entry `protobuf:"bytes,4,rep,name=files" json:"files,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
Contents map[string]*Entry `protobuf:"bytes,4,rep,name=contents" json:"contents,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@@ -202,9 +202,9 @@ func (x *Manifest) GetCommit() string {
return ""
}
func (x *Manifest) GetFiles() map[string]*Entry {
func (x *Manifest) GetContents() map[string]*Entry {
if x != nil {
return x.Files
return x.Contents
}
return nil
}
@@ -216,15 +216,14 @@ const file_schema_proto_rawDesc = "" +
"\fschema.proto\"J\n" +
"\x05Entry\x12\x19\n" +
"\x04type\x18\x01 \x01(\x0e2\x05.TypeR\x04type\x12\x12\n" +
"\x04size\x18\x02 \x01(\x04R\x04size\x12\x12\n" +
"\x04data\x18\x03 \x01(\fR\x04data\"\xc3\x01\n" +
"\x04size\x18\x02 \x01(\rR\x04size\x12\x12\n" +
"\x04data\x18\x03 \x01(\fR\x04data\"\xcf\x01\n" +
"\bManifest\x12\x19\n" +
"\brepo_url\x18\x01 \x01(\tR\arepoUrl\x12\x16\n" +
"\x06branch\x18\x02 \x01(\tR\x06branch\x12\x16\n" +
"\x06commit\x18\x03 \x01(\tR\x06commit\x12*\n" +
"\x05files\x18\x04 \x03(\v2\x14.Manifest.FilesEntryR\x05files\x1a@\n" +
"\n" +
"FilesEntry\x12\x10\n" +
"\x06commit\x18\x03 \x01(\tR\x06commit\x123\n" +
"\bcontents\x18\x04 \x03(\v2\x17.Manifest.ContentsEntryR\bcontents\x1aC\n" +
"\rContentsEntry\x12\x10\n" +
"\x03key\x18\x01 \x01(\tR\x03key\x12\x1c\n" +
"\x05value\x18\x02 \x01(\v2\x06.EntryR\x05value:\x028\x01*Q\n" +
"\x04Type\x12\v\n" +
@@ -253,12 +252,12 @@ var file_schema_proto_goTypes = []any{
(Type)(0), // 0: Type
(*Entry)(nil), // 1: Entry
(*Manifest)(nil), // 2: Manifest
nil, // 3: Manifest.FilesEntry
nil, // 3: Manifest.ContentsEntry
}
var file_schema_proto_depIdxs = []int32{
0, // 0: Entry.type:type_name -> Type
3, // 1: Manifest.files:type_name -> Manifest.FilesEntry
1, // 2: Manifest.FilesEntry.value:type_name -> Entry
3, // 1: Manifest.contents:type_name -> Manifest.ContentsEntry
1, // 2: Manifest.ContentsEntry.value:type_name -> Entry
3, // [3:3] is the sub-list for method output_type
3, // [3:3] is the sub-list for method input_type
3, // [3:3] is the sub-list for extension type_name

View File

@@ -17,7 +17,7 @@ enum Type {
message Entry {
Type type = 1;
uint64 size = 2;
uint32 size = 2;
bytes data = 3;
}
@@ -25,5 +25,5 @@ message Manifest {
string repo_url = 1;
string branch = 2;
string commit = 3;
map<string, Entry> files = 4;
map<string, Entry> contents = 4;
}