mirror of
https://codeberg.org/git-pages/git-pages.git
synced 2026-05-14 03:01:48 +00:00
Refactor Protobuf schema.
This is to prepare for making manifest debug representation accessible. - change `Entry.size` to `uint32` so that it's serialized as a number in protoJSON export - rename `Manifest.files` to `Manifest.contents` - leave size and data for the root directory empty, same as with non-root directories fetched from git
This commit is contained in:
18
src/fetch.go
18
src/fetch.go
@@ -62,16 +62,12 @@ func FetchRepository(ctx context.Context, repoURL string, branch string) (*Manif
|
||||
defer walker.Close()
|
||||
|
||||
manifest := Manifest{
|
||||
RepoUrl: proto.String(repoURL),
|
||||
Branch: proto.String(branch),
|
||||
Commit: proto.String(ref.Hash().String()),
|
||||
Files: make(map[string]*Entry),
|
||||
}
|
||||
manifest.Files[""] = &Entry{
|
||||
Type: Type_Directory.Enum(),
|
||||
Size: proto.Uint64(0),
|
||||
Data: []byte{},
|
||||
RepoUrl: proto.String(repoURL),
|
||||
Branch: proto.String(branch),
|
||||
Commit: proto.String(ref.Hash().String()),
|
||||
Contents: make(map[string]*Entry),
|
||||
}
|
||||
manifest.Contents[""] = &Entry{Type: Type_Directory.Enum()}
|
||||
for {
|
||||
name, entry, err := walker.Next()
|
||||
if err == io.EOF {
|
||||
@@ -101,14 +97,14 @@ func FetchRepository(ctx context.Context, repoURL string, branch string) (*Manif
|
||||
} else {
|
||||
manifestEntry.Type = Type_InlineFile.Enum()
|
||||
}
|
||||
manifestEntry.Size = proto.Uint64(uint64(blob.Size))
|
||||
manifestEntry.Size = proto.Uint32(uint32(blob.Size))
|
||||
manifestEntry.Data = data
|
||||
} else if entry.Mode == filemode.Dir {
|
||||
manifestEntry.Type = Type_Directory.Enum()
|
||||
} else {
|
||||
manifestEntry.Type = Type_Invalid.Enum()
|
||||
}
|
||||
manifest.Files[name] = &manifestEntry
|
||||
manifest.Contents[name] = &manifestEntry
|
||||
}
|
||||
}
|
||||
return &manifest, nil
|
||||
|
||||
@@ -15,10 +15,10 @@ import (
|
||||
)
|
||||
|
||||
func IsManifestEmpty(manifest *Manifest) bool {
|
||||
if len(manifest.Files) > 1 {
|
||||
if len(manifest.Contents) > 1 {
|
||||
return false
|
||||
}
|
||||
for name, entry := range manifest.Files {
|
||||
for name, entry := range manifest.Contents {
|
||||
if name == "" && entry.GetType() == Type_Directory {
|
||||
return true
|
||||
}
|
||||
@@ -28,11 +28,11 @@ func IsManifestEmpty(manifest *Manifest) bool {
|
||||
|
||||
// Returns `true` if `left` and `right` contain the same files with the same types and data.
|
||||
func CompareManifest(left *Manifest, right *Manifest) bool {
|
||||
if len(left.Files) != len(right.Files) {
|
||||
if len(left.Contents) != len(right.Contents) {
|
||||
return false
|
||||
}
|
||||
for name, leftEntry := range left.Files {
|
||||
rightEntry := right.Files[name]
|
||||
for name, leftEntry := range left.Contents {
|
||||
rightEntry := right.Contents[name]
|
||||
if rightEntry == nil {
|
||||
return false
|
||||
}
|
||||
@@ -71,7 +71,7 @@ again:
|
||||
parts := strings.Split(inPath, "/")
|
||||
for i := 1; i <= len(parts); i++ {
|
||||
linkPath := path.Join(parts[:i]...)
|
||||
entry := manifest.Files[linkPath]
|
||||
entry := manifest.Contents[linkPath]
|
||||
if entry != nil && entry.GetType() == Type_Symlink {
|
||||
inPath = path.Join(
|
||||
path.Dir(linkPath),
|
||||
@@ -90,24 +90,24 @@ again:
|
||||
}
|
||||
}
|
||||
|
||||
const ExternalSizeMin uint64 = 256
|
||||
const ExternalSizeMin uint32 = 256
|
||||
|
||||
func ExternalizeFiles(manifest *Manifest) *Manifest {
|
||||
newManifest := Manifest{
|
||||
RepoUrl: manifest.RepoUrl,
|
||||
Branch: manifest.Branch,
|
||||
Commit: manifest.Commit,
|
||||
Files: make(map[string]*Entry),
|
||||
RepoUrl: manifest.RepoUrl,
|
||||
Branch: manifest.Branch,
|
||||
Commit: manifest.Commit,
|
||||
Contents: make(map[string]*Entry),
|
||||
}
|
||||
for name, entry := range manifest.Files {
|
||||
for name, entry := range manifest.Contents {
|
||||
if entry.GetType() == Type_InlineFile && entry.GetSize() > ExternalSizeMin {
|
||||
newManifest.Files[name] = &Entry{
|
||||
newManifest.Contents[name] = &Entry{
|
||||
Type: Type_ExternalFile.Enum(),
|
||||
Size: entry.Size,
|
||||
Data: fmt.Appendf(nil, "sha256-%x", sha256.Sum256(entry.Data)),
|
||||
}
|
||||
} else {
|
||||
newManifest.Files[name] = entry
|
||||
newManifest.Contents[name] = entry
|
||||
}
|
||||
}
|
||||
return &newManifest
|
||||
@@ -129,11 +129,11 @@ func StoreManifest(name string, manifest *Manifest) (*Manifest, error) {
|
||||
}
|
||||
|
||||
wg := sync.WaitGroup{}
|
||||
ch := make(chan error, len(extManifest.Files))
|
||||
for name, entry := range extManifest.Files {
|
||||
ch := make(chan error, len(extManifest.Contents))
|
||||
for name, entry := range extManifest.Contents {
|
||||
if entry.GetType() == Type_ExternalFile {
|
||||
wg.Go(func() {
|
||||
err := backend.PutBlob(string(entry.Data), manifest.Files[name].Data)
|
||||
err := backend.PutBlob(string(entry.Data), manifest.Contents[name].Data)
|
||||
if err != nil {
|
||||
ch <- fmt.Errorf("put blob %s: %w", name, err)
|
||||
}
|
||||
|
||||
@@ -12,7 +12,7 @@ import (
|
||||
|
||||
func readToManifest(root *os.Root) (*Manifest, error) {
|
||||
manifest := Manifest{}
|
||||
manifest.Files = make(map[string]*Entry)
|
||||
manifest.Contents = make(map[string]*Entry)
|
||||
err := fs.WalkDir(root.FS(), ".", func(path string, dirEntry fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -27,7 +27,7 @@ func readToManifest(root *os.Root) (*Manifest, error) {
|
||||
return err
|
||||
}
|
||||
manifestEntry.Type = Type_InlineFile.Enum()
|
||||
manifestEntry.Size = proto.Uint64(uint64(len(data)))
|
||||
manifestEntry.Size = proto.Uint32(uint32(len(data)))
|
||||
manifestEntry.Data = data
|
||||
} else if dirEntry.Type().Type() == fs.ModeSymlink {
|
||||
target, err := root.Readlink(path)
|
||||
@@ -35,7 +35,7 @@ func readToManifest(root *os.Root) (*Manifest, error) {
|
||||
return err
|
||||
}
|
||||
manifestEntry.Type = Type_Symlink.Enum()
|
||||
manifestEntry.Size = proto.Uint64(uint64(len(target)))
|
||||
manifestEntry.Size = proto.Uint32(uint32(len(target)))
|
||||
manifestEntry.Data = []byte(target)
|
||||
} else {
|
||||
log.Printf("migrate v1: illegal %s/%s\n", root.Name(), path)
|
||||
@@ -43,7 +43,7 @@ func readToManifest(root *os.Root) (*Manifest, error) {
|
||||
if path == "." {
|
||||
path = ""
|
||||
}
|
||||
manifest.Files[path] = &manifestEntry
|
||||
manifest.Contents[path] = &manifestEntry
|
||||
return nil
|
||||
})
|
||||
return &manifest, err
|
||||
|
||||
@@ -64,7 +64,7 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
|
||||
fmt.Fprintln(w, err)
|
||||
return err
|
||||
}
|
||||
entry = manifest.Files[entryPath]
|
||||
entry = manifest.Contents[entryPath]
|
||||
if entry == nil || entry.GetType() == Type_Invalid {
|
||||
is404 = true
|
||||
if entryPath == notFoundPage {
|
||||
|
||||
@@ -84,7 +84,7 @@ func (Type) EnumDescriptor() ([]byte, []int) {
|
||||
type Entry struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
Type *Type `protobuf:"varint,1,opt,name=type,enum=Type" json:"type,omitempty"`
|
||||
Size *uint64 `protobuf:"varint,2,opt,name=size" json:"size,omitempty"`
|
||||
Size *uint32 `protobuf:"varint,2,opt,name=size" json:"size,omitempty"`
|
||||
Data []byte `protobuf:"bytes,3,opt,name=data" json:"data,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
@@ -127,7 +127,7 @@ func (x *Entry) GetType() Type {
|
||||
return Type_Invalid
|
||||
}
|
||||
|
||||
func (x *Entry) GetSize() uint64 {
|
||||
func (x *Entry) GetSize() uint32 {
|
||||
if x != nil && x.Size != nil {
|
||||
return *x.Size
|
||||
}
|
||||
@@ -146,7 +146,7 @@ type Manifest struct {
|
||||
RepoUrl *string `protobuf:"bytes,1,opt,name=repo_url,json=repoUrl" json:"repo_url,omitempty"`
|
||||
Branch *string `protobuf:"bytes,2,opt,name=branch" json:"branch,omitempty"`
|
||||
Commit *string `protobuf:"bytes,3,opt,name=commit" json:"commit,omitempty"`
|
||||
Files map[string]*Entry `protobuf:"bytes,4,rep,name=files" json:"files,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
|
||||
Contents map[string]*Entry `protobuf:"bytes,4,rep,name=contents" json:"contents,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
@@ -202,9 +202,9 @@ func (x *Manifest) GetCommit() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *Manifest) GetFiles() map[string]*Entry {
|
||||
func (x *Manifest) GetContents() map[string]*Entry {
|
||||
if x != nil {
|
||||
return x.Files
|
||||
return x.Contents
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -216,15 +216,14 @@ const file_schema_proto_rawDesc = "" +
|
||||
"\fschema.proto\"J\n" +
|
||||
"\x05Entry\x12\x19\n" +
|
||||
"\x04type\x18\x01 \x01(\x0e2\x05.TypeR\x04type\x12\x12\n" +
|
||||
"\x04size\x18\x02 \x01(\x04R\x04size\x12\x12\n" +
|
||||
"\x04data\x18\x03 \x01(\fR\x04data\"\xc3\x01\n" +
|
||||
"\x04size\x18\x02 \x01(\rR\x04size\x12\x12\n" +
|
||||
"\x04data\x18\x03 \x01(\fR\x04data\"\xcf\x01\n" +
|
||||
"\bManifest\x12\x19\n" +
|
||||
"\brepo_url\x18\x01 \x01(\tR\arepoUrl\x12\x16\n" +
|
||||
"\x06branch\x18\x02 \x01(\tR\x06branch\x12\x16\n" +
|
||||
"\x06commit\x18\x03 \x01(\tR\x06commit\x12*\n" +
|
||||
"\x05files\x18\x04 \x03(\v2\x14.Manifest.FilesEntryR\x05files\x1a@\n" +
|
||||
"\n" +
|
||||
"FilesEntry\x12\x10\n" +
|
||||
"\x06commit\x18\x03 \x01(\tR\x06commit\x123\n" +
|
||||
"\bcontents\x18\x04 \x03(\v2\x17.Manifest.ContentsEntryR\bcontents\x1aC\n" +
|
||||
"\rContentsEntry\x12\x10\n" +
|
||||
"\x03key\x18\x01 \x01(\tR\x03key\x12\x1c\n" +
|
||||
"\x05value\x18\x02 \x01(\v2\x06.EntryR\x05value:\x028\x01*Q\n" +
|
||||
"\x04Type\x12\v\n" +
|
||||
@@ -253,12 +252,12 @@ var file_schema_proto_goTypes = []any{
|
||||
(Type)(0), // 0: Type
|
||||
(*Entry)(nil), // 1: Entry
|
||||
(*Manifest)(nil), // 2: Manifest
|
||||
nil, // 3: Manifest.FilesEntry
|
||||
nil, // 3: Manifest.ContentsEntry
|
||||
}
|
||||
var file_schema_proto_depIdxs = []int32{
|
||||
0, // 0: Entry.type:type_name -> Type
|
||||
3, // 1: Manifest.files:type_name -> Manifest.FilesEntry
|
||||
1, // 2: Manifest.FilesEntry.value:type_name -> Entry
|
||||
3, // 1: Manifest.contents:type_name -> Manifest.ContentsEntry
|
||||
1, // 2: Manifest.ContentsEntry.value:type_name -> Entry
|
||||
3, // [3:3] is the sub-list for method output_type
|
||||
3, // [3:3] is the sub-list for method input_type
|
||||
3, // [3:3] is the sub-list for extension type_name
|
||||
|
||||
@@ -17,7 +17,7 @@ enum Type {
|
||||
|
||||
message Entry {
|
||||
Type type = 1;
|
||||
uint64 size = 2;
|
||||
uint32 size = 2;
|
||||
bytes data = 3;
|
||||
}
|
||||
|
||||
@@ -25,5 +25,5 @@ message Manifest {
|
||||
string repo_url = 1;
|
||||
string branch = 2;
|
||||
string commit = 3;
|
||||
map<string, Entry> files = 4;
|
||||
map<string, Entry> contents = 4;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user