Record non-fatal problems in manifest and report them.

This feature keeps complex features like `_redirects` debuggable.

README.md

@@ -64,7 +64,7 @@ Features
     - If the URL matches `https://<hostname>/<project-name>/...` and a site was published at `<project-name>`, this project-specific site is selected.
     - If the URL matches `https://<hostname>/...` and the previous rule did not apply, the index site is selected.
     - Site paths starting with `.git-pages/...` are reserved.
-        - The `.git-pages/manifest.json` path returns a [ProtoJSON](https://protobuf.dev/programming-guides/json/) representation of the deployed site manifest.
+        - The `.git-pages/manifest.json` path returns a [ProtoJSON](https://protobuf.dev/programming-guides/json/) representation of the deployed site manifest. It enumerates site structure, redirect rules, and errors that were not severe enough to abort publishing.
 * In response to a `PUT` or `POST` request, the server retrieves updates a site with new content. The URL of the request must be the root URL of the site that is being published.
     - If the `PUT` method receives an `application/x-www-form-urlencoded` body, it contains a repository URL to be shallowly cloned. The `X-Pages-Branch` header contains the branch to be checked out; the `pages` branch is used if the header is absent.
     - If the `PUT` method receives an `application/x-tar` or `application/zip` body, it contains an archive to be extracted.

src/extract.go

@@ -28,6 +28,7 @@ func ExtractTar(reader io.Reader) (*Manifest, error) {
 			return nil, err
 		}
 
+		fileName := strings.TrimSuffix(header.Name, "/")
 		manifestEntry := Entry{}
 		switch header.Typeflag {
 		case tar.TypeReg:
@@ -51,9 +52,10 @@ func ExtractTar(reader io.Reader) (*Manifest, error) {
 			manifestEntry.Type = Type_Directory.Enum()
 
 		default:
-			manifestEntry.Type = Type_Invalid.Enum()
+			AddProblem(&manifest, fileName, "unsupported type '%c'", header.Typeflag)
+			continue
 		}
-		manifest.Contents[strings.TrimSuffix(header.Name, "/")] = &manifestEntry
+		manifest.Contents[fileName] = &manifestEntry
 	}
 	return &manifest, nil
 }

src/fetch.go

@@ -104,7 +104,8 @@ func FetchRepository(ctx context.Context, repoURL string, branch string) (*Manif
 			} else if entry.Mode == filemode.Dir {
 				manifestEntry.Type = Type_Directory.Enum()
 			} else {
-				manifestEntry.Type = Type_Invalid.Enum()
+				AddProblem(&manifest, name, "unsupported mode %#o", entry.Mode)
+				continue
 			}
 			manifest.Contents[name] = &manifestEntry
 		}

src/manifest.go

@@ -62,6 +62,24 @@ func DecodeManifest(data []byte) (*Manifest, error) {
 	return &manifest, err
 }
 
+func AddProblem(manifest *Manifest, path, format string, args ...any) error {
+	cause := fmt.Sprintf(format, args...)
+	manifest.Problems = append(manifest.Problems, &Problem{
+		Path:  proto.String(path),
+		Cause: proto.String(cause),
+	})
+	return fmt.Errorf("%s: %s", path, cause)
+}
+
+func GetProblemReport(manifest *Manifest) []string {
+	var report []string
+	for _, problem := range manifest.Problems {
+		report = append(report,
+			fmt.Sprintf("%s: %s", problem.GetPath(), problem.GetCause()))
+	}
+	return report
+}
+
 func ManifestDebugJSON(manifest *Manifest) string {
 	result, err := protojson.MarshalOptions{
 		Multiline:         true,
@@ -128,6 +146,7 @@ func ExternalizeFiles(manifest *Manifest) *Manifest {
 		Commit:    manifest.Commit,
 		Contents:  make(map[string]*Entry),
 		Redirects: manifest.Redirects,
+		Problems:  manifest.Problems,
 	}
 	var totalSize uint32
 	for name, entry := range manifest.Contents {

src/pages.go

@@ -243,6 +243,11 @@ func putPage(w http.ResponseWriter, r *http.Request) error {
 	if result.manifest != nil {
 		if result.manifest.Commit != nil {
 			fmt.Fprintln(w, *result.manifest.Commit)
+		} else {
+			fmt.Fprintln(w, "(archive)")
+		}
+		for _, problem := range GetProblemReport(result.manifest) {
+			fmt.Fprintln(w, problem)
 		}
 	} else if result.err != nil {
 		fmt.Fprintln(w, result.err)
@@ -375,6 +380,15 @@ func postPage(w http.ResponseWriter, r *http.Request) error {
 		w.WriteHeader(http.StatusOK)
 		fmt.Fprintln(w, "deleted")
 	}
+	if result.manifest != nil {
+		report := GetProblemReport(result.manifest)
+		if len(report) > 0 {
+			fmt.Fprintln(w, "problems:")
+		}
+		for _, problem := range report {
+			fmt.Fprintf(w, "- %s\n", problem)
+		}
+	}
 	return nil
 }
 

src/redirects.go

@@ -68,17 +68,20 @@ func ProcessRedirects(manifest *Manifest) error {
 	if redirectsEntry == nil {
 		return nil
 	} else if redirectsEntry.GetType() != Type_InlineFile {
-		return fmt.Errorf("%q is not a regular file", redirectsFile)
+		return AddProblem(manifest, redirectsFile,
+			"not a regular file")
 	}
 
 	rules, err := redirects.ParseString(string(redirectsEntry.GetData()))
 	if err != nil {
-		return fmt.Errorf("syntax error: %w", err)
+		return AddProblem(manifest, redirectsFile,
+			"syntax error: %s", err)
 	}
 
 	for index, rule := range rules {
 		if err := validateRule(rule); err != nil {
-			return fmt.Errorf("rule #%d: %w (in %q)", index+1, err, unparseRule(rule))
+			return AddProblem(manifest, redirectsFile,
+				"rule #%d %q: %s", index+1, unparseRule(rule), err)
 		}
 		manifest.Redirects = append(manifest.Redirects, &Redirect{
 			From:   proto.String(rule.From),

src/schema.pb.go

@@ -209,21 +209,78 @@ func (x *Redirect) GetForce() bool {
 	return false
 }
 
-type Manifest struct {
+type Problem struct {
 	state         protoimpl.MessageState `protogen:"open.v1"`
-	RepoUrl       *string                `protobuf:"bytes,1,opt,name=repo_url,json=repoUrl" json:"repo_url,omitempty"`
-	Branch        *string                `protobuf:"bytes,2,opt,name=branch" json:"branch,omitempty"`
-	Commit        *string                `protobuf:"bytes,3,opt,name=commit" json:"commit,omitempty"`
-	Contents      map[string]*Entry      `protobuf:"bytes,4,rep,name=contents" json:"contents,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
-	TotalSize     *uint32                `protobuf:"varint,5,opt,name=total_size,json=totalSize" json:"total_size,omitempty"`
-	Redirects     []*Redirect            `protobuf:"bytes,6,rep,name=redirects" json:"redirects,omitempty"`
+	Path          *string                `protobuf:"bytes,1,opt,name=path" json:"path,omitempty"`
+	Cause         *string                `protobuf:"bytes,2,opt,name=cause" json:"cause,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *Problem) Reset() {
+	*x = Problem{}
+	mi := &file_schema_proto_msgTypes[2]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *Problem) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Problem) ProtoMessage() {}
+
+func (x *Problem) ProtoReflect() protoreflect.Message {
+	mi := &file_schema_proto_msgTypes[2]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use Problem.ProtoReflect.Descriptor instead.
+func (*Problem) Descriptor() ([]byte, []int) {
+	return file_schema_proto_rawDescGZIP(), []int{2}
+}
+
+func (x *Problem) GetPath() string {
+	if x != nil && x.Path != nil {
+		return *x.Path
+	}
+	return ""
+}
+
+func (x *Problem) GetCause() string {
+	if x != nil && x.Cause != nil {
+		return *x.Cause
+	}
+	return ""
+}
+
+type Manifest struct {
+	state protoimpl.MessageState `protogen:"open.v1"`
+	// Source metadata
+	RepoUrl *string `protobuf:"bytes,1,opt,name=repo_url,json=repoUrl" json:"repo_url,omitempty"`
+	Branch  *string `protobuf:"bytes,2,opt,name=branch" json:"branch,omitempty"`
+	Commit  *string `protobuf:"bytes,3,opt,name=commit" json:"commit,omitempty"`
+	// Contents
+	Contents  map[string]*Entry `protobuf:"bytes,4,rep,name=contents" json:"contents,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+	TotalSize *uint32           `protobuf:"varint,5,opt,name=total_size,json=totalSize" json:"total_size,omitempty"`
+	// Netlify-style `_redirects`
+	Redirects []*Redirect `protobuf:"bytes,6,rep,name=redirects" json:"redirects,omitempty"`
+	// Diagnostics for non-fatal errors
+	Problems      []*Problem `protobuf:"bytes,7,rep,name=problems" json:"problems,omitempty"`
 	unknownFields protoimpl.UnknownFields
 	sizeCache     protoimpl.SizeCache
 }
 
 func (x *Manifest) Reset() {
 	*x = Manifest{}
-	mi := &file_schema_proto_msgTypes[2]
+	mi := &file_schema_proto_msgTypes[3]
 	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 	ms.StoreMessageInfo(mi)
 }
@@ -235,7 +292,7 @@ func (x *Manifest) String() string {
 func (*Manifest) ProtoMessage() {}
 
 func (x *Manifest) ProtoReflect() protoreflect.Message {
-	mi := &file_schema_proto_msgTypes[2]
+	mi := &file_schema_proto_msgTypes[3]
 	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -248,7 +305,7 @@ func (x *Manifest) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use Manifest.ProtoReflect.Descriptor instead.
 func (*Manifest) Descriptor() ([]byte, []int) {
-	return file_schema_proto_rawDescGZIP(), []int{2}
+	return file_schema_proto_rawDescGZIP(), []int{3}
 }
 
 func (x *Manifest) GetRepoUrl() string {
@@ -293,6 +350,13 @@ func (x *Manifest) GetRedirects() []*Redirect {
 	return nil
 }
 
+func (x *Manifest) GetProblems() []*Problem {
+	if x != nil {
+		return x.Problems
+	}
+	return nil
+}
+
 var File_schema_proto protoreflect.FileDescriptor
 
 const file_schema_proto_rawDesc = "" +
@@ -306,7 +370,10 @@ const file_schema_proto_rawDesc = "" +
 	"\x04from\x18\x01 \x01(\tR\x04from\x12\x0e\n" +
 	"\x02to\x18\x02 \x01(\tR\x02to\x12\x16\n" +
 	"\x06status\x18\x03 \x01(\rR\x06status\x12\x14\n" +
-	"\x05force\x18\x04 \x01(\bR\x05force\"\x97\x02\n" +
+	"\x05force\x18\x04 \x01(\bR\x05force\"3\n" +
+	"\aProblem\x12\x12\n" +
+	"\x04path\x18\x01 \x01(\tR\x04path\x12\x14\n" +
+	"\x05cause\x18\x02 \x01(\tR\x05cause\"\xbd\x02\n" +
 	"\bManifest\x12\x19\n" +
 	"\brepo_url\x18\x01 \x01(\tR\arepoUrl\x12\x16\n" +
 	"\x06branch\x18\x02 \x01(\tR\x06branch\x12\x16\n" +
@@ -314,7 +381,8 @@ const file_schema_proto_rawDesc = "" +
 	"\bcontents\x18\x04 \x03(\v2\x17.Manifest.ContentsEntryR\bcontents\x12\x1d\n" +
 	"\n" +
 	"total_size\x18\x05 \x01(\rR\ttotalSize\x12'\n" +
-	"\tredirects\x18\x06 \x03(\v2\t.RedirectR\tredirects\x1aC\n" +
+	"\tredirects\x18\x06 \x03(\v2\t.RedirectR\tredirects\x12$\n" +
+	"\bproblems\x18\a \x03(\v2\b.ProblemR\bproblems\x1aC\n" +
 	"\rContentsEntry\x12\x10\n" +
 	"\x03key\x18\x01 \x01(\tR\x03key\x12\x1c\n" +
 	"\x05value\x18\x02 \x01(\v2\x06.EntryR\x05value:\x028\x01*Q\n" +
@@ -339,24 +407,26 @@ func file_schema_proto_rawDescGZIP() []byte {
 }
 
 var file_schema_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
-var file_schema_proto_msgTypes = make([]protoimpl.MessageInfo, 4)
+var file_schema_proto_msgTypes = make([]protoimpl.MessageInfo, 5)
 var file_schema_proto_goTypes = []any{
 	(Type)(0),        // 0: Type
 	(*Entry)(nil),    // 1: Entry
 	(*Redirect)(nil), // 2: Redirect
-	(*Manifest)(nil), // 3: Manifest
-	nil,              // 4: Manifest.ContentsEntry
+	(*Problem)(nil),  // 3: Problem
+	(*Manifest)(nil), // 4: Manifest
+	nil,              // 5: Manifest.ContentsEntry
 }
 var file_schema_proto_depIdxs = []int32{
 	0, // 0: Entry.type:type_name -> Type
-	4, // 1: Manifest.contents:type_name -> Manifest.ContentsEntry
+	5, // 1: Manifest.contents:type_name -> Manifest.ContentsEntry
 	2, // 2: Manifest.redirects:type_name -> Redirect
-	1, // 3: Manifest.ContentsEntry.value:type_name -> Entry
-	4, // [4:4] is the sub-list for method output_type
-	4, // [4:4] is the sub-list for method input_type
-	4, // [4:4] is the sub-list for extension type_name
-	4, // [4:4] is the sub-list for extension extendee
-	0, // [0:4] is the sub-list for field type_name
+	3, // 3: Manifest.problems:type_name -> Problem
+	1, // 4: Manifest.ContentsEntry.value:type_name -> Entry
+	5, // [5:5] is the sub-list for method output_type
+	5, // [5:5] is the sub-list for method input_type
+	5, // [5:5] is the sub-list for extension type_name
+	5, // [5:5] is the sub-list for extension extendee
+	0, // [0:5] is the sub-list for field type_name
 }
 
 func init() { file_schema_proto_init() }
@@ -370,7 +440,7 @@ func file_schema_proto_init() {
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
 			RawDescriptor: unsafe.Slice(unsafe.StringData(file_schema_proto_rawDesc), len(file_schema_proto_rawDesc)),
 			NumEnums:      1,
-			NumMessages:   4,
+			NumMessages:   5,
 			NumExtensions: 0,
 			NumServices:   0,
 		},

src/schema.proto

@@ -17,10 +17,19 @@ enum Type {
 
 message Entry {
 	Type type = 1;
+	// Only present for `type == InlineFile` and `type == ExternalFile`
 	uint32 size = 2;
+	// Meaning depends on `type`:
+	//  * If `type == InlineFile`, contains file data.
+	//  * If `type == ExternalFile`, contains blob name (an otherwise unspecified
+	//    cryptographically secure content hash).
+	//  * If `type == Symlink`, contains link target.
+	//  * Otherwise not present.
 	bytes data = 3;
 }
 
+// See https://docs.netlify.com/manage/routing/redirects/overview/ for details.
+// Only a subset of the Netlify specification is representable here.
 message Redirect {
 	string from = 1;
 	string to = 2;
@@ -28,11 +37,24 @@ message Redirect {
 	bool force = 4;
 }
 
+message Problem {
+	string path = 1;
+	string cause = 2;
+}
+
 message Manifest {
+	// Source metadata
 	string repo_url = 1;
 	string branch = 2;
 	string commit = 3;
+
+	// Contents
 	map<string, Entry> contents = 4;
 	uint32 total_size = 5;
+
+	// Netlify-style `_redirects`
 	repeated Redirect redirects = 6;
+
+	// Diagnostics for non-fatal errors
+	repeated Problem problems = 7;
 }