From d2b5144182cd55fd6d32c19fea423b64f022bc86 Mon Sep 17 00:00:00 2001 From: miyuko Date: Sat, 21 Mar 2026 00:13:24 +0000 Subject: [PATCH] Warn when a Git repository is uploaded with Git LFS-tracked files. --- src/fetch.go | 18 +++++++++++++ src/gitattributes.go | 61 ++++++++++++++++++++++++++++++++++++++++++++ src/headers.go | 16 ++++++++---- src/manifest.go | 43 +++++++++++++++++++++++++++++-- src/redirects.go | 17 ++++++++---- 5 files changed, 143 insertions(+), 12 deletions(-) create mode 100644 src/gitattributes.go diff --git a/src/fetch.go b/src/fetch.go index 2756c3f..c6caa55 100644 --- a/src/fetch.go +++ b/src/fetch.go @@ -9,6 +9,7 @@ import ( "net/url" "os" "slices" + "strings" "github.com/c2h5oh/datasize" "github.com/go-git/go-billy/v6/osfs" @@ -209,6 +210,8 @@ func FetchRepository( datasize.ByteSize(dataBytesTransferred).HR(), ) + warnAboutGitLFS(ctx, manifest) + return manifest, nil } @@ -254,3 +257,18 @@ func readGitBlob( return nil } + +func warnAboutGitLFS(ctx context.Context, manifest *Manifest) { + gitattributes := ReadGitAttributes(ctx, manifest) + for _, name := range slices.Sorted(maps.Keys(manifest.GetContents())) { + entry := manifest.GetContents()[name] + if !IsEntryRegularFile(entry) { + continue + } + parts := strings.Split(name, "/") + attrs, _ := gitattributes.Match(parts, nil) + if attr, ok := attrs["filter"]; ok && attr.Value() == "lfs" { + AddProblem(manifest, name, "git-pages does not support Git LFS; move this file into Git or use incremental uploads") + } + } +} diff --git a/src/gitattributes.go b/src/gitattributes.go new file mode 100644 index 0000000..9e300ac --- /dev/null +++ b/src/gitattributes.go @@ -0,0 +1,61 @@ +package git_pages + +import ( + "bytes" + "cmp" + "context" + "slices" + "strings" + + "github.com/go-git/go-git/v6/plumbing/format/gitattributes" +) + +func ReadGitAttributes(ctx context.Context, manifest *Manifest) gitattributes.Matcher { + type entryPair struct { + parts []string + entry *Entry + } + + // Collect all .gitattributes files. + var files []entryPair + for name, entry := range manifest.GetContents() { + switch entry.GetType() { + case Type_InlineFile, Type_ExternalFile: + parts := strings.Split(name, "/") + if parts[len(parts)-1] == ".gitattributes" { + files = append(files, entryPair{parts, entry}) + } + } + } + + // Sort the file list by depth, then by name. + slices.SortFunc(files, func(a entryPair, b entryPair) int { + return cmp.Or( + cmp.Compare(len(a.parts), len(b.parts)), + slices.Compare(a.parts, b.parts), + ) + }) + + // Gather all .gitattributes rules, sorted by depth. + var rules []gitattributes.MatchAttribute + for _, pair := range files { + parts, entry := pair.parts, pair.entry + data, err := GetEntryContents(ctx, entry) + if err != nil { + continue + } + dirs := parts[:len(parts)-1] + isRoot := len(parts) == 1 + newRules, err := gitattributes.ReadAttributes(bytes.NewReader(data), dirs, isRoot) + if err != nil { + AddProblem(manifest, strings.Join(parts, "/"), "parsing .gitattributes: %v", err) + continue + } + rules = append(rules, newRules...) + } + + // gitattributes.Matcher applies rules in reverse. + slices.Reverse(rules) + matcher := gitattributes.NewMatcher(rules) + return matcher +} diff --git a/src/headers.go b/src/headers.go index 1385835..c162568 100644 --- a/src/headers.go +++ b/src/headers.go @@ -1,6 +1,7 @@ package git_pages import ( + "context" "errors" "fmt" "net/http" @@ -85,17 +86,22 @@ func validateHeaderRule(rule headers.Rule) error { } // Parses redirects file and injects rules into the manifest. -func ProcessHeadersFile(manifest *Manifest) error { +func ProcessHeadersFile(ctx context.Context, manifest *Manifest) error { headersEntry := manifest.Contents[HeadersFileName] delete(manifest.Contents, HeadersFileName) if headersEntry == nil { return nil - } else if headersEntry.GetType() != Type_InlineFile { - return AddProblem(manifest, HeadersFileName, - "not a regular file") } - rules, err := headers.ParseString(string(headersEntry.GetData())) + data, err := GetEntryContents(ctx, headersEntry) + if errors.Is(err, ErrNotRegularFile) { + return AddProblem(manifest, HeadersFileName, + "not a regular file") + } else if err != nil { + return err + } + + rules, err := headers.ParseString(string(data)) if err != nil { return AddProblem(manifest, HeadersFileName, "syntax error: %s", err) diff --git a/src/manifest.go b/src/manifest.go index b6e3788..771381d 100644 --- a/src/manifest.go +++ b/src/manifest.go @@ -8,6 +8,7 @@ import ( "crypto/sha256" "errors" "fmt" + "io" "mime" "net/http" "path" @@ -144,6 +145,44 @@ func AddProblem(manifest *Manifest, pathName, format string, args ...any) error return fmt.Errorf("%s: %s", pathName, cause) } +func IsEntryRegularFile(entry *Entry) bool { + return entry.GetType() == Type_InlineFile || + entry.GetType() == Type_ExternalFile +} + +var ErrNotRegularFile = errors.New("not a regular file") + +func GetEntryContents(ctx context.Context, entry *Entry) (data []byte, err error) { + switch entry.GetType() { + case Type_InlineFile: + data = entry.GetData() + case Type_ExternalFile: + reader, _, err := backend.GetBlob(ctx, string(entry.GetData())) + if err != nil { + return nil, err + } + data, err = io.ReadAll(reader) + if err != nil { + return nil, err + } + default: + return nil, ErrNotRegularFile + } + + switch entry.GetTransform() { + case Transform_Identity: + case Transform_Zstd: + data, err = zstdDecoder.DecodeAll(data, []byte{}) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("unexpected transform") + } + + return +} + // EnsureLeadingDirectories adds directory entries for any parent directories // that are implicitly referenced by files in the manifest but don't have // explicit directory entries. (This can be the case if an archive is created @@ -275,7 +314,7 @@ func CompressFiles(ctx context.Context, manifest *Manifest) { // (Perhaps in the future they could be exposed at `.git-pages/status.txt`?) func PrepareManifest(ctx context.Context, manifest *Manifest) error { // Parse Netlify-style `_redirects`. - if err := ProcessRedirectsFile(manifest); err != nil { + if err := ProcessRedirectsFile(ctx, manifest); err != nil { logc.Printf(ctx, "redirects err: %s\n", err) } else if len(manifest.Redirects) > 0 { logc.Printf(ctx, "redirects ok: %d rules\n", len(manifest.Redirects)) @@ -285,7 +324,7 @@ func PrepareManifest(ctx context.Context, manifest *Manifest) error { LintRedirects(manifest) // Parse Netlify-style `_headers`. - if err := ProcessHeadersFile(manifest); err != nil { + if err := ProcessHeadersFile(ctx, manifest); err != nil { logc.Printf(ctx, "headers err: %s\n", err) } else if len(manifest.Headers) > 0 { logc.Printf(ctx, "headers ok: %d rules\n", len(manifest.Headers)) diff --git a/src/redirects.go b/src/redirects.go index 2670c33..8a3bd2c 100644 --- a/src/redirects.go +++ b/src/redirects.go @@ -1,6 +1,8 @@ package git_pages import ( + "context" + "errors" "fmt" "net/http" "net/url" @@ -96,17 +98,22 @@ func validateRedirectRule(rule *redirects.Rule) error { } // Parses redirects file and injects rules into the manifest. -func ProcessRedirectsFile(manifest *Manifest) error { +func ProcessRedirectsFile(ctx context.Context, manifest *Manifest) error { redirectsEntry := manifest.Contents[RedirectsFileName] delete(manifest.Contents, RedirectsFileName) if redirectsEntry == nil { return nil - } else if redirectsEntry.GetType() != Type_InlineFile { - return AddProblem(manifest, RedirectsFileName, - "not a regular file") } - rules, err := redirects.ParseString(string(redirectsEntry.GetData())) + data, err := GetEntryContents(ctx, redirectsEntry) + if errors.Is(err, ErrNotRegularFile) { + return AddProblem(manifest, RedirectsFileName, + "not a regular file") + } else if err != nil { + return err + } + + rules, err := redirects.ParseString(string(data)) if err != nil { return AddProblem(manifest, RedirectsFileName, "syntax error: %s", err)