From 2557ec750f1da3147be8848295bc2900dddd3096 Mon Sep 17 00:00:00 2001 From: Catherine Date: Wed, 21 Oct 2026 15:24:47 +0000 Subject: [PATCH] Refactor handling of domain name and site name. Define a proper abstraction for domain names and site names, and use i elsewhere in he codebase. This still leaves domain patterns as an abstraction for future work. The existing implementation is particularly awkward now. --- src/analysis.go | 27 +++++++--- src/audit.go | 48 ++++++----------- src/auth.go | 77 ++++++++-------------------- src/backend.go | 20 ++++---- src/backend_fs.go | 68 ++++++++++++------------ src/backend_s3.go | 86 +++++++++++++++---------------- src/domains.go | 29 ----------- src/existence.go | 36 ++++++------- src/garbage.go | 2 +- src/main.go | 48 ++++++++++------- src/manifest.go | 13 +++-- src/migrate.go | 20 ++++---- src/names/domain.go | 97 +++++++++++++++++++++++++++++++++++ src/names/domain_test.go | 70 +++++++++++++++++++++++++ src/names/error.go | 9 ++++ src/names/project.go | 20 ++++---- src/names/project_test.go | 67 +++++------------------- src/names/site.go | 60 ++++++++++++++++++++++ src/names/site_test.go | 29 +++++++++++ src/names/utils.go | 48 +++++++++++++++++ src/observe.go | 33 ++++++------ src/pages.go | 105 ++++++++++++++++++++------------------ src/pages_test.go | 55 -------------------- src/proxy.go | 26 +++++----- src/redirects.go | 7 +-- src/scrub.go | 5 +- src/update.go | 67 ++++++++++++------------ src/util.go | 7 +++ src/wildcard.go | 9 ++-- 29 files changed, 681 insertions(+), 507 deletions(-) delete mode 100644 src/domains.go create mode 100644 src/names/domain.go create mode 100644 src/names/domain_test.go create mode 100644 src/names/error.go create mode 100644 src/names/site.go create mode 100644 src/names/site_test.go create mode 100644 src/names/utils.go delete mode 100644 src/pages_test.go diff --git a/src/analysis.go b/src/analysis.go index bc19945..dee2d09 100644 --- a/src/analysis.go +++ b/src/analysis.go @@ -4,6 +4,8 @@ import ( "context" "fmt" "strings" + + "codeberg.org/git-pages/git-pages/src/names" ) type StorageSize struct { @@ -26,11 +28,21 @@ func AnalyzeStorage(ctx context.Context) ([]*StorageSize, error) { auditBlobs map[string]int64 } - thickStats := map[string]*storageData{} + thickStats := map[names.Domain]*storageData{} thinStats := []*StorageSize{} - getStats := func(domain string) *storageData { - domain = EffectiveTLDPlusOne(domain) + extraSuffixes := []names.Domain{} + for _, wildcard := range wildcards { + extraSuffixes = append(extraSuffixes, + Must(names.ParseDomain(strings.Join(wildcard.Domain, "."), names.ParseDomainRelaxed))) + if len(wildcard.PreviewDomain) > 0 { + extraSuffixes = append(extraSuffixes, + Must(names.ParseDomain(strings.Join(wildcard.PreviewDomain, "."), names.ParseDomainRelaxed))) + } + } + + getStats := func(domain names.Domain) *storageData { + domain = domain.EffectiveTLDPlusOne(extraSuffixes) // FIXME if _, found := thickStats[domain]; !found { thickStats[domain] = &storageData{ siteBlobs: map[string]int64{}, @@ -40,7 +52,7 @@ func AnalyzeStorage(ctx context.Context) ([]*StorageSize, error) { return thickStats[domain] } - totalStats := getStats("*") + totalStats := getStats(names.Domain{}) logc.Printf(ctx, "analyze: enumerating manifests") for item, err := range backend.GetAllManifests(ctx) { @@ -48,8 +60,7 @@ func AnalyzeStorage(ctx context.Context) ([]*StorageSize, error) { if err != nil { return nil, fmt.Errorf("analyze err: %w", err) } - domain, _, _ := strings.Cut(metadata.Name, "/") - stats := getStats(domain) + stats := getStats(metadata.Name.Domain) stats.siteManifests += metadata.Size totalStats.siteManifests += metadata.Size for _, entry := range manifest.GetContents() { @@ -67,7 +78,7 @@ func AnalyzeStorage(ctx context.Context) ([]*StorageSize, error) { if err != nil { return nil, fmt.Errorf("analyze err: %w", err) } - domain := record.GetDomain() + domain := Must(names.ParseDomain(record.GetDomain(), names.ParseDomainRelaxed)) stats := getStats(domain) recordSize := int64(len(EncodeAuditRecord(record))) stats.auditRecords += recordSize @@ -89,7 +100,7 @@ func AnalyzeStorage(ctx context.Context) ([]*StorageSize, error) { // Now aggregate the information. for domain, stats := range thickStats { - sizes := StorageSize{Domain: domain} + sizes := StorageSize{Domain: domain.String()} sizes.CurrentSize += stats.siteManifests for _, size := range stats.siteBlobs { sizes.CurrentSize += size diff --git a/src/audit.go b/src/audit.go index 737cdc2..f0f39a2 100644 --- a/src/audit.go +++ b/src/audit.go @@ -15,6 +15,7 @@ import ( "strings" "time" + "codeberg.org/git-pages/git-pages/src/names" exponential "github.com/jpillora/backoff" "github.com/kankanreno/go-snowflake" "github.com/prometheus/client_golang/prometheus" @@ -396,65 +397,50 @@ func notifyAudit(ctx context.Context, id AuditID) { } func (audited *auditedBackend) CommitManifest( - ctx context.Context, name string, manifest *Manifest, opts ModifyManifestOptions, + ctx context.Context, site names.Site, manifest *Manifest, opts ModifyManifestOptions, ) error { - domain, project, ok := strings.Cut(name, "/") - if !ok { - panic("malformed manifest name") - } - if err := audited.appendNewAuditRecord(ctx, &AuditRecord{ Event: AuditEvent_CommitManifest.Enum(), - Domain: proto.String(domain), - Project: proto.String(project), + Domain: proto.String(site.Domain.String()), + Project: proto.String(site.Project.String()), Manifest: manifest, }); err != nil { return err } - return audited.Backend.CommitManifest(ctx, name, manifest, opts) + return audited.Backend.CommitManifest(ctx, site, manifest, opts) } func (audited *auditedBackend) DeleteManifest( - ctx context.Context, name string, opts ModifyManifestOptions, + ctx context.Context, site names.Site, opts ModifyManifestOptions, ) error { - domain, project, ok := strings.Cut(name, "/") - if !ok { - panic("malformed manifest name") - } - if err := audited.appendNewAuditRecord(ctx, &AuditRecord{ Event: AuditEvent_DeleteManifest.Enum(), - Domain: proto.String(domain), - Project: proto.String(project), + Domain: proto.String(site.Domain.String()), + Project: proto.String(site.Project.String()), }); err != nil { return err } - return audited.Backend.DeleteManifest(ctx, name, opts) + return audited.Backend.DeleteManifest(ctx, site, opts) } -func (audited *auditedBackend) ExpireManifest(ctx context.Context, name string) error { - domain, project, ok := strings.Cut(name, "/") - if !ok { - panic("malformed manifest name") - } - +func (audited *auditedBackend) ExpireManifest(ctx context.Context, site names.Site) error { if err := audited.appendNewAuditRecord(ctx, &AuditRecord{ Event: AuditEvent_ExpireManifest.Enum(), - Domain: proto.String(domain), - Project: proto.String(project), + Domain: proto.String(site.Domain.String()), + Project: proto.String(site.Project.String()), }); err != nil { return err } - return audited.Backend.ExpireManifest(ctx, name) + return audited.Backend.ExpireManifest(ctx, site) } -func (audited *auditedBackend) FreezeDomain(ctx context.Context, domain string) error { +func (audited *auditedBackend) FreezeDomain(ctx context.Context, domain names.Domain) error { if err := audited.appendNewAuditRecord(ctx, &AuditRecord{ Event: AuditEvent_FreezeDomain.Enum(), - Domain: proto.String(domain), + Domain: proto.String(domain.String()), }); err != nil { return err } @@ -462,10 +448,10 @@ func (audited *auditedBackend) FreezeDomain(ctx context.Context, domain string) return audited.Backend.FreezeDomain(ctx, domain) } -func (audited *auditedBackend) UnfreezeDomain(ctx context.Context, domain string) error { +func (audited *auditedBackend) UnfreezeDomain(ctx context.Context, domain names.Domain) error { if err := audited.appendNewAuditRecord(ctx, &AuditRecord{ Event: AuditEvent_UnfreezeDomain.Enum(), - Domain: proto.String(domain), + Domain: proto.String(domain.String()), }); err != nil { return err } diff --git a/src/auth.go b/src/auth.go index a906cf2..523101f 100644 --- a/src/auth.go +++ b/src/auth.go @@ -14,7 +14,6 @@ import ( "time" "codeberg.org/git-pages/git-pages/src/names" - "golang.org/x/net/idna" ) type AuthError struct { @@ -45,40 +44,6 @@ func authorizeInsecure(r *http.Request) *Authorization { return nil } -var idnaProfile = idna.New(idna.MapForLookup(), idna.BidiRule()) - -func GetHost(r *http.Request) (string, error) { - host, _, err := net.SplitHostPort(r.Host) - if err != nil { - host = r.Host - } - // this also rejects invalid characters and labels - host, err = idnaProfile.ToASCII(host) - if err != nil { - if config.Feature("relaxed-idna") { - // unfortunately, the go IDNA library has some significant issues around its - // Unicode TR46 implementation: https://github.com/golang/go/issues/76804 - // we would like to allow *just* the _ here, but adding `idna.StrictDomainName(false)` - // would also accept domains like `*.foo.bar` which should clearly be disallowed. - // as a workaround, accept a domain name if it is valid with all `_` characters - // replaced with an alphanumeric character (we use `a`); this allows e.g. `foo_bar.xxx` - // and `foo__bar.xxx`, as well as `_foo.xxx` and `foo_.xxx`. labels starting with - // an underscore are explicitly rejected below. - _, err = idnaProfile.ToASCII(strings.ReplaceAll(host, "_", "a")) - } - if err != nil { - return "", AuthError{http.StatusBadRequest, - fmt.Sprintf("malformed host name %q", host)} - } - } - if strings.HasPrefix(host, ".") || strings.HasPrefix(host, "_") { - return "", AuthError{http.StatusBadRequest, - fmt.Sprintf("reserved host name %q", host)} - } - host = strings.TrimSuffix(host, ".") - return host, nil -} - type Authorization struct { // If `nil`, any URL is allowed. If not, only those in the set are allowed. repoURLs []string @@ -98,7 +63,7 @@ func (auth *Authorization) ForgeRepoURL() string { } func authorizeDNSChallenge(r *http.Request) (*Authorization, error) { - host, err := GetHost(r) + domain, err := GetDomain(r) if err != nil { return nil, err } @@ -143,14 +108,14 @@ func authorizeDNSChallenge(r *http.Request) (*Authorization, error) { param = password } - challengeHostname := fmt.Sprintf("_git-pages-challenge.%s", host) + challengeHostname := fmt.Sprintf("_git-pages-challenge.%s", domain) actualChallenges, err := net.LookupTXT(challengeHostname) if err != nil { return nil, AuthError{http.StatusUnauthorized, fmt.Sprintf("failed to look up DNS challenge: %s TXT", challengeHostname)} } - expectedChallenge := fmt.Sprintf("%x", sha256.Sum256(fmt.Appendf(nil, "%s %s", host, param))) + expectedChallenge := fmt.Sprintf("%x", sha256.Sum256(fmt.Appendf(nil, "%s %s", domain, param))) if !slices.Contains(actualChallenges, expectedChallenge) { return nil, AuthError{http.StatusUnauthorized, fmt.Sprintf( "defeated by DNS challenge: %s TXT %v does not include %s", @@ -167,7 +132,7 @@ func authorizeDNSChallenge(r *http.Request) (*Authorization, error) { } func authorizeDNSAllowlist(r *http.Request, scope string) (*Authorization, error) { - host, err := GetHost(r) + domain, err := GetDomain(r) if err != nil { return nil, err } @@ -177,7 +142,7 @@ func authorizeDNSAllowlist(r *http.Request, scope string) (*Authorization, error return nil, err } - allowlistHostname := fmt.Sprintf("_%s.%s", scope, host) + allowlistHostname := fmt.Sprintf("_%s.%s", scope, domain) records, err := net.LookupTXT(allowlistHostname) if err != nil { return nil, AuthError{http.StatusUnauthorized, @@ -223,12 +188,12 @@ func authorizeDNSAllowlist(r *http.Request, scope string) (*Authorization, error // used for `/.git-pages/...` metadata func authorizeWildcardMatchHost(r *http.Request, pattern *WildcardPattern) (*Authorization, error) { - host, err := GetHost(r) + domain, err := GetDomain(r) if err != nil { return nil, err } - if _, found := pattern.Matches(host, WildcardDomainAny); found { + if _, found := pattern.Matches(domain, WildcardDomainAny); found { return &Authorization{ repoURLs: []string{}, branch: "", @@ -236,14 +201,14 @@ func authorizeWildcardMatchHost(r *http.Request, pattern *WildcardPattern) (*Aut } else { return nil, AuthError{ http.StatusUnauthorized, - fmt.Sprintf("domain %s does not match wildcard %s", host, pattern.GetHost()), + fmt.Sprintf("domain %s does not match wildcard %s", domain, pattern.GetHost()), } } } // used for updates to site content func authorizeWildcardMatchSite(r *http.Request, pattern *WildcardPattern) (*Authorization, error) { - host, err := GetHost(r) + domain, err := GetDomain(r) if err != nil { return nil, err } @@ -253,13 +218,13 @@ func authorizeWildcardMatchSite(r *http.Request, pattern *WildcardPattern) (*Aut return nil, err } - if userName, found := pattern.Matches(host, WildcardDomainPrimary); found { + if userName, found := pattern.Matches(domain, WildcardDomainPrimary); found { repoURL, branch := pattern.ApplyTemplate(userName, projectName.String()) return &Authorization{repoURLs: []string{repoURL}, branch: branch}, nil } else { return nil, AuthError{ http.StatusUnauthorized, - fmt.Sprintf("domain %s does not match wildcard %s", host, pattern.GetHost()), + fmt.Sprintf("domain %s does not match wildcard %s", domain, pattern.GetHost()), } } } @@ -267,29 +232,29 @@ func authorizeWildcardMatchSite(r *http.Request, pattern *WildcardPattern) (*Aut // used for compatibility with Codeberg Pages v2 // see https://docs.codeberg.org/codeberg-pages/using-custom-domain/ func authorizeCodebergPagesV2(r *http.Request) (*Authorization, error) { - host, err := GetHost(r) + domain, err := GetDomain(r) if err != nil { return nil, err } dnsRecords := []string{} - cnameRecord, err := net.LookupCNAME(host) + cnameRecord, err := net.LookupCNAME(domain.String()) // "LookupCNAME does not return an error if host does not contain DNS "CNAME" records, // as long as host resolves to address records. - if err == nil && cnameRecord != host { + if err == nil && cnameRecord != domain.String() { // LookupCNAME() returns a domain with the root label, i.e. `username.codeberg.page.`, // with the trailing dot dnsRecords = append(dnsRecords, strings.TrimSuffix(cnameRecord, ".")) } - txtRecords, err := net.LookupTXT(host) + txtRecords, err := net.LookupTXT(domain.String()) if err == nil { dnsRecords = append(dnsRecords, txtRecords...) } if len(dnsRecords) > 0 { - logc.Printf(r.Context(), "auth: %s TXT/CNAME: %q\n", host, dnsRecords) + logc.Printf(r.Context(), "auth: %s TXT/CNAME: %q\n", domain, dnsRecords) } for _, dnsRecord := range dnsRecords { @@ -328,7 +293,7 @@ func authorizeCodebergPagesV2(r *http.Request) (*Authorization, error) { return nil, AuthError{ http.StatusUnauthorized, - fmt.Sprintf("domain %s does not have Codeberg Pages TXT or CNAME records", host), + fmt.Sprintf("domain %s does not have Codeberg Pages TXT or CNAME records", domain), } } @@ -610,7 +575,7 @@ func authorizeForgeWildcard(r *http.Request) (*Authorization, error) { return nil, AuthError{http.StatusUnauthorized, "missing Forge-Authorization header"} } - host, err := GetHost(r) + host, err := GetDomain(r) if err != nil { return nil, err } @@ -777,15 +742,15 @@ func AuthorizeDeletion(r *http.Request) (*Authorization, error) { } func CheckForbiddenDomain(r *http.Request) error { - host, err := GetHost(r) + domain, err := GetDomain(r) if err != nil { return err } - host = strings.ToLower(host) for _, reservedDomain := range config.Limits.ForbiddenDomains { reservedDomain = strings.ToLower(reservedDomain) - if host == reservedDomain || strings.HasSuffix(host, fmt.Sprintf(".%s", reservedDomain)) { + if domain.String() == reservedDomain || + strings.HasSuffix(domain.String(), fmt.Sprintf(".%s", reservedDomain)) { return AuthError{http.StatusForbidden, "forbidden domain"} } } diff --git a/src/backend.go b/src/backend.go index 5b80714..01edfe1 100644 --- a/src/backend.go +++ b/src/backend.go @@ -8,6 +8,8 @@ import ( "iter" "strings" "time" + + "codeberg.org/git-pages/git-pages/src/names" ) var ErrObjectNotFound = errors.New("not found") @@ -46,7 +48,7 @@ type GetManifestOptions struct { } type ManifestMetadata struct { - Name string + Name names.Site Size int64 LastModified time.Time ETag string @@ -103,7 +105,7 @@ type Backend interface { EnumerateBlobs(ctx context.Context) iter.Seq2[BlobMetadata, error] // Retrieve a manifest. - GetManifest(ctx context.Context, name string, opts GetManifestOptions) ( + GetManifest(ctx context.Context, site names.Site, opts GetManifestOptions) ( manifest *Manifest, metadata ManifestMetadata, err error, ) @@ -119,13 +121,13 @@ type Backend interface { // Commit a manifest. This is an atomic operation; `GetManifest` calls will return either // the old version or the new version of the manifest, never anything else. - CommitManifest(ctx context.Context, name string, manifest *Manifest, opts ModifyManifestOptions) error + CommitManifest(ctx context.Context, site names.Site, manifest *Manifest, opts ModifyManifestOptions) error // Delete a manifest. This operation is initiated via the API. - DeleteManifest(ctx context.Context, name string, opts ModifyManifestOptions) error + DeleteManifest(ctx context.Context, site names.Site, opts ModifyManifestOptions) error // Expire a manifest. This operation is initiated via a scheduled job. - ExpireManifest(ctx context.Context, name string) error + ExpireManifest(ctx context.Context, site names.Site) error // Iterate through metadata of all manifests. Whether manifests that are newly added during // iteration will appear in the results is unspecified. @@ -139,17 +141,17 @@ type Backend interface { HasSiteListChanged(ctx context.Context, since time.Time) (changed bool, lastChanged time.Time, err error) // Check whether a domain has any deployments. - CheckDomain(ctx context.Context, domain string) (found bool, err error) + CheckDomain(ctx context.Context, domain names.Domain) (found bool, err error) // Create a domain. This allows us to start serving content for the domain. - CreateDomain(ctx context.Context, domain string) error + CreateDomain(ctx context.Context, domain names.Domain) error // Freeze a domain. This allows a site to be administratively locked, e.g. if it // is discovered serving abusive content. - FreezeDomain(ctx context.Context, domain string) error + FreezeDomain(ctx context.Context, domain names.Domain) error // Thaw a domain. This removes the previously placed administrative lock (if any). - UnfreezeDomain(ctx context.Context, domain string) error + UnfreezeDomain(ctx context.Context, domain names.Domain) error // Append a record to the audit log. AppendAuditLog(ctx context.Context, id AuditID, record *AuditRecord) error diff --git a/src/backend_fs.go b/src/backend_fs.go index ddb3731..1976eb2 100644 --- a/src/backend_fs.go +++ b/src/backend_fs.go @@ -13,6 +13,7 @@ import ( "strings" "time" + "codeberg.org/git-pages/git-pages/src/names" sys "codeberg.org/git-pages/git-pages/src/sys" ) @@ -219,11 +220,11 @@ func (fs *FSBackend) EnumerateBlobs(ctx context.Context) iter.Seq2[BlobMetadata, } func (fs *FSBackend) GetManifest( - ctx context.Context, name string, opts GetManifestOptions, + ctx context.Context, site names.Site, opts GetManifestOptions, ) ( manifest *Manifest, metadata ManifestMetadata, err error, ) { - stat, err := fs.siteRoot.Stat(name) + stat, err := fs.siteRoot.Stat(site.String()) if errors.Is(err, os.ErrNotExist) { err = fmt.Errorf("%w: %s", ErrObjectNotFound, err.(*os.PathError).Path) return @@ -231,14 +232,14 @@ func (fs *FSBackend) GetManifest( err = fmt.Errorf("stat: %w", err) return } - data, err := fs.siteRoot.ReadFile(name) + data, err := fs.siteRoot.ReadFile(site.String()) if err != nil { err = fmt.Errorf("read: %w", err) return } manifest, err = DecodeManifest(data) if err != nil { - err = fmt.Errorf("decode %s: %w", name, err) + err = fmt.Errorf("decode %s: %w", site, err) return } return manifest, ManifestMetadata{ @@ -266,11 +267,11 @@ func (fs *FSBackend) StageManifest(ctx context.Context, manifest *Manifest) erro return nil } -func domainFrozenMarkerName(domain string) string { - return filepath.Join(domain, ".frozen") +func domainFrozenMarkerName(domain names.Domain) string { + return filepath.Join(domain.String(), ".frozen") } -func (fs *FSBackend) checkDomainFrozen(ctx context.Context, domain string) error { +func (fs *FSBackend) checkDomainFrozen(ctx context.Context, domain names.Domain) error { if _, err := fs.siteRoot.Stat(domainFrozenMarkerName(domain)); err == nil { return ErrDomainFrozen } else if !errors.Is(err, os.ErrNotExist) { @@ -293,8 +294,8 @@ type manifestLockGuard struct { file *os.File } -func lockManifest(fs *os.Root, name string) (*manifestLockGuard, error) { - file, err := fs.Open(name) +func lockManifest(fs *os.Root, site names.Site) (*manifestLockGuard, error) { + file, err := fs.Open(site.String()) if errors.Is(err, os.ErrNotExist) { return &manifestLockGuard{nil}, nil } else if err != nil { @@ -315,10 +316,10 @@ func (guard *manifestLockGuard) Unlock() { } func (fs *FSBackend) checkManifestPrecondition( - ctx context.Context, name string, opts ModifyManifestOptions, + ctx context.Context, site names.Site, opts ModifyManifestOptions, ) error { if !opts.IfUnmodifiedSince.IsZero() { - stat, err := fs.siteRoot.Stat(name) + stat, err := fs.siteRoot.Stat(site.String()) if err != nil { return fmt.Errorf("stat: %w", err) } @@ -329,7 +330,7 @@ func (fs *FSBackend) checkManifestPrecondition( } if opts.IfMatch != "" { - data, err := fs.siteRoot.ReadFile(name) + data, err := fs.siteRoot.ReadFile(site.String()) if err != nil { return fmt.Errorf("read: %w", err) } @@ -343,22 +344,21 @@ func (fs *FSBackend) checkManifestPrecondition( } func (fs *FSBackend) CommitManifest( - ctx context.Context, name string, manifest *Manifest, opts ModifyManifestOptions, + ctx context.Context, site names.Site, manifest *Manifest, opts ModifyManifestOptions, ) error { if fs.hasAtomicCAS { - if guard, err := lockManifest(fs.siteRoot, name); err != nil { + if guard, err := lockManifest(fs.siteRoot, site); err != nil { return err } else { defer guard.Unlock() } } - domain := filepath.Dir(name) - if err := fs.checkDomainFrozen(ctx, domain); err != nil { + if err := fs.checkDomainFrozen(ctx, site.Domain); err != nil { return err } - if err := fs.checkManifestPrecondition(ctx, name, opts); err != nil { + if err := fs.checkManifestPrecondition(ctx, site, opts); err != nil { return err } @@ -369,11 +369,11 @@ func (fs *FSBackend) CommitManifest( return fmt.Errorf("manifest not staged") } - if err := fs.siteRoot.MkdirAll(domain, 0o755); err != nil { + if err := fs.siteRoot.MkdirAll(site.Domain.String(), 0o755); err != nil { return fmt.Errorf("mkdir: %w", err) } - if err := fs.siteRoot.Rename(manifestHashName, name); err != nil { + if err := fs.siteRoot.Rename(manifestHashName, site.String()); err != nil { return fmt.Errorf("rename: %w", err) } @@ -381,26 +381,25 @@ func (fs *FSBackend) CommitManifest( } func (fs *FSBackend) DeleteManifest( - ctx context.Context, name string, opts ModifyManifestOptions, + ctx context.Context, site names.Site, opts ModifyManifestOptions, ) error { if fs.hasAtomicCAS { - if guard, err := lockManifest(fs.siteRoot, name); err != nil { + if guard, err := lockManifest(fs.siteRoot, site); err != nil { return err } else { defer guard.Unlock() } } - domain := filepath.Dir(name) - if err := fs.checkDomainFrozen(ctx, domain); err != nil { + if err := fs.checkDomainFrozen(ctx, site.Domain); err != nil { return err } - if err := fs.checkManifestPrecondition(ctx, name, opts); err != nil { + if err := fs.checkManifestPrecondition(ctx, site, opts); err != nil { return err } - err := fs.siteRoot.Remove(name) + err := fs.siteRoot.Remove(site.String()) if errors.Is(err, os.ErrNotExist) { return nil } else { @@ -409,16 +408,15 @@ func (fs *FSBackend) DeleteManifest( } func (fs *FSBackend) ExpireManifest( - ctx context.Context, name string, + ctx context.Context, site names.Site, ) error { - return fs.DeleteManifest(ctx, name, ModifyManifestOptions{}) + return fs.DeleteManifest(ctx, site, ModifyManifestOptions{}) } func (fs *FSBackend) EnumerateManifests(ctx context.Context) iter.Seq2[*ManifestMetadata, error] { return func(yield func(*ManifestMetadata, error) bool) { iofs.WalkDir(fs.siteRoot.FS(), ".", func(path string, entry iofs.DirEntry, err error) error { - _, project, _ := strings.Cut(path, "/") var metadata *ManifestMetadata if err != nil { // report error @@ -427,7 +425,7 @@ func (fs *FSBackend) EnumerateManifests(ctx context.Context) iter.Seq2[*Manifest return nil } else if strings.HasPrefix(path, ".tmp") { // temporary - } else if project == "" || strings.HasPrefix(project, ".") && project != ".index" { + } else if site, err := names.ParseSite(path); err != nil { // skip internal return nil } else if info, err := entry.Info(); err != nil { @@ -435,7 +433,7 @@ func (fs *FSBackend) EnumerateManifests(ctx context.Context) iter.Seq2[*Manifest } else { // report blob metadata = &ManifestMetadata{ - Name: path, + Name: site, Size: info.Size(), LastModified: info.ModTime(), } @@ -465,8 +463,8 @@ func (fs *FSBackend) GetAllManifests(ctx context.Context) iter.Seq2[tuple[*Manif } } -func (fs *FSBackend) CheckDomain(ctx context.Context, domain string) (bool, error) { - _, err := fs.siteRoot.Stat(domain) +func (fs *FSBackend) CheckDomain(ctx context.Context, domain names.Domain) (bool, error) { + _, err := fs.siteRoot.Stat(domain.String()) if errors.Is(err, os.ErrNotExist) { return false, nil } else if err == nil { @@ -476,15 +474,15 @@ func (fs *FSBackend) CheckDomain(ctx context.Context, domain string) (bool, erro } } -func (fs *FSBackend) CreateDomain(ctx context.Context, domain string) error { +func (fs *FSBackend) CreateDomain(ctx context.Context, domain names.Domain) error { return nil // no-op } -func (fs *FSBackend) FreezeDomain(ctx context.Context, domain string) error { +func (fs *FSBackend) FreezeDomain(ctx context.Context, domain names.Domain) error { return fs.siteRoot.WriteFile(domainFrozenMarkerName(domain), []byte{}, 0o644) } -func (fs *FSBackend) UnfreezeDomain(ctx context.Context, domain string) error { +func (fs *FSBackend) UnfreezeDomain(ctx context.Context, domain names.Domain) error { err := fs.siteRoot.Remove(domainFrozenMarkerName(domain)) if errors.Is(err, os.ErrNotExist) { return nil diff --git a/src/backend_s3.go b/src/backend_s3.go index aca097c..d990da7 100644 --- a/src/backend_s3.go +++ b/src/backend_s3.go @@ -13,6 +13,7 @@ import ( "sync" "time" + "codeberg.org/git-pages/git-pages/src/names" "github.com/c2h5oh/datasize" "github.com/maypok86/otter/v2" "github.com/minio/minio-go/v7" @@ -128,7 +129,7 @@ type S3Backend struct { client *minio.Client bucket string blobCache *observedCache[string, *CachedBlob] - siteCache *observedCache[string, *CachedManifest] + siteCache *observedCache[names.Site, *CachedManifest] featureCache *otter.Cache[BackendFeature, bool] } @@ -224,7 +225,7 @@ func NewS3Backend(ctx context.Context, config *S3Config) (*S3Backend, error) { EvictionNumberCounter: manifestCacheEvictionsCount, } siteCache, err := newObservedCache(makeCacheOptions(&config.SiteCache, - func(key string, value *CachedManifest) uint32 { return value.weight }), + func(key names.Site, value *CachedManifest) uint32 { return value.weight }), siteCacheMetrics) if err != nil { return nil, err @@ -410,8 +411,8 @@ func (s3 *S3Backend) EnumerateBlobs(ctx context.Context) iter.Seq2[BlobMetadata, } } -func manifestObjectName(name string) string { - return fmt.Sprintf("site/%s", name) +func manifestObjectName(object string) string { + return fmt.Sprintf("site/%s", object) } func stagedManifestObjectName(manifestData []byte) string { @@ -423,7 +424,7 @@ type s3ManifestLoader struct { } func (l s3ManifestLoader) Load( - ctx context.Context, key string, + ctx context.Context, key names.Site, ) ( *CachedManifest, error, ) { @@ -431,7 +432,7 @@ func (l s3ManifestLoader) Load( } func (l s3ManifestLoader) Reload( - ctx context.Context, key string, oldValue *CachedManifest, + ctx context.Context, key names.Site, oldValue *CachedManifest, ) ( *CachedManifest, error, ) { @@ -439,18 +440,18 @@ func (l s3ManifestLoader) Reload( } func (l s3ManifestLoader) load( - ctx context.Context, name string, oldManifest *CachedManifest, + ctx context.Context, site names.Site, oldManifest *CachedManifest, ) ( *CachedManifest, error, ) { - logc.Printf(ctx, "s3: get manifest %s\n", name) + logc.Printf(ctx, "s3: get manifest %s\n", site) loader := func() (*CachedManifest, error) { opts := minio.GetObjectOptions{} if oldManifest != nil && oldManifest.metadata.ETag != "" { opts.SetMatchETagExcept(oldManifest.metadata.ETag) } - object, err := l.s3.client.GetObject(ctx, l.s3.bucket, manifestObjectName(name), opts) + object, err := l.s3.client.GetObject(ctx, l.s3.bucket, manifestObjectName(site.String()), opts) // Note that many errors (e.g. NoSuchKey) will be reported only after this point. if err != nil { return nil, err @@ -469,7 +470,7 @@ func (l s3ManifestLoader) load( manifest, err := DecodeManifest(data) if err != nil { - err = fmt.Errorf("decode %s: %w", name, err) + err = fmt.Errorf("decode %s: %w", site, err) return nil, err } @@ -512,19 +513,19 @@ func (l s3ManifestLoader) load( } func (s3 *S3Backend) GetManifest( - ctx context.Context, name string, opts GetManifestOptions, + ctx context.Context, site names.Site, opts GetManifestOptions, ) ( manifest *Manifest, metadata ManifestMetadata, err error, ) { if opts.BypassCache { - entry, found := s3.siteCache.Cache.GetEntry(name) + entry, found := s3.siteCache.Cache.GetEntry(site) if found && entry.RefreshableAt().Before(time.Now()) { - s3.siteCache.Cache.Invalidate(name) + s3.siteCache.Cache.Invalidate(site) } } var cached *CachedManifest - cached, err = s3.siteCache.Get(ctx, name, s3ManifestLoader{s3}) + cached, err = s3.siteCache.Get(ctx, site, s3ManifestLoader{s3}) if err != nil { return } else { @@ -543,11 +544,11 @@ func (s3 *S3Backend) StageManifest(ctx context.Context, manifest *Manifest) erro return err } -func domainFrozenObjectName(domain string) string { - return manifestObjectName(fmt.Sprintf("%s/.frozen", domain)) +func domainFrozenObjectName(domain names.Domain) string { + return manifestObjectName(fmt.Sprintf("%s/.frozen", domain.String())) } -func (s3 *S3Backend) checkDomainFrozen(ctx context.Context, domain string) error { +func (s3 *S3Backend) checkDomainFrozen(ctx context.Context, domain names.Domain) error { _, err := s3.client.StatObject(ctx, s3.bucket, domainFrozenObjectName(domain), minio.GetObjectOptions{}) if err == nil { @@ -577,9 +578,9 @@ func (s3 *S3Backend) HasAtomicCAS(ctx context.Context) bool { } func (s3 *S3Backend) checkManifestPrecondition( - ctx context.Context, name string, opts ModifyManifestOptions, + ctx context.Context, site names.Site, opts ModifyManifestOptions, ) (exists bool, err error) { - stat, err := s3.client.StatObject(ctx, s3.bucket, manifestObjectName(name), + stat, err := s3.client.StatObject(ctx, s3.bucket, manifestObjectName(site.String()), minio.GetObjectOptions{}) if err != nil { errResp := minio.ToErrorResponse(err) @@ -603,17 +604,16 @@ func (s3 *S3Backend) checkManifestPrecondition( } func (s3 *S3Backend) CommitManifest( - ctx context.Context, name string, manifest *Manifest, opts ModifyManifestOptions, + ctx context.Context, site names.Site, manifest *Manifest, opts ModifyManifestOptions, ) error { data := EncodeManifest(manifest) - logc.Printf(ctx, "s3: commit manifest %x -> %s", sha256.Sum256(data), name) + logc.Printf(ctx, "s3: commit manifest %x -> %s", sha256.Sum256(data), site.String()) - domain, _, _ := strings.Cut(name, "/") - if err := s3.checkDomainFrozen(ctx, domain); err != nil { + if err := s3.checkDomainFrozen(ctx, site.Domain); err != nil { return err } - existed, err := s3.checkManifestPrecondition(ctx, name, opts) + existed, err := s3.checkManifestPrecondition(ctx, site, opts) if err != nil { return err } @@ -628,11 +628,11 @@ func (s3 *S3Backend) CommitManifest( // It does reliably work on MinIO at least. putOptions.SetMatchETag(opts.IfMatch) } - _, putErr := s3.client.PutObject(ctx, s3.bucket, manifestObjectName(name), + _, putErr := s3.client.PutObject(ctx, s3.bucket, manifestObjectName(site.String()), bytes.NewReader(data), int64(len(data)), putOptions) removeErr := s3.client.RemoveObject(ctx, s3.bucket, stagedManifestObjectName(data), minio.RemoveObjectOptions{}) - s3.siteCache.Cache.Invalidate(name) + s3.siteCache.Cache.Invalidate(site) if putErr != nil { if errResp := minio.ToErrorResponse(putErr); errResp.Code == "PreconditionFailed" { return ErrPreconditionFailed @@ -653,26 +653,25 @@ func (s3 *S3Backend) CommitManifest( } func (s3 *S3Backend) DeleteManifest( - ctx context.Context, name string, opts ModifyManifestOptions, + ctx context.Context, site names.Site, opts ModifyManifestOptions, ) error { - logc.Printf(ctx, "s3: delete manifest %s\n", name) + logc.Printf(ctx, "s3: delete manifest %s\n", site) - domain, _, _ := strings.Cut(name, "/") - if err := s3.checkDomainFrozen(ctx, domain); err != nil { + if err := s3.checkDomainFrozen(ctx, site.Domain); err != nil { return err } - existed, err := s3.checkManifestPrecondition(ctx, name, opts) + existed, err := s3.checkManifestPrecondition(ctx, site, opts) if err != nil { return err } - err = s3.client.RemoveObject(ctx, s3.bucket, manifestObjectName(name), + err = s3.client.RemoveObject(ctx, s3.bucket, manifestObjectName(site.String()), minio.RemoveObjectOptions{}) if err != nil { return err } - s3.siteCache.Cache.Invalidate(name) + s3.siteCache.Cache.Invalidate(site) if existed { if err := s3.bumpLastSiteUpdateTimestamp(ctx); err != nil { @@ -684,9 +683,9 @@ func (s3 *S3Backend) DeleteManifest( } func (s3 *S3Backend) ExpireManifest( - ctx context.Context, name string, + ctx context.Context, site names.Site, ) error { - return s3.DeleteManifest(ctx, name, ModifyManifestOptions{}) + return s3.DeleteManifest(ctx, site, ModifyManifestOptions{}) } func (s3 *S3Backend) EnumerateManifests(ctx context.Context) iter.Seq2[*ManifestMetadata, error] { @@ -705,14 +704,13 @@ func (s3 *S3Backend) EnumerateManifests(ctx context.Context) iter.Seq2[*Manifest var err error if err = object.Err; err == nil { key := strings.TrimPrefix(object.Key, prefix) - _, project, _ := strings.Cut(key, "/") if strings.HasSuffix(key, "/") { continue // directory; skip - } else if project == "" || strings.HasPrefix(project, ".") && project != ".index" { - continue // internal; skip + } else if site, err := names.ParseSite(key); err != nil { + continue // not a site (marker, etc); skip } else { metadata = &ManifestMetadata{ - Name: key, + Name: site, Size: object.Size, LastModified: object.LastModified, ETag: object.ETag, @@ -768,11 +766,11 @@ func (s3 *S3Backend) GetAllManifests(ctx context.Context) iter.Seq2[tuple[*Manif } } -func domainCheckObjectName(domain string) string { +func domainCheckObjectName(domain names.Domain) string { return manifestObjectName(fmt.Sprintf("%s/.exists", domain)) } -func (s3 *S3Backend) CheckDomain(ctx context.Context, domain string) (exists bool, err error) { +func (s3 *S3Backend) CheckDomain(ctx context.Context, domain names.Domain) (exists bool, err error) { logc.Printf(ctx, "s3: check domain %s\n", domain) _, err = s3.client.StatObject(ctx, s3.bucket, domainCheckObjectName(domain), @@ -803,7 +801,7 @@ func (s3 *S3Backend) CheckDomain(ctx context.Context, domain string) (exists boo return } -func (s3 *S3Backend) CreateDomain(ctx context.Context, domain string) error { +func (s3 *S3Backend) CreateDomain(ctx context.Context, domain names.Domain) error { logc.Printf(ctx, "s3: create domain %s\n", domain) _, err := s3.client.PutObject(ctx, s3.bucket, domainCheckObjectName(domain), @@ -811,7 +809,7 @@ func (s3 *S3Backend) CreateDomain(ctx context.Context, domain string) error { return err } -func (s3 *S3Backend) FreezeDomain(ctx context.Context, domain string) error { +func (s3 *S3Backend) FreezeDomain(ctx context.Context, domain names.Domain) error { logc.Printf(ctx, "s3: freeze domain %s\n", domain) _, err := s3.client.PutObject(ctx, s3.bucket, domainFrozenObjectName(domain), @@ -819,7 +817,7 @@ func (s3 *S3Backend) FreezeDomain(ctx context.Context, domain string) error { return err } -func (s3 *S3Backend) UnfreezeDomain(ctx context.Context, domain string) error { +func (s3 *S3Backend) UnfreezeDomain(ctx context.Context, domain names.Domain) error { logc.Printf(ctx, "s3: unfreeze domain %s\n", domain) err := s3.client.RemoveObject(ctx, s3.bucket, domainFrozenObjectName(domain), diff --git a/src/domains.go b/src/domains.go deleted file mode 100644 index cdb2fb2..0000000 --- a/src/domains.go +++ /dev/null @@ -1,29 +0,0 @@ -package git_pages - -import ( - "slices" - "strings" - - "golang.org/x/net/publicsuffix" -) - -func EffectiveTLDPlusOne(domain string) string { - domainParts := strings.Split(domain, ".") - for _, pattern := range wildcards { - for _, suffix := range [][]string{pattern.PreviewDomain, pattern.Domain} { - if len(domainParts) < len(suffix)+1 { - continue - } - if slices.Equal(domainParts[len(domainParts)-len(suffix):], suffix) { - return strings.Join(domainParts[len(domainParts)-len(suffix)-1:], ".") - } - } - } - - result, err := publicsuffix.EffectiveTLDPlusOne(domain) - if err == nil { - return result - } - - return domain -} diff --git a/src/existence.go b/src/existence.go index e38033a..629e0a7 100644 --- a/src/existence.go +++ b/src/existence.go @@ -9,10 +9,10 @@ package git_pages import ( "context" "fmt" - "strings" "sync" "time" + "codeberg.org/git-pages/git-pages/src/names" "github.com/bits-and-blooms/bloom/v3" exponential "github.com/jpillora/backoff" ) @@ -45,13 +45,13 @@ func (value Existence) String() string { type ExistenceCache interface { // Check if we might be serving the site. - CheckSite(ctx context.Context, site string) (result Existence) + CheckSite(ctx context.Context, site names.Site) (result Existence) // Check if we might be serving the domain. - CheckDomain(ctx context.Context, domain string) (result Existence) + CheckDomain(ctx context.Context, domain names.Domain) (result Existence) // Add the site to the cache. - AddSite(ctx context.Context, site string) + AddSite(ctx context.Context, site names.Site) } func CreateExistenceCache(ctx context.Context) (ExistenceCache, error) { @@ -149,10 +149,8 @@ func (c *bloomExistenceCache) refresh(ctx context.Context) error { if err != nil { return fmt.Errorf("enum manifests: %w", err) } - site := metadata.Name - domain, _, _ := strings.Cut(site, "/") - sites.AddString(site) - domains.AddString(domain) + sites.AddString(metadata.Name.String()) + domains.AddString(metadata.Name.Domain.String()) siteCount++ } @@ -165,14 +163,14 @@ func (c *bloomExistenceCache) refresh(ctx context.Context) error { return nil } -func (c *bloomExistenceCache) CheckSite(ctx context.Context, site string) (result Existence) { +func (c *bloomExistenceCache) CheckSite(ctx context.Context, site names.Site) (result Existence) { select { case c.accessCh <- struct{}{}: default: } c.filterMu.Lock() - if c.sites.TestString(site) { + if c.sites.TestString(site.String()) { result = ExistencePossible } else { result = ExistenceImpossible @@ -183,14 +181,14 @@ func (c *bloomExistenceCache) CheckSite(ctx context.Context, site string) (resul return } -func (c *bloomExistenceCache) CheckDomain(ctx context.Context, domain string) (result Existence) { +func (c *bloomExistenceCache) CheckDomain(ctx context.Context, domain names.Domain) (result Existence) { select { case c.accessCh <- struct{}{}: default: } c.filterMu.Lock() - if c.domains.TestString(domain) { + if c.domains.TestString(domain.String()) { result = ExistencePossible } else { result = ExistenceImpossible @@ -201,15 +199,13 @@ func (c *bloomExistenceCache) CheckDomain(ctx context.Context, domain string) (r return } -func (c *bloomExistenceCache) AddSite(ctx context.Context, site string) { +func (c *bloomExistenceCache) AddSite(ctx context.Context, site names.Site) { c.refreshMu.Lock() defer c.refreshMu.Unlock() - domain, _, _ := strings.Cut(site, "/") - c.filterMu.Lock() - c.sites.AddString(site) - c.domains.AddString(domain) + c.sites.AddString(site.String()) + c.domains.AddString(site.Domain.String()) c.filterMu.Unlock() logc.Printf(ctx, "existence: added site %s", site) @@ -217,12 +213,12 @@ func (c *bloomExistenceCache) AddSite(ctx context.Context, site string) { type dummyExistenceCache struct{} -func (d dummyExistenceCache) CheckSite(context.Context, string) Existence { +func (d dummyExistenceCache) CheckSite(context.Context, names.Site) Existence { return ExistencePossible } -func (d dummyExistenceCache) CheckDomain(context.Context, string) Existence { +func (d dummyExistenceCache) CheckDomain(context.Context, names.Domain) Existence { return ExistencePossible } -func (d dummyExistenceCache) AddSite(context.Context, string) {} +func (d dummyExistenceCache) AddSite(context.Context, names.Site) {} diff --git a/src/garbage.go b/src/garbage.go index 8d14901..2b31c20 100644 --- a/src/garbage.go +++ b/src/garbage.go @@ -49,7 +49,7 @@ func TraceGarbage(ctx context.Context) error { if err != nil { return fmt.Errorf("trace err: %w", err) } - traceManifest("site", metadata.Name, manifest) + traceManifest("site", metadata.Name.String(), manifest) } // Enumerate blobs live via audit records. diff --git a/src/main.go b/src/main.go index 3dd3b66..cdedaf4 100644 --- a/src/main.go +++ b/src/main.go @@ -175,17 +175,28 @@ func serve(ctx context.Context, listener net.Listener, handler http.Handler) { } } -func webRootArg(arg string) string { +func domainArg(arg string) (domain names.Domain) { + var err error + if domain, err = names.ParseDomain(arg, names.ParseDomainRelaxed); err != nil { + logc.Fatalf(context.Background(), "invalid site argument: %s\n", err) + } + return +} + +func siteArg(arg string) (site names.Site) { + var err error switch strings.Count(arg, "/") { case 0: - return arg + "/.index" + site.Domain = domainArg(arg) case 1: - return arg + if site, err = names.ParseSite(arg); err != nil { + logc.Fatalf(context.Background(), "invalid site argument: %s\n", err) + } default: logc.Fatalln(context.Background(), - "webroot argument must be either 'domain.tld' or 'domain.tld/dir") - return "" + "site argument must be either 'domain.tld' or 'domain.tld/dir") } + return } func fileOutputArg() (writer io.WriteCloser) { @@ -450,7 +461,7 @@ func Main(versionInfo string) { io.Copy(fileOutputArg(), reader) case *getManifest != "": - webRoot := webRootArg(*getManifest) + webRoot := siteArg(*getManifest) manifest, _, err := backend.GetManifest(ctx, webRoot, GetManifestOptions{}) if err != nil { logc.Fatalln(ctx, err) @@ -458,7 +469,7 @@ func Main(versionInfo string) { fmt.Fprintln(fileOutputArg(), string(ManifestJSON(manifest))) case *getArchive != "": - webRoot := webRootArg(*getArchive) + webRoot := siteArg(*getArchive) manifest, metadata, err := backend.GetManifest(ctx, webRoot, GetManifestOptions{}) if err != nil { @@ -473,7 +484,7 @@ func Main(versionInfo string) { logc.Fatalln(ctx, "update source must be provided as the argument") } - webRoot := webRootArg(*updateSite) + webRoot := siteArg(*updateSite) sourceURL, err := url.Parse(flag.Arg(0)) if err != nil { @@ -530,7 +541,7 @@ func Main(versionInfo string) { } case *deleteSite != "": - webRoot := webRootArg(*deleteSite) + webRoot := siteArg(*deleteSite) err := backend.DeleteManifest(ctx, webRoot, ModifyManifestOptions{}) if err != nil { logc.Fatalln(ctx, err) @@ -539,13 +550,13 @@ func Main(versionInfo string) { logc.Println(ctx, "deleted") case *freezeDomain != "" || *unfreezeDomain != "": - var domain string + var domain names.Domain var freeze bool if *freezeDomain != "" { - domain = *freezeDomain + domain = domainArg(*freezeDomain) freeze = true } else { - domain = *unfreezeDomain + domain = domainArg(*unfreezeDomain) freeze = false } @@ -562,14 +573,15 @@ func Main(versionInfo string) { } case *purgeDomain != "": + domain := domainArg(*freezeDomain) + purgeCount := 0 for metadata, err := range backend.EnumerateManifests(ctx) { if err != nil { logc.Fatalln(ctx, err) } - domain, _, _ := strings.Cut(metadata.Name, "/") - if domain == *purgeDomain { + if metadata.Name.Domain == domain { err := backend.DeleteManifest(ctx, metadata.Name, ModifyManifestOptions{}) if err != nil { logc.Fatalln(ctx, err) @@ -578,7 +590,7 @@ func Main(versionInfo string) { } } - logc.Printf(ctx, "admin: purged %s (%d sites)\n", *purgeDomain, purgeCount) + logc.Printf(ctx, "admin: purged %s (%d sites)\n", domain, purgeCount) case *auditLog: records := []*AuditRecord{} @@ -663,12 +675,12 @@ func Main(versionInfo string) { logc.Fatalln(ctx, "no manifest in audit record") } - webRoot := path.Join(record.GetDomain(), record.GetProject()) + site := Must(names.ParseSite(fmt.Sprintf("%s/%s", record.GetDomain(), record.GetProject()))) err = backend.StageManifest(ctx, record.GetManifest()) if err != nil { logc.Fatalln(ctx, err) } - err = backend.CommitManifest(ctx, webRoot, record.GetManifest(), ModifyManifestOptions{}) + err = backend.CommitManifest(ctx, site, record.GetManifest(), ModifyManifestOptions{}) if err != nil { logc.Fatalln(ctx, err) } @@ -681,7 +693,7 @@ func Main(versionInfo string) { } if project != "*" { - if _, err := names.ParseProjectName(project); err != nil { + if _, err := names.ParseProject(project); err != nil { logc.Fatalf(ctx, "audit detach: project name: %v\n", err) } } diff --git a/src/manifest.go b/src/manifest.go index 1464bdc..2f4980a 100644 --- a/src/manifest.go +++ b/src/manifest.go @@ -17,6 +17,7 @@ import ( "sync" "time" + "codeberg.org/git-pages/git-pages/src/names" "github.com/c2h5oh/datasize" "github.com/go-git/go-git/v6/plumbing" format "github.com/go-git/go-git/v6/plumbing/format/config" @@ -338,11 +339,9 @@ func CompressFiles(ctx context.Context, manifest *Manifest) { // Apply post-processing steps to the manifest. // At the moment, there isn't a good way to report errors except to log them on the terminal. // (Perhaps in the future they could be exposed at `.git-pages/status.txt`?) -func PrepareManifest(ctx context.Context, webRoot string, manifest *Manifest) error { - _, project, _ := strings.Cut(webRoot, "/") - +func PrepareManifest(ctx context.Context, site names.Site, manifest *Manifest) error { // Parse Netlify-style `_redirects`. - if err := ProcessRedirectsFile(ctx, project, manifest); err != nil { + if err := ProcessRedirectsFile(ctx, site.Project, manifest); err != nil { logc.Printf(ctx, "redirects err: %s\n", err) } else if len(manifest.Redirects) > 0 { logc.Printf(ctx, "redirects ok: %d rules\n", len(manifest.Redirects)) @@ -379,9 +378,9 @@ var putBlobSemaphore = make(chan struct{}) // Uploads inline file data over certain size to the storage backend. Returns a copy of // the manifest updated to refer to an external content-addressable store. func StoreManifest( - ctx context.Context, name string, manifest *Manifest, opts ModifyManifestOptions, + ctx context.Context, site names.Site, manifest *Manifest, opts ModifyManifestOptions, ) (*Manifest, error) { - span, ctx := ObserveFunction(ctx, "StoreManifest", "manifest.name", name) + span, ctx := ObserveFunction(ctx, "StoreManifest", "manifest.name", site) defer span.Finish() extManifest := &Manifest{} @@ -466,7 +465,7 @@ func StoreManifest( return nil, err // currently ignores all but 1st error } - if err := backend.CommitManifest(ctx, name, extManifest, opts); err != nil { + if err := backend.CommitManifest(ctx, site, extManifest, opts); err != nil { if errors.Is(err, ErrDomainFrozen) { return nil, err } else { diff --git a/src/migrate.go b/src/migrate.go index c129d30..c2d06b7 100644 --- a/src/migrate.go +++ b/src/migrate.go @@ -4,7 +4,8 @@ import ( "context" "fmt" "slices" - "strings" + + "codeberg.org/git-pages/git-pages/src/names" ) func RunMigration(ctx context.Context, name string) error { @@ -22,19 +23,19 @@ func createDomainMarkers(ctx context.Context) error { return nil } - var manifests []string + var sites []names.Site for metadata, err := range backend.EnumerateManifests(ctx) { if err != nil { return fmt.Errorf("enum manifests: %w", err) } - manifests = append(manifests, metadata.Name) + sites = append(sites, metadata.Name) } - slices.Sort(manifests) - var domains []string - for _, manifest := range manifests { - domain, _, _ := strings.Cut(manifest, "/") - if len(domains) == 0 || domains[len(domains)-1] != domain { - domains = append(domains, domain) + slices.SortFunc(sites, names.CompareSite) + + var domains []names.Domain + for _, site := range sites { + if len(domains) == 0 || domains[len(domains)-1] != site.Domain { + domains = append(domains, site.Domain) } } for idx, domain := range domains { @@ -46,6 +47,7 @@ func createDomainMarkers(ctx context.Context) error { if err := backend.EnableFeature(ctx, FeatureCheckDomainMarker); err != nil { return err } + logc.Printf(ctx, "created markers for %d domains", len(domains)) return nil } diff --git a/src/names/domain.go b/src/names/domain.go new file mode 100644 index 0000000..c8855e0 --- /dev/null +++ b/src/names/domain.go @@ -0,0 +1,97 @@ +package names + +import ( + "fmt" + "slices" + "strings" + + "golang.org/x/net/idna" + "golang.org/x/net/publicsuffix" +) + +var ErrInvalidDomainName = NameError{"invalid domain name"} + +// Domain of a git-pages site. +type Domain struct { + name string +} + +// IsValid checks whether the domain name is valid. The zero value is the only invalid value. +func (d Domain) IsValid() bool { + return d.name != "" +} + +// MustValid ensures the domain name is valid by panicking if it is not. +func (d Domain) MustValid() Domain { + if !d.IsValid() { + panic("nil domain name") + } + return d +} + +// String returns the normalized domain name. +func (d Domain) String() string { + return d.MustValid().name +} + +var idnaProfile = idna.New(idna.MapForLookup(), idna.BidiRule()) + +type ParseDomainMode int + +const ( + // RFC-compliant parsing mode. + ParseDomainStrict ParseDomainMode = iota + // Parsing mode that accepts `_` symbols in the middle and at the end of a label, and behaves + // like ParseDomainNormal otherwise. + ParseDomainRelaxed +) + +// ParseDomain parses and normalizes a domain name according to the requirements set out in +// Section 5 of RFC 5891, as well as the Bidi Rule. +// +// The mappings include normalization and mapping case, width and other compatibility mappings. +func ParseDomain(name string, mode ParseDomainMode) (Domain, error) { + // this also rejects invalid characters and labels + name, err := idnaProfile.ToASCII(name) + if err != nil { + if mode == ParseDomainRelaxed { + // unfortunately, the go IDNA library has some significant issues around its + // Unicode TR46 implementation: https://github.com/golang/go/issues/76804 + // we would like to allow *just* the _ here, but adding `idna.StrictDomainName(false)` + // would also accept domains like `*.foo.bar` which should clearly be disallowed. + // as a workaround, accept a domain name if it is valid with all `_` characters + // replaced with an alphanumeric character (we use `a`); this allows e.g. `foo_bar.xxx` + // and `foo__bar.xxx`, as well as `_foo.xxx` and `foo_.xxx`. labels starting with + // an underscore are explicitly rejected below. + _, err = idnaProfile.ToASCII(strings.ReplaceAll(name, "_", "a")) + } + if err != nil { + return Domain{}, fmt.Errorf("%w: malformed host name %q", ErrInvalidDomainName, name) + } + } + if strings.HasPrefix(name, ".") || strings.HasPrefix(name, "_") { + return Domain{}, fmt.Errorf("%w: reserved host name %q", ErrInvalidDomainName, name) + } + + return Domain{strings.TrimSuffix(name, ".")}, nil +} + +func (d Domain) EffectiveTLDPlusOne(extraSuffixes []Domain) Domain { + domainParts := strings.Split(d.name, ".") + for _, suffix := range extraSuffixes { + suffixParts := strings.Split(suffix.name, ".") + if len(domainParts) < len(suffixParts)+1 { + continue + } + if slices.Equal(domainParts[len(domainParts)-len(suffixParts):], suffixParts) { + return Domain{strings.Join(domainParts[len(domainParts)-len(suffixParts)-1:], ".")} + } + } + + eTLDp1, err := publicsuffix.EffectiveTLDPlusOne(d.String()) + if err != nil { + panic(fmt.Sprintf("internal error: %s", err)) + } + + return Domain{eTLDp1} +} diff --git a/src/names/domain_test.go b/src/names/domain_test.go new file mode 100644 index 0000000..9c12508 --- /dev/null +++ b/src/names/domain_test.go @@ -0,0 +1,70 @@ +package names + +import ( + "errors" + "fmt" + "testing" +) + +func checkDomainParseOk(t *testing.T, mode ParseDomainMode, value string, expect string) { + t.Helper() + if actual, err := ParseDomain(value, mode); err != nil { + t.Error(err) + } else { + if actual.name != expect { + t.Errorf("unexpected value: %v != %v", actual, expect) + } + } +} + +func checkDomainParseErr(t *testing.T, mode ParseDomainMode, value string, expect error) { + t.Helper() + _, err := ParseDomain(value, mode) + if !errors.Is(err, expect) { + t.Errorf("unexpected error: %v, expected %v", err, expect) + } +} + +func TestDomainParse(t *testing.T) { + for _, mode := range []ParseDomainMode{ParseDomainStrict, ParseDomainRelaxed} { + checkDomainParseOk(t, mode, "foo.bar", "foo.bar") + checkDomainParseOk(t, mode, "foo.bar.", "foo.bar") + checkDomainParseOk(t, mode, "foo-baz.bar", "foo-baz.bar") + checkDomainParseOk(t, mode, "foo--baz.bar", "foo--baz.bar") + checkDomainParseOk(t, mode, "ß.bar", "xn--zca.bar") + checkDomainParseOk(t, mode, "xn--zca.bar", "xn--zca.bar") + + checkDomainParseErr(t, mode, ".foo.bar", ErrInvalidDomainName) + checkDomainParseErr(t, mode, "..foo.bar", ErrInvalidDomainName) + checkDomainParseErr(t, mode, "foo-.bar", ErrInvalidDomainName) + checkDomainParseErr(t, mode, "-foo.bar", ErrInvalidDomainName) + checkDomainParseErr(t, mode, "_foo.bar", ErrInvalidDomainName) + checkDomainParseErr(t, mode, "*.foo.bar", ErrInvalidDomainName) + } + + for _, label := range []string{"foo_", "foo_baz", "foo__baz"} { + domain := fmt.Sprintf("%s.bar", label) + checkDomainParseErr(t, ParseDomainStrict, domain, ErrInvalidDomainName) + checkDomainParseOk(t, ParseDomainRelaxed, domain, domain) + } +} + +func TestDomainProperties(t *testing.T) { + checkEq(t, Domain{}.IsValid(), false) + checkEq(t, Domain{"example.org"}.IsValid(), true) + + checkPanic(t, func() Domain { return Domain{}.MustValid() }) + checkEq(t, Domain{"example.org"}.MustValid(), Domain{"example.org"}) + + checkPanic(t, func() string { return Domain{}.String() }) + checkEq(t, Domain{"example.org"}.String(), "example.org") + + checkEq(t, Domain{"example.org"}.EffectiveTLDPlusOne([]Domain{}).String(), + "example.org") + checkEq(t, Domain{"foo.example.org"}.EffectiveTLDPlusOne([]Domain{}).String(), + "example.org") + checkEq(t, Domain{"example.co.uk"}.EffectiveTLDPlusOne([]Domain{}).String(), + "example.co.uk") + checkEq(t, Domain{"foo.example.co.uk"}.EffectiveTLDPlusOne([]Domain{}).String(), + "example.co.uk") +} diff --git a/src/names/error.go b/src/names/error.go new file mode 100644 index 0000000..33e0d0a --- /dev/null +++ b/src/names/error.go @@ -0,0 +1,9 @@ +package names + +type NameError struct { + text string +} + +func (err NameError) Error() string { + return err.text +} diff --git a/src/names/project.go b/src/names/project.go index 39ab9b5..3ad93e7 100644 --- a/src/names/project.go +++ b/src/names/project.go @@ -1,17 +1,15 @@ package names import ( - "errors" "fmt" "net/url" "strconv" "strings" ) -var ErrInvalidProjectName = errors.New("invalid project name") -var ErrInvalidProjectURL = errors.New("invalid project URL") +var ErrInvalidProjectName = NameError{"invalid project name"} +var ErrInvalidProjectURL = NameError{"invalid project URL"} -const indexProject = ".index" const previewSeparator = "@" // Name of a git-pages project. The index project corresponds to the root URL of a domain, and @@ -22,6 +20,8 @@ type Project struct { name string } +var IndexProject = Project{".index"} + // IsValid checks whether the project name is valid. The zero value is the only invalid value. func (p Project) IsValid() bool { return p.name != "" @@ -42,7 +42,7 @@ func (p Project) String() string { // IsIndex checks whether this name corresponds to the index project. func (p Project) IsIndex() bool { - return p.String() == indexProject + return p.MustValid() == IndexProject } // GetPullRequest returns (baseName, prNumber, isPreview), where isPreview is true if this is @@ -89,11 +89,11 @@ func validateProjectName(name string) (proj Project, err error) { return } -// ParseProjectName parses the given string as a project name. The string must not be empty, and +// ParseProject parses the given string as a project name. The string must not be empty, and // ".index" represents the index project. -func ParseProjectName(name string) (Project, error) { - if name == indexProject { - return Project{indexProject}, nil +func ParseProject(name string) (Project, error) { + if name == IndexProject.name { + return IndexProject, nil } return validateProjectName(name) @@ -107,7 +107,7 @@ func ParseProjectURL(url *url.URL) (Project, error) { if strings.Contains(path, "/") { return Project{}, fmt.Errorf("%w: must contain zero or one path segments", ErrInvalidProjectURL) } else if path == "" { - return Project{indexProject}, nil + return IndexProject, nil } else if name, err := validateProjectName(path); err != nil { return Project{}, fmt.Errorf("%w: %w", ErrInvalidProjectURL, err) } else { diff --git a/src/names/project_test.go b/src/names/project_test.go index b318dba..ac309e0 100644 --- a/src/names/project_test.go +++ b/src/names/project_test.go @@ -1,67 +1,24 @@ package names import ( - "errors" "net/url" "testing" ) -func must[T any](value T, err error) T { - if err != nil { - panic(err) - } - return value -} - -func checkEq[T comparable](t *testing.T, value T, expect T) { - t.Helper() - if value != expect { - t.Errorf("unexpected: %v != %v", value, expect) - } -} - -func checkPanic[T any](t *testing.T, f func() T) { - defer func() { - if r := recover(); r == nil { - t.Errorf("function did not panic") - } - }() - - f() -} - -func checkOk[T any](t *testing.T, parse func(T) (Project, error), value T, expect Project) { - t.Helper() - actual, err := parse(value) - if err != nil { - t.Errorf("unexpected error: %v", err) - } else if actual != expect { - t.Errorf("unexpected result: %v, expected %v", actual, expect) - } -} - -func checkErr[T any](t *testing.T, parse func(T) (Project, error), value T, expect error) { - t.Helper() - _, err := parse(value) - if !errors.Is(err, expect) { - t.Errorf("unexpected error: %v, expected %v", err, expect) - } -} - func TestParseProjectName(t *testing.T) { - checkOk(t, ParseProjectName, ".index", Project{".index"}) - checkOk(t, ParseProjectName, "foo", Project{"foo"}) - checkOk(t, ParseProjectName, "foo@10", Project{"foo@10"}) + checkOk(t, ParseProject, ".index", Project{".index"}) + checkOk(t, ParseProject, "foo", Project{"foo"}) + checkOk(t, ParseProject, "foo@10", Project{"foo@10"}) - checkErr(t, ParseProjectName, ".foo", ErrInvalidProjectName) - checkErr(t, ParseProjectName, "meow/miau", ErrInvalidProjectName) - checkErr(t, ParseProjectName, "meow\\miau", ErrInvalidProjectName) - checkErr(t, ParseProjectName, "meow%miau", ErrInvalidProjectName) - checkErr(t, ParseProjectName, "meow*", ErrInvalidProjectName) - checkErr(t, ParseProjectName, "foo@10@11", ErrInvalidProjectName) - checkErr(t, ParseProjectName, "foo@awa", ErrInvalidProjectName) - checkErr(t, ParseProjectName, "foo@0", ErrInvalidProjectName) - checkErr(t, ParseProjectName, "foo@-10", ErrInvalidProjectName) + checkErr(t, ParseProject, ".foo", ErrInvalidProjectName) + checkErr(t, ParseProject, "meow/miau", ErrInvalidProjectName) + checkErr(t, ParseProject, "meow\\miau", ErrInvalidProjectName) + checkErr(t, ParseProject, "meow%miau", ErrInvalidProjectName) + checkErr(t, ParseProject, "meow*", ErrInvalidProjectName) + checkErr(t, ParseProject, "foo@10@11", ErrInvalidProjectName) + checkErr(t, ParseProject, "foo@awa", ErrInvalidProjectName) + checkErr(t, ParseProject, "foo@0", ErrInvalidProjectName) + checkErr(t, ParseProject, "foo@-10", ErrInvalidProjectName) } func TestParseProjectURL(t *testing.T) { diff --git a/src/names/site.go b/src/names/site.go new file mode 100644 index 0000000..b1864b1 --- /dev/null +++ b/src/names/site.go @@ -0,0 +1,60 @@ +package names + +import ( + "cmp" + "fmt" + "strings" +) + +var ErrInvalidSiteName = NameError{"invalid site name"} + +const siteSeparator = "/" + +// Name of a git-pages site. Consists of a Domain and a Project name. +type Site struct { + Domain Domain + Project Project +} + +func (s Site) IsValid() bool { + return s.Domain.IsValid() && s.Project.IsValid() +} + +// MustValid ensures the domain name is valid by panicking if it is not. +func (s Site) MustValid() Site { + if !s.IsValid() { + panic("nil site name") + } + return s +} + +// String returns the Domain and Project names separated by a "/". +func (s Site) String() string { + return fmt.Sprint(s.Domain.MustValid(), siteSeparator, s.Project.MustValid()) +} + +// CompareSite orders sites first by domain, then by project. +func CompareSite(a Site, b Site) int { + if n := cmp.Compare(a.Domain.name, b.Domain.name); n != 0 { + return n + } else { + return cmp.Compare(a.Project.name, b.Project.name) + } +} + +// ParseSite parses the format returned by String. +func ParseSite(value string) (s Site, err error) { + if domainPart, projectPart, ok := strings.Cut(value, siteSeparator); ok { + if s.Domain, err = ParseDomain(domainPart, ParseDomainRelaxed); err != nil { + s = Site{} + return + } + if s.Project, err = ParseProject(projectPart); err != nil { + s = Site{} + return + } + } else { + err = ErrInvalidSiteName + } + return +} diff --git a/src/names/site_test.go b/src/names/site_test.go new file mode 100644 index 0000000..2d7218a --- /dev/null +++ b/src/names/site_test.go @@ -0,0 +1,29 @@ +package names + +import ( + "testing" +) + +func TestParseSite(t *testing.T) { + checkOk(t, ParseSite, "example.org/foo", Site{Domain{"example.org"}, Project{"foo"}}) + checkOk(t, ParseSite, "example.org/.index", Site{Domain{"example.org"}, Project{".index"}}) + checkOk(t, ParseSite, "foo_bar.example.org/.index", Site{Domain{"foo_bar.example.org"}, Project{".index"}}) + + checkErr(t, ParseSite, "example.org", ErrInvalidSiteName) + checkErr(t, ParseSite, "example.org/a*", ErrInvalidProjectName) + checkErr(t, ParseSite, "_foo.example.org/.index", ErrInvalidDomainName) +} + +func TestSiteProperties(t *testing.T) { + checkEq(t, Site{}.IsValid(), false) + checkEq(t, Site{Domain{"example.org"}, Project{}}.IsValid(), false) + checkEq(t, Site{Domain{}, Project{"foo"}}.IsValid(), false) + checkEq(t, Site{Domain{"example.org"}, Project{"foo"}}.IsValid(), true) + + checkPanic(t, func() Site { return Site{}.MustValid() }) + checkEq(t, Site{Domain{"example.org"}, Project{"foo"}}.MustValid().Project, Project{"foo"}) + + checkPanic(t, func() string { return Site{}.String() }) + checkEq(t, Site{Domain{"example.org"}, Project{"foo"}}.String(), "example.org/foo") + checkEq(t, Site{Domain{"example.org"}, IndexProject}.String(), "example.org/.index") +} diff --git a/src/names/utils.go b/src/names/utils.go new file mode 100644 index 0000000..f033c0a --- /dev/null +++ b/src/names/utils.go @@ -0,0 +1,48 @@ +package names + +import ( + "errors" + "testing" +) + +func must[T any](value T, err error) T { + if err != nil { + panic(err) + } + return value +} + +func checkEq[T comparable](t *testing.T, value T, expect T) { + t.Helper() + if value != expect { + t.Errorf("unexpected: %v != %v", value, expect) + } +} + +func checkPanic[T any](t *testing.T, f func() T) { + defer func() { + if r := recover(); r == nil { + t.Errorf("function did not panic") + } + }() + + f() +} + +func checkOk[T any, U comparable](t *testing.T, parse func(T) (U, error), value T, expect U) { + t.Helper() + actual, err := parse(value) + if err != nil { + t.Errorf("unexpected error: %v", err) + } else if actual != expect { + t.Errorf("unexpected result: %v, expected %v", actual, expect) + } +} + +func checkErr[T any, U any](t *testing.T, parse func(T) (U, error), value T, expect error) { + t.Helper() + _, err := parse(value) + if !errors.Is(err, expect) { + t.Errorf("unexpected error: %v, expected %v", err, expect) + } +} diff --git a/src/observe.go b/src/observe.go index 97ff495..323dec7 100644 --- a/src/observe.go +++ b/src/observe.go @@ -17,6 +17,7 @@ import ( slogmulti "github.com/samber/slog-multi" + "codeberg.org/git-pages/git-pages/src/names" syslog "codeberg.org/git-pages/go-slog-syslog" "github.com/prometheus/client_golang/prometheus" @@ -259,15 +260,15 @@ func (backend *observedBackend) EnumerateBlobs(ctx context.Context) iter.Seq2[Bl } func (backend *observedBackend) GetManifest( - ctx context.Context, name string, opts GetManifestOptions, + ctx context.Context, site names.Site, opts GetManifestOptions, ) ( manifest *Manifest, metadata ManifestMetadata, err error, ) { span, ctx := ObserveFunction(ctx, "GetManifest", - "manifest.name", name, + "manifest.name", site, "manifest.bypass_cache", opts.BypassCache, ) - if manifest, metadata, err = backend.inner.GetManifest(ctx, name, opts); err == nil { + if manifest, metadata, err = backend.inner.GetManifest(ctx, site, opts); err == nil { manifestsRetrievedCount.Inc() } span.Finish() @@ -285,23 +286,23 @@ func (backend *observedBackend) HasAtomicCAS(ctx context.Context) bool { return backend.inner.HasAtomicCAS(ctx) } -func (backend *observedBackend) CommitManifest(ctx context.Context, name string, manifest *Manifest, opts ModifyManifestOptions) (err error) { - span, ctx := ObserveFunction(ctx, "CommitManifest", "manifest.name", name) - err = backend.inner.CommitManifest(ctx, name, manifest, opts) +func (backend *observedBackend) CommitManifest(ctx context.Context, site names.Site, manifest *Manifest, opts ModifyManifestOptions) (err error) { + span, ctx := ObserveFunction(ctx, "CommitManifest", "manifest.name", site) + err = backend.inner.CommitManifest(ctx, site, manifest, opts) span.Finish() return } -func (backend *observedBackend) DeleteManifest(ctx context.Context, name string, opts ModifyManifestOptions) (err error) { - span, ctx := ObserveFunction(ctx, "DeleteManifest", "manifest.name", name) - err = backend.inner.DeleteManifest(ctx, name, opts) +func (backend *observedBackend) DeleteManifest(ctx context.Context, site names.Site, opts ModifyManifestOptions) (err error) { + span, ctx := ObserveFunction(ctx, "DeleteManifest", "manifest.name", site) + err = backend.inner.DeleteManifest(ctx, site, opts) span.Finish() return } -func (backend *observedBackend) ExpireManifest(ctx context.Context, name string) (err error) { - span, ctx := ObserveFunction(ctx, "ExpireManifest", "manifest.name", name) - err = backend.inner.ExpireManifest(ctx, name) +func (backend *observedBackend) ExpireManifest(ctx context.Context, site names.Site) (err error) { + span, ctx := ObserveFunction(ctx, "ExpireManifest", "manifest.name", site) + err = backend.inner.ExpireManifest(ctx, site) span.Finish() return } @@ -330,28 +331,28 @@ func (backend *observedBackend) GetAllManifests(ctx context.Context) iter.Seq2[t } } -func (backend *observedBackend) CheckDomain(ctx context.Context, domain string) (found bool, err error) { +func (backend *observedBackend) CheckDomain(ctx context.Context, domain names.Domain) (found bool, err error) { span, ctx := ObserveFunction(ctx, "CheckDomain", "domain.name", domain) found, err = backend.inner.CheckDomain(ctx, domain) span.Finish() return } -func (backend *observedBackend) CreateDomain(ctx context.Context, domain string) (err error) { +func (backend *observedBackend) CreateDomain(ctx context.Context, domain names.Domain) (err error) { span, ctx := ObserveFunction(ctx, "CreateDomain", "domain.name", domain) err = backend.inner.CreateDomain(ctx, domain) span.Finish() return } -func (backend *observedBackend) FreezeDomain(ctx context.Context, domain string) (err error) { +func (backend *observedBackend) FreezeDomain(ctx context.Context, domain names.Domain) (err error) { span, ctx := ObserveFunction(ctx, "FreezeDomain", "domain.name", domain) err = backend.inner.FreezeDomain(ctx, domain) span.Finish() return } -func (backend *observedBackend) UnfreezeDomain(ctx context.Context, domain string) (err error) { +func (backend *observedBackend) UnfreezeDomain(ctx context.Context, domain names.Domain) (err error) { span, ctx := ObserveFunction(ctx, "UnfreezeDomain", "domain.name", domain) err = backend.inner.UnfreezeDomain(ctx, domain) span.Finish() diff --git a/src/pages.go b/src/pages.go index 000021e..35498ae 100644 --- a/src/pages.go +++ b/src/pages.go @@ -9,6 +9,7 @@ import ( "fmt" "io" "maps" + "net" "net/http" "net/url" "os" @@ -77,26 +78,33 @@ func copyForgeAuthToPrincipal(principal *Principal, auth *Authorization) { } } -func normalizeHost(host string) string { - return strings.ToLower(host) -} - -func makeWebRoot(host string, projectName string) string { - return path.Join(normalizeHost(host), projectName) -} - -func getWebRoot(r *http.Request) (string, error) { - host, err := GetHost(r) +func GetDomain(r *http.Request) (domain names.Domain, err error) { + host, _, err := net.SplitHostPort(r.Host) if err != nil { - return "", err + host = r.Host } - projectName, err := names.ParseProjectURL(r.URL) - if err != nil { - return "", err + mode := names.ParseDomainStrict + if config.Feature("relaxed-idna") { + mode = names.ParseDomainRelaxed } - return makeWebRoot(host, projectName.String()), nil + return names.ParseDomain(host, mode) +} + +func GetSite(r *http.Request) (site names.Site, err error) { + domain, err := GetDomain(r) + if err != nil { + return + } + + project, err := names.ParseProjectURL(r.URL) + if err != nil { + return + } + + site = names.Site{Domain: domain, Project: project} + return } func writeRedirect(w http.ResponseWriter, code int, path string) { @@ -126,11 +134,10 @@ func getPage(w http.ResponseWriter, r *http.Request) error { bypassCache := cacheControl.NoCache || cacheControl.MaxAge == 0 - host, err := GetHost(r) + domain, err := GetDomain(r) if err != nil { return err } - host = normalizeHost(host) type indexManifestResult struct { manifest *Manifest @@ -139,14 +146,13 @@ func getPage(w http.ResponseWriter, r *http.Request) error { } indexManifestCh := make(chan indexManifestResult, 1) go func() { - webRoot := makeWebRoot(host, ".index") - if existenceCache.CheckSite(r.Context(), webRoot).IsImpossible() { + site := names.Site{Domain: domain, Project: names.IndexProject} + if existenceCache.CheckSite(r.Context(), site).IsImpossible() { close(indexManifestCh) return } manifest, metadata, err := backend.GetManifest( - r.Context(), webRoot, - GetManifestOptions{BypassCache: bypassCache}, + r.Context(), site, GetManifestOptions{BypassCache: bypassCache}, ) indexManifestCh <- (indexManifestResult{manifest, metadata, err}) }() @@ -155,24 +161,25 @@ func getPage(w http.ResponseWriter, r *http.Request) error { sitePath = strings.TrimPrefix(r.URL.Path, "/") projectPathPrefix := "/" // 2xx and 4xx redirect target URLs must start with this prefix if projectName, projectPath, hasProjectSlash := strings.Cut(sitePath, "/"); projectName != "" { - webRoot := makeWebRoot(host, projectName) - if _, err = names.ParseProjectName(projectName); err == nil && - existenceCache.CheckSite(r.Context(), webRoot).IsPossible() { - var projectManifest *Manifest - var projectMetadata ManifestMetadata - projectManifest, projectMetadata, err = backend.GetManifest( - r.Context(), webRoot, - GetManifestOptions{BypassCache: bypassCache}, - ) - if err == nil { - if !hasProjectSlash { - newURL := *r.URL - newURL.Path += "/" - writeRedirect(w, http.StatusFound, newURL.String()) - return nil + var project names.Project + if project, err = names.ParseProject(projectName); err == nil { + site := names.Site{Domain: domain, Project: project} + if existenceCache.CheckSite(r.Context(), site).IsPossible() { + var projectManifest *Manifest + var projectMetadata ManifestMetadata + projectManifest, projectMetadata, err = backend.GetManifest( + r.Context(), site, GetManifestOptions{BypassCache: bypassCache}, + ) + if err == nil { + if !hasProjectSlash { + newURL := *r.URL + newURL.Path += "/" + writeRedirect(w, http.StatusFound, newURL.String()) + return nil + } + sitePath, manifest, metadata = projectPath, projectManifest, projectMetadata + projectPathPrefix = fmt.Sprintf("/%s/", projectName) } - sitePath, manifest, metadata = projectPath, projectManifest, projectMetadata - projectPathPrefix = fmt.Sprintf("/%s/", projectName) } } } @@ -181,7 +188,7 @@ func getPage(w http.ResponseWriter, r *http.Request) error { manifest, metadata, err = result.manifest, result.metadata, result.err if manifest == nil && (err == nil || errors.Is(err, ErrObjectNotFound)) { if fallback != nil { - logc.Printf(r.Context(), "fallback: %s via %s", host, config.Fallback.ProxyTo) + logc.Printf(r.Context(), "fallback: %s via %s", domain, config.Fallback.ProxyTo) fallback.ServeHTTP(w, r) return nil } else { @@ -552,7 +559,7 @@ func putPage(w http.ResponseWriter, r *http.Request) error { } } - webRoot, err := getWebRoot(r) + site, err := GetSite(r) if err != nil { return err } @@ -596,7 +603,7 @@ func putPage(w http.ResponseWriter, r *http.Request) error { return nil } - result = UpdateFromRepository(ctx, webRoot, repoURL, branch, opts) + result = UpdateFromRepository(ctx, site, repoURL, branch, opts) default: auth, err := AuthorizeUpdateFromArchive(r) @@ -620,7 +627,7 @@ func putPage(w http.ResponseWriter, r *http.Request) error { // request body contains archive reader := http.MaxBytesReader(w, r.Body, int64(config.Limits.MaxSiteSize.Bytes())) - result = UpdateFromArchive(ctx, webRoot, repoURL, contentType, reader, opts) + result = UpdateFromArchive(ctx, site, repoURL, contentType, reader, opts) } return reportUpdateResult(w, r, result) @@ -636,7 +643,7 @@ func patchPage(w http.ResponseWriter, r *http.Request) error { } } - webRoot, err := getWebRoot(r) + site, err := GetSite(r) if err != nil { return err } @@ -694,7 +701,7 @@ func patchPage(w http.ResponseWriter, r *http.Request) error { contentType := getMediaType(r.Header.Get("Content-Type")) reader := http.MaxBytesReader(w, r.Body, int64(config.Limits.MaxSiteSize.Bytes())) - result := PartialUpdateFromArchive(ctx, webRoot, contentType, reader, parents, opts) + result := PartialUpdateFromArchive(ctx, site, contentType, reader, parents, opts) return reportUpdateResult(w, r, result) } @@ -777,7 +784,7 @@ func reportUpdateResult(w http.ResponseWriter, r *http.Request, result UpdateRes } func deletePage(w http.ResponseWriter, r *http.Request) error { - webRoot, err := getWebRoot(r) + site, err := GetSite(r) if err != nil { return err } @@ -794,7 +801,7 @@ func deletePage(w http.ResponseWriter, r *http.Request) error { return nil } - if err = backend.DeleteManifest(r.Context(), webRoot, ModifyManifestOptions{}); err != nil { + if err = backend.DeleteManifest(r.Context(), site, ModifyManifestOptions{}); err != nil { w.WriteHeader(http.StatusInternalServerError) fmt.Fprintln(w, err) } else { @@ -811,7 +818,7 @@ func postPage(w http.ResponseWriter, r *http.Request) error { requestTimeout := 3 * time.Second requestTimer := time.NewTimer(requestTimeout) - webRoot, err := getWebRoot(r) + site, err := GetSite(r) if err != nil { return err } @@ -898,7 +905,7 @@ func postPage(w http.ResponseWriter, r *http.Request) error { principal := GetPrincipal(r.Context()) principal.RepoUrl = &repoURL - result := UpdateFromRepository(ctx, webRoot, repoURL, auth.branch, UpdateOptions{}) + result := UpdateFromRepository(ctx, site, repoURL, auth.branch, UpdateOptions{}) resultChan <- result observeSiteUpdate("webhook", &result) }(context.WithoutCancel(r.Context())) @@ -993,7 +1000,7 @@ func ServePages(w http.ResponseWriter, r *http.Request) { err = fmt.Errorf("method %s not allowed", r.Method) } if err != nil { - if errors.Is(err, names.ErrInvalidProjectURL) { + if _, ok := errors.AsType[names.NameError](err); ok { http.Error(w, prettyErrMsg(err), http.StatusBadRequest) } var authErr AuthError diff --git a/src/pages_test.go b/src/pages_test.go deleted file mode 100644 index c9875f7..0000000 --- a/src/pages_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package git_pages - -import ( - "net/http" - "strings" - "testing" -) - -func checkHost(t *testing.T, host string, expectOk string, expectErr string) { - host, err := GetHost(&http.Request{Host: host}) - if expectErr != "" { - if err == nil || !strings.HasPrefix(err.Error(), expectErr) { - t.Errorf("%s: expect err %s, got err %s", host, expectErr, err) - } - } - if expectOk != "" { - if err != nil { - t.Errorf("%s: expect ok %s, got err %s", host, expectOk, err) - } else if host != expectOk { - t.Errorf("%s: expect ok %s, got ok %s", host, expectOk, host) - } - } -} - -func TestHelloName(t *testing.T) { - config = &Config{Features: []string{}} - - checkHost(t, "foo.bar", "foo.bar", "") - checkHost(t, "foo-baz.bar", "foo-baz.bar", "") - checkHost(t, "foo--baz.bar", "foo--baz.bar", "") - checkHost(t, "foo.bar.", "foo.bar", "") - checkHost(t, ".foo.bar", "", "reserved host name") - checkHost(t, "..foo.bar", "", "reserved host name") - - checkHost(t, "ß.bar", "xn--zca.bar", "") - checkHost(t, "xn--zca.bar", "xn--zca.bar", "") - - checkHost(t, "foo-.bar", "", "malformed host name") - checkHost(t, "-foo.bar", "", "malformed host name") - checkHost(t, "foo_.bar", "", "malformed host name") - checkHost(t, "_foo.bar", "", "malformed host name") - checkHost(t, "foo_baz.bar", "", "malformed host name") - checkHost(t, "foo__baz.bar", "", "malformed host name") - checkHost(t, "*.foo.bar", "", "malformed host name") - - config = &Config{Features: []string{"relaxed-idna"}} - - checkHost(t, "foo-.bar", "", "malformed host name") - checkHost(t, "-foo.bar", "", "malformed host name") - checkHost(t, "foo_.bar", "foo_.bar", "") - checkHost(t, "_foo.bar", "", "reserved host name") - checkHost(t, "foo_baz.bar", "foo_baz.bar", "") - checkHost(t, "foo__baz.bar", "foo__baz.bar", "") - checkHost(t, "*.foo.bar", "", "malformed host name") -} diff --git a/src/proxy.go b/src/proxy.go index f7e7e8f..42da65f 100644 --- a/src/proxy.go +++ b/src/proxy.go @@ -9,15 +9,19 @@ import ( "net" "net/http" "slices" - "strings" + + "codeberg.org/git-pages/git-pages/src/names" ) func ServeProxy(w http.ResponseWriter, r *http.Request) { - domain := r.URL.Query().Get("domain") - if domain == "" { + domainParam := r.URL.Query().Get("domain") + if domainParam == "" { proxyListDomains(w, r) - } else { + } else if domain, err := names.ParseDomain(domainParam, names.ParseDomainRelaxed); err == nil { proxyCheckDomain(w, r, domain) + } else { + w.WriteHeader(http.StatusBadRequest) + fmt.Fprintln(w, err) } } @@ -28,28 +32,26 @@ func proxyListDomains(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusInternalServerError) fmt.Fprintf(w, "%s", err) } - domain, _, _ := strings.Cut(metadata.Name, "/") - domainSet[domain] = Unit + domainSet[metadata.Name.Domain.String()] = Unit } w.Header().Add("Content-Type", "application/json") json.NewEncoder(w).Encode(slices.Collect(maps.Keys(domainSet))) } -func proxyCheckDomain(w http.ResponseWriter, r *http.Request, domain string) { +func proxyCheckDomain(w http.ResponseWriter, r *http.Request, domain names.Domain) { + var err error + // Save the backend some effort from queries that are essentially guaranteed to fail. // While TLS certificates may be provisionsed for IP addresses under special circumstances[^1], // this isn't really what git-pages is designed for, and object store accesses can cost money. // [^1]: https://letsencrypt.org/2025/07/01/issuing-our-first-ip-address-certificate - if ip := net.ParseIP(domain); ip != nil { + if ip := net.ParseIP(domain.String()); ip != nil { logc.Println(r.Context(), "proxy:", domain, 404, "(bare IP)") w.WriteHeader(http.StatusNotFound) return } - var err error - domain = strings.ToLower(domain) - // Run a cheap check as to whether we might be serving the domain. found := existenceCache.CheckDomain(r.Context(), domain).IsPossible() @@ -65,7 +67,7 @@ func proxyCheckDomain(w http.ResponseWriter, r *http.Request, domain string) { // Pages v2, which would under some circumstances return certificates with subjectAltName // not valid for the SNI. Go's TLS stack makes `tls.Dial` return an error for these, // thankfully making it unnecessary to examine X.509 certificates manually here.) - found, err = tryDialWithSNI(r.Context(), domain) + found, err = tryDialWithSNI(r.Context(), domain.String()) if err != nil { logc.Printf(r.Context(), "proxy err: check SNI: %s\n", err) } diff --git a/src/redirects.go b/src/redirects.go index be48f5e..abcc2a7 100644 --- a/src/redirects.go +++ b/src/redirects.go @@ -9,6 +9,7 @@ import ( "slices" "strings" + "codeberg.org/git-pages/git-pages/src/names" "codeberg.org/git-pages/go-redirects" "google.golang.org/protobuf/proto" ) @@ -43,7 +44,7 @@ func Is3xxHTTPStatus(status int) bool { return status >= 300 && status <= 399 } -func validateRedirectRule(project string, rule *redirects.Rule) error { +func validateRedirectRule(project names.Project, rule *redirects.Rule) error { if len(rule.Params) > 0 { return fmt.Errorf("rules with parameters are not supported") } @@ -81,7 +82,7 @@ func validateRedirectRule(project string, rule *redirects.Rule) error { // (which was the old, incorrect interpretation). This broke every SPA on git-pages that was // deployed to a non-index project. Unfortunately, we can't do much more than diagnosing // the problematic pattern at deploy time. - if rule.Status == http.StatusOK && project != ".index" && toURL.Host == "" && + if rule.Status == http.StatusOK && !project.IsIndex() && toURL.Host == "" && !strings.HasPrefix(toURL.Path, fmt.Sprintf("/%s/", project)) { return fmt.Errorf("'to' URL must start with '/%s/' for a 200 status rule", project) } @@ -89,7 +90,7 @@ func validateRedirectRule(project string, rule *redirects.Rule) error { } // Parses redirects file and injects rules into the manifest. -func ProcessRedirectsFile(ctx context.Context, project string, manifest *Manifest) error { +func ProcessRedirectsFile(ctx context.Context, project names.Project, manifest *Manifest) error { redirectsEntry := manifest.Contents[RedirectsFileName] delete(manifest.Contents, RedirectsFileName) if redirectsEntry == nil { diff --git a/src/scrub.go b/src/scrub.go index 30ac526..6e1294b 100644 --- a/src/scrub.go +++ b/src/scrub.go @@ -10,17 +10,18 @@ import ( "strings" "time" + "codeberg.org/git-pages/git-pages/src/names" "google.golang.org/protobuf/proto" timestamppb "google.golang.org/protobuf/types/known/timestamppb" ) -func createPlaceholderManifest(ctx context.Context, name string) *Manifest { +func createPlaceholderManifest(ctx context.Context, site names.Site) *Manifest { manifest := NewManifest() manifest.Corrupted = proto.Bool(true) AddFile(manifest, "index.html", []byte(""+ "

Data corruption

"+ - "

The site "+name+" has suffered data corruption and must be "+ + "

The site "+site.String()+" has suffered data corruption and must be "+ "re-uploaded. Contact the server administrator for more information.

"), ) manifest.Redirects = []*RedirectRule{ diff --git a/src/update.go b/src/update.go index 9024e7c..6b192ed 100644 --- a/src/update.go +++ b/src/update.go @@ -5,9 +5,9 @@ import ( "errors" "fmt" "io" - "strings" "time" + "codeberg.org/git-pages/git-pages/src/names" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -52,7 +52,7 @@ type UpdateResult struct { var errExpireExistingSite = fmt.Errorf("cannot expire an existing site") func Update( - ctx context.Context, webRoot string, oldManifest, newManifest *Manifest, + ctx context.Context, site names.Site, oldManifest, newManifest *Manifest, opts ModifyManifestOptions, ) UpdateResult { var err error @@ -62,7 +62,7 @@ func Update( if oldManifest != nil && oldManifest.ExpiresAt == nil && newManifest.ExpiresAt != nil { err = errExpireExistingSite } else if IsManifestEmpty(newManifest) { - storedManifest, err = newManifest, backend.DeleteManifest(ctx, webRoot, opts) + storedManifest, err = newManifest, backend.DeleteManifest(ctx, site, opts) if err == nil { if oldManifest == nil { outcome = UpdateNoChange @@ -70,12 +70,11 @@ func Update( outcome = UpdateDeleted } } - } else if err = PrepareManifest(ctx, webRoot, newManifest); err == nil { - storedManifest, err = StoreManifest(ctx, webRoot, newManifest, opts) + } else if err = PrepareManifest(ctx, site, newManifest); err == nil { + storedManifest, err = StoreManifest(ctx, site, newManifest, opts) if err == nil { - domain, _, _ := strings.Cut(webRoot, "/") - err = backend.CreateDomain(ctx, domain) - existenceCache.AddSite(ctx, webRoot) + err = backend.CreateDomain(ctx, site.Domain) + existenceCache.AddSite(ctx, site) } if err == nil { if oldManifest == nil { @@ -101,15 +100,15 @@ func Update( status = "unchanged" } if newManifest.ExpiresAt != nil { - logc.Printf(ctx, "expire %s: at %s", webRoot, newManifest.ExpiresAt.AsTime()) + logc.Printf(ctx, "expire %s: at %s", site, newManifest.ExpiresAt.AsTime()) } if storedManifest.Commit != nil { - logc.Printf(ctx, "update %s ok: %s %s", webRoot, *storedManifest.Commit, status) + logc.Printf(ctx, "update %s ok: %s %s", site, *storedManifest.Commit, status) } else { - logc.Printf(ctx, "update %s ok: %s", webRoot, status) + logc.Printf(ctx, "update %s ok: %s", site, status) } } else { - logc.Printf(ctx, "update %s err: %s", webRoot, err) + logc.Printf(ctx, "update %s err: %s", site, err) } return UpdateResult{outcome, storedManifest, err} @@ -117,7 +116,7 @@ func Update( func UpdateFromRepository( ctx context.Context, - webRoot string, + site names.Site, repoURL string, branch string, opts UpdateOptions, @@ -126,11 +125,11 @@ func UpdateFromRepository( defer span.Finish() defer observeUpdateResult(result) - logc.Printf(ctx, "update %s: %s %s\n", webRoot, repoURL, branch) + logc.Printf(ctx, "update %s: %s %s\n", site, repoURL, branch) - oldManifest, _, err := backend.GetManifest(ctx, webRoot, GetManifestOptions{}) + oldManifest, _, err := backend.GetManifest(ctx, site, GetManifestOptions{}) if err != nil && !errors.Is(err, ErrObjectNotFound) { - logc.Printf(ctx, "update %s err: %s", webRoot, err) + logc.Printf(ctx, "update %s err: %s", site, err) result = UpdateResult{UpdateError, nil, err} return } @@ -142,7 +141,7 @@ func UpdateFromRepository( result = UpdateResult{UpdateError, nil, err} } else { opts.Apply(newManifest) - result = Update(ctx, webRoot, oldManifest, newManifest, ModifyManifestOptions{}) + result = Update(ctx, site, oldManifest, newManifest, ModifyManifestOptions{}) } return @@ -152,7 +151,7 @@ var errArchiveFormat = errors.New("unsupported archive format") func UpdateFromArchive( ctx context.Context, - webRoot string, + site names.Site, repoURL string, contentType string, reader io.Reader, @@ -163,9 +162,9 @@ func UpdateFromArchive( defer span.Finish() defer observeUpdateResult(result) - oldManifest, _, err := backend.GetManifest(ctx, webRoot, GetManifestOptions{}) + oldManifest, _, err := backend.GetManifest(ctx, site, GetManifestOptions{}) if err != nil && !errors.Is(err, ErrObjectNotFound) { - logc.Printf(ctx, "update %s err: %s", webRoot, err) + logc.Printf(ctx, "update %s err: %s", site, err) result = UpdateResult{UpdateError, nil, err} return } @@ -177,37 +176,37 @@ func UpdateFromArchive( var newManifest *Manifest switch contentType { case "application/x-tar": - logc.Printf(ctx, "update %s: (tar)", webRoot) + logc.Printf(ctx, "update %s: (tar)", site) newManifest, err = extractTar(ctx, reader) // yellow? case "application/x-tar+gzip": - logc.Printf(ctx, "update %s: (tar.gz)", webRoot) + logc.Printf(ctx, "update %s: (tar.gz)", site) newManifest, err = ExtractGzip(ctx, reader, extractTar) // definitely yellow. case "application/x-tar+zstd": - logc.Printf(ctx, "update %s: (tar.zst)", webRoot) + logc.Printf(ctx, "update %s: (tar.zst)", site) newManifest, err = ExtractZstd(ctx, reader, extractTar) case "application/zip": - logc.Printf(ctx, "update %s: (zip)", webRoot) + logc.Printf(ctx, "update %s: (zip)", site) newManifest, err = ExtractZip(ctx, reader, oldManifest) default: err = errArchiveFormat } if err != nil { - logc.Printf(ctx, "update %s err: %s", webRoot, err) + logc.Printf(ctx, "update %s err: %s", site, err) result = UpdateResult{UpdateError, nil, err} } else { if repoURL != "" { newManifest.RepoUrl = &repoURL } opts.Apply(newManifest) - result = Update(ctx, webRoot, oldManifest, newManifest, ModifyManifestOptions{}) + result = Update(ctx, site, oldManifest, newManifest, ModifyManifestOptions{}) } return } func PartialUpdateFromArchive( ctx context.Context, - webRoot string, + site names.Site, contentType string, reader io.Reader, parents CreateParentsMode, @@ -220,10 +219,10 @@ func PartialUpdateFromArchive( // Here the old manifest is used both as a substrate to which a patch is applied, as well // as a "load linked" operation for a future "store conditional" update which, taken together, // create an atomic compare-and-swap operation. - oldManifest, oldMetadata, err := backend.GetManifest(ctx, webRoot, + oldManifest, oldMetadata, err := backend.GetManifest(ctx, site, GetManifestOptions{BypassCache: true}) if err != nil { - logc.Printf(ctx, "patch %s err: %s", webRoot, err) + logc.Printf(ctx, "patch %s err: %s", site, err) result = UpdateResult{UpdateError, nil, err} return } @@ -246,24 +245,24 @@ func PartialUpdateFromArchive( var newManifest *Manifest switch contentType { case "application/x-tar": - logc.Printf(ctx, "patch %s: (tar)", webRoot) + logc.Printf(ctx, "patch %s: (tar)", site) newManifest, err = applyTarPatch(ctx, reader) case "application/x-tar+gzip": - logc.Printf(ctx, "patch %s: (tar.gz)", webRoot) + logc.Printf(ctx, "patch %s: (tar.gz)", site) newManifest, err = ExtractGzip(ctx, reader, applyTarPatch) case "application/x-tar+zstd": - logc.Printf(ctx, "patch %s: (tar.zst)", webRoot) + logc.Printf(ctx, "patch %s: (tar.zst)", site) newManifest, err = ExtractZstd(ctx, reader, applyTarPatch) default: err = errArchiveFormat } if err != nil { - logc.Printf(ctx, "patch %s err: %s", webRoot, err) + logc.Printf(ctx, "patch %s err: %s", site, err) result = UpdateResult{UpdateError, nil, err} } else { opts.Apply(newManifest) - result = Update(ctx, webRoot, oldManifest, newManifest, + result = Update(ctx, site, oldManifest, newManifest, ModifyManifestOptions{ IfUnmodifiedSince: oldMetadata.LastModified, IfMatch: oldMetadata.ETag, diff --git a/src/util.go b/src/util.go index e89e2d2..ac8af5a 100644 --- a/src/util.go +++ b/src/util.go @@ -19,6 +19,13 @@ func (t tuple[A, B]) Splat() (A, B) { return t.A, t.B } +func Must[T any](value T, err error) T { + if err != nil { + panic(err) + } + return value +} + type BoundedReader struct { inner io.Reader fuel int64 diff --git a/src/wildcard.go b/src/wildcard.go index 626c115..a6369c6 100644 --- a/src/wildcard.go +++ b/src/wildcard.go @@ -5,6 +5,7 @@ import ( "slices" "strings" + "codeberg.org/git-pages/git-pages/src/names" "github.com/valyala/fasttemplate" ) @@ -34,7 +35,7 @@ const ( // Returns `subdomain, found` where if `found == true`, `subdomain` contains the part of `host` // corresponding to the * in the domain pattern. -func (pattern *WildcardPattern) Matches(host string, kind WildcardDomainKind) (string, bool) { +func (pattern *WildcardPattern) Matches(domain names.Domain, kind WildcardDomainKind) (string, bool) { var suffixParts []string switch kind { case WildcardDomainPrimary: @@ -42,9 +43,9 @@ func (pattern *WildcardPattern) Matches(host string, kind WildcardDomainKind) (s case WildcardDomainPreview: suffixParts = pattern.PreviewDomain case WildcardDomainAny: - if userName, found := pattern.Matches(host, WildcardDomainPrimary); found { + if userName, found := pattern.Matches(domain, WildcardDomainPrimary); found { return userName, found - } else if userName, found := pattern.Matches(host, WildcardDomainPreview); found { + } else if userName, found := pattern.Matches(domain, WildcardDomainPreview); found { return userName, found } return "", false @@ -52,7 +53,7 @@ func (pattern *WildcardPattern) Matches(host string, kind WildcardDomainKind) (s panic("invalid wildcard domain kind") } - hostParts := strings.Split(host, ".") + hostParts := strings.Split(domain.String(), ".") if len(suffixParts) == 0 { return "", false } else if len(hostParts) != len(suffixParts)+1 {