Refactor handling of domain name and site name.

Define a proper abstraction for domain names and site names, and use i
elsewhere in he codebase.

This still leaves domain patterns as an abstraction for future work.
The existing implementation is particularly awkward now.
This commit is contained in:
Catherine
2026-10-21 18:32:26 +00:00
parent e4e3a43825
commit 2557ec750f
29 changed files with 681 additions and 507 deletions
+19 -8
View File
@@ -4,6 +4,8 @@ import (
"context"
"fmt"
"strings"
"codeberg.org/git-pages/git-pages/src/names"
)
type StorageSize struct {
@@ -26,11 +28,21 @@ func AnalyzeStorage(ctx context.Context) ([]*StorageSize, error) {
auditBlobs map[string]int64
}
thickStats := map[string]*storageData{}
thickStats := map[names.Domain]*storageData{}
thinStats := []*StorageSize{}
getStats := func(domain string) *storageData {
domain = EffectiveTLDPlusOne(domain)
extraSuffixes := []names.Domain{}
for _, wildcard := range wildcards {
extraSuffixes = append(extraSuffixes,
Must(names.ParseDomain(strings.Join(wildcard.Domain, "."), names.ParseDomainRelaxed)))
if len(wildcard.PreviewDomain) > 0 {
extraSuffixes = append(extraSuffixes,
Must(names.ParseDomain(strings.Join(wildcard.PreviewDomain, "."), names.ParseDomainRelaxed)))
}
}
getStats := func(domain names.Domain) *storageData {
domain = domain.EffectiveTLDPlusOne(extraSuffixes) // FIXME
if _, found := thickStats[domain]; !found {
thickStats[domain] = &storageData{
siteBlobs: map[string]int64{},
@@ -40,7 +52,7 @@ func AnalyzeStorage(ctx context.Context) ([]*StorageSize, error) {
return thickStats[domain]
}
totalStats := getStats("*")
totalStats := getStats(names.Domain{})
logc.Printf(ctx, "analyze: enumerating manifests")
for item, err := range backend.GetAllManifests(ctx) {
@@ -48,8 +60,7 @@ func AnalyzeStorage(ctx context.Context) ([]*StorageSize, error) {
if err != nil {
return nil, fmt.Errorf("analyze err: %w", err)
}
domain, _, _ := strings.Cut(metadata.Name, "/")
stats := getStats(domain)
stats := getStats(metadata.Name.Domain)
stats.siteManifests += metadata.Size
totalStats.siteManifests += metadata.Size
for _, entry := range manifest.GetContents() {
@@ -67,7 +78,7 @@ func AnalyzeStorage(ctx context.Context) ([]*StorageSize, error) {
if err != nil {
return nil, fmt.Errorf("analyze err: %w", err)
}
domain := record.GetDomain()
domain := Must(names.ParseDomain(record.GetDomain(), names.ParseDomainRelaxed))
stats := getStats(domain)
recordSize := int64(len(EncodeAuditRecord(record)))
stats.auditRecords += recordSize
@@ -89,7 +100,7 @@ func AnalyzeStorage(ctx context.Context) ([]*StorageSize, error) {
// Now aggregate the information.
for domain, stats := range thickStats {
sizes := StorageSize{Domain: domain}
sizes := StorageSize{Domain: domain.String()}
sizes.CurrentSize += stats.siteManifests
for _, size := range stats.siteBlobs {
sizes.CurrentSize += size
+17 -31
View File
@@ -15,6 +15,7 @@ import (
"strings"
"time"
"codeberg.org/git-pages/git-pages/src/names"
exponential "github.com/jpillora/backoff"
"github.com/kankanreno/go-snowflake"
"github.com/prometheus/client_golang/prometheus"
@@ -396,65 +397,50 @@ func notifyAudit(ctx context.Context, id AuditID) {
}
func (audited *auditedBackend) CommitManifest(
ctx context.Context, name string, manifest *Manifest, opts ModifyManifestOptions,
ctx context.Context, site names.Site, manifest *Manifest, opts ModifyManifestOptions,
) error {
domain, project, ok := strings.Cut(name, "/")
if !ok {
panic("malformed manifest name")
}
if err := audited.appendNewAuditRecord(ctx, &AuditRecord{
Event: AuditEvent_CommitManifest.Enum(),
Domain: proto.String(domain),
Project: proto.String(project),
Domain: proto.String(site.Domain.String()),
Project: proto.String(site.Project.String()),
Manifest: manifest,
}); err != nil {
return err
}
return audited.Backend.CommitManifest(ctx, name, manifest, opts)
return audited.Backend.CommitManifest(ctx, site, manifest, opts)
}
func (audited *auditedBackend) DeleteManifest(
ctx context.Context, name string, opts ModifyManifestOptions,
ctx context.Context, site names.Site, opts ModifyManifestOptions,
) error {
domain, project, ok := strings.Cut(name, "/")
if !ok {
panic("malformed manifest name")
}
if err := audited.appendNewAuditRecord(ctx, &AuditRecord{
Event: AuditEvent_DeleteManifest.Enum(),
Domain: proto.String(domain),
Project: proto.String(project),
Domain: proto.String(site.Domain.String()),
Project: proto.String(site.Project.String()),
}); err != nil {
return err
}
return audited.Backend.DeleteManifest(ctx, name, opts)
return audited.Backend.DeleteManifest(ctx, site, opts)
}
func (audited *auditedBackend) ExpireManifest(ctx context.Context, name string) error {
domain, project, ok := strings.Cut(name, "/")
if !ok {
panic("malformed manifest name")
}
func (audited *auditedBackend) ExpireManifest(ctx context.Context, site names.Site) error {
if err := audited.appendNewAuditRecord(ctx, &AuditRecord{
Event: AuditEvent_ExpireManifest.Enum(),
Domain: proto.String(domain),
Project: proto.String(project),
Domain: proto.String(site.Domain.String()),
Project: proto.String(site.Project.String()),
}); err != nil {
return err
}
return audited.Backend.ExpireManifest(ctx, name)
return audited.Backend.ExpireManifest(ctx, site)
}
func (audited *auditedBackend) FreezeDomain(ctx context.Context, domain string) error {
func (audited *auditedBackend) FreezeDomain(ctx context.Context, domain names.Domain) error {
if err := audited.appendNewAuditRecord(ctx, &AuditRecord{
Event: AuditEvent_FreezeDomain.Enum(),
Domain: proto.String(domain),
Domain: proto.String(domain.String()),
}); err != nil {
return err
}
@@ -462,10 +448,10 @@ func (audited *auditedBackend) FreezeDomain(ctx context.Context, domain string)
return audited.Backend.FreezeDomain(ctx, domain)
}
func (audited *auditedBackend) UnfreezeDomain(ctx context.Context, domain string) error {
func (audited *auditedBackend) UnfreezeDomain(ctx context.Context, domain names.Domain) error {
if err := audited.appendNewAuditRecord(ctx, &AuditRecord{
Event: AuditEvent_UnfreezeDomain.Enum(),
Domain: proto.String(domain),
Domain: proto.String(domain.String()),
}); err != nil {
return err
}
+21 -56
View File
@@ -14,7 +14,6 @@ import (
"time"
"codeberg.org/git-pages/git-pages/src/names"
"golang.org/x/net/idna"
)
type AuthError struct {
@@ -45,40 +44,6 @@ func authorizeInsecure(r *http.Request) *Authorization {
return nil
}
var idnaProfile = idna.New(idna.MapForLookup(), idna.BidiRule())
func GetHost(r *http.Request) (string, error) {
host, _, err := net.SplitHostPort(r.Host)
if err != nil {
host = r.Host
}
// this also rejects invalid characters and labels
host, err = idnaProfile.ToASCII(host)
if err != nil {
if config.Feature("relaxed-idna") {
// unfortunately, the go IDNA library has some significant issues around its
// Unicode TR46 implementation: https://github.com/golang/go/issues/76804
// we would like to allow *just* the _ here, but adding `idna.StrictDomainName(false)`
// would also accept domains like `*.foo.bar` which should clearly be disallowed.
// as a workaround, accept a domain name if it is valid with all `_` characters
// replaced with an alphanumeric character (we use `a`); this allows e.g. `foo_bar.xxx`
// and `foo__bar.xxx`, as well as `_foo.xxx` and `foo_.xxx`. labels starting with
// an underscore are explicitly rejected below.
_, err = idnaProfile.ToASCII(strings.ReplaceAll(host, "_", "a"))
}
if err != nil {
return "", AuthError{http.StatusBadRequest,
fmt.Sprintf("malformed host name %q", host)}
}
}
if strings.HasPrefix(host, ".") || strings.HasPrefix(host, "_") {
return "", AuthError{http.StatusBadRequest,
fmt.Sprintf("reserved host name %q", host)}
}
host = strings.TrimSuffix(host, ".")
return host, nil
}
type Authorization struct {
// If `nil`, any URL is allowed. If not, only those in the set are allowed.
repoURLs []string
@@ -98,7 +63,7 @@ func (auth *Authorization) ForgeRepoURL() string {
}
func authorizeDNSChallenge(r *http.Request) (*Authorization, error) {
host, err := GetHost(r)
domain, err := GetDomain(r)
if err != nil {
return nil, err
}
@@ -143,14 +108,14 @@ func authorizeDNSChallenge(r *http.Request) (*Authorization, error) {
param = password
}
challengeHostname := fmt.Sprintf("_git-pages-challenge.%s", host)
challengeHostname := fmt.Sprintf("_git-pages-challenge.%s", domain)
actualChallenges, err := net.LookupTXT(challengeHostname)
if err != nil {
return nil, AuthError{http.StatusUnauthorized,
fmt.Sprintf("failed to look up DNS challenge: %s TXT", challengeHostname)}
}
expectedChallenge := fmt.Sprintf("%x", sha256.Sum256(fmt.Appendf(nil, "%s %s", host, param)))
expectedChallenge := fmt.Sprintf("%x", sha256.Sum256(fmt.Appendf(nil, "%s %s", domain, param)))
if !slices.Contains(actualChallenges, expectedChallenge) {
return nil, AuthError{http.StatusUnauthorized, fmt.Sprintf(
"defeated by DNS challenge: %s TXT %v does not include %s",
@@ -167,7 +132,7 @@ func authorizeDNSChallenge(r *http.Request) (*Authorization, error) {
}
func authorizeDNSAllowlist(r *http.Request, scope string) (*Authorization, error) {
host, err := GetHost(r)
domain, err := GetDomain(r)
if err != nil {
return nil, err
}
@@ -177,7 +142,7 @@ func authorizeDNSAllowlist(r *http.Request, scope string) (*Authorization, error
return nil, err
}
allowlistHostname := fmt.Sprintf("_%s.%s", scope, host)
allowlistHostname := fmt.Sprintf("_%s.%s", scope, domain)
records, err := net.LookupTXT(allowlistHostname)
if err != nil {
return nil, AuthError{http.StatusUnauthorized,
@@ -223,12 +188,12 @@ func authorizeDNSAllowlist(r *http.Request, scope string) (*Authorization, error
// used for `/.git-pages/...` metadata
func authorizeWildcardMatchHost(r *http.Request, pattern *WildcardPattern) (*Authorization, error) {
host, err := GetHost(r)
domain, err := GetDomain(r)
if err != nil {
return nil, err
}
if _, found := pattern.Matches(host, WildcardDomainAny); found {
if _, found := pattern.Matches(domain, WildcardDomainAny); found {
return &Authorization{
repoURLs: []string{},
branch: "",
@@ -236,14 +201,14 @@ func authorizeWildcardMatchHost(r *http.Request, pattern *WildcardPattern) (*Aut
} else {
return nil, AuthError{
http.StatusUnauthorized,
fmt.Sprintf("domain %s does not match wildcard %s", host, pattern.GetHost()),
fmt.Sprintf("domain %s does not match wildcard %s", domain, pattern.GetHost()),
}
}
}
// used for updates to site content
func authorizeWildcardMatchSite(r *http.Request, pattern *WildcardPattern) (*Authorization, error) {
host, err := GetHost(r)
domain, err := GetDomain(r)
if err != nil {
return nil, err
}
@@ -253,13 +218,13 @@ func authorizeWildcardMatchSite(r *http.Request, pattern *WildcardPattern) (*Aut
return nil, err
}
if userName, found := pattern.Matches(host, WildcardDomainPrimary); found {
if userName, found := pattern.Matches(domain, WildcardDomainPrimary); found {
repoURL, branch := pattern.ApplyTemplate(userName, projectName.String())
return &Authorization{repoURLs: []string{repoURL}, branch: branch}, nil
} else {
return nil, AuthError{
http.StatusUnauthorized,
fmt.Sprintf("domain %s does not match wildcard %s", host, pattern.GetHost()),
fmt.Sprintf("domain %s does not match wildcard %s", domain, pattern.GetHost()),
}
}
}
@@ -267,29 +232,29 @@ func authorizeWildcardMatchSite(r *http.Request, pattern *WildcardPattern) (*Aut
// used for compatibility with Codeberg Pages v2
// see https://docs.codeberg.org/codeberg-pages/using-custom-domain/
func authorizeCodebergPagesV2(r *http.Request) (*Authorization, error) {
host, err := GetHost(r)
domain, err := GetDomain(r)
if err != nil {
return nil, err
}
dnsRecords := []string{}
cnameRecord, err := net.LookupCNAME(host)
cnameRecord, err := net.LookupCNAME(domain.String())
// "LookupCNAME does not return an error if host does not contain DNS "CNAME" records,
// as long as host resolves to address records.
if err == nil && cnameRecord != host {
if err == nil && cnameRecord != domain.String() {
// LookupCNAME() returns a domain with the root label, i.e. `username.codeberg.page.`,
// with the trailing dot
dnsRecords = append(dnsRecords, strings.TrimSuffix(cnameRecord, "."))
}
txtRecords, err := net.LookupTXT(host)
txtRecords, err := net.LookupTXT(domain.String())
if err == nil {
dnsRecords = append(dnsRecords, txtRecords...)
}
if len(dnsRecords) > 0 {
logc.Printf(r.Context(), "auth: %s TXT/CNAME: %q\n", host, dnsRecords)
logc.Printf(r.Context(), "auth: %s TXT/CNAME: %q\n", domain, dnsRecords)
}
for _, dnsRecord := range dnsRecords {
@@ -328,7 +293,7 @@ func authorizeCodebergPagesV2(r *http.Request) (*Authorization, error) {
return nil, AuthError{
http.StatusUnauthorized,
fmt.Sprintf("domain %s does not have Codeberg Pages TXT or CNAME records", host),
fmt.Sprintf("domain %s does not have Codeberg Pages TXT or CNAME records", domain),
}
}
@@ -610,7 +575,7 @@ func authorizeForgeWildcard(r *http.Request) (*Authorization, error) {
return nil, AuthError{http.StatusUnauthorized, "missing Forge-Authorization header"}
}
host, err := GetHost(r)
host, err := GetDomain(r)
if err != nil {
return nil, err
}
@@ -777,15 +742,15 @@ func AuthorizeDeletion(r *http.Request) (*Authorization, error) {
}
func CheckForbiddenDomain(r *http.Request) error {
host, err := GetHost(r)
domain, err := GetDomain(r)
if err != nil {
return err
}
host = strings.ToLower(host)
for _, reservedDomain := range config.Limits.ForbiddenDomains {
reservedDomain = strings.ToLower(reservedDomain)
if host == reservedDomain || strings.HasSuffix(host, fmt.Sprintf(".%s", reservedDomain)) {
if domain.String() == reservedDomain ||
strings.HasSuffix(domain.String(), fmt.Sprintf(".%s", reservedDomain)) {
return AuthError{http.StatusForbidden, "forbidden domain"}
}
}
+11 -9
View File
@@ -8,6 +8,8 @@ import (
"iter"
"strings"
"time"
"codeberg.org/git-pages/git-pages/src/names"
)
var ErrObjectNotFound = errors.New("not found")
@@ -46,7 +48,7 @@ type GetManifestOptions struct {
}
type ManifestMetadata struct {
Name string
Name names.Site
Size int64
LastModified time.Time
ETag string
@@ -103,7 +105,7 @@ type Backend interface {
EnumerateBlobs(ctx context.Context) iter.Seq2[BlobMetadata, error]
// Retrieve a manifest.
GetManifest(ctx context.Context, name string, opts GetManifestOptions) (
GetManifest(ctx context.Context, site names.Site, opts GetManifestOptions) (
manifest *Manifest, metadata ManifestMetadata, err error,
)
@@ -119,13 +121,13 @@ type Backend interface {
// Commit a manifest. This is an atomic operation; `GetManifest` calls will return either
// the old version or the new version of the manifest, never anything else.
CommitManifest(ctx context.Context, name string, manifest *Manifest, opts ModifyManifestOptions) error
CommitManifest(ctx context.Context, site names.Site, manifest *Manifest, opts ModifyManifestOptions) error
// Delete a manifest. This operation is initiated via the API.
DeleteManifest(ctx context.Context, name string, opts ModifyManifestOptions) error
DeleteManifest(ctx context.Context, site names.Site, opts ModifyManifestOptions) error
// Expire a manifest. This operation is initiated via a scheduled job.
ExpireManifest(ctx context.Context, name string) error
ExpireManifest(ctx context.Context, site names.Site) error
// Iterate through metadata of all manifests. Whether manifests that are newly added during
// iteration will appear in the results is unspecified.
@@ -139,17 +141,17 @@ type Backend interface {
HasSiteListChanged(ctx context.Context, since time.Time) (changed bool, lastChanged time.Time, err error)
// Check whether a domain has any deployments.
CheckDomain(ctx context.Context, domain string) (found bool, err error)
CheckDomain(ctx context.Context, domain names.Domain) (found bool, err error)
// Create a domain. This allows us to start serving content for the domain.
CreateDomain(ctx context.Context, domain string) error
CreateDomain(ctx context.Context, domain names.Domain) error
// Freeze a domain. This allows a site to be administratively locked, e.g. if it
// is discovered serving abusive content.
FreezeDomain(ctx context.Context, domain string) error
FreezeDomain(ctx context.Context, domain names.Domain) error
// Thaw a domain. This removes the previously placed administrative lock (if any).
UnfreezeDomain(ctx context.Context, domain string) error
UnfreezeDomain(ctx context.Context, domain names.Domain) error
// Append a record to the audit log.
AppendAuditLog(ctx context.Context, id AuditID, record *AuditRecord) error
+33 -35
View File
@@ -13,6 +13,7 @@ import (
"strings"
"time"
"codeberg.org/git-pages/git-pages/src/names"
sys "codeberg.org/git-pages/git-pages/src/sys"
)
@@ -219,11 +220,11 @@ func (fs *FSBackend) EnumerateBlobs(ctx context.Context) iter.Seq2[BlobMetadata,
}
func (fs *FSBackend) GetManifest(
ctx context.Context, name string, opts GetManifestOptions,
ctx context.Context, site names.Site, opts GetManifestOptions,
) (
manifest *Manifest, metadata ManifestMetadata, err error,
) {
stat, err := fs.siteRoot.Stat(name)
stat, err := fs.siteRoot.Stat(site.String())
if errors.Is(err, os.ErrNotExist) {
err = fmt.Errorf("%w: %s", ErrObjectNotFound, err.(*os.PathError).Path)
return
@@ -231,14 +232,14 @@ func (fs *FSBackend) GetManifest(
err = fmt.Errorf("stat: %w", err)
return
}
data, err := fs.siteRoot.ReadFile(name)
data, err := fs.siteRoot.ReadFile(site.String())
if err != nil {
err = fmt.Errorf("read: %w", err)
return
}
manifest, err = DecodeManifest(data)
if err != nil {
err = fmt.Errorf("decode %s: %w", name, err)
err = fmt.Errorf("decode %s: %w", site, err)
return
}
return manifest, ManifestMetadata{
@@ -266,11 +267,11 @@ func (fs *FSBackend) StageManifest(ctx context.Context, manifest *Manifest) erro
return nil
}
func domainFrozenMarkerName(domain string) string {
return filepath.Join(domain, ".frozen")
func domainFrozenMarkerName(domain names.Domain) string {
return filepath.Join(domain.String(), ".frozen")
}
func (fs *FSBackend) checkDomainFrozen(ctx context.Context, domain string) error {
func (fs *FSBackend) checkDomainFrozen(ctx context.Context, domain names.Domain) error {
if _, err := fs.siteRoot.Stat(domainFrozenMarkerName(domain)); err == nil {
return ErrDomainFrozen
} else if !errors.Is(err, os.ErrNotExist) {
@@ -293,8 +294,8 @@ type manifestLockGuard struct {
file *os.File
}
func lockManifest(fs *os.Root, name string) (*manifestLockGuard, error) {
file, err := fs.Open(name)
func lockManifest(fs *os.Root, site names.Site) (*manifestLockGuard, error) {
file, err := fs.Open(site.String())
if errors.Is(err, os.ErrNotExist) {
return &manifestLockGuard{nil}, nil
} else if err != nil {
@@ -315,10 +316,10 @@ func (guard *manifestLockGuard) Unlock() {
}
func (fs *FSBackend) checkManifestPrecondition(
ctx context.Context, name string, opts ModifyManifestOptions,
ctx context.Context, site names.Site, opts ModifyManifestOptions,
) error {
if !opts.IfUnmodifiedSince.IsZero() {
stat, err := fs.siteRoot.Stat(name)
stat, err := fs.siteRoot.Stat(site.String())
if err != nil {
return fmt.Errorf("stat: %w", err)
}
@@ -329,7 +330,7 @@ func (fs *FSBackend) checkManifestPrecondition(
}
if opts.IfMatch != "" {
data, err := fs.siteRoot.ReadFile(name)
data, err := fs.siteRoot.ReadFile(site.String())
if err != nil {
return fmt.Errorf("read: %w", err)
}
@@ -343,22 +344,21 @@ func (fs *FSBackend) checkManifestPrecondition(
}
func (fs *FSBackend) CommitManifest(
ctx context.Context, name string, manifest *Manifest, opts ModifyManifestOptions,
ctx context.Context, site names.Site, manifest *Manifest, opts ModifyManifestOptions,
) error {
if fs.hasAtomicCAS {
if guard, err := lockManifest(fs.siteRoot, name); err != nil {
if guard, err := lockManifest(fs.siteRoot, site); err != nil {
return err
} else {
defer guard.Unlock()
}
}
domain := filepath.Dir(name)
if err := fs.checkDomainFrozen(ctx, domain); err != nil {
if err := fs.checkDomainFrozen(ctx, site.Domain); err != nil {
return err
}
if err := fs.checkManifestPrecondition(ctx, name, opts); err != nil {
if err := fs.checkManifestPrecondition(ctx, site, opts); err != nil {
return err
}
@@ -369,11 +369,11 @@ func (fs *FSBackend) CommitManifest(
return fmt.Errorf("manifest not staged")
}
if err := fs.siteRoot.MkdirAll(domain, 0o755); err != nil {
if err := fs.siteRoot.MkdirAll(site.Domain.String(), 0o755); err != nil {
return fmt.Errorf("mkdir: %w", err)
}
if err := fs.siteRoot.Rename(manifestHashName, name); err != nil {
if err := fs.siteRoot.Rename(manifestHashName, site.String()); err != nil {
return fmt.Errorf("rename: %w", err)
}
@@ -381,26 +381,25 @@ func (fs *FSBackend) CommitManifest(
}
func (fs *FSBackend) DeleteManifest(
ctx context.Context, name string, opts ModifyManifestOptions,
ctx context.Context, site names.Site, opts ModifyManifestOptions,
) error {
if fs.hasAtomicCAS {
if guard, err := lockManifest(fs.siteRoot, name); err != nil {
if guard, err := lockManifest(fs.siteRoot, site); err != nil {
return err
} else {
defer guard.Unlock()
}
}
domain := filepath.Dir(name)
if err := fs.checkDomainFrozen(ctx, domain); err != nil {
if err := fs.checkDomainFrozen(ctx, site.Domain); err != nil {
return err
}
if err := fs.checkManifestPrecondition(ctx, name, opts); err != nil {
if err := fs.checkManifestPrecondition(ctx, site, opts); err != nil {
return err
}
err := fs.siteRoot.Remove(name)
err := fs.siteRoot.Remove(site.String())
if errors.Is(err, os.ErrNotExist) {
return nil
} else {
@@ -409,16 +408,15 @@ func (fs *FSBackend) DeleteManifest(
}
func (fs *FSBackend) ExpireManifest(
ctx context.Context, name string,
ctx context.Context, site names.Site,
) error {
return fs.DeleteManifest(ctx, name, ModifyManifestOptions{})
return fs.DeleteManifest(ctx, site, ModifyManifestOptions{})
}
func (fs *FSBackend) EnumerateManifests(ctx context.Context) iter.Seq2[*ManifestMetadata, error] {
return func(yield func(*ManifestMetadata, error) bool) {
iofs.WalkDir(fs.siteRoot.FS(), ".",
func(path string, entry iofs.DirEntry, err error) error {
_, project, _ := strings.Cut(path, "/")
var metadata *ManifestMetadata
if err != nil {
// report error
@@ -427,7 +425,7 @@ func (fs *FSBackend) EnumerateManifests(ctx context.Context) iter.Seq2[*Manifest
return nil
} else if strings.HasPrefix(path, ".tmp") {
// temporary
} else if project == "" || strings.HasPrefix(project, ".") && project != ".index" {
} else if site, err := names.ParseSite(path); err != nil {
// skip internal
return nil
} else if info, err := entry.Info(); err != nil {
@@ -435,7 +433,7 @@ func (fs *FSBackend) EnumerateManifests(ctx context.Context) iter.Seq2[*Manifest
} else {
// report blob
metadata = &ManifestMetadata{
Name: path,
Name: site,
Size: info.Size(),
LastModified: info.ModTime(),
}
@@ -465,8 +463,8 @@ func (fs *FSBackend) GetAllManifests(ctx context.Context) iter.Seq2[tuple[*Manif
}
}
func (fs *FSBackend) CheckDomain(ctx context.Context, domain string) (bool, error) {
_, err := fs.siteRoot.Stat(domain)
func (fs *FSBackend) CheckDomain(ctx context.Context, domain names.Domain) (bool, error) {
_, err := fs.siteRoot.Stat(domain.String())
if errors.Is(err, os.ErrNotExist) {
return false, nil
} else if err == nil {
@@ -476,15 +474,15 @@ func (fs *FSBackend) CheckDomain(ctx context.Context, domain string) (bool, erro
}
}
func (fs *FSBackend) CreateDomain(ctx context.Context, domain string) error {
func (fs *FSBackend) CreateDomain(ctx context.Context, domain names.Domain) error {
return nil // no-op
}
func (fs *FSBackend) FreezeDomain(ctx context.Context, domain string) error {
func (fs *FSBackend) FreezeDomain(ctx context.Context, domain names.Domain) error {
return fs.siteRoot.WriteFile(domainFrozenMarkerName(domain), []byte{}, 0o644)
}
func (fs *FSBackend) UnfreezeDomain(ctx context.Context, domain string) error {
func (fs *FSBackend) UnfreezeDomain(ctx context.Context, domain names.Domain) error {
err := fs.siteRoot.Remove(domainFrozenMarkerName(domain))
if errors.Is(err, os.ErrNotExist) {
return nil
+42 -44
View File
@@ -13,6 +13,7 @@ import (
"sync"
"time"
"codeberg.org/git-pages/git-pages/src/names"
"github.com/c2h5oh/datasize"
"github.com/maypok86/otter/v2"
"github.com/minio/minio-go/v7"
@@ -128,7 +129,7 @@ type S3Backend struct {
client *minio.Client
bucket string
blobCache *observedCache[string, *CachedBlob]
siteCache *observedCache[string, *CachedManifest]
siteCache *observedCache[names.Site, *CachedManifest]
featureCache *otter.Cache[BackendFeature, bool]
}
@@ -224,7 +225,7 @@ func NewS3Backend(ctx context.Context, config *S3Config) (*S3Backend, error) {
EvictionNumberCounter: manifestCacheEvictionsCount,
}
siteCache, err := newObservedCache(makeCacheOptions(&config.SiteCache,
func(key string, value *CachedManifest) uint32 { return value.weight }),
func(key names.Site, value *CachedManifest) uint32 { return value.weight }),
siteCacheMetrics)
if err != nil {
return nil, err
@@ -410,8 +411,8 @@ func (s3 *S3Backend) EnumerateBlobs(ctx context.Context) iter.Seq2[BlobMetadata,
}
}
func manifestObjectName(name string) string {
return fmt.Sprintf("site/%s", name)
func manifestObjectName(object string) string {
return fmt.Sprintf("site/%s", object)
}
func stagedManifestObjectName(manifestData []byte) string {
@@ -423,7 +424,7 @@ type s3ManifestLoader struct {
}
func (l s3ManifestLoader) Load(
ctx context.Context, key string,
ctx context.Context, key names.Site,
) (
*CachedManifest, error,
) {
@@ -431,7 +432,7 @@ func (l s3ManifestLoader) Load(
}
func (l s3ManifestLoader) Reload(
ctx context.Context, key string, oldValue *CachedManifest,
ctx context.Context, key names.Site, oldValue *CachedManifest,
) (
*CachedManifest, error,
) {
@@ -439,18 +440,18 @@ func (l s3ManifestLoader) Reload(
}
func (l s3ManifestLoader) load(
ctx context.Context, name string, oldManifest *CachedManifest,
ctx context.Context, site names.Site, oldManifest *CachedManifest,
) (
*CachedManifest, error,
) {
logc.Printf(ctx, "s3: get manifest %s\n", name)
logc.Printf(ctx, "s3: get manifest %s\n", site)
loader := func() (*CachedManifest, error) {
opts := minio.GetObjectOptions{}
if oldManifest != nil && oldManifest.metadata.ETag != "" {
opts.SetMatchETagExcept(oldManifest.metadata.ETag)
}
object, err := l.s3.client.GetObject(ctx, l.s3.bucket, manifestObjectName(name), opts)
object, err := l.s3.client.GetObject(ctx, l.s3.bucket, manifestObjectName(site.String()), opts)
// Note that many errors (e.g. NoSuchKey) will be reported only after this point.
if err != nil {
return nil, err
@@ -469,7 +470,7 @@ func (l s3ManifestLoader) load(
manifest, err := DecodeManifest(data)
if err != nil {
err = fmt.Errorf("decode %s: %w", name, err)
err = fmt.Errorf("decode %s: %w", site, err)
return nil, err
}
@@ -512,19 +513,19 @@ func (l s3ManifestLoader) load(
}
func (s3 *S3Backend) GetManifest(
ctx context.Context, name string, opts GetManifestOptions,
ctx context.Context, site names.Site, opts GetManifestOptions,
) (
manifest *Manifest, metadata ManifestMetadata, err error,
) {
if opts.BypassCache {
entry, found := s3.siteCache.Cache.GetEntry(name)
entry, found := s3.siteCache.Cache.GetEntry(site)
if found && entry.RefreshableAt().Before(time.Now()) {
s3.siteCache.Cache.Invalidate(name)
s3.siteCache.Cache.Invalidate(site)
}
}
var cached *CachedManifest
cached, err = s3.siteCache.Get(ctx, name, s3ManifestLoader{s3})
cached, err = s3.siteCache.Get(ctx, site, s3ManifestLoader{s3})
if err != nil {
return
} else {
@@ -543,11 +544,11 @@ func (s3 *S3Backend) StageManifest(ctx context.Context, manifest *Manifest) erro
return err
}
func domainFrozenObjectName(domain string) string {
return manifestObjectName(fmt.Sprintf("%s/.frozen", domain))
func domainFrozenObjectName(domain names.Domain) string {
return manifestObjectName(fmt.Sprintf("%s/.frozen", domain.String()))
}
func (s3 *S3Backend) checkDomainFrozen(ctx context.Context, domain string) error {
func (s3 *S3Backend) checkDomainFrozen(ctx context.Context, domain names.Domain) error {
_, err := s3.client.StatObject(ctx, s3.bucket, domainFrozenObjectName(domain),
minio.GetObjectOptions{})
if err == nil {
@@ -577,9 +578,9 @@ func (s3 *S3Backend) HasAtomicCAS(ctx context.Context) bool {
}
func (s3 *S3Backend) checkManifestPrecondition(
ctx context.Context, name string, opts ModifyManifestOptions,
ctx context.Context, site names.Site, opts ModifyManifestOptions,
) (exists bool, err error) {
stat, err := s3.client.StatObject(ctx, s3.bucket, manifestObjectName(name),
stat, err := s3.client.StatObject(ctx, s3.bucket, manifestObjectName(site.String()),
minio.GetObjectOptions{})
if err != nil {
errResp := minio.ToErrorResponse(err)
@@ -603,17 +604,16 @@ func (s3 *S3Backend) checkManifestPrecondition(
}
func (s3 *S3Backend) CommitManifest(
ctx context.Context, name string, manifest *Manifest, opts ModifyManifestOptions,
ctx context.Context, site names.Site, manifest *Manifest, opts ModifyManifestOptions,
) error {
data := EncodeManifest(manifest)
logc.Printf(ctx, "s3: commit manifest %x -> %s", sha256.Sum256(data), name)
logc.Printf(ctx, "s3: commit manifest %x -> %s", sha256.Sum256(data), site.String())
domain, _, _ := strings.Cut(name, "/")
if err := s3.checkDomainFrozen(ctx, domain); err != nil {
if err := s3.checkDomainFrozen(ctx, site.Domain); err != nil {
return err
}
existed, err := s3.checkManifestPrecondition(ctx, name, opts)
existed, err := s3.checkManifestPrecondition(ctx, site, opts)
if err != nil {
return err
}
@@ -628,11 +628,11 @@ func (s3 *S3Backend) CommitManifest(
// It does reliably work on MinIO at least.
putOptions.SetMatchETag(opts.IfMatch)
}
_, putErr := s3.client.PutObject(ctx, s3.bucket, manifestObjectName(name),
_, putErr := s3.client.PutObject(ctx, s3.bucket, manifestObjectName(site.String()),
bytes.NewReader(data), int64(len(data)), putOptions)
removeErr := s3.client.RemoveObject(ctx, s3.bucket, stagedManifestObjectName(data),
minio.RemoveObjectOptions{})
s3.siteCache.Cache.Invalidate(name)
s3.siteCache.Cache.Invalidate(site)
if putErr != nil {
if errResp := minio.ToErrorResponse(putErr); errResp.Code == "PreconditionFailed" {
return ErrPreconditionFailed
@@ -653,26 +653,25 @@ func (s3 *S3Backend) CommitManifest(
}
func (s3 *S3Backend) DeleteManifest(
ctx context.Context, name string, opts ModifyManifestOptions,
ctx context.Context, site names.Site, opts ModifyManifestOptions,
) error {
logc.Printf(ctx, "s3: delete manifest %s\n", name)
logc.Printf(ctx, "s3: delete manifest %s\n", site)
domain, _, _ := strings.Cut(name, "/")
if err := s3.checkDomainFrozen(ctx, domain); err != nil {
if err := s3.checkDomainFrozen(ctx, site.Domain); err != nil {
return err
}
existed, err := s3.checkManifestPrecondition(ctx, name, opts)
existed, err := s3.checkManifestPrecondition(ctx, site, opts)
if err != nil {
return err
}
err = s3.client.RemoveObject(ctx, s3.bucket, manifestObjectName(name),
err = s3.client.RemoveObject(ctx, s3.bucket, manifestObjectName(site.String()),
minio.RemoveObjectOptions{})
if err != nil {
return err
}
s3.siteCache.Cache.Invalidate(name)
s3.siteCache.Cache.Invalidate(site)
if existed {
if err := s3.bumpLastSiteUpdateTimestamp(ctx); err != nil {
@@ -684,9 +683,9 @@ func (s3 *S3Backend) DeleteManifest(
}
func (s3 *S3Backend) ExpireManifest(
ctx context.Context, name string,
ctx context.Context, site names.Site,
) error {
return s3.DeleteManifest(ctx, name, ModifyManifestOptions{})
return s3.DeleteManifest(ctx, site, ModifyManifestOptions{})
}
func (s3 *S3Backend) EnumerateManifests(ctx context.Context) iter.Seq2[*ManifestMetadata, error] {
@@ -705,14 +704,13 @@ func (s3 *S3Backend) EnumerateManifests(ctx context.Context) iter.Seq2[*Manifest
var err error
if err = object.Err; err == nil {
key := strings.TrimPrefix(object.Key, prefix)
_, project, _ := strings.Cut(key, "/")
if strings.HasSuffix(key, "/") {
continue // directory; skip
} else if project == "" || strings.HasPrefix(project, ".") && project != ".index" {
continue // internal; skip
} else if site, err := names.ParseSite(key); err != nil {
continue // not a site (marker, etc); skip
} else {
metadata = &ManifestMetadata{
Name: key,
Name: site,
Size: object.Size,
LastModified: object.LastModified,
ETag: object.ETag,
@@ -768,11 +766,11 @@ func (s3 *S3Backend) GetAllManifests(ctx context.Context) iter.Seq2[tuple[*Manif
}
}
func domainCheckObjectName(domain string) string {
func domainCheckObjectName(domain names.Domain) string {
return manifestObjectName(fmt.Sprintf("%s/.exists", domain))
}
func (s3 *S3Backend) CheckDomain(ctx context.Context, domain string) (exists bool, err error) {
func (s3 *S3Backend) CheckDomain(ctx context.Context, domain names.Domain) (exists bool, err error) {
logc.Printf(ctx, "s3: check domain %s\n", domain)
_, err = s3.client.StatObject(ctx, s3.bucket, domainCheckObjectName(domain),
@@ -803,7 +801,7 @@ func (s3 *S3Backend) CheckDomain(ctx context.Context, domain string) (exists boo
return
}
func (s3 *S3Backend) CreateDomain(ctx context.Context, domain string) error {
func (s3 *S3Backend) CreateDomain(ctx context.Context, domain names.Domain) error {
logc.Printf(ctx, "s3: create domain %s\n", domain)
_, err := s3.client.PutObject(ctx, s3.bucket, domainCheckObjectName(domain),
@@ -811,7 +809,7 @@ func (s3 *S3Backend) CreateDomain(ctx context.Context, domain string) error {
return err
}
func (s3 *S3Backend) FreezeDomain(ctx context.Context, domain string) error {
func (s3 *S3Backend) FreezeDomain(ctx context.Context, domain names.Domain) error {
logc.Printf(ctx, "s3: freeze domain %s\n", domain)
_, err := s3.client.PutObject(ctx, s3.bucket, domainFrozenObjectName(domain),
@@ -819,7 +817,7 @@ func (s3 *S3Backend) FreezeDomain(ctx context.Context, domain string) error {
return err
}
func (s3 *S3Backend) UnfreezeDomain(ctx context.Context, domain string) error {
func (s3 *S3Backend) UnfreezeDomain(ctx context.Context, domain names.Domain) error {
logc.Printf(ctx, "s3: unfreeze domain %s\n", domain)
err := s3.client.RemoveObject(ctx, s3.bucket, domainFrozenObjectName(domain),
-29
View File
@@ -1,29 +0,0 @@
package git_pages
import (
"slices"
"strings"
"golang.org/x/net/publicsuffix"
)
func EffectiveTLDPlusOne(domain string) string {
domainParts := strings.Split(domain, ".")
for _, pattern := range wildcards {
for _, suffix := range [][]string{pattern.PreviewDomain, pattern.Domain} {
if len(domainParts) < len(suffix)+1 {
continue
}
if slices.Equal(domainParts[len(domainParts)-len(suffix):], suffix) {
return strings.Join(domainParts[len(domainParts)-len(suffix)-1:], ".")
}
}
}
result, err := publicsuffix.EffectiveTLDPlusOne(domain)
if err == nil {
return result
}
return domain
}
+16 -20
View File
@@ -9,10 +9,10 @@ package git_pages
import (
"context"
"fmt"
"strings"
"sync"
"time"
"codeberg.org/git-pages/git-pages/src/names"
"github.com/bits-and-blooms/bloom/v3"
exponential "github.com/jpillora/backoff"
)
@@ -45,13 +45,13 @@ func (value Existence) String() string {
type ExistenceCache interface {
// Check if we might be serving the site.
CheckSite(ctx context.Context, site string) (result Existence)
CheckSite(ctx context.Context, site names.Site) (result Existence)
// Check if we might be serving the domain.
CheckDomain(ctx context.Context, domain string) (result Existence)
CheckDomain(ctx context.Context, domain names.Domain) (result Existence)
// Add the site to the cache.
AddSite(ctx context.Context, site string)
AddSite(ctx context.Context, site names.Site)
}
func CreateExistenceCache(ctx context.Context) (ExistenceCache, error) {
@@ -149,10 +149,8 @@ func (c *bloomExistenceCache) refresh(ctx context.Context) error {
if err != nil {
return fmt.Errorf("enum manifests: %w", err)
}
site := metadata.Name
domain, _, _ := strings.Cut(site, "/")
sites.AddString(site)
domains.AddString(domain)
sites.AddString(metadata.Name.String())
domains.AddString(metadata.Name.Domain.String())
siteCount++
}
@@ -165,14 +163,14 @@ func (c *bloomExistenceCache) refresh(ctx context.Context) error {
return nil
}
func (c *bloomExistenceCache) CheckSite(ctx context.Context, site string) (result Existence) {
func (c *bloomExistenceCache) CheckSite(ctx context.Context, site names.Site) (result Existence) {
select {
case c.accessCh <- struct{}{}:
default:
}
c.filterMu.Lock()
if c.sites.TestString(site) {
if c.sites.TestString(site.String()) {
result = ExistencePossible
} else {
result = ExistenceImpossible
@@ -183,14 +181,14 @@ func (c *bloomExistenceCache) CheckSite(ctx context.Context, site string) (resul
return
}
func (c *bloomExistenceCache) CheckDomain(ctx context.Context, domain string) (result Existence) {
func (c *bloomExistenceCache) CheckDomain(ctx context.Context, domain names.Domain) (result Existence) {
select {
case c.accessCh <- struct{}{}:
default:
}
c.filterMu.Lock()
if c.domains.TestString(domain) {
if c.domains.TestString(domain.String()) {
result = ExistencePossible
} else {
result = ExistenceImpossible
@@ -201,15 +199,13 @@ func (c *bloomExistenceCache) CheckDomain(ctx context.Context, domain string) (r
return
}
func (c *bloomExistenceCache) AddSite(ctx context.Context, site string) {
func (c *bloomExistenceCache) AddSite(ctx context.Context, site names.Site) {
c.refreshMu.Lock()
defer c.refreshMu.Unlock()
domain, _, _ := strings.Cut(site, "/")
c.filterMu.Lock()
c.sites.AddString(site)
c.domains.AddString(domain)
c.sites.AddString(site.String())
c.domains.AddString(site.Domain.String())
c.filterMu.Unlock()
logc.Printf(ctx, "existence: added site %s", site)
@@ -217,12 +213,12 @@ func (c *bloomExistenceCache) AddSite(ctx context.Context, site string) {
type dummyExistenceCache struct{}
func (d dummyExistenceCache) CheckSite(context.Context, string) Existence {
func (d dummyExistenceCache) CheckSite(context.Context, names.Site) Existence {
return ExistencePossible
}
func (d dummyExistenceCache) CheckDomain(context.Context, string) Existence {
func (d dummyExistenceCache) CheckDomain(context.Context, names.Domain) Existence {
return ExistencePossible
}
func (d dummyExistenceCache) AddSite(context.Context, string) {}
func (d dummyExistenceCache) AddSite(context.Context, names.Site) {}
+1 -1
View File
@@ -49,7 +49,7 @@ func TraceGarbage(ctx context.Context) error {
if err != nil {
return fmt.Errorf("trace err: %w", err)
}
traceManifest("site", metadata.Name, manifest)
traceManifest("site", metadata.Name.String(), manifest)
}
// Enumerate blobs live via audit records.
+30 -18
View File
@@ -175,17 +175,28 @@ func serve(ctx context.Context, listener net.Listener, handler http.Handler) {
}
}
func webRootArg(arg string) string {
func domainArg(arg string) (domain names.Domain) {
var err error
if domain, err = names.ParseDomain(arg, names.ParseDomainRelaxed); err != nil {
logc.Fatalf(context.Background(), "invalid site argument: %s\n", err)
}
return
}
func siteArg(arg string) (site names.Site) {
var err error
switch strings.Count(arg, "/") {
case 0:
return arg + "/.index"
site.Domain = domainArg(arg)
case 1:
return arg
if site, err = names.ParseSite(arg); err != nil {
logc.Fatalf(context.Background(), "invalid site argument: %s\n", err)
}
default:
logc.Fatalln(context.Background(),
"webroot argument must be either 'domain.tld' or 'domain.tld/dir")
return ""
"site argument must be either 'domain.tld' or 'domain.tld/dir")
}
return
}
func fileOutputArg() (writer io.WriteCloser) {
@@ -450,7 +461,7 @@ func Main(versionInfo string) {
io.Copy(fileOutputArg(), reader)
case *getManifest != "":
webRoot := webRootArg(*getManifest)
webRoot := siteArg(*getManifest)
manifest, _, err := backend.GetManifest(ctx, webRoot, GetManifestOptions{})
if err != nil {
logc.Fatalln(ctx, err)
@@ -458,7 +469,7 @@ func Main(versionInfo string) {
fmt.Fprintln(fileOutputArg(), string(ManifestJSON(manifest)))
case *getArchive != "":
webRoot := webRootArg(*getArchive)
webRoot := siteArg(*getArchive)
manifest, metadata, err :=
backend.GetManifest(ctx, webRoot, GetManifestOptions{})
if err != nil {
@@ -473,7 +484,7 @@ func Main(versionInfo string) {
logc.Fatalln(ctx, "update source must be provided as the argument")
}
webRoot := webRootArg(*updateSite)
webRoot := siteArg(*updateSite)
sourceURL, err := url.Parse(flag.Arg(0))
if err != nil {
@@ -530,7 +541,7 @@ func Main(versionInfo string) {
}
case *deleteSite != "":
webRoot := webRootArg(*deleteSite)
webRoot := siteArg(*deleteSite)
err := backend.DeleteManifest(ctx, webRoot, ModifyManifestOptions{})
if err != nil {
logc.Fatalln(ctx, err)
@@ -539,13 +550,13 @@ func Main(versionInfo string) {
logc.Println(ctx, "deleted")
case *freezeDomain != "" || *unfreezeDomain != "":
var domain string
var domain names.Domain
var freeze bool
if *freezeDomain != "" {
domain = *freezeDomain
domain = domainArg(*freezeDomain)
freeze = true
} else {
domain = *unfreezeDomain
domain = domainArg(*unfreezeDomain)
freeze = false
}
@@ -562,14 +573,15 @@ func Main(versionInfo string) {
}
case *purgeDomain != "":
domain := domainArg(*freezeDomain)
purgeCount := 0
for metadata, err := range backend.EnumerateManifests(ctx) {
if err != nil {
logc.Fatalln(ctx, err)
}
domain, _, _ := strings.Cut(metadata.Name, "/")
if domain == *purgeDomain {
if metadata.Name.Domain == domain {
err := backend.DeleteManifest(ctx, metadata.Name, ModifyManifestOptions{})
if err != nil {
logc.Fatalln(ctx, err)
@@ -578,7 +590,7 @@ func Main(versionInfo string) {
}
}
logc.Printf(ctx, "admin: purged %s (%d sites)\n", *purgeDomain, purgeCount)
logc.Printf(ctx, "admin: purged %s (%d sites)\n", domain, purgeCount)
case *auditLog:
records := []*AuditRecord{}
@@ -663,12 +675,12 @@ func Main(versionInfo string) {
logc.Fatalln(ctx, "no manifest in audit record")
}
webRoot := path.Join(record.GetDomain(), record.GetProject())
site := Must(names.ParseSite(fmt.Sprintf("%s/%s", record.GetDomain(), record.GetProject())))
err = backend.StageManifest(ctx, record.GetManifest())
if err != nil {
logc.Fatalln(ctx, err)
}
err = backend.CommitManifest(ctx, webRoot, record.GetManifest(), ModifyManifestOptions{})
err = backend.CommitManifest(ctx, site, record.GetManifest(), ModifyManifestOptions{})
if err != nil {
logc.Fatalln(ctx, err)
}
@@ -681,7 +693,7 @@ func Main(versionInfo string) {
}
if project != "*" {
if _, err := names.ParseProjectName(project); err != nil {
if _, err := names.ParseProject(project); err != nil {
logc.Fatalf(ctx, "audit detach: project name: %v\n", err)
}
}
+6 -7
View File
@@ -17,6 +17,7 @@ import (
"sync"
"time"
"codeberg.org/git-pages/git-pages/src/names"
"github.com/c2h5oh/datasize"
"github.com/go-git/go-git/v6/plumbing"
format "github.com/go-git/go-git/v6/plumbing/format/config"
@@ -338,11 +339,9 @@ func CompressFiles(ctx context.Context, manifest *Manifest) {
// Apply post-processing steps to the manifest.
// At the moment, there isn't a good way to report errors except to log them on the terminal.
// (Perhaps in the future they could be exposed at `.git-pages/status.txt`?)
func PrepareManifest(ctx context.Context, webRoot string, manifest *Manifest) error {
_, project, _ := strings.Cut(webRoot, "/")
func PrepareManifest(ctx context.Context, site names.Site, manifest *Manifest) error {
// Parse Netlify-style `_redirects`.
if err := ProcessRedirectsFile(ctx, project, manifest); err != nil {
if err := ProcessRedirectsFile(ctx, site.Project, manifest); err != nil {
logc.Printf(ctx, "redirects err: %s\n", err)
} else if len(manifest.Redirects) > 0 {
logc.Printf(ctx, "redirects ok: %d rules\n", len(manifest.Redirects))
@@ -379,9 +378,9 @@ var putBlobSemaphore = make(chan struct{})
// Uploads inline file data over certain size to the storage backend. Returns a copy of
// the manifest updated to refer to an external content-addressable store.
func StoreManifest(
ctx context.Context, name string, manifest *Manifest, opts ModifyManifestOptions,
ctx context.Context, site names.Site, manifest *Manifest, opts ModifyManifestOptions,
) (*Manifest, error) {
span, ctx := ObserveFunction(ctx, "StoreManifest", "manifest.name", name)
span, ctx := ObserveFunction(ctx, "StoreManifest", "manifest.name", site)
defer span.Finish()
extManifest := &Manifest{}
@@ -466,7 +465,7 @@ func StoreManifest(
return nil, err // currently ignores all but 1st error
}
if err := backend.CommitManifest(ctx, name, extManifest, opts); err != nil {
if err := backend.CommitManifest(ctx, site, extManifest, opts); err != nil {
if errors.Is(err, ErrDomainFrozen) {
return nil, err
} else {
+11 -9
View File
@@ -4,7 +4,8 @@ import (
"context"
"fmt"
"slices"
"strings"
"codeberg.org/git-pages/git-pages/src/names"
)
func RunMigration(ctx context.Context, name string) error {
@@ -22,19 +23,19 @@ func createDomainMarkers(ctx context.Context) error {
return nil
}
var manifests []string
var sites []names.Site
for metadata, err := range backend.EnumerateManifests(ctx) {
if err != nil {
return fmt.Errorf("enum manifests: %w", err)
}
manifests = append(manifests, metadata.Name)
sites = append(sites, metadata.Name)
}
slices.Sort(manifests)
var domains []string
for _, manifest := range manifests {
domain, _, _ := strings.Cut(manifest, "/")
if len(domains) == 0 || domains[len(domains)-1] != domain {
domains = append(domains, domain)
slices.SortFunc(sites, names.CompareSite)
var domains []names.Domain
for _, site := range sites {
if len(domains) == 0 || domains[len(domains)-1] != site.Domain {
domains = append(domains, site.Domain)
}
}
for idx, domain := range domains {
@@ -46,6 +47,7 @@ func createDomainMarkers(ctx context.Context) error {
if err := backend.EnableFeature(ctx, FeatureCheckDomainMarker); err != nil {
return err
}
logc.Printf(ctx, "created markers for %d domains", len(domains))
return nil
}
+97
View File
@@ -0,0 +1,97 @@
package names
import (
"fmt"
"slices"
"strings"
"golang.org/x/net/idna"
"golang.org/x/net/publicsuffix"
)
var ErrInvalidDomainName = NameError{"invalid domain name"}
// Domain of a git-pages site.
type Domain struct {
name string
}
// IsValid checks whether the domain name is valid. The zero value is the only invalid value.
func (d Domain) IsValid() bool {
return d.name != ""
}
// MustValid ensures the domain name is valid by panicking if it is not.
func (d Domain) MustValid() Domain {
if !d.IsValid() {
panic("nil domain name")
}
return d
}
// String returns the normalized domain name.
func (d Domain) String() string {
return d.MustValid().name
}
var idnaProfile = idna.New(idna.MapForLookup(), idna.BidiRule())
type ParseDomainMode int
const (
// RFC-compliant parsing mode.
ParseDomainStrict ParseDomainMode = iota
// Parsing mode that accepts `_` symbols in the middle and at the end of a label, and behaves
// like ParseDomainNormal otherwise.
ParseDomainRelaxed
)
// ParseDomain parses and normalizes a domain name according to the requirements set out in
// Section 5 of RFC 5891, as well as the Bidi Rule.
//
// The mappings include normalization and mapping case, width and other compatibility mappings.
func ParseDomain(name string, mode ParseDomainMode) (Domain, error) {
// this also rejects invalid characters and labels
name, err := idnaProfile.ToASCII(name)
if err != nil {
if mode == ParseDomainRelaxed {
// unfortunately, the go IDNA library has some significant issues around its
// Unicode TR46 implementation: https://github.com/golang/go/issues/76804
// we would like to allow *just* the _ here, but adding `idna.StrictDomainName(false)`
// would also accept domains like `*.foo.bar` which should clearly be disallowed.
// as a workaround, accept a domain name if it is valid with all `_` characters
// replaced with an alphanumeric character (we use `a`); this allows e.g. `foo_bar.xxx`
// and `foo__bar.xxx`, as well as `_foo.xxx` and `foo_.xxx`. labels starting with
// an underscore are explicitly rejected below.
_, err = idnaProfile.ToASCII(strings.ReplaceAll(name, "_", "a"))
}
if err != nil {
return Domain{}, fmt.Errorf("%w: malformed host name %q", ErrInvalidDomainName, name)
}
}
if strings.HasPrefix(name, ".") || strings.HasPrefix(name, "_") {
return Domain{}, fmt.Errorf("%w: reserved host name %q", ErrInvalidDomainName, name)
}
return Domain{strings.TrimSuffix(name, ".")}, nil
}
func (d Domain) EffectiveTLDPlusOne(extraSuffixes []Domain) Domain {
domainParts := strings.Split(d.name, ".")
for _, suffix := range extraSuffixes {
suffixParts := strings.Split(suffix.name, ".")
if len(domainParts) < len(suffixParts)+1 {
continue
}
if slices.Equal(domainParts[len(domainParts)-len(suffixParts):], suffixParts) {
return Domain{strings.Join(domainParts[len(domainParts)-len(suffixParts)-1:], ".")}
}
}
eTLDp1, err := publicsuffix.EffectiveTLDPlusOne(d.String())
if err != nil {
panic(fmt.Sprintf("internal error: %s", err))
}
return Domain{eTLDp1}
}
+70
View File
@@ -0,0 +1,70 @@
package names
import (
"errors"
"fmt"
"testing"
)
func checkDomainParseOk(t *testing.T, mode ParseDomainMode, value string, expect string) {
t.Helper()
if actual, err := ParseDomain(value, mode); err != nil {
t.Error(err)
} else {
if actual.name != expect {
t.Errorf("unexpected value: %v != %v", actual, expect)
}
}
}
func checkDomainParseErr(t *testing.T, mode ParseDomainMode, value string, expect error) {
t.Helper()
_, err := ParseDomain(value, mode)
if !errors.Is(err, expect) {
t.Errorf("unexpected error: %v, expected %v", err, expect)
}
}
func TestDomainParse(t *testing.T) {
for _, mode := range []ParseDomainMode{ParseDomainStrict, ParseDomainRelaxed} {
checkDomainParseOk(t, mode, "foo.bar", "foo.bar")
checkDomainParseOk(t, mode, "foo.bar.", "foo.bar")
checkDomainParseOk(t, mode, "foo-baz.bar", "foo-baz.bar")
checkDomainParseOk(t, mode, "foo--baz.bar", "foo--baz.bar")
checkDomainParseOk(t, mode, "ß.bar", "xn--zca.bar")
checkDomainParseOk(t, mode, "xn--zca.bar", "xn--zca.bar")
checkDomainParseErr(t, mode, ".foo.bar", ErrInvalidDomainName)
checkDomainParseErr(t, mode, "..foo.bar", ErrInvalidDomainName)
checkDomainParseErr(t, mode, "foo-.bar", ErrInvalidDomainName)
checkDomainParseErr(t, mode, "-foo.bar", ErrInvalidDomainName)
checkDomainParseErr(t, mode, "_foo.bar", ErrInvalidDomainName)
checkDomainParseErr(t, mode, "*.foo.bar", ErrInvalidDomainName)
}
for _, label := range []string{"foo_", "foo_baz", "foo__baz"} {
domain := fmt.Sprintf("%s.bar", label)
checkDomainParseErr(t, ParseDomainStrict, domain, ErrInvalidDomainName)
checkDomainParseOk(t, ParseDomainRelaxed, domain, domain)
}
}
func TestDomainProperties(t *testing.T) {
checkEq(t, Domain{}.IsValid(), false)
checkEq(t, Domain{"example.org"}.IsValid(), true)
checkPanic(t, func() Domain { return Domain{}.MustValid() })
checkEq(t, Domain{"example.org"}.MustValid(), Domain{"example.org"})
checkPanic(t, func() string { return Domain{}.String() })
checkEq(t, Domain{"example.org"}.String(), "example.org")
checkEq(t, Domain{"example.org"}.EffectiveTLDPlusOne([]Domain{}).String(),
"example.org")
checkEq(t, Domain{"foo.example.org"}.EffectiveTLDPlusOne([]Domain{}).String(),
"example.org")
checkEq(t, Domain{"example.co.uk"}.EffectiveTLDPlusOne([]Domain{}).String(),
"example.co.uk")
checkEq(t, Domain{"foo.example.co.uk"}.EffectiveTLDPlusOne([]Domain{}).String(),
"example.co.uk")
}
+9
View File
@@ -0,0 +1,9 @@
package names
type NameError struct {
text string
}
func (err NameError) Error() string {
return err.text
}
+10 -10
View File
@@ -1,17 +1,15 @@
package names
import (
"errors"
"fmt"
"net/url"
"strconv"
"strings"
)
var ErrInvalidProjectName = errors.New("invalid project name")
var ErrInvalidProjectURL = errors.New("invalid project URL")
var ErrInvalidProjectName = NameError{"invalid project name"}
var ErrInvalidProjectURL = NameError{"invalid project URL"}
const indexProject = ".index"
const previewSeparator = "@"
// Name of a git-pages project. The index project corresponds to the root URL of a domain, and
@@ -22,6 +20,8 @@ type Project struct {
name string
}
var IndexProject = Project{".index"}
// IsValid checks whether the project name is valid. The zero value is the only invalid value.
func (p Project) IsValid() bool {
return p.name != ""
@@ -42,7 +42,7 @@ func (p Project) String() string {
// IsIndex checks whether this name corresponds to the index project.
func (p Project) IsIndex() bool {
return p.String() == indexProject
return p.MustValid() == IndexProject
}
// GetPullRequest returns (baseName, prNumber, isPreview), where isPreview is true if this is
@@ -89,11 +89,11 @@ func validateProjectName(name string) (proj Project, err error) {
return
}
// ParseProjectName parses the given string as a project name. The string must not be empty, and
// ParseProject parses the given string as a project name. The string must not be empty, and
// ".index" represents the index project.
func ParseProjectName(name string) (Project, error) {
if name == indexProject {
return Project{indexProject}, nil
func ParseProject(name string) (Project, error) {
if name == IndexProject.name {
return IndexProject, nil
}
return validateProjectName(name)
@@ -107,7 +107,7 @@ func ParseProjectURL(url *url.URL) (Project, error) {
if strings.Contains(path, "/") {
return Project{}, fmt.Errorf("%w: must contain zero or one path segments", ErrInvalidProjectURL)
} else if path == "" {
return Project{indexProject}, nil
return IndexProject, nil
} else if name, err := validateProjectName(path); err != nil {
return Project{}, fmt.Errorf("%w: %w", ErrInvalidProjectURL, err)
} else {
+12 -55
View File
@@ -1,67 +1,24 @@
package names
import (
"errors"
"net/url"
"testing"
)
func must[T any](value T, err error) T {
if err != nil {
panic(err)
}
return value
}
func checkEq[T comparable](t *testing.T, value T, expect T) {
t.Helper()
if value != expect {
t.Errorf("unexpected: %v != %v", value, expect)
}
}
func checkPanic[T any](t *testing.T, f func() T) {
defer func() {
if r := recover(); r == nil {
t.Errorf("function did not panic")
}
}()
f()
}
func checkOk[T any](t *testing.T, parse func(T) (Project, error), value T, expect Project) {
t.Helper()
actual, err := parse(value)
if err != nil {
t.Errorf("unexpected error: %v", err)
} else if actual != expect {
t.Errorf("unexpected result: %v, expected %v", actual, expect)
}
}
func checkErr[T any](t *testing.T, parse func(T) (Project, error), value T, expect error) {
t.Helper()
_, err := parse(value)
if !errors.Is(err, expect) {
t.Errorf("unexpected error: %v, expected %v", err, expect)
}
}
func TestParseProjectName(t *testing.T) {
checkOk(t, ParseProjectName, ".index", Project{".index"})
checkOk(t, ParseProjectName, "foo", Project{"foo"})
checkOk(t, ParseProjectName, "foo@10", Project{"foo@10"})
checkOk(t, ParseProject, ".index", Project{".index"})
checkOk(t, ParseProject, "foo", Project{"foo"})
checkOk(t, ParseProject, "foo@10", Project{"foo@10"})
checkErr(t, ParseProjectName, ".foo", ErrInvalidProjectName)
checkErr(t, ParseProjectName, "meow/miau", ErrInvalidProjectName)
checkErr(t, ParseProjectName, "meow\\miau", ErrInvalidProjectName)
checkErr(t, ParseProjectName, "meow%miau", ErrInvalidProjectName)
checkErr(t, ParseProjectName, "meow*", ErrInvalidProjectName)
checkErr(t, ParseProjectName, "foo@10@11", ErrInvalidProjectName)
checkErr(t, ParseProjectName, "foo@awa", ErrInvalidProjectName)
checkErr(t, ParseProjectName, "foo@0", ErrInvalidProjectName)
checkErr(t, ParseProjectName, "foo@-10", ErrInvalidProjectName)
checkErr(t, ParseProject, ".foo", ErrInvalidProjectName)
checkErr(t, ParseProject, "meow/miau", ErrInvalidProjectName)
checkErr(t, ParseProject, "meow\\miau", ErrInvalidProjectName)
checkErr(t, ParseProject, "meow%miau", ErrInvalidProjectName)
checkErr(t, ParseProject, "meow*", ErrInvalidProjectName)
checkErr(t, ParseProject, "foo@10@11", ErrInvalidProjectName)
checkErr(t, ParseProject, "foo@awa", ErrInvalidProjectName)
checkErr(t, ParseProject, "foo@0", ErrInvalidProjectName)
checkErr(t, ParseProject, "foo@-10", ErrInvalidProjectName)
}
func TestParseProjectURL(t *testing.T) {
+60
View File
@@ -0,0 +1,60 @@
package names
import (
"cmp"
"fmt"
"strings"
)
var ErrInvalidSiteName = NameError{"invalid site name"}
const siteSeparator = "/"
// Name of a git-pages site. Consists of a Domain and a Project name.
type Site struct {
Domain Domain
Project Project
}
func (s Site) IsValid() bool {
return s.Domain.IsValid() && s.Project.IsValid()
}
// MustValid ensures the domain name is valid by panicking if it is not.
func (s Site) MustValid() Site {
if !s.IsValid() {
panic("nil site name")
}
return s
}
// String returns the Domain and Project names separated by a "/".
func (s Site) String() string {
return fmt.Sprint(s.Domain.MustValid(), siteSeparator, s.Project.MustValid())
}
// CompareSite orders sites first by domain, then by project.
func CompareSite(a Site, b Site) int {
if n := cmp.Compare(a.Domain.name, b.Domain.name); n != 0 {
return n
} else {
return cmp.Compare(a.Project.name, b.Project.name)
}
}
// ParseSite parses the format returned by String.
func ParseSite(value string) (s Site, err error) {
if domainPart, projectPart, ok := strings.Cut(value, siteSeparator); ok {
if s.Domain, err = ParseDomain(domainPart, ParseDomainRelaxed); err != nil {
s = Site{}
return
}
if s.Project, err = ParseProject(projectPart); err != nil {
s = Site{}
return
}
} else {
err = ErrInvalidSiteName
}
return
}
+29
View File
@@ -0,0 +1,29 @@
package names
import (
"testing"
)
func TestParseSite(t *testing.T) {
checkOk(t, ParseSite, "example.org/foo", Site{Domain{"example.org"}, Project{"foo"}})
checkOk(t, ParseSite, "example.org/.index", Site{Domain{"example.org"}, Project{".index"}})
checkOk(t, ParseSite, "foo_bar.example.org/.index", Site{Domain{"foo_bar.example.org"}, Project{".index"}})
checkErr(t, ParseSite, "example.org", ErrInvalidSiteName)
checkErr(t, ParseSite, "example.org/a*", ErrInvalidProjectName)
checkErr(t, ParseSite, "_foo.example.org/.index", ErrInvalidDomainName)
}
func TestSiteProperties(t *testing.T) {
checkEq(t, Site{}.IsValid(), false)
checkEq(t, Site{Domain{"example.org"}, Project{}}.IsValid(), false)
checkEq(t, Site{Domain{}, Project{"foo"}}.IsValid(), false)
checkEq(t, Site{Domain{"example.org"}, Project{"foo"}}.IsValid(), true)
checkPanic(t, func() Site { return Site{}.MustValid() })
checkEq(t, Site{Domain{"example.org"}, Project{"foo"}}.MustValid().Project, Project{"foo"})
checkPanic(t, func() string { return Site{}.String() })
checkEq(t, Site{Domain{"example.org"}, Project{"foo"}}.String(), "example.org/foo")
checkEq(t, Site{Domain{"example.org"}, IndexProject}.String(), "example.org/.index")
}
+48
View File
@@ -0,0 +1,48 @@
package names
import (
"errors"
"testing"
)
func must[T any](value T, err error) T {
if err != nil {
panic(err)
}
return value
}
func checkEq[T comparable](t *testing.T, value T, expect T) {
t.Helper()
if value != expect {
t.Errorf("unexpected: %v != %v", value, expect)
}
}
func checkPanic[T any](t *testing.T, f func() T) {
defer func() {
if r := recover(); r == nil {
t.Errorf("function did not panic")
}
}()
f()
}
func checkOk[T any, U comparable](t *testing.T, parse func(T) (U, error), value T, expect U) {
t.Helper()
actual, err := parse(value)
if err != nil {
t.Errorf("unexpected error: %v", err)
} else if actual != expect {
t.Errorf("unexpected result: %v, expected %v", actual, expect)
}
}
func checkErr[T any, U any](t *testing.T, parse func(T) (U, error), value T, expect error) {
t.Helper()
_, err := parse(value)
if !errors.Is(err, expect) {
t.Errorf("unexpected error: %v, expected %v", err, expect)
}
}
+17 -16
View File
@@ -17,6 +17,7 @@ import (
slogmulti "github.com/samber/slog-multi"
"codeberg.org/git-pages/git-pages/src/names"
syslog "codeberg.org/git-pages/go-slog-syslog"
"github.com/prometheus/client_golang/prometheus"
@@ -259,15 +260,15 @@ func (backend *observedBackend) EnumerateBlobs(ctx context.Context) iter.Seq2[Bl
}
func (backend *observedBackend) GetManifest(
ctx context.Context, name string, opts GetManifestOptions,
ctx context.Context, site names.Site, opts GetManifestOptions,
) (
manifest *Manifest, metadata ManifestMetadata, err error,
) {
span, ctx := ObserveFunction(ctx, "GetManifest",
"manifest.name", name,
"manifest.name", site,
"manifest.bypass_cache", opts.BypassCache,
)
if manifest, metadata, err = backend.inner.GetManifest(ctx, name, opts); err == nil {
if manifest, metadata, err = backend.inner.GetManifest(ctx, site, opts); err == nil {
manifestsRetrievedCount.Inc()
}
span.Finish()
@@ -285,23 +286,23 @@ func (backend *observedBackend) HasAtomicCAS(ctx context.Context) bool {
return backend.inner.HasAtomicCAS(ctx)
}
func (backend *observedBackend) CommitManifest(ctx context.Context, name string, manifest *Manifest, opts ModifyManifestOptions) (err error) {
span, ctx := ObserveFunction(ctx, "CommitManifest", "manifest.name", name)
err = backend.inner.CommitManifest(ctx, name, manifest, opts)
func (backend *observedBackend) CommitManifest(ctx context.Context, site names.Site, manifest *Manifest, opts ModifyManifestOptions) (err error) {
span, ctx := ObserveFunction(ctx, "CommitManifest", "manifest.name", site)
err = backend.inner.CommitManifest(ctx, site, manifest, opts)
span.Finish()
return
}
func (backend *observedBackend) DeleteManifest(ctx context.Context, name string, opts ModifyManifestOptions) (err error) {
span, ctx := ObserveFunction(ctx, "DeleteManifest", "manifest.name", name)
err = backend.inner.DeleteManifest(ctx, name, opts)
func (backend *observedBackend) DeleteManifest(ctx context.Context, site names.Site, opts ModifyManifestOptions) (err error) {
span, ctx := ObserveFunction(ctx, "DeleteManifest", "manifest.name", site)
err = backend.inner.DeleteManifest(ctx, site, opts)
span.Finish()
return
}
func (backend *observedBackend) ExpireManifest(ctx context.Context, name string) (err error) {
span, ctx := ObserveFunction(ctx, "ExpireManifest", "manifest.name", name)
err = backend.inner.ExpireManifest(ctx, name)
func (backend *observedBackend) ExpireManifest(ctx context.Context, site names.Site) (err error) {
span, ctx := ObserveFunction(ctx, "ExpireManifest", "manifest.name", site)
err = backend.inner.ExpireManifest(ctx, site)
span.Finish()
return
}
@@ -330,28 +331,28 @@ func (backend *observedBackend) GetAllManifests(ctx context.Context) iter.Seq2[t
}
}
func (backend *observedBackend) CheckDomain(ctx context.Context, domain string) (found bool, err error) {
func (backend *observedBackend) CheckDomain(ctx context.Context, domain names.Domain) (found bool, err error) {
span, ctx := ObserveFunction(ctx, "CheckDomain", "domain.name", domain)
found, err = backend.inner.CheckDomain(ctx, domain)
span.Finish()
return
}
func (backend *observedBackend) CreateDomain(ctx context.Context, domain string) (err error) {
func (backend *observedBackend) CreateDomain(ctx context.Context, domain names.Domain) (err error) {
span, ctx := ObserveFunction(ctx, "CreateDomain", "domain.name", domain)
err = backend.inner.CreateDomain(ctx, domain)
span.Finish()
return
}
func (backend *observedBackend) FreezeDomain(ctx context.Context, domain string) (err error) {
func (backend *observedBackend) FreezeDomain(ctx context.Context, domain names.Domain) (err error) {
span, ctx := ObserveFunction(ctx, "FreezeDomain", "domain.name", domain)
err = backend.inner.FreezeDomain(ctx, domain)
span.Finish()
return
}
func (backend *observedBackend) UnfreezeDomain(ctx context.Context, domain string) (err error) {
func (backend *observedBackend) UnfreezeDomain(ctx context.Context, domain names.Domain) (err error) {
span, ctx := ObserveFunction(ctx, "UnfreezeDomain", "domain.name", domain)
err = backend.inner.UnfreezeDomain(ctx, domain)
span.Finish()
+56 -49
View File
@@ -9,6 +9,7 @@ import (
"fmt"
"io"
"maps"
"net"
"net/http"
"net/url"
"os"
@@ -77,26 +78,33 @@ func copyForgeAuthToPrincipal(principal *Principal, auth *Authorization) {
}
}
func normalizeHost(host string) string {
return strings.ToLower(host)
}
func makeWebRoot(host string, projectName string) string {
return path.Join(normalizeHost(host), projectName)
}
func getWebRoot(r *http.Request) (string, error) {
host, err := GetHost(r)
func GetDomain(r *http.Request) (domain names.Domain, err error) {
host, _, err := net.SplitHostPort(r.Host)
if err != nil {
return "", err
host = r.Host
}
projectName, err := names.ParseProjectURL(r.URL)
if err != nil {
return "", err
mode := names.ParseDomainStrict
if config.Feature("relaxed-idna") {
mode = names.ParseDomainRelaxed
}
return makeWebRoot(host, projectName.String()), nil
return names.ParseDomain(host, mode)
}
func GetSite(r *http.Request) (site names.Site, err error) {
domain, err := GetDomain(r)
if err != nil {
return
}
project, err := names.ParseProjectURL(r.URL)
if err != nil {
return
}
site = names.Site{Domain: domain, Project: project}
return
}
func writeRedirect(w http.ResponseWriter, code int, path string) {
@@ -126,11 +134,10 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
bypassCache := cacheControl.NoCache || cacheControl.MaxAge == 0
host, err := GetHost(r)
domain, err := GetDomain(r)
if err != nil {
return err
}
host = normalizeHost(host)
type indexManifestResult struct {
manifest *Manifest
@@ -139,14 +146,13 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
}
indexManifestCh := make(chan indexManifestResult, 1)
go func() {
webRoot := makeWebRoot(host, ".index")
if existenceCache.CheckSite(r.Context(), webRoot).IsImpossible() {
site := names.Site{Domain: domain, Project: names.IndexProject}
if existenceCache.CheckSite(r.Context(), site).IsImpossible() {
close(indexManifestCh)
return
}
manifest, metadata, err := backend.GetManifest(
r.Context(), webRoot,
GetManifestOptions{BypassCache: bypassCache},
r.Context(), site, GetManifestOptions{BypassCache: bypassCache},
)
indexManifestCh <- (indexManifestResult{manifest, metadata, err})
}()
@@ -155,24 +161,25 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
sitePath = strings.TrimPrefix(r.URL.Path, "/")
projectPathPrefix := "/" // 2xx and 4xx redirect target URLs must start with this prefix
if projectName, projectPath, hasProjectSlash := strings.Cut(sitePath, "/"); projectName != "" {
webRoot := makeWebRoot(host, projectName)
if _, err = names.ParseProjectName(projectName); err == nil &&
existenceCache.CheckSite(r.Context(), webRoot).IsPossible() {
var projectManifest *Manifest
var projectMetadata ManifestMetadata
projectManifest, projectMetadata, err = backend.GetManifest(
r.Context(), webRoot,
GetManifestOptions{BypassCache: bypassCache},
)
if err == nil {
if !hasProjectSlash {
newURL := *r.URL
newURL.Path += "/"
writeRedirect(w, http.StatusFound, newURL.String())
return nil
var project names.Project
if project, err = names.ParseProject(projectName); err == nil {
site := names.Site{Domain: domain, Project: project}
if existenceCache.CheckSite(r.Context(), site).IsPossible() {
var projectManifest *Manifest
var projectMetadata ManifestMetadata
projectManifest, projectMetadata, err = backend.GetManifest(
r.Context(), site, GetManifestOptions{BypassCache: bypassCache},
)
if err == nil {
if !hasProjectSlash {
newURL := *r.URL
newURL.Path += "/"
writeRedirect(w, http.StatusFound, newURL.String())
return nil
}
sitePath, manifest, metadata = projectPath, projectManifest, projectMetadata
projectPathPrefix = fmt.Sprintf("/%s/", projectName)
}
sitePath, manifest, metadata = projectPath, projectManifest, projectMetadata
projectPathPrefix = fmt.Sprintf("/%s/", projectName)
}
}
}
@@ -181,7 +188,7 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
manifest, metadata, err = result.manifest, result.metadata, result.err
if manifest == nil && (err == nil || errors.Is(err, ErrObjectNotFound)) {
if fallback != nil {
logc.Printf(r.Context(), "fallback: %s via %s", host, config.Fallback.ProxyTo)
logc.Printf(r.Context(), "fallback: %s via %s", domain, config.Fallback.ProxyTo)
fallback.ServeHTTP(w, r)
return nil
} else {
@@ -552,7 +559,7 @@ func putPage(w http.ResponseWriter, r *http.Request) error {
}
}
webRoot, err := getWebRoot(r)
site, err := GetSite(r)
if err != nil {
return err
}
@@ -596,7 +603,7 @@ func putPage(w http.ResponseWriter, r *http.Request) error {
return nil
}
result = UpdateFromRepository(ctx, webRoot, repoURL, branch, opts)
result = UpdateFromRepository(ctx, site, repoURL, branch, opts)
default:
auth, err := AuthorizeUpdateFromArchive(r)
@@ -620,7 +627,7 @@ func putPage(w http.ResponseWriter, r *http.Request) error {
// request body contains archive
reader := http.MaxBytesReader(w, r.Body, int64(config.Limits.MaxSiteSize.Bytes()))
result = UpdateFromArchive(ctx, webRoot, repoURL, contentType, reader, opts)
result = UpdateFromArchive(ctx, site, repoURL, contentType, reader, opts)
}
return reportUpdateResult(w, r, result)
@@ -636,7 +643,7 @@ func patchPage(w http.ResponseWriter, r *http.Request) error {
}
}
webRoot, err := getWebRoot(r)
site, err := GetSite(r)
if err != nil {
return err
}
@@ -694,7 +701,7 @@ func patchPage(w http.ResponseWriter, r *http.Request) error {
contentType := getMediaType(r.Header.Get("Content-Type"))
reader := http.MaxBytesReader(w, r.Body, int64(config.Limits.MaxSiteSize.Bytes()))
result := PartialUpdateFromArchive(ctx, webRoot, contentType, reader, parents, opts)
result := PartialUpdateFromArchive(ctx, site, contentType, reader, parents, opts)
return reportUpdateResult(w, r, result)
}
@@ -777,7 +784,7 @@ func reportUpdateResult(w http.ResponseWriter, r *http.Request, result UpdateRes
}
func deletePage(w http.ResponseWriter, r *http.Request) error {
webRoot, err := getWebRoot(r)
site, err := GetSite(r)
if err != nil {
return err
}
@@ -794,7 +801,7 @@ func deletePage(w http.ResponseWriter, r *http.Request) error {
return nil
}
if err = backend.DeleteManifest(r.Context(), webRoot, ModifyManifestOptions{}); err != nil {
if err = backend.DeleteManifest(r.Context(), site, ModifyManifestOptions{}); err != nil {
w.WriteHeader(http.StatusInternalServerError)
fmt.Fprintln(w, err)
} else {
@@ -811,7 +818,7 @@ func postPage(w http.ResponseWriter, r *http.Request) error {
requestTimeout := 3 * time.Second
requestTimer := time.NewTimer(requestTimeout)
webRoot, err := getWebRoot(r)
site, err := GetSite(r)
if err != nil {
return err
}
@@ -898,7 +905,7 @@ func postPage(w http.ResponseWriter, r *http.Request) error {
principal := GetPrincipal(r.Context())
principal.RepoUrl = &repoURL
result := UpdateFromRepository(ctx, webRoot, repoURL, auth.branch, UpdateOptions{})
result := UpdateFromRepository(ctx, site, repoURL, auth.branch, UpdateOptions{})
resultChan <- result
observeSiteUpdate("webhook", &result)
}(context.WithoutCancel(r.Context()))
@@ -993,7 +1000,7 @@ func ServePages(w http.ResponseWriter, r *http.Request) {
err = fmt.Errorf("method %s not allowed", r.Method)
}
if err != nil {
if errors.Is(err, names.ErrInvalidProjectURL) {
if _, ok := errors.AsType[names.NameError](err); ok {
http.Error(w, prettyErrMsg(err), http.StatusBadRequest)
}
var authErr AuthError
-55
View File
@@ -1,55 +0,0 @@
package git_pages
import (
"net/http"
"strings"
"testing"
)
func checkHost(t *testing.T, host string, expectOk string, expectErr string) {
host, err := GetHost(&http.Request{Host: host})
if expectErr != "" {
if err == nil || !strings.HasPrefix(err.Error(), expectErr) {
t.Errorf("%s: expect err %s, got err %s", host, expectErr, err)
}
}
if expectOk != "" {
if err != nil {
t.Errorf("%s: expect ok %s, got err %s", host, expectOk, err)
} else if host != expectOk {
t.Errorf("%s: expect ok %s, got ok %s", host, expectOk, host)
}
}
}
func TestHelloName(t *testing.T) {
config = &Config{Features: []string{}}
checkHost(t, "foo.bar", "foo.bar", "")
checkHost(t, "foo-baz.bar", "foo-baz.bar", "")
checkHost(t, "foo--baz.bar", "foo--baz.bar", "")
checkHost(t, "foo.bar.", "foo.bar", "")
checkHost(t, ".foo.bar", "", "reserved host name")
checkHost(t, "..foo.bar", "", "reserved host name")
checkHost(t, "ß.bar", "xn--zca.bar", "")
checkHost(t, "xn--zca.bar", "xn--zca.bar", "")
checkHost(t, "foo-.bar", "", "malformed host name")
checkHost(t, "-foo.bar", "", "malformed host name")
checkHost(t, "foo_.bar", "", "malformed host name")
checkHost(t, "_foo.bar", "", "malformed host name")
checkHost(t, "foo_baz.bar", "", "malformed host name")
checkHost(t, "foo__baz.bar", "", "malformed host name")
checkHost(t, "*.foo.bar", "", "malformed host name")
config = &Config{Features: []string{"relaxed-idna"}}
checkHost(t, "foo-.bar", "", "malformed host name")
checkHost(t, "-foo.bar", "", "malformed host name")
checkHost(t, "foo_.bar", "foo_.bar", "")
checkHost(t, "_foo.bar", "", "reserved host name")
checkHost(t, "foo_baz.bar", "foo_baz.bar", "")
checkHost(t, "foo__baz.bar", "foo__baz.bar", "")
checkHost(t, "*.foo.bar", "", "malformed host name")
}
+14 -12
View File
@@ -9,15 +9,19 @@ import (
"net"
"net/http"
"slices"
"strings"
"codeberg.org/git-pages/git-pages/src/names"
)
func ServeProxy(w http.ResponseWriter, r *http.Request) {
domain := r.URL.Query().Get("domain")
if domain == "" {
domainParam := r.URL.Query().Get("domain")
if domainParam == "" {
proxyListDomains(w, r)
} else {
} else if domain, err := names.ParseDomain(domainParam, names.ParseDomainRelaxed); err == nil {
proxyCheckDomain(w, r, domain)
} else {
w.WriteHeader(http.StatusBadRequest)
fmt.Fprintln(w, err)
}
}
@@ -28,28 +32,26 @@ func proxyListDomains(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
fmt.Fprintf(w, "%s", err)
}
domain, _, _ := strings.Cut(metadata.Name, "/")
domainSet[domain] = Unit
domainSet[metadata.Name.Domain.String()] = Unit
}
w.Header().Add("Content-Type", "application/json")
json.NewEncoder(w).Encode(slices.Collect(maps.Keys(domainSet)))
}
func proxyCheckDomain(w http.ResponseWriter, r *http.Request, domain string) {
func proxyCheckDomain(w http.ResponseWriter, r *http.Request, domain names.Domain) {
var err error
// Save the backend some effort from queries that are essentially guaranteed to fail.
// While TLS certificates may be provisionsed for IP addresses under special circumstances[^1],
// this isn't really what git-pages is designed for, and object store accesses can cost money.
// [^1]: https://letsencrypt.org/2025/07/01/issuing-our-first-ip-address-certificate
if ip := net.ParseIP(domain); ip != nil {
if ip := net.ParseIP(domain.String()); ip != nil {
logc.Println(r.Context(), "proxy:", domain, 404, "(bare IP)")
w.WriteHeader(http.StatusNotFound)
return
}
var err error
domain = strings.ToLower(domain)
// Run a cheap check as to whether we might be serving the domain.
found := existenceCache.CheckDomain(r.Context(), domain).IsPossible()
@@ -65,7 +67,7 @@ func proxyCheckDomain(w http.ResponseWriter, r *http.Request, domain string) {
// Pages v2, which would under some circumstances return certificates with subjectAltName
// not valid for the SNI. Go's TLS stack makes `tls.Dial` return an error for these,
// thankfully making it unnecessary to examine X.509 certificates manually here.)
found, err = tryDialWithSNI(r.Context(), domain)
found, err = tryDialWithSNI(r.Context(), domain.String())
if err != nil {
logc.Printf(r.Context(), "proxy err: check SNI: %s\n", err)
}
+4 -3
View File
@@ -9,6 +9,7 @@ import (
"slices"
"strings"
"codeberg.org/git-pages/git-pages/src/names"
"codeberg.org/git-pages/go-redirects"
"google.golang.org/protobuf/proto"
)
@@ -43,7 +44,7 @@ func Is3xxHTTPStatus(status int) bool {
return status >= 300 && status <= 399
}
func validateRedirectRule(project string, rule *redirects.Rule) error {
func validateRedirectRule(project names.Project, rule *redirects.Rule) error {
if len(rule.Params) > 0 {
return fmt.Errorf("rules with parameters are not supported")
}
@@ -81,7 +82,7 @@ func validateRedirectRule(project string, rule *redirects.Rule) error {
// (which was the old, incorrect interpretation). This broke every SPA on git-pages that was
// deployed to a non-index project. Unfortunately, we can't do much more than diagnosing
// the problematic pattern at deploy time.
if rule.Status == http.StatusOK && project != ".index" && toURL.Host == "" &&
if rule.Status == http.StatusOK && !project.IsIndex() && toURL.Host == "" &&
!strings.HasPrefix(toURL.Path, fmt.Sprintf("/%s/", project)) {
return fmt.Errorf("'to' URL must start with '/%s/' for a 200 status rule", project)
}
@@ -89,7 +90,7 @@ func validateRedirectRule(project string, rule *redirects.Rule) error {
}
// Parses redirects file and injects rules into the manifest.
func ProcessRedirectsFile(ctx context.Context, project string, manifest *Manifest) error {
func ProcessRedirectsFile(ctx context.Context, project names.Project, manifest *Manifest) error {
redirectsEntry := manifest.Contents[RedirectsFileName]
delete(manifest.Contents, RedirectsFileName)
if redirectsEntry == nil {
+3 -2
View File
@@ -10,17 +10,18 @@ import (
"strings"
"time"
"codeberg.org/git-pages/git-pages/src/names"
"google.golang.org/protobuf/proto"
timestamppb "google.golang.org/protobuf/types/known/timestamppb"
)
func createPlaceholderManifest(ctx context.Context, name string) *Manifest {
func createPlaceholderManifest(ctx context.Context, site names.Site) *Manifest {
manifest := NewManifest()
manifest.Corrupted = proto.Bool(true)
AddFile(manifest, "index.html",
[]byte("<!DOCTYPE html>"+
"<h1>Data corruption</h1>"+
"<p>The site <code>"+name+"</code> has suffered data corruption and must be "+
"<p>The site <code>"+site.String()+"</code> has suffered data corruption and must be "+
"re-uploaded. Contact the server administrator for more information.</p>"),
)
manifest.Redirects = []*RedirectRule{
+33 -34
View File
@@ -5,9 +5,9 @@ import (
"errors"
"fmt"
"io"
"strings"
"time"
"codeberg.org/git-pages/git-pages/src/names"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/timestamppb"
)
@@ -52,7 +52,7 @@ type UpdateResult struct {
var errExpireExistingSite = fmt.Errorf("cannot expire an existing site")
func Update(
ctx context.Context, webRoot string, oldManifest, newManifest *Manifest,
ctx context.Context, site names.Site, oldManifest, newManifest *Manifest,
opts ModifyManifestOptions,
) UpdateResult {
var err error
@@ -62,7 +62,7 @@ func Update(
if oldManifest != nil && oldManifest.ExpiresAt == nil && newManifest.ExpiresAt != nil {
err = errExpireExistingSite
} else if IsManifestEmpty(newManifest) {
storedManifest, err = newManifest, backend.DeleteManifest(ctx, webRoot, opts)
storedManifest, err = newManifest, backend.DeleteManifest(ctx, site, opts)
if err == nil {
if oldManifest == nil {
outcome = UpdateNoChange
@@ -70,12 +70,11 @@ func Update(
outcome = UpdateDeleted
}
}
} else if err = PrepareManifest(ctx, webRoot, newManifest); err == nil {
storedManifest, err = StoreManifest(ctx, webRoot, newManifest, opts)
} else if err = PrepareManifest(ctx, site, newManifest); err == nil {
storedManifest, err = StoreManifest(ctx, site, newManifest, opts)
if err == nil {
domain, _, _ := strings.Cut(webRoot, "/")
err = backend.CreateDomain(ctx, domain)
existenceCache.AddSite(ctx, webRoot)
err = backend.CreateDomain(ctx, site.Domain)
existenceCache.AddSite(ctx, site)
}
if err == nil {
if oldManifest == nil {
@@ -101,15 +100,15 @@ func Update(
status = "unchanged"
}
if newManifest.ExpiresAt != nil {
logc.Printf(ctx, "expire %s: at %s", webRoot, newManifest.ExpiresAt.AsTime())
logc.Printf(ctx, "expire %s: at %s", site, newManifest.ExpiresAt.AsTime())
}
if storedManifest.Commit != nil {
logc.Printf(ctx, "update %s ok: %s %s", webRoot, *storedManifest.Commit, status)
logc.Printf(ctx, "update %s ok: %s %s", site, *storedManifest.Commit, status)
} else {
logc.Printf(ctx, "update %s ok: %s", webRoot, status)
logc.Printf(ctx, "update %s ok: %s", site, status)
}
} else {
logc.Printf(ctx, "update %s err: %s", webRoot, err)
logc.Printf(ctx, "update %s err: %s", site, err)
}
return UpdateResult{outcome, storedManifest, err}
@@ -117,7 +116,7 @@ func Update(
func UpdateFromRepository(
ctx context.Context,
webRoot string,
site names.Site,
repoURL string,
branch string,
opts UpdateOptions,
@@ -126,11 +125,11 @@ func UpdateFromRepository(
defer span.Finish()
defer observeUpdateResult(result)
logc.Printf(ctx, "update %s: %s %s\n", webRoot, repoURL, branch)
logc.Printf(ctx, "update %s: %s %s\n", site, repoURL, branch)
oldManifest, _, err := backend.GetManifest(ctx, webRoot, GetManifestOptions{})
oldManifest, _, err := backend.GetManifest(ctx, site, GetManifestOptions{})
if err != nil && !errors.Is(err, ErrObjectNotFound) {
logc.Printf(ctx, "update %s err: %s", webRoot, err)
logc.Printf(ctx, "update %s err: %s", site, err)
result = UpdateResult{UpdateError, nil, err}
return
}
@@ -142,7 +141,7 @@ func UpdateFromRepository(
result = UpdateResult{UpdateError, nil, err}
} else {
opts.Apply(newManifest)
result = Update(ctx, webRoot, oldManifest, newManifest, ModifyManifestOptions{})
result = Update(ctx, site, oldManifest, newManifest, ModifyManifestOptions{})
}
return
@@ -152,7 +151,7 @@ var errArchiveFormat = errors.New("unsupported archive format")
func UpdateFromArchive(
ctx context.Context,
webRoot string,
site names.Site,
repoURL string,
contentType string,
reader io.Reader,
@@ -163,9 +162,9 @@ func UpdateFromArchive(
defer span.Finish()
defer observeUpdateResult(result)
oldManifest, _, err := backend.GetManifest(ctx, webRoot, GetManifestOptions{})
oldManifest, _, err := backend.GetManifest(ctx, site, GetManifestOptions{})
if err != nil && !errors.Is(err, ErrObjectNotFound) {
logc.Printf(ctx, "update %s err: %s", webRoot, err)
logc.Printf(ctx, "update %s err: %s", site, err)
result = UpdateResult{UpdateError, nil, err}
return
}
@@ -177,37 +176,37 @@ func UpdateFromArchive(
var newManifest *Manifest
switch contentType {
case "application/x-tar":
logc.Printf(ctx, "update %s: (tar)", webRoot)
logc.Printf(ctx, "update %s: (tar)", site)
newManifest, err = extractTar(ctx, reader) // yellow?
case "application/x-tar+gzip":
logc.Printf(ctx, "update %s: (tar.gz)", webRoot)
logc.Printf(ctx, "update %s: (tar.gz)", site)
newManifest, err = ExtractGzip(ctx, reader, extractTar) // definitely yellow.
case "application/x-tar+zstd":
logc.Printf(ctx, "update %s: (tar.zst)", webRoot)
logc.Printf(ctx, "update %s: (tar.zst)", site)
newManifest, err = ExtractZstd(ctx, reader, extractTar)
case "application/zip":
logc.Printf(ctx, "update %s: (zip)", webRoot)
logc.Printf(ctx, "update %s: (zip)", site)
newManifest, err = ExtractZip(ctx, reader, oldManifest)
default:
err = errArchiveFormat
}
if err != nil {
logc.Printf(ctx, "update %s err: %s", webRoot, err)
logc.Printf(ctx, "update %s err: %s", site, err)
result = UpdateResult{UpdateError, nil, err}
} else {
if repoURL != "" {
newManifest.RepoUrl = &repoURL
}
opts.Apply(newManifest)
result = Update(ctx, webRoot, oldManifest, newManifest, ModifyManifestOptions{})
result = Update(ctx, site, oldManifest, newManifest, ModifyManifestOptions{})
}
return
}
func PartialUpdateFromArchive(
ctx context.Context,
webRoot string,
site names.Site,
contentType string,
reader io.Reader,
parents CreateParentsMode,
@@ -220,10 +219,10 @@ func PartialUpdateFromArchive(
// Here the old manifest is used both as a substrate to which a patch is applied, as well
// as a "load linked" operation for a future "store conditional" update which, taken together,
// create an atomic compare-and-swap operation.
oldManifest, oldMetadata, err := backend.GetManifest(ctx, webRoot,
oldManifest, oldMetadata, err := backend.GetManifest(ctx, site,
GetManifestOptions{BypassCache: true})
if err != nil {
logc.Printf(ctx, "patch %s err: %s", webRoot, err)
logc.Printf(ctx, "patch %s err: %s", site, err)
result = UpdateResult{UpdateError, nil, err}
return
}
@@ -246,24 +245,24 @@ func PartialUpdateFromArchive(
var newManifest *Manifest
switch contentType {
case "application/x-tar":
logc.Printf(ctx, "patch %s: (tar)", webRoot)
logc.Printf(ctx, "patch %s: (tar)", site)
newManifest, err = applyTarPatch(ctx, reader)
case "application/x-tar+gzip":
logc.Printf(ctx, "patch %s: (tar.gz)", webRoot)
logc.Printf(ctx, "patch %s: (tar.gz)", site)
newManifest, err = ExtractGzip(ctx, reader, applyTarPatch)
case "application/x-tar+zstd":
logc.Printf(ctx, "patch %s: (tar.zst)", webRoot)
logc.Printf(ctx, "patch %s: (tar.zst)", site)
newManifest, err = ExtractZstd(ctx, reader, applyTarPatch)
default:
err = errArchiveFormat
}
if err != nil {
logc.Printf(ctx, "patch %s err: %s", webRoot, err)
logc.Printf(ctx, "patch %s err: %s", site, err)
result = UpdateResult{UpdateError, nil, err}
} else {
opts.Apply(newManifest)
result = Update(ctx, webRoot, oldManifest, newManifest,
result = Update(ctx, site, oldManifest, newManifest,
ModifyManifestOptions{
IfUnmodifiedSince: oldMetadata.LastModified,
IfMatch: oldMetadata.ETag,
+7
View File
@@ -19,6 +19,13 @@ func (t tuple[A, B]) Splat() (A, B) {
return t.A, t.B
}
func Must[T any](value T, err error) T {
if err != nil {
panic(err)
}
return value
}
type BoundedReader struct {
inner io.Reader
fuel int64
+5 -4
View File
@@ -5,6 +5,7 @@ import (
"slices"
"strings"
"codeberg.org/git-pages/git-pages/src/names"
"github.com/valyala/fasttemplate"
)
@@ -34,7 +35,7 @@ const (
// Returns `subdomain, found` where if `found == true`, `subdomain` contains the part of `host`
// corresponding to the * in the domain pattern.
func (pattern *WildcardPattern) Matches(host string, kind WildcardDomainKind) (string, bool) {
func (pattern *WildcardPattern) Matches(domain names.Domain, kind WildcardDomainKind) (string, bool) {
var suffixParts []string
switch kind {
case WildcardDomainPrimary:
@@ -42,9 +43,9 @@ func (pattern *WildcardPattern) Matches(host string, kind WildcardDomainKind) (s
case WildcardDomainPreview:
suffixParts = pattern.PreviewDomain
case WildcardDomainAny:
if userName, found := pattern.Matches(host, WildcardDomainPrimary); found {
if userName, found := pattern.Matches(domain, WildcardDomainPrimary); found {
return userName, found
} else if userName, found := pattern.Matches(host, WildcardDomainPreview); found {
} else if userName, found := pattern.Matches(domain, WildcardDomainPreview); found {
return userName, found
}
return "", false
@@ -52,7 +53,7 @@ func (pattern *WildcardPattern) Matches(host string, kind WildcardDomainKind) (s
panic("invalid wildcard domain kind")
}
hostParts := strings.Split(host, ".")
hostParts := strings.Split(domain.String(), ".")
if len(suffixParts) == 0 {
return "", false
} else if len(hostParts) != len(suffixParts)+1 {