at-container-registry/pkg/appview/readme/fetcher.go

// Package readme provides fetching and rendering of README files from Git hosting platforms.
package readme

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"regexp"
	"strings"
	"time"

	"github.com/microcosm-cc/bluemonday"
	"github.com/yuin/goldmark"
	"github.com/yuin/goldmark/extension"
	"github.com/yuin/goldmark/parser"
	"github.com/yuin/goldmark/renderer/html"
)

// Fetcher fetches and renders README content from URLs
type Fetcher struct {
	httpClient *http.Client
	markdown   goldmark.Markdown
	sanitizer  *bluemonday.Policy
}

// NewFetcher creates a new README fetcher
func NewFetcher() *Fetcher {
	// Configure markdown renderer with GitHub-flavored markdown
	md := goldmark.New(
		goldmark.WithExtensions(
			extension.GFM,         // GitHub Flavored Markdown
			extension.Typographer, // Smart quotes, dashes, etc.
		),
		goldmark.WithParserOptions(
			parser.WithAutoHeadingID(), // Auto-generate heading IDs
		),
		goldmark.WithRendererOptions(
			html.WithHardWraps(), // Line breaks create <br>
			html.WithXHTML(),     // XHTML-compliant output
			// html.WithUnsafe(), // Uncomment ONLY if you want to allow raw HTML in markdown (not recommended)
		),
	)

	// Configure HTML sanitizer as a safety net
	// This catches any HTML that makes it through (if WithUnsafe() is enabled)
	sanitizer := bluemonday.UGCPolicy()
	// Allow additional attributes for better markdown rendering
	sanitizer.AllowAttrs("class").Globally()
	sanitizer.AllowAttrs("id").Globally()
	sanitizer.AllowAttrs("align").OnElements("img", "div", "p", "span")

	return &Fetcher{
		httpClient: &http.Client{
			Timeout: 10 * time.Second,
			CheckRedirect: func(req *http.Request, via []*http.Request) error {
				// Allow up to 5 redirects
				if len(via) >= 5 {
					return fmt.Errorf("too many redirects")
				}
				return nil
			},
		},
		markdown:  md,
		sanitizer: sanitizer,
	}
}

// FetchAndRender fetches a README from a URL and renders it as HTML
// Returns the rendered HTML and any error
func (f *Fetcher) FetchAndRender(ctx context.Context, readmeURL string) (string, error) {
	// Fetch content (includes URL validation, Content-Type check, and HTML detection)
	content, baseURL, err := f.fetchContent(ctx, readmeURL)
	if err != nil {
		return "", err
	}

	// Render markdown to HTML
	html, err := f.renderMarkdown(content, baseURL)
	if err != nil {
		return "", fmt.Errorf("failed to render markdown: %w", err)
	}

	return html, nil
}

// FetchRaw fetches raw README content from a URL without rendering
// Returns raw bytes with Content-Type and HTML validation
// Use this when you need to store the raw markdown (e.g., in PDS records)
func (f *Fetcher) FetchRaw(ctx context.Context, readmeURL string) ([]byte, error) {
	// Fetch content (includes URL validation, Content-Type check, and HTML detection)
	content, _, err := f.fetchContent(ctx, readmeURL)
	if err != nil {
		return nil, err
	}

	return content, nil
}

// fetchContent fetches the raw content from a URL
func (f *Fetcher) fetchContent(ctx context.Context, urlStr string) ([]byte, string, error) {
	// Validate URL
	if urlStr == "" {
		return nil, "", fmt.Errorf("empty README URL")
	}

	parsedURL, err := url.Parse(urlStr)
	if err != nil {
		return nil, "", fmt.Errorf("invalid README URL: %w", err)
	}

	// Only allow HTTP/HTTPS
	if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" {
		return nil, "", fmt.Errorf("invalid URL scheme: %s", parsedURL.Scheme)
	}

	req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil)
	if err != nil {
		return nil, "", fmt.Errorf("failed to create request: %w", err)
	}

	// Set user agent
	req.Header.Set("User-Agent", "ATCR-README-Fetcher/1.0")

	resp, err := f.httpClient.Do(req)
	if err != nil {
		return nil, "", fmt.Errorf("failed to fetch URL: %w", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return nil, "", fmt.Errorf("unexpected status code: %d", resp.StatusCode)
	}

	// Reject HTML content types (catches proper error pages)
	contentType := resp.Header.Get("Content-Type")
	if contentType != "" {
		ct := strings.ToLower(contentType)
		if strings.Contains(ct, "text/html") || strings.Contains(ct, "application/xhtml") {
			return nil, "", fmt.Errorf("unsupported content type: %s (expected markdown or plain text)", contentType)
		}
	}

	// Limit content size to 1MB
	limitedReader := io.LimitReader(resp.Body, 1*1024*1024)
	content, err := io.ReadAll(limitedReader)
	if err != nil {
		return nil, "", fmt.Errorf("failed to read response body: %w", err)
	}

	// Detect HTML content by checking for common markers (catches soft 404s)
	if LooksLikeHTML(content) {
		return nil, "", fmt.Errorf("detected HTML content instead of markdown")
	}

	// Get base URL for relative link resolution
	baseURL := getBaseURL(resp.Request.URL)

	return content, baseURL, nil
}

// LooksLikeHTML checks if content appears to be HTML rather than markdown
// Exported for use by other packages that fetch README content
func LooksLikeHTML(content []byte) bool {
	if len(content) == 0 {
		return false
	}

	// Check first 512 bytes for HTML markers
	checkLen := min(len(content), 512)

	trimmed := strings.TrimSpace(string(content[:checkLen]))
	lower := strings.ToLower(trimmed)

	return strings.HasPrefix(lower, "<!doctype") ||
		strings.HasPrefix(lower, "<html") ||
		strings.HasPrefix(lower, "<?xml")
}

// renderMarkdown renders markdown content to sanitized HTML
func (f *Fetcher) renderMarkdown(content []byte, baseURL string) (string, error) {
	var buf bytes.Buffer

	if err := f.markdown.Convert(content, &buf); err != nil {
		return "", err
	}

	// Rewrite relative URLs to absolute
	html := buf.String()
	if baseURL != "" {
		html = rewriteRelativeURLs(html, baseURL)
	}

	// Sanitize HTML
	sanitized := f.sanitizer.Sanitize(html)

	return sanitized, nil
}

// getBaseURL extracts the base URL for relative link resolution
func getBaseURL(u *url.URL) string {
	if u == nil {
		return ""
	}

	// For GitHub raw URLs, convert to blob URL base for relative links
	// e.g., https://raw.githubusercontent.com/user/repo/main/README.md
	//    -> https://github.com/user/repo/blob/main/
	if u.Host == "raw.githubusercontent.com" {
		parts := strings.Split(strings.TrimPrefix(u.Path, "/"), "/")
		if len(parts) >= 3 {
			user := parts[0]
			repo := parts[1]
			branch := parts[2]
			return fmt.Sprintf("https://github.com/%s/%s/blob/%s/", user, repo, branch)
		}
	}

	// For other URLs, use the directory containing the file
	path := u.Path
	lastSlash := strings.LastIndex(path, "/")
	if lastSlash >= 0 {
		path = path[:lastSlash+1]
	}
	return fmt.Sprintf("%s://%s%s", u.Scheme, u.Host, path)
}

// Is404 returns true if the error indicates a 404 Not Found response
func Is404(err error) bool {
	return err != nil && strings.Contains(err.Error(), "unexpected status code: 404")
}

// RenderMarkdown renders a markdown string to sanitized HTML
// This is used for rendering repo page descriptions stored in the database
func (f *Fetcher) RenderMarkdown(content []byte) (string, error) {
	// Render markdown to HTML (no base URL for repo page descriptions)
	return f.renderMarkdown(content, "")
}

// Regex patterns for matching relative URLs that need rewriting
// These match src="..." or href="..." where the URL is relative (not absolute, not data:, not #anchor)
var (
	// Match src="filename" where filename doesn't start with http://, https://, //, /, #, data:, or mailto:
	relativeSrcPattern = regexp.MustCompile(`src="([^"/:][^"]*)"`)
	// Match href="filename" where filename doesn't start with http://, https://, //, /, #, data:, or mailto:
	relativeHrefPattern = regexp.MustCompile(`href="([^"/:][^"]*)"`)
)

// rewriteRelativeURLs converts relative URLs to absolute URLs
func rewriteRelativeURLs(html, baseURL string) string {
	if baseURL == "" {
		return html
	}

	base, err := url.Parse(baseURL)
	if err != nil {
		return html
	}

	// Handle root-relative URLs (starting with /) first
	// Must be done before bare relative URLs to avoid double-processing
	if base.Scheme != "" && base.Host != "" {
		root := fmt.Sprintf("%s://%s/", base.Scheme, base.Host)
		// Replace src="/" and href="/" but not src="//" (protocol-relative URLs)
		html = strings.ReplaceAll(html, `src="/`, fmt.Sprintf(`src="%s`, root))
		html = strings.ReplaceAll(html, `href="/`, fmt.Sprintf(`href="%s`, root))
	}

	// Handle explicit relative paths (./something and ../something)
	html = strings.ReplaceAll(html, `src="./`, fmt.Sprintf(`src="%s`, baseURL))
	html = strings.ReplaceAll(html, `href="./`, fmt.Sprintf(`href="%s`, baseURL))
	html = strings.ReplaceAll(html, `src="../`, fmt.Sprintf(`src="%s../`, baseURL))
	html = strings.ReplaceAll(html, `href="../`, fmt.Sprintf(`href="%s../`, baseURL))

	// Handle bare relative URLs (e.g., src="image.png" without ./ prefix)
	// Skip URLs that are already absolute (start with http://, https://, or //)
	// Skip anchors (#), data URLs (data:), and mailto links
	html = relativeSrcPattern.ReplaceAllStringFunc(html, func(match string) string {
		// Extract the URL from src="..."
		url := match[5 : len(match)-1] // Remove 'src="' and '"'

		// Skip if already processed or is a special URL type
		if strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") ||
			strings.HasPrefix(url, "//") || strings.HasPrefix(url, "#") ||
			strings.HasPrefix(url, "data:") || strings.HasPrefix(url, "mailto:") {
			return match
		}

		return fmt.Sprintf(`src="%s%s"`, baseURL, url)
	})

	html = relativeHrefPattern.ReplaceAllStringFunc(html, func(match string) string {
		// Extract the URL from href="..."
		url := match[6 : len(match)-1] // Remove 'href="' and '"'

		// Skip if already processed or is a special URL type
		if strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") ||
			strings.HasPrefix(url, "//") || strings.HasPrefix(url, "#") ||
			strings.HasPrefix(url, "data:") || strings.HasPrefix(url, "mailto:") {
			return match
		}

		return fmt.Sprintf(`href="%s%s"`, baseURL, url)
	})

	return html
}