at-container-registry/pkg/appview/readme/fetcher.go

package readme

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/microcosm-cc/bluemonday"
	"github.com/yuin/goldmark"
	"github.com/yuin/goldmark/extension"
	"github.com/yuin/goldmark/parser"
	"github.com/yuin/goldmark/renderer/html"
)

// Fetcher fetches and renders README content from URLs
type Fetcher struct {
	httpClient *http.Client
	markdown   goldmark.Markdown
	sanitizer  *bluemonday.Policy
}

// NewFetcher creates a new README fetcher
func NewFetcher() *Fetcher {
	// Configure markdown renderer with GitHub-flavored markdown
	md := goldmark.New(
		goldmark.WithExtensions(
			extension.GFM,         // GitHub Flavored Markdown
			extension.Typographer, // Smart quotes, dashes, etc.
		),
		goldmark.WithParserOptions(
			parser.WithAutoHeadingID(), // Auto-generate heading IDs
		),
		goldmark.WithRendererOptions(
			html.WithHardWraps(), // Line breaks create <br>
			html.WithXHTML(),     // XHTML-compliant output
			// html.WithUnsafe(), // Uncomment ONLY if you want to allow raw HTML in markdown (not recommended)
		),
	)

	// Configure HTML sanitizer as a safety net
	// This catches any HTML that makes it through (if WithUnsafe() is enabled)
	sanitizer := bluemonday.UGCPolicy()
	// Allow additional attributes for better markdown rendering
	sanitizer.AllowAttrs("class").Globally()
	sanitizer.AllowAttrs("id").Globally()
	sanitizer.AllowAttrs("align").OnElements("img", "div", "p", "span")

	return &Fetcher{
		httpClient: &http.Client{
			Timeout: 10 * time.Second,
			CheckRedirect: func(req *http.Request, via []*http.Request) error {
				// Allow up to 5 redirects
				if len(via) >= 5 {
					return fmt.Errorf("too many redirects")
				}
				return nil
			},
		},
		markdown:  md,
		sanitizer: sanitizer,
	}
}

// FetchAndRender fetches a README from a URL and renders it as HTML
// Returns the rendered HTML and any error
func (f *Fetcher) FetchAndRender(ctx context.Context, readmeURL string) (string, error) {
	// Validate URL
	if readmeURL == "" {
		return "", fmt.Errorf("empty README URL")
	}

	parsedURL, err := url.Parse(readmeURL)
	if err != nil {
		return "", fmt.Errorf("invalid README URL: %w", err)
	}

	// Only allow HTTP/HTTPS
	if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" {
		return "", fmt.Errorf("invalid URL scheme: %s", parsedURL.Scheme)
	}

	// Fetch content
	content, baseURL, err := f.fetchContent(ctx, readmeURL)
	if err != nil {
		return "", err
	}

	// Render markdown to HTML
	html, err := f.renderMarkdown(content, baseURL)
	if err != nil {
		return "", fmt.Errorf("failed to render markdown: %w", err)
	}

	return html, nil
}

// fetchContent fetches the raw content from a URL
func (f *Fetcher) fetchContent(ctx context.Context, urlStr string) ([]byte, string, error) {
	req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil)
	if err != nil {
		return nil, "", fmt.Errorf("failed to create request: %w", err)
	}

	// Set user agent
	req.Header.Set("User-Agent", "ATCR-README-Fetcher/1.0")

	resp, err := f.httpClient.Do(req)
	if err != nil {
		return nil, "", fmt.Errorf("failed to fetch URL: %w", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return nil, "", fmt.Errorf("unexpected status code: %d", resp.StatusCode)
	}

	// Limit content size to 1MB
	limitedReader := io.LimitReader(resp.Body, 1*1024*1024)
	content, err := io.ReadAll(limitedReader)
	if err != nil {
		return nil, "", fmt.Errorf("failed to read response body: %w", err)
	}

	// Get base URL for relative link resolution
	baseURL := getBaseURL(resp.Request.URL)

	return content, baseURL, nil
}

// renderMarkdown renders markdown content to sanitized HTML
func (f *Fetcher) renderMarkdown(content []byte, baseURL string) (string, error) {
	var buf bytes.Buffer

	if err := f.markdown.Convert(content, &buf); err != nil {
		return "", err
	}

	// Rewrite relative URLs to absolute
	html := buf.String()
	if baseURL != "" {
		html = rewriteRelativeURLs(html, baseURL)
	}

	// Sanitize HTML
	sanitized := f.sanitizer.Sanitize(html)

	return sanitized, nil
}

// getBaseURL extracts the base URL for relative link resolution
func getBaseURL(u *url.URL) string {
	if u == nil {
		return ""
	}

	// For GitHub raw URLs, convert to blob URL base for relative links
	// e.g., https://raw.githubusercontent.com/user/repo/main/README.md
	//    -> https://github.com/user/repo/blob/main/
	if u.Host == "raw.githubusercontent.com" {
		parts := strings.Split(strings.TrimPrefix(u.Path, "/"), "/")
		if len(parts) >= 3 {
			user := parts[0]
			repo := parts[1]
			branch := parts[2]
			return fmt.Sprintf("https://github.com/%s/%s/blob/%s/", user, repo, branch)
		}
	}

	// For other URLs, use the directory containing the file
	path := u.Path
	lastSlash := strings.LastIndex(path, "/")
	if lastSlash >= 0 {
		path = path[:lastSlash+1]
	}
	return fmt.Sprintf("%s://%s%s", u.Scheme, u.Host, path)
}

// rewriteRelativeURLs converts relative URLs to absolute URLs
func rewriteRelativeURLs(html, baseURL string) string {
	if baseURL == "" {
		return html
	}

	base, err := url.Parse(baseURL)
	if err != nil {
		return html
	}

	// Simple string replacement for common patterns
	// This is a basic implementation - for production, consider using an HTML parser
	html = strings.ReplaceAll(html, `src="./`, fmt.Sprintf(`src="%s`, baseURL))
	html = strings.ReplaceAll(html, `href="./`, fmt.Sprintf(`href="%s`, baseURL))
	html = strings.ReplaceAll(html, `src="../`, fmt.Sprintf(`src="%s../`, baseURL))
	html = strings.ReplaceAll(html, `href="../`, fmt.Sprintf(`href="%s../`, baseURL))

	// Handle root-relative URLs (starting with /)
	if base.Scheme != "" && base.Host != "" {
		root := fmt.Sprintf("%s://%s/", base.Scheme, base.Host)
		// Replace src="/" and href="/" but not src="//" (absolute URLs)
		html = strings.ReplaceAll(html, `src="/`, fmt.Sprintf(`src="%s`, root))
		html = strings.ReplaceAll(html, `href="/`, fmt.Sprintf(`href="%s`, root))
	}

	return html
}