at-container-registry/deploy/upcloud/provision.go

package main

import (
	"bufio"
	"context"
	crypto_rand "crypto/rand"
	"crypto/sha256"
	"encoding/base64"
	"encoding/hex"
	"fmt"
	"net"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/UpCloudLtd/upcloud-go-api/v8/upcloud"
	"github.com/UpCloudLtd/upcloud-go-api/v8/upcloud/request"
	"github.com/UpCloudLtd/upcloud-go-api/v8/upcloud/service"
	"github.com/spf13/cobra"
)

var provisionCmd = &cobra.Command{
	Use:   "provision",
	Short: "Create all infrastructure (servers, network, LB, firewall)",
	RunE: func(cmd *cobra.Command, args []string) error {
		token, _ := cmd.Root().PersistentFlags().GetString("token")
		zone, _ := cmd.Flags().GetString("zone")
		plan, _ := cmd.Flags().GetString("plan")
		scannerPlan, _ := cmd.Flags().GetString("scanner-plan")
		sshKey, _ := cmd.Flags().GetString("ssh-key")
		s3Secret, _ := cmd.Flags().GetString("s3-secret")
		withScanner, _ := cmd.Flags().GetBool("with-scanner")
		withLabeler, _ := cmd.Flags().GetBool("with-labeler")
		withBilling, _ := cmd.Flags().GetBool("with-billing")
		return cmdProvision(token, zone, plan, scannerPlan, sshKey, s3Secret, withScanner, withLabeler, withBilling)
	},
}

func init() {
	provisionCmd.Flags().String("zone", "", "UpCloud zone (interactive picker if omitted)")
	provisionCmd.Flags().String("plan", "", "Server plan for appview and hold (interactive picker if omitted)")
	provisionCmd.Flags().String("scanner-plan", "", fmt.Sprintf("Server plan for the scanner's own server (default %s, or whichever plan offers %d CPU / %d GB RAM / %d GB disk if that name is gone)", defaultScannerPlan, scannerPlanSpec.Cores, scannerPlanSpec.MemoryMB/1024, scannerPlanSpec.DiskGB))
	provisionCmd.Flags().String("ssh-key", "", "Path to SSH public key file (required when creating new servers)")
	provisionCmd.Flags().String("s3-secret", "", "S3 secret access key (for existing object storage)")
	provisionCmd.Flags().Bool("with-scanner", false, "Deploy the vulnerability scanner on its own server")
	provisionCmd.Flags().Bool("with-labeler", false, "Deploy content moderation labeler alongside appview")
	provisionCmd.Flags().Bool("with-billing", false, "Compile Stripe billing into the appview (pkg/billing, `billing` build tag). Off by default.")
	rootCmd.AddCommand(provisionCmd)
}

func cmdProvision(token, zone, plan, scannerPlan, sshKeyPath, s3Secret string, withScanner, withLabeler, withBilling bool) error {
	cfg, err := loadConfig(zone, plan, scannerPlan, sshKeyPath, s3Secret)
	if err != nil {
		return err
	}

	naming := cfg.Naming()

	svc, err := newService(token)
	if err != nil {
		return err
	}

	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
	defer cancel()

	// Load existing state or start fresh
	state, err := loadState()
	if err != nil {
		state = &InfraState{}
	}

	// Use zone from state if not provided via flags
	if cfg.Zone == "" && state.Zone != "" {
		cfg.Zone = state.Zone
	}

	// Only need interactive picker if we still need to create resources
	needsServers := state.Appview.UUID == "" || state.Hold.UUID == ""
	if cfg.Zone == "" || (needsServers && cfg.Plan == "") {
		if err := resolveInteractive(ctx, svc, cfg); err != nil {
			return err
		}
	}

	// The scanner gets its own server, on its own plan. Resolve it only when
	// that server still has to be created; an existing one keeps its plan.
	scannerEnabled := withScanner || state.ScannerEnabled
	needsScannerServer := scannerEnabled && state.Scanner.UUID == ""
	if needsScannerServer {
		if err := resolveScannerPlan(ctx, svc, cfg); err != nil {
			return err
		}
	}

	if state.Zone == "" {
		state.Zone = cfg.Zone
	}
	state.ClientName = cfg.ClientName
	state.RepoBranch = cfg.RepoBranch

	// Scanner setup
	if withScanner {
		state.ScannerEnabled = true
		if state.ScannerSecret == "" {
			secret, err := generateScannerSecret()
			if err != nil {
				return fmt.Errorf("generate scanner secret: %w", err)
			}
			state.ScannerSecret = secret
			fmt.Printf("Generated scanner shared secret\n")
		}
		_ = saveState(state)
	}

	// Labeler setup
	if withLabeler {
		state.LabelerEnabled = true
		_ = saveState(state)
	}

	fmt.Printf("Provisioning %s infrastructure in zone %s...\n", naming.DisplayName(), cfg.Zone)
	if needsServers {
		fmt.Printf("Server plan: %s\n", cfg.Plan)
	}
	if needsScannerServer {
		fmt.Printf("Scanner server plan: %s\n", cfg.ScannerPlan)
	}
	fmt.Println()

	// S3 secret key — from flag for existing storage, from API for new
	s3SecretKey := cfg.S3SecretKey

	// 1. Object storage
	if state.ObjectStorage.UUID != "" {
		fmt.Printf("Object storage: %s (exists)\n", state.ObjectStorage.UUID)
		// Refresh discoverable fields if missing (e.g. pre-seeded UUID only)
		if state.ObjectStorage.Endpoint == "" || state.ObjectStorage.Bucket == "" {
			fmt.Println("  Discovering endpoint, bucket, access key...")
			discovered, err := lookupObjectStorage(ctx, svc, state.ObjectStorage.UUID)
			if err != nil {
				return err
			}
			state.ObjectStorage.Endpoint = discovered.Endpoint
			state.ObjectStorage.Region = discovered.Region
			if discovered.Bucket != "" {
				state.ObjectStorage.Bucket = discovered.Bucket
			}
			if discovered.AccessKeyID != "" {
				state.ObjectStorage.AccessKeyID = discovered.AccessKeyID
			}
			_ = saveState(state)
		}
	} else {
		fmt.Println("Creating object storage...")
		objState, secretKey, err := provisionObjectStorage(ctx, svc, cfg.Zone, naming.S3Name())
		if err != nil {
			return fmt.Errorf("object storage: %w", err)
		}
		state.ObjectStorage = objState
		s3SecretKey = secretKey
		_ = saveState(state)
		fmt.Printf("  S3 Secret Key: %s\n", secretKey)
	}

	fmt.Printf("  Endpoint:   %s\n", state.ObjectStorage.Endpoint)
	fmt.Printf("  Region:     %s\n", state.ObjectStorage.Region)
	fmt.Printf("  Bucket:     %s\n", state.ObjectStorage.Bucket)
	fmt.Printf("  Access Key: %s\n\n", state.ObjectStorage.AccessKeyID)

	// Hold domain is zone-based (e.g. us-chi1.cove.seamark.dev)
	holdDomain := cfg.Zone + ".cove." + cfg.BaseDomain
	// Labeler domain is a fixed subdomain on the base domain (e.g. labeler.seamark.dev)
	labelerDomain := "labeler." + cfg.BaseDomain

	// Build config template values
	vals := &ConfigValues{
		S3Endpoint:    state.ObjectStorage.Endpoint,
		S3Region:      state.ObjectStorage.Region,
		S3Bucket:      state.ObjectStorage.Bucket,
		S3AccessKey:   state.ObjectStorage.AccessKeyID,
		S3SecretKey:   s3SecretKey,
		Zone:          cfg.Zone,
		HoldDomain:    holdDomain,
		HoldDid:       "did:web:" + holdDomain,
		LabelerDomain: labelerDomain,
		BasePath:      naming.BasePath(),
		ScannerSecret: state.ScannerSecret,
	}

	// 2. Private network
	if state.Network.UUID != "" {
		fmt.Printf("Network: %s (exists)\n", state.Network.UUID)
	} else {
		fmt.Println("Creating private network...")
		network, err := svc.CreateNetwork(ctx, &request.CreateNetworkRequest{
			Name: naming.NetworkName(),
			Zone: cfg.Zone,
			IPNetworks: upcloud.IPNetworkSlice{
				{
					Address:          privateNetworkCIDR,
					DHCP:             upcloud.True,
					DHCPDefaultRoute: upcloud.False,
					DHCPDns:          []string{"8.8.8.8", "1.1.1.1"},
					Family:           upcloud.IPAddressFamilyIPv4,
					Gateway:          "",
				},
			},
		})
		if err != nil {
			return fmt.Errorf("create network: %w", err)
		}
		state.Network = StateRef{UUID: network.UUID}
		_ = saveState(state)
		fmt.Printf("  Network: %s (%s)\n", network.UUID, privateNetworkCIDR)
	}

	// Find Debian template (needed for server creation)
	templateUUID, err := findDebianTemplate(ctx, svc)
	if err != nil {
		return err
	}

	// 3. Appview server
	appviewCreated := false
	if state.Appview.UUID != "" {
		fmt.Printf("Appview: %s (exists)\n", state.Appview.UUID)
		appviewScript, err := generateAppviewCloudInit(cfg, vals, state.LabelerEnabled)
		if err != nil {
			return err
		}
		if err := syncCloudInit("appview", state.Appview.PublicIP, appviewScript); err != nil {
			return err
		}
		appviewConfigYAML, err := renderConfig(appviewConfigTmpl, vals)
		if err != nil {
			return fmt.Errorf("render appview config: %w", err)
		}
		if err := syncConfigKeys("appview", state.Appview.PublicIP, naming.AppviewConfigPath(), appviewConfigYAML, true); err != nil {
			return fmt.Errorf("appview config sync: %w", err)
		}
		if state.LabelerEnabled {
			labelerConfigYAML, err := renderConfig(labelerConfigTmpl, vals)
			if err != nil {
				return fmt.Errorf("render labeler config: %w", err)
			}
			if err := syncConfigKeys("labeler", state.Appview.PublicIP, naming.LabelerConfigPath(), labelerConfigYAML, true); err != nil {
				return fmt.Errorf("labeler config sync: %w", err)
			}
		}
	} else {
		fmt.Println("Creating appview server...")
		appviewUserData, err := generateAppviewCloudInit(cfg, vals, state.LabelerEnabled)
		if err != nil {
			return err
		}
		appview, err := createServer(ctx, svc, cfg, cfg.Plan, templateUUID, state.Network.UUID, naming.Appview(), appviewUserData)
		if err != nil {
			return fmt.Errorf("create appview: %w", err)
		}
		state.Appview = *appview
		_ = saveState(state)
		appviewCreated = true
		fmt.Printf("  Appview: %s (public: %s, private: %s)\n", appview.UUID, appview.PublicIP, appview.PrivateIP)
	}

	// 4. Hold server
	holdCreated := false
	if state.Hold.UUID != "" {
		fmt.Printf("Hold: %s (exists)\n", state.Hold.UUID)
		holdScript, err := generateHoldCloudInit(cfg, vals)
		if err != nil {
			return err
		}
		if err := syncCloudInit("hold", state.Hold.PublicIP, holdScript); err != nil {
			return err
		}
		holdConfigYAML, err := renderConfig(holdConfigTmpl, vals)
		if err != nil {
			return fmt.Errorf("render hold config: %w", err)
		}
		if err := syncConfigKeys("hold", state.Hold.PublicIP, naming.HoldConfigPath(), holdConfigYAML, true); err != nil {
			return fmt.Errorf("hold config sync: %w", err)
		}
	} else {
		fmt.Println("Creating hold server...")
		holdUserData, err := generateHoldCloudInit(cfg, vals)
		if err != nil {
			return err
		}
		hold, err := createServer(ctx, svc, cfg, cfg.Plan, templateUUID, state.Network.UUID, naming.Hold(), holdUserData)
		if err != nil {
			return fmt.Errorf("create hold: %w", err)
		}
		state.Hold = *hold
		_ = saveState(state)
		holdCreated = true
		fmt.Printf("  Hold: %s (public: %s, private: %s)\n", hold.UUID, hold.PublicIP, hold.PrivateIP)
	}

	// 4b. Scanner server. Its own box: the scanner is memory-bound (Grype's
	// database working set plus Syft's in-memory catalog) and sharing a 1 GB
	// host with the hold swapped that host unresponsive for eleven hours on
	// 2026-09-12. It reaches the hold over the private network, so the hold's
	// private IP has to be known before its config can be rendered.
	scannerCreated := false
	if scannerEnabled {
		vals.ScannerHoldURL = scannerHoldURL(state.Hold.PrivateIP)
		if state.Scanner.UUID != "" {
			fmt.Printf("Scanner: %s (exists)\n", state.Scanner.UUID)
			scannerScript, err := generateScannerCloudInit(cfg, vals)
			if err != nil {
				return err
			}
			if err := syncCloudInit("scanner", state.Scanner.PublicIP, scannerScript); err != nil {
				return err
			}
			scannerConfigYAML, err := renderConfig(scannerConfigTmpl, vals)
			if err != nil {
				return fmt.Errorf("render scanner config: %w", err)
			}
			if err := syncConfigKeys("scanner", state.Scanner.PublicIP, naming.ScannerConfigPath(), scannerConfigYAML, true); err != nil {
				return fmt.Errorf("scanner config sync: %w", err)
			}
		} else {
			fmt.Println("Creating scanner server...")
			scannerUserData, err := generateScannerCloudInit(cfg, vals)
			if err != nil {
				return err
			}
			scanner, err := createServer(ctx, svc, cfg, cfg.ScannerPlan, templateUUID, state.Network.UUID, naming.Scanner(), scannerUserData)
			if err != nil {
				return fmt.Errorf("create scanner: %w", err)
			}
			state.Scanner = *scanner
			_ = saveState(state)
			scannerCreated = true
			fmt.Printf("  Scanner: %s (public: %s, private: %s)\n", scanner.UUID, scanner.PublicIP, scanner.PrivateIP)
		}
	}

	// 5. Firewall rules (idempotent — replaces all rules)
	fmt.Println("Configuring firewall rules...")
	firewalled := []struct {
		name string
		uuid string
	}{
		{"appview", state.Appview.UUID},
		{"hold", state.Hold.UUID},
	}
	if state.Scanner.UUID != "" {
		firewalled = append(firewalled, struct {
			name string
			uuid string
		}{"scanner", state.Scanner.UUID})
	}
	for _, s := range firewalled {
		if err := createFirewallRules(ctx, svc, s.uuid, privateNetworkCIDR); err != nil {
			return fmt.Errorf("firewall %s: %w", s.name, err)
		}
	}

	// 6. Load balancer
	if state.LB.UUID != "" {
		fmt.Printf("Load balancer: %s (exists)\n", state.LB.UUID)
	} else {
		fmt.Println("Creating load balancer (Essentials tier)...")
		lb, err := createLoadBalancer(ctx, svc, cfg, naming, state.Network.UUID, state.Appview.PrivateIP, state.Hold.PrivateIP, holdDomain, labelerDomain, state.LabelerEnabled)
		if err != nil {
			return fmt.Errorf("create LB: %w", err)
		}
		state.LB = StateRef{UUID: lb.UUID}
		_ = saveState(state)
	}

	// Always reconcile forwarded headers rule (handles existing LBs)
	if err := ensureLBForwardedHeaders(ctx, svc, state.LB.UUID); err != nil {
		return fmt.Errorf("LB forwarded headers: %w", err)
	}

	// Same reason: HTTP/2 and the client timeout must reach an LB that already
	// exists, not just one created by this run.
	if err := ensureLBHTTP2(ctx, svc, state.LB.UUID); err != nil {
		return fmt.Errorf("LB http2: %w", err)
	}

	// Ensure route-hold rule includes forwarded headers action
	if err := ensureLBHoldForwardedHeaders(ctx, svc, state.LB.UUID, holdDomain); err != nil {
		return fmt.Errorf("LB hold forwarded headers: %w", err)
	}

	// Ensure labeler backend + route-labeler rule when labeler is enabled
	if state.LabelerEnabled {
		if err := ensureLBLabelerRoute(ctx, svc, state.LB.UUID, state.Appview.PrivateIP, labelerDomain); err != nil {
			return fmt.Errorf("LB labeler route: %w", err)
		}
	}

	// Always reconcile scanner block rule
	if err := ensureLBScannerBlock(ctx, svc, state.LB.UUID); err != nil {
		return fmt.Errorf("LB scanner block: %w", err)
	}

	// Always reconcile TLS certs (handles partial failures and re-runs)
	tlsDomains := []string{cfg.BaseDomain}
	tlsDomains = append(tlsDomains, cfg.RegistryDomains...)
	tlsDomains = append(tlsDomains, holdDomain)
	if state.LabelerEnabled {
		tlsDomains = append(tlsDomains, labelerDomain)
	}
	if err := ensureLBCertificates(ctx, svc, state.LB.UUID, tlsDomains); err != nil {
		return fmt.Errorf("LB certificates: %w", err)
	}

	// Fetch LB DNS name for output
	lbDNS := ""
	if state.LB.UUID != "" {
		lb, err := svc.GetLoadBalancer(ctx, &request.GetLoadBalancerRequest{UUID: state.LB.UUID})
		if err == nil {
			for _, n := range lb.Networks {
				if n.Type == upcloud.LoadBalancerNetworkTypePublic {
					lbDNS = n.DNSName
				}
			}
		}
	}

	// 7. Build locally and deploy binaries to new servers
	if appviewCreated || holdCreated || scannerCreated {
		rootDir := projectRoot()

		if err := runMakeBuildTrixie(rootDir, withBilling); err != nil {
			return fmt.Errorf("build: %w", err)
		}
		if err := verifyAppviewBilling(filepath.Join(rootDir, "bin", "atcr-appview"), withBilling); err != nil {
			return err
		}

		fmt.Println("\nWaiting for cloud-init to complete on new servers...")
		if appviewCreated {
			if err := waitForSetup(state.Appview.PublicIP, "appview"); err != nil {
				return err
			}
		}
		if holdCreated {
			if err := waitForSetup(state.Hold.PublicIP, "hold"); err != nil {
				return err
			}
		}
		if scannerCreated {
			if err := waitForSetup(state.Scanner.PublicIP, "scanner"); err != nil {
				return err
			}
		}

		fmt.Println("\nDeploying binaries...")
		if appviewCreated {
			localPath := filepath.Join(rootDir, "bin", "atcr-appview")
			remotePath := naming.InstallDir() + "/bin/" + naming.Appview()
			if err := scpFile(localPath, state.Appview.PublicIP, remotePath); err != nil {
				return fmt.Errorf("upload appview: %w", err)
			}
			if state.LabelerEnabled {
				labelerLocal := filepath.Join(rootDir, "bin", "atcr-labeler")
				labelerRemote := naming.InstallDir() + "/bin/" + naming.Labeler()
				if err := scpFile(labelerLocal, state.Appview.PublicIP, labelerRemote); err != nil {
					return fmt.Errorf("upload labeler: %w", err)
				}
			}
		}
		if holdCreated {
			localPath := filepath.Join(rootDir, "bin", "atcr-hold")
			remotePath := naming.InstallDir() + "/bin/" + naming.Hold()
			if err := scpFile(localPath, state.Hold.PublicIP, remotePath); err != nil {
				return fmt.Errorf("upload hold: %w", err)
			}
		}
		if scannerCreated {
			scannerLocal := filepath.Join(rootDir, "bin", "atcr-scanner")
			scannerRemote := naming.InstallDir() + "/bin/" + naming.Scanner()
			if err := scpFile(scannerLocal, state.Scanner.PublicIP, scannerRemote); err != nil {
				return fmt.Errorf("upload scanner: %w", err)
			}
		}
	}

	// Labeler binary: build and upload when labeler is enabled but appview was
	// not freshly created (the appviewCreated branch above already handled it).
	if state.LabelerEnabled && !appviewCreated {
		rootDir := projectRoot()
		labelerLocal := filepath.Join(rootDir, "bin", "atcr-labeler")
		if err := runMakeBuildTrixie(rootDir, withBilling); err != nil {
			return fmt.Errorf("build labeler: %w", err)
		}
		labelerRemote := naming.InstallDir() + "/bin/" + naming.Labeler()
		fmt.Println("Deploying labeler binary...")
		if err := scpFile(labelerLocal, state.Appview.PublicIP, labelerRemote); err != nil {
			return fmt.Errorf("upload labeler: %w", err)
		}
	}

	fmt.Println("\n=== Provisioning Complete ===")
	fmt.Println()
	fmt.Println("DNS records needed:")
	if lbDNS != "" {
		fmt.Printf("  CNAME  %-24s → %s\n", cfg.BaseDomain, lbDNS)
		for _, rd := range cfg.RegistryDomains {
			fmt.Printf("  CNAME  %-24s → %s\n", rd, lbDNS)
		}
		fmt.Printf("  CNAME  %-24s → %s\n", holdDomain, lbDNS)
		if state.LabelerEnabled {
			fmt.Printf("  CNAME  %-24s → %s\n", labelerDomain, lbDNS)
		}
	} else {
		fmt.Println("  (LB DNS name not yet available — check 'status' in a few minutes)")
	}
	fmt.Println()
	fmt.Println("SSH access:")
	fmt.Printf("  ssh root@%s   # appview\n", state.Appview.PublicIP)
	fmt.Printf("  ssh root@%s   # hold\n", state.Hold.PublicIP)
	if state.Scanner.UUID != "" {
		fmt.Printf("  ssh root@%s   # scanner\n", state.Scanner.PublicIP)
	}
	fmt.Println()
	fmt.Println("Next steps:")
	if appviewCreated || holdCreated || scannerCreated {
		fmt.Println("  1. Edit configs if needed, then start services:")
	} else {
		fmt.Println("  1. Start services:")
	}
	services := []string{naming.Appview(), naming.Hold()}
	if state.ScannerEnabled {
		services = append(services, naming.Scanner())
	}
	if state.LabelerEnabled {
		services = append(services, naming.Labeler())
	}
	fmt.Printf("     systemctl start %s\n", strings.Join(services, " / "))
	fmt.Println("  2. Configure DNS records above")

	return nil
}

// provisionObjectStorage creates a new Managed Object Storage with a user, access key, and bucket.
// Returns the state and the secret key separately (only available at creation time).
func provisionObjectStorage(ctx context.Context, svc *service.Service, zone, s3Name string) (ObjectStorageState, string, error) {
	// Map compute zone to object storage region (e.g. us-chi1 → us-east-1)
	region := objectStorageRegion(zone)

	storage, err := svc.CreateManagedObjectStorage(ctx, &request.CreateManagedObjectStorageRequest{
		Name:             s3Name,
		Region:           region,
		ConfiguredStatus: upcloud.ManagedObjectStorageConfiguredStatusStarted,
		Networks: []upcloud.ManagedObjectStorageNetwork{
			{
				Family: upcloud.IPAddressFamilyIPv4,
				Name:   "public",
				Type:   "public",
			},
		},
	})
	if err != nil {
		return ObjectStorageState{}, "", fmt.Errorf("create storage: %w", err)
	}
	fmt.Printf("  Created: %s (region: %s)\n", storage.UUID, region)

	// Find endpoint
	var endpoint string
	for _, ep := range storage.Endpoints {
		if ep.DomainName != "" {
			endpoint = "https://" + ep.DomainName
			break
		}
	}

	// Create user
	_, err = svc.CreateManagedObjectStorageUser(ctx, &request.CreateManagedObjectStorageUserRequest{
		ServiceUUID: storage.UUID,
		Username:    s3Name,
	})
	if err != nil {
		return ObjectStorageState{}, "", fmt.Errorf("create user: %w", err)
	}

	// Attach admin policy
	err = svc.AttachManagedObjectStorageUserPolicy(ctx, &request.AttachManagedObjectStorageUserPolicyRequest{
		ServiceUUID: storage.UUID,
		Username:    s3Name,
		Name:        "admin",
	})
	if err != nil {
		return ObjectStorageState{}, "", fmt.Errorf("attach policy: %w", err)
	}

	// Create access key (secret is only returned here)
	accessKey, err := svc.CreateManagedObjectStorageUserAccessKey(ctx, &request.CreateManagedObjectStorageUserAccessKeyRequest{
		ServiceUUID: storage.UUID,
		Username:    s3Name,
	})
	if err != nil {
		return ObjectStorageState{}, "", fmt.Errorf("create access key: %w", err)
	}

	secretKey := ""
	if accessKey.SecretAccessKey != nil {
		secretKey = *accessKey.SecretAccessKey
	}

	// Create bucket
	_, err = svc.CreateManagedObjectStorageBucket(ctx, &request.CreateManagedObjectStorageBucketRequest{
		ServiceUUID: storage.UUID,
		Name:        s3Name,
	})
	if err != nil {
		return ObjectStorageState{}, "", fmt.Errorf("create bucket: %w", err)
	}

	return ObjectStorageState{
		UUID:        storage.UUID,
		Endpoint:    endpoint,
		Region:      region,
		Bucket:      s3Name,
		AccessKeyID: accessKey.AccessKeyID,
	}, secretKey, nil
}

// objectStorageRegion maps a compute zone to the nearest object storage region.
func objectStorageRegion(zone string) string {
	switch {
	case strings.HasPrefix(zone, "us-"):
		return "us-east-1"
	case strings.HasPrefix(zone, "de-"):
		return "europe-1"
	case strings.HasPrefix(zone, "fi-"):
		return "europe-1"
	case strings.HasPrefix(zone, "nl-"):
		return "europe-1"
	case strings.HasPrefix(zone, "es-"):
		return "europe-1"
	case strings.HasPrefix(zone, "pl-"):
		return "europe-1"
	case strings.HasPrefix(zone, "se-"):
		return "europe-1"
	case strings.HasPrefix(zone, "au-"):
		return "australia-1"
	case strings.HasPrefix(zone, "sg-"):
		return "singapore-1"
	default:
		return "us-east-1"
	}
}

func createServer(ctx context.Context, svc *service.Service, cfg *InfraConfig, plan, templateUUID, networkUUID, title, userData string) (*ServerState, error) {
	if cfg.SSHPublicKey == "" {
		return nil, fmt.Errorf("creating server %s requires --ssh-key (path to SSH public key file)", title)
	}
	if plan == "" {
		return nil, fmt.Errorf("creating server %s: no plan selected", title)
	}
	storageTier := "maxiops"
	if isStarterPlan(plan) {
		storageTier = "standard"
	}

	// Look up the plan's storage size from the API instead of hardcoding.
	diskSize := 25 // fallback
	plans, err := svc.GetPlans(ctx)
	if err == nil {
		for _, p := range plans.Plans {
			if p.Name == plan {
				diskSize = p.StorageSize
				break
			}
		}
	}

	details, err := svc.CreateServer(ctx, &request.CreateServerRequest{
		Zone:             cfg.Zone,
		Title:            title,
		Hostname:         title,
		Plan:             plan,
		Metadata:         upcloud.True,
		UserData:         userData,
		Firewall:         "on",
		PasswordDelivery: "none",
		StorageDevices: request.CreateServerStorageDeviceSlice{
			{
				Action:  "clone",
				Storage: templateUUID,
				Title:   title + "-disk",
				Size:    diskSize,
				Tier:    storageTier,
			},
		},
		Networking: &request.CreateServerNetworking{
			Interfaces: request.CreateServerInterfaceSlice{
				{
					Index: 1,
					Type:  upcloud.IPAddressAccessPublic,
					IPAddresses: request.CreateServerIPAddressSlice{
						{Family: upcloud.IPAddressFamilyIPv4},
					},
				},
				{
					Index:   2,
					Type:    upcloud.IPAddressAccessPrivate,
					Network: networkUUID,
					IPAddresses: request.CreateServerIPAddressSlice{
						{Family: upcloud.IPAddressFamilyIPv4},
					},
				},
			},
		},
		LoginUser: &request.LoginUser{
			CreatePassword: "no",
			SSHKeys:        request.SSHKeySlice{cfg.SSHPublicKey},
		},
	})
	if err != nil {
		return nil, err
	}

	fmt.Printf("  Waiting for server %s to start...\n", details.UUID)
	details, err = svc.WaitForServerState(ctx, &request.WaitForServerStateRequest{
		UUID:         details.UUID,
		DesiredState: upcloud.ServerStateStarted,
	})
	if err != nil {
		return nil, fmt.Errorf("wait for server: %w", err)
	}

	s := &ServerState{UUID: details.UUID}
	for _, iface := range details.Networking.Interfaces {
		for _, addr := range iface.IPAddresses {
			if addr.Family == upcloud.IPAddressFamilyIPv4 {
				switch iface.Type {
				case upcloud.IPAddressAccessPublic:
					s.PublicIP = addr.Address
				case upcloud.IPAddressAccessPrivate:
					s.PrivateIP = addr.Address
				}
			}
		}
	}

	return s, nil
}

func createFirewallRules(ctx context.Context, svc *service.Service, serverUUID, privateCIDR string) error {
	networkBase := strings.TrimSuffix(privateCIDR, "/24")
	networkBase = strings.TrimSuffix(networkBase, ".0")

	return svc.CreateFirewallRules(ctx, &request.CreateFirewallRulesRequest{
		ServerUUID: serverUUID,
		FirewallRules: request.FirewallRuleSlice{
			{
				Direction:            upcloud.FirewallRuleDirectionIn,
				Action:               upcloud.FirewallRuleActionAccept,
				Family:               upcloud.IPAddressFamilyIPv4,
				Protocol:             upcloud.FirewallRuleProtocolTCP,
				DestinationPortStart: "22",
				DestinationPortEnd:   "22",
				Position:             1,
				Comment:              "Allow SSH",
			},
			{
				Direction:          upcloud.FirewallRuleDirectionIn,
				Action:             upcloud.FirewallRuleActionAccept,
				Family:             upcloud.IPAddressFamilyIPv4,
				SourceAddressStart: networkBase + ".0",
				SourceAddressEnd:   networkBase + ".255",
				Position:           2,
				Comment:            "Allow private network",
			},
			{
				Direction:       upcloud.FirewallRuleDirectionIn,
				Action:          upcloud.FirewallRuleActionAccept,
				Family:          upcloud.IPAddressFamilyIPv4,
				Protocol:        upcloud.FirewallRuleProtocolUDP,
				SourcePortStart: "123",
				SourcePortEnd:   "123",
				Position:        3,
				Comment:         "Allow NTP replies",
			},
			{
				Direction: upcloud.FirewallRuleDirectionIn,
				Action:    upcloud.FirewallRuleActionDrop,
				Position:  4,
				Comment:   "Drop all other inbound",
			},
		},
	})
}

// lbClientTimeout is the frontend's client-side timeout, in seconds. It is
// stated explicitly rather than left to the UpCloud default so the value the
// deployment relies on is visible here: the admin panel's slow endpoints are
// cut at this boundary, and the 504 that produces is what the UI surfaces as
// "Server error". Changing it changes that behaviour, so it is a deliberate
// knob, not an incidental one.
const lbClientTimeout = 10

// httpFrontendProperties are the https frontend's properties.
//
// HTTP/2 stays OFF on this frontend, and ensureLBHTTP2 turns it off on an LB
// that has it on. It was on until 2026-09-12, for the browser's sake (h2
// multiplexes the admin crew tab's fan-out over one connection instead of
// six). Two things changed the answer:
//
//   - Re-enabling it that day caused a 25-minute /auth/token outage. The
//     appview and the hold call each other through this LB on long-lived Go
//     HTTP clients; the frontend reconfigure left those connections open at
//     TCP level but dead, Go's HTTP/2 transport multiplexed every call onto
//     them, and no transport here sets ReadIdleTimeout to notice. Only
//     restarting both services recovered. Over HTTP/1.1 a dead connection
//     fails one request and the pool opens another.
//   - The 64 KB HTTP/2 stream window capped pushes at ~2.5 MB/s through the
//     LB; median push fell from 16.5 s to 3.1 s when it went off.
//
// The registry and web hostnames are fronted by Bunny now, so the browser
// case this bought is served by the CDN's own h2 edge rather than by the LB.
func httpFrontendProperties() *upcloud.LoadBalancerFrontendProperties {
	return &upcloud.LoadBalancerFrontendProperties{
		HTTP2Enabled:  new(false),
		TimeoutClient: lbClientTimeout,
	}
}

// Backends are deliberately left at HTTP/1.1.
//
// UpCloud refuses http2_enabled on a backend that is not TLS:
//
//	invalid_params_properties.http2_enabled='Tls must be enabled to enable HTTP2.'
//
// This LB reaches both origins over the private network in cleartext, so the
// option is simply unavailable here; requesting it fails the whole provision
// run with a 400. Backend HTTP/2 would first require terminating TLS at the
// origins, which is a larger change than the frontend win justifies.
//
// The frontend is where it mattered anyway: that is the leg a browser speaks,
// and the ~6-connections-per-origin limit it removes is what made one slow
// endpoint block every other request to the same host.
//
// Both origins still wrap their handler in h2c, so if backend TLS is ever
// added, enabling this becomes a config change rather than a code change. Note
// the hold would still have to opt out even then: it serves
// com.atproto.sync.subscribeRepos over WebSocket to external relays and to the
// scanner, and WebSocket over HTTP/2 needs the RFC 8441 Extended CONNECT that
// Go's http2 server does not implement for Upgrade:.

func createLoadBalancer(ctx context.Context, svc *service.Service, cfg *InfraConfig, naming Naming, networkUUID, appviewIP, holdIP, holdDomain, labelerDomain string, withLabeler bool) (*upcloud.LoadBalancer, error) {
	frontendRules := []request.LoadBalancerFrontendRule{
		{
			Name:     "set-forwarded-headers",
			Priority: 1,
			Matchers: []upcloud.LoadBalancerMatcher{},
			Actions: []upcloud.LoadBalancerAction{
				request.NewLoadBalancerSetForwardedHeadersAction(),
			},
		},
		{
			Name:     "route-hold",
			Priority: 10,
			Matchers: []upcloud.LoadBalancerMatcher{
				{
					Type: upcloud.LoadBalancerMatcherTypeHost,
					Host: &upcloud.LoadBalancerMatcherHost{
						Value: holdDomain,
					},
				},
			},
			Actions: []upcloud.LoadBalancerAction{
				request.NewLoadBalancerSetForwardedHeadersAction(),
				{
					Type: upcloud.LoadBalancerActionTypeUseBackend,
					UseBackend: &upcloud.LoadBalancerActionUseBackend{
						Backend: "hold",
					},
				},
			},
		},
	}

	backends := []request.LoadBalancerBackend{
		{
			Name: "appview",
			Members: []request.LoadBalancerBackendMember{
				{
					Name:        "appview-1",
					Type:        upcloud.LoadBalancerBackendMemberTypeStatic,
					IP:          appviewIP,
					Port:        5000,
					Weight:      100,
					MaxSessions: 1000,
					Enabled:     true,
				},
			},
			Properties: &upcloud.LoadBalancerBackendProperties{
				HealthCheckType: upcloud.LoadBalancerHealthCheckTypeHTTP,
				HealthCheckURL:  "/health",
			},
		},
		{
			Name: "hold",
			Members: []request.LoadBalancerBackendMember{
				{
					Name:        "hold-1",
					Type:        upcloud.LoadBalancerBackendMemberTypeStatic,
					IP:          holdIP,
					Port:        8080,
					Weight:      100,
					MaxSessions: 1000,
					Enabled:     true,
				},
			},
			Properties: &upcloud.LoadBalancerBackendProperties{
				HealthCheckType: upcloud.LoadBalancerHealthCheckTypeHTTP,
				HealthCheckURL:  "/xrpc/_health",
			},
		},
	}

	if withLabeler {
		frontendRules = append(frontendRules, labelerFrontendRule(labelerDomain))
		backends = append(backends, labelerBackend(appviewIP))
	}

	lb, err := svc.CreateLoadBalancer(ctx, &request.CreateLoadBalancerRequest{
		Name:             naming.LBName(),
		Plan:             "essentials",
		Zone:             cfg.Zone,
		ConfiguredStatus: upcloud.LoadBalancerConfiguredStatusStarted,
		Networks: []request.LoadBalancerNetwork{
			{
				Name:   "public",
				Type:   upcloud.LoadBalancerNetworkTypePublic,
				Family: upcloud.LoadBalancerAddressFamilyIPv4,
			},
			{
				Name:   "private",
				Type:   upcloud.LoadBalancerNetworkTypePrivate,
				Family: upcloud.LoadBalancerAddressFamilyIPv4,
				UUID:   networkUUID,
			},
		},
		Frontends: []request.LoadBalancerFrontend{
			{
				Name:           "https",
				Mode:           upcloud.LoadBalancerModeHTTP,
				Port:           443,
				DefaultBackend: "appview",
				Properties:     httpFrontendProperties(),
				Networks: []upcloud.LoadBalancerFrontendNetwork{
					{Name: "public"},
				},
				Rules: frontendRules,
			},
			{
				Name:           "http-redirect",
				Mode:           upcloud.LoadBalancerModeHTTP,
				Port:           80,
				DefaultBackend: "appview",
				Networks: []upcloud.LoadBalancerFrontendNetwork{
					{Name: "public"},
				},
				Rules: []request.LoadBalancerFrontendRule{
					{
						Name:     "redirect-https",
						Priority: 10,
						Matchers: []upcloud.LoadBalancerMatcher{
							{
								Type: upcloud.LoadBalancerMatcherTypeSrcPort,
								SrcPort: &upcloud.LoadBalancerMatcherInteger{
									Method: upcloud.LoadBalancerIntegerMatcherMethodEqual,
									Value:  80,
								},
							},
						},
						Actions: []upcloud.LoadBalancerAction{
							{
								Type: upcloud.LoadBalancerActionTypeHTTPRedirect,
								HTTPRedirect: &upcloud.LoadBalancerActionHTTPRedirect{
									Scheme: upcloud.LoadBalancerActionHTTPRedirectSchemeHTTPS,
								},
							},
						},
					},
				},
			},
		},
		Resolvers: []request.LoadBalancerResolver{},
		Backends:  backends,
	})
	if err != nil {
		return nil, err
	}

	return lb, nil
}

// labelerBackend builds the labeler LB backend pointing at the appview server's
// private IP on the labeler listen port.
func labelerBackend(appviewIP string) request.LoadBalancerBackend {
	return request.LoadBalancerBackend{
		Name: "labeler",
		Members: []request.LoadBalancerBackendMember{
			{
				Name:        "labeler-1",
				Type:        upcloud.LoadBalancerBackendMemberTypeStatic,
				IP:          appviewIP,
				Port:        5002,
				Weight:      100,
				MaxSessions: 1000,
				Enabled:     true,
			},
		},
		Properties: &upcloud.LoadBalancerBackendProperties{
			HealthCheckType: upcloud.LoadBalancerHealthCheckTypeHTTP,
			HealthCheckURL:  "/.well-known/did.json",
		},
	}
}

// labelerFrontendRule returns a host-match rule routing labelerDomain to the
// labeler backend with forwarded headers.
func labelerFrontendRule(labelerDomain string) request.LoadBalancerFrontendRule {
	return request.LoadBalancerFrontendRule{
		Name:     "route-labeler",
		Priority: 20,
		Matchers: []upcloud.LoadBalancerMatcher{
			{
				Type: upcloud.LoadBalancerMatcherTypeHost,
				Host: &upcloud.LoadBalancerMatcherHost{
					Value: labelerDomain,
				},
			},
		},
		Actions: []upcloud.LoadBalancerAction{
			request.NewLoadBalancerSetForwardedHeadersAction(),
			{
				Type: upcloud.LoadBalancerActionTypeUseBackend,
				UseBackend: &upcloud.LoadBalancerActionUseBackend{
					Backend: "labeler",
				},
			},
		},
	}
}

// ensureLBCertificates reconciles TLS certificate bundles on the load balancer.
// It skips domains that already have a TLS config attached and creates missing
// ones, but only for domains whose DNS resolves to this LB. A dynamic
// (Let's Encrypt) bundle is validated over HTTP at the domain, so a domain
// fronted by a CDN cannot be issued here anyway; requesting it only leaves a
// bundle stuck in validation. The .cr registry domains moved behind Bunny on
// 2026-09-12 and their bundles were removed by hand; on 2026-09-13 this
// function put them straight back. Hence the check.
func ensureLBCertificates(ctx context.Context, svc *service.Service, lbUUID string, tlsDomains []string) error {
	lb, err := svc.GetLoadBalancer(ctx, &request.GetLoadBalancerRequest{UUID: lbUUID})
	if err != nil {
		return fmt.Errorf("get load balancer: %w", err)
	}

	lbAddrs := map[string]bool{}
	for _, n := range lb.Networks {
		if n.Type == upcloud.LoadBalancerNetworkTypePublic && n.DNSName != "" {
			for _, a := range lookupHost(n.DNSName) {
				lbAddrs[a] = true
			}
		}
	}

	// Build set of existing TLS config names on the "https" frontend
	existing := make(map[string]bool)
	for _, fe := range lb.Frontends {
		if fe.Name == "https" {
			for _, tc := range fe.TLSConfigs {
				existing[tc.Name] = true
			}
		}
	}

	for _, domain := range tlsDomains {
		certName := "tls-" + strings.ReplaceAll(domain, ".", "-")
		if existing[certName] {
			fmt.Printf("  TLS certificate: %s (exists)\n", domain)
			continue
		}
		if !pointsAt(domain, lbAddrs) {
			fmt.Printf("  TLS certificate: %s skipped (DNS does not point at this LB; fronted elsewhere or not yet delegated)\n", domain)
			continue
		}

		bundle, err := svc.CreateLoadBalancerCertificateBundle(ctx, &request.CreateLoadBalancerCertificateBundleRequest{
			Type:      upcloud.LoadBalancerCertificateBundleTypeDynamic,
			Name:      certName,
			KeyType:   "ecdsa",
			Hostnames: []string{domain},
		})
		if err != nil {
			return fmt.Errorf("create TLS cert for %s: %w", domain, err)
		}

		_, err = svc.CreateLoadBalancerFrontendTLSConfig(ctx, &request.CreateLoadBalancerFrontendTLSConfigRequest{
			ServiceUUID:  lbUUID,
			FrontendName: "https",
			Config: request.LoadBalancerFrontendTLSConfig{
				Name:                  certName,
				CertificateBundleUUID: bundle.UUID,
			},
		})
		if err != nil {
			return fmt.Errorf("attach TLS cert %s to frontend: %w", domain, err)
		}
		fmt.Printf("  TLS certificate: %s\n", domain)
	}

	return nil
}

// lookupHost resolves name to its IPv4/IPv6 addresses; failures resolve to none.
func lookupHost(name string) []string {
	addrs, err := net.LookupHost(name)
	if err != nil {
		return nil
	}
	return addrs
}

// pointsAt reports whether any address domain resolves to is one of lbAddrs.
// An empty lbAddrs (the LB's DNS name did not resolve) matches nothing, so
// no bundle is requested when the precondition cannot be checked.
func pointsAt(domain string, lbAddrs map[string]bool) bool {
	for _, a := range lookupHost(domain) {
		if lbAddrs[a] {
			return true
		}
	}
	return false
}

// ensureLBForwardedHeaders ensures the "https" frontend has a set_forwarded_headers rule.
// This makes the LB set X-Forwarded-For, X-Forwarded-Proto, and X-Forwarded-Port headers,
// overwriting any pre-existing values (prevents spoofing).
// ensureLBHTTP2 reconciles the HTTP/2 flag (off, see httpFrontendProperties)
// and the client timeout onto an LB that already exists.
//
// createLoadBalancer only runs when there is no LB yet, so without this the
// properties above would reach a fresh deployment and never an existing one.
// This follows ensureLBForwardedHeaders: read what is there, change only what
// differs, and say what it did. Running it twice is a no-op.
func ensureLBHTTP2(ctx context.Context, svc *service.Service, lbUUID string) error {
	fe, err := svc.GetLoadBalancerFrontend(ctx, &request.GetLoadBalancerFrontendRequest{
		ServiceUUID: lbUUID,
		Name:        "https",
	})
	if err != nil {
		return fmt.Errorf("get https frontend: %w", err)
	}

	want := httpFrontendProperties()
	wantHTTP2 := want.HTTP2Enabled != nil && *want.HTTP2Enabled
	haveHTTP2 := fe.Properties != nil && fe.Properties.HTTP2Enabled != nil && *fe.Properties.HTTP2Enabled
	haveTimeout := fe.Properties != nil && fe.Properties.TimeoutClient == want.TimeoutClient
	if haveHTTP2 == wantHTTP2 && haveTimeout {
		fmt.Printf("  Frontend HTTP/2 %s + timeout_client=%ds: already set\n", onOff(wantHTTP2), want.TimeoutClient)
	} else {
		if _, err := svc.ModifyLoadBalancerFrontend(ctx, &request.ModifyLoadBalancerFrontendRequest{
			ServiceUUID: lbUUID,
			Name:        "https",
			Frontend: request.ModifyLoadBalancerFrontend{
				Properties: want,
			},
		}); err != nil {
			return fmt.Errorf("modify https frontend: %w", err)
		}
		fmt.Printf("  Frontend HTTP/2: %s (timeout_client=%ds)\n", onOff(wantHTTP2), want.TimeoutClient)
	}

	return nil
}

func onOff(b bool) string {
	if b {
		return "on"
	}
	return "off"
}

func ensureLBForwardedHeaders(ctx context.Context, svc *service.Service, lbUUID string) error {
	rules, err := svc.GetLoadBalancerFrontendRules(ctx, &request.GetLoadBalancerFrontendRulesRequest{
		ServiceUUID:  lbUUID,
		FrontendName: "https",
	})
	if err != nil {
		return fmt.Errorf("get frontend rules: %w", err)
	}

	for _, r := range rules {
		if r.Name == "set-forwarded-headers" {
			// Verify it has the set_forwarded_headers action
			for _, a := range r.Actions {
				if a.SetForwardedHeaders != nil {
					fmt.Println("  Forwarded headers rule: exists and valid")
					return nil
				}
			}
			// Rule exists but is misconfigured — delete and recreate
			fmt.Println("  Forwarded headers rule: exists but misconfigured, recreating")
			if err := svc.DeleteLoadBalancerFrontendRule(ctx, &request.DeleteLoadBalancerFrontendRuleRequest{
				ServiceUUID:  lbUUID,
				FrontendName: "https",
				Name:         r.Name,
			}); err != nil {
				return fmt.Errorf("delete misconfigured forwarded headers rule: %w", err)
			}
			break
		}
	}

	_, err = svc.CreateLoadBalancerFrontendRule(ctx, &request.CreateLoadBalancerFrontendRuleRequest{
		ServiceUUID:  lbUUID,
		FrontendName: "https",
		Rule: request.LoadBalancerFrontendRule{
			Name:     "set-forwarded-headers",
			Priority: 1,
			Matchers: []upcloud.LoadBalancerMatcher{},
			Actions: []upcloud.LoadBalancerAction{
				request.NewLoadBalancerSetForwardedHeadersAction(),
			},
		},
	})
	if err != nil {
		return fmt.Errorf("create forwarded headers rule: %w", err)
	}
	fmt.Println("  Forwarded headers rule: created")

	return nil
}

// ensureLBHoldForwardedHeaders ensures the "route-hold" rule includes a
// set_forwarded_headers action alongside use_backend. Without this, the LB
// doesn't set X-Forwarded-For on hold-routed traffic.
func ensureLBHoldForwardedHeaders(ctx context.Context, svc *service.Service, lbUUID, holdDomain string) error {
	rules, err := svc.GetLoadBalancerFrontendRules(ctx, &request.GetLoadBalancerFrontendRulesRequest{
		ServiceUUID:  lbUUID,
		FrontendName: "https",
	})
	if err != nil {
		return fmt.Errorf("get frontend rules: %w", err)
	}

	for _, r := range rules {
		if r.Name == "route-hold" {
			hasForwarded := false
			for _, a := range r.Actions {
				if a.SetForwardedHeaders != nil {
					hasForwarded = true
					break
				}
			}
			if hasForwarded {
				fmt.Println("  Route-hold forwarded headers: exists")
				return nil
			}
			// Delete and recreate with both actions
			fmt.Println("  Route-hold forwarded headers: missing, recreating rule")
			if err := svc.DeleteLoadBalancerFrontendRule(ctx, &request.DeleteLoadBalancerFrontendRuleRequest{
				ServiceUUID:  lbUUID,
				FrontendName: "https",
				Name:         r.Name,
			}); err != nil {
				return fmt.Errorf("delete route-hold rule: %w", err)
			}
			break
		}
	}

	_, err = svc.CreateLoadBalancerFrontendRule(ctx, &request.CreateLoadBalancerFrontendRuleRequest{
		ServiceUUID:  lbUUID,
		FrontendName: "https",
		Rule: request.LoadBalancerFrontendRule{
			Name:     "route-hold",
			Priority: 10,
			Matchers: []upcloud.LoadBalancerMatcher{
				{
					Type: upcloud.LoadBalancerMatcherTypeHost,
					Host: &upcloud.LoadBalancerMatcherHost{
						Value: holdDomain,
					},
				},
			},
			Actions: []upcloud.LoadBalancerAction{
				request.NewLoadBalancerSetForwardedHeadersAction(),
				{
					Type: upcloud.LoadBalancerActionTypeUseBackend,
					UseBackend: &upcloud.LoadBalancerActionUseBackend{
						Backend: "hold",
					},
				},
			},
		},
	})
	if err != nil {
		return fmt.Errorf("create route-hold rule: %w", err)
	}
	fmt.Println("  Route-hold forwarded headers: created")

	return nil
}

// ensureLBLabelerRoute idempotently ensures the LB has a "labeler" backend
// pointing at the appview server's private IP and a "route-labeler" frontend
// rule matching labelerDomain. Used to add labeler routing to a pre-existing LB
// during a re-provision with --with-labeler.
func ensureLBLabelerRoute(ctx context.Context, svc *service.Service, lbUUID, appviewIP, labelerDomain string) error {
	// 1. Ensure backend exists
	backends, err := svc.GetLoadBalancerBackends(ctx, &request.GetLoadBalancerBackendsRequest{ServiceUUID: lbUUID})
	if err != nil {
		return fmt.Errorf("get backends: %w", err)
	}
	hasBackend := false
	for _, b := range backends {
		if b.Name == "labeler" {
			hasBackend = true
			break
		}
	}
	if !hasBackend {
		_, err := svc.CreateLoadBalancerBackend(ctx, &request.CreateLoadBalancerBackendRequest{
			ServiceUUID: lbUUID,
			Backend:     labelerBackend(appviewIP),
		})
		if err != nil {
			return fmt.Errorf("create labeler backend: %w", err)
		}
		fmt.Println("  Labeler backend: created")
	} else {
		fmt.Println("  Labeler backend: exists")
	}

	// 2. Ensure frontend rule exists with correct host matcher
	rules, err := svc.GetLoadBalancerFrontendRules(ctx, &request.GetLoadBalancerFrontendRulesRequest{
		ServiceUUID:  lbUUID,
		FrontendName: "https",
	})
	if err != nil {
		return fmt.Errorf("get frontend rules: %w", err)
	}
	for _, r := range rules {
		if r.Name == "route-labeler" {
			// Verify the host matcher and use_backend action are correct
			hostOK := false
			for _, m := range r.Matchers {
				if m.Host != nil && m.Host.Value == labelerDomain {
					hostOK = true
					break
				}
			}
			backendOK := false
			for _, a := range r.Actions {
				if a.UseBackend != nil && a.UseBackend.Backend == "labeler" {
					backendOK = true
					break
				}
			}
			if hostOK && backendOK {
				fmt.Println("  Route-labeler rule: exists and valid")
				return nil
			}
			fmt.Println("  Route-labeler rule: exists but misconfigured, recreating")
			if err := svc.DeleteLoadBalancerFrontendRule(ctx, &request.DeleteLoadBalancerFrontendRuleRequest{
				ServiceUUID:  lbUUID,
				FrontendName: "https",
				Name:         r.Name,
			}); err != nil {
				return fmt.Errorf("delete route-labeler rule: %w", err)
			}
			break
		}
	}

	if _, err := svc.CreateLoadBalancerFrontendRule(ctx, &request.CreateLoadBalancerFrontendRuleRequest{
		ServiceUUID:  lbUUID,
		FrontendName: "https",
		Rule:         labelerFrontendRule(labelerDomain),
	}); err != nil {
		return fmt.Errorf("create route-labeler rule: %w", err)
	}
	fmt.Println("  Route-labeler rule: created")
	return nil
}

// ensureLBScannerBlock ensures the "https" frontend has a rule that returns 403
// for common scanner paths (.php, .asp, .aspx, .jsp, .cgi, .env).
func ensureLBScannerBlock(ctx context.Context, svc *service.Service, lbUUID string) error {
	rules, err := svc.GetLoadBalancerFrontendRules(ctx, &request.GetLoadBalancerFrontendRulesRequest{
		ServiceUUID:  lbUUID,
		FrontendName: "https",
	})
	if err != nil {
		return fmt.Errorf("get frontend rules: %w", err)
	}

	for _, r := range rules {
		if r.Name == "block-scanners" {
			for _, a := range r.Actions {
				if a.HTTPReturn != nil {
					fmt.Println("  Scanner block rule: exists and valid")
					return nil
				}
			}
			fmt.Println("  Scanner block rule: exists but misconfigured, recreating")
			if err := svc.DeleteLoadBalancerFrontendRule(ctx, &request.DeleteLoadBalancerFrontendRuleRequest{
				ServiceUUID:  lbUUID,
				FrontendName: "https",
				Name:         r.Name,
			}); err != nil {
				return fmt.Errorf("delete misconfigured scanner block rule: %w", err)
			}
			break
		}
	}

	ignoreCase := true
	_, err = svc.CreateLoadBalancerFrontendRule(ctx, &request.CreateLoadBalancerFrontendRuleRequest{
		ServiceUUID:  lbUUID,
		FrontendName: "https",
		Rule: request.LoadBalancerFrontendRule{
			Name:     "block-scanners",
			Priority: 2,
			Matchers: []upcloud.LoadBalancerMatcher{
				request.NewLoadBalancerPathMatcher(
					upcloud.LoadBalancerStringMatcherMethodRegexp,
					`\.(php|asp|aspx|jsp|cgi|env)$`,
					&ignoreCase,
				),
			},
			Actions: []upcloud.LoadBalancerAction{
				{
					Type: upcloud.LoadBalancerActionTypeHTTPReturn,
					HTTPReturn: &upcloud.LoadBalancerActionHTTPReturn{
						Status:      403,
						ContentType: "text/plain",
						Payload:     base64.StdEncoding.EncodeToString([]byte("Forbidden")),
					},
				},
			},
		},
	})
	if err != nil {
		return fmt.Errorf("create scanner block rule: %w", err)
	}
	fmt.Println("  Scanner block rule: created")

	return nil
}

// lookupObjectStorage discovers details of an existing Managed Object Storage.
func lookupObjectStorage(ctx context.Context, svc *service.Service, uuid string) (ObjectStorageState, error) {
	storage, err := svc.GetManagedObjectStorage(ctx, &request.GetManagedObjectStorageRequest{
		UUID: uuid,
	})
	if err != nil {
		return ObjectStorageState{}, fmt.Errorf("get object storage %s: %w", uuid, err)
	}

	var endpoint string
	for _, ep := range storage.Endpoints {
		if ep.DomainName != "" {
			endpoint = "https://" + ep.DomainName
			break
		}
	}

	var bucket string
	buckets, err := svc.GetManagedObjectStorageBucketMetrics(ctx, &request.GetManagedObjectStorageBucketMetricsRequest{
		ServiceUUID: uuid,
	})
	if err == nil {
		for _, b := range buckets {
			if !b.Deleted {
				bucket = b.Name
				break
			}
		}
	}

	var accessKeyID string
	users, err := svc.GetManagedObjectStorageUsers(ctx, &request.GetManagedObjectStorageUsersRequest{
		ServiceUUID: uuid,
	})
	if err == nil {
		for _, u := range users {
			for _, k := range u.AccessKeys {
				if k.Status == "Active" {
					accessKeyID = k.AccessKeyID
					break
				}
			}
			if accessKeyID != "" {
				break
			}
		}
	}

	return ObjectStorageState{
		UUID:        uuid,
		Endpoint:    endpoint,
		Region:      storage.Region,
		Bucket:      bucket,
		AccessKeyID: accessKeyID,
	}, nil
}

func findDebianTemplate(ctx context.Context, svc *service.Service) (string, error) {
	storages, err := svc.GetStorages(ctx, &request.GetStoragesRequest{
		Type: "template",
	})
	if err != nil {
		return "", fmt.Errorf("list templates: %w", err)
	}

	var debian13, debian12 string
	for _, s := range storages.Storages {
		title := strings.ToLower(s.Title)
		if strings.Contains(title, "debian") {
			if strings.Contains(title, "13") || strings.Contains(title, "trixie") {
				debian13 = s.UUID
			} else if strings.Contains(title, "12") || strings.Contains(title, "bookworm") {
				debian12 = s.UUID
			}
		}
	}

	if debian13 != "" {
		return debian13, nil
	}
	if debian12 != "" {
		fmt.Println("  Debian 13 not available, using Debian 12")
		return debian12, nil
	}

	return "", fmt.Errorf("no Debian template found — check UpCloud template list")
}

const cloudInitPath = "/var/lib/cloud/instance/scripts/part-001"

// syncCloudInit compares a locally-generated cloud-init script against what's
// on the server. If they differ (or the remote is missing), it prompts the
// user and re-runs the script over SSH.
func syncCloudInit(name, ip, localScript string) error {
	// Fetch the remote script
	remoteScript, err := runSSH(ip, fmt.Sprintf("cat %s 2>/dev/null || echo '__MISSING__'", cloudInitPath), false)
	if err != nil {
		fmt.Printf("  cloud-init: could not reach %s (%v)\n", name, err)
		return nil
	}
	remoteScript = strings.TrimSpace(remoteScript)

	if remoteScript == "__MISSING__" {
		fmt.Printf("  cloud-init: not found on %s (server may need initial setup)\n", name)
	} else {
		localHash := fmt.Sprintf("%x", sha256.Sum256([]byte(strings.TrimSpace(localScript))))
		remoteHash := fmt.Sprintf("%x", sha256.Sum256([]byte(remoteScript)))
		if localHash == remoteHash {
			fmt.Printf("  cloud-init: up to date\n")
			return nil
		}
		fmt.Printf("  cloud-init: differs from local\n")
	}

	fmt.Printf("  Re-run cloud-init on %s? [Y/n] ", name)
	answer, ok := readLine()
	if !ok {
		// No answer at all (stdin closed, or a non-interactive run) must not
		// re-run setup on a production server. Only a typed yes does.
		fmt.Printf("  no answer, skipped\n")
		if err := writeRemoteCloudInit(ip, localScript); err != nil {
			fmt.Printf("  WARNING: could not update remote cloud-init reference: %v\n", err)
		}
		return nil
	}
	if answer != "" && answer != "y" && answer != "yes" {
		fmt.Printf("  Skipped\n")
		// Still update the remote reference so next provision sees an accurate diff
		if err := writeRemoteCloudInit(ip, localScript); err != nil {
			fmt.Printf("  WARNING: could not update remote cloud-init reference: %v\n", err)
		}
		return nil
	}

	// Write the reference file first so next provision can detect real diffs,
	// regardless of whether the script execution succeeds or fails.
	if err := writeRemoteCloudInit(ip, localScript); err != nil {
		fmt.Printf("  WARNING: could not update remote cloud-init reference: %v\n", err)
	}

	fmt.Printf("  Running cloud-init on %s (%s)... (this may take several minutes)\n", name, ip)
	output, err := runSSH(ip, localScript, true)
	if err != nil {
		fmt.Printf("  ERROR: %v\n", err)
		fmt.Printf("  Output:\n%s\n", output)
		return fmt.Errorf("cloud-init %s failed", name)
	}

	fmt.Printf("  %s: cloud-init complete\n", name)
	return nil
}

// scannerHoldURL is the hold's WebSocket address from the scanner server: the
// hold's private IP over the SDN, never the load balancer (whose 10s client
// timeout would cut the long-lived scan socket).
func scannerHoldURL(holdPrivateIP string) string {
	if holdPrivateIP == "" {
		return ""
	}
	return "ws://" + holdPrivateIP + ":8080"
}

// stdin is shared by every prompt in the tool. Each prompt used to wrap
// os.Stdin in its own bufio.Scanner, and the first one swallowed everything
// that was already buffered (all of a piped "n\nn\n"), so the second prompt
// read EOF, took the default, and re-ran cloud-init on the production hold
// (2026-09-13). One reader keeps the unread lines for the next prompt.
var stdin = bufio.NewReader(os.Stdin)

// readLine returns the next line typed on stdin, lower-cased and trimmed, and
// false when stdin has nothing more to give.
func readLine() (string, bool) {
	line, err := stdin.ReadString('\n')
	if err != nil && line == "" {
		return "", false
	}
	return strings.TrimSpace(strings.ToLower(line)), true
}

// generateScannerSecret generates a random 32-byte hex-encoded shared secret
// for authenticating scanner-to-hold WebSocket connections.
func generateScannerSecret() (string, error) {
	b := make([]byte, 32)
	if _, err := crypto_rand.Read(b); err != nil {
		return "", err
	}
	return hex.EncodeToString(b), nil
}

// writeRemoteCloudInit writes the local cloud-init script to the remote server
// so that subsequent provision runs can accurately detect real changes.
// Uses base64 encoding to avoid heredoc nesting issues (the cloud-init script
// itself contains heredocs like CFGEOF and SVCEOF).
func writeRemoteCloudInit(ip, script string) error {
	encoded := base64.StdEncoding.EncodeToString([]byte(script))
	cmd := fmt.Sprintf("mkdir -p $(dirname %s) && echo '%s' | base64 -d > %s", cloudInitPath, encoded, cloudInitPath)
	_, err := runSSH(ip, cmd, false)
	return err
}

// waitForSetup polls SSH availability on a newly created server, then waits
// for cloud-init to complete before returning.
func waitForSetup(ip, name string) error {
	fmt.Printf("  %s (%s): waiting for SSH...\n", name, ip)
	for i := range 30 {
		_, err := runSSH(ip, "echo ssh_ready", false)
		if err == nil {
			break
		}
		if i == 29 {
			return fmt.Errorf("SSH not available after 5 minutes on %s (%s)", name, ip)
		}
		time.Sleep(10 * time.Second)
	}

	// Poll rather than `cloud-init status --wait`: runSSH caps one command at
	// five minutes, and a fresh server's first boot (apt upgrade plus git,
	// gcc, nodejs, npm) takes longer than that. The 2026-09-13 scanner
	// provision timed out here with the server healthy and half installed.
	fmt.Printf("  %s: waiting for cloud-init...\n", name)
	deadline := time.Now().Add(20 * time.Minute)
	for {
		out, err := runSSH(ip, "cloud-init status 2>/dev/null | head -1", false)
		status := strings.TrimSpace(out)
		switch {
		case err == nil && strings.HasPrefix(status, "status: done"):
			fmt.Printf("  %s: ready\n", name)
			return nil
		case err == nil && strings.HasPrefix(status, "status: error"):
			return fmt.Errorf("cloud-init on %s finished with errors; see /var/log/cloud-init-output.log on the server", name)
		}
		if time.Now().After(deadline) {
			return fmt.Errorf("cloud-init on %s not done after 20 minutes (last status %q)", name, status)
		}
		time.Sleep(15 * time.Second)
	}
}