fix star not being filled in. add ability to deploy scanner on the same server as the hold

This commit is contained in:
Evan Jarrett
2026-02-13 20:41:36 -06:00
parent 8048921f5e
commit ec2063ef52
22 changed files with 850 additions and 161 deletions
+110 -2
View File
@@ -16,12 +16,18 @@ var appviewServiceTmpl string
//go:embed systemd/hold.service.tmpl
var holdServiceTmpl string
//go:embed systemd/scanner.service.tmpl
var scannerServiceTmpl string
//go:embed configs/appview.yaml.tmpl
var appviewConfigTmpl string
//go:embed configs/hold.yaml.tmpl
var holdConfigTmpl string
//go:embed configs/scanner.yaml.tmpl
var scannerConfigTmpl string
//go:embed configs/cloudinit.sh.tmpl
var cloudInitTmpl string
@@ -41,6 +47,9 @@ type ConfigValues struct {
HoldDomain string // e.g. "us-chi1.cove.seamark.dev"
HoldDid string // e.g. "did:web:us-chi1.cove.seamark.dev"
BasePath string // e.g. "/var/lib/seamark"
// Scanner (auto-generated shared secret)
ScannerSecret string // hex-encoded 32-byte secret; empty disables scanning
}
// renderConfig executes a Go template with the given values.
@@ -78,6 +87,30 @@ func renderServiceUnit(tmplStr string, p serviceUnitParams) (string, error) {
return buf.String(), nil
}
// scannerServiceUnitParams holds values for rendering the scanner systemd unit.
// Extends the standard fields with HoldServiceName for the After= dependency.
type scannerServiceUnitParams struct {
DisplayName string // e.g. "Seamark"
User string // e.g. "seamark"
BinaryPath string // e.g. "/opt/seamark/bin/seamark-scanner"
ConfigPath string // e.g. "/etc/seamark/scanner.yaml"
DataDir string // e.g. "/var/lib/seamark"
ServiceName string // e.g. "seamark-scanner"
HoldServiceName string // e.g. "seamark-hold" (After= dependency)
}
func renderScannerServiceUnit(p scannerServiceUnitParams) (string, error) {
t, err := template.New("scanner-service").Parse(scannerServiceTmpl)
if err != nil {
return "", fmt.Errorf("parse scanner service template: %w", err)
}
var buf bytes.Buffer
if err := t.Execute(&buf, p); err != nil {
return "", fmt.Errorf("render scanner service template: %w", err)
}
return buf.String(), nil
}
// generateAppviewCloudInit generates the cloud-init user-data script for the appview server.
func generateAppviewCloudInit(cfg *InfraConfig, vals *ConfigValues, goVersion string) (string, error) {
naming := cfg.Naming()
@@ -119,7 +152,9 @@ func generateAppviewCloudInit(cfg *InfraConfig, vals *ConfigValues, goVersion st
}
// generateHoldCloudInit generates the cloud-init user-data script for the hold server.
func generateHoldCloudInit(cfg *InfraConfig, vals *ConfigValues, goVersion string) (string, error) {
// When withScanner is true, a second phase is appended that builds the scanner binary,
// creates scanner data directories, and installs a scanner systemd service.
func generateHoldCloudInit(cfg *InfraConfig, vals *ConfigValues, goVersion string, withScanner bool) (string, error) {
naming := cfg.Naming()
configYAML, err := renderConfig(holdConfigTmpl, vals)
@@ -139,7 +174,7 @@ func generateHoldCloudInit(cfg *InfraConfig, vals *ConfigValues, goVersion strin
return "", fmt.Errorf("hold service unit: %w", err)
}
return generateCloudInit(cloudInitParams{
script, err := generateCloudInit(cloudInitParams{
GoVersion: goVersion,
BinaryName: naming.Hold(),
BuildCmd: "hold",
@@ -156,6 +191,79 @@ func generateHoldCloudInit(cfg *InfraConfig, vals *ConfigValues, goVersion strin
LogFile: naming.LogFile(),
DisplayName: naming.DisplayName(),
})
if err != nil {
return "", err
}
if !withScanner {
return script, nil
}
// Render scanner config YAML
scannerConfigYAML, err := renderConfig(scannerConfigTmpl, vals)
if err != nil {
return "", fmt.Errorf("scanner config: %w", err)
}
// Append scanner build and setup phase
scannerUnit, err := renderScannerServiceUnit(scannerServiceUnitParams{
DisplayName: naming.DisplayName(),
User: naming.SystemUser(),
BinaryPath: naming.InstallDir() + "/bin/" + naming.Scanner(),
ConfigPath: naming.ScannerConfigPath(),
DataDir: naming.BasePath(),
ServiceName: naming.Scanner(),
HoldServiceName: naming.Hold(),
})
if err != nil {
return "", fmt.Errorf("scanner service unit: %w", err)
}
// Escape single quotes for heredoc embedding
scannerUnit = strings.ReplaceAll(scannerUnit, "'", "'\\''")
scannerConfigYAML = strings.ReplaceAll(scannerConfigYAML, "'", "'\\''")
scannerPhase := fmt.Sprintf(`
# === Scanner Setup ===
echo "Building scanner..."
cd %s/scanner
CGO_ENABLED=1 go build \
-ldflags="-s -w" \
-trimpath \
-o ../bin/%s ./cmd/scanner
cd %s
# Scanner data dirs
mkdir -p %s/vulndb %s/tmp
chown -R %s:%s %s
# Scanner config
cat > %s << 'CFGEOF'
%s
CFGEOF
# Scanner systemd service
cat > /etc/systemd/system/%s.service << 'SVCEOF'
%s
SVCEOF
systemctl daemon-reload
systemctl enable %s
echo "=== Scanner setup complete ==="
`,
naming.InstallDir(),
naming.Scanner(),
naming.InstallDir(),
naming.ScannerDataDir(), naming.ScannerDataDir(),
naming.SystemUser(), naming.SystemUser(), naming.ScannerDataDir(),
naming.ScannerConfigPath(),
scannerConfigYAML,
naming.Scanner(),
scannerUnit,
naming.Scanner(),
)
return script + scannerPhase, nil
}
type cloudInitParams struct {
+1 -1
View File
@@ -50,5 +50,5 @@ quota:
defaults:
new_crew_tier: deckhand
scanner:
secret: ""
secret: "{{.ScannerSecret}}"
+21
View File
@@ -0,0 +1,21 @@
version: "0.1"
log_level: info
log_shipper:
backend: ""
url: ""
batch_size: 100
flush_interval: 5s
username: ""
password: ""
server:
addr: :9090
hold:
url: "ws://localhost:8080"
secret: "{{.ScannerSecret}}"
scanner:
workers: 2
queue_size: 100
vuln:
enabled: true
db_path: "{{.BasePath}}/scanner/vulndb"
tmp_dir: "{{.BasePath}}/scanner/tmp"
BIN
View File
Binary file not shown.
+9
View File
@@ -48,5 +48,14 @@ func (n Naming) NetworkName() string { return n.ClientName + "-private" }
// LBName returns the load balancer name (e.g. "seamark-lb").
func (n Naming) LBName() string { return n.ClientName + "-lb" }
// Scanner returns the scanner binary/service name (e.g. "seamark-scanner").
func (n Naming) Scanner() string { return n.ClientName + "-scanner" }
// ScannerConfigPath returns the scanner config file path.
func (n Naming) ScannerConfigPath() string { return n.ConfigDir() + "/scanner.yaml" }
// ScannerDataDir returns the scanner data directory (e.g. "/var/lib/seamark/scanner").
func (n Naming) ScannerDataDir() string { return n.BasePath() + "/scanner" }
// S3Name returns the name used for S3 storage, user, and bucket.
func (n Naming) S3Name() string { return n.ClientName }
+56 -14
View File
@@ -3,8 +3,10 @@ package main
import (
"bufio"
"context"
crypto_rand "crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"fmt"
"os"
"strings"
@@ -25,7 +27,8 @@ var provisionCmd = &cobra.Command{
plan, _ := cmd.Flags().GetString("plan")
sshKey, _ := cmd.Flags().GetString("ssh-key")
s3Secret, _ := cmd.Flags().GetString("s3-secret")
return cmdProvision(token, zone, plan, sshKey, s3Secret)
withScanner, _ := cmd.Flags().GetBool("with-scanner")
return cmdProvision(token, zone, plan, sshKey, s3Secret, withScanner)
},
}
@@ -34,11 +37,12 @@ func init() {
provisionCmd.Flags().String("plan", "", "Server plan (interactive picker if omitted)")
provisionCmd.Flags().String("ssh-key", "", "Path to SSH public key file (required)")
provisionCmd.Flags().String("s3-secret", "", "S3 secret access key (for existing object storage)")
provisionCmd.Flags().Bool("with-scanner", false, "Deploy vulnerability scanner alongside hold")
provisionCmd.MarkFlagRequired("ssh-key")
rootCmd.AddCommand(provisionCmd)
}
func cmdProvision(token, zone, plan, sshKeyPath, s3Secret string) error {
func cmdProvision(token, zone, plan, sshKeyPath, s3Secret string, withScanner bool) error {
cfg, err := loadConfig(zone, plan, sshKeyPath, s3Secret)
if err != nil {
return err
@@ -79,6 +83,20 @@ func cmdProvision(token, zone, plan, sshKeyPath, s3Secret string) error {
state.ClientName = cfg.ClientName
state.RepoBranch = cfg.RepoBranch
// Scanner setup
if withScanner {
state.ScannerEnabled = true
if state.ScannerSecret == "" {
secret, err := generateScannerSecret()
if err != nil {
return fmt.Errorf("generate scanner secret: %w", err)
}
state.ScannerSecret = secret
fmt.Printf("Generated scanner shared secret\n")
}
saveState(state)
}
goVersion, err := requiredGoVersion()
if err != nil {
return err
@@ -136,15 +154,16 @@ func cmdProvision(token, zone, plan, sshKeyPath, s3Secret string) error {
// Build config template values
vals := &ConfigValues{
S3Endpoint: state.ObjectStorage.Endpoint,
S3Region: state.ObjectStorage.Region,
S3Bucket: state.ObjectStorage.Bucket,
S3AccessKey: state.ObjectStorage.AccessKeyID,
S3SecretKey: s3SecretKey,
Zone: cfg.Zone,
HoldDomain: holdDomain,
HoldDid: "did:web:" + holdDomain,
BasePath: naming.BasePath(),
S3Endpoint: state.ObjectStorage.Endpoint,
S3Region: state.ObjectStorage.Region,
S3Bucket: state.ObjectStorage.Bucket,
S3AccessKey: state.ObjectStorage.AccessKeyID,
S3SecretKey: s3SecretKey,
Zone: cfg.Zone,
HoldDomain: holdDomain,
HoldDid: "did:web:" + holdDomain,
BasePath: naming.BasePath(),
ScannerSecret: state.ScannerSecret,
}
// 2. Private network
@@ -215,7 +234,7 @@ func cmdProvision(token, zone, plan, sshKeyPath, s3Secret string) error {
// 4. Hold server
if state.Hold.UUID != "" {
fmt.Printf("Hold: %s (exists)\n", state.Hold.UUID)
holdScript, err := generateHoldCloudInit(cfg, vals, goVersion)
holdScript, err := generateHoldCloudInit(cfg, vals, goVersion, state.ScannerEnabled)
if err != nil {
return err
}
@@ -229,9 +248,18 @@ func cmdProvision(token, zone, plan, sshKeyPath, s3Secret string) error {
if err := syncConfigKeys("hold", state.Hold.PublicIP, naming.HoldConfigPath(), holdConfigYAML); err != nil {
return fmt.Errorf("hold config sync: %w", err)
}
if state.ScannerEnabled {
scannerConfigYAML, err := renderConfig(scannerConfigTmpl, vals)
if err != nil {
return fmt.Errorf("render scanner config: %w", err)
}
if err := syncConfigKeys("scanner", state.Hold.PublicIP, naming.ScannerConfigPath(), scannerConfigYAML); err != nil {
return fmt.Errorf("scanner config sync: %w", err)
}
}
} else {
fmt.Println("Creating hold server...")
holdUserData, err := generateHoldCloudInit(cfg, vals, goVersion)
holdUserData, err := generateHoldCloudInit(cfg, vals, goVersion, state.ScannerEnabled)
if err != nil {
return err
}
@@ -316,7 +344,11 @@ func cmdProvision(token, zone, plan, sshKeyPath, s3Secret string) error {
fmt.Println()
fmt.Println("Next steps:")
fmt.Println(" 1. Wait ~5 min for cloud-init to complete")
fmt.Printf(" 2. systemctl start %s / %s\n", naming.Appview(), naming.Hold())
if state.ScannerEnabled {
fmt.Printf(" 2. systemctl start %s / %s / %s\n", naming.Appview(), naming.Hold(), naming.Scanner())
} else {
fmt.Printf(" 2. systemctl start %s / %s\n", naming.Appview(), naming.Hold())
}
fmt.Println(" 3. Configure DNS records above")
return nil
@@ -932,6 +964,16 @@ func syncCloudInit(name, ip, localScript string) error {
return nil
}
// generateScannerSecret generates a random 32-byte hex-encoded shared secret
// for authenticating scanner-to-hold WebSocket connections.
func generateScannerSecret() (string, error) {
b := make([]byte, 32)
if _, err := crypto_rand.Read(b); err != nil {
return "", err
}
return hex.EncodeToString(b), nil
}
// writeRemoteCloudInit writes the local cloud-init script to the remote server
// so that subsequent provision runs can accurately detect real changes.
// Uses base64 encoding to avoid heredoc nesting issues (the cloud-init script
+4 -2
View File
@@ -16,8 +16,10 @@ type InfraState struct {
Network StateRef `json:"network"`
Appview ServerState `json:"appview"`
Hold ServerState `json:"hold"`
LB StateRef `json:"loadbalancer"`
ObjectStorage ObjectStorageState `json:"object_storage"`
LB StateRef `json:"loadbalancer"`
ObjectStorage ObjectStorageState `json:"object_storage"`
ScannerEnabled bool `json:"scanner_enabled,omitempty"`
ScannerSecret string `json:"scanner_secret,omitempty"`
}
// Naming returns a Naming helper, defaulting to "seamark" if ClientName is empty.
+25
View File
@@ -90,6 +90,31 @@ func cmdStatus(token string) error {
fmt.Println()
}
// Scanner status (runs on hold server)
if state.ScannerEnabled {
fmt.Printf("Scanner (on hold server)\n")
if state.Hold.PublicIP != "" {
output, err := runSSH(state.Hold.PublicIP, fmt.Sprintf(
"systemctl is-active %s 2>/dev/null || echo 'inactive'; curl -sf http://localhost:9090/healthz > /dev/null 2>&1 && echo 'health:ok' || echo 'health:fail'",
naming.Scanner(),
), false)
if err != nil {
fmt.Printf(" Service: unreachable\n")
} else {
lines := strings.Split(strings.TrimSpace(output), "\n")
for _, line := range lines {
line = strings.TrimSpace(line)
if line == "active" || line == "inactive" {
fmt.Printf(" Service: %s\n", line)
} else if strings.HasPrefix(line, "health:") {
fmt.Printf(" Health: %s\n", strings.TrimPrefix(line, "health:"))
}
}
}
}
fmt.Println()
}
// LB status
if state.LB.UUID != "" {
fmt.Printf("Load Balancer: %s\n", state.LB.UUID)
@@ -0,0 +1,25 @@
[Unit]
Description={{.DisplayName}} Scanner (Vulnerability Scanning)
After=network-online.target {{.HoldServiceName}}.service
Wants=network-online.target
[Service]
Type=simple
User={{.User}}
Group={{.User}}
ExecStart={{.BinaryPath}} serve --config {{.ConfigPath}}
Restart=on-failure
RestartSec=10
ReadWritePaths={{.DataDir}}
ProtectSystem=strict
ProtectHome=yes
NoNewPrivileges=yes
PrivateTmp=yes
StandardOutput=journal
StandardError=journal
SyslogIdentifier={{.ServiceName}}
[Install]
WantedBy=multi-user.target
+102 -13
View File
@@ -22,7 +22,8 @@ var updateCmd = &cobra.Command{
if len(args) > 0 {
target = args[0]
}
return cmdUpdate(target)
withScanner, _ := cmd.Flags().GetBool("with-scanner")
return cmdUpdate(target, withScanner)
},
}
@@ -37,11 +38,12 @@ var sshCmd = &cobra.Command{
}
func init() {
updateCmd.Flags().Bool("with-scanner", false, "Enable and deploy vulnerability scanner alongside hold")
rootCmd.AddCommand(updateCmd)
rootCmd.AddCommand(sshCmd)
}
func cmdUpdate(target string) error {
func cmdUpdate(target string, withScanner bool) error {
state, err := loadState()
if err != nil {
return err
@@ -55,6 +57,20 @@ func cmdUpdate(target string) error {
return err
}
// Enable scanner retroactively via --with-scanner on update
if withScanner && !state.ScannerEnabled {
state.ScannerEnabled = true
if state.ScannerSecret == "" {
secret, err := generateScannerSecret()
if err != nil {
return fmt.Errorf("generate scanner secret: %w", err)
}
state.ScannerSecret = secret
fmt.Printf("Generated scanner shared secret\n")
}
saveState(state)
}
vals := configValsFromState(state)
targets := map[string]struct {
@@ -134,6 +150,64 @@ func cmdUpdate(target string) error {
daemonReload = "systemctl daemon-reload"
}
// Scanner additions for hold server
scannerBuild := ""
scannerRestart := ""
scannerHealthCheck := ""
if name == "hold" && state.ScannerEnabled {
// Sync scanner config keys
scannerConfigYAML, err := renderConfig(scannerConfigTmpl, vals)
if err != nil {
return fmt.Errorf("render scanner config: %w", err)
}
if err := syncConfigKeys("scanner", t.ip, naming.ScannerConfigPath(), scannerConfigYAML); err != nil {
return fmt.Errorf("scanner config sync: %w", err)
}
// Sync scanner service unit
scannerUnit, err := renderScannerServiceUnit(scannerServiceUnitParams{
DisplayName: naming.DisplayName(),
User: naming.SystemUser(),
BinaryPath: naming.InstallDir() + "/bin/" + naming.Scanner(),
ConfigPath: naming.ScannerConfigPath(),
DataDir: naming.BasePath(),
ServiceName: naming.Scanner(),
HoldServiceName: naming.Hold(),
})
if err != nil {
return fmt.Errorf("render scanner service unit: %w", err)
}
scannerUnitChanged, err := syncServiceUnit("scanner", t.ip, naming.Scanner(), scannerUnit)
if err != nil {
return fmt.Errorf("scanner service unit sync: %w", err)
}
if scannerUnitChanged {
daemonReload = "systemctl daemon-reload"
}
scannerBuild = fmt.Sprintf(`
# Build scanner
cd %s/scanner
CGO_ENABLED=1 go build \
-ldflags="-s -w" \
-trimpath \
-o ../bin/%s ./cmd/scanner
cd %s
# Ensure scanner data dirs exist
mkdir -p %s/vulndb %s/tmp
chown -R %s:%s %s
`, naming.InstallDir(), naming.Scanner(), naming.InstallDir(),
naming.ScannerDataDir(), naming.ScannerDataDir(),
naming.SystemUser(), naming.SystemUser(), naming.ScannerDataDir())
scannerRestart = fmt.Sprintf("\nsystemctl restart %s", naming.Scanner())
scannerHealthCheck = fmt.Sprintf(`
sleep 2
curl -sf http://localhost:9090/healthz > /dev/null && echo "SCANNER_HEALTH_OK" || echo "SCANNER_HEALTH_FAIL"
`)
}
updateScript := fmt.Sprintf(`set -euo pipefail
export PATH=$PATH:/usr/local/go/bin
export GOTMPDIR=/var/tmp
@@ -156,11 +230,15 @@ CGO_ENABLED=1 go build \
-tags sqlite_omit_load_extension -trimpath \
-o bin/%s ./cmd/%s
%s
%s
systemctl restart %s
%s
sleep 2
curl -sf %s > /dev/null && echo "HEALTH_OK" || echo "HEALTH_FAIL"
`, goVersion, naming.InstallDir(), branch, t.binaryName, t.buildCmd, daemonReload, t.serviceName, t.healthURL)
%s
`, goVersion, naming.InstallDir(), branch, t.binaryName, t.buildCmd,
scannerBuild, daemonReload, t.serviceName, scannerRestart,
t.healthURL, scannerHealthCheck)
output, err := runSSH(t.ip, updateScript, true)
if err != nil {
@@ -177,6 +255,16 @@ curl -sf %s > /dev/null && echo "HEALTH_OK" || echo "HEALTH_FAIL"
} else {
fmt.Printf(" %s: updated (health check inconclusive)\n", name)
}
// Scanner health reporting
if name == "hold" && state.ScannerEnabled {
if strings.Contains(output, "SCANNER_HEALTH_OK") {
fmt.Printf(" scanner: updated and healthy\n")
} else if strings.Contains(output, "SCANNER_HEALTH_FAIL") {
fmt.Printf(" scanner: updated but health check failed!\n")
fmt.Printf(" Check: ssh root@%s journalctl -u %s -n 50\n", t.ip, naming.Scanner())
}
}
}
return nil
@@ -191,15 +279,16 @@ func configValsFromState(state *InfraState) *ConfigValues {
holdDomain := state.Zone + ".cove." + baseDomain
return &ConfigValues{
S3Endpoint: state.ObjectStorage.Endpoint,
S3Region: state.ObjectStorage.Region,
S3Bucket: state.ObjectStorage.Bucket,
S3AccessKey: state.ObjectStorage.AccessKeyID,
S3SecretKey: "", // not persisted in state; existing value on server is preserved
Zone: state.Zone,
HoldDomain: holdDomain,
HoldDid: "did:web:" + holdDomain,
BasePath: naming.BasePath(),
S3Endpoint: state.ObjectStorage.Endpoint,
S3Region: state.ObjectStorage.Region,
S3Bucket: state.ObjectStorage.Bucket,
S3AccessKey: state.ObjectStorage.AccessKeyID,
S3SecretKey: "", // not persisted in state; existing value on server is preserved
Zone: state.Zone,
HoldDomain: holdDomain,
HoldDid: "did:web:" + holdDomain,
BasePath: naming.BasePath(),
ScannerSecret: state.ScannerSecret,
}
}
+1
View File
@@ -246,6 +246,7 @@ github.com/brianvoe/gofakeit/v6 v6.25.0/go.mod h1:Xj58BMSnFqcn/fAQeSK+/PLtC5kSb7
github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
github.com/campoy/embedmd v1.0.0/go.mod h1:oxyr9RCiSXg0M3VJ3ks0UGfp98BpSSGr0kpiX3MzVl8=
github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw=
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/charmbracelet/colorprofile v0.4.1/go.mod h1:U1d9Dljmdf9DLegaJ0nGZNJvoXAhayhmidOdcBwAvKk=
+35 -14
View File
@@ -5,6 +5,7 @@ import (
"html/template"
"log/slog"
"net/http"
"net/url"
"strings"
"sync"
"time"
@@ -229,6 +230,24 @@ func (h *RepositoryPageHandler) ServeHTTP(w http.ResponseWriter, r *http.Request
artifactType = manifests[0].ArtifactType
}
// Collect digests for batch scan-result request
var scanDigests []string
var scanHoldEndpoint string
for _, m := range manifests {
if !m.IsManifestList && m.Manifest.HoldEndpoint != "" {
if scanHoldEndpoint == "" {
scanHoldEndpoint = m.Manifest.HoldEndpoint
}
if m.Manifest.HoldEndpoint == scanHoldEndpoint {
scanDigests = append(scanDigests, strings.TrimPrefix(m.Manifest.Digest, "sha256:"))
}
}
}
var scanBatchParams string
if len(scanDigests) > 0 {
scanBatchParams = "holdEndpoint=" + url.QueryEscape(scanHoldEndpoint) + "&digests=" + strings.Join(scanDigests, ",")
}
// Build page meta
title := owner.Handle + "/" + repository + " - " + h.ClientShortName
if repo.Title != "" {
@@ -264,21 +283,23 @@ func (h *RepositoryPageHandler) ServeHTTP(w http.ResponseWriter, r *http.Request
PullCount int
IsStarred bool
IsOwner bool // Whether current user owns this repository
ReadmeHTML template.HTML
ArtifactType string // Dominant artifact type: container-image, helm-chart, unknown
ReadmeHTML template.HTML
ArtifactType string // Dominant artifact type: container-image, helm-chart, unknown
ScanBatchParams template.HTML // Pre-encoded query string for batch scan-result endpoint
}{
PageData: NewPageData(r, &h.BaseUIHandler),
Meta: meta,
Owner: owner,
Repository: repo,
Tags: tagsWithPlatforms,
Manifests: manifests,
StarCount: stats.StarCount,
PullCount: stats.PullCount,
IsStarred: isStarred,
IsOwner: isOwner,
ReadmeHTML: readmeHTML,
ArtifactType: artifactType,
PageData: NewPageData(r, &h.BaseUIHandler),
Meta: meta,
Owner: owner,
Repository: repo,
Tags: tagsWithPlatforms,
Manifests: manifests,
StarCount: stats.StarCount,
PullCount: stats.PullCount,
IsStarred: isStarred,
IsOwner: isOwner,
ReadmeHTML: readmeHTML,
ArtifactType: artifactType,
ScanBatchParams: template.HTML(scanBatchParams),
}
if err := h.Templates.ExecuteTemplate(w, "repository", data); err != nil {
+122
View File
@@ -1,13 +1,16 @@
package handlers
import (
"bytes"
"context"
"encoding/json"
"fmt"
"html/template"
"log/slog"
"net/http"
"net/url"
"strings"
"sync"
"time"
"atcr.io/pkg/atproto"
@@ -123,3 +126,122 @@ func (h *ScanResultHandler) renderBadge(w http.ResponseWriter, data vulnBadgeDat
slog.Warn("Failed to render vuln badge", "error", err)
}
}
// fetchScanRecord fetches a scan record from a hold's PDS and returns badge data.
func fetchScanRecord(ctx context.Context, holdEndpoint, holdDID, hexDigest string) vulnBadgeData {
rkey := hexDigest
fullDigest := "sha256:" + hexDigest
scanURL := fmt.Sprintf("%s/xrpc/com.atproto.repo.getRecord?repo=%s&collection=%s&rkey=%s",
holdEndpoint,
url.QueryEscape(holdDID),
url.QueryEscape(atproto.ScanCollection),
url.QueryEscape(rkey),
)
req, err := http.NewRequestWithContext(ctx, "GET", scanURL, nil)
if err != nil {
return vulnBadgeData{Error: true}
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return vulnBadgeData{Error: true}
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return vulnBadgeData{Error: true}
}
var envelope struct {
Value json.RawMessage `json:"value"`
}
if err := json.NewDecoder(resp.Body).Decode(&envelope); err != nil {
return vulnBadgeData{Error: true}
}
var scanRecord atproto.ScanRecord
if err := json.Unmarshal(envelope.Value, &scanRecord); err != nil {
return vulnBadgeData{Error: true}
}
return vulnBadgeData{
Critical: scanRecord.Critical,
High: scanRecord.High,
Medium: scanRecord.Medium,
Low: scanRecord.Low,
Total: scanRecord.Total,
ScannedAt: scanRecord.ScannedAt,
Found: true,
Digest: fullDigest,
HoldEndpoint: holdEndpoint,
}
}
// BatchScanResultHandler handles a single HTMX request that fetches scan results
// for multiple manifests concurrently and returns OOB swap fragments.
type BatchScanResultHandler struct {
BaseUIHandler
}
func (h *BatchScanResultHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
holdEndpoint := r.URL.Query().Get("holdEndpoint")
digestsParam := r.URL.Query().Get("digests")
if holdEndpoint == "" || digestsParam == "" {
w.Header().Set("Content-Type", "text/html")
return
}
digests := strings.Split(digestsParam, ",")
if len(digests) > 50 {
digests = digests[:50]
}
holdDID := atproto.ResolveHoldDIDFromURL(holdEndpoint)
if holdDID == "" {
// Can't resolve hold — render empty OOB spans
w.Header().Set("Content-Type", "text/html")
for _, d := range digests {
fmt.Fprintf(w, `<span id="scan-badge-%s" hx-swap-oob="outerHTML"></span>`, template.HTMLEscapeString(d))
}
return
}
// Fetch scan records concurrently with a semaphore to limit parallelism
type result struct {
hexDigest string
data vulnBadgeData
}
results := make([]result, len(digests))
sem := make(chan struct{}, 10)
var wg sync.WaitGroup
for i, hexDigest := range digests {
results[i].hexDigest = hexDigest
wg.Add(1)
go func(idx int, hex string) {
defer wg.Done()
sem <- struct{}{}
defer func() { <-sem }()
ctx, cancel := context.WithTimeout(r.Context(), 3*time.Second)
defer cancel()
results[idx].data = fetchScanRecord(ctx, holdEndpoint, holdDID, hex)
}(i, hexDigest)
}
wg.Wait()
// Render all OOB fragments
w.Header().Set("Content-Type", "text/html")
for _, res := range results {
var buf bytes.Buffer
if err := h.Templates.ExecuteTemplate(&buf, "vuln-badge", res.data); err != nil {
slog.Warn("Failed to render vuln badge in batch", "digest", res.hexDigest, "error", err)
}
fmt.Fprintf(w, `<span id="scan-badge-%s" hx-swap-oob="outerHTML">%s</span>`,
template.HTMLEscapeString(res.hexDigest), buf.String())
}
}
+147
View File
@@ -253,3 +253,150 @@ func TestScanResult_OnlyCriticalShown(t *testing.T) {
t.Error("Should not contain 'L:0' for zero low count")
}
}
// --- Batch scan result tests ---
func setupBatchScanResultHandler(t *testing.T) *handlers.BatchScanResultHandler {
t.Helper()
templates, err := appview.Templates(nil)
if err != nil {
t.Fatalf("Failed to load templates: %v", err)
}
return &handlers.BatchScanResultHandler{
BaseUIHandler: handlers.BaseUIHandler{
Templates: templates,
},
}
}
func TestBatchScanResult_MultipleDigests(t *testing.T) {
// Mock hold that returns different results based on rkey
hold := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
rkey := r.URL.Query().Get("rkey")
w.Header().Set("Content-Type", "application/json")
switch rkey {
case "abc123":
w.Write([]byte(mockScanRecord(2, 5, 10, 3, 20)))
case "def456":
w.Write([]byte(mockScanRecord(0, 0, 0, 0, 0)))
default:
http.Error(w, "not found", http.StatusNotFound)
}
}))
defer hold.Close()
handler := setupBatchScanResultHandler(t)
req := httptest.NewRequest("GET",
"/api/scan-results?holdEndpoint="+hold.URL+"&digests=abc123,def456,unknown789", nil)
rr := httptest.NewRecorder()
handler.ServeHTTP(rr, req)
if rr.Code != http.StatusOK {
t.Errorf("Expected status %d, got %d", http.StatusOK, rr.Code)
}
body := rr.Body.String()
// All three digests should have OOB spans
if !strings.Contains(body, `id="scan-badge-abc123"`) {
t.Error("Expected OOB span for abc123")
}
if !strings.Contains(body, `id="scan-badge-def456"`) {
t.Error("Expected OOB span for def456")
}
if !strings.Contains(body, `id="scan-badge-unknown789"`) {
t.Error("Expected OOB span for unknown789")
}
// All should have hx-swap-oob attribute
if !strings.Contains(body, `hx-swap-oob="outerHTML"`) {
t.Error("Expected hx-swap-oob attribute in response")
}
// abc123 should have vulnerability badges
if !strings.Contains(body, "C:2") {
t.Error("Expected body to contain 'C:2' for abc123")
}
// def456 should have clean badge
if !strings.Contains(body, "Clean") {
t.Error("Expected body to contain 'Clean' for def456")
}
}
func TestBatchScanResult_EmptyParams(t *testing.T) {
handler := setupBatchScanResultHandler(t)
// No params
req := httptest.NewRequest("GET", "/api/scan-results", nil)
rr := httptest.NewRecorder()
handler.ServeHTTP(rr, req)
body := strings.TrimSpace(rr.Body.String())
if body != "" {
t.Errorf("Expected empty body for missing params, got: %q", body)
}
}
func TestBatchScanResult_MissingDigests(t *testing.T) {
handler := setupBatchScanResultHandler(t)
req := httptest.NewRequest("GET", "/api/scan-results?holdEndpoint=https://hold.example.com", nil)
rr := httptest.NewRecorder()
handler.ServeHTTP(rr, req)
body := strings.TrimSpace(rr.Body.String())
if body != "" {
t.Errorf("Expected empty body for missing digests, got: %q", body)
}
}
func TestBatchScanResult_HoldUnreachable(t *testing.T) {
hold := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
hold.Close()
handler := setupBatchScanResultHandler(t)
req := httptest.NewRequest("GET",
"/api/scan-results?holdEndpoint="+hold.URL+"&digests=abc123,def456", nil)
rr := httptest.NewRecorder()
handler.ServeHTTP(rr, req)
body := rr.Body.String()
// Should still have OOB spans (empty content since hold is unreachable)
if !strings.Contains(body, `id="scan-badge-abc123"`) {
t.Error("Expected OOB span for abc123 even when hold is unreachable")
}
if !strings.Contains(body, `id="scan-badge-def456"`) {
t.Error("Expected OOB span for def456 even when hold is unreachable")
}
// Should NOT contain vulnerability badges
if strings.Contains(body, "badge-error") || strings.Contains(body, "Clean") {
t.Error("Unreachable hold should not render badge content")
}
}
func TestBatchScanResult_SingleDigest(t *testing.T) {
hold := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.Write([]byte(mockScanRecord(1, 0, 0, 0, 1)))
}))
defer hold.Close()
handler := setupBatchScanResultHandler(t)
req := httptest.NewRequest("GET",
"/api/scan-results?holdEndpoint="+hold.URL+"&digests=abc123", nil)
rr := httptest.NewRecorder()
handler.ServeHTTP(rr, req)
body := rr.Body.String()
if !strings.Contains(body, `id="scan-badge-abc123"`) {
t.Error("Expected OOB span for abc123")
}
if !strings.Contains(body, "C:1") {
t.Error("Expected body to contain 'C:1'")
}
}
+1
View File
@@ -124,6 +124,7 @@ func RegisterUIRoutes(router chi.Router, deps UIDependencies) {
// Vulnerability scan result API endpoints (HTMX lazy loading + modal content)
router.Get("/api/scan-result", (&uihandlers.ScanResultHandler{BaseUIHandler: base}).ServeHTTP)
router.Get("/api/scan-results", (&uihandlers.BatchScanResultHandler{BaseUIHandler: base}).ServeHTTP)
router.Get("/api/vuln-details", (&uihandlers.VulnDetailsHandler{BaseUIHandler: base}).ServeHTTP)
// Attestation details API endpoint (HTMX modal content)
+2 -2
View File
@@ -21,12 +21,12 @@
hx-on::before-request="this.disabled=true"
hx-on::after-request="if(event.detail.xhr.status===401) window.location='/auth/oauth/login'"
aria-label="{{ if .IsStarred }}Unstar{{ else }}Star{{ end }} {{ .Handle }}/{{ .Repository }}">
<svg class="icon size-4 text-amber-400 stroke-amber-400 transition-transform group-hover:scale-110{{ if .IsStarred }} fill-amber-400{{ end }}" id="star-icon" aria-hidden="true"><use href="/icons.svg#star"></use></svg>
<svg class="icon size-4 text-amber-400 stroke-amber-400 transition-transform group-hover:scale-110{{ if .IsStarred }} fill-amber-400!{{ end }}" id="star-icon" aria-hidden="true"><use href="/icons.svg#star"></use></svg>
<span id="star-count">{{ .StarCount }}</span>
</button>
{{ else }}
<span class="flex items-center gap-2 text-base-content/60">
<svg class="icon size-[1.1rem] text-amber-400 stroke-amber-400{{ if .IsStarred }} fill-amber-400{{ end }}" aria-hidden="true"><use href="/icons.svg#star"></use></svg>
<svg class="icon size-[1.1rem] text-amber-400 stroke-amber-400{{ if .IsStarred }} fill-amber-400!{{ end }}" aria-hidden="true"><use href="/icons.svg#star"></use></svg>
<span class="font-semibold text-base-content">{{ .StarCount }}</span>
</span>
{{ end }}
+8 -5
View File
@@ -220,12 +220,9 @@
{{ else if not .Reachable }}
<span class="badge badge-sm badge-warning">{{ icon "alert-triangle" "size-3" }} Offline</span>
{{ end }}
{{/* Vulnerability scan badge (lazy-loaded from hold) */}}
{{/* Vulnerability scan badge placeholder (batch-loaded via OOB swap) */}}
{{ if and (not .IsManifestList) .Manifest.HoldEndpoint }}
<span hx-get="/api/scan-result?digest={{ .Manifest.Digest | urlquery }}&holdEndpoint={{ .Manifest.HoldEndpoint | urlquery }}"
hx-trigger="load delay:1s"
hx-swap="outerHTML">
</span>
<span id="scan-badge-{{ trimPrefix "sha256:" .Manifest.Digest }}"></span>
{{ end }}
</div>
<div class="flex items-center gap-2">
@@ -270,6 +267,12 @@
</div>
{{ end }}
</div>
{{ if $.ScanBatchParams }}
<div hx-get="/api/scan-results?{{ $.ScanBatchParams }}"
hx-trigger="load delay:500ms"
hx-swap="none"
style="display:none"></div>
{{ end }}
{{ else }}
<p class="text-base-content/60">No manifests available</p>
{{ end }}
+47 -18
View File
@@ -17,6 +17,8 @@ import (
"atcr.io/scanner/internal/scan"
)
var configFile string
var rootCmd = &cobra.Command{
Use: "atcr-scanner",
Short: "ATCR Scanner - container image vulnerability scanner",
@@ -31,36 +33,30 @@ The scanner connects to a hold service via WebSocket, receives scan jobs,
generates SBOMs with Syft, scans for vulnerabilities with Grype, and sends
results back over the same WebSocket connection.
Configuration via environment variables (SCANNER_ prefix):
SCANNER_HOLD_URL Hold service URL (required)
SCANNER_SHARED_SECRET Shared secret for auth (required)
SCANNER_WORKERS Worker count (default: 2)
SCANNER_QUEUE_SIZE Max queue depth (default: 100)
SCANNER_VULN_ENABLED Enable Grype scanning (default: true)
SCANNER_VULN_DB_PATH Grype DB location (default: /var/lib/atcr-scanner/vulndb)
SCANNER_TMP_DIR Temp dir for extraction (default: /var/lib/atcr-scanner/tmp)
SCANNER_ADDR Health endpoint addr (default: :9090)`,
Configuration is loaded in layers: defaults -> YAML file -> environment variables.
Use --config to specify a YAML configuration file.
Environment variables always override file values (SCANNER_ prefix).`,
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, args []string) error {
cfg, err := config.Load()
cfg, err := config.LoadConfig(configFile)
if err != nil {
return fmt.Errorf("failed to load config: %w", err)
}
slog.Info("Starting ATCR scanner",
"hold_url", cfg.HoldURL,
"workers", cfg.Workers,
"queue_size", cfg.QueueSize,
"vuln_enabled", cfg.VulnEnabled)
"hold_url", cfg.Hold.URL,
"workers", cfg.Scanner.Workers,
"queue_size", cfg.Scanner.QueueSize,
"vuln_enabled", cfg.Vuln.Enabled)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// Create priority queue
q := queue.NewJobQueue(cfg.QueueSize)
q := queue.NewJobQueue(cfg.Scanner.QueueSize)
// Create hold WebSocket client
holdClient := client.NewHoldClient(cfg.HoldURL, cfg.SharedSecret, q)
holdClient := client.NewHoldClient(cfg.Hold.URL, cfg.Hold.Secret, q)
// Start WebSocket connection (feeds queue)
go holdClient.Connect()
@@ -75,9 +71,9 @@ Configuration via environment variables (SCANNER_ prefix):
w.WriteHeader(http.StatusOK)
w.Write([]byte("ok"))
})
healthServer := &http.Server{Addr: cfg.Addr, Handler: mux}
healthServer := &http.Server{Addr: cfg.Server.Addr, Handler: mux}
go func() {
slog.Info("Health endpoint listening", "addr", cfg.Addr)
slog.Info("Health endpoint listening", "addr", cfg.Server.Addr)
if err := healthServer.ListenAndServe(); err != nil && err != http.ErrServerClosed {
slog.Error("Health server error", "error", err)
}
@@ -101,8 +97,41 @@ Configuration via environment variables (SCANNER_ prefix):
},
}
var configCmd = &cobra.Command{
Use: "config",
Short: "Configuration management commands",
}
var configInitCmd = &cobra.Command{
Use: "init [path]",
Short: "Generate an example configuration file",
Long: `Generate an example YAML configuration file with all available options.
If path is provided, writes to that file. Otherwise writes to stdout.`,
Args: cobra.MaximumNArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
yamlBytes, err := config.ExampleYAML()
if err != nil {
return fmt.Errorf("failed to generate example config: %w", err)
}
if len(args) == 1 {
if err := os.WriteFile(args[0], yamlBytes, 0644); err != nil {
return fmt.Errorf("failed to write config file: %w", err)
}
fmt.Fprintf(os.Stderr, "Wrote example config to %s\n", args[0])
return nil
}
fmt.Print(string(yamlBytes))
return nil
},
}
func init() {
serveCmd.Flags().StringVarP(&configFile, "config", "c", "", "path to YAML configuration file")
configCmd.AddCommand(configInitCmd)
rootCmd.AddCommand(serveCmd)
rootCmd.AddCommand(configCmd)
}
func main() {
+6 -1
View File
@@ -9,9 +9,12 @@ require (
github.com/spf13/cobra v1.10.2
)
require go.yaml.in/yaml/v4 v4.0.0-rc.4 // indirect
exclude google.golang.org/grpc/stats/opentelemetry v0.0.0-20240907200651-3ffb98b2c93a
require (
atcr.io v0.0.0
cel.dev/expr v0.25.1 // indirect
cloud.google.com/go v0.123.0 // indirect
cloud.google.com/go/auth v0.18.1 // indirect
@@ -248,7 +251,7 @@ require (
github.com/spf13/afero v1.15.0 // indirect
github.com/spf13/cast v1.10.0 // indirect
github.com/spf13/pflag v1.0.10 // indirect
github.com/spf13/viper v1.21.0 // indirect
github.com/spf13/viper v1.21.0
github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/sylabs/sif/v2 v2.23.0 // indirect
@@ -304,3 +307,5 @@ require (
modernc.org/memory v1.11.0 // indirect
modernc.org/sqlite v1.45.0 // indirect
)
replace atcr.io => ../
+16 -14
View File
@@ -243,8 +243,8 @@ github.com/bodgit/windows v1.0.1 h1:tF7K6KOluPYygXa3Z2594zxlkbKPAOvqr97etrGNIz4=
github.com/bodgit/windows v1.0.1/go.mod h1:a6JLwrB4KrTR5hBpp8FI9/9W9jJfeQ2h4XDXU74ZCdM=
github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M=
github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0=
github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
@@ -571,8 +571,8 @@ github.com/gpustack/gguf-parser-go v0.24.0 h1:tdJceXYp9e5RhE9RwVYIuUpir72Jz2D68N
github.com/gpustack/gguf-parser-go v0.24.0/go.mod h1:y4TwTtDqFWTK+xvprOjRUh+dowgU2TKCX37vRKvGiZ0=
github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo=
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.1 h1:e9Rjr40Z98/clHv5Yg79Is0NtosR5LXRvdr7o/6NwbA=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.1/go.mod h1:tIxuGz/9mpox++sgp9fJjHO0+q1X9/UOWd798aAm22M=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.8 h1:NpbJl/eVbvrGE0MJ6X16X9SAifesl6Fwxg/YmCvubRI=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.8/go.mod h1:mi7YA+gCzVem12exXy46ZespvGtX/lZmD/RLnQhVW7U=
github.com/hako/durafmt v0.0.0-20210608085754-5c1018a4e16b h1:wDUNC2eKiL35DbLvsDhiblTUXHxcOPwQSCzi7xpQUN4=
github.com/hako/durafmt v0.0.0-20210608085754-5c1018a4e16b/go.mod h1:VzxiSdG6j1pi7rwGm/xYI5RbtpBgM8sARDXlvEvxlu0=
github.com/hashicorp/aws-sdk-go-base/v2 v2.0.0-beta.70 h1:0HADrxxqaQkGycO1JoUUA+B4FnIkuo8d2bz/hSaTFFQ=
@@ -845,8 +845,8 @@ github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8b
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws=
github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
@@ -1009,12 +1009,12 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 h1:7iP2uCb
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0/go.mod h1:c7hN3ddxs/z6q9xwvfLPk+UHlWRQyaeR1LdgfL/66l0=
go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms=
go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0 h1:1fTNlAIJZGWLP5FVu0fikVry1IsiUnXjf7QFvoNN3Xw=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0/go.mod h1:zjPK58DtkqQFn+YUMbx0M2XV3QgKU0gS9LeGohREyK4=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.35.0 h1:xJ2qHD0C1BeYVTLLR9sX12+Qb95kfeD/byKj6Ky1pXg=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.35.0/go.mod h1:u5BF1xyjstDowA1R5QAO9JHzqK+ublenEW/dyqTjBVk=
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0 h1:5gn2urDL/FBnK8OkCfD1j3/ER79rUuTYmCvlXBKeYL8=
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0/go.mod h1:0fBG6ZJxhqByfFZDwSwpZGzJU671HkwpWaNe2t4VUPI=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 h1:QKdN8ly8zEMrByybbQgv8cWBcdAarwmIPZ6FThrWXJs=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0/go.mod h1:bTdK1nhqF76qiPoCCdyFIV+N/sRHYXYCTQc+3VCi3MI=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.40.0 h1:wVZXIWjQSeSmMoxF74LzAnpVQOAFDo3pPji9Y4SOFKc=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.40.0/go.mod h1:khvBS2IggMFNwZK/6lEeHg/W57h/IX6J4URh57fuI40=
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.40.0 h1:ZrPRak/kS4xI3AVXy8F7pipuDXmDsrO8Lg+yQjBLjw0=
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.40.0/go.mod h1:3y6kQCWztq6hyW8Z9YxQDDm0Je9AJoFar2G0yDcmhRk=
go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g=
go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc=
go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8=
@@ -1024,8 +1024,8 @@ go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2
go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw=
go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA=
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
go.opentelemetry.io/proto/otlp v1.7.1 h1:gTOMpGDb0WTBOP8JaO72iL3auEZhVmAQg4ipjOVAtj4=
go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE=
go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A=
go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
@@ -1033,6 +1033,8 @@ go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9i
go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
go.yaml.in/yaml/v4 v4.0.0-rc.4 h1:UP4+v6fFrBIb1l934bDl//mmnoIZEDK0idg1+AIvX5U=
go.yaml.in/yaml/v4 v4.0.0-rc.4/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
go4.org v0.0.0-20260112195520-a5071408f32f h1:ziUVAjmTPwQMBmYR1tbdRFJPtTcQUI12fH9QQjfb0Sw=
go4.org v0.0.0-20260112195520-a5071408f32f/go.mod h1:ZRJnO5ZI4zAwMFp+dS1+V6J6MSyAowhRqAE+DPa1Xp0=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+104 -67
View File
@@ -1,83 +1,120 @@
// Package config provides environment-based configuration for the scanner service.
// Package config provides Viper-based configuration for the scanner service.
package config
import (
"fmt"
"os"
"strconv"
"github.com/spf13/viper"
"atcr.io/pkg/config"
)
// Config holds all scanner configuration
// Config holds all scanner configuration.
type Config struct {
// Addr is the HTTP address for the health endpoint
Addr string
// HoldURL is the WebSocket URL of the hold service
HoldURL string
// SharedSecret is the shared secret for scanner authentication
SharedSecret string
// Workers is the number of concurrent scan workers
Workers int
// QueueSize is the maximum priority queue depth
QueueSize int
// VulnEnabled enables Grype vulnerability scanning
VulnEnabled bool
// VulnDBPath is the directory for the Grype vulnerability database
VulnDBPath string
// TmpDir is the directory for temporary layer extraction
TmpDir string
Version string `yaml:"version" comment:"Configuration format version."`
LogLevel string `yaml:"log_level" comment:"Log level: debug, info, warn, error."`
LogShipper config.LogShipperConfig `yaml:"log_shipper" comment:"Remote log shipping settings."`
Server ServerConfig `yaml:"server" comment:"Health endpoint settings."`
Hold HoldConfig `yaml:"hold" comment:"Hold service connection settings."`
Scanner ScannerConfig `yaml:"scanner" comment:"Worker pool settings."`
Vuln VulnConfig `yaml:"vuln" comment:"Vulnerability scanning (Grype) settings."`
}
// Load reads configuration from environment variables with SCANNER_ prefix
func Load() (*Config, error) {
cfg := &Config{
Addr: envOr("SCANNER_ADDR", ":9090"),
HoldURL: os.Getenv("SCANNER_HOLD_URL"),
SharedSecret: os.Getenv("SCANNER_SHARED_SECRET"),
Workers: envIntOr("SCANNER_WORKERS", 2),
QueueSize: envIntOr("SCANNER_QUEUE_SIZE", 100),
VulnEnabled: envBoolOr("SCANNER_VULN_ENABLED", true),
VulnDBPath: envOr("SCANNER_VULN_DB_PATH", "/var/lib/atcr-scanner/vulndb"),
TmpDir: envOr("SCANNER_TMP_DIR", "/var/lib/atcr-scanner/tmp"),
// ServerConfig defines the health endpoint settings.
type ServerConfig struct {
// Listen address for the health endpoint.
Addr string `yaml:"addr" comment:"Listen address for the health endpoint, e.g. \":9090\"."`
}
// HoldConfig defines the hold service connection.
type HoldConfig struct {
// WebSocket URL of the hold service.
URL string `yaml:"url" comment:"WebSocket URL of the hold service (REQUIRED), e.g. \"ws://localhost:8080\"."`
// Shared secret for scanner authentication.
Secret string `yaml:"secret" comment:"Shared secret for scanner WebSocket auth (REQUIRED)."`
}
// ScannerConfig defines worker pool settings.
type ScannerConfig struct {
// Number of concurrent scan workers.
Workers int `yaml:"workers" comment:"Number of concurrent scan workers."`
// Maximum priority queue depth.
QueueSize int `yaml:"queue_size" comment:"Maximum priority queue depth."`
}
// VulnConfig defines vulnerability scanning settings.
type VulnConfig struct {
// Enable Grype vulnerability scanning.
Enabled bool `yaml:"enabled" comment:"Enable Grype vulnerability scanning."`
// Directory for the Grype vulnerability database.
DBPath string `yaml:"db_path" comment:"Directory for the Grype vulnerability database."`
// Directory for temporary layer extraction.
TmpDir string `yaml:"tmp_dir" comment:"Directory for temporary layer extraction."`
}
// setScannerDefaults registers all default values on the given Viper instance.
func setScannerDefaults(v *viper.Viper) {
v.SetDefault("version", "0.1")
v.SetDefault("log_level", "info")
// Server defaults
v.SetDefault("server.addr", ":9090")
// Hold defaults
v.SetDefault("hold.url", "")
v.SetDefault("hold.secret", "")
// Scanner defaults
v.SetDefault("scanner.workers", 2)
v.SetDefault("scanner.queue_size", 100)
// Vuln defaults
v.SetDefault("vuln.enabled", true)
v.SetDefault("vuln.db_path", "/var/lib/atcr-scanner/vulndb")
v.SetDefault("vuln.tmp_dir", "/var/lib/atcr-scanner/tmp")
// Log shipper defaults
v.SetDefault("log_shipper.batch_size", 100)
v.SetDefault("log_shipper.flush_interval", "5s")
}
// DefaultConfig returns a Config populated with all default values (no validation).
func DefaultConfig() *Config {
v := config.NewViper("SCANNER", "")
setScannerDefaults(v)
cfg := &Config{}
_ = v.Unmarshal(cfg, config.UnmarshalOption())
return cfg
}
// ExampleYAML returns a fully-commented YAML configuration with default values.
func ExampleYAML() ([]byte, error) {
return config.MarshalCommentedYAML("ATCR Scanner Configuration", DefaultConfig())
}
// LoadConfig builds a complete configuration using Viper layered loading:
// defaults -> YAML file -> environment variables.
// yamlPath is optional; empty string means env-only (backward compatible).
func LoadConfig(yamlPath string) (*Config, error) {
v := config.NewViper("SCANNER", yamlPath)
setScannerDefaults(v)
cfg := &Config{}
if err := v.Unmarshal(cfg, config.UnmarshalOption()); err != nil {
return nil, fmt.Errorf("failed to unmarshal config: %w", err)
}
if cfg.HoldURL == "" {
return nil, fmt.Errorf("SCANNER_HOLD_URL is required")
if cfg.Hold.URL == "" {
return nil, fmt.Errorf("hold.url is required (env: SCANNER_HOLD_URL)")
}
if cfg.SharedSecret == "" {
return nil, fmt.Errorf("SCANNER_SHARED_SECRET is required")
if cfg.Hold.Secret == "" {
return nil, fmt.Errorf("hold.secret is required (env: SCANNER_HOLD_SECRET)")
}
return cfg, nil
}
func envOr(key, fallback string) string {
if v := os.Getenv(key); v != "" {
return v
}
return fallback
}
func envIntOr(key string, fallback int) int {
if v := os.Getenv(key); v != "" {
if n, err := strconv.Atoi(v); err == nil {
return n
}
}
return fallback
}
func envBoolOr(key string, fallback bool) bool {
if v := os.Getenv(key); v != "" {
if b, err := strconv.ParseBool(v); err == nil {
return b
}
}
return fallback
}
+8 -8
View File
@@ -36,21 +36,21 @@ func NewWorkerPool(cfg *config.Config, q *queue.JobQueue, c *client.HoldClient)
// Start launches worker goroutines
func (wp *WorkerPool) Start(ctx context.Context) {
// Initialize vuln database on startup if enabled
if wp.cfg.VulnEnabled {
if wp.cfg.Vuln.Enabled {
go func() {
if err := initializeVulnDatabase(wp.cfg.VulnDBPath, wp.cfg.TmpDir); err != nil {
if err := initializeVulnDatabase(wp.cfg.Vuln.DBPath, wp.cfg.Vuln.TmpDir); err != nil {
slog.Error("Failed to initialize vulnerability database", "error", err)
slog.Warn("Vulnerability scanning will be disabled until database is available")
}
}()
}
for i := 0; i < wp.cfg.Workers; i++ {
for i := 0; i < wp.cfg.Scanner.Workers; i++ {
wp.wg.Add(1)
go wp.worker(ctx, i)
}
slog.Info("Scanner worker pool started", "workers", wp.cfg.Workers)
slog.Info("Scanner worker pool started", "workers", wp.cfg.Scanner.Workers)
}
// Wait blocks until all workers finish
@@ -100,13 +100,13 @@ func (wp *WorkerPool) processJob(ctx context.Context, job *scanner.ScanJob) (*sc
startTime := time.Now()
// Ensure tmp dir exists
if err := ensureDir(wp.cfg.TmpDir); err != nil {
if err := ensureDir(wp.cfg.Vuln.TmpDir); err != nil {
return nil, fmt.Errorf("failed to create tmp dir: %w", err)
}
// Step 1: Extract image layers from hold via presigned URLs
slog.Info("Extracting image layers", "repository", job.Repository)
imageDir, cleanup, err := extractLayers(job, wp.cfg.TmpDir)
imageDir, cleanup, err := extractLayers(job, wp.cfg.Vuln.TmpDir)
if err != nil {
return nil, fmt.Errorf("failed to extract layers: %w", err)
}
@@ -126,9 +126,9 @@ func (wp *WorkerPool) processJob(ctx context.Context, job *scanner.ScanJob) (*sc
}
// Step 3: Scan SBOM with Grype (if enabled)
if wp.cfg.VulnEnabled {
if wp.cfg.Vuln.Enabled {
slog.Info("Scanning for vulnerabilities", "repository", job.Repository)
vulnJSON, vulnDigest, summary, err := scanVulnerabilities(ctx, sbomResult, wp.cfg.VulnDBPath)
vulnJSON, vulnDigest, summary, err := scanVulnerabilities(ctx, sbomResult, wp.cfg.Vuln.DBPath)
if err != nil {
return nil, fmt.Errorf("failed to scan vulnerabilities: %w", err)
}