automations/scripts/harden-jumphost.sh

#!/usr/bin/env bash
#
# harden-jumphost.sh
#
# Hardens a box for use as an SSH jump host (bastion) on Alpine, Debian, or
# Alma Linux. Layered on the same PQ-hybrid posture as harden-ssh.sh, plus
# jump-host specifics. All distro differences go through scripts/oslib.sh.
#
# Two groups, two privilege levels:
#   ssh-admins   -- full TTY shell on the jump host (maintenance only). No
#                   forwarding. For fixing the box, not reaching elsewhere.
#   ssh-jumpers  -- ProxyJump ONLY. No TTY, no shell, no SFTP, no agent
#                   forwarding. Only direct-tcpip to whitelisted targets.
#
# How the restriction works:
#   - Global default: DisableForwarding yes, PermitTTY no, ForceCommand
#     <nologin>. A user in neither group can do nothing.
#   - Match Group ssh-admins: re-enables PermitTTY, clears ForceCommand.
#   - Match Group ssh-jumpers: enables AllowTcpForwarding + a PermitOpen
#     whitelist, but keeps PermitTTY no + ForceCommand <nologin>. ProxyJump
#     (direct-tcpip) works because it never opens a session channel, so
#     ForceCommand never fires.
#
# Usage:
#   bash harden-jumphost.sh
#   SSH_PORT=2222 bash harden-jumphost.sh
#   JUMP_TARGETS="10.0.0.5:22 10.0.0.6:22" bash harden-jumphost.sh
#   ALLOWED_IP=1.2.3.4 bash harden-jumphost.sh
#   FORCE=1 bash harden-jumphost.sh

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=scripts/oslib.sh
. "$SCRIPT_DIR/oslib.sh"

# ============================================================================
# CONFIG
# ============================================================================
: "${SSH_PORT:=22}"
: "${ALLOWED_IP:=}"
: "${FORCE:=0}"
# Space-separated host:port list jumpers can reach via ProxyJump. Empty means
# deny-all. e.g. "10.0.0.5:22 10.0.0.6:22".
: "${JUMP_TARGETS:=}"
: "${KEY_COMMENT:=root@$(hostname)-$(date +%Y%m%d)}"

log()  { _log  "$@"; }
warn() { _warn "$@"; }
die()  { _die  "$@"; }

[[ $EUID -eq 0 ]] || die "Run as root."
os_detect
log "Detected OS: ${OS_ID} (family ${OS_FAMILY}, init ${INIT_SYSTEM})"

# ----------------------------------------------------------------------------
# 1. Packages
# ----------------------------------------------------------------------------
log "Installing OpenSSH + sshguard + iptables..."
install_openssh || die "OpenSSH packages failed to install; cannot harden. Fix the package error above, then re-run."
# sshguard is best-effort (see harden-ssh.sh): never let a missing brute-force
# package abort the whole bastion hardening.
install_bruteforce_protection \
    || warn "sshguard not installed; brute-force protection is OFF. Add it later with: dnf install -y epel-release sshguard. Continuing with the rest of the hardening."
ensure_gum || warn "gum not installed; sshuser will use its CLI mode."

SFTP_PATH="$(sftp_server_path)"
NOLOGIN="$(nologin_path)"      # /sbin/nologin (Alpine/Alma) or /usr/sbin/nologin (Debian)

# Install the sshuser tool alongside this script if present.
if [[ -f "$SCRIPT_DIR/sshuser.sh" ]]; then
    install -m 0755 "$SCRIPT_DIR/sshuser.sh" /usr/local/bin/sshuser
    log "Installed /usr/local/bin/sshuser"
fi

# ----------------------------------------------------------------------------
# 2. PQ KEX detection
# ----------------------------------------------------------------------------
log "Checking OpenSSH version supports PQ KEX..."
SSH_VER=$(ssh -V 2>&1 | grep -oE 'OpenSSH_[0-9]+\.[0-9]+' | head -1 | sed 's/OpenSSH_//')
SSH_MAJOR=${SSH_VER%%.*}
SSH_MINOR=${SSH_VER##*.}
HAS_MLKEM=0; HAS_SNTRUP=0
[[ $SSH_MAJOR -gt 9 || ( $SSH_MAJOR -eq 9 && $SSH_MINOR -ge 0 ) ]] && HAS_SNTRUP=1
[[ $SSH_MAJOR -gt 9 || ( $SSH_MAJOR -eq 9 && $SSH_MINOR -ge 9 ) ]] && HAS_MLKEM=1
[[ $HAS_SNTRUP -eq 1 || $HAS_MLKEM -eq 1 ]] || die "OpenSSH ${SSH_VER} has no PQ KEX. Need >= 9.0."
log "OpenSSH ${SSH_VER}: ML-KEM=${HAS_MLKEM} sntrup761=${HAS_SNTRUP}"

KEX_LIST=""
[[ $HAS_MLKEM  -eq 1 ]] && KEX_LIST="mlkem768x25519-sha256"
[[ $HAS_SNTRUP -eq 1 ]] && KEX_LIST="${KEX_LIST:+$KEX_LIST,}sntrup761x25519-sha512"

# ----------------------------------------------------------------------------
# 3. Host keys (Ed25519 only)
# ----------------------------------------------------------------------------
log "Regenerating host keys (Ed25519 only)..."
rm -f /etc/ssh/ssh_host_rsa_key* /etc/ssh/ssh_host_ecdsa_key* /etc/ssh/ssh_host_dsa_key*
if [[ ! -f /etc/ssh/ssh_host_ed25519_key ]]; then
    ssh-keygen -q -t ed25519 -f /etc/ssh/ssh_host_ed25519_key -N "" \
        -C "host@$(hostname)-$(date +%Y%m%d)"
fi
chmod 600 /etc/ssh/ssh_host_ed25519_key
chmod 644 /etc/ssh/ssh_host_ed25519_key.pub
log "Host key fingerprint:"
ssh-keygen -l -f /etc/ssh/ssh_host_ed25519_key.pub | sed 's/^/    /'
sshd_disable_keygen   # Alpine-only; no-op on systemd

# ----------------------------------------------------------------------------
# 4. Groups
# ----------------------------------------------------------------------------
log "Ensuring groups ssh-admins and ssh-jumpers exist..."
group_add_system ssh-admins
group_add_system ssh-jumpers

# ----------------------------------------------------------------------------
# 5. Root keypair (for ssh-admins maintenance access)
# ----------------------------------------------------------------------------
log "Generating Ed25519 keypair for root..."
mkdir -p /root/.ssh
chmod 700 /root/.ssh
touch /root/.ssh/authorized_keys
chmod 600 /root/.ssh/authorized_keys
user_add_to_group root ssh-admins

TMP_KEY=$(mktemp -u /tmp/root_ed25519.XXXXXX)
ssh-keygen -q -t ed25519 -f "$TMP_KEY" -N "" -C "$KEY_COMMENT"
ROOT_PUB=$(cat "${TMP_KEY}.pub")
ROOT_PRIV=$(cat "$TMP_KEY")
grep -qxF "$ROOT_PUB" /root/.ssh/authorized_keys || echo "$ROOT_PUB" >> /root/.ssh/authorized_keys

# Seed root (ssh-admins) with the shared admin keys from globals/ so the
# bastion has a known, secure default login. Best-effort.
if [[ "${SEED_KEYS:-1}" == "1" && -f "$SCRIPT_DIR/lib.sh" ]]; then
    # shellcheck source=scripts/lib.sh
    . "$SCRIPT_DIR/lib.sh"
    load_globals
    if declare -f resolve_ssh_keys >/dev/null 2>&1; then
        SEEDED=0
        while IFS= read -r k; do
            [[ -n "$k" ]] || continue
            grep -qxF "$k" /root/.ssh/authorized_keys || { echo "$k" >> /root/.ssh/authorized_keys; SEEDED=$((SEEDED+1)); }
        done <<< "$(resolve_ssh_keys 2>/dev/null || true)"
        [[ "$SEEDED" -gt 0 ]] && log "Seeded ${SEEDED} admin key(s) into /root/.ssh/authorized_keys from globals."
    fi
fi

# ----------------------------------------------------------------------------
# 6. PermitOpen line from JUMP_TARGETS (space-separated -> comma-separated)
# ----------------------------------------------------------------------------
PERMIT_OPEN_LINE="none"
[[ -n "$JUMP_TARGETS" ]] && PERMIT_OPEN_LINE=$(echo "$JUMP_TARGETS" | tr -s ' ' ',' | sed 's/^,//;s/,$//')

# ----------------------------------------------------------------------------
# 7. sshd_config
# ----------------------------------------------------------------------------
log "Writing /etc/ssh/sshd_config..."
[[ -f /etc/ssh/sshd_config.orig ]] || cp /etc/ssh/sshd_config /etc/ssh/sshd_config.orig

cat > /etc/ssh/sshd_config <<EOF
# Generated by harden-jumphost.sh on ${OS_ID} -- $(date -u +%FT%TZ)
# Original config preserved at /etc/ssh/sshd_config.orig

Port ${SSH_PORT}
AddressFamily any
ListenAddress 0.0.0.0
ListenAddress ::
PidFile /run/sshd.pid

# VERBOSE so the auth log records key fingerprints and direct-tcpip targets
# (used by the login notifier to report the key and best-effort jump target).
LogLevel VERBOSE

# --- Host key: Ed25519 only ---
HostKey /etc/ssh/ssh_host_ed25519_key

# --- Post-quantum hybrid KEX only ---
KexAlgorithms ${KEX_LIST}

# --- Modern ciphers and MACs ---
Ciphers chacha20-poly1305@openssh.com,aes256-gcm@openssh.com,aes128-gcm@openssh.com
MACs    hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com,umac-128-etm@openssh.com

# --- Algorithms for host and client signatures ---
HostKeyAlgorithms ssh-ed25519,ssh-ed25519-cert-v01@openssh.com
PubkeyAcceptedAlgorithms ssh-ed25519,ssh-ed25519-cert-v01@openssh.com,sk-ssh-ed25519@openssh.com,sk-ssh-ed25519-cert-v01@openssh.com,rsa-sha2-512,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256,rsa-sha2-256-cert-v01@openssh.com
RequiredRSASize 4096

# --- Authentication ---
PermitRootLogin prohibit-password
PubkeyAuthentication yes
PasswordAuthentication no
PermitEmptyPasswords no
ChallengeResponseAuthentication no
KbdInteractiveAuthentication no
UsePAM yes
AuthenticationMethods publickey
MaxAuthTries 3
MaxSessions 10
LoginGraceTime 30s
# Expose the authenticated key (file at \$SSH_USER_AUTH) for the notifier.
ExposeAuthInfo yes

# --- Default posture: deny everything ---
# Anyone not matched below gets nothing. The per-group Match blocks
# selectively re-enable what each group needs.
DisableForwarding yes
PermitTTY no
ForceCommand ${NOLOGIN}
X11Forwarding no
GatewayPorts no
PermitTunnel no
PermitUserRC no
PermitListen none
PermitOpen none
AllowAgentForwarding no
AllowTcpForwarding no
AllowStreamLocalForwarding no
PrintMotd no
TCPKeepAlive no
ClientAliveInterval 300
ClientAliveCountMax 2

# --- Misc ---
StrictModes yes
IgnoreRhosts yes
HostbasedAuthentication no
Compression no
Banner none

# --- Subsystems ---
# SFTP off by default on a jump host; jumpers must never have it. If you
# enable it for admins, log transfers to AUTHPRIV at INFO:
# Subsystem sftp ${SFTP_PATH} -f AUTHPRIV -l INFO

# --- Allowlist ---
AllowGroups ssh-admins ssh-jumpers

# ============================================================================
# Match blocks
# ============================================================================

# --- Admins: full shell, no forwarding, no SFTP ---
Match Group ssh-admins
    PermitTTY yes
    ForceCommand none
    DisableForwarding yes
    AllowTcpForwarding no
    AllowAgentForwarding no
    AllowStreamLocalForwarding no
    X11Forwarding no
    PermitOpen none

# --- Jumpers: ProxyJump only, no shell, whitelisted destinations ---
Match Group ssh-jumpers
    PermitTTY no
    ForceCommand ${NOLOGIN}
    AllowTcpForwarding yes
    PermitOpen ${PERMIT_OPEN_LINE}
    AllowAgentForwarding no
    AllowStreamLocalForwarding no
    DisableForwarding no
    X11Forwarding no
    GatewayPorts no
    PermitTunnel no
EOF

# ----------------------------------------------------------------------------
# 8. Validate config
# ----------------------------------------------------------------------------
log "Validating sshd config..."
if ! sshd -t 2>/tmp/sshd-test.err; then
    cat /tmp/sshd-test.err >&2
    cp /etc/ssh/sshd_config.orig /etc/ssh/sshd_config
    die "sshd config invalid; restored original. NOT reloading."
fi
rm -f /tmp/sshd-test.err

# ----------------------------------------------------------------------------
# 9. sshguard
# ----------------------------------------------------------------------------
log "Configuring sshguard..."
mkdir -p /etc/sshguard
WHITELIST=/etc/sshguard/whitelist
{
    echo "127.0.0.1"; echo "::1"
    [[ -n "$ALLOWED_IP" ]] && echo "$ALLOWED_IP"
} > "$WHITELIST"

SSHGUARD_BACKEND="$(sshguard_backend)"
SSHGUARD_LOGREADER="$(sshguard_logreader)"
[[ -x "${SSHGUARD_BACKEND}" ]] || warn "sshguard backend not found at ${SSHGUARD_BACKEND}; brute-force blocking may be inactive."

cat > /etc/sshguard/sshguard.conf <<EOF
BACKEND="${SSHGUARD_BACKEND}"
${SSHGUARD_LOGREADER:+LOGREADER="${SSHGUARD_LOGREADER}"}
THRESHOLD=30
BLOCK_TIME=300
DETECTION_TIME=1800
PID_FILE=/run/sshguard.pid
WHITELIST_FILE=${WHITELIST}
EOF

# INPUT -> sshguard jump. When the host firewall (harden-firewall.sh) is enabled
# it owns the whole INPUT chain -- including this jump -- and persists it via the
# distro's native iptables package, so we install the firewall and skip the
# standalone boot hook. Otherwise fall back to the minimal init-agnostic boot
# hook that just (re)inserts the jump at every boot.
: "${ENABLE_FIREWALL:=1}"
if [[ "$ENABLE_FIREWALL" == "1" && -f "$SCRIPT_DIR/harden-firewall.sh" ]]; then
    log "Installing host firewall (deny-by-default INPUT; carries the sshguard jump)..."
    SSH_PORT="$SSH_PORT" OPEN_PORTS="${OPEN_PORTS:-}" \
    FW_SSH_SOURCE="${FW_SSH_SOURCE:-}" FW_ALLOW_PING="${FW_ALLOW_PING:-1}" \
    FORCE=1 bash "$SCRIPT_DIR/harden-firewall.sh" apply \
        || warn "harden-firewall.sh failed; INPUT left unfiltered. Re-run it manually."
else
    HOOK=$(mktemp)
    cat > "$HOOK" <<'EOF'
#!/bin/sh
SSH_PORT=$(awk '/^Port / {print $2; exit}' /etc/ssh/sshd_config)
SSH_PORT=${SSH_PORT:-22}
for ipt in iptables ip6tables; do
    command -v "$ipt" >/dev/null 2>&1 || continue
    $ipt -N sshguard 2>/dev/null || true
    $ipt -C INPUT -p tcp --dport "$SSH_PORT" -j sshguard 2>/dev/null \
        || $ipt -I INPUT -p tcp --dport "$SSH_PORT" -j sshguard
done
EOF
    install_boot_hook sshguard-iptables "$HOOK"
    rm -f "$HOOK"
fi

svc_enable_start sshguard || warn "Could not start sshguard; check sshguard.conf on this distro."

# ----------------------------------------------------------------------------
# 9b. Optional: SSH login notifier (pam_exec -> ntfy)
# ----------------------------------------------------------------------------
# Enabled when NTFY_URL is provided. On a bastion we default to notifying for
# the two SSH groups and tag the alert with this host's region.
if [[ -n "${NTFY_URL:-}" ]]; then
    : "${NOTIFY_GROUPS:=ssh-admins ssh-jumpers}"
    : "${NTFY_REGION:=$(host_region)}"
    log "Installing SSH login notifier (ntfy)..."
    install_login_notifier "$SCRIPT_DIR/ntfy-ssh-login.sh" || warn "Notifier install had issues."
fi

# ----------------------------------------------------------------------------
# 9c. Daily unattended updates (default ON -- recommended for an SSH-only
# bastion; set AUTO_UPDATE=0 to skip). New Alpine *branches* are reported, not
# auto-applied.
# ----------------------------------------------------------------------------
if [[ "${AUTO_UPDATE:-1}" == "1" && -f "$SCRIPT_DIR/auto-update.sh" ]]; then
    log "Scheduling daily auto-update (reboot only when idle)..."
    AUTO_REBOOT="${AUTO_REBOOT:-idle}" \
    ALLOW_RELEASE_UPGRADE="${ALLOW_RELEASE_UPGRADE:-0}" \
    NOTIFY="${NOTIFY:-1}" \
        bash "$SCRIPT_DIR/auto-update.sh" install || warn "Could not schedule auto-update."
fi

# ----------------------------------------------------------------------------
# 10. Enable sshd
# ----------------------------------------------------------------------------
SSHD_SVC="$(sshd_service)"
log "Enabling ${SSHD_SVC} at boot..."
svc_enable "$SSHD_SVC"

cat <<EOF

================================================================
JUMP HOST SETUP COMPLETE (${OS_ID})

Groups created:
  ssh-admins   -- full shell on the jump host (root added)
  ssh-jumpers  -- ProxyJump only, no shell

Add a jumper user (using the installed tool):
  sshuser add -u alice -r jumper -k "ssh-ed25519 AAA..."

Or an admin:
  sshuser add -u bob -r admin -k "ssh-ed25519 AAA..."

Allowed jump targets (PermitOpen):
  ${PERMIT_OPEN_LINE}

To change targets: re-run with JUMP_TARGETS set, or edit the Match
block in /etc/ssh/sshd_config directly.

COPY THIS PRIVATE KEY TO YOUR CLIENT *NOW* (admin/root key):

  ssh -i ~/.ssh/id_ed25519_jump -p ${SSH_PORT} root@<host>

----- BEGIN ROOT PRIVATE KEY (Ed25519) -----
${ROOT_PRIV}
----- END ROOT PRIVATE KEY -----

Public key (already in /root/.ssh/authorized_keys):
${ROOT_PUB}

Host fingerprint:
$(ssh-keygen -l -f /etc/ssh/ssh_host_ed25519_key.pub)

Client usage examples:
  # Admin shell on the jump host:
  ssh -i ~/.ssh/id_ed25519_jump -p ${SSH_PORT} root@<jumphost>
  # ProxyJump through to an internal target:
  ssh -J root@<jumphost>:${SSH_PORT} -i ~/.ssh/id_ed25519_target user@<target>

================================================================
EOF

shred -u "$TMP_KEY" "${TMP_KEY}.pub" 2>/dev/null || rm -f "$TMP_KEY" "${TMP_KEY}.pub"

if [[ "$FORCE" != "1" ]]; then
    cat <<EOF
sshd config has passed validation. Ready to reload sshd.

Open a SECOND terminal and verify the new key/port/KEX work BEFORE
answering yes:
  ssh -i ~/.ssh/<saved key> -p ${SSH_PORT} root@<host>

Reload sshd now? [y/N]
EOF
    read -r ans
    if [[ "${ans,,}" != "y" && "${ans,,}" != "yes" ]]; then
        warn "Skipping reload. Reload ${SSHD_SVC} manually when ready."
        exit 0
    fi
fi

log "Reloading ${SSHD_SVC}..."
svc_reload "$SSHD_SVC"
log "Done."