fix(harden): keep hardening and the ntfy notifier alive when sshguard can't install

On a fresh AlmaLinux 9.8 box, install_bruteforce_protection ran unguarded under
'set -euo pipefail'. When sshguard (from EPEL) wasn't installable at that moment,
the single failed dnf aborted the ENTIRE harden run before it wrote sshd_config
or installed the pam_exec login notifier -- leaving a stock, unhardened box and a
silently-missing ntfy hook.

- oslib: install the iptables backend best-effort first, then sshguard, and
  return sshguard's status so callers can treat it as non-fatal.
- harden-ssh/harden-jumphost: install_openssh now dies with a clear message on
  failure; sshguard is '|| warn' so sshd hardening and the notifier still apply.
- install_login_notifier verifies the script + pam hook landed and logs
  'Login notifier ACTIVE' (or a loud warning) instead of failing silently.
- ntfy-ssh-login.sh: NTFY_DEBUG=1 logs delivery attempts + curl errors to
  /var/log/ssh-notify.log so the next silent failure leaves a trace.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

scripts/harden-jumphost.sh

@@ -57,8 +57,11 @@ log "Detected OS: ${OS_ID} (family ${OS_FAMILY}, init ${INIT_SYSTEM})"
 # 1. Packages
 # ----------------------------------------------------------------------------
 log "Installing OpenSSH + sshguard + iptables..."
-install_openssh
-install_bruteforce_protection
+install_openssh || die "OpenSSH packages failed to install; cannot harden. Fix the package error above, then re-run."
+# sshguard is best-effort (see harden-ssh.sh): never let a missing brute-force
+# package abort the whole bastion hardening.
+install_bruteforce_protection \
+    || warn "sshguard not installed; brute-force protection is OFF. Add it later with: dnf install -y epel-release sshguard. Continuing with the rest of the hardening."
 ensure_gum || warn "gum not installed; sshuser will use its CLI mode."
 
 SFTP_PATH="$(sftp_server_path)"

scripts/harden-ssh.sh

@@ -64,7 +64,7 @@ log "Detected OS: ${OS_ID} (family ${OS_FAMILY}, init ${INIT_SYSTEM})"
 # ----------------------------------------------------------------------------
 if ! command -v ssh >/dev/null 2>&1; then
     log "ssh not found; installing openssh..."
-    install_openssh
+    install_openssh || die "Could not install OpenSSH; cannot harden. Fix the package error above, then re-run."
 fi
 
 log "Checking OpenSSH version supports PQ KEX..."
@@ -89,8 +89,12 @@ KEX_LIST=""
 # 2. Install packages (OS-gated inside oslib)
 # ----------------------------------------------------------------------------
 log "Installing OpenSSH server + sshguard + iptables..."
-install_openssh
-install_bruteforce_protection
+install_openssh || die "OpenSSH packages failed to install; cannot harden SSH. Fix the package error above, then re-run."
+# sshguard is best-effort: a host where it can't install right now (e.g. EPEL
+# momentarily unreachable) must still get the sshd_config hardening AND the login
+# notifier -- not a silently half-configured box. Warn and press on.
+install_bruteforce_protection \
+    || warn "sshguard not installed; brute-force protection is OFF. Add it later with: dnf install -y epel-release sshguard. Continuing with the rest of the hardening."
 
 # The external SFTP subsystem binary path differs per distro.
 SFTP_PATH="$(sftp_server_path)"

scripts/ntfy-ssh-login.sh

@@ -123,5 +123,20 @@ tags="warning"
 [ -n "${NTFY_REGION:-}" ] && tags="${tags},${NTFY_REGION}"
 set -- "$@" -H "X-Tags: ${tags}"
 
-curl "$@" -d "$body" "$NTFY_URL" >/dev/null 2>&1 || true
+# Deliver. Failures are non-fatal -- a login must never be blocked by a notifier
+# hiccup. Set NTFY_DEBUG=1 in the conf to log attempts + curl errors to
+# /var/log/ssh-notify.log, so a silent failure (SELinux, egress, bad token, ...)
+# leaves a trace instead of vanishing.
+if [ "${NTFY_DEBUG:-0}" = "1" ]; then
+    log=/var/log/ssh-notify.log
+    printf '%s login user=%s rhost=%s -> %s\n' \
+        "$(date -u +%FT%TZ 2>/dev/null || echo)" "$user" "$rhost" "$NTFY_URL" >> "$log" 2>/dev/null || true
+    if curl "$@" -d "$body" "$NTFY_URL" >>"$log" 2>&1; then
+        echo "  -> delivered" >> "$log" 2>/dev/null || true
+    else
+        echo "  -> curl FAILED (exit $?)" >> "$log" 2>/dev/null || true
+    fi
+else
+    curl "$@" -d "$body" "$NTFY_URL" >/dev/null 2>&1 || true
+fi
 exit 0

scripts/oslib.sh

@@ -403,14 +403,21 @@ install_openssh() {
 }
 
 # Install sshguard + an iptables firewall backend. On RHEL/Alma sshguard lives
-# in EPEL, so enable that first.
+# in EPEL, so enable that first. The iptables backend is installed best-effort
+# FIRST (it's usually already present as iptables-nft), then sshguard, and the
+# function RETURNS sshguard's status -- so a caller can treat a sshguard miss
+# (e.g. EPEL momentarily unreachable) as non-fatal and still apply the rest of
+# the hardening instead of aborting the whole run.
 install_bruteforce_protection() {
     _require_detected
     case "$OS_FAMILY" in
-        alpine) pkg_install sshguard iptables ip6tables ;;
-        debian) pkg_install sshguard iptables ;;
-        rhel)   pkg_install epel-release || true
-                pkg_install sshguard iptables ;;
+        alpine) pkg_install iptables ip6tables || true
+                pkg_install sshguard ;;
+        debian) pkg_install iptables || true
+                pkg_install sshguard ;;
+        rhel)   pkg_install iptables || true        # el9+: provided by iptables-nft
+                pkg_install epel-release || true     # sshguard lives in EPEL
+                pkg_install sshguard ;;
     esac
 }
 
@@ -538,6 +545,8 @@ NTFY_PRIORITY="${NTFY_PRIORITY:-min}"
 NTFY_REGION="${NTFY_REGION:-}"
 NOTIFY_GROUPS="${NOTIFY_GROUPS:-}"
 NOTIFY_PRIORITY_MAP="${NOTIFY_PRIORITY_MAP:-}"
+# Set to 1 to log every delivery attempt (and curl errors) to /var/log/ssh-notify.log.
+NTFY_DEBUG="${NTFY_DEBUG:-0}"
 CONF
         )
         chmod 600 /etc/ssh-notify.conf
@@ -556,6 +565,17 @@ CONF
         _warn "$pam not found; add this line to your sshd PAM stack manually:"
         _warn "  $line"
     fi
+
+    # Verify the hook actually landed and report loudly. A notifier that fails to
+    # install silently is worse than none -- you'd believe logins are watched
+    # when they aren't (exactly the trap that hid this on the first Alma run).
+    if [[ -x /opt/scripts/ntfy-ssh-login.sh ]] \
+        && grep -qF '/opt/scripts/ntfy-ssh-login.sh' "$pam" 2>/dev/null; then
+        _log "Login notifier ACTIVE -> ${NTFY_URL:-<NTFY_URL unset!>}"
+        return 0
+    fi
+    _warn "Login notifier did NOT fully install (script or pam hook missing)."
+    return 1
 }
 
 # Locate the sshguard iptables backend binary (path varies by packaging).