fix(alpine): create /usr/local/sbin, and run the PAM sshd so pam_exec fires

Two Alpine-only failures from an irc-1 run.

1. harden-firewall.sh died with

     line 161: /usr/local/sbin/firewall-apply: No such file or directory

   and left INPUT unfiltered. Alpine's baselayout ships /usr/local/{bin,lib,
   share} but no sbin, and nothing created it. Create it before writing the
   engine; same guard in oslib's install_boot_hook / install_daily_job (which
   `install` into /usr/local/sbin, and `install` does not make parent dirs) and
   for /usr/local/bin in harden-jumphost.

2. The ntfy login notifier never fired despite reporting ACTIVE. Alpine keeps
   PAM support in a SEPARATE binary: openssh-server gives /usr/sbin/sshd (no
   PAM), openssh-server-pam gives /usr/sbin/sshd.pam. The OpenRC init only
   picks the PAM one in start_pre (checkconfig -> update_command), and its
   reload/stop match the process with `start-stop-daemon --exec "$command"`.
   So a host already running /usr/sbin/sshd never swaps: reload signals a
   non-matching process, stop matches nothing, a later start hits "address
   already in use". sshd keeps serving without PAM, so /etc/pam.d/sshd -- and
   the pam_exec hook in it -- is never consulted.

   Add oslib sshd_wanted_binary / sshd_running_binary / sshd_apply_config, and
   end the harden scripts with sshd_apply_config instead of svc_reload: reload
   as before, but on Alpine stop by pidfile and start again when the running
   binary isn't the one the config calls for. Established sessions are separate
   processes and survive; only the listener blinks.

   install_login_notifier now checks that PAM can actually be reached on Alpine
   (sshd.pam present, UsePAM yes) and warns with the fix instead of reporting a
   hook that can never run as ACTIVE -- the same "silent notifier" trap the
   Alma run hit.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

scripts/harden-firewall.sh

@@ -158,6 +158,9 @@ seed_open_ports() {
 # ============================================================================
 write_apply() {
     local save_cmd; save_cmd="$(fw_save_cmd)"
+    # Alpine's baselayout ships /usr/local/{bin,lib,share} but NOT sbin, so the
+    # redirect below fails with "No such file or directory" unless we make it.
+    install -d -m 0755 "${APPLY%/*}"
     cat > "$APPLY" <<'ENGINE'
 #!/bin/sh
 # Managed by harden-firewall.sh -- do not edit by hand (regenerated on install).

scripts/harden-jumphost.sh

@@ -69,6 +69,7 @@ NOLOGIN="$(nologin_path)"      # /sbin/nologin (Alpine/Alma) or /usr/sbin/nologi
 
 # Install the sshuser tool alongside this script if present.
 if [[ -f "$SCRIPT_DIR/sshuser.sh" ]]; then
+    install -d -m 0755 /usr/local/bin
     install -m 0755 "$SCRIPT_DIR/sshuser.sh" /usr/local/bin/sshuser
     log "Installed /usr/local/bin/sshuser"
 fi
@@ -427,5 +428,7 @@ EOF
 fi
 
 log "Reloading ${SSHD_SVC}..."
-svc_reload "$SSHD_SVC"
+# Reload -- and on Alpine swap in the PAM sshd build if the running one
+# predates this config (oslib explains why that can happen).
+sshd_apply_config
 log "Done."

scripts/harden-ssh.sh

@@ -390,6 +390,8 @@ EOF
 fi
 
 log "Reloading ${SSHD_SVC}..."
-svc_reload "$SSHD_SVC"
+# Reload -- and on Alpine swap in the PAM sshd build if the running one
+# predates this config (oslib explains why that can happen).
+sshd_apply_config
 log "Done. Your session, if any, should remain alive (reload preserves connections)."
 log "Test from another machine before closing this session."

scripts/oslib.sh

@@ -188,6 +188,77 @@ sshd_disable_keygen() {
     fi
 }
 
+# ----------------------------------------------------------------------------
+# Alpine keeps PAM support in a SEPARATE binary.
+#
+#   openssh-server      -> /usr/sbin/sshd       (built WITHOUT PAM)
+#   openssh-server-pam  -> /usr/sbin/sshd.pam   (built WITH PAM)
+#
+# The OpenRC init picks between them in start_pre (checkconfig -> update_command:
+# "sshd.pam if it is executable and the config says UsePAM yes"), and its
+# reload/stop match the running process with `start-stop-daemon --exec "$command"`.
+# So a host that is ALREADY running /usr/sbin/sshd never swaps over on its own:
+# reload signals a process that doesn't match (nothing happens), stop matches
+# nothing, and a follow-up start hits "address already in use". The daemon keeps
+# serving without PAM -- which means /etc/pam.d/sshd, and therefore the pam_exec
+# login notifier, is silently never consulted. Every other distro builds PAM into
+# the one sshd binary, so this is Alpine-only.
+# ----------------------------------------------------------------------------
+sshd_wanted_binary() {  # echo the sshd binary this host's config should be running
+    _require_detected
+    if [[ "$OS_FAMILY" == alpine ]] && [[ -x /usr/sbin/sshd.pam ]]         && grep -qiE '^[[:space:]]*UsePAM[[:space:]]+yes' /etc/ssh/sshd_config 2>/dev/null; then
+        echo /usr/sbin/sshd.pam
+    else
+        echo /usr/sbin/sshd
+    fi
+}
+
+sshd_running_binary() {  # echo the executable behind the running sshd master ('' if unknown)
+    local pid="" p
+    if [[ -r /run/sshd.pid ]]; then pid="$(cat /run/sshd.pid 2>/dev/null || true)"; fi
+    if [[ -z "$pid" ]]; then
+        for p in sshd.pam sshd; do
+            pid="$(pgrep -x "$p" 2>/dev/null | head -n1 || true)"
+            [[ -n "$pid" ]] && break
+        done
+    fi
+    [[ -n "$pid" ]] || return 0
+    readlink -f "/proc/$pid/exe" 2>/dev/null || true
+}
+
+# Apply a freshly written sshd_config. Normally a reload (keeps connections).
+# On Alpine, when the running binary is not the one the config calls for, the
+# service is stopped by pidfile and started again so the PAM build takes over --
+# established sessions are separate processes and survive; only the listener
+# blinks.
+sshd_apply_config() {
+    _require_detected
+    local svc; svc="$(sshd_service)"
+    if [[ "$OS_FAMILY" == alpine ]]; then
+        local want run pid i
+        want="$(sshd_wanted_binary)"
+        run="$(sshd_running_binary)"
+        if [[ -n "$run" && "$run" != "$want" ]]; then
+            _warn "sshd is running $run, but this config needs $want -- restarting to swap it in."
+            _warn "(Existing SSH sessions survive; the listener is down for about a second.)"
+            pid=""
+            if [[ -r /run/sshd.pid ]]; then pid="$(cat /run/sshd.pid 2>/dev/null || true)"; fi
+            rc-service "$svc" stop >/dev/null 2>&1 || true
+            if [[ -n "$pid" ]]; then
+                kill "$pid" 2>/dev/null || true
+                i=0
+                while kill -0 "$pid" 2>/dev/null && [[ "$i" -lt 10 ]]; do sleep 1; i=$((i + 1)); done
+                kill -9 "$pid" 2>/dev/null || true
+            fi
+            rm -f /run/sshd.pid
+            svc_start "$svc" || _die "sshd failed to start as $want. Check: rc-service $svc start"
+            _log "sshd restarted as $(sshd_running_binary)."
+            return 0
+        fi
+    fi
+    svc_reload "$svc"
+}
+
 # ============================================================================
 # Users & groups (busybox adduser/addgroup vs shadow useradd/groupadd)
 # ============================================================================
@@ -286,6 +357,7 @@ install_boot_hook() {  # install_boot_hook <name> <path-to-script>
             rc-update add local default >/dev/null 2>&1 || true
             "/etc/local.d/${name}.start" || true ;;
         systemd)
+            install -d -m 0755 /usr/local/sbin
             install -m 0755 "$src" "/usr/local/sbin/${name}"
             cat > "/etc/systemd/system/${name}.service" <<UNIT
 [Unit]
@@ -316,6 +388,8 @@ install_daily_job() {  # install_daily_job <name> <script-src> [run-args...]
     local args="$*"
     # The job scripts use bash; ensure it's present (Alpine images often lack it).
     command -v bash >/dev/null 2>&1 || pkg_install bash || true
+    # Alpine's baselayout ships /usr/local/{bin,lib,share} but NOT sbin.
+    install -d -m 0755 /usr/local/sbin
     install -m 0755 "$src" "/usr/local/sbin/$name"
     # Co-install oslib.sh so a script that sources it still works standalone.
     local srcdir; srcdir="$(dirname "$src")"
@@ -587,6 +661,17 @@ CONF
     # Verify the hook actually landed and report loudly. A notifier that fails to
     # install silently is worse than none -- you'd believe logins are watched
     # when they aren't (exactly the trap that hid this on the first Alma run).
+    # "Wired into /etc/pam.d/sshd" is not the same as "will fire": on Alpine the
+    # PAM stack is only read when sshd IS the PAM build (see sshd_wanted_binary).
+    # Check that too, or we'd report ACTIVE for a hook that can never run.
+    if [[ "${OS_FAMILY:-}" == alpine ]]; then
+        if [[ ! -x /usr/sbin/sshd.pam ]]; then
+            _warn "Alpine: /usr/sbin/sshd.pam is missing, so /etc/pam.d/sshd is never read and this notifier cannot fire."
+            _warn "  Fix: apk add openssh-server-pam && rc-service $(sshd_service) restart"
+        elif ! grep -qiE '^[[:space:]]*UsePAM[[:space:]]+yes' /etc/ssh/sshd_config 2>/dev/null; then
+            _warn "Alpine: sshd_config lacks 'UsePAM yes', so the PAM build is never selected and this notifier cannot fire."
+        fi
+    fi
     if [[ -x /opt/scripts/ntfy-ssh-login.sh ]] \
         && grep -qF '/opt/scripts/ntfy-ssh-login.sh' "$pam" 2>/dev/null; then
         _log "Login notifier ACTIVE -> ${NTFY_URL:-<NTFY_URL unset!>}"