feat: unified launcher, multi-OS hardening, login alerts & auto-updates

Restructure around a single entry point (automations.sh) with a Gum wizard and a self-extracting bundle for repo-less installs. Add scripts/oslib.sh so the provisioning scripts (setup-host, harden-ssh, harden-jumphost, sshuser) run on Alpine/Debian/Alma; seed root keys from globals/. - ntfy SSH-login alerts (user, source IP, key, region, jump target) via pam_exec - daily auto-updates with AUTO_REBOOT=idle (reboots only when no SSH active) and opt-in Alpine stable-branch upgrades - generic + per-deployment cloud-init; Gitea release workflow on tag - README/LICENSE/.gitignore/.gitattributes (force LF); repo URLs -> Gitea
2026-06-12 14:56:02 -05:00
parent 85eeb79971
commit 7faa9098de
58 changed files with 6225 additions and 284 deletions
@@ -0,0 +1,520 @@
+#!/usr/bin/env bash
+#
+# oslib.sh -- OS abstraction layer for Alpine, Debian, and Alma Linux.
+#
+# Source it; it has no side effects beyond defining functions and, after you
+# call os_detect, exporting OS_ID / OS_FAMILY / INIT_SYSTEM.
+#
+#   . "$(dirname "$0")/oslib.sh"
+#   os_detect
+#   pkg_install curl jq
+#
+# Every distro-specific decision lives here, behind a function or a per-OS
+# `case "$OS_FAMILY"`. Consumers (harden-ssh.sh, harden-jumphost.sh,
+# sshuser.sh, setup-host.sh) should never call apk/apt/dnf, rc-service, or
+# systemctl directly -- they call these helpers instead. That keeps the
+# OS-specific surface in ONE auditable file.
+#
+# Supported targets:
+#   OS_ID      OS_FAMILY   INIT_SYSTEM   package mgr
+#   ------     ---------   -----------   -----------
+#   alpine     alpine      openrc        apk
+#   debian     debian      systemd       apt-get   (also ubuntu)
+#   alma       rhel        systemd       dnf       (also rocky/rhel/centos)
+
+# Reuse the log helpers if a consumer already defined them; otherwise define.
+if ! declare -f _log >/dev/null 2>&1; then
+    _log()  { printf '\033[1;32m[+]\033[0m %s\n' "$*"; }
+    _warn() { printf '\033[1;33m[!]\033[0m %s\n' "$*" >&2; }
+    _die()  { printf '\033[1;31m[x]\033[0m %s\n' "$*" >&2; exit 1; }
+fi
+
+# ============================================================================
+# Detection
+# ============================================================================
+os_detect() {
+    local id="" like="" osr="${OS_RELEASE_FILE:-/etc/os-release}"
+    if [[ -r "$osr" ]]; then
+        # shellcheck disable=SC1091
+        id="$(. "$osr" 2>/dev/null && echo "${ID:-}")"
+        like="$(. "$osr" 2>/dev/null && echo "${ID_LIKE:-}")"
+    elif [[ -f /etc/alpine-release ]]; then
+        id=alpine
+    fi
+
+    case "$id" in
+        alpine)                         OS_ID=alpine; OS_FAMILY=alpine ;;
+        debian|ubuntu|raspbian)         OS_ID=debian; OS_FAMILY=debian ;;
+        almalinux|alma|rocky|rhel|centos|fedora) OS_ID=alma; OS_FAMILY=rhel ;;
+        *)
+            # Fall back to ID_LIKE for derivatives we didn't name explicitly.
+            case " $like " in
+                *" alpine "*)            OS_ID=alpine; OS_FAMILY=alpine ;;
+                *" debian "*|*" ubuntu "*) OS_ID=debian; OS_FAMILY=debian ;;
+                *" rhel "*|*" fedora "*|*" centos "*) OS_ID=alma; OS_FAMILY=rhel ;;
+                *) _die "Unsupported OS (ID='$id', ID_LIKE='$like'). Supported: Alpine, Debian, Alma." ;;
+            esac ;;
+    esac
+
+    case "$OS_FAMILY" in
+        alpine) INIT_SYSTEM=openrc ;;
+        *)      INIT_SYSTEM=systemd ;;
+    esac
+    export OS_ID OS_FAMILY INIT_SYSTEM
+}
+
+_require_detected() { [[ -n "${OS_FAMILY:-}" ]] || os_detect; }
+
+# ============================================================================
+# Packages
+# ============================================================================
+pkg_update() {
+    _require_detected
+    case "$OS_FAMILY" in
+        alpine) apk update -q ;;
+        debian) apt-get update -qq ;;
+        rhel)   dnf -q makecache || true ;;
+    esac
+}
+
+pkg_install() {  # pkg_install <name>...
+    _require_detected
+    case "$OS_FAMILY" in
+        alpine) apk add -q "$@" ;;
+        debian) DEBIAN_FRONTEND=noninteractive apt-get install -y -qq "$@" ;;
+        rhel)   dnf install -y -q "$@" ;;
+    esac
+}
+
+pkg_remove() {  # pkg_remove <name>...
+    _require_detected
+    case "$OS_FAMILY" in
+        alpine) apk del -q "$@" 2>/dev/null || true ;;
+        debian) DEBIAN_FRONTEND=noninteractive apt-get remove -y -qq "$@" 2>/dev/null || true ;;
+        rhel)   dnf remove -y -q "$@" 2>/dev/null || true ;;
+    esac
+}
+
+pkg_installed() {  # pkg_installed <name> -> 0 if installed
+    _require_detected
+    case "$OS_FAMILY" in
+        alpine) apk info -e "$1" >/dev/null 2>&1 ;;
+        debian) dpkg -s "$1" >/dev/null 2>&1 ;;
+        rhel)   rpm -q "$1" >/dev/null 2>&1 ;;
+    esac
+}
+
+# Logical package-name differences across families. Echo the right name(s).
+pkg_name() {  # pkg_name <logical>
+    _require_detected
+    case "$1" in
+        openssh-server)
+            # Alpine: the PAM-enabled variant so the session stack runs.
+            case "$OS_FAMILY" in alpine) echo openssh-server-pam ;; *) echo openssh-server ;; esac ;;
+        openssh-client)
+            case "$OS_FAMILY" in
+                alpine) echo openssh-client ;;
+                debian) echo openssh-client ;;   # Debian/Ubuntu: singular
+                rhel)   echo openssh-clients ;;  # RHEL/Alma: plural
+            esac ;;
+        sftp-server)
+            # The external SFTP subsystem binary's package. Alpine ships it
+            # separately; Debian/Alma bundle it inside openssh-server.
+            case "$OS_FAMILY" in alpine) echo openssh-sftp-server ;; *) echo "" ;; esac ;;
+        *) echo "$1" ;;
+    esac
+}
+
+# ============================================================================
+# Services (OpenRC vs systemd)
+# ============================================================================
+svc_enable() {  # enable at boot
+    _require_detected
+    case "$INIT_SYSTEM" in
+        openrc)  rc-update add "$1" default >/dev/null 2>&1 || true ;;
+        systemd) systemctl enable "$1" >/dev/null 2>&1 || true ;;
+    esac
+}
+
+svc_start()   { _require_detected; [[ "$INIT_SYSTEM" == openrc ]] && rc-service "$1" start   || systemctl start   "$1"; }
+svc_restart() { _require_detected; [[ "$INIT_SYSTEM" == openrc ]] && { rc-service "$1" restart || rc-service "$1" start; } || systemctl restart "$1"; }
+svc_reload()  { _require_detected; [[ "$INIT_SYSTEM" == openrc ]] && { rc-service "$1" reload  || rc-service "$1" restart; } || { systemctl reload "$1" || systemctl restart "$1"; }; }
+
+svc_enable_start() { svc_enable "$1"; svc_start "$1"; }
+
+# The sshd service is named differently per distro.
+sshd_service() {
+    _require_detected
+    [[ "$OS_FAMILY" == debian ]] && echo ssh || echo sshd
+}
+
+# ============================================================================
+# SSH-specific paths
+# ============================================================================
+# External sftp-server binary path. The user asked for the external subsystem
+# (not internal-sftp), and the path differs by distro.
+sftp_server_path() {
+    _require_detected
+    local p
+    case "$OS_FAMILY" in
+        alpine) p=/usr/lib/ssh/sftp-server ;;
+        debian) p=/usr/lib/openssh/sftp-server ;;
+        rhel)   p=/usr/libexec/openssh/sftp-server ;;
+    esac
+    # Verify; fall back to a search so an unusual layout still works.
+    if [[ ! -x "$p" ]]; then
+        local found
+        found="$(command -v sftp-server 2>/dev/null)" || true
+        [[ -z "$found" ]] && for c in /usr/lib/ssh/sftp-server /usr/lib/openssh/sftp-server \
+                                       /usr/libexec/openssh/sftp-server /usr/libexec/sftp-server; do
+            [[ -x "$c" ]] && { found="$c"; break; }
+        done
+        [[ -n "$found" ]] && p="$found"
+    fi
+    echo "$p"
+}
+
+# After writing sshd_config, Alpine's OpenRC init can regenerate RSA/ECDSA
+# host keys on every start. Pin it off. No-op on systemd distros (they only
+# generate host keys at install time, via ssh-keygen -A).
+sshd_disable_keygen() {
+    _require_detected
+    [[ "$OS_FAMILY" == alpine ]] || return 0
+    [[ -f /etc/conf.d/sshd ]] || return 0
+    if grep -q '^sshd_disable_keygen=' /etc/conf.d/sshd; then
+        sed -i 's/^sshd_disable_keygen=.*/sshd_disable_keygen="yes"/' /etc/conf.d/sshd
+    else
+        echo 'sshd_disable_keygen="yes"' >> /etc/conf.d/sshd
+    fi
+}
+
+# ============================================================================
+# Users & groups (busybox adduser/addgroup vs shadow useradd/groupadd)
+# ============================================================================
+group_add_system() {  # group_add_system <group>
+    _require_detected
+    getent group "$1" >/dev/null && return 0
+    case "$OS_FAMILY" in
+        alpine) addgroup -S "$1" ;;
+        *)      groupadd -r "$1" ;;
+    esac
+}
+
+user_add_to_group() {  # user_add_to_group <user> <group>
+    _require_detected
+    case "$OS_FAMILY" in
+        alpine) addgroup "$1" "$2" 2>/dev/null || adduser "$1" "$2" 2>/dev/null || true ;;
+        *)      usermod -aG "$2" "$1" ;;
+    esac
+}
+
+# Create a no-login user (for ssh jumpers) with the right nologin shell.
+user_add_nologin() {  # user_add_nologin <name>
+    _require_detected
+    getent passwd "$1" >/dev/null && return 0
+    case "$OS_FAMILY" in
+        alpine) adduser -D -s /sbin/nologin "$1" ;;
+        *)      useradd -m -s /usr/sbin/nologin "$1" 2>/dev/null \
+                    || useradd -m -s /sbin/nologin "$1" ;;
+    esac
+}
+
+# Default interactive shell present on the distro (for admin users).
+default_shell() {
+    _require_detected
+    if [[ -x /bin/bash ]]; then echo /bin/bash
+    elif [[ "$OS_FAMILY" == alpine ]]; then echo /bin/ash
+    else echo /bin/sh; fi
+}
+
+# Path to nologin (consistent across distros, but verify).
+nologin_path() {
+    for p in /sbin/nologin /usr/sbin/nologin; do [[ -x "$p" ]] && { echo "$p"; return; }; done
+    echo /sbin/nologin
+}
+
+# ============================================================================
+# Hostname
+# ============================================================================
+set_hostname() {  # set_hostname <fqdn>
+    _require_detected
+    local fqdn="$1" short="${1%%.*}"
+    if command -v hostnamectl >/dev/null 2>&1; then
+        hostnamectl set-hostname "$fqdn"
+    else
+        # Alpine / no-systemd path.
+        echo "$short" > /etc/hostname     # Alpine stores the short name
+        hostname "$short" 2>/dev/null || true
+        command -v rc-service >/dev/null 2>&1 && rc-service hostname restart >/dev/null 2>&1 || true
+    fi
+    # Maintain /etc/hosts so `hostname -f` resolves.
+    _update_etc_hosts "$fqdn" "$short"
+}
+
+# Echo the region segment of this host's FQDN (e.g. "us-evi-1" from
+# ssh-1.us-evi-1.srvno.de), or nothing when the name carries no region
+# (e.g. sto-1.srvno.de). Used to tag login notifications by location.
+host_region() {
+    local fqdn seg
+    fqdn="$(hostname -f 2>/dev/null || hostname 2>/dev/null || echo)"
+    seg="$(printf '%s' "$fqdn" | cut -d. -f2)"
+    printf '%s' "$seg" | grep -qE '^[a-z]{2}-[a-z]{2,4}-[0-9]+$' && printf '%s' "$seg" || printf ''
+}
+
+_update_etc_hosts() {  # <fqdn> <short>
+    local fqdn="$1" short="$2"
+    touch /etc/hosts
+    # Debian convention: 127.0.1.1 for the box's own FQDN.
+    local line="127.0.1.1 ${fqdn} ${short}"
+    if grep -qE '^127\.0\.1\.1[[:space:]]' /etc/hosts; then
+        sed -i "s|^127\.0\.1\.1[[:space:]].*|${line}|" /etc/hosts
+    else
+        printf '%s\n' "$line" >> /etc/hosts
+    fi
+}
+
+# ============================================================================
+# Boot hooks -- run a script once at every boot, init-system agnostic.
+# Used to (re)install the iptables INPUT->sshguard jump.
+# ============================================================================
+install_boot_hook() {  # install_boot_hook <name> <path-to-script>
+    _require_detected
+    local name="$1" src="$2"
+    case "$INIT_SYSTEM" in
+        openrc)
+            install -m 0755 "$src" "/etc/local.d/${name}.start"
+            rc-update add local default >/dev/null 2>&1 || true
+            "/etc/local.d/${name}.start" || true ;;
+        systemd)
+            install -m 0755 "$src" "/usr/local/sbin/${name}"
+            cat > "/etc/systemd/system/${name}.service" <<UNIT
+[Unit]
+Description=${name} (oslib boot hook)
+After=network.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/sbin/${name}
+RemainAfterExit=yes
+
+[Install]
+WantedBy=multi-user.target
+UNIT
+            systemctl daemon-reload
+            systemctl enable "${name}.service" >/dev/null 2>&1 || true
+            systemctl start  "${name}.service" || true ;;
+    esac
+}
+
+# ============================================================================
+# Daily scheduled jobs -- busybox crond (/etc/periodic/daily) on Alpine, a
+# systemd timer elsewhere.
+# ============================================================================
+install_daily_job() {  # install_daily_job <name> <script-src> [run-args...]
+    _require_detected
+    local name="$1" src="$2"; shift 2
+    local args="$*"
+    # The job scripts use bash; ensure it's present (Alpine images often lack it).
+    command -v bash >/dev/null 2>&1 || pkg_install bash || true
+    install -m 0755 "$src" "/usr/local/sbin/$name"
+    # Co-install oslib.sh so a script that sources it still works standalone.
+    local srcdir; srcdir="$(dirname "$src")"
+    [[ -f "$srcdir/oslib.sh" ]] && install -m 0644 "$srcdir/oslib.sh" /usr/local/sbin/oslib.sh
+    case "$INIT_SYSTEM" in
+        openrc)
+            command -v crond >/dev/null 2>&1 || pkg_install busybox-suid 2>/dev/null || true
+            cat > "/etc/periodic/daily/$name" <<HOOK
+#!/bin/sh
+exec /usr/local/sbin/$name $args
+HOOK
+            chmod +x "/etc/periodic/daily/$name"
+            rc-update add crond default >/dev/null 2>&1 || true
+            rc-service crond status >/dev/null 2>&1 || rc-service crond start >/dev/null 2>&1 || true ;;
+        systemd)
+            cat > "/etc/systemd/system/$name.service" <<UNIT
+[Unit]
+Description=$name (daily job)
+After=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/sbin/$name $args
+UNIT
+            cat > "/etc/systemd/system/$name.timer" <<UNIT
+[Unit]
+Description=Run $name daily
+
+[Timer]
+OnCalendar=daily
+Persistent=true
+RandomizedDelaySec=1h
+
+[Install]
+WantedBy=timers.target
+UNIT
+            systemctl daemon-reload
+            systemctl enable --now "$name.timer" >/dev/null 2>&1 || true ;;
+    esac
+}
+
+remove_daily_job() {  # remove_daily_job <name>
+    _require_detected
+    case "$INIT_SYSTEM" in
+        openrc)  rm -f "/etc/periodic/daily/$1" ;;
+        systemd) systemctl disable --now "$1.timer" >/dev/null 2>&1 || true
+                 rm -f "/etc/systemd/system/$1.timer" "/etc/systemd/system/$1.service"
+                 systemctl daemon-reload ;;
+    esac
+    rm -f "/usr/local/sbin/$1"
+}
+
+# ============================================================================
+# Brute-force protection (sshguard) -- log source & firewall backend differ.
+# ============================================================================
+# A sshguard LOGREADER line appropriate for the distro's logging.
+sshguard_logreader() {
+    _require_detected
+    local svc; svc="$(sshd_service)"
+    case "$OS_FAMILY" in
+        # Alpine uses busybox syslogd -> /var/log/messages (no journald).
+        alpine) echo "LANG=C tail -F -n0 /var/log/messages" ;;
+        # Debian & Alma run systemd-journald.
+        *)      echo "LANG=C journalctl -afb -p info -n1 -u ${svc} -o cat" ;;
+    esac
+}
+
+# ============================================================================
+# High-level installers (compose the primitives above; OS knowledge lives here)
+# ============================================================================
+# Install OpenSSH server + client + the external SFTP subsystem. On Alpine we
+# swap the non-PAM server for the PAM-enabled one so the session stack runs.
+install_openssh() {
+    _require_detected
+    if [[ "$OS_FAMILY" == alpine ]]; then
+        if pkg_installed openssh-server && ! pkg_installed openssh-server-pam; then
+            pkg_remove openssh-server
+        fi
+    fi
+    local sftp_pkg; sftp_pkg="$(pkg_name sftp-server)"
+    # shellcheck disable=SC2046
+    pkg_install $(pkg_name openssh-server) $(pkg_name openssh-client) ${sftp_pkg:+$sftp_pkg}
+    # Alpine needs linux-pam present for the PAM server build.
+    [[ "$OS_FAMILY" == alpine ]] && pkg_install linux-pam openrc
+}
+
+# Install sshguard + an iptables firewall backend. On RHEL/Alma sshguard lives
+# in EPEL, so enable that first.
+install_bruteforce_protection() {
+    _require_detected
+    case "$OS_FAMILY" in
+        alpine) pkg_install sshguard iptables ip6tables ;;
+        debian) pkg_install sshguard iptables ;;
+        rhel)   pkg_install epel-release || true
+                pkg_install sshguard iptables ;;
+    esac
+}
+
+# ============================================================================
+# gum (Charm TUI) -- multi-OS installer. Best-effort; callers that can fall
+# back to CLI prompts should not treat failure as fatal.
+# ============================================================================
+ensure_gum() {
+    command -v gum >/dev/null 2>&1 && return 0
+    _require_detected
+    case "$OS_FAMILY" in
+        alpine)
+            apk add -q gum 2>/dev/null && return 0
+            # community repo may be disabled; enable it for this release, retry.
+            local rel mirror
+            rel="$(cut -d. -f1,2 < /etc/alpine-release 2>/dev/null || echo edge)"
+            mirror="https://dl-cdn.alpinelinux.org/alpine/v${rel}/community"
+            grep -qF "$mirror" /etc/apk/repositories 2>/dev/null || echo "$mirror" >> /etc/apk/repositories
+            apk update -q && apk add -q gum ;;
+        debian)
+            # Charm's apt repo.
+            pkg_install curl gnupg ca-certificates || true
+            mkdir -p /etc/apt/keyrings
+            curl -fsSL https://repo.charm.sh/apt/gpg.key | gpg --dearmor -o /etc/apt/keyrings/charm.gpg
+            echo "deb [signed-by=/etc/apt/keyrings/charm.gpg] https://repo.charm.sh/apt/ * *" \
+                > /etc/apt/sources.list.d/charm.list
+            apt-get update -qq && pkg_install gum ;;
+        rhel)
+            cat > /etc/yum.repos.d/charm.repo <<'REPO'
+[charm]
+name=Charm
+baseurl=https://repo.charm.sh/yum/
+enabled=1
+gpgcheck=1
+gpgkey=https://repo.charm.sh/yum/gpg.key
+REPO
+            pkg_install gum ;;
+    esac
+    command -v gum >/dev/null 2>&1
+}
+
+# ============================================================================
+# SSH login notifier (pam_exec -> ntfy)
+# ============================================================================
+# Install the ntfy login hook: drop the script in /opt/scripts, write
+# /etc/ssh-notify.conf from the NTFY_* / NOTIFY_* environment, and add the
+# pam_exec line to /etc/pam.d/sshd (idempotent). Requires curl.
+#
+#   install_login_notifier <path-to-ntfy-ssh-login.sh>
+#
+# Honored env: NTFY_URL (required to be useful), NTFY_TOKEN, NTFY_EMAIL,
+# NTFY_TITLE, NTFY_PRIORITY, NTFY_REGION, NOTIFY_GROUPS, NOTIFY_PRIORITY_MAP.
+# Set NTFY_FORCE_CONF=1 to overwrite an existing /etc/ssh-notify.conf.
+install_login_notifier() {
+    _require_detected
+    local src="$1"
+    [[ -f "$src" ]] || { _warn "notifier script not found: $src"; return 1; }
+    command -v curl >/dev/null 2>&1 || pkg_install curl || _warn "curl not installed; notifier needs it."
+
+    install -d -m 0755 /opt/scripts
+    install -m 0755 "$src" /opt/scripts/ntfy-ssh-login.sh
+
+    if [[ ! -f /etc/ssh-notify.conf || "${NTFY_FORCE_CONF:-0}" == "1" ]]; then
+        ( umask 077
+          cat > /etc/ssh-notify.conf <<CONF
+# Generated by oslib install_login_notifier -- $(date -u +%FT%TZ)
+NTFY_URL="${NTFY_URL:-}"
+NTFY_TOKEN="${NTFY_TOKEN:-}"
+NTFY_EMAIL="${NTFY_EMAIL:-}"
+NTFY_TITLE="${NTFY_TITLE:-Bastion Notification}"
+NTFY_PRIORITY="${NTFY_PRIORITY:-min}"
+NTFY_REGION="${NTFY_REGION:-}"
+NOTIFY_GROUPS="${NOTIFY_GROUPS:-}"
+NOTIFY_PRIORITY_MAP="${NOTIFY_PRIORITY_MAP:-}"
+CONF
+        )
+        chmod 600 /etc/ssh-notify.conf
+        _log "Wrote /etc/ssh-notify.conf (mode 0600)."
+    else
+        _log "/etc/ssh-notify.conf exists; left untouched (set NTFY_FORCE_CONF=1 to replace)."
+    fi
+
+    # Wire pam_exec into the sshd PAM stack (idempotent).
+    local pam=/etc/pam.d/sshd
+    local line='session    optional    pam_exec.so /opt/scripts/ntfy-ssh-login.sh'
+    if [[ -f "$pam" ]]; then
+        grep -qF '/opt/scripts/ntfy-ssh-login.sh' "$pam" || echo "$line" >> "$pam"
+        _log "Enabled pam_exec login notifier in $pam."
+    else
+        _warn "$pam not found; add this line to your sshd PAM stack manually:"
+        _warn "  $line"
+    fi
+}
+
+# Locate the sshguard iptables backend binary (path varies by packaging).
+sshguard_backend() {
+    local c
+    for c in /usr/libexec/sshguard/sshg-fw-iptables \
+             /usr/libexec/sshg-fw-iptables \
+             /usr/lib/sshguard/sshg-fw-iptables \
+             /usr/libexec/sshguard/sshg-fw-nft \
+             /usr/sbin/sshg-fw-iptables; do
+        [[ -x "$c" ]] && { echo "$c"; return; }
+    done
+    # Sensible default if nothing matched; caller may warn.
+    echo /usr/libexec/sshg-fw-iptables
+}