57_Wolve
7faa9098de
Restructure around a single entry point (automations.sh) with a Gum wizard and a self-extracting bundle for repo-less installs. Add scripts/oslib.sh so the provisioning scripts (setup-host, harden-ssh, harden-jumphost, sshuser) run on Alpine/Debian/Alma; seed root keys from globals/. - ntfy SSH-login alerts (user, source IP, key, region, jump target) via pam_exec - daily auto-updates with AUTO_REBOOT=idle (reboots only when no SSH active) and opt-in Alpine stable-branch upgrades - generic + per-deployment cloud-init; Gitea release workflow on tag - README/LICENSE/.gitignore/.gitattributes (force LF); repo URLs -> Gitea
311 lines
14 KiB
Bash
311 lines
14 KiB
Bash
#!/usr/bin/env bash
|
|
#
|
|
# auto-update.sh -- unattended package updates + new-OS-release check, for
|
|
# Alpine, Debian, and Alma. Designed for SSH-only bastion hosts, where the
|
|
# blast radius of a routine upgrade is tiny.
|
|
#
|
|
# Policy:
|
|
# - Apply all in-branch package upgrades automatically (apk/apt/dnf).
|
|
# - Do NOT auto-jump to a new Alpine branch (e.g. 3.21 -> 3.22): that
|
|
# rewrites the repo branch and is a human decision. Instead, NOTIFY that
|
|
# one is available.
|
|
# - Detect when a reboot is needed (kernel/libc/openssl). Reboot only if
|
|
# AUTO_REBOOT=1, otherwise just report it.
|
|
# - Send an ntfy summary, reusing /etc/ssh-notify.conf (same creds as the
|
|
# login notifier).
|
|
#
|
|
# Subcommands:
|
|
# run (default) do the update pass and notify
|
|
# install schedule this script to run daily (oslib install_daily_job)
|
|
# uninstall remove the daily schedule
|
|
#
|
|
# Env (also settable in /etc/auto-update.conf, which the daily run reads;
|
|
# environment overrides the file):
|
|
# AUTO_REBOOT=0 when a reboot is needed:
|
|
# 0 = never (just flag/notify)
|
|
# 1 = always
|
|
# idle = only when NO SSH connections are active
|
|
# (safe for a bastion -- won't cut a live
|
|
# admin session or a ProxyJump tunnel; a
|
|
# deferred reboot is retried each day)
|
|
# ALLOW_RELEASE_UPGRADE=0 (Alpine) also jump to a newer STABLE branch when
|
|
# one is posted (e.g. 3.21 -> 3.22). Off by default;
|
|
# when off, a new branch is only reported.
|
|
# NOTIFY=1 send an ntfy summary (0 to disable)
|
|
# SSH_NOTIFY_CONF=/etc/ssh-notify.conf
|
|
# DRY_RUN=0 print what would happen; touch nothing (for testing)
|
|
# LOG=/var/log/auto-update.log
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
# Locate oslib.sh. When this script is installed standalone (e.g. to
|
|
# /usr/local/sbin by install_daily_job), oslib is co-installed alongside it.
|
|
_oslib_loaded=0
|
|
for _cand in "$SCRIPT_DIR/oslib.sh" /usr/local/sbin/oslib.sh /opt/automations/scripts/oslib.sh; do
|
|
if [[ -f "$_cand" ]]; then . "$_cand"; _oslib_loaded=1; break; fi
|
|
done
|
|
[[ "$_oslib_loaded" == 1 ]] || { echo "[x] oslib.sh not found next to auto-update.sh" >&2; exit 1; }
|
|
SELF="$SCRIPT_DIR/$(basename "${BASH_SOURCE[0]}")" # resolved path to this file
|
|
|
|
# Load defaults from the conf for any var not already set in the environment
|
|
# (precedence: environment > conf > built-in). This is how the daily cron run
|
|
# picks up AUTO_REBOOT etc. without baking them into the schedule.
|
|
: "${AUTO_UPDATE_CONF:=/etc/auto-update.conf}"
|
|
if [[ -r "$AUTO_UPDATE_CONF" ]]; then
|
|
while IFS= read -r _line; do
|
|
[[ "$_line" =~ ^[[:space:]]*# || -z "${_line//[[:space:]]/}" ]] && continue
|
|
_k="${_line%%=*}"; _v="${_line#*=}"; _k="${_k//[[:space:]]/}"
|
|
[[ "$_k" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] || continue
|
|
[[ -n "${!_k:-}" ]] && continue # env already set -> wins
|
|
_v="${_v%\"}"; _v="${_v#\"}" # strip surrounding quotes
|
|
printf -v "$_k" '%s' "$_v"
|
|
done < "$AUTO_UPDATE_CONF"
|
|
fi
|
|
|
|
: "${AUTO_REBOOT:=0}"
|
|
: "${ALLOW_RELEASE_UPGRADE:=0}"
|
|
: "${NOTIFY:=1}"
|
|
: "${SSH_NOTIFY_CONF:=/etc/ssh-notify.conf}"
|
|
: "${DRY_RUN:=0}"
|
|
: "${LOG:=/var/log/auto-update.log}"
|
|
# Pending-reboot marker. In /run (tmpfs), so it is cleared by a real reboot;
|
|
# its presence on a later run means a deferred reboot is still pending.
|
|
: "${REBOOT_FLAG:=/run/automations-reboot}"
|
|
|
|
log() { _log "$@"; }
|
|
warn() { _warn "$@"; }
|
|
die() { _die "$@"; }
|
|
|
|
run() { if [[ "$DRY_RUN" == "1" ]]; then echo "DRY: $*"; else eval "$@"; fi; }
|
|
|
|
# ============================================================================
|
|
# Update pass
|
|
# ============================================================================
|
|
UPGRADED=0
|
|
REBOOT=0
|
|
NEW_RELEASE="" # a newer stable branch exists (e.g. 3.22)
|
|
UPGRADED_TO="" # we actually upgraded to this branch (ALLOW_RELEASE_UPGRADE)
|
|
CUR_BRANCH=""
|
|
|
|
apply_updates() {
|
|
case "$OS_FAMILY" in
|
|
alpine)
|
|
run "apk update"
|
|
local out
|
|
if [[ "$DRY_RUN" == "1" ]]; then out="$(apk version -l '<' 2>/dev/null || true)"; else out="$(apk upgrade --no-self-upgrade 2>&1 | tee -a "$LOG")"; fi
|
|
UPGRADED=$(printf '%s\n' "$out" | grep -cE '^\([0-9]+/[0-9]+\) (Upgrading|Installing)' || true)
|
|
# A kernel / libc / crypto / busybox change wants a reboot.
|
|
printf '%s\n' "$out" | grep -qiE 'linux-(lts|virt|edge|rpi)|(^|[^a-z])musl|openssl|libcrypto|busybox' && REBOOT=1
|
|
;;
|
|
debian)
|
|
run "apt-get update -qq"
|
|
local before after
|
|
before="$(dpkg -l 2>/dev/null | grep -c '^ii' || echo 0)"
|
|
run "DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold upgrade"
|
|
run "DEBIAN_FRONTEND=noninteractive apt-get -y autoremove"
|
|
after="$(dpkg -l 2>/dev/null | grep -c '^ii' || echo 0)"
|
|
UPGRADED=$(( before > after ? 0 : after - before )) # rough; reboot flag is the signal that matters
|
|
[[ -f /var/run/reboot-required ]] && REBOOT=1
|
|
;;
|
|
rhel)
|
|
run "dnf -y upgrade --refresh"
|
|
command -v needs-restarting >/dev/null 2>&1 || run "dnf -y install dnf-utils"
|
|
if command -v needs-restarting >/dev/null 2>&1; then
|
|
needs-restarting -r >/dev/null 2>&1 || REBOOT=1
|
|
fi
|
|
;;
|
|
esac
|
|
return 0 # never let a non-matching grep above fail the function (set -e)
|
|
}
|
|
|
|
# ============================================================================
|
|
# New-release check (Alpine: is a newer stable BRANCH posted?)
|
|
# ============================================================================
|
|
check_new_release() {
|
|
[[ "$OS_FAMILY" == alpine ]] || return 0
|
|
command -v curl >/dev/null 2>&1 || return 0
|
|
local cur cur_branch repo base latest
|
|
cur="$(cat /etc/alpine-release 2>/dev/null || echo 0.0.0)"
|
|
cur_branch="$(printf '%s' "$cur" | cut -d. -f1,2)"
|
|
CUR_BRANCH="$cur_branch"
|
|
repo="$(grep -m1 -E '^https?://' /etc/apk/repositories 2>/dev/null || true)"
|
|
[[ -n "$repo" ]] || return 0
|
|
base="$(printf '%s' "$repo" | sed -E 's#(/alpine/).*#\1#')"
|
|
# Newest vX.Y branch advertised on the mirror.
|
|
latest="$(curl -fsSL "$base" 2>/dev/null | grep -oE 'v[0-9]+\.[0-9]+' | tr -d v | sort -uV | tail -1 || true)"
|
|
[[ -n "$latest" ]] || return 0
|
|
# If the newest branch sorts strictly above the current branch, flag it.
|
|
if [[ "$latest" != "$cur_branch" ]] && \
|
|
[[ "$(printf '%s\n%s\n' "$cur_branch" "$latest" | sort -V | tail -1)" == "$latest" ]]; then
|
|
NEW_RELEASE="$latest"
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
# Jump to the newest STABLE Alpine branch (ALLOW_RELEASE_UPGRADE=1). Only the
|
|
# vX.Y stable branches are ever considered -- `edge` is never a target. Repoints
|
|
# /etc/apk/repositories from the current vX.Y to the new one (leaving any edge
|
|
# or non-versioned lines alone) and runs `apk upgrade --available`.
|
|
do_release_upgrade() {
|
|
[[ "$OS_FAMILY" == alpine ]] || { warn "Release upgrade is Alpine-only; skipping."; return 0; }
|
|
[[ -n "$NEW_RELEASE" ]] || return 0
|
|
[[ "$NEW_RELEASE" =~ ^[0-9]+\.[0-9]+$ ]] || { warn "Refusing non-stable release target '$NEW_RELEASE'."; return 0; }
|
|
|
|
log "Upgrading Alpine ${CUR_BRANCH} -> ${NEW_RELEASE} (ALLOW_RELEASE_UPGRADE=1)..."
|
|
if [[ "$DRY_RUN" == "1" ]]; then
|
|
echo "DRY: sed repositories /v${CUR_BRANCH}/ -> /v${NEW_RELEASE}/ ; apk update ; apk upgrade --available"
|
|
UPGRADED_TO="$NEW_RELEASE"; REBOOT=1; return 0
|
|
fi
|
|
cp -a /etc/apk/repositories "/etc/apk/repositories.bak.$(date -u +%Y%m%d%H%M%S)"
|
|
# Only touch versioned (vX.Y) lines; edge / non-versioned lines are left as-is.
|
|
sed -i -E "s#/v[0-9]+\.[0-9]+/#/v${NEW_RELEASE}/#g" /etc/apk/repositories
|
|
apk update
|
|
apk upgrade --available 2>&1 | tee -a "$LOG" || true
|
|
UPGRADED_TO="$NEW_RELEASE"
|
|
REBOOT=1 # a branch jump replaces kernel/musl/openssl -- always reboot
|
|
log "Now on Alpine $(cat /etc/alpine-release 2>/dev/null || echo '?')."
|
|
return 0
|
|
}
|
|
|
|
# ============================================================================
|
|
# Notify (reuse the login-notifier's ntfy config)
|
|
# ============================================================================
|
|
send_notice() {
|
|
[[ "$NOTIFY" == "1" ]] || return 0
|
|
[[ -r "$SSH_NOTIFY_CONF" ]] || return 0
|
|
# shellcheck disable=SC1090
|
|
. "$SSH_NOTIFY_CONF"
|
|
[[ -n "${NTFY_URL:-}" ]] || return 0
|
|
command -v curl >/dev/null 2>&1 || return 0
|
|
|
|
local host prio tags body
|
|
host="$(hostname -f 2>/dev/null || hostname 2>/dev/null || echo unknown)"
|
|
prio="min"
|
|
[[ "$REBOOT" == "1" || -n "$NEW_RELEASE" ]] && prio="default"
|
|
[[ -n "$UPGRADED_TO" ]] && prio="high"
|
|
|
|
body="Auto-update on ${host} (${OS_ID})
|
|
packages upgraded: ${UPGRADED}"
|
|
[[ "$REBOOT" == "1" ]] && body="${body}
|
|
reboot needed: yes$( [[ "$AUTO_REBOOT" == "1" ]] && echo ' (rebooting)' )"
|
|
if [[ -n "$UPGRADED_TO" ]]; then
|
|
body="${body}
|
|
UPGRADED Alpine release: ${CUR_BRANCH} -> ${UPGRADED_TO}"
|
|
elif [[ -n "$NEW_RELEASE" ]]; then
|
|
body="${body}
|
|
NEW Alpine release available: ${NEW_RELEASE} (current branch ${CUR_BRANCH})"
|
|
fi
|
|
|
|
set -- -fsS -m 5 -H "X-Title: Host Update" -H "X-Priority: ${prio}"
|
|
[[ -n "${NTFY_TOKEN:-}" ]] && set -- "$@" -H "Authorization: Bearer ${NTFY_TOKEN}"
|
|
[[ -n "${NTFY_EMAIL:-}" ]] && set -- "$@" -H "X-Email: ${NTFY_EMAIL}"
|
|
local t="update"; [[ -n "${NTFY_REGION:-}" ]] && t="${t},${NTFY_REGION}"
|
|
[[ -n "$NEW_RELEASE" && -z "$UPGRADED_TO" ]] && t="${t},new_release"
|
|
[[ -n "$UPGRADED_TO" ]] && t="${t},release_upgraded"
|
|
set -- "$@" -H "X-Tags: ${t}"
|
|
if [[ "$DRY_RUN" == "1" ]]; then
|
|
echo "DRY: curl $* -d <body> $NTFY_URL"
|
|
else
|
|
curl "$@" -d "$body" "$NTFY_URL" >/dev/null 2>&1 || true
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
# Are there any active inbound SSH connections? Counts established TCP
|
|
# connections on the sshd port -- this catches ProxyJump tunnels too, which
|
|
# never open a login session (so `who` would miss them).
|
|
ssh_sessions_active() {
|
|
local port n
|
|
port="$(awk '/^Port /{print $2; exit}' /etc/ssh/sshd_config 2>/dev/null || true)"
|
|
port="${port:-22}"
|
|
if command -v ss >/dev/null 2>&1; then
|
|
n="$(ss -Htn state established "sport = :$port" 2>/dev/null | grep -c . || true)"
|
|
elif command -v netstat >/dev/null 2>&1; then
|
|
n="$(netstat -tn 2>/dev/null | awk -v p=":$port" '$NF=="ESTABLISHED" && $4 ~ p"$"' | grep -c . || true)"
|
|
else
|
|
# last resort: sshd per-connection worker processes
|
|
n="$(ps ax 2>/dev/null | grep -E '[s]shd:.*@|[s]shd: ' | grep -c . || true)"
|
|
fi
|
|
[ "${n:-0}" -gt 0 ]
|
|
}
|
|
|
|
# ============================================================================
|
|
# Subcommands
|
|
# ============================================================================
|
|
do_run() {
|
|
[[ $EUID -eq 0 ]] || die "Run as root."
|
|
os_detect
|
|
# A reboot pending from a previous (deferred) run -- the flag is in /run,
|
|
# wiped on a real reboot, so its presence means "still pending".
|
|
[[ -f "$REBOOT_FLAG" ]] && REBOOT=1
|
|
[[ "$DRY_RUN" == "1" ]] || { install -d -m 0755 "$(dirname "$LOG")" 2>/dev/null || true; echo "=== auto-update $(date -u +%FT%TZ) ===" >> "$LOG"; }
|
|
log "Applying updates (${OS_ID})..."
|
|
apply_updates
|
|
check_new_release
|
|
if [[ -n "$NEW_RELEASE" && "$ALLOW_RELEASE_UPGRADE" == "1" ]]; then
|
|
do_release_upgrade
|
|
fi
|
|
|
|
# Record the pending reboot so a deferral survives to the next daily run.
|
|
if [[ "$REBOOT" == "1" && "$DRY_RUN" != "1" ]]; then : > "$REBOOT_FLAG" 2>/dev/null || true; fi
|
|
|
|
log "Upgraded: ${UPGRADED} | reboot: ${REBOOT} | new release: ${NEW_RELEASE:-none}${UPGRADED_TO:+ | upgraded to: $UPGRADED_TO}"
|
|
send_notice
|
|
|
|
if [[ "$REBOOT" == "1" ]]; then
|
|
case "$AUTO_REBOOT" in
|
|
1)
|
|
log "Rebooting (AUTO_REBOOT=1)..."; run "reboot" ;;
|
|
idle)
|
|
if ssh_sessions_active; then
|
|
warn "Reboot needed, but SSH connections are active; deferring until idle (retried daily)."
|
|
else
|
|
log "No active SSH connections; rebooting (AUTO_REBOOT=idle)..."; run "reboot"
|
|
fi ;;
|
|
*)
|
|
warn "A reboot is recommended (kernel/libc/crypto updated). Set AUTO_REBOOT=1 or idle to automate." ;;
|
|
esac
|
|
fi
|
|
}
|
|
|
|
# Write /etc/auto-update.conf so the scheduled run inherits these defaults.
|
|
write_autoupdate_conf() {
|
|
local f="$AUTO_UPDATE_CONF"
|
|
if [[ -f "$f" && "${AU_FORCE_CONF:-0}" != "1" ]]; then
|
|
log "$f exists; leaving it (set AU_FORCE_CONF=1 to overwrite)."
|
|
return 0
|
|
fi
|
|
cat > "$f" <<CONF
|
|
# Defaults for the daily auto-update job (scripts/auto-update.sh).
|
|
# Environment variables still override these at runtime.
|
|
AUTO_REBOOT="${AUTO_REBOOT}" # 0 | 1 | idle
|
|
ALLOW_RELEASE_UPGRADE="${ALLOW_RELEASE_UPGRADE}" # Alpine stable-branch jump
|
|
NOTIFY="${NOTIFY}"
|
|
CONF
|
|
chmod 644 "$f"
|
|
log "Wrote $f"
|
|
}
|
|
|
|
do_install() {
|
|
[[ $EUID -eq 0 ]] || die "Run as root."
|
|
os_detect
|
|
write_autoupdate_conf
|
|
install_daily_job auto-update "$SELF" run
|
|
log "Scheduled daily auto-update (AUTO_REBOOT=${AUTO_REBOOT})."
|
|
}
|
|
|
|
do_uninstall() {
|
|
[[ $EUID -eq 0 ]] || die "Run as root."
|
|
os_detect
|
|
remove_daily_job auto-update
|
|
log "Removed daily auto-update."
|
|
}
|
|
|
|
case "${1:-run}" in
|
|
run) do_run ;;
|
|
install) do_install ;;
|
|
uninstall) do_uninstall ;;
|
|
*) die "Usage: auto-update.sh [run|install|uninstall]" ;;
|
|
esac
|