#!/usr/bin/env bash # # auto-update.sh -- unattended package updates + new-OS-release check, for # Alpine, Debian, and Alma. Designed for SSH-only bastion hosts, where the # blast radius of a routine upgrade is tiny. # # Policy: # - Apply all in-branch package upgrades automatically (apk/apt/dnf). # - Do NOT auto-jump to a new Alpine branch (e.g. 3.21 -> 3.22): that # rewrites the repo branch and is a human decision. Instead, NOTIFY that # one is available. # - Detect when a reboot is needed (kernel/libc/openssl). Reboot only if # AUTO_REBOOT=1, otherwise just report it. # - Send an ntfy summary, reusing /etc/ssh-notify.conf (same creds as the # login notifier). # # Subcommands: # run (default) do the update pass and notify # install schedule this script to run daily (oslib install_daily_job) # uninstall remove the daily schedule # # Env (also settable in /etc/auto-update.conf, which the daily run reads; # environment overrides the file): # AUTO_REBOOT=0 when a reboot is needed: # 0 = never (just flag/notify) # 1 = always # idle = only when NO SSH connections are active # (safe for a bastion -- won't cut a live # admin session or a ProxyJump tunnel; a # deferred reboot is retried each day) # ALLOW_RELEASE_UPGRADE=0 (Alpine) also jump to a newer STABLE branch when # one is posted (e.g. 3.21 -> 3.22). Off by default; # when off, a new branch is only reported. # NOTIFY=1 send an ntfy summary (0 to disable) # SSH_NOTIFY_CONF=/etc/ssh-notify.conf # DRY_RUN=0 print what would happen; touch nothing (for testing) # LOG=/var/log/auto-update.log set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # Locate oslib.sh. When this script is installed standalone (e.g. to # /usr/local/sbin by install_daily_job), oslib is co-installed alongside it. _oslib_loaded=0 for _cand in "$SCRIPT_DIR/oslib.sh" /usr/local/sbin/oslib.sh /opt/automations/scripts/oslib.sh; do if [[ -f "$_cand" ]]; then . "$_cand"; _oslib_loaded=1; break; fi done [[ "$_oslib_loaded" == 1 ]] || { echo "[x] oslib.sh not found next to auto-update.sh" >&2; exit 1; } SELF="$SCRIPT_DIR/$(basename "${BASH_SOURCE[0]}")" # resolved path to this file # Load defaults from the conf for any var not already set in the environment # (precedence: environment > conf > built-in). This is how the daily cron run # picks up AUTO_REBOOT etc. without baking them into the schedule. : "${AUTO_UPDATE_CONF:=/etc/auto-update.conf}" if [[ -r "$AUTO_UPDATE_CONF" ]]; then while IFS= read -r _line; do [[ "$_line" =~ ^[[:space:]]*# || -z "${_line//[[:space:]]/}" ]] && continue _k="${_line%%=*}"; _v="${_line#*=}"; _k="${_k//[[:space:]]/}" [[ "$_k" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] || continue [[ -n "${!_k:-}" ]] && continue # env already set -> wins _v="${_v%\"}"; _v="${_v#\"}" # strip surrounding quotes printf -v "$_k" '%s' "$_v" done < "$AUTO_UPDATE_CONF" fi : "${AUTO_REBOOT:=0}" : "${ALLOW_RELEASE_UPGRADE:=0}" : "${NOTIFY:=1}" : "${SSH_NOTIFY_CONF:=/etc/ssh-notify.conf}" : "${DRY_RUN:=0}" : "${LOG:=/var/log/auto-update.log}" # Pending-reboot marker. In /run (tmpfs), so it is cleared by a real reboot; # its presence on a later run means a deferred reboot is still pending. : "${REBOOT_FLAG:=/run/automations-reboot}" log() { _log "$@"; } warn() { _warn "$@"; } die() { _die "$@"; } run() { if [[ "$DRY_RUN" == "1" ]]; then echo "DRY: $*"; else eval "$@"; fi; } # ============================================================================ # Update pass # ============================================================================ UPGRADED=0 REBOOT=0 NEW_RELEASE="" # a newer stable branch exists (e.g. 3.22) UPGRADED_TO="" # we actually upgraded to this branch (ALLOW_RELEASE_UPGRADE) CUR_BRANCH="" apply_updates() { case "$OS_FAMILY" in alpine) run "apk update" local out if [[ "$DRY_RUN" == "1" ]]; then out="$(apk version -l '<' 2>/dev/null || true)"; else out="$(apk upgrade --no-self-upgrade 2>&1 | tee -a "$LOG")"; fi UPGRADED=$(printf '%s\n' "$out" | grep -cE '^\([0-9]+/[0-9]+\) (Upgrading|Installing)' || true) # A kernel / libc / crypto / busybox change wants a reboot. printf '%s\n' "$out" | grep -qiE 'linux-(lts|virt|edge|rpi)|(^|[^a-z])musl|openssl|libcrypto|busybox' && REBOOT=1 ;; debian) run "apt-get update -qq" local before after before="$(dpkg -l 2>/dev/null | grep -c '^ii' || echo 0)" run "DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold upgrade" run "DEBIAN_FRONTEND=noninteractive apt-get -y autoremove" after="$(dpkg -l 2>/dev/null | grep -c '^ii' || echo 0)" UPGRADED=$(( before > after ? 0 : after - before )) # rough; reboot flag is the signal that matters [[ -f /var/run/reboot-required ]] && REBOOT=1 ;; rhel) run "dnf -y upgrade --refresh" command -v needs-restarting >/dev/null 2>&1 || run "dnf -y install dnf-utils" if command -v needs-restarting >/dev/null 2>&1; then needs-restarting -r >/dev/null 2>&1 || REBOOT=1 fi ;; esac return 0 # never let a non-matching grep above fail the function (set -e) } # ============================================================================ # New-release check (Alpine: is a newer stable BRANCH posted?) # ============================================================================ check_new_release() { [[ "$OS_FAMILY" == alpine ]] || return 0 command -v curl >/dev/null 2>&1 || return 0 local cur cur_branch repo base latest cur="$(cat /etc/alpine-release 2>/dev/null || echo 0.0.0)" cur_branch="$(printf '%s' "$cur" | cut -d. -f1,2)" CUR_BRANCH="$cur_branch" repo="$(grep -m1 -E '^https?://' /etc/apk/repositories 2>/dev/null || true)" [[ -n "$repo" ]] || return 0 base="$(printf '%s' "$repo" | sed -E 's#(/alpine/).*#\1#')" # Newest vX.Y branch advertised on the mirror. latest="$(curl -fsSL "$base" 2>/dev/null | grep -oE 'v[0-9]+\.[0-9]+' | tr -d v | sort -uV | tail -1 || true)" [[ -n "$latest" ]] || return 0 # If the newest branch sorts strictly above the current branch, flag it. if [[ "$latest" != "$cur_branch" ]] && \ [[ "$(printf '%s\n%s\n' "$cur_branch" "$latest" | sort -V | tail -1)" == "$latest" ]]; then NEW_RELEASE="$latest" fi return 0 } # Jump to the newest STABLE Alpine branch (ALLOW_RELEASE_UPGRADE=1). Only the # vX.Y stable branches are ever considered -- `edge` is never a target. Repoints # /etc/apk/repositories from the current vX.Y to the new one (leaving any edge # or non-versioned lines alone) and runs `apk upgrade --available`. do_release_upgrade() { [[ "$OS_FAMILY" == alpine ]] || { warn "Release upgrade is Alpine-only; skipping."; return 0; } [[ -n "$NEW_RELEASE" ]] || return 0 [[ "$NEW_RELEASE" =~ ^[0-9]+\.[0-9]+$ ]] || { warn "Refusing non-stable release target '$NEW_RELEASE'."; return 0; } log "Upgrading Alpine ${CUR_BRANCH} -> ${NEW_RELEASE} (ALLOW_RELEASE_UPGRADE=1)..." if [[ "$DRY_RUN" == "1" ]]; then echo "DRY: sed repositories /v${CUR_BRANCH}/ -> /v${NEW_RELEASE}/ ; apk update ; apk upgrade --available" UPGRADED_TO="$NEW_RELEASE"; REBOOT=1; return 0 fi cp -a /etc/apk/repositories "/etc/apk/repositories.bak.$(date -u +%Y%m%d%H%M%S)" # Only touch versioned (vX.Y) lines; edge / non-versioned lines are left as-is. sed -i -E "s#/v[0-9]+\.[0-9]+/#/v${NEW_RELEASE}/#g" /etc/apk/repositories apk update apk upgrade --available 2>&1 | tee -a "$LOG" || true UPGRADED_TO="$NEW_RELEASE" REBOOT=1 # a branch jump replaces kernel/musl/openssl -- always reboot log "Now on Alpine $(cat /etc/alpine-release 2>/dev/null || echo '?')." return 0 } # ============================================================================ # Notify (reuse the login-notifier's ntfy config) # ============================================================================ send_notice() { [[ "$NOTIFY" == "1" ]] || return 0 [[ -r "$SSH_NOTIFY_CONF" ]] || return 0 # shellcheck disable=SC1090 . "$SSH_NOTIFY_CONF" [[ -n "${NTFY_URL:-}" ]] || return 0 command -v curl >/dev/null 2>&1 || return 0 local host prio tags body host="$(hostname -f 2>/dev/null || hostname 2>/dev/null || echo unknown)" prio="min" [[ "$REBOOT" == "1" || -n "$NEW_RELEASE" ]] && prio="default" [[ -n "$UPGRADED_TO" ]] && prio="high" body="Auto-update on ${host} (${OS_ID}) packages upgraded: ${UPGRADED}" [[ "$REBOOT" == "1" ]] && body="${body} reboot needed: yes$( [[ "$AUTO_REBOOT" == "1" ]] && echo ' (rebooting)' )" if [[ -n "$UPGRADED_TO" ]]; then body="${body} UPGRADED Alpine release: ${CUR_BRANCH} -> ${UPGRADED_TO}" elif [[ -n "$NEW_RELEASE" ]]; then body="${body} NEW Alpine release available: ${NEW_RELEASE} (current branch ${CUR_BRANCH})" fi set -- -fsS -m 5 -H "X-Title: Host Update" -H "X-Priority: ${prio}" [[ -n "${NTFY_TOKEN:-}" ]] && set -- "$@" -H "Authorization: Bearer ${NTFY_TOKEN}" [[ -n "${NTFY_EMAIL:-}" ]] && set -- "$@" -H "X-Email: ${NTFY_EMAIL}" local t="update"; [[ -n "${NTFY_REGION:-}" ]] && t="${t},${NTFY_REGION}" [[ -n "$NEW_RELEASE" && -z "$UPGRADED_TO" ]] && t="${t},new_release" [[ -n "$UPGRADED_TO" ]] && t="${t},release_upgraded" set -- "$@" -H "X-Tags: ${t}" if [[ "$DRY_RUN" == "1" ]]; then echo "DRY: curl $* -d $NTFY_URL" else curl "$@" -d "$body" "$NTFY_URL" >/dev/null 2>&1 || true fi return 0 } # Are there any active inbound SSH connections? Counts established TCP # connections on the sshd port -- this catches ProxyJump tunnels too, which # never open a login session (so `who` would miss them). ssh_sessions_active() { local port n port="$(awk '/^Port /{print $2; exit}' /etc/ssh/sshd_config 2>/dev/null || true)" port="${port:-22}" if command -v ss >/dev/null 2>&1; then n="$(ss -Htn state established "sport = :$port" 2>/dev/null | grep -c . || true)" elif command -v netstat >/dev/null 2>&1; then n="$(netstat -tn 2>/dev/null | awk -v p=":$port" '$NF=="ESTABLISHED" && $4 ~ p"$"' | grep -c . || true)" else # last resort: sshd per-connection worker processes n="$(ps ax 2>/dev/null | grep -E '[s]shd:.*@|[s]shd: ' | grep -c . || true)" fi [ "${n:-0}" -gt 0 ] } # ============================================================================ # Subcommands # ============================================================================ do_run() { [[ $EUID -eq 0 ]] || die "Run as root." os_detect # A reboot pending from a previous (deferred) run -- the flag is in /run, # wiped on a real reboot, so its presence means "still pending". [[ -f "$REBOOT_FLAG" ]] && REBOOT=1 [[ "$DRY_RUN" == "1" ]] || { install -d -m 0755 "$(dirname "$LOG")" 2>/dev/null || true; echo "=== auto-update $(date -u +%FT%TZ) ===" >> "$LOG"; } log "Applying updates (${OS_ID})..." apply_updates check_new_release if [[ -n "$NEW_RELEASE" && "$ALLOW_RELEASE_UPGRADE" == "1" ]]; then do_release_upgrade fi # Record the pending reboot so a deferral survives to the next daily run. if [[ "$REBOOT" == "1" && "$DRY_RUN" != "1" ]]; then : > "$REBOOT_FLAG" 2>/dev/null || true; fi log "Upgraded: ${UPGRADED} | reboot: ${REBOOT} | new release: ${NEW_RELEASE:-none}${UPGRADED_TO:+ | upgraded to: $UPGRADED_TO}" send_notice if [[ "$REBOOT" == "1" ]]; then case "$AUTO_REBOOT" in 1) log "Rebooting (AUTO_REBOOT=1)..."; run "reboot" ;; idle) if ssh_sessions_active; then warn "Reboot needed, but SSH connections are active; deferring until idle (retried daily)." else log "No active SSH connections; rebooting (AUTO_REBOOT=idle)..."; run "reboot" fi ;; *) warn "A reboot is recommended (kernel/libc/crypto updated). Set AUTO_REBOOT=1 or idle to automate." ;; esac fi } # Write /etc/auto-update.conf so the scheduled run inherits these defaults. write_autoupdate_conf() { local f="$AUTO_UPDATE_CONF" if [[ -f "$f" && "${AU_FORCE_CONF:-0}" != "1" ]]; then log "$f exists; leaving it (set AU_FORCE_CONF=1 to overwrite)." return 0 fi cat > "$f" <