automations/deployments/ergo/ergoctl

#!/usr/bin/env bash
#
# ergoctl -- administer the Ergo IRC stack from the host.
#
# Installed by deploy.sh as $STACK_DIR/ergoctl with a wrapper at
# /usr/local/bin/ergoctl. Talks to Ergo over the loopback plaintext listener
# (127.0.0.1:6667) as the 'admin' oper, using the password in secrets/admin.pass.
# Config changes are validated in a throwaway container (ergo run --smoke on a
# copy) and applied with REHASH; on failure the previous file is restored.
#
# Run `ergoctl help` for the command list.

set -euo pipefail

SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

log()  { printf '\033[1;32m[+]\033[0m %s\n' "$*"; }
warn() { printf '\033[1;33m[!]\033[0m %s\n' "$*" >&2; }
die()  { printf '\033[1;31m[x]\033[0m %s\n' "$*" >&2; exit 1; }

# shellcheck source=ergolib.sh
. "$SELF_DIR/ergolib.sh"
load_conf
: "${STACK_DIR:=$SELF_DIR}"
ergo_set_paths
CONF="$IRCD_DIR/ircd.yaml"
MOTD="$IRCD_DIR/ergo.motd"

usage() {
    cat <<EOF
ergoctl -- administer the Ergo IRC server (stack: $STACK_DIR)

Status & logs
  status                     containers, versions, TLS cert, user count, schedule
  users                      LUSERS summary
  logs [-f] [N]              ergo log (health-probe noise filtered; default last 100)
  version                    running and pinned version, latest release
  cert [show|sync]           cert on disk vs served on :6697; 'sync' pulls Caddy's now

Configuration (validated, then REHASH; reverted on failure)
  edit                       \$EDITOR ircd/ircd.yaml
  motd                       \$EDITOR ircd/ergo.motd
  rehash                     reload config, MOTD and TLS certs (no disconnects)
  restart | stop | start     the ergo container (restart drops every user)
  caddy-restart              apply Caddyfile / conf.d changes
  debug on|off               switch the log level (info <-> debug)

Operators
  oper list
  oper add <name> [class]    class: chat-moderator (default) | server-admin; prints the password once
  oper passwd <name>         new random password for an oper
  oper rm <name>
  passwd [show|rotate]       the 'admin' oper password (secrets/admin.pass)

Moderation (as the admin oper)
  announce <text>            NOTICE to all users
  kill <nick> [reason]
  ban add <ip|cidr|nick!user@host|account> [duration] [reason]   e.g. ban add 203.0.113.9 1d spam
                             (a bare name is treated as an ACCOUNT to suspend, not a
                              connected nick -- use 'ban info <nick>' to get their IP first)
  ban del <target> | ban list | ban info <target>
  defcon [1-5]               show or set the DEFCON level
  cmd <raw IRC line>         anything else, e.g.  cmd NS SAREGISTER alice hunter2
                             (replies are printed; control characters stripped)

Data
  backup [--live] [dir]      tar.gz of ircd.yaml, ircd.db, ergo.motd (default: stop Ergo
                             briefly for a consistent copy; --live = crash-consistent).
                             Encrypted with age when $STACK_DIR/age-recipients.txt exists.
  restore <file>             restore a backup (stops Ergo, restarts, waits for health)

Updates (update.sh)
  update [check|update|run|certsync|caddy|install|uninstall]

  shell                      sh inside the ergo container
EOF
}

require_root() { [[ $EUID -eq 0 ]] || die "Run as root."; }
require_stack() { [[ -f "$STACK_DIR/docker-compose.yml" && -f "$CONF" ]] || die "No deployed stack at $STACK_DIR (set STACK_DIR)."; }

# Validate the current ./ircd with the running image; on failure print the log.
validate_or_fail() {  # <what>
    if ergo_validate_config; then rm -f "$VALIDATE_LOG"; return 0; fi
    warn "$1 did not load in $(ergo_image):"
    tail -n 15 "$VALIDATE_LOG" >&2 || true
    rm -f "$VALIDATE_LOG"
    return 1
}

# Apply a config change: validate, then REHASH; revert the file(s) on failure.
# $1 = description; $2.. = files that were changed (each has a .last-good copy).
apply_change() {
    local what="$1"; shift
    local f rc=0
    if ! validate_or_fail "$what"; then
        for f in "$@"; do [[ -f "$f.last-good" ]] && cat "$f.last-good" > "$f"; done
        die "$what rejected; previous version restored."
    fi
    if ! ergo_running; then
        log "$what saved (Ergo is not running; it will load on start)."
        for f in "$@"; do rm -f "$f.last-good"; done
        return 0
    fi
    # Take the status from the call itself: `$?` after an `if` whose condition
    # failed and that has no else branch is 0, which would make the SIGHUP
    # fallback below unreachable.
    irc_rehash || rc=$?
    if (( rc == 0 )); then
        log "$what applied (rehash complete)."
        for f in "$@"; do rm -f "$f.last-good"; done
        return 0
    fi
    # Ergo refuses to rehash settings that are fixed at startup. It phrases them
    # all as "... after launching the server", so match that, not one variant.
    if [[ "$REHASH_MSG" == *"after launching the server"* || "$REHASH_MSG" == *"cannot be changed after launching"* ]]; then
        warn "Rehash refused: $REHASH_MSG"
        warn "The file is valid and saved; apply it with:  ergoctl restart   (disconnects everyone)"
        for f in "$@"; do rm -f "$f.last-good"; done
        return 0
    fi
    if (( rc == 2 )); then
        warn "Could not REHASH over IRC ($REHASH_MSG); sending SIGHUP instead."
        dc kill -s HUP ergo >/dev/null 2>&1 || true
        sleep 2
        if dc logs --since 15s ergo 2>/dev/null | grep -qiE 'rehash.*(complete|success)'; then
            log "$what applied (rehash via SIGHUP)."; for f in "$@"; do rm -f "$f.last-good"; done; return 0
        fi
    fi
    for f in "$@"; do [[ -f "$f.last-good" ]] && cat "$f.last-good" > "$f"; done
    die "Rehash failed: ${REHASH_MSG}. Previous version restored (Ergo kept running on the old config)."
}

snapshot_file() { cp -p "$1" "$1.last-good"; }

edit_file() {  # <file> <what>
    local f="$1" what="$2" tmp
    require_root; require_stack
    tmp="$(mktemp)"; cat "$f" > "$tmp"
    "${EDITOR:-vi}" "$tmp"
    if cmp -s "$f" "$tmp"; then rm -f "$tmp"; log "No changes."; return 0; fi
    snapshot_file "$f"
    cat "$tmp" > "$f"; rm -f "$tmp"
    apply_change "$what" "$f"
}

oper_name_ok() { [[ "$1" =~ ^[a-z][a-z0-9_-]{1,31}$ ]]; }

cmd_status() {
    require_stack
    echo "Stack:    $STACK_DIR  (domain $(ergo_domain), network $(env_get NETWORK_NAME))"
    dc ps 2>/dev/null || true
    echo
    local st cur latest
    st="$(svc_state ergo || true)"; cur="$(ergo_version_running || true)"
    latest="$(normver "$(ergo_latest_tag || true)")"
    printf 'Ergo:     %s | running %s | pinned %s | latest %s\n' "${st:-not created}" "${cur:-?}" "$(ergo_tag)" "${latest:-?}"
    if [[ "$(env_get PLAINTEXT)" == "1" ]]; then echo "          PUBLIC PLAINTEXT LISTENER on :6667 (PLAINTEXT=1)"; fi
    cmd_cert show
    if ergo_running; then
        echo; cmd_users || true
    fi
    echo
    printf 'Policy:   %s | schedule: ' "${UPDATE_POLICY:-$(env_get UPDATE_POLICY)}"
    if [[ -x /etc/periodic/15min/ergo-certsync || -f /etc/systemd/system/ergo-certsync.timer ]]; then printf 'certsync 15min'; else printf 'certsync MISSING'; fi
    if [[ -x /etc/periodic/daily/ergo-update || -f /etc/systemd/system/ergo-update.timer ]]; then printf ', update daily\n'; else printf ', update NOT scheduled\n'; fi
    local s; s="$(state_get certsync)"; [[ -z "$s" ]] || echo "certsync: LAST FAILURE: $s"
}

cmd_users() {
    local out
    out="$(irc_raw --quiet 1 -- LUSERS)" || die "Could not talk to Ergo on 127.0.0.1:6667 (is it running?)."
    # ":server 251 nick :There are N users..." -> "There are N users..."
    printf '%s\n' "$out" | awk '$2=="251"||$2=="252"||$2=="254"||$2=="265"||$2=="266" {
        sub(/^:[^ ]+ [0-9]+ [^ ]+ /, ""); sub(/^:/, ""); sub(/ :/, " "); print "          " $0 }'
}

cmd_logs() {
    require_stack
    local follow="" n=100
    while [[ $# -gt 0 ]]; do
        case "$1" in -f|--follow) follow="-f" ;; *[0-9]*) n="$1" ;; esac; shift
    done
    # Each healthcheck probe logs a 'Client connecting: real IP 127.0.0.1' line; hide those.
    dc logs --tail "$n" $follow ergo 2>&1 | grep -vE 'real IP 127\.0\.0\.1, proxied IP <nil>|Preexisting TLS cert' || true
}

cmd_cert() {
    require_stack
    local f="$IRCD_DIR/fullchain.pem" served
    case "${1:-show}" in
        sync) require_root; bash "$STACK_DIR/update.sh" certsync ;;
        *)
            if [[ ! -f "$f" ]]; then echo "TLS:      no cert in ircd/ yet"; return 0; fi
            if cert_is_selfsigned "$f"; then
                echo "TLS:      SELF-SIGNED (Caddy has not issued a cert yet -- DNS/80/443?)  expires $(cert_enddate "$f")"
            else
                echo "TLS:      $(cert_subject_cn "$f")  expires $(cert_enddate "$f")  (from Caddy)"
            fi
            if ergo_running; then
                served="$(served_fingerprint || true)"
                if [[ -z "$served" ]]; then echo "          :6697 did not complete a TLS handshake"
                elif [[ "$served" == "$(cert_fingerprint "$f")" ]]; then echo "          :6697 serves this cert"
                else echo "          :6697 serves a DIFFERENT cert -- run: ergoctl rehash"; fi
            fi ;;
    esac
}

cmd_oper() {
    require_root; require_stack
    local sub="${1:-list}" name="${2:-}" class="${3:-chat-moderator}" pw hash
    case "$sub" in
        list) yaml_oper_list "$CONF" | awk '{printf "  %-20s %s\n", $1, $2}' ;;
        add)
            [[ -n "$name" ]] || die "Usage: ergoctl oper add <name> [chat-moderator|server-admin]"
            oper_name_ok "$name" || die "Oper names: lowercase letters, digits, _ - (2-32 chars)."
            yaml_oper_has "$CONF" "$name" && die "Oper '$name' already exists."
            grep -qE "^    \"?${class}\"?:" "$CONF" || warn "Class '$class' is not defined under oper-classes in ircd.yaml -- the rehash will fail unless you add it."
            pw="$(ergo_random_password 24)"
            hash="$(ergo_genpasswd "$(ergo_image)" "$pw")" || die "genpasswd failed."
            snapshot_file "$CONF"
            yaml_oper_add "$CONF" "$name" "$class" "$hash" || { cat "$CONF.last-good" > "$CONF"; die "Could not insert the oper block."; }
            apply_change "oper '$name'" "$CONF"
            echo
            echo "  Oper '$name' ($class) added. Log in with:   /OPER $name $pw"
            echo "  (shown once; rotate with: ergoctl oper passwd $name)" ;;
        passwd)
            [[ -n "$name" ]] || die "Usage: ergoctl oper passwd <name>"
            yaml_oper_has "$CONF" "$name" || die "No oper '$name' in ircd.yaml."
            pw="$(ergo_random_password 24)"
            hash="$(ergo_genpasswd "$(ergo_image)" "$pw")" || die "genpasswd failed."
            snapshot_file "$CONF"
            yaml_oper_set_password "$CONF" "$name" "$hash" || { cat "$CONF.last-good" > "$CONF"; die "Oper '$name' has no password line to replace."; }
            apply_change "oper '$name' password" "$CONF"
            if [[ "$name" == "admin" ]]; then ( umask 077; printf '%s\n' "$pw" > "$ADMIN_PASS_FILE" ); fi
            echo
            echo "  New password for '$name':   /OPER $name $pw" ;;
        rm|remove|del)
            [[ -n "$name" ]] || die "Usage: ergoctl oper rm <name>"
            [[ "$name" != "admin" ]] || die "Refusing to remove 'admin' (ergoctl uses it). Rotate its password instead."
            yaml_oper_has "$CONF" "$name" || die "No oper '$name' in ircd.yaml."
            snapshot_file "$CONF"
            yaml_oper_rm "$CONF" "$name" || { cat "$CONF.last-good" > "$CONF"; die "Could not remove the oper block."; }
            apply_change "removal of oper '$name'" "$CONF" ;;
        *) die "Usage: ergoctl oper list|add|passwd|rm" ;;
    esac
}

cmd_passwd() {
    require_root; require_stack
    case "${1:-show}" in
        show)   [[ -f "$ADMIN_PASS_FILE" ]] || die "No $ADMIN_PASS_FILE -- run: ergoctl passwd rotate"
                echo "/OPER admin $(head -n1 "$ADMIN_PASS_FILE")" ;;
        rotate) cmd_oper passwd admin
                if irc_raw --oper --quiet 1 -- >/dev/null 2>&1; then log "Verified: the new admin password works."; else warn "Could not verify the new password over IRC (is Ergo running?)."; fi ;;
        *) die "Usage: ergoctl passwd [show|rotate]" ;;
    esac
}

cmd_cmd() {  # raw line as oper
    require_root; require_stack
    [[ $# -gt 0 ]] || die "Usage: ergoctl cmd <raw IRC line>"
    local out rc=0
    out="$(irc_raw --oper --quiet 2 -- "$*")" || rc=$?
    case "$rc" in
        0) ;;
        3) die "OPER failed (wrong admin password? see secrets/admin.pass; the server logs details under type 'opers')." ;;
        4) die "Cannot connect to 127.0.0.1:6667 -- is Ergo running?" ;;
        *) die "Registration with the server failed (rc=$rc)." ;;
    esac
    # Drop the routine registration burst and show only what the command produced.
    # The MODE filter is anchored to our own throwaway nick so that MODE replies
    # caused by the command itself (e.g. SAMODE #chan +m) still print.
    printf '%s\n' "$out" | grep -vE '^:[^ ]+ (00[1-5]|25[0-9]|26[56]|37[256]|422|381) |^:[^ ]+ MODE ergoctl[0-9]+ ' || true
}

cmd_backup() {
    require_root; require_stack
    local live=0 dest="$BACKUP_DIR" ts stage out was_running=0 users=""
    while [[ $# -gt 0 ]]; do case "$1" in --live) live=1 ;; *) dest="$1" ;; esac; shift; done
    install -d -m 0700 "$dest"
    # A recipients file means "encrypt these" -- refuse rather than silently
    # writing account hashes and the oper hash out in the clear.
    if [[ -f "$STACK_DIR/age-recipients.txt" ]] && ! command -v age >/dev/null 2>&1; then
        die "$STACK_DIR/age-recipients.txt exists but 'age' is not installed, so the backup would be UNENCRYPTED. Install it (apk add age / apt install age / dnf install age) or move the recipients file away."
    fi
    ts="$(date -u +%Y%m%d-%H%M%S)"
    stage="$(mktemp -d)"
    if ergo_running; then
        was_running=1
        if [[ "$live" == 0 ]]; then
            users="$(irc_raw --quiet 1 -- LUSERS 2>/dev/null | awk '$2=="251" {print}' | sed 's/.*:There are //' || true)"
            log "Stopping Ergo for a consistent snapshot (${users:-users unknown})..."
            irc_raw --oper --quiet 1 -- 'NOTICE $$* :Brief restart for a backup -- back in a few seconds.' >/dev/null 2>&1 || true
            dc stop -t 15 ergo >/dev/null 2>&1 || true
        else
            warn "Live backup: ircd.db is copied while Ergo writes to it (crash-consistent; may miss the last second)."
        fi
    fi
    cp -p "$CONF" "$stage/ircd.yaml"
    [[ -f "$IRCD_DIR/ircd.db" ]] && cp -p "$IRCD_DIR/ircd.db" "$stage/ircd.db"
    [[ -f "$MOTD" ]] && cp -p "$MOTD" "$stage/ergo.motd"
    cat > "$stage/meta" <<EOF
ERGO_TAG=$(ergo_tag)
ERGO_DOMAIN=$(ergo_domain)
DATE=$(date -u +%FT%TZ)
HOST=$(hostname -f 2>/dev/null || hostname)
CONSISTENT=$(( live == 0 ))
EOF
    if [[ "$was_running" == 1 && "$live" == 0 ]]; then dc start ergo >/dev/null 2>&1 || true; fi
    out="$dest/ergo-backup-${ts}.tar.gz"
    ( umask 077; tar -czf "$out" -C "$stage" . )
    rm -rf "$stage"
    if [[ -f "$STACK_DIR/age-recipients.txt" ]]; then
        age -R "$STACK_DIR/age-recipients.txt" -o "$out.age" "$out" || { rm -f "$out.age"; die "age encryption failed; the plaintext tar is still at $out."; }
        rm -f "$out"; out="$out.age"
    fi
    chmod 0600 "$out"
    log "Backup written: $out ($(wc -c < "$out") bytes). Copy it off this host."
    [[ "$out" == *.age ]] || warn "Unencrypted: it holds account hashes and the oper hash. Put an age public key in $STACK_DIR/age-recipients.txt (and install 'age') to encrypt future backups."
}

cmd_restore() {
    require_root; require_stack
    local file="${1:-}" stage tag
    [[ -f "$file" ]] || die "Usage: ergoctl restore <backup.tar.gz[.age]>"
    stage="$(mktemp -d)"
    if [[ "$file" == *.age ]]; then
        command -v age >/dev/null 2>&1 || die "age is required to decrypt $file."
        age -d -i "${AGE_IDENTITY:?set AGE_IDENTITY=/path/to/age/key}" "$file" | tar -xzf - -C "$stage"
    else
        tar -xzf "$file" -C "$stage"
    fi
    [[ -f "$stage/ircd.yaml" && -f "$stage/ircd.db" ]] || { rm -rf "$stage"; die "Backup lacks ircd.yaml/ircd.db."; }
    tag="$(grep '^ERGO_TAG=' "$stage/meta" 2>/dev/null | cut -d= -f2 || true)"
    if [[ -n "$tag" ]] && ver_gt "$(normver "$tag")" "$(normver "$(ergo_tag)")"; then
        warn "Backup was taken with Ergo $tag but $(ergo_tag) is pinned -- an older Ergo cannot read a newer database schema. Update first (ergoctl update update) or pin ERGO_TAG=$tag."
        read -r -p "Continue anyway? [y/N] " ans; [[ "${ans,,}" == y* ]] || { rm -rf "$stage"; die "Aborted."; }
    fi
    log "Stopping Ergo and restoring from $file..."
    dc stop -t 15 ergo >/dev/null 2>&1 || true
    install -d -m 0700 "$BACKUP_DIR"
    [[ -f "$IRCD_DIR/ircd.db" ]] && cp -p "$IRCD_DIR/ircd.db" "$BACKUP_DIR/ircd.db.pre-restore.$(date -u +%Y%m%d%H%M%S)"
    install -m 0600 -o "$(ergo_uid)" -g "$(ergo_gid)" "$stage/ircd.yaml" "$CONF"
    install -m 0600 -o "$(ergo_uid)" -g "$(ergo_gid)" "$stage/ircd.db" "$IRCD_DIR/ircd.db"
    [[ -f "$stage/ergo.motd" ]] && install -m 0644 -o "$(ergo_uid)" -g "$(ergo_gid)" "$stage/ergo.motd" "$MOTD"
    rm -rf "$stage"
    dc up -d --no-deps ergo >/dev/null 2>&1 || true
    if wait_healthy ergo 120 && irc_probe; then log "Restored; Ergo is healthy."; else dc logs --tail 20 ergo >&2 || true; die "Ergo is not healthy after the restore."; fi
}

cmd_debug() {
    require_root; require_stack
    local level
    case "${1:-}" in on) level=debug ;; off) level=info ;; *) die "Usage: ergoctl debug on|off" ;; esac
    snapshot_file "$CONF"
    yaml_set_log_level "$CONF" "$level" || { cat "$CONF.last-good" > "$CONF"; die "No 'level:' line found in the logging block."; }
    apply_change "log level $level" "$CONF"
}

case "${1:-help}" in
    help|-h|--help) usage ;;
    status)   cmd_status ;;
    users)    require_stack; cmd_users ;;
    logs)     shift; cmd_logs "$@" ;;
    version)  require_stack; printf 'running %s | pinned %s | latest %s\n' "$(ergo_version_running)" "$(ergo_tag)" "$(ergo_latest_tag || echo '?')" ;;
    cert)     shift; cmd_cert "$@" ;;
    edit)     edit_file "$CONF" "ircd.yaml" ;;
    motd)     edit_file "$MOTD" "MOTD" ;;
    rehash)   require_root; require_stack
              if irc_rehash; then log "Rehash complete."; else
                  if [[ "$REHASH_MSG" == *"could not reach"* ]]; then warn "$REHASH_MSG -- sending SIGHUP."; dc kill -s HUP ergo; else die "Rehash failed: $REHASH_MSG"; fi
              fi ;;
    restart)  require_root; require_stack; irc_raw --oper --quiet 1 -- 'NOTICE $$* :Server restarting now.' >/dev/null 2>&1 || true
              dc restart -t 15 ergo; wait_healthy ergo 120 && log "Ergo is healthy." || die "Ergo did not become healthy (ergoctl logs)." ;;
    stop)     require_root; require_stack; dc stop -t 15 ergo ;;
    start)    require_root; require_stack; dc start ergo; wait_healthy ergo 120 && log "Ergo is healthy." || die "Ergo did not become healthy (ergoctl logs)." ;;
    caddy-restart) require_root; require_stack
              dc run --rm --no-deps -T caddy caddy validate --config /etc/caddy/Caddyfile --adapter caddyfile >/dev/null || die "Caddyfile invalid (see caddy/etc/Caddyfile, conf.d/)."
              dc restart caddy; wait_healthy caddy 60 && log "Caddy is healthy." || die "Caddy did not become healthy." ;;
    debug)    shift; cmd_debug "$@" ;;
    oper)     shift; cmd_oper "$@" ;;
    passwd)   shift; cmd_passwd "$@" ;;
    cmd)      shift; cmd_cmd "$@" ;;
    announce) shift; [[ $# -gt 0 ]] || die "Usage: ergoctl announce <text>"; cmd_cmd "NOTICE \$\$* :$*" ;;
    kill)     shift; [[ $# -ge 1 ]] || die "Usage: ergoctl kill <nick> [reason]"; n="$1"; shift; cmd_cmd "KILL $n :${*:-Killed by an operator}" ;;
    ban)      shift; sub="${1:-list}"; shift || true
              case "$sub" in
                  add)  [[ $# -ge 1 ]] || die "Usage: ergoctl ban add <ip|cidr|nick!user@host|account> [duration e.g. 1d] [reason] -- a bare name means an ACCOUNT; use 'ban info <nick>' for a connected user's IP"
                        t="$1"; shift; d=""; if [[ "${1:-}" =~ ^[0-9]+[ymwdhs]$|^[0-9]+mo$ ]]; then d="DURATION $1"; shift; fi
                        cmd_cmd "UBAN ADD $t $d ${*:+:$*}" ;;
                  del)  [[ $# -ge 1 ]] || die "Usage: ergoctl ban del <target>"; cmd_cmd "UBAN DEL $1" ;;
                  info) [[ $# -ge 1 ]] || die "Usage: ergoctl ban info <target>"; cmd_cmd "UBAN INFO $1" ;;
                  list|*) cmd_cmd "UBAN LIST" ;;
              esac ;;
    defcon)   shift; cmd_cmd "DEFCON ${1:-}" ;;
    backup)   shift; cmd_backup "$@" ;;
    restore)  shift; cmd_restore "$@" ;;
    update)   shift; require_root; require_stack; exec bash "$STACK_DIR/update.sh" "${1:-check}" ;;
    shell)    require_root; require_stack; dc exec ergo sh ;;
    *)        usage; exit 1 ;;
esac