diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..9833e4a --- /dev/null +++ b/.gitattributes @@ -0,0 +1,15 @@ +# Force LF for everything textual so scripts edited on Windows still run on +# Alpine/Debian/Alma. A CRLF in a shebang line breaks the interpreter. +* text=auto eol=lf + +*.sh text eol=lf +*.yml text eol=lf +*.yaml text eol=lf +*.conf text eol=lf +*.env text eol=lf +*.example text eol=lf +Caddyfile text eol=lf +*.md text eol=lf + +# Treat the embedded-archive deploy scripts as text too (they're base64+shell). +deployments/*/deploy.sh text eol=lf diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml new file mode 100644 index 0000000..19d9406 --- /dev/null +++ b/.gitea/workflows/release.yml @@ -0,0 +1,41 @@ +# Gitea Actions: on a version tag (vX.Y.Z), build the self-extracting bundle +# and publish it as a release asset. Make the release public (Gitea per-repo +# setting) and the bundle is downloadable without granting repo access. +# +# Requirements: +# - A registered Gitea act_runner whose labels include the `runs-on` value +# below (adjust if yours differ, e.g. self-hosted). +# - A repo/org secret named TOKEN_GITEA holding a Gitea access token with +# write:repository scope (Settings -> Applications -> Generate Token). +# NOTE: Gitea reserves the GITEA_ prefix, so the name can't start with it; +# TOKEN_GITEA is fine. + +name: Release bundle + +on: + push: + tags: + - 'v*' + +jobs: + bundle: + runs-on: ubuntu-latest + steps: + - name: Checkout (full history + tags) + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Build self-extracting bundle + run: | + bash build-bundle.sh "${GITHUB_REF_NAME}" + ls -l dist/ + + - name: Create release and attach the bundle + uses: https://gitea.com/actions/release-action@main + with: + files: |- + dist/automations-bundle.sh + api_key: '${{ secrets.TOKEN_GITEA }}' + # title/body default to the tag; uncomment to customize: + # title: 'automations ${{ github.ref_name }}' diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..41626af --- /dev/null +++ b/.gitignore @@ -0,0 +1,33 @@ +# ── Secrets / runtime config ──────────────────────────────────────────────── +# Populated env / config files (keep the *.example templates). +.env +**/.env +globals/globals.env +ssh-notify.conf +auto-update.conf +!*.env.example +!**/.env.example +!*.conf.example + +# ── Private keys (NEVER commit) ───────────────────────────────────────────── +# age identities and SSH private keys. globals/age-pubkey.txt and +# globals/authorized_keys are PUBLIC and intentionally tracked. +*-private-key.txt +*age-key* +*.age.key +id_ed25519 +id_ed25519_* +*.pem + +# ── Backups ───────────────────────────────────────────────────────────────── +*.tar.gz.age +*-backup-*.tar.gz* + +# ── Build output ──────────────────────────────────────────────────────────── +dist/ + +# ── Editor / OS noise ─────────────────────────────────────────────────────── +*.tmp +*.swp +.DS_Store +Thumbs.db diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..f477a2d --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 William Gill + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..ba5f775 --- /dev/null +++ b/README.md @@ -0,0 +1,197 @@ +# automations + +Deployment and automation scripts for self-hosted, security-hardened +infrastructure. The host-provisioning scripts run on **Alpine, Debian, and +Alma Linux** (every distro difference is gated in +[`scripts/oslib.sh`](scripts/oslib.sh)); each Docker stack runs behind Caddy +with automatic Let's Encrypt TLS, orchestrated with Docker Compose. + +## One command to run anything + +```bash +curl -fsSL https://git.anomalous.dev/57_Wolve/automations/raw/branch/main/automations.sh \ + | REPO_URL=https://git.anomalous.dev/57_Wolve/automations.git bash +``` + +Or, from a clone: + +```bash +./automations.sh +``` + +[`automations.sh`](automations.sh) opens a **Gum** wizard (auto-installed) that +lets you: + +- **Deploy on this host** — pick any deployment or generic script and run it, + prompting for the values it needs. +- **Build artifacts locally** — regenerate a deployment's self-contained + `deploy.sh` and optionally `scp` it to a target host. + +Shared defaults (email, repo URL, SSH key source, backup recipient) come from +[`globals/`](globals/) so you set them once. + +## Self-contained bundle (no repo access) + +For hosts that shouldn't have repo/git access, package the whole repo into one +self-extracting script and serve it from a webserver or a **public Gitea +release** (the repo itself can stay private): + +```bash +./build-bundle.sh # -> dist/automations-bundle.sh (HEAD) +./build-bundle.sh v1.2.0 # a specific tag +./build-bundle.sh worktree # current working tree +``` + +On the target host, download then run it (it extracts itself — it can't read a +pipe, so download first): + +```bash +curl -fsSLO https://your-host/automations-bundle.sh +bash automations-bundle.sh # launcher wizard +bash automations-bundle.sh bash scripts/setup-host.sh # run one script +SSH_PORT=2222 bash automations-bundle.sh bash scripts/harden-jumphost.sh +``` + +It extracts to `INSTALL_DIR` (default `/opt/automations`) and runs the launcher +or the command you pass. The payload excludes ignored files, so no secrets are +embedded. + +**Releases via Gitea Actions:** pushing a `vX.Y.Z` tag runs +[`.gitea/workflows/release.yml`](.gitea/workflows/release.yml), which builds the +bundle and attaches it to a release. Create a secret named **`TOKEN_GITEA`** +(a Gitea access token with `write:repository`) — note Gitea reserves the +`GITEA_` prefix, so the secret can't be called `GITEA_TOKEN`. Adjust `runs-on` +to match your runner. Mark the release public and `automations-bundle.sh` is +fetchable without repo access. + +## Layout + +The rule: **generic, run-anywhere scripts live in `scripts/`; deployment-specific +files live under `deployments//`.** Shared values live in `globals/`. + +``` +automations.sh # the launcher (one-liner entry point) +cloud-init/ # generic base + jumphost cloud-init (any of the 3 distros) +globals/ # shared assets: age-pubkey, authorized_keys, motd, globals.env +scripts/ # generic scripts (Alpine/Debian/Alma) +deployments// # one folder per stack +``` + +### `scripts/` — generic (Alpine / Debian / Alma) + +| Script | What it does | +|--------|--------------| +| [`setup-host.sh`](scripts/setup-host.sh) | Set hostname per the naming schema (derives FQDN + Node ID) and render the shared MOTD with auto-computed border spacing. | +| [`harden-ssh.sh`](scripts/harden-ssh.sh) | SSH hardening: post-quantum hybrid KEX, fresh Ed25519 host keys, key-only auth, external SFTP subsystem, sshguard. | +| [`harden-jumphost.sh`](scripts/harden-jumphost.sh) | Bastion hardening on top of `harden-ssh`: `ssh-admins` (shell) vs `ssh-jumpers` (ProxyJump-only) with a PermitOpen allow-list. | +| [`sshuser.sh`](scripts/sshuser.sh) | Add/edit/remove SSH users on a hardened jump host (Gum TUI or CLI flags). Installed standalone as `sshuser`. | +| [`ntfy-ssh-login.sh`](scripts/ntfy-ssh-login.sh) | `pam_exec` hook that posts SSH logins to ntfy (user, source IP, key used, best-effort jump target), gated by group. Config: [`ssh-notify.conf.example`](scripts/ssh-notify.conf.example). | +| [`auto-update.sh`](scripts/auto-update.sh) | Daily unattended package updates; reports (doesn't auto-jump) a new Alpine branch; reboot detection; ntfy summary. `install`/`run`/`uninstall`. | +| [`oslib.sh`](scripts/oslib.sh) | OS-abstraction layer (detection, package manager, init system, SFTP path, hostname, boot hooks, sshguard, login notifier). Sourced, not run. **One file holds every distro difference.** | +| [`lib.sh`](scripts/lib.sh) | Launcher helpers (`ensure_gum`, `load_globals`, `resolve_ssh_keys`); sources `oslib.sh`. Not run directly. | + +### `deployments/` — per stack + +| Deployment | What it is | Depends on | +|------------|------------|------------| +| [`pocket-id`](deployments/pocket-id/) | OIDC provider (Caddy + Anubis PoW gate + Pocket-ID). | — | +| [`beszel`](deployments/beszel/) | Server monitoring hub. OIDC via pocket-id (post-deploy). | pocket-id (OIDC, optional) | +| [`headscale`](deployments/headscale/) | Self-hosted Tailscale control server, OIDC login. | pocket-id (OIDC) | +| [`webfinger`](deployments/webfinger/) | Serves `/.well-known/webfinger` for OIDC discovery; redirects the rest. | pocket-id (issuer) | +| [`simplex`](deployments/simplex/) | SimpleX SMP + XFTP relay with Tor hidden services + encrypted backups. | globals/age-pubkey.txt | + +## Conventions + +- **Alpine + Docker Compose + Caddy/Let's Encrypt** across every stack. +- **`build.sh` → `deploy.sh`**: each stack's `build.sh` embeds its + `docker-compose.yml` / `Caddyfile` / `.env.example` into a single + self-contained `deploy.sh` (base64 tar.gz). That one file can be `scp`'d to a + host and run on its own. Rebuild after editing the loose files. + (simplex is the exception — it deploys via `install-simplex.sh`.) +- **Secrets** live in `.env` (generated on first deploy) and `globals/globals.env`, + both git-ignored. Only `*.example` templates and public keys are committed. +- **Non-interactive**: every `deploy.sh` honors `SKIP_PROMPTS=1` with values + supplied via the environment — which is how the per-deployment + `cloud-init.yml` templates stand a stack up unattended. + +## Cloud-init + +- **Provision a host**: [`cloud-init/base.yml`](cloud-init/base.yml) (hostname + + MOTD + SSH hardening) or [`cloud-init/jumphost.yml`](cloud-init/jumphost.yml) + (bastion). Distro-agnostic — they install prerequisites for whatever the image + is, then run the scripts. +- **Stand up a stack**: each deployment ships its own `cloud-init.yml` (e.g. + [`deployments/pocket-id/cloud-init.yml`](deployments/pocket-id/cloud-init.yml)). + +Fill in `REPO_URL` and the values at the top of the `runcmd` block, paste as +instance user-data, and the host configures itself on first boot. + +## Multi-OS notes + +The host-provisioning scripts (`setup-host`, `harden-ssh`, `harden-jumphost`, +`sshuser`) and the four Docker stacks (pocket-id, beszel, headscale, webfinger) +run on Alpine, Debian, and Alma. Distro differences live in +[`scripts/oslib.sh`](scripts/oslib.sh) — package manager (`apk`/`apt`/`dnf`), +init system (OpenRC/systemd), sshd service name, the per-distro `sftp-server` +path, hostname, boot hooks, and the sshguard log source/backend. + +**simplex** remains **Alpine-targeted** — it depends on `awall` and Tor hidden +services with Alpine-specific wiring, so it isn't part of the tri-distro set. + +## SSH login notifications + +The harden scripts can install a `pam_exec` hook +([`scripts/ntfy-ssh-login.sh`](scripts/ntfy-ssh-login.sh)) that posts every SSH +login to an [ntfy](https://ntfy.sh) topic. Enable it by passing `NTFY_URL` (and +optionally `NTFY_TOKEN`) when running `harden-ssh.sh` / `harden-jumphost.sh`, or +via the launcher / jumphost cloud-init. Each alert reports: + +- the **user** and **source IP**, +- the **SSH key** they authenticated with (fingerprint, via `ExposeAuthInfo`), +- a **region tag** so you know which bastion fired it (derived from the host's + FQDN, e.g. `us-evi-1`), +- best-effort the **next hop** of a ProxyJump (see caveat below). + +Filtering is by group: `NOTIFY_GROUPS` limits alerts to certain +groups/security levels, and `NOTIFY_PRIORITY_MAP` sets a per-group ntfy +priority. Config lives at `/etc/ssh-notify.conf` (mode 0600; +[`scripts/ssh-notify.conf.example`](scripts/ssh-notify.conf.example) documents +every key). A publish token is optional — leave it empty for a read-gated topic. + +> **Jump-target caveat:** a ProxyJump opens a *direct-tcpip* channel (no +> session), so the destination never reaches `pam_exec`. The bastion only logs +> it at `LogLevel VERBOSE`/`DEBUG`; `harden-jumphost.sh` sets `VERBOSE` and the +> notifier parses the log best-effort. If the target isn't in the log it is +> simply omitted. + +## Daily updates + +[`scripts/auto-update.sh`](scripts/auto-update.sh) keeps a host patched +unattended — ideal for an SSH-only bastion, where a routine upgrade can barely +break anything. `harden-jumphost.sh` schedules it **by default** (set +`AUTO_UPDATE=0` to skip); `harden-ssh.sh` and `cloud-init/base.yml` take +`AUTO_UPDATE=1`. It runs daily via busybox `crond` (`/etc/periodic/daily`) on +Alpine or a systemd timer on Debian/Alma. + +Each run: +- applies all **in-branch** package upgrades (`apk`/`apt`/`dnf`); +- **reports** a new Alpine *branch* (e.g. 3.21 → 3.22) by default — that + rewrites the repo branch, so it's opt-in. Set `ALLOW_RELEASE_UPGRADE=1` to + also **apply** it: it repoints `/etc/apk/repositories` to the newest **stable** + `vX.Y` (never `edge`), runs `apk upgrade --available`, and forces a reboot + flag. Debian/Alma stay report-only; +- detects when a **reboot** is needed (kernel/libc/openssl). `AUTO_REBOOT` + controls it: `0` = never (just flag), `1` = always, **`idle` = only when no + SSH connections are active** — so a bastion reboots itself once the admins and + ProxyJump tunnels have cleared, never mid-session. `harden-jumphost.sh` + defaults the bastion to `idle`. A deferred reboot is tracked in `/run` and + retried each day until it happens; +- sends an ntfy summary (reusing `/etc/ssh-notify.conf`). + +Schedule it standalone with `auto-update.sh install` (or via the launcher), run +a pass now with `auto-update.sh run`, and preview safely with +`DRY_RUN=1 auto-update.sh run`. + +## License + +[MIT](LICENSE). diff --git a/automations.sh b/automations.sh new file mode 100644 index 0000000..b5ad668 --- /dev/null +++ b/automations.sh @@ -0,0 +1,211 @@ +#!/usr/bin/env bash +# +# automations.sh -- one command to run or deploy anything in this repo. +# +# Run it two ways: +# +# 1. One-liner on a fresh target host (clones the repo, then launches): +# curl -fsSL https://git.anomalous.dev/57_Wolve/automations/raw/branch/main/automations.sh \ +# | REPO_URL=https://git.anomalous.dev/57_Wolve/automations.git bash +# +# 2. From a clone: +# ./automations.sh +# +# It opens a Gum wizard (auto-installed) that lets you: +# • Mode: deploy on THIS host, or build deploy.sh artifacts locally. +# • Pick any deployment (pocket-id, beszel, headscale, webfinger, simplex) +# or any generic script (harden-ssh, harden-jumphost, sshuser). +# Shared defaults come from globals/ (see globals/README.md). +# +# Non-interactive: set SKIP_PROMPTS=1 plus the needed vars and pipe the menu +# choices in, or just call the underlying deployments//deploy.sh +# directly -- they all honor SKIP_PROMPTS=1. + +set -euo pipefail + +# ---------------------------------------------------------------------------- +# Self-locate, or bootstrap by cloning the repo (one-liner / piped form). +# ---------------------------------------------------------------------------- +_self="${BASH_SOURCE[0]:-}" +if [[ -n "$_self" && -f "$(cd "$(dirname "$_self")" 2>/dev/null && pwd)/scripts/lib.sh" ]]; then + ROOT="$(cd "$(dirname "$_self")" && pwd)" +else + # Piped via curl: we don't have the repo on disk. Clone it, then re-exec. + : "${REPO_URL:=}" + : "${REPO_BRANCH:=main}" + [[ -n "$REPO_URL" ]] || { + echo "[x] Running standalone (piped). Set REPO_URL=... so I can clone the repo." >&2 + exit 1 + } + command -v git >/dev/null 2>&1 || { command -v apk >/dev/null 2>&1 && apk add -q git; } + _tmp="$(mktemp -d -t automations.XXXXXX)" + echo "[+] Cloning $REPO_URL ($REPO_BRANCH)..." + git clone --depth 1 --branch "$REPO_BRANCH" "$REPO_URL" "$_tmp" + exec bash "$_tmp/automations.sh" "$@" +fi + +# shellcheck source=scripts/lib.sh +. "$ROOT/scripts/lib.sh" +load_globals + +DEPLOYMENTS=(pocket-id beszel headscale webfinger simplex) +SCRIPTS=(setup-host harden-ssh harden-jumphost sshuser auto-update) + +# ---------------------------------------------------------------------------- +# Prompt helpers (gum). `ask` records each answer in ENVS for passing onward. +# ---------------------------------------------------------------------------- +ENVS=() +ask() { #