From 76783f3d711ea306041d874de2f1d780e1abc3e4 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 15 Jun 2026 13:25:27 -0700 Subject: [PATCH] test: add FUSE database load/durability/perf benchmark (#9980) * test: add FUSE database load/durability/perf benchmark Runs MySQL (InnoDB) and SQLite with ~1GB datadirs on a SeaweedFS FUSE mount. Two parts: - durability: normal shutdown, kill -9, and crash-during-write all keep every fsync-committed row (verified by integrity check + row count + contiguous prefix + per-row CRC). - performance: FUSE vs the same local disk. fsync/commit latency is the dominant cost (~0.13ms -> ~1.18ms), so small transactions run ~9-12x slower while bulk loads and warm reads stay close. Harness is path-independent (runtime under $SEAWEED_BENCH_WORK) and only touches its own processes on non-default ports. * test/benchmark/fuse_db: portable to Linux + crash-safe progress - export MYSQL_BIN; mysql_bench.py falls back to PATH when unset - unmount via fusermount/fusermount3 (non-root Linux), then umount/diskutil - atomic progress write (tmp+fsync+rename); treat empty progress file as 0 - reuse a single PRNG in the perf probe so RNG init doesn't skew timings * test/benchmark/fuse_db: validate inputs, add subprocess timeout - mysql_bench.py: 1800s timeout on mysql CLI calls; reject db names that aren't plain identifiers (interpolated into SQL) - sqlite_gen.py / sqlite_verify.py: allowlist journal/synchronous modes and the verify mode so a typo can't silently weaken durability or relax checks - run_mysql.sh: durable atomic progress write (tmp+fsync+rename), matching sqlite_gen.py; quote $LB in both crash-test verify calls --- test/benchmark/fuse_db/README.md | 99 +++++++++ test/benchmark/fuse_db/bin/compare.sh | 44 ++++ test/benchmark/fuse_db/bin/fsbench.py | 52 +++++ test/benchmark/fuse_db/bin/lib.sh | 214 ++++++++++++++++++++ test/benchmark/fuse_db/bin/mysql_bench.py | 48 +++++ test/benchmark/fuse_db/bin/run_mysql.sh | 120 +++++++++++ test/benchmark/fuse_db/bin/run_sqlite.sh | 67 ++++++ test/benchmark/fuse_db/bin/sqlite_bench.py | 51 +++++ test/benchmark/fuse_db/bin/sqlite_gen.py | 62 ++++++ test/benchmark/fuse_db/bin/sqlite_verify.py | 70 +++++++ 10 files changed, 827 insertions(+) create mode 100644 test/benchmark/fuse_db/README.md create mode 100755 test/benchmark/fuse_db/bin/compare.sh create mode 100644 test/benchmark/fuse_db/bin/fsbench.py create mode 100755 test/benchmark/fuse_db/bin/lib.sh create mode 100644 test/benchmark/fuse_db/bin/mysql_bench.py create mode 100755 test/benchmark/fuse_db/bin/run_mysql.sh create mode 100755 test/benchmark/fuse_db/bin/run_sqlite.sh create mode 100644 test/benchmark/fuse_db/bin/sqlite_bench.py create mode 100644 test/benchmark/fuse_db/bin/sqlite_gen.py create mode 100644 test/benchmark/fuse_db/bin/sqlite_verify.py diff --git a/test/benchmark/fuse_db/README.md b/test/benchmark/fuse_db/README.md new file mode 100644 index 000000000..264a035cc --- /dev/null +++ b/test/benchmark/fuse_db/README.md @@ -0,0 +1,99 @@ +# FUSE database load / durability / performance benchmark + +Runs **MySQL (InnoDB)** and **SQLite** with their data files on a SeaweedFS **FUSE mount**, +with ~1 GB datasets, and answers two questions: + +1. **Durability** — does any committed data get lost across normal shutdown, an unexpected + `kill -9`, or a crash during active writes? +2. **Performance** — how much slower is the FUSE mount than the local disk for the same + workload (bulk load, OLTP commits, scans, raw fsync)? + +Each row carries a CRC32 of an incompressible (so ~1 GB actually hits the volumes) +deterministic payload; verification checks `integrity_check`/`CHECK TABLE`, exact row count, +contiguous-id prefix, and a per-row CRC recompute. + +## Results (single node, macOS arm64 + macFUSE, weed 4.34, local NVMe, 2026-06-15) + +### Durability — 6/6 PASS, no committed data lost, no corruption +| scenario | what | SQLite | MySQL | +|---|---|---|---| +| A normal shutdown | graceful DB stop, unmount, cluster stop, restart, verify | PASS | PASS | +| B unexpected (`kill -9`) | kill -9 db+mount+cluster, restart, recover, verify | PASS | PASS | +| C crash during writes | kill -9 mid-load, restart, recover, verify committed prefix | PASS | PASS | + +Scenario C confirmed correct recovery: the in-flight transaction rolled back, every +fsync-committed row survived as a contiguous prefix (InnoDB redo recovery ~9 s). + +### Performance — FUSE vs host (same disk, same durability settings) +Raw filesystem: +| metric | host | FUSE | FUSE slower | +|---|---|---|---| +| seq write +fsync | 2336 MB/s | 369 MB/s | 6.3x | +| seq read (warm) | 3435 MB/s | 1422 MB/s | 2.4x | +| fsync latency | 0.13 ms | 1.18 ms | 9x | + +SQLite (1 GB, journal=DELETE, synchronous=FULL): +| metric | host | FUSE | FUSE slower | +|---|---|---|---| +| bulk load | 5.8 s (177 MB/s) | 11.6 s (88 MB/s) | 2.0x | +| OLTP 1-row txns | 1987 tx/s | 171 tx/s | 11.6x | +| full scan (warm) | 316 MB/s | 427 MB/s | ~equal (python-bound) | + +MySQL/InnoDB (1 GB, trx_commit=1, flush_method=fsync): +| metric | host | FUSE | FUSE slower | +|---|---|---|---| +| bulk load | 24.4 s (42 MB/s) | 32.9 s (31 MB/s) | 1.35x | +| OLTP 1-row commits | 10542 tx/s | 1144 tx/s | 9.2x | +| full scan | 2419 MB/s | 430 MB/s | 5.6x | + +**Takeaway:** the cost is dominated by fsync/commit latency (0.13 → 1.18 ms, ~9x) because +every durable commit uploads the chunk to the volume + persists the filer chunk-manifest over +loopback. Small fsync'd transactions are ~9-12x slower; bulk loads (commits amortized) are +1.3-2x slower; warm/sequential reads are close to host. These are single-node loopback +numbers — a real cluster (remote volumes, replication) adds network RTT + replica fsync per +commit. + +## Durability caveat (test scope) +`kill -9` terminates the **processes** while the OS keeps running, so everything that reached +the OS page cache (all fsync'd data) survives — this faithfully tests process/daemon crashes +and is what passed 6/6. It does **not** simulate true **power loss** (page cache lost). The +FUSE mount uploads chunks **without** `fsync=true` (`weed/mount/weedfs_write.go` builds +`UploadOption` with no `Fsync`), so the volume does not fsync `.dat` per upload and the filer +leveldb likely does not sync per write; under a real power cut, recently-"committed" data that +only reached the page cache could be lost. A VM hard-reset / power-loss test on real hardware +is the follow-up to close that gap. + +## Requirements +- `weed` on `$PATH` (or set `WEED=/path/to/weed`) +- macFUSE (macOS) or libfuse (Linux) +- `python3`, `sqlite3` +- MySQL/MariaDB install; set `MYSQL_BASE` to its prefix (default macOS Homebrew + `/opt/homebrew/opt/mysql`; must contain `bin/mysqld`, `bin/mysql`, `bin/mysqladmin`). + For `mysql_bench.py` on a non-default install, also set `MYSQL_BIN=/path/to/mysql`. + +## Run +Runtime artifacts (cluster, mount, logs) go to `$SEAWEED_BENCH_WORK` +(default `/tmp/seaweedfs_fuse_db_bench`), kept out of the repo. + + cd test/benchmark/fuse_db + bash bin/run_sqlite.sh > results/sqlite.log 2>&1 # SQLite durability suite + bash bin/run_mysql.sh > results/mysql.log 2>&1 # MySQL durability suite + bash bin/compare.sh > results/compare.log 2>&1 # FUSE vs host performance + +The harness manages only its own processes (via pidfiles) on non-default ports +(9555/9560/9565, mysqld 3308); it never runs `pkill weed`, so other SeaweedFS instances on +the box are untouched. + +## Files + bin/lib.sh cluster/mount/mysqld lifecycle helpers (start/stop/kill, clean state) + bin/run_sqlite.sh SQLite 1GB load + A/B/C crash scenarios + bin/run_mysql.sh MySQL 1GB load + A/B/C crash scenarios + bin/compare.sh FUSE-vs-host driver + bin/sqlite_gen.py deterministic incompressible SQLite loader (progress/ref files) + bin/sqlite_verify.py SQLite integrity + count + prefix + per-row CRC verifier + bin/sqlite_bench.py SQLite load/OLTP/scan timing probe + bin/mysql_bench.py MySQL load/OLTP/scan timing probe (via mysql CLI) + bin/fsbench.py raw seq-write / fsync-latency / seq-read microbenchmark + +The numbers above are the reference baseline; rerun the scripts to reproduce. Run output is +written to results/*.log, which is gitignored (repo-wide *.log rule) -- not checked in. diff --git a/test/benchmark/fuse_db/bin/compare.sh b/test/benchmark/fuse_db/bin/compare.sh new file mode 100755 index 000000000..35a965f7c --- /dev/null +++ b/test/benchmark/fuse_db/bin/compare.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# Identical SQLite + MySQL + raw-fs workloads on HOST (local APFS) vs the SeaweedFS +# FUSE mount. Same disk underneath, same DB durability settings -> the delta is the +# SeaweedFS+FUSE stack. +BIN="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$BIN/lib.sh" +HOSTDIR=$WORK/hostdir # local disk, NOT under the FUSE mount +ROWS=262144; SBATCH=4096; MBATCH=8192; OLTP=2000 + +mysqld_init_at() { rm -rf "$1"; mkdir -p "$1"; "$MYSQLD" --initialize-insecure --datadir="$1" \ + --basedir="$MYSQL_BASE" --log-error="$LOGS/mysql_init_cmp.log"; } +mysqld_run() { + "$MYSQLD" --datadir="$1" --basedir="$MYSQL_BASE" --socket="$MYSQL_SOCK" --port=$MYSQL_PORT \ + --bind-address=127.0.0.1 --innodb_flush_log_at_trx_commit=1 --innodb_flush_method=fsync \ + --innodb_doublewrite=ON --innodb_buffer_pool_size=256M --secure-file-priv="" \ + --log-error="$LOGS/mysqld_cmp.log" >> "$LOGS/mysqld_cmp.out" 2>&1 & + echo $! > "$PIDS/mysql.pid"; mysql_wait; } + +echo "###### FUSE vs HOST $(date) ######" +echo "ROWS=$ROWS payload=4096B incompressible SQLite(journal=DELETE,sync=FULL) InnoDB(trx_commit=1,flush=fsync)" + +# ===================== HOST (local APFS, no SeaweedFS) ===================== +clean_state >/dev/null 2>&1 +rm -rf "$HOSTDIR"; mkdir -p "$HOSTDIR/sqlite" "$HOSTDIR/fsb" +echo; echo "===== HOST: raw fs ====="; python3 "$BIN/fsbench.py" "$HOSTDIR/fsb" +echo; echo "===== HOST: SQLite ====="; python3 "$BIN/sqlite_bench.py" "$HOSTDIR/sqlite/b.db" $ROWS $SBATCH $OLTP +echo; echo "===== HOST: MySQL =====" +mysqld_init_at "$HOSTDIR/mysql/data"; mysqld_run "$HOSTDIR/mysql/data" +python3 "$BIN/mysql_bench.py" "$MYSQL_SOCK" cmp $ROWS $MBATCH $OLTP +mysql_stop_graceful + +# ===================== FUSE (SeaweedFS mount) ===================== +cluster_start || exit 1; mount_start || exit 1 +mkdir -p "$MNT/sqlite" "$MNT/fsb" +echo; echo "===== FUSE: raw fs ====="; python3 "$BIN/fsbench.py" "$MNT/fsb" +echo; echo "===== FUSE: SQLite ====="; python3 "$BIN/sqlite_bench.py" "$MNT/sqlite/b.db" $ROWS $SBATCH $OLTP +echo; echo "===== FUSE: MySQL =====" +mysqld_init_at "$MNT/mysql/data"; mysqld_run "$MNT/mysql/data" +python3 "$BIN/mysql_bench.py" "$MYSQL_SOCK" cmp $ROWS $MBATCH $OLTP +mysql_stop_graceful + +stop_all +rm -rf "$HOSTDIR" +echo; echo "###### compare done ######" diff --git a/test/benchmark/fuse_db/bin/fsbench.py b/test/benchmark/fuse_db/bin/fsbench.py new file mode 100644 index 000000000..78ba2e808 --- /dev/null +++ b/test/benchmark/fuse_db/bin/fsbench.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +# Raw filesystem microbenchmark in one directory. Prints TSV: METRICvalueunit +# SEQWRITE MB/s (512MB stream + final fsync) +# FSYNC ops/s (500 x: pwrite 4KB at random offset in a 64MB file, then fsync) +# SEQREAD MB/s (read the 512MB file back; warm cache) +import sys, os, time, random + +d = sys.argv[1] +os.makedirs(d, exist_ok=True) +big = os.path.join(d, "fsb_big.bin") +sm = os.path.join(d, "fsb_sync.bin") + +# SEQWRITE 512MB + fsync +n = 512 * 1024 * 1024 +buf = os.urandom(8 * 1024 * 1024) +t = time.time(); f = open(big, "wb"); w = 0 +while w < n: + f.write(buf); w += len(buf) +f.flush(); os.fsync(f.fileno()); f.close() +dt = time.time() - t +print(f"SEQWRITE\t{w/1048576/dt:.0f}\tMB/s") + +# FSYNC latency: 500 x (pwrite 4KB random offset + fsync) on a 64MB file +fd = os.open(sm, os.O_RDWR | os.O_CREAT, 0o644) +os.ftruncate(fd, 64 * 1024 * 1024) +os.fsync(fd) +page = os.urandom(4096) +N = 500 +rnd = random.Random(12345) +t = time.time() +for _ in range(N): + off = rnd.randrange(0, 64 * 1024 * 1024 - 4096) & ~0xfff + os.pwrite(fd, page, off) + os.fsync(fd) +dt = time.time() - t +os.close(fd) +print(f"FSYNC\t{N/dt:.0f}\tops/s") +print(f"FSYNCLAT\t{dt/N*1000:.2f}\tms") + +# SEQREAD (warm) +t = time.time(); tot = 0 +with open(big, "rb") as f: + while True: + b = f.read(8 * 1024 * 1024) + if not b: break + tot += len(b) +dt = time.time() - t +print(f"SEQREAD\t{tot/1048576/dt:.0f}\tMB/s") + +for p in (big, sm): + try: os.remove(p) + except OSError: pass diff --git a/test/benchmark/fuse_db/bin/lib.sh b/test/benchmark/fuse_db/bin/lib.sh new file mode 100755 index 000000000..5f8ff1672 --- /dev/null +++ b/test/benchmark/fuse_db/bin/lib.sh @@ -0,0 +1,214 @@ +# Shared config + helpers for the SeaweedFS FUSE load/durability test. +# IMPORTANT: this harness manages ONLY its own processes via pidfiles and its own +# (non-default) ports. It never runs `pkill weed`, so any other SeaweedFS instances +# on the box (e.g. default ports 9333/8888) are safe. +# +# Config via env: +# SEAWEED_BENCH_WORK runtime dir for cluster/mount/logs (default /tmp/...; kept out of the repo) +# WEED path to the weed binary (default: from $PATH) +# MYSQL_BASE MySQL install prefix (default macOS Homebrew; on Linux e.g. /usr) + +set -u + +WORK="${SEAWEED_BENCH_WORK:-/tmp/seaweedfs_fuse_db_bench}" +WEED="${WEED:-$(command -v weed || echo weed)}" +RUN=$WORK +CLUSTER=$RUN/cluster # weed server -dir (master+volume+filer state) -- local disk +MNT=$RUN/mnt # FUSE mount point (the SeaweedFS filesystem under test) +LOGS=$RUN/logs +LOCAL=$RUN/local # non-FUSE scratch: pidfiles, mysql socket, references +PIDS=$LOCAL/pids +RESULTS=$RUN/results + +# Isolated, non-default ports (avoid colliding with any running cluster) +MASTER_PORT=9555 +VOLUME_PORT=9560 +FILER_PORT=9565 +FILER_GRPC=19565 + +# MySQL (macOS Homebrew default; override MYSQL_BASE on Linux, e.g. /usr) +MYSQL_BASE="${MYSQL_BASE:-/opt/homebrew/opt/mysql}" +MYSQLD=$MYSQL_BASE/bin/mysqld +MYSQL=$MYSQL_BASE/bin/mysql +export MYSQL_BIN="${MYSQL_BIN:-$MYSQL}" # propagated to mysql_bench.py +MYSQLADMIN=$MYSQL_BASE/bin/mysqladmin +MYSQL_PORT=3308 +MYSQL_SOCK=$LOCAL/mysql.sock # socket on LOCAL disk, datadir on FUSE +MYSQL_DATADIR=$MNT/mysql/data # on the FUSE mount -- the thing under test + +ts() { date +%H:%M:%S; } +say() { echo "[$(ts)] $*"; } + +ensure_dirs() { mkdir -p "$CLUSTER" "$MNT" "$LOGS" "$LOCAL" "$PIDS" "$RESULTS" "$LOCAL/mountcache"; } + +# Portable unmount: fusermount (Linux non-root) -> umount -> diskutil (macOS). +fuse_umount() { fusermount -u "$1" 2>/dev/null || fusermount3 -u "$1" 2>/dev/null || umount "$1" 2>/dev/null || diskutil unmount "$1" 2>/dev/null; } +fuse_umount_force() { fusermount -uz "$1" 2>/dev/null || fusermount3 -uz "$1" 2>/dev/null || umount -f "$1" 2>/dev/null || diskutil unmount force "$1" 2>/dev/null; } + +# My (and only my) ports. Used for the kill-by-port safety net in clean_state. +MY_PORTS="$MASTER_PORT $VOLUME_PORT $FILER_PORT 19555 19560 $FILER_GRPC $MYSQL_PORT" +port_busy() { lsof -ti TCP:"$1" >/dev/null 2>&1; } +wait_port_free() { # $1=port $2=max-halfsec + local n=${2:-40}; for i in $(seq 1 "$n"); do port_busy "$1" || return 0; sleep 0.5; done; return 1; } + +# ---- cluster (master+volume+filer in one process) ---- +cluster_start() { + ensure_dirs + if ! wait_port_free "$MASTER_PORT" 30; then + say "ERROR master port $MASTER_PORT still busy (pid $(lsof -ti TCP:$MASTER_PORT|tr '\n' ' ')); not starting"; return 1 + fi + say "starting weed server (master:$MASTER_PORT volume:$VOLUME_PORT filer:$FILER_PORT)" + "$WEED" server -ip=127.0.0.1 -dir="$CLUSTER" \ + -master.port=$MASTER_PORT -volume.port=$VOLUME_PORT \ + -filer -filer.port=$FILER_PORT \ + -volume.max=50 -master.volumeSizeLimitMB=1024 \ + >> "$LOGS/cluster.log" 2>&1 & + echo $! > "$PIDS/cluster.pid" + cluster_wait +} +cluster_wait() { + local pid; pid=$(cat "$PIDS/cluster.pid" 2>/dev/null) + for i in $(seq 1 120); do + # the process we launched must still be alive -- guards against a foreign + # process answering on the port after ours fataled (e.g. bind conflict) + if ! kill -0 "$pid" 2>/dev/null; then + say "ERROR cluster process $pid died during startup"; tail -15 "$LOGS/cluster.log"; return 1 + fi + if curl -s "http://127.0.0.1:$MASTER_PORT/cluster/status" >/dev/null 2>&1 \ + && curl -s "http://127.0.0.1:$FILER_PORT/" >/dev/null 2>&1; then + say "cluster ready (pid $pid)"; return 0 + fi + sleep 0.5 + done + say "ERROR cluster not ready"; tail -20 "$LOGS/cluster.log"; return 1 +} +cluster_stop_graceful() { + local pid; pid=$(cat "$PIDS/cluster.pid" 2>/dev/null) || return 0 + [ -n "${pid:-}" ] || return 0 + say "graceful stop cluster pid $pid (SIGTERM)" + kill -TERM "$pid" 2>/dev/null + for i in $(seq 1 60); do kill -0 "$pid" 2>/dev/null || { rm -f "$PIDS/cluster.pid"; return 0; }; sleep 0.5; done + say "cluster still alive after 30s; SIGKILL"; kill -9 "$pid" 2>/dev/null; rm -f "$PIDS/cluster.pid" +} +cluster_kill_hard() { + local pid; pid=$(cat "$PIDS/cluster.pid" 2>/dev/null) || return 0 + [ -n "${pid:-}" ] && { say "HARD kill -9 cluster pid $pid"; kill -9 "$pid" 2>/dev/null; } + rm -f "$PIDS/cluster.pid" +} + +# ---- FUSE mount ---- +mount_start() { + ensure_dirs + say "mounting FUSE at $MNT (filer 127.0.0.1:$FILER_PORT)" + "$WEED" mount -filer=127.0.0.1:$FILER_PORT -dir="$MNT" \ + -dirAutoCreate -cacheDir="$LOCAL/mountcache" \ + >> "$LOGS/mount.log" 2>&1 & + echo $! > "$PIDS/mount.pid" + mount_wait +} +mount_wait() { + local pid; pid=$(cat "$PIDS/mount.pid" 2>/dev/null) + for i in $(seq 1 120); do + if ! kill -0 "$pid" 2>/dev/null; then + say "ERROR mount process $pid died during startup"; tail -15 "$LOGS/mount.log"; return 1 + fi + if mount | grep -q "$MNT" && ls "$MNT" >/dev/null 2>&1; then say "mount ready (pid $pid)"; return 0; fi + sleep 0.5 + done + say "ERROR mount not ready"; tail -20 "$LOGS/mount.log"; return 1 +} +mount_stop_graceful() { + say "graceful unmount $MNT" + fuse_umount "$MNT" + local pid; pid=$(cat "$PIDS/mount.pid" 2>/dev/null) + if [ -n "${pid:-}" ]; then + for i in $(seq 1 40); do kill -0 "$pid" 2>/dev/null || break; sleep 0.5; done + kill -0 "$pid" 2>/dev/null && { kill -TERM "$pid" 2>/dev/null; sleep 1; } + fi + rm -f "$PIDS/mount.pid" +} +mount_kill_hard() { + local pid; pid=$(cat "$PIDS/mount.pid" 2>/dev/null) + [ -n "${pid:-}" ] && { say "HARD kill -9 mount pid $pid"; kill -9 "$pid" 2>/dev/null; } + rm -f "$PIDS/mount.pid" + # clean up the stale mountpoint + fuse_umount_force "$MNT" +} + +# ---- MySQL ---- +mysql_init() { + ensure_dirs + rm -rf "$MNT/mysql"; mkdir -p "$MNT/mysql" + say "initializing MySQL datadir on FUSE: $MYSQL_DATADIR" + "$MYSQLD" --initialize-insecure --datadir="$MYSQL_DATADIR" --basedir="$MYSQL_BASE" \ + --log-error="$LOGS/mysql-init.log" +} +mysql_start() { + say "starting mysqld (port $MYSQL_PORT, datadir on FUSE, flush_log_at_trx_commit=1, flush_method=fsync)" + "$MYSQLD" --datadir="$MYSQL_DATADIR" --basedir="$MYSQL_BASE" \ + --socket="$MYSQL_SOCK" --port=$MYSQL_PORT --bind-address=127.0.0.1 \ + --innodb_flush_log_at_trx_commit=1 --innodb_flush_method=fsync \ + --innodb_doublewrite=ON --innodb_buffer_pool_size=256M \ + --secure-file-priv="" --log-error="$LOGS/mysqld.log" \ + >> "$LOGS/mysqld.out" 2>&1 & + echo $! > "$PIDS/mysql.pid" + mysql_wait +} +mysql_wait() { + for i in $(seq 1 120); do + "$MYSQLADMIN" --socket="$MYSQL_SOCK" -uroot ping >/dev/null 2>&1 && { say "mysqld ready"; return 0; } + sleep 0.5 + done + say "ERROR mysqld not ready"; tail -30 "$LOGS/mysqld.log"; return 1 +} +mysql_stop_graceful() { + say "graceful mysqld shutdown" + "$MYSQLADMIN" --socket="$MYSQL_SOCK" -uroot shutdown 2>/dev/null + local pid; pid=$(cat "$PIDS/mysql.pid" 2>/dev/null) + if [ -n "${pid:-}" ]; then for i in $(seq 1 60); do kill -0 "$pid" 2>/dev/null || break; sleep 0.5; done; fi + rm -f "$PIDS/mysql.pid" +} +mysql_kill() { + local pid; pid=$(cat "$PIDS/mysql.pid" 2>/dev/null) + [ -n "${pid:-}" ] && { say "HARD kill -9 mysqld pid $pid"; kill -9 "$pid" 2>/dev/null; } + rm -f "$PIDS/mysql.pid" +} +mq() { "$MYSQL" --socket="$MYSQL_SOCK" -uroot "$@"; } + +# ---- whole-node unexpected shutdown (simultaneous -9) ---- +hard_kill_all() { + say "=== UNEXPECTED SHUTDOWN: kill -9 db + mount + cluster ===" + mysql_kill + mount_kill_hard + cluster_kill_hard + kill_my_ports # safety net in case any pidfile was stale +} + +# stop everything I started (graceful, pidfile-based) +stop_all() { mysql_stop_graceful; mount_stop_graceful; cluster_stop_graceful; } + +# safety net: kill anything still bound to MY (non-default) ports +kill_my_ports() { + for p in $MY_PORTS; do + local pids; pids=$(lsof -ti TCP:"$p" 2>/dev/null) + [ -n "$pids" ] && { say "killing leftover pid(s) $pids on my port $p"; kill -9 $pids 2>/dev/null; } + done +} + +# ---- clean slate (wipe all test state; never touches user data) ---- +clean_state() { + say "cleaning test state" + stop_all + fuse_umount_force "$MNT" + kill_my_ports + for p in $MASTER_PORT $VOLUME_PORT $FILER_PORT; do wait_port_free "$p" 40 || say "WARN port $p still busy"; done + rm -rf "$CLUSTER" "$MNT" "$LOCAL/mountcache" "$PIDS" 2>/dev/null + ensure_dirs +} + +status_report() { + echo "---- status ----" + echo "df $MNT:"; df -h "$MNT" 2>/dev/null | tail -1 + echo "cluster dir size:"; du -sh "$CLUSTER" 2>/dev/null + echo "volumes:"; curl -s "http://127.0.0.1:$MASTER_PORT/dir/status" 2>/dev/null | python3 -c 'import sys,json;d=json.load(sys.stdin);print(" topology free/max:",d.get("Topology",{}).get("Free"),d.get("Topology",{}).get("Max"))' 2>/dev/null +} diff --git a/test/benchmark/fuse_db/bin/mysql_bench.py b/test/benchmark/fuse_db/bin/mysql_bench.py new file mode 100644 index 000000000..32c16cec8 --- /dev/null +++ b/test/benchmark/fuse_db/bin/mysql_bench.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +# MySQL performance probe via the mysql CLI (no driver needed). Times each phase +# in Python. Prints TSV: METRICsecsrate +# LOAD : bulk-load rows (4096B incompressible AES payload), -row txns +# OLTP : single-row autocommit INSERTs (each fsyncs redo, trx_commit=1) -> tx/s +# SCAN : SUM(CRC32(payload)) over the table (reads every payload) -> MB/s +import os, re, sys, subprocess, time + +sock = sys.argv[1] +db = sys.argv[2] +rows = int(sys.argv[3]) +batch = int(sys.argv[4]) +oltp = int(sys.argv[5]) +MYSQL = os.environ.get("MYSQL_BIN", "mysql") # lib.sh exports MYSQL_BIN; else use PATH + +if not re.fullmatch(r"[A-Za-z0-9_]+", db): # db is interpolated into SQL + raise SystemExit(f"invalid database name: {db!r}") + +def run(sql): + r = subprocess.run([MYSQL, "--socket", sock, "-uroot"], input=sql.encode(), + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, timeout=1800) + if r.returncode != 0: + sys.stderr.write(r.stderr.decode()[:500]); raise SystemExit("mysql failed") + +run(f"DROP DATABASE IF EXISTS {db}; CREATE DATABASE {db};" + f"CREATE TABLE {db}.t(id INT PRIMARY KEY,payload BLOB,chk BIGINT) ENGINE=InnoDB;" + f"CREATE TABLE {db}.o(id INT AUTO_INCREMENT PRIMARY KEY,v INT) ENGINE=InnoDB;") + +mb = rows * 4096 / 1048576 +stmts = ["SET SESSION cte_max_recursion_depth=1000000;"] +o = 0 +while o < rows: + b = min(batch, rows - o) + stmts.append( + f"INSERT INTO {db}.t(id,payload,chk) " + f"WITH RECURSIVE seq(n) AS (SELECT 0 UNION ALL SELECT n+1 FROM seq WHERE n<{b-1}) " + f"SELECT n+{o},AES_ENCRYPT(REPEAT('a',4080),SHA2(n+{o},256))," + f"CRC32(AES_ENCRYPT(REPEAT('a',4080),SHA2(n+{o},256))) FROM seq;") + o += b +t = time.time(); run("\n".join(stmts)); dt = time.time() - t +print(f"LOAD\t{dt:.1f}\t{mb/dt:.0f} MB/s") + +sql = "\n".join(f"INSERT INTO {db}.o(v) VALUES({i});" for i in range(oltp)) +t = time.time(); run(sql); dt = time.time() - t +print(f"OLTP\t{dt:.2f}\t{oltp/dt:.0f} tx/s") + +t = time.time(); run(f"SELECT SUM(CRC32(payload)) FROM {db}.t;"); dt = time.time() - t +print(f"SCAN\t{dt:.1f}\t{mb/dt:.0f} MB/s") diff --git a/test/benchmark/fuse_db/bin/run_mysql.sh b/test/benchmark/fuse_db/bin/run_mysql.sh new file mode 100755 index 000000000..5ab4540cd --- /dev/null +++ b/test/benchmark/fuse_db/bin/run_mysql.sh @@ -0,0 +1,120 @@ +#!/bin/bash +# MySQL (InnoDB) ~1GB load test on the SeaweedFS FUSE mount + crash suite. +# Datadir lives on the FUSE mount; innodb_flush_log_at_trx_commit=1 + +# innodb_flush_method=fsync => every commit fsyncs the redo log through FUSE. +BIN="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$BIN/lib.sh" + +TOTAL=262144 # 262144 * 4096B payload = ~1GiB row data (InnoDB on-disk larger) +BATCH=8192 # 32 committed transactions +PROG=$LOCAL/mysql_progress.txt +PASS=0; FAIL=0 +chk() { if [ "$1" -eq 0 ]; then echo ">>> $2: PASS"; PASS=$((PASS+1)); else echo ">>> $2: FAIL"; FAIL=$((FAIL+1)); fi; } + +mysql_schema() { + mq -e "CREATE DATABASE IF NOT EXISTS loadtest; + DROP TABLE IF EXISTS loadtest.t; + CREATE TABLE loadtest.t(id INT PRIMARY KEY, payload BLOB, chk BIGINT) ENGINE=InnoDB;" +} +mysql_load() { # loads TOTAL rows in BATCH-sized committed inserts; progress -> $PROG + local O=0 B rem + : > "$PROG" + while [ $O -lt $TOTAL ]; do + B=$BATCH; rem=$((TOTAL-O)); [ $rem -lt $B ] && B=$rem + # incompressible + deterministic payload: AES-128-ECB (no IV -> deterministic) + # of a fixed 4080B plaintext, keyed per-id -> 4096B ciphertext that defeats + # gzip on the volumes. Identical expr for payload and chk, so CRC32(payload) + # recomputed at verify must equal stored chk. + mq -e "SET SESSION cte_max_recursion_depth=1000000; + INSERT INTO loadtest.t(id,payload,chk) + WITH RECURSIVE seq(n) AS (SELECT 0 UNION ALL SELECT n+1 FROM seq WHERE n < $((B-1))) + SELECT n+$O, + AES_ENCRYPT(REPEAT('a',4080), SHA2(n+$O,256)), + CRC32(AES_ENCRYPT(REPEAT('a',4080), SHA2(n+$O,256))) + FROM seq;" \ + || { echo "INSERT failed at offset $O"; return 1; } + # durable + atomic progress (matches sqlite_gen.py) so the crash-test lower + # bound is never stale or torn: write tmp, fsync, rename. + O=$((O+B)); printf '%s' "$O" > "$PROG.tmp" + python3 -c "import os,sys; fd=os.open(sys.argv[1],os.O_RDONLY); os.fsync(fd); os.close(fd)" "$PROG.tmp" + mv -f "$PROG.tmp" "$PROG" + done +} +mysql_verify() { # $1=expected $2=mode(exact|atleast) ; returns 0 on PASS + local exp=$1 mode=$2 cnt sumchk corrupt minid maxid checktbl ok=0 + read -r cnt sumchk corrupt minid maxid <CRC32(payload)),0),COALESCE(MIN(id),-1),COALESCE(MAX(id),-1) FROM loadtest.t;") +EOF + checktbl=$(mq -N -e "CHECK TABLE loadtest.t;" | awk '{print $NF}') + echo " rows=$cnt corrupt=$corrupt minid=$minid maxid=$maxid CHECK=$checktbl (want $mode $exp)" + [ "$checktbl" = "OK" ] || { echo " -> CHECK TABLE failed"; ok=1; } + [ "$corrupt" = "0" ] || { echo " -> payload corruption in $corrupt rows"; ok=1; } + if [ "$mode" = exact ]; then + [ "$cnt" = "$exp" ] || { echo " -> row count mismatch"; ok=1; } + { [ "$minid" = 0 ] && [ "$maxid" = "$((exp-1))" ]; } || { echo " -> not contiguous 0..$((exp-1))"; ok=1; } + else + [ "$cnt" -ge "$exp" ] 2>/dev/null || { echo " -> LOST COMMITTED DATA (cnt<$exp)"; ok=1; } + { [ "$minid" = 0 ] && [ "$maxid" = "$((cnt-1))" ]; } || { echo " -> not contiguous prefix"; ok=1; } + fi + return $ok +} + +echo "######## MySQL FUSE load test $(date) ########" + +# ---------- fresh cluster + mysql + 1GB load ---------- +clean_state; cluster_start || exit 1; mount_start || exit 1 +mysql_init || { echo "mysql_init failed"; tail -20 "$LOGS/mysql-init.log"; exit 1; } +mysql_start || exit 1 +mysql_schema +say "loading ~1GB into InnoDB" +t0=$SECONDS +mysql_load || echo "load reported an error" +say "load done in $((SECONDS-t0))s" +mq -N -e "SELECT CONCAT('innodb rows=',COUNT(*),' data_len=',ROUND(SUM(LENGTH(payload))/1048576),'MB') FROM loadtest.t;" +du -sh "$MYSQL_DATADIR"; status_report + +# ---------- Scenario A: normal (graceful) shutdown + restart ---------- +echo; echo "==================== SCENARIO A: NORMAL SHUTDOWN ====================" +mysql_stop_graceful +mount_stop_graceful +cluster_stop_graceful +sleep 1 +cluster_start || exit 1; mount_start || exit 1; mysql_start || exit 1 +mysql_verify $TOTAL exact; chk $? "A/normal-shutdown" + +# ---------- Scenario B: unexpected shutdown (kill -9 everything) ---------- +echo; echo "============== SCENARIO B: UNEXPECTED SHUTDOWN (kill -9) ==============" +hard_kill_all +sleep 1 +cluster_start || exit 1; mount_start || exit 1 +say "restarting mysqld -> InnoDB crash recovery runs" +mysql_start || { echo "mysqld failed to recover"; tail -30 "$LOGS/mysqld.log"; chk 1 "B/unexpected-shutdown"; } +if mq -e "SELECT 1" >/dev/null 2>&1; then mysql_verify $TOTAL exact; chk $? "B/unexpected-shutdown"; fi + +# ---------- Scenario C: crash DURING active writes ---------- +echo; echo "=========== SCENARIO C: CRASH DURING ACTIVE WRITES (kill -9) ===========" +mysql_stop_graceful +clean_state; cluster_start || exit 1; mount_start || exit 1 +mysql_init || exit 1; mysql_start || exit 1; mysql_schema +say "loading in background; will crash storage + mysqld mid-write" +mysql_load > "$LOGS/mysql_load_c.log" 2>&1 & +LPID=$! +while :; do + p=$(cat "$PROG" 2>/dev/null || echo 0); p=${p:-0} + [ "$p" -ge $((TOTAL/2)) ] && break + kill -0 $LPID 2>/dev/null || { say "loader exited early"; break; } + sleep 0.2 +done +LB=$(cat "$PROG" 2>/dev/null); LB=${LB:-0} # empty file (killed mid-write) -> 0 +say "durable committed lower-bound at crash = $LB rows; crashing now" +hard_kill_all +kill -9 $LPID 2>/dev/null; wait $LPID 2>/dev/null +sleep 1 +cluster_start || exit 1; mount_start || exit 1 +say "restarting mysqld -> InnoDB recovery; in-flight txn must roll back, committed must survive" +mysql_start || { echo "recovery failed"; tail -30 "$LOGS/mysqld.log"; chk 1 "C/crash-during-write"; } +if mq -e "SELECT 1" >/dev/null 2>&1; then mysql_verify "$LB" atleast; chk $? "C/crash-during-write"; fi + +echo; echo "######## MySQL RESULT: PASS=$PASS FAIL=$FAIL ########" +mysql_stop_graceful; mount_stop_graceful; cluster_stop_graceful +exit $FAIL diff --git a/test/benchmark/fuse_db/bin/run_sqlite.sh b/test/benchmark/fuse_db/bin/run_sqlite.sh new file mode 100755 index 000000000..5d32a7ccc --- /dev/null +++ b/test/benchmark/fuse_db/bin/run_sqlite.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# SQLite ~1GB load test on the SeaweedFS FUSE mount + crash suite. +BIN="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$BIN/lib.sh" + +TOTAL=262144 # 262144 * 4096B payload = ~1GiB of row data +BATCH=4096 # 64 committed transactions (each fsync'd, synchronous=FULL) +DB=$MNT/sqlite/load.db +PROG=$LOCAL/sqlite_progress.txt +REF=$LOCAL/sqlite_ref.txt +PASS=0; FAIL=0 +chk() { if [ "$1" -eq 0 ]; then echo ">>> $2: PASS"; PASS=$((PASS+1)); else echo ">>> $2: FAIL"; FAIL=$((FAIL+1)); fi; } + +echo "######## SQLite FUSE load test $(date) ########" + +# ---------- fresh cluster + 1GB load ---------- +clean_state; cluster_start || exit 1; mount_start || exit 1 +mkdir -p "$MNT/sqlite" +say "loading ~1GB into SQLite (journal=DELETE synchronous=FULL)" +t0=$SECONDS +python3 "$BIN/sqlite_gen.py" "$DB" $TOTAL $BATCH "$PROG" "$REF" FULL DELETE | tail -3 +say "load done in $((SECONDS-t0))s; file size:"; ls -lh "$DB"; du -sh "$MNT/sqlite" +status_report + +# ---------- Scenario A: normal (graceful) shutdown + restart ---------- +echo; echo "==================== SCENARIO A: NORMAL SHUTDOWN ====================" +mount_stop_graceful +cluster_stop_graceful +sleep 1 +cluster_start || exit 1; mount_start || exit 1 +say "verifying after graceful restart" +python3 "$BIN/sqlite_verify.py" "$DB" $TOTAL exact; chk $? "A/normal-shutdown" + +# ---------- Scenario B: unexpected shutdown (kill -9) + restart ---------- +echo; echo "============== SCENARIO B: UNEXPECTED SHUTDOWN (kill -9) ==============" +hard_kill_all +sleep 1 +cluster_start || exit 1; mount_start || exit 1 +say "verifying after kill -9 + restart" +python3 "$BIN/sqlite_verify.py" "$DB" $TOTAL exact; chk $? "B/unexpected-shutdown" + +# ---------- Scenario C: crash DURING active writes ---------- +echo; echo "=========== SCENARIO C: CRASH DURING ACTIVE WRITES (kill -9) ===========" +clean_state; cluster_start || exit 1; mount_start || exit 1 +mkdir -p "$MNT/sqlite"; : > "$PROG" +say "starting loader in background; will crash storage mid-write" +python3 "$BIN/sqlite_gen.py" "$DB" $TOTAL $BATCH "$PROG" "" FULL DELETE > "$LOGS/sqlite_gen_c.log" 2>&1 & +GENPID=$! +# wait until at least half is durably committed +while :; do + p=$(cat "$PROG" 2>/dev/null || echo 0); p=${p:-0} + [ "$p" -ge $((TOTAL/2)) ] && break + kill -0 $GENPID 2>/dev/null || { say "loader exited early"; break; } + sleep 0.2 +done +LB=$(cat "$PROG" 2>/dev/null); LB=${LB:-0} # empty file (killed mid-write) -> 0 +say "durable committed lower-bound at crash = $LB rows; crashing storage now" +hard_kill_all # storage dies under the still-writing loader +kill -9 $GENPID 2>/dev/null; wait $GENPID 2>/dev/null +sleep 1 +cluster_start || exit 1; mount_start || exit 1 +say "verifying recovery (committed prefix must survive, integrity must hold)" +python3 "$BIN/sqlite_verify.py" "$DB" "$LB" atleast; chk $? "C/crash-during-write" + +echo; echo "######## SQLite RESULT: PASS=$PASS FAIL=$FAIL ########" +mount_stop_graceful; cluster_stop_graceful +exit $FAIL diff --git a/test/benchmark/fuse_db/bin/sqlite_bench.py b/test/benchmark/fuse_db/bin/sqlite_bench.py new file mode 100644 index 000000000..bf5ae51db --- /dev/null +++ b/test/benchmark/fuse_db/bin/sqlite_bench.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +# SQLite performance probe at one db path. Prints TSV: METRICsecsrate +# LOAD : bulk-load rows (4096B incompressible payload) in -row txns +# OLTP : single-row INSERTs, each its own fsync'd autocommit txn -> tx/s +# SCAN : full table scan reading every payload (warm) -> MB/s +# journal=DELETE, synchronous=FULL (durable: fsync per commit). +import sys, os, sqlite3, time, random + +db = sys.argv[1] +rows = int(sys.argv[2]) +batch = int(sys.argv[3]) +oltp = int(sys.argv[4]) + +for ext in ("", "-journal", "-wal", "-shm"): + try: os.remove(db + ext) + except OSError: pass + +con = sqlite3.connect(db, isolation_level=None) +con.execute("PRAGMA journal_mode=DELETE;") +con.execute("PRAGMA synchronous=FULL;") +con.execute("CREATE TABLE t(id INTEGER PRIMARY KEY, payload BLOB, chk INTEGER);") + +# LOAD +mb = rows * 4096 / 1048576 +t = time.time(); done = 0 +rnd = random.Random(12345) # reuse one PRNG; this probe doesn't verify payloads +while done < rows: + end = min(done + batch, rows) + data = [(i, rnd.randbytes(4096), 0) for i in range(done, end)] + con.execute("BEGIN") + con.executemany("INSERT INTO t VALUES(?,?,?)", data) + con.execute("COMMIT") + done = end +dt = time.time() - t +print(f"LOAD\t{dt:.1f}\t{mb/dt:.0f} MB/s") + +# OLTP: each INSERT autocommits (isolation_level=None) -> 1 fsync'd txn each +con.execute("CREATE TABLE o(id INTEGER PRIMARY KEY, v INTEGER);") +t = time.time() +for i in range(oltp): + con.execute("INSERT INTO o(v) VALUES(?)", (i,)) +dt = time.time() - t +print(f"OLTP\t{dt:.2f}\t{oltp/dt:.0f} tx/s") + +# SCAN (warm): read every payload +t = time.time(); nbytes = 0 +for rid, p, c in con.execute("SELECT id,payload,chk FROM t"): + nbytes += len(p) +dt = time.time() - t +print(f"SCAN\t{dt:.1f}\t{nbytes/1048576/dt:.0f} MB/s") +con.close() diff --git a/test/benchmark/fuse_db/bin/sqlite_gen.py b/test/benchmark/fuse_db/bin/sqlite_gen.py new file mode 100644 index 000000000..9eb9b9d9e --- /dev/null +++ b/test/benchmark/fuse_db/bin/sqlite_gen.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +# Deterministic ~1GB SQLite load on the FUSE mount. +# Each row: id, payload (4096 deterministic bytes derived from id), chk=crc32(payload). +# Commits in batches; after each commit, writes the committed row count to a +# progress file on LOCAL disk and fsyncs it -> a durable record of what was +# committed, used to verify "no committed data lost" after a crash. +import sys, os, sqlite3, zlib, random + +db_path = sys.argv[1] +total_rows = int(sys.argv[2]) +batch = int(sys.argv[3]) +progress_path= sys.argv[4] # LOCAL disk +ref_path = sys.argv[5] if len(sys.argv) > 5 else None +sync_mode = sys.argv[6] if len(sys.argv) > 6 else "FULL" +journal_mode = sys.argv[7] if len(sys.argv) > 7 else "DELETE" + +# Validate before interpolating into PRAGMAs: a typo must not silently weaken +# durability and invalidate the comparison / crash-survival claims. +if sync_mode.upper() not in {"OFF", "NORMAL", "FULL", "EXTRA"}: + raise SystemExit(f"invalid synchronous mode: {sync_mode!r}") +if journal_mode.upper() not in {"DELETE", "TRUNCATE", "PERSIST", "MEMORY", "WAL", "OFF"}: + raise SystemExit(f"invalid journal mode: {journal_mode!r}") + +def payload(i: int) -> bytes: + # deterministic but incompressible (PRNG seeded by id) -> 4096 bytes that + # actually consume ~1GB on the volume servers (defeats gzip storage). + return random.Random(i).randbytes(4096) + +con = sqlite3.connect(db_path, isolation_level=None) +con.execute(f"PRAGMA journal_mode={journal_mode};") +con.execute(f"PRAGMA synchronous={sync_mode};") +con.execute("CREATE TABLE IF NOT EXISTS t(id INTEGER PRIMARY KEY, payload BLOB, chk INTEGER);") + +def write_progress(n): + # atomic: write tmp + fsync + rename, so a kill -9 mid-write can't leave an + # empty/torn progress file (the crash-test lower bound must stay trustworthy). + tmp = progress_path + ".tmp" + with open(tmp, "w") as f: + f.write(str(n)); f.flush(); os.fsync(f.fileno()) + os.replace(tmp, progress_path) + +done = 0 +sum_chk = 0 +sum_id = 0 +while done < total_rows: + end = min(done + batch, total_rows) + rows = [] + for i in range(done, end): + p = payload(i); c = zlib.crc32(p) + rows.append((i, p, c)); sum_chk += c; sum_id += i + con.execute("BEGIN;") + con.executemany("INSERT INTO t(id,payload,chk) VALUES(?,?,?);", rows) + con.execute("COMMIT;") + done = end + write_progress(done) + print(f"committed {done}/{total_rows}", flush=True) + +if ref_path: + with open(ref_path, "w") as f: + f.write(f"{total_rows} {sum_chk} {sum_id}\n"); f.flush(); os.fsync(f.fileno()) +con.close() +print("DONE", flush=True) diff --git a/test/benchmark/fuse_db/bin/sqlite_verify.py b/test/benchmark/fuse_db/bin/sqlite_verify.py new file mode 100644 index 000000000..9a5f88e69 --- /dev/null +++ b/test/benchmark/fuse_db/bin/sqlite_verify.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +# Verify a SQLite DB after a crash/restart. +# - PRAGMA integrity_check (structural consistency of the b-tree / pages) +# - row count / sum(chk) / sum(id) vs expected +# - recompute crc32(payload) for every row vs stored chk (byte corruption) +# +# usage: sqlite_verify.py [mode] +# mode=exact (default): count must == expected_rows (normal/unexpected shutdown +# of a fully-loaded DB -- nothing may be lost or extra). +# mode=atleast: expected_rows is a LOWER BOUND (durable committed progress at +# crash time). count must be >= expected_rows and form a contiguous +# prefix 0..count-1 (the in-flight txn may or may not have committed, +# but no committed row may be lost and recovery must be consistent). +import sys, sqlite3, zlib, random + +db_path = sys.argv[1] +expected_rows = int(sys.argv[2]) +mode = sys.argv[3] if len(sys.argv) > 3 else "exact" +if mode not in ("exact", "atleast"): # a typo must not silently relax checks + raise SystemExit(f"invalid mode: {mode!r} (want exact|atleast)") + +def payload(i: int) -> bytes: + return random.Random(i).randbytes(4096) + +con = sqlite3.connect(db_path) +ok = True + +ic = con.execute("PRAGMA integrity_check;").fetchone()[0] +print(f"integrity_check: {ic}") +if ic != "ok": ok = False + +cnt = con.execute("SELECT COUNT(*) FROM t;").fetchone()[0] +minid = con.execute("SELECT MIN(id) FROM t;").fetchone()[0] +maxid = con.execute("SELECT MAX(id) FROM t;").fetchone()[0] +sum_chk= con.execute("SELECT COALESCE(SUM(chk),0) FROM t;").fetchone()[0] +sum_id = con.execute("SELECT COALESCE(SUM(id),0) FROM t;").fetchone()[0] + +if mode == "exact": + target = expected_rows + if cnt != expected_rows: + print(f"rows: got={cnt} expected={expected_rows} MISMATCH"); ok = False + else: + print(f"rows: got={cnt} expected={expected_rows} OK") +else: # atleast + print(f"rows: got={cnt} committed-lower-bound={expected_rows} " + f"{'OK' if cnt >= expected_rows else 'LOST COMMITTED DATA'}") + if cnt < expected_rows: ok = False + target = cnt + # contiguous prefix 0..cnt-1 ? + if cnt > 0 and (minid != 0 or maxid != cnt - 1): + print(f"prefix: MIN={minid} MAX={maxid} expected 0..{cnt-1} NON-CONTIGUOUS"); ok = False + else: + print(f"prefix: 0..{maxid} contiguous OK") + +exp_sum_chk = sum(zlib.crc32(payload(i)) for i in range(target)) +exp_sum_id = target * (target - 1) // 2 +print(f"sum(chk): got={sum_chk} expected={exp_sum_chk} {'OK' if sum_chk==exp_sum_chk else 'MISMATCH'}") +print(f"sum(id): got={sum_id} expected={exp_sum_id} {'OK' if sum_id==exp_sum_id else 'MISMATCH'}") +if sum_chk != exp_sum_chk or sum_id != exp_sum_id: ok = False + +corrupt = 0; scanned = 0 +for rid, p, c in con.execute("SELECT id,payload,chk FROM t;"): + scanned += 1 + if zlib.crc32(p) != c: corrupt += 1 +print(f"payload byte-check: scanned={scanned} corrupt={corrupt} {'OK' if corrupt==0 else 'CORRUPT'}") +if corrupt != 0: ok = False + +con.close() +print("VERIFY: " + ("PASS" if ok else "FAIL")) +sys.exit(0 if ok else 1)