mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-07-21 07:23:02 +00:00
Chris Lu
76783f3d71
* test: add FUSE database load/durability/perf benchmark Runs MySQL (InnoDB) and SQLite with ~1GB datadirs on a SeaweedFS FUSE mount. Two parts: - durability: normal shutdown, kill -9, and crash-during-write all keep every fsync-committed row (verified by integrity check + row count + contiguous prefix + per-row CRC). - performance: FUSE vs the same local disk. fsync/commit latency is the dominant cost (~0.13ms -> ~1.18ms), so small transactions run ~9-12x slower while bulk loads and warm reads stay close. Harness is path-independent (runtime under $SEAWEED_BENCH_WORK) and only touches its own processes on non-default ports. * test/benchmark/fuse_db: portable to Linux + crash-safe progress - export MYSQL_BIN; mysql_bench.py falls back to PATH when unset - unmount via fusermount/fusermount3 (non-root Linux), then umount/diskutil - atomic progress write (tmp+fsync+rename); treat empty progress file as 0 - reuse a single PRNG in the perf probe so RNG init doesn't skew timings * test/benchmark/fuse_db: validate inputs, add subprocess timeout - mysql_bench.py: 1800s timeout on mysql CLI calls; reject db names that aren't plain identifiers (interpolated into SQL) - sqlite_gen.py / sqlite_verify.py: allowlist journal/synchronous modes and the verify mode so a typo can't silently weaken durability or relax checks - run_mysql.sh: durable atomic progress write (tmp+fsync+rename), matching sqlite_gen.py; quote $LB in both crash-test verify calls
63 lines
2.6 KiB
Python
63 lines
2.6 KiB
Python
#!/usr/bin/env python3
|
|
# Deterministic ~1GB SQLite load on the FUSE mount.
|
|
# Each row: id, payload (4096 deterministic bytes derived from id), chk=crc32(payload).
|
|
# Commits in batches; after each commit, writes the committed row count to a
|
|
# progress file on LOCAL disk and fsyncs it -> a durable record of what was
|
|
# committed, used to verify "no committed data lost" after a crash.
|
|
import sys, os, sqlite3, zlib, random
|
|
|
|
db_path = sys.argv[1]
|
|
total_rows = int(sys.argv[2])
|
|
batch = int(sys.argv[3])
|
|
progress_path= sys.argv[4] # LOCAL disk
|
|
ref_path = sys.argv[5] if len(sys.argv) > 5 else None
|
|
sync_mode = sys.argv[6] if len(sys.argv) > 6 else "FULL"
|
|
journal_mode = sys.argv[7] if len(sys.argv) > 7 else "DELETE"
|
|
|
|
# Validate before interpolating into PRAGMAs: a typo must not silently weaken
|
|
# durability and invalidate the comparison / crash-survival claims.
|
|
if sync_mode.upper() not in {"OFF", "NORMAL", "FULL", "EXTRA"}:
|
|
raise SystemExit(f"invalid synchronous mode: {sync_mode!r}")
|
|
if journal_mode.upper() not in {"DELETE", "TRUNCATE", "PERSIST", "MEMORY", "WAL", "OFF"}:
|
|
raise SystemExit(f"invalid journal mode: {journal_mode!r}")
|
|
|
|
def payload(i: int) -> bytes:
|
|
# deterministic but incompressible (PRNG seeded by id) -> 4096 bytes that
|
|
# actually consume ~1GB on the volume servers (defeats gzip storage).
|
|
return random.Random(i).randbytes(4096)
|
|
|
|
con = sqlite3.connect(db_path, isolation_level=None)
|
|
con.execute(f"PRAGMA journal_mode={journal_mode};")
|
|
con.execute(f"PRAGMA synchronous={sync_mode};")
|
|
con.execute("CREATE TABLE IF NOT EXISTS t(id INTEGER PRIMARY KEY, payload BLOB, chk INTEGER);")
|
|
|
|
def write_progress(n):
|
|
# atomic: write tmp + fsync + rename, so a kill -9 mid-write can't leave an
|
|
# empty/torn progress file (the crash-test lower bound must stay trustworthy).
|
|
tmp = progress_path + ".tmp"
|
|
with open(tmp, "w") as f:
|
|
f.write(str(n)); f.flush(); os.fsync(f.fileno())
|
|
os.replace(tmp, progress_path)
|
|
|
|
done = 0
|
|
sum_chk = 0
|
|
sum_id = 0
|
|
while done < total_rows:
|
|
end = min(done + batch, total_rows)
|
|
rows = []
|
|
for i in range(done, end):
|
|
p = payload(i); c = zlib.crc32(p)
|
|
rows.append((i, p, c)); sum_chk += c; sum_id += i
|
|
con.execute("BEGIN;")
|
|
con.executemany("INSERT INTO t(id,payload,chk) VALUES(?,?,?);", rows)
|
|
con.execute("COMMIT;")
|
|
done = end
|
|
write_progress(done)
|
|
print(f"committed {done}/{total_rows}", flush=True)
|
|
|
|
if ref_path:
|
|
with open(ref_path, "w") as f:
|
|
f.write(f"{total_rows} {sum_chk} {sum_id}\n"); f.flush(); os.fsync(f.fileno())
|
|
con.close()
|
|
print("DONE", flush=True)
|