mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-08-16 12:16:36 +00:00
Chris LuandGitHub
76783f3d71
* test: add FUSE database load/durability/perf benchmark Runs MySQL (InnoDB) and SQLite with ~1GB datadirs on a SeaweedFS FUSE mount. Two parts: - durability: normal shutdown, kill -9, and crash-during-write all keep every fsync-committed row (verified by integrity check + row count + contiguous prefix + per-row CRC). - performance: FUSE vs the same local disk. fsync/commit latency is the dominant cost (~0.13ms -> ~1.18ms), so small transactions run ~9-12x slower while bulk loads and warm reads stay close. Harness is path-independent (runtime under $SEAWEED_BENCH_WORK) and only touches its own processes on non-default ports. * test/benchmark/fuse_db: portable to Linux + crash-safe progress - export MYSQL_BIN; mysql_bench.py falls back to PATH when unset - unmount via fusermount/fusermount3 (non-root Linux), then umount/diskutil - atomic progress write (tmp+fsync+rename); treat empty progress file as 0 - reuse a single PRNG in the perf probe so RNG init doesn't skew timings * test/benchmark/fuse_db: validate inputs, add subprocess timeout - mysql_bench.py: 1800s timeout on mysql CLI calls; reject db names that aren't plain identifiers (interpolated into SQL) - sqlite_gen.py / sqlite_verify.py: allowlist journal/synchronous modes and the verify mode so a typo can't silently weaken durability or relax checks - run_mysql.sh: durable atomic progress write (tmp+fsync+rename), matching sqlite_gen.py; quote $LB in both crash-test verify calls
53 lines
1.5 KiB
Python
53 lines
1.5 KiB
Python
#!/usr/bin/env python3
|
|
# Raw filesystem microbenchmark in one directory. Prints TSV: METRIC<TAB>value<TAB>unit
|
|
# SEQWRITE MB/s (512MB stream + final fsync)
|
|
# FSYNC ops/s (500 x: pwrite 4KB at random offset in a 64MB file, then fsync)
|
|
# SEQREAD MB/s (read the 512MB file back; warm cache)
|
|
import sys, os, time, random
|
|
|
|
d = sys.argv[1]
|
|
os.makedirs(d, exist_ok=True)
|
|
big = os.path.join(d, "fsb_big.bin")
|
|
sm = os.path.join(d, "fsb_sync.bin")
|
|
|
|
# SEQWRITE 512MB + fsync
|
|
n = 512 * 1024 * 1024
|
|
buf = os.urandom(8 * 1024 * 1024)
|
|
t = time.time(); f = open(big, "wb"); w = 0
|
|
while w < n:
|
|
f.write(buf); w += len(buf)
|
|
f.flush(); os.fsync(f.fileno()); f.close()
|
|
dt = time.time() - t
|
|
print(f"SEQWRITE\t{w/1048576/dt:.0f}\tMB/s")
|
|
|
|
# FSYNC latency: 500 x (pwrite 4KB random offset + fsync) on a 64MB file
|
|
fd = os.open(sm, os.O_RDWR | os.O_CREAT, 0o644)
|
|
os.ftruncate(fd, 64 * 1024 * 1024)
|
|
os.fsync(fd)
|
|
page = os.urandom(4096)
|
|
N = 500
|
|
rnd = random.Random(12345)
|
|
t = time.time()
|
|
for _ in range(N):
|
|
off = rnd.randrange(0, 64 * 1024 * 1024 - 4096) & ~0xfff
|
|
os.pwrite(fd, page, off)
|
|
os.fsync(fd)
|
|
dt = time.time() - t
|
|
os.close(fd)
|
|
print(f"FSYNC\t{N/dt:.0f}\tops/s")
|
|
print(f"FSYNCLAT\t{dt/N*1000:.2f}\tms")
|
|
|
|
# SEQREAD (warm)
|
|
t = time.time(); tot = 0
|
|
with open(big, "rb") as f:
|
|
while True:
|
|
b = f.read(8 * 1024 * 1024)
|
|
if not b: break
|
|
tot += len(b)
|
|
dt = time.time() - t
|
|
print(f"SEQREAD\t{tot/1048576/dt:.0f}\tMB/s")
|
|
|
|
for p in (big, sm):
|
|
try: os.remove(p)
|
|
except OSError: pass
|