mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-07-06 08:06:31 +00:00
Chris Lu
afcc491517
test(mount): fix fd leak that deadlocked the DLM handoff check The cross-mount handoff checks held a file open on mount 2 via fd 9 to keep the distributed lock, then started the SMB writer in a background subshell. The subshell inherited fd 9, so the SMB writer kept the file open and waited on a lock held by its own descriptor; the put could never complete, and the two checks were parked as expected-fail. Close fd 9 in the subshell (9>&-) so the writer does not hold the file. The waiter now acquires the freed lock within ~1s, so the two checks are real assertions and the xfail machinery is gone.
219 lines
7.9 KiB
Bash
Executable File
219 lines
7.9 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# Locking / concurrency test battery for Samba on a SeaweedFS FUSE mount.
|
|
#
|
|
# Covers the challenges a network-filesystem backend has to get right:
|
|
# 1. POSIX fcntl byte-range locking on the FUSE mount (SetLk/GetLk)
|
|
# 2. Distributed locking (-dlm): a write held open on one mount blocks a
|
|
# writer on another mount until it is released
|
|
# 3. Distributed locking integrity: concurrent writers to the same file from
|
|
# two mounts produce intact (non-torn) data
|
|
# 4. Concurrent writers to distinct files all succeed
|
|
#
|
|
# Required env:
|
|
# SMB_USER, SMB_PASS samba credentials
|
|
# MOUNT_SHARE dir on the smbd-backed FUSE mount (mount 1)
|
|
# MOUNT2_SHARE dir on the second FUSE mount (mount 2)
|
|
# Optional env:
|
|
# SMB_HOST (127.0.0.1), SMB_SHARE (seaweedfs), SMB_PORT (445)
|
|
set -uo pipefail
|
|
|
|
SMB_HOST="${SMB_HOST:-127.0.0.1}"
|
|
SMB_SHARE="${SMB_SHARE:-seaweedfs}"
|
|
SMB_PORT="${SMB_PORT:-445}"
|
|
SMB_USER="${SMB_USER:?SMB_USER is required}"
|
|
SMB_PASS="${SMB_PASS:?SMB_PASS is required}"
|
|
MOUNT_SHARE="${MOUNT_SHARE:?MOUNT_SHARE is required}"
|
|
MOUNT2_SHARE="${MOUNT2_SHARE:?MOUNT2_SHARE is required}"
|
|
|
|
WORK="$(mktemp -d /tmp/samba-locktest.XXXXXX)"
|
|
trap 'rm -rf "${WORK}"' EXIT
|
|
|
|
PASS=0
|
|
FAIL=0
|
|
pass() { printf ' [PASS] %s\n' "$1"; PASS=$((PASS + 1)); }
|
|
fail() { printf ' [FAIL] %s\n' "$1"; FAIL=$((FAIL + 1)); }
|
|
|
|
smb() {
|
|
smbclient "//${SMB_HOST}/${SMB_SHARE}" -p "${SMB_PORT}" \
|
|
-U "${SMB_USER}%${SMB_PASS}" -m SMB3 -c "$1"
|
|
}
|
|
md5() { md5sum "$1" | awk '{print $1}'; }
|
|
|
|
# 1. POSIX fcntl byte-range locking on the FUSE mount ------------------------
|
|
# Exercises the mount's SetLk/GetLk via two processes contending over fcntl
|
|
# (F_SETLK) byte-range locks. python3's fcntl.lockf issues real POSIX locks.
|
|
echo "==> 1. POSIX fcntl byte-range locking (FUSE mount SetLk/GetLk)"
|
|
lockfile="${MOUNT_SHARE}/fcntl_lock.dat"
|
|
: >"${lockfile}"
|
|
fcntl_out="$(python3 - "${lockfile}" <<'PY'
|
|
import fcntl, os, sys
|
|
|
|
path = sys.argv[1]
|
|
parent_to_child_r, parent_to_child_w = os.pipe() # release signal
|
|
child_to_parent_r, child_to_parent_w = os.pipe() # locked signal
|
|
|
|
pid = os.fork()
|
|
if pid == 0: # child: hold an exclusive lock on [0,100)
|
|
fd = os.open(path, os.O_RDWR | os.O_CREAT, 0o644)
|
|
fcntl.lockf(fd, fcntl.LOCK_EX, 100, 0, 0)
|
|
os.write(child_to_parent_w, b"L")
|
|
os.read(parent_to_child_r, 1) # wait until parent says release
|
|
fcntl.lockf(fd, fcntl.LOCK_UN, 100, 0, 0)
|
|
os.close(fd)
|
|
os._exit(0)
|
|
|
|
# parent
|
|
os.read(child_to_parent_r, 1) # wait until child holds the lock
|
|
fd = os.open(path, os.O_RDWR | os.O_CREAT, 0o644)
|
|
results = []
|
|
|
|
# a. a conflicting exclusive lock must be denied while the child holds it
|
|
try:
|
|
fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB, 100, 0, 0)
|
|
fcntl.lockf(fd, fcntl.LOCK_UN, 100, 0, 0)
|
|
results.append(("conflicting exclusive lock denied while held", False))
|
|
except OSError:
|
|
results.append(("conflicting exclusive lock denied while held", True))
|
|
|
|
# b. a non-overlapping range must be grantable
|
|
try:
|
|
fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB, 100, 200, 0)
|
|
fcntl.lockf(fd, fcntl.LOCK_UN, 100, 200, 0)
|
|
results.append(("non-overlapping range lock granted", True))
|
|
except OSError:
|
|
results.append(("non-overlapping range lock granted", False))
|
|
|
|
# c. after the holder releases, the lock must be acquirable
|
|
os.write(parent_to_child_w, b"R")
|
|
os.waitpid(pid, 0)
|
|
try:
|
|
fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB, 100, 0, 0)
|
|
fcntl.lockf(fd, fcntl.LOCK_UN, 100, 0, 0)
|
|
results.append(("lock acquirable after holder releases", True))
|
|
except OSError:
|
|
results.append(("lock acquirable after holder releases", False))
|
|
|
|
for name, ok in results:
|
|
print((" [PASS] " if ok else " [FAIL] ") + name)
|
|
sys.exit(0 if all(ok for _, ok in results) else 1)
|
|
PY
|
|
)"
|
|
echo "${fcntl_out}"
|
|
PASS=$((PASS + $(grep -c '\[PASS\]' <<<"${fcntl_out}")))
|
|
FAIL=$((FAIL + $(grep -c '\[FAIL\]' <<<"${fcntl_out}")))
|
|
|
|
# 2. Distributed lock blocks a cross-mount writer, then hands it off ----------
|
|
# mount 2 holds a file open for writing (holding the DLM lock on its path).
|
|
# An SMB put of the same file goes through mount 1 and must (a) block while
|
|
# mount 2 holds it and (b) succeed once mount 2 releases, leaving the SMB
|
|
# writer's payload on disk. smbclient gets a long client timeout (-t) so we are
|
|
# testing the lock handoff itself, not smbclient's own ~20s default timeout.
|
|
echo "==> 2. distributed lock: cross-mount write coordination"
|
|
dlmfile="dlm_coord.bin"
|
|
newdata="${WORK}/dlm_new.bin"
|
|
head -c 4096 /dev/urandom >"${newdata}"
|
|
|
|
# Hold the file open for writing on mount 2 via fd 9 -> holds the DLM lock.
|
|
exec 9>"${MOUNT2_SHARE}/${dlmfile}"
|
|
printf 'held-by-mount2' >&9
|
|
|
|
# Start the SMB write; record its real exit code when it returns. The subshell
|
|
# must NOT inherit fd 9 (9>&-): otherwise the SMB writer keeps the file open and
|
|
# waits on a DLM lock held by its own inherited descriptor, deadlocking the
|
|
# handoff this test is meant to exercise.
|
|
rm -f "${WORK}/dlm_put.rc"
|
|
(
|
|
smbclient "//${SMB_HOST}/${SMB_SHARE}" -p "${SMB_PORT}" \
|
|
-U "${SMB_USER}%${SMB_PASS}" -m SMB3 -t 120 \
|
|
-c "put ${newdata} ${dlmfile}" >/dev/null 2>&1
|
|
echo "$?" >"${WORK}/dlm_put.rc"
|
|
) 9>&- &
|
|
smb_bg=$!
|
|
|
|
sleep 4
|
|
if [[ ! -f "${WORK}/dlm_put.rc" ]]; then
|
|
pass "SMB write blocks while another mount holds the file open"
|
|
else
|
|
fail "SMB write returned early instead of blocking (rc=$(cat "${WORK}/dlm_put.rc"))"
|
|
fi
|
|
|
|
# Release mount 2's DLM lock; the blocked SMB write must now complete.
|
|
exec 9>&-
|
|
|
|
# Wait (bounded) for the SMB put to finish so a stuck handoff fails the test
|
|
# instead of hanging the suite.
|
|
put_rc="timeout"
|
|
for _ in $(seq 1 20); do
|
|
if [[ -f "${WORK}/dlm_put.rc" ]]; then
|
|
put_rc="$(cat "${WORK}/dlm_put.rc")"
|
|
break
|
|
fi
|
|
sleep 1
|
|
done
|
|
kill "${smb_bg}" 2>/dev/null
|
|
wait "${smb_bg}" 2>/dev/null
|
|
|
|
if [[ "${put_rc}" == "0" ]]; then
|
|
pass "blocked SMB write succeeds after the other mount releases"
|
|
else
|
|
fail "blocked SMB write succeeds after the other mount releases (rc=${put_rc})"
|
|
fi
|
|
|
|
# A correct handoff leaves the SMB writer's payload on disk: mount 1 acquired
|
|
# the lock and wrote after mount 2 released.
|
|
got="${WORK}/dlm_got.bin"
|
|
if smb "get ${dlmfile} ${got}" >/dev/null 2>&1 && [[ "$(md5 "${got}")" == "$(md5 "${newdata}")" ]]; then
|
|
pass "post-release content is the SMB writer's payload (correct handoff)"
|
|
else
|
|
fail "post-release content is the SMB writer's payload (correct handoff)"
|
|
fi
|
|
|
|
# 3. Distributed lock integrity: concurrent writers, same file ---------------
|
|
# An SMB writer (mount 1) and a direct writer (mount 2) race on one file. DLM
|
|
# serializes them, so the result must be exactly one of the two payloads.
|
|
echo "==> 3. distributed lock: concurrent writers produce intact data"
|
|
racefile="dlm_race.bin"
|
|
payloadA="${WORK}/dlm_raceA.bin"
|
|
head -c 1048576 /dev/urandom >"${payloadA}"
|
|
payloadB="direct-write-from-mount2-payload"
|
|
(smb "put ${payloadA} ${racefile}" >/dev/null 2>&1) &
|
|
(printf '%s' "${payloadB}" >"${MOUNT2_SHARE}/${racefile}") &
|
|
wait
|
|
racegot="${WORK}/dlm_race_got.bin"
|
|
if smb "get ${racefile} ${racegot}" >/dev/null 2>&1 &&
|
|
{ [[ "$(md5 "${racegot}")" == "$(md5 "${payloadA}")" ]] || [[ "$(cat "${racegot}")" == "${payloadB}" ]]; }; then
|
|
pass "concurrent same-file writers leave one intact payload"
|
|
else
|
|
fail "concurrent same-file writers leave one intact payload"
|
|
fi
|
|
|
|
# 4. Concurrent writers to distinct files ------------------------------------
|
|
echo "==> 4. concurrent writers to distinct files"
|
|
n=6
|
|
declare -a srcs=()
|
|
for i in $(seq 1 "${n}"); do
|
|
s="${WORK}/cc_${i}.bin"
|
|
head -c 1048576 /dev/urandom >"${s}"
|
|
srcs+=("${s}")
|
|
(smb "put ${s} concurrent_${i}.bin" >/dev/null 2>&1) &
|
|
done
|
|
wait
|
|
all_ok=true
|
|
for i in $(seq 1 "${n}"); do
|
|
g="${WORK}/cc_got_${i}.bin"
|
|
if ! smb "get concurrent_${i}.bin ${g}" >/dev/null 2>&1 ||
|
|
[[ "$(md5 "${srcs[$((i - 1))]}")" != "$(md5 "${g}")" ]]; then
|
|
all_ok=false
|
|
fi
|
|
done
|
|
if ${all_ok}; then
|
|
pass "${n} concurrent distinct-file writes all intact"
|
|
else
|
|
fail "${n} concurrent distinct-file writes all intact"
|
|
fi
|
|
|
|
echo
|
|
echo "==> Summary: ${PASS} passed, ${FAIL} failed"
|
|
[[ "${FAIL}" -eq 0 ]]
|