mirror of
https://github.com/versity/scoutfs.git
synced 2026-09-27 10:24:31 +00:00
Zach Brown
8ddf9b8c8c
The userspace fencing process wasn't careful about handling underlying directories that disappear while it was working. On the server/fenced side, fencing requests can linger after they've been resolved by writing 1 to fenced or error. The script could come back around to see the directory before the server finally removes it, causing all later uses of the request dir to fail. We saw this in the logs as a bunch of cat errors for the various request files. On the local fence script side, all the mounts can be in the process of being unmounted so both the /sys/fs dirs and the mount it self can be removed while we're working. For both, when we're working with the /sys/fs files we read them without logging errors and then test that the dir still exists before using what we read. When fencing a mount, we stop if findmnt doesn't find the mount and then raise a umount error if the /sys/fs dir exists after umount fails. And while we're at it, we have each scripts logging append instead of truncating (if, say, it's a log file instead of an interactive tty). Signed-off-by: Zach Brown <zab@versity.com>
104 lines
2.4 KiB
Bash
Executable File
104 lines
2.4 KiB
Bash
Executable File
#!/usr/bin/bash
|
|
|
|
message_output()
|
|
{
|
|
printf "[%s] %s\n" "$(date '+%F %T.%N')" "$@"
|
|
}
|
|
|
|
error_message()
|
|
{
|
|
message_output "$@" >> /dev/stderr
|
|
}
|
|
|
|
error_exit()
|
|
{
|
|
error_message "$@, exiting"
|
|
exit 1
|
|
}
|
|
|
|
log_message()
|
|
{
|
|
message_output "$@"
|
|
}
|
|
|
|
# restart if we catch hup to re-read the config
|
|
hup_restart()
|
|
{
|
|
log_message "caught SIGHUP, restarting"
|
|
exec "$@"
|
|
}
|
|
trap hup_restart SIGHUP
|
|
|
|
# defaults
|
|
SCOUTFS_FENCED_CONFIG_FILE=${SCOUTFS_FENCED_CONFIG_FILE:-/etc/scoutfs/scoutfs-fenced.conf}
|
|
SCOUTFS_FENCED_DELAY=2
|
|
#SCOUTFS_FENCED_RUN
|
|
#SCOUTFS_FENCED_RUN_ARGS
|
|
|
|
test -n "$SCOUTFS_FENCED_CONFIG_FILE" || \
|
|
error_exit "SCOUTFS_FENCED_CONFIG_FILE isn't set"
|
|
test -r "$SCOUTFS_FENCED_CONFIG_FILE" || \
|
|
error_exit "SCOUTFS_FENCED_CONFIG_FILE isn't readable file"
|
|
|
|
log_message "reading config file $SCOUTFS_FENCED_CONFIG_FILE"
|
|
|
|
. "$SCOUTFS_FENCED_CONFIG_FILE" || \
|
|
error_exit "error sourcing $SCOUTFS_FENCED_CONFIG_FILE as bash script"
|
|
|
|
for conf in "${!SCOUTFS_FENCED_@}"; do
|
|
log_message " config var $conf=${!conf}"
|
|
done
|
|
|
|
test -n "$SCOUTFS_FENCED_RUN" || \
|
|
error_exit "SCOUTFS_FENCED_RUN must be set"
|
|
test -x "$SCOUTFS_FENCED_RUN" || \
|
|
error_exit "SCOUTFS_FENCED_RUN '$SCOUTFS_FENCED_RUN' isn't executable"
|
|
|
|
#
|
|
# Main loop watching for fence request across all filesystems. The
|
|
# server can shut down without waiting for pending fence requests to
|
|
# finish. All of the interaction with the fence directory and files can
|
|
# fail at any moment. We will generate log messages when the dir or
|
|
# files disappear.
|
|
#
|
|
|
|
# silence error messages
|
|
quiet_cat()
|
|
{
|
|
cat "$@" 2>/dev/null
|
|
}
|
|
|
|
while sleep $SCOUTFS_FENCED_DELAY; do
|
|
shopt -s nullglob
|
|
for fence in /sys/fs/scoutfs/*/fence/*; do
|
|
|
|
srv=$(basename $(dirname $(dirname $fence)))
|
|
fenced="$(quiet_cat $fence/fenced)"
|
|
error="$(quiet_cat $fence/error)"
|
|
rid="$(quiet_cat $fence/rid)"
|
|
ip="$(quiet_cat $fence/ipv4_addr)"
|
|
reason="$(quiet_cat $fence/reason)"
|
|
|
|
# request dirs can linger then disappear after fenced/error is set
|
|
if [ ! -d "$fence" -o "$fenced" == "1" -o "$error" == "1" ]; then
|
|
continue
|
|
fi
|
|
|
|
log_message "server $srv fencing rid $rid at IP $ip for $reason"
|
|
|
|
# export _REQ_ vars for run to use
|
|
export SCOUTFS_FENCED_REQ_RID="$rid"
|
|
export SCOUTFS_FENCED_REQ_IP="$ip"
|
|
|
|
$SCOUTFS_FENCED_RUN $SCOUTFS_FENCED_RUN_ARGS
|
|
rc=$?
|
|
if [ "$rc" != 0 ]; then
|
|
log_message "server $srv fencing rid $rid saw error status $rc"
|
|
echo 1 > "$fence/error"
|
|
continue
|
|
fi
|
|
|
|
echo 1 > "$fence/fenced"
|
|
done
|
|
done
|