diff --git a/utils/fenced/ipmi-remote-host b/utils/fenced/ipmi-remote-host new file mode 100644 index 00000000..5e004ac9 --- /dev/null +++ b/utils/fenced/ipmi-remote-host @@ -0,0 +1,140 @@ +#!/usr/bin/bash +# /usr/libexec/scoutfs-fenced/run/ipmi-remote-host + +# ipmi configuration +SCOUTFS_IPMI_CONFIG_FILE=${SCOUTFS_IPMI_CONFIG_FILE:-/etc/scoutfs/scoutfs-ipmi.conf} +SCOUTFS_IPMI_HOSTS_FILE=${SCOUTFS_IPMI_HOSTS_FILE:-/etc/scoutfs/scoutfs-ipmi-hosts.conf} + +## hosts file format +## SCOUTFS_HOST_IP IPMI_ADDRESS +## ex: +# 192.168.1.1 192.168.10.1 + +# command setup +IPMI_POWER="/sbin/ipmipower" +SSH_CMD="ssh -o ConnectTimeout=3 -o BatchMode=yes -o StrictHostKeyChecking=no" +LOGGER="/bin/logger -p local3.crit -t scoutfs-fenced" + +$LOGGER "ipmi fence script invoked: IP: $SCOUTFS_FENCED_REQ_IP RID: $SCOUTFS_FENCED_REQ_RID TEST: $IPMITEST" + +echo_fail() { + echo "$@" >&2 + $LOGGER "fence failed: $@" + exit 1 +} + +echo_log() { + echo "$@" >&2 + $LOGGER "fence info: $@" +} + +echo_test_pass() { + echo -e "\xE2\x9C\x94 $@" +} + +echo_test_fail() { + echo -e "\xE2\x9D\x8C $@" +} + +test -n "$SCOUTFS_IPMI_CONFIG_FILE" || \ + echo_fail "SCOUTFS_IPMI_CONFIG_FILE isn't set" +test -r "$SCOUTFS_IPMI_CONFIG_FILE" || \ + echo_fail "$SCOUTFS_IPMI_CONFIG_FILE isn't readable file" +. "$SCOUTFS_IPMI_CONFIG_FILE" +test -n "$SCOUTFS_IPMI_HOSTS_FILE" || \ + echo_fail "SCOUTFS_IPMI_HOSTS_FILE isn't set" +test -r "$SCOUTFS_IPMI_HOSTS_FILE" || \ + echo_fail "$SCOUTFS_IPMI_HOSTS_FILE isn't readable file" +test -x "$IPMI_POWER" || \ + echo_fail "$IPMI_POWER not found, need to install freeimpi?" + +export ip="$SCOUTFS_FENCED_REQ_IP" +export rid="$SCOUTFS_FENCED_REQ_RID" + +getIPMIhost () { + host=$(awk -v ip="$1" '$1 == ip {print $2}' "$SCOUTFS_IPMI_HOSTS_FILE") || \ + echo_fail "lookup ipmi host failed" + echo "$host" +} + +powerOffHost() { + # older versions of ipmipower inverted wait-until-off/wait-until-on, so specify both + $IPMI_POWER $IPMI_OPTS -h "$1" --wait-until-off --wait-until-on --off || \ + echo_fail "ipmi power off $1 failed" + + ipmioutput=$($IPMI_POWER $IPMI_OPTS -h "$1" --stat) || \ + echo_fail "ipmi power stat $1 failed" + + if [[ ! "$ipmioutput" =~ off ]]; then + echo_fail "ipmi stat $1 not off" + fi + + $LOGGER "ipmi fence power down $1 success" + + exit 0 +} + +if [ -n "$IPMITEST" ]; then + for i in $(awk '!/^($|[[:space:]]*#)/ {print $1}' "$SCOUTFS_IPMI_HOSTS_FILE"); do + if ! $SSH_CMD "$i" /bin/true; then + echo_test_fail "ssh $i" + else + echo_test_pass "ssh $i" + fi + host=$(getIPMIhost "$i") + if [ -z "$host" ]; then + echo_test_fail "ipmi config $i $host" + else + if ! $IPMI_POWER $IPMI_OPTS -h "$host" --stat; then + echo_test_fail "ipmi $i" + else + echo_test_pass "ipmi $i" + fi + fi + done + exit 0 +fi + +if [ -z "$ip" ]; then + echo_fail "no IP given for fencing" +fi + +host=$(getIPMIhost "$ip") +if [ -z "$host" ]; then + echo_fail "no IPMI host found for fence IP" +fi + +# first check via ssh if the mount still exists +# if ssh succeeds, we will only power down the node if mounted +if ! output=$($SSH_CMD "$ip" "echo BEGIN; LC_ALL=C egrep -m 1 '(^0x*|^$rid$)' /sys/kernel/boot_params/version /sys/fs/scoutfs/f*r*/rid; echo END"); then + # ssh not working, just power down host + powerOffHost "$host" +fi + +if [[ ! "$output" =~ BEGIN ]]; then + # ssh failure + echo_log "no BEGIN" + powerOffHost "$host" +fi + +if [[ ! "$output" =~ \/boot_params\/ ]]; then + # ssh failure + echo_log "no boot params" + powerOffHost "$host" +fi + +if [[ ! "$output" =~ END ]]; then + # ssh failure + echo_log "no END" + powerOffHost "$host" +fi + +if [[ "$output" =~ "rid:$rid" ]]; then + # rid still mounted, power down + echo_log "rid $rid still mounted" + powerOffHost "$host" +fi + +$LOGGER "ipmi fence host $ip/$host success (rid $rid not mounted)" +exit 0 + diff --git a/utils/fenced/local-force-unmount b/utils/fenced/local-force-unmount new file mode 100644 index 00000000..f211b5b3 --- /dev/null +++ b/utils/fenced/local-force-unmount @@ -0,0 +1,36 @@ +#!/usr/bin/bash +# /usr/libexec/scoutfs-fenced/run/local-force-umount + +echo_fail() { + echo "$@" > /dev/stderr + exit 1 +} + +rid="$SCOUTFS_FENCED_REQ_RID" + +# +# Look for a local mount with the rid to fence. Typically we'll at +# least find the mount with the server that requested the fence that +# we're processing. But it's possible that mounts are unmounted +# before, or while, we're running. +# +mnts=$(findmnt -l -n -t scoutfs -o TARGET) || \ + echo_fail "findmnt -t scoutfs failed" > /dev/stderr + +for mnt in $mnts; do + mnt_rid=$(scoutfs statfs -p "$mnt" -s rid) || \ + echo_fail "scoutfs statfs $mnt failed" + + if [ "$mnt_rid" == "$rid" ]; then + umount -f "$mnt" || \ + echo_fail "umout -f $mnt" + + exit 0 + fi +done + +# +# If the mount doesn't exist on this host then it can't access the +# devices by definition and can be considered fenced. +# +exit 0 diff --git a/utils/fenced/powerman-remote-host b/utils/fenced/powerman-remote-host new file mode 100644 index 00000000..4090b0cb --- /dev/null +++ b/utils/fenced/powerman-remote-host @@ -0,0 +1,139 @@ +#!/usr/bin/bash +# /usr/libexec/scoutfs-fenced/run/powerman-remote-host + +# powerman configuration +SCOUTFS_PM_CONFIG_FILE=${SCOUTFS_PM_CONFIG_FILE:-/etc/scoutfs/scoutfs-pm.conf} +SCOUTFS_PM_HOSTS_FILE=${SCOUTFS_PM_HOSTS_FILE:-/etc/scoutfs/scoutfs-pm-hosts.conf} + +## hosts file format +## SCOUTFS_HOST_IP POWERMAN_NODE_NAME +## ex: +# 192.168.1.1 dm1 + +# command setup +PM_CMD="/usr/bin/pm" +SSH_CMD="ssh -o ConnectTimeout=3 -o BatchMode=yes -o StrictHostKeyChecking=no" +LOGGER="/bin/logger -p local3.crit -t scoutfs-fenced" + +$LOGGER "ipmi fence script invoked: IP: $SCOUTFS_FENCED_REQ_IP RID: $SCOUTFS_FENCED_REQ_RID TEST: $IPMITEST" + +echo_fail() { + echo "$@" >&2 + $LOGGER "fence failed: $@" + exit 1 +} + +echo_log() { + echo "$@" >&2 + $LOGGER "fence info: $@" +} + +echo_test_pass() { + echo -e "\xE2\x9C\x94 $@" +} + +echo_test_fail() { + echo -e "\xE2\x9D\x8C $@" +} + +test -n "$SCOUTFS_PM_CONFIG_FILE" || \ + echo_fail "SCOUTFS_PM_CONFIG_FILE isn't set" +test -r "$SCOUTFS_PM_CONFIG_FILE" || \ + echo_fail "$SCOUTFS_PM_CONFIG_FILE isn't readable file" +. "$SCOUTFS_PM_CONFIG_FILE" +test -n "$SCOUTFS_PM_HOSTS_FILE" || \ + echo_fail "SCOUTFS_PM_HOSTS_FILE isn't set" +test -r "$SCOUTFS_PM_HOSTS_FILE" || \ + echo_fail "$SCOUTFS_PM_HOSTS_FILE isn't readable file" +test -x "$PM_CMD" || \ + echo_fail "$PMCMD not found, need to install powerman?" + +export ip="$SCOUTFS_FENCED_REQ_IP" +fence_rid="$SCOUTFS_FENCED_REQ_RID" + +getPMhost () { + host=$(awk -v ip="$1" '$1 == ip {print $2}' "$SCOUTFS_PM_HOSTS_FILE") || \ + echo_fail "lookup pm host failed" + echo "$host" +} + +powerOffHost() { + $PM_CMD $PM_OPTS "$1" -0 || \ + echo_fail "pm power off $host failed" + + pmoutput=$($PM_CMD $PM_OPTS "$1" -q | grep "$1") || \ + echo_fail "powerman power stat $1 failed" + + if [[ ! "$pmoutput" =~ off ]]; then + echo_fail "powerman stat $1 not off" + fi + + $LOGGER "powerman fence power down $1 success" + + exit 0 +} + +if [ -n "$PMTEST" ]; then + for i in $(awk '!/^($|[[:space:]]*#)/ {print $1}' "$SCOUTFS_PM_HOSTS_FILE"); do + if ! $SSH_CMD "$i" /bin/true; then + echo_test_fail "ssh $i" + else + echo_test_pass "ssh $i" + fi + host=$(getPMhost "$i") + if [ -z "$host" ]; then + echo_test_fail "pm config $i $host" + else + if ! $PM_CMD $PM_OPTS "$host" -q; then + echo_test_fail "pm $i" + else + echo_test_pass "pm $i" + fi + fi + done + exit 0 +fi + +if [ -z "$ip" ]; then + echo_fail "no IP given for fencing" +fi + +host=$(getPMhost "$ip") +if [ -z "$host" ]; then + echo_fail "no host found for fence IP" +fi + +# first check via ssh if the mount still exists +# if ssh succeeds, we will only power down the node if mounted +if ! output=$($SSH_CMD "$ip" "echo BEGIN; LC_ALL=C egrep -m 1 '(^0x*|^$rid$)' /sys/kernel/boot_params/version /sys/fs/scoutfs/f*r*/rid; echo END"); then + # ssh not working, just power down host + powerOffHost "$host" +fi + +if [[ ! "$output" =~ BEGIN ]]; then + # ssh failure + echo_log "no BEGIN" + powerOffHost "$host" +fi + +if [[ ! "$output" =~ \/boot_params\/ ]]; then + # ssh failure + echo_log "no boot params" + powerOffHost "$host" +fi + +if [[ ! "$output" =~ END ]]; then + # ssh failure + echo_log "no END" + powerOffHost "$host" +fi + +if [[ "$output" =~ "rid:$rid" ]]; then + # rid still mounted, power down + echo_log "rid $rid still mounted" + powerOffHost "$host" +fi + +$LOGGER "powerman fence host $ip/$host success (rid $rid not mounted)" +exit 0 + diff --git a/utils/fenced/scoutfs-ipmi-hosts.conf b/utils/fenced/scoutfs-ipmi-hosts.conf new file mode 100644 index 00000000..5e8cfd46 --- /dev/null +++ b/utils/fenced/scoutfs-ipmi-hosts.conf @@ -0,0 +1,11 @@ +# /etc/scoutfs/scoutfs-ipmi-hosts.conf + +## config file format +## +## SCOUTFS_HOST_IP must match the interface used for scoutfs +## leader/follower communications +## +## SCOUTFS_HOST_IP IPMI_ADDRESS +## ex: +#192.168.1.1 192.168.10.1 + diff --git a/utils/fenced/scoutfs-ipmi.conf b/utils/fenced/scoutfs-ipmi.conf new file mode 100644 index 00000000..9b7386d5 --- /dev/null +++ b/utils/fenced/scoutfs-ipmi.conf @@ -0,0 +1,10 @@ +#!/usr/bin/bash +# /etc/scoutfs/scoutfs-ipmi.conf + +IPMI_USER="admin" +IPMI_PASSWORD="password" +IPMI_OPTS="-D LAN_2_0 -u $IPMI_USER -p $IPMI_PASSWORD" + +# some Intel BMCs need -I 17 +# IPMI_OPTS="-D LAN_2_0 -u $IPMI_USER -p $IPMI_PASSWORD -I 17" + diff --git a/utils/fenced/scoutfs-pm-hosts.conf b/utils/fenced/scoutfs-pm-hosts.conf new file mode 100644 index 00000000..4bb5fc34 --- /dev/null +++ b/utils/fenced/scoutfs-pm-hosts.conf @@ -0,0 +1,11 @@ +# /etc/scoutfs/scoutfs-ipmi-hosts.conf + +## config file format +## +## SCOUTFS_HOST_IP must match the interface used for scoutfs +## leader/follower communications +## +## SCOUTFS_HOST_IP POWERMAN_NODE_NAME +## ex: +#192.168.1.1 node1 + diff --git a/utils/fenced/scoutfs-pm.conf b/utils/fenced/scoutfs-pm.conf new file mode 100644 index 00000000..112d355f --- /dev/null +++ b/utils/fenced/scoutfs-pm.conf @@ -0,0 +1,8 @@ +#!/usr/bin/bash +# /etc/scoutfs/scoutfs-pm.conf + +PM_OPTS="" + +# optionally specify remote powerman server +#PM_OPTS="-h pm-server.localdomain" + diff --git a/utils/scoutfs-utils.spec.in b/utils/scoutfs-utils.spec.in index 4568b73f..f004323e 100644 --- a/utils/scoutfs-utils.spec.in +++ b/utils/scoutfs-utils.spec.in @@ -55,14 +55,21 @@ install -m 755 -D src/scoutfs $RPM_BUILD_ROOT%{_sbindir}/scoutfs install -m 644 -D src/ioctl.h $RPM_BUILD_ROOT%{_includedir}/scoutfs/ioctl.h install -m 644 -D src/format.h $RPM_BUILD_ROOT%{_includedir}/scoutfs/format.h install -m 755 -D fenced/scoutfs-fenced $RPM_BUILD_ROOT%{_libexecdir}/scoutfs-fenced/scoutfs-fenced +install -m 755 -D fenced/local-force-unmount $RPM_BUILD_ROOT%{_libexecdir}/scoutfs-fenced/run/local-force-unmount +install -m 755 -D fenced/ipmi-remote-host $RPM_BUILD_ROOT%{_libexecdir}/scoutfs-fenced/run/ipmi-remote-host +install -m 755 -D fenced/powerman-remote-host $RPM_BUILD_ROOT%{_libexecdir}/scoutfs-fenced/run/powerman-remote-host install -m 644 -D fenced/scoutfs-fenced.service $RPM_BUILD_ROOT%{_unitdir}/scoutfs-fenced.service install -m 644 -D fenced/scoutfs-fenced.conf.example $RPM_BUILD_ROOT%{_sysconfdir}/scoutfs/scoutfs-fenced.conf.example +install -m 644 -D fenced/scoutfs-ipmi.conf $RPM_BUILD_ROOT%{_sysconfdir}/scoutfs/scoutfs-ipmi.conf +install -m 644 -D fenced/scoutfs-ipmi-hosts.conf $RPM_BUILD_ROOT%{_sysconfdir}/scoutfs/scoutfs-ipmi-hosts.conf +install -m 644 -D fenced/scoutfs-pm.conf $RPM_BUILD_ROOT%{_sysconfdir}/scoutfs/scoutfs-pm.conf +install -m 644 -D fenced/scoutfs-pm-hosts.conf $RPM_BUILD_ROOT%{_sysconfdir}/scoutfs/scoutfs-pm-hosts.conf %files %defattr(644,root,root,755) %{_mandir}/man*/scoutfs*.gz %{_unitdir}/scoutfs-fenced.service -%{_sysconfdir}/scoutfs +%config(noreplace) %{_sysconfdir}/scoutfs %defattr(755,root,root,755) %{_sbindir}/scoutfs %{_libexecdir}/scoutfs-fenced