add local,ipmi,powerman fenced scripts to utils rpm

This adds the fenced scripts so that we have a place to track these and
get updates out to users. This latest version of scripts has the checks
to validate that power off succeeded and not just assume based on power
command return status.

Signed-off-by: Ben McClelland <ben.mcclelland@versity.com>

utils/fenced/ipmi-remote-host

@@ -0,0 +1,140 @@
+#!/usr/bin/bash
+# /usr/libexec/scoutfs-fenced/run/ipmi-remote-host
+
+# ipmi configuration
+SCOUTFS_IPMI_CONFIG_FILE=${SCOUTFS_IPMI_CONFIG_FILE:-/etc/scoutfs/scoutfs-ipmi.conf}
+SCOUTFS_IPMI_HOSTS_FILE=${SCOUTFS_IPMI_HOSTS_FILE:-/etc/scoutfs/scoutfs-ipmi-hosts.conf}
+
+## hosts file format
+## SCOUTFS_HOST_IP IPMI_ADDRESS
+## ex:
+#  192.168.1.1     192.168.10.1
+
+# command setup
+IPMI_POWER="/sbin/ipmipower"
+SSH_CMD="ssh -o ConnectTimeout=3 -o BatchMode=yes -o StrictHostKeyChecking=no"
+LOGGER="/bin/logger -p local3.crit -t scoutfs-fenced"
+
+$LOGGER "ipmi fence script invoked: IP: $SCOUTFS_FENCED_REQ_IP RID: $SCOUTFS_FENCED_REQ_RID TEST: $IPMITEST"
+
+echo_fail() {
+    echo "$@" >&2
+    $LOGGER "fence failed: $@"
+    exit 1
+}
+
+echo_log() {
+    echo "$@" >&2
+    $LOGGER "fence info: $@"
+}
+
+echo_test_pass() {
+    echo -e "\xE2\x9C\x94 $@"
+}
+
+echo_test_fail() {
+    echo -e "\xE2\x9D\x8C $@"
+}
+
+test -n "$SCOUTFS_IPMI_CONFIG_FILE" || \
+    echo_fail "SCOUTFS_IPMI_CONFIG_FILE isn't set"
+test -r "$SCOUTFS_IPMI_CONFIG_FILE" || \
+    echo_fail "$SCOUTFS_IPMI_CONFIG_FILE isn't readable file"
+. "$SCOUTFS_IPMI_CONFIG_FILE"
+test -n "$SCOUTFS_IPMI_HOSTS_FILE" || \
+    echo_fail "SCOUTFS_IPMI_HOSTS_FILE isn't set"
+test -r "$SCOUTFS_IPMI_HOSTS_FILE" || \
+    echo_fail "$SCOUTFS_IPMI_HOSTS_FILE isn't readable file"
+test -x "$IPMI_POWER" || \
+    echo_fail "$IPMI_POWER not found, need to install freeimpi?"
+
+export ip="$SCOUTFS_FENCED_REQ_IP"
+export rid="$SCOUTFS_FENCED_REQ_RID"
+
+getIPMIhost () {
+    host=$(awk -v ip="$1" '$1 == ip {print $2}' "$SCOUTFS_IPMI_HOSTS_FILE") || \
+        echo_fail "lookup ipmi host failed"
+    echo "$host"
+}
+
+powerOffHost() {
+    # older versions of ipmipower inverted wait-until-off/wait-until-on, so specify both
+    $IPMI_POWER $IPMI_OPTS -h "$1" --wait-until-off --wait-until-on --off || \
+        echo_fail "ipmi power off $1 failed"
+
+    ipmioutput=$($IPMI_POWER $IPMI_OPTS -h "$1" --stat) || \
+        echo_fail "ipmi power stat $1 failed"
+
+    if [[ ! "$ipmioutput" =~ off ]]; then
+        echo_fail "ipmi stat $1 not off"
+    fi
+
+    $LOGGER "ipmi fence power down $1 success"
+
+    exit 0
+}
+
+if [ -n "$IPMITEST" ]; then
+    for i in $(awk '!/^($|[[:space:]]*#)/ {print $1}' "$SCOUTFS_IPMI_HOSTS_FILE"); do
+        if ! $SSH_CMD "$i" /bin/true; then
+            echo_test_fail "ssh $i"
+        else
+            echo_test_pass "ssh $i"
+        fi
+	host=$(getIPMIhost "$i")
+        if [ -z "$host" ]; then
+            echo_test_fail "ipmi config $i $host"
+        else
+            if ! $IPMI_POWER $IPMI_OPTS -h "$host" --stat; then
+                echo_test_fail "ipmi $i"
+            else
+                echo_test_pass "ipmi $i"
+            fi
+        fi
+    done
+    exit 0
+fi
+
+if [ -z "$ip" ]; then
+    echo_fail "no IP given for fencing"
+fi
+
+host=$(getIPMIhost "$ip")
+if [ -z "$host" ]; then
+    echo_fail "no IPMI host found for fence IP"
+fi
+
+# first check via ssh if the mount still exists
+# if ssh succeeds, we will only power down the node if mounted
+if ! output=$($SSH_CMD "$ip" "echo BEGIN; LC_ALL=C egrep -m 1 '(^0x*|^$rid$)' /sys/kernel/boot_params/version /sys/fs/scoutfs/f*r*/rid; echo END"); then
+    # ssh not working, just power down host
+    powerOffHost "$host"
+fi
+
+if [[ ! "$output" =~ BEGIN ]]; then
+    # ssh failure
+    echo_log "no BEGIN"
+    powerOffHost "$host"
+fi
+
+if [[ ! "$output" =~ \/boot_params\/ ]]; then
+    # ssh failure
+    echo_log "no boot params"
+    powerOffHost "$host"
+fi
+
+if [[ ! "$output" =~ END ]]; then
+    # ssh failure
+    echo_log "no END"
+    powerOffHost "$host"
+fi
+
+if [[ "$output" =~ "rid:$rid" ]]; then
+    # rid still mounted, power down
+    echo_log "rid $rid still mounted"
+    powerOffHost "$host"
+fi
+
+$LOGGER "ipmi fence host $ip/$host success (rid $rid not mounted)"
+exit 0
+

utils/fenced/local-force-unmount

@@ -0,0 +1,36 @@
+#!/usr/bin/bash
+# /usr/libexec/scoutfs-fenced/run/local-force-umount
+
+echo_fail() {
+	echo "$@" > /dev/stderr
+	exit 1
+}
+
+rid="$SCOUTFS_FENCED_REQ_RID"
+
+#
+# Look for a local mount with the rid to fence.  Typically we'll at
+# least find the mount with the server that requested the fence that
+# we're processing.   But it's possible that mounts are unmounted
+# before, or while, we're running.
+#
+mnts=$(findmnt -l -n -t scoutfs -o TARGET) || \
+	echo_fail "findmnt -t scoutfs failed" > /dev/stderr
+
+for mnt in $mnts; do
+	mnt_rid=$(scoutfs statfs -p "$mnt" -s rid) || \
+		echo_fail "scoutfs statfs $mnt failed"
+
+	if [ "$mnt_rid" == "$rid" ]; then
+		umount -f "$mnt" || \
+			echo_fail "umout -f $mnt"
+
+		exit 0
+	fi
+done
+
+#
+# If the mount doesn't exist on this host then it can't access the
+# devices by definition and can be considered fenced.
+#
+exit 0

utils/fenced/powerman-remote-host

@@ -0,0 +1,139 @@
+#!/usr/bin/bash
+# /usr/libexec/scoutfs-fenced/run/powerman-remote-host
+
+# powerman configuration
+SCOUTFS_PM_CONFIG_FILE=${SCOUTFS_PM_CONFIG_FILE:-/etc/scoutfs/scoutfs-pm.conf}
+SCOUTFS_PM_HOSTS_FILE=${SCOUTFS_PM_HOSTS_FILE:-/etc/scoutfs/scoutfs-pm-hosts.conf}
+
+## hosts file format
+## SCOUTFS_HOST_IP POWERMAN_NODE_NAME
+## ex:
+#  192.168.1.1     dm1
+
+# command setup
+PM_CMD="/usr/bin/pm"
+SSH_CMD="ssh -o ConnectTimeout=3 -o BatchMode=yes -o StrictHostKeyChecking=no"
+LOGGER="/bin/logger -p local3.crit -t scoutfs-fenced"
+
+$LOGGER "ipmi fence script invoked: IP: $SCOUTFS_FENCED_REQ_IP RID: $SCOUTFS_FENCED_REQ_RID TEST: $IPMITEST"
+
+echo_fail() {
+    echo "$@" >&2
+    $LOGGER "fence failed: $@"
+    exit 1
+}
+
+echo_log() {
+    echo "$@" >&2
+    $LOGGER "fence info: $@"
+}
+
+echo_test_pass() {
+    echo -e "\xE2\x9C\x94 $@"
+}
+
+echo_test_fail() {
+    echo -e "\xE2\x9D\x8C $@"
+}
+
+test -n "$SCOUTFS_PM_CONFIG_FILE" || \
+    echo_fail "SCOUTFS_PM_CONFIG_FILE isn't set"
+test -r "$SCOUTFS_PM_CONFIG_FILE" || \
+    echo_fail "$SCOUTFS_PM_CONFIG_FILE isn't readable file"
+. "$SCOUTFS_PM_CONFIG_FILE"
+test -n "$SCOUTFS_PM_HOSTS_FILE" || \
+    echo_fail "SCOUTFS_PM_HOSTS_FILE isn't set"
+test -r "$SCOUTFS_PM_HOSTS_FILE" || \
+    echo_fail "$SCOUTFS_PM_HOSTS_FILE isn't readable file"
+test -x "$PM_CMD" || \
+    echo_fail "$PMCMD not found, need to install powerman?"
+
+export ip="$SCOUTFS_FENCED_REQ_IP"
+fence_rid="$SCOUTFS_FENCED_REQ_RID"
+
+getPMhost () {
+    host=$(awk -v ip="$1" '$1 == ip {print $2}' "$SCOUTFS_PM_HOSTS_FILE") || \
+        echo_fail "lookup pm host failed"
+    echo "$host"
+}
+
+powerOffHost() {
+    $PM_CMD $PM_OPTS "$1" -0 || \
+        echo_fail "pm power off $host failed"
+
+    pmoutput=$($PM_CMD $PM_OPTS "$1" -q | grep "$1") || \
+        echo_fail "powerman power stat $1 failed"
+
+    if [[ ! "$pmoutput" =~ off ]]; then
+        echo_fail "powerman stat $1 not off"
+    fi
+
+    $LOGGER "powerman fence power down $1 success"
+
+    exit 0
+}
+
+if [ -n "$PMTEST" ]; then
+    for i in $(awk '!/^($|[[:space:]]*#)/ {print $1}' "$SCOUTFS_PM_HOSTS_FILE"); do
+        if ! $SSH_CMD "$i" /bin/true; then
+            echo_test_fail "ssh $i"
+        else
+            echo_test_pass "ssh $i"
+        fi
+	host=$(getPMhost "$i")
+        if [ -z "$host" ]; then
+            echo_test_fail "pm config $i $host"
+        else
+            if ! $PM_CMD $PM_OPTS "$host" -q; then
+                echo_test_fail "pm $i"
+            else
+                echo_test_pass "pm $i"
+            fi
+        fi
+    done
+    exit 0
+fi
+
+if [ -z "$ip" ]; then
+    echo_fail "no IP given for fencing"
+fi
+
+host=$(getPMhost "$ip")
+if [ -z "$host" ]; then
+    echo_fail "no host found for fence IP"
+fi
+
+# first check via ssh if the mount still exists
+# if ssh succeeds, we will only power down the node if mounted
+if ! output=$($SSH_CMD "$ip" "echo BEGIN; LC_ALL=C egrep -m 1 '(^0x*|^$rid$)' /sys/kernel/boot_params/version /sys/fs/scoutfs/f*r*/rid; echo END"); then
+    # ssh not working, just power down host
+    powerOffHost "$host"
+fi
+
+if [[ ! "$output" =~ BEGIN ]]; then
+    # ssh failure
+    echo_log "no BEGIN"
+    powerOffHost "$host"
+fi
+
+if [[ ! "$output" =~ \/boot_params\/ ]]; then
+    # ssh failure
+    echo_log "no boot params"
+    powerOffHost "$host"
+fi
+
+if [[ ! "$output" =~ END ]]; then
+    # ssh failure
+    echo_log "no END"
+    powerOffHost "$host"
+fi
+
+if [[ "$output" =~ "rid:$rid" ]]; then
+    # rid still mounted, power down
+    echo_log "rid $rid still mounted"
+    powerOffHost "$host"
+fi
+
+$LOGGER "powerman fence host $ip/$host success (rid $rid not mounted)"
+exit 0
+

utils/fenced/scoutfs-ipmi-hosts.conf

@@ -0,0 +1,11 @@
+# /etc/scoutfs/scoutfs-ipmi-hosts.conf
+
+## config file format
+##
+## SCOUTFS_HOST_IP must match the interface used for scoutfs
+## leader/follower communications
+##
+## SCOUTFS_HOST_IP IPMI_ADDRESS
+## ex:
+#192.168.1.1     192.168.10.1
+

utils/fenced/scoutfs-ipmi.conf

@@ -0,0 +1,10 @@
+#!/usr/bin/bash
+# /etc/scoutfs/scoutfs-ipmi.conf
+
+IPMI_USER="admin"
+IPMI_PASSWORD="password"
+IPMI_OPTS="-D LAN_2_0 -u $IPMI_USER -p $IPMI_PASSWORD"
+
+# some Intel BMCs need -I 17
+# IPMI_OPTS="-D LAN_2_0 -u $IPMI_USER -p $IPMI_PASSWORD -I 17"
+

utils/fenced/scoutfs-pm-hosts.conf

@@ -0,0 +1,11 @@
+# /etc/scoutfs/scoutfs-ipmi-hosts.conf
+
+## config file format
+##
+## SCOUTFS_HOST_IP must match the interface used for scoutfs
+## leader/follower communications
+##
+## SCOUTFS_HOST_IP POWERMAN_NODE_NAME
+## ex:
+#192.168.1.1     node1
+

utils/fenced/scoutfs-pm.conf

@@ -0,0 +1,8 @@
+#!/usr/bin/bash
+# /etc/scoutfs/scoutfs-pm.conf
+
+PM_OPTS=""
+
+# optionally specify remote powerman server
+#PM_OPTS="-h pm-server.localdomain"
+

utils/scoutfs-utils.spec.in

@@ -55,14 +55,21 @@ install -m 755 -D src/scoutfs $RPM_BUILD_ROOT%{_sbindir}/scoutfs
 install -m 644 -D src/ioctl.h $RPM_BUILD_ROOT%{_includedir}/scoutfs/ioctl.h
 install -m 644 -D src/format.h $RPM_BUILD_ROOT%{_includedir}/scoutfs/format.h
 install -m 755 -D fenced/scoutfs-fenced $RPM_BUILD_ROOT%{_libexecdir}/scoutfs-fenced/scoutfs-fenced
+install -m 755 -D fenced/local-force-unmount $RPM_BUILD_ROOT%{_libexecdir}/scoutfs-fenced/run/local-force-unmount
+install -m 755 -D fenced/ipmi-remote-host $RPM_BUILD_ROOT%{_libexecdir}/scoutfs-fenced/run/ipmi-remote-host
+install -m 755 -D fenced/powerman-remote-host $RPM_BUILD_ROOT%{_libexecdir}/scoutfs-fenced/run/powerman-remote-host
 install -m 644 -D fenced/scoutfs-fenced.service $RPM_BUILD_ROOT%{_unitdir}/scoutfs-fenced.service
 install -m 644 -D fenced/scoutfs-fenced.conf.example $RPM_BUILD_ROOT%{_sysconfdir}/scoutfs/scoutfs-fenced.conf.example
+install -m 644 -D fenced/scoutfs-ipmi.conf $RPM_BUILD_ROOT%{_sysconfdir}/scoutfs/scoutfs-ipmi.conf
+install -m 644 -D fenced/scoutfs-ipmi-hosts.conf $RPM_BUILD_ROOT%{_sysconfdir}/scoutfs/scoutfs-ipmi-hosts.conf
+install -m 644 -D fenced/scoutfs-pm.conf $RPM_BUILD_ROOT%{_sysconfdir}/scoutfs/scoutfs-pm.conf
+install -m 644 -D fenced/scoutfs-pm-hosts.conf $RPM_BUILD_ROOT%{_sysconfdir}/scoutfs/scoutfs-pm-hosts.conf
 
 %files
 %defattr(644,root,root,755)
 %{_mandir}/man*/scoutfs*.gz
 %{_unitdir}/scoutfs-fenced.service
-%{_sysconfdir}/scoutfs
+%config(noreplace) %{_sysconfdir}/scoutfs
 %defattr(755,root,root,755)
 %{_sbindir}/scoutfs
 %{_libexecdir}/scoutfs-fenced