Compare commits

..

1 Commits

Author SHA1 Message Date
Ben McClelland b898c89c11 add local,ipmi,powerman fenced scripts to utils rpm
This adds the fenced scripts so that we have a place to track these and
get updates out to users. This latest version of scripts has the checks
to validate that power off succeeded and not just assume based on power
command return status.

Signed-off-by: Ben McClelland <ben.mcclelland@versity.com>
2022-07-06 15:11:21 -07:00
14 changed files with 389 additions and 94 deletions
-16
View File
@@ -1,22 +1,6 @@
Versity ScoutFS Release Notes
=============================
---
v1.6
\
*Jul 7, 2022*
* **Fix memory leaks in rare corner cases**
\
Analysis tools found a few corner cases that leaked small structures,
generally around error handling or startup and shutdown.
* **Add --skip-likely-huge scoutfs print command option**
\
Add an option to scoutfs print to reduce the size of the output
so that it can be used to see system-wide metadata without being
overwhelmed by file-level details.
---
v1.5
\
+4 -7
View File
@@ -355,7 +355,6 @@ static int submit_send(struct super_block *sb,
}
if (rid != 0) {
spin_unlock(&conn->lock);
kfree(msend);
return -ENOTCONN;
}
}
@@ -1346,12 +1345,10 @@ scoutfs_net_alloc_conn(struct super_block *sb,
if (!conn)
return NULL;
if (info_size) {
conn->info = kzalloc(info_size, GFP_NOFS);
if (!conn->info) {
kfree(conn);
return NULL;
}
conn->info = kzalloc(info_size, GFP_NOFS);
if (!conn->info) {
kfree(conn);
return NULL;
}
conn->workq = alloc_workqueue("scoutfs_net_%s",
-17
View File
@@ -157,15 +157,6 @@ static int free_rid(struct omap_rid_list *list, struct omap_rid_entry *entry)
return nr;
}
static void free_rid_list(struct omap_rid_list *list)
{
struct omap_rid_entry *entry;
struct omap_rid_entry *tmp;
list_for_each_entry_safe(entry, tmp, &list->head, head)
free_rid(list, entry);
}
static int copy_rids(struct omap_rid_list *to, struct omap_rid_list *from, spinlock_t *from_lock)
{
struct omap_rid_entry *entry;
@@ -813,10 +804,6 @@ void scoutfs_omap_server_shutdown(struct super_block *sb)
llist_for_each_entry_safe(req, tmp, requests, llnode)
kfree(req);
spin_lock(&ominf->lock);
free_rid_list(&ominf->rids);
spin_unlock(&ominf->lock);
synchronize_rcu();
}
@@ -877,10 +864,6 @@ void scoutfs_omap_destroy(struct super_block *sb)
rhashtable_walk_stop(&iter);
rhashtable_walk_exit(&iter);
spin_lock(&ominf->lock);
free_rid_list(&ominf->rids);
spin_unlock(&ominf->lock);
rhashtable_destroy(&ominf->group_ht);
rhashtable_destroy(&ominf->req_ht);
kfree(ominf);
+2 -2
View File
@@ -496,7 +496,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent)
ret = assign_random_id(sbi);
if (ret < 0)
goto out;
return ret;
spin_lock_init(&sbi->next_ino_lock);
spin_lock_init(&sbi->data_wait_root.lock);
@@ -505,7 +505,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent)
/* parse options early for use during setup */
ret = scoutfs_options_early_setup(sb, data);
if (ret < 0)
goto out;
return ret;
scoutfs_options_read(sb, &opts);
ret = sb_set_blocksize(sb, SCOUTFS_BLOCK_SM_SIZE);
+140
View File
@@ -0,0 +1,140 @@
#!/usr/bin/bash
# /usr/libexec/scoutfs-fenced/run/ipmi-remote-host
# ipmi configuration
SCOUTFS_IPMI_CONFIG_FILE=${SCOUTFS_IPMI_CONFIG_FILE:-/etc/scoutfs/scoutfs-ipmi.conf}
SCOUTFS_IPMI_HOSTS_FILE=${SCOUTFS_IPMI_HOSTS_FILE:-/etc/scoutfs/scoutfs-ipmi-hosts.conf}
## hosts file format
## SCOUTFS_HOST_IP IPMI_ADDRESS
## ex:
# 192.168.1.1 192.168.10.1
# command setup
IPMI_POWER="/sbin/ipmipower"
SSH_CMD="ssh -o ConnectTimeout=3 -o BatchMode=yes -o StrictHostKeyChecking=no"
LOGGER="/bin/logger -p local3.crit -t scoutfs-fenced"
$LOGGER "ipmi fence script invoked: IP: $SCOUTFS_FENCED_REQ_IP RID: $SCOUTFS_FENCED_REQ_RID TEST: $IPMITEST"
echo_fail() {
echo "$@" >&2
$LOGGER "fence failed: $@"
exit 1
}
echo_log() {
echo "$@" >&2
$LOGGER "fence info: $@"
}
echo_test_pass() {
echo -e "\xE2\x9C\x94 $@"
}
echo_test_fail() {
echo -e "\xE2\x9D\x8C $@"
}
test -n "$SCOUTFS_IPMI_CONFIG_FILE" || \
echo_fail "SCOUTFS_IPMI_CONFIG_FILE isn't set"
test -r "$SCOUTFS_IPMI_CONFIG_FILE" || \
echo_fail "$SCOUTFS_IPMI_CONFIG_FILE isn't readable file"
. "$SCOUTFS_IPMI_CONFIG_FILE"
test -n "$SCOUTFS_IPMI_HOSTS_FILE" || \
echo_fail "SCOUTFS_IPMI_HOSTS_FILE isn't set"
test -r "$SCOUTFS_IPMI_HOSTS_FILE" || \
echo_fail "$SCOUTFS_IPMI_HOSTS_FILE isn't readable file"
test -x "$IPMI_POWER" || \
echo_fail "$IPMI_POWER not found, need to install freeimpi?"
export ip="$SCOUTFS_FENCED_REQ_IP"
export rid="$SCOUTFS_FENCED_REQ_RID"
getIPMIhost () {
host=$(awk -v ip="$1" '$1 == ip {print $2}' "$SCOUTFS_IPMI_HOSTS_FILE") || \
echo_fail "lookup ipmi host failed"
echo "$host"
}
powerOffHost() {
# older versions of ipmipower inverted wait-until-off/wait-until-on, so specify both
$IPMI_POWER $IPMI_OPTS -h "$1" --wait-until-off --wait-until-on --off || \
echo_fail "ipmi power off $1 failed"
ipmioutput=$($IPMI_POWER $IPMI_OPTS -h "$1" --stat) || \
echo_fail "ipmi power stat $1 failed"
if [[ ! "$ipmioutput" =~ off ]]; then
echo_fail "ipmi stat $1 not off"
fi
$LOGGER "ipmi fence power down $1 success"
exit 0
}
if [ -n "$IPMITEST" ]; then
for i in $(awk '!/^($|[[:space:]]*#)/ {print $1}' "$SCOUTFS_IPMI_HOSTS_FILE"); do
if ! $SSH_CMD "$i" /bin/true; then
echo_test_fail "ssh $i"
else
echo_test_pass "ssh $i"
fi
host=$(getIPMIhost "$i")
if [ -z "$host" ]; then
echo_test_fail "ipmi config $i $host"
else
if ! $IPMI_POWER $IPMI_OPTS -h "$host" --stat; then
echo_test_fail "ipmi $i"
else
echo_test_pass "ipmi $i"
fi
fi
done
exit 0
fi
if [ -z "$ip" ]; then
echo_fail "no IP given for fencing"
fi
host=$(getIPMIhost "$ip")
if [ -z "$host" ]; then
echo_fail "no IPMI host found for fence IP"
fi
# first check via ssh if the mount still exists
# if ssh succeeds, we will only power down the node if mounted
if ! output=$($SSH_CMD "$ip" "echo BEGIN; LC_ALL=C egrep -m 1 '(^0x*|^$rid$)' /sys/kernel/boot_params/version /sys/fs/scoutfs/f*r*/rid; echo END"); then
# ssh not working, just power down host
powerOffHost "$host"
fi
if [[ ! "$output" =~ BEGIN ]]; then
# ssh failure
echo_log "no BEGIN"
powerOffHost "$host"
fi
if [[ ! "$output" =~ \/boot_params\/ ]]; then
# ssh failure
echo_log "no boot params"
powerOffHost "$host"
fi
if [[ ! "$output" =~ END ]]; then
# ssh failure
echo_log "no END"
powerOffHost "$host"
fi
if [[ "$output" =~ "rid:$rid" ]]; then
# rid still mounted, power down
echo_log "rid $rid still mounted"
powerOffHost "$host"
fi
$LOGGER "ipmi fence host $ip/$host success (rid $rid not mounted)"
exit 0
+36
View File
@@ -0,0 +1,36 @@
#!/usr/bin/bash
# /usr/libexec/scoutfs-fenced/run/local-force-umount
echo_fail() {
echo "$@" > /dev/stderr
exit 1
}
rid="$SCOUTFS_FENCED_REQ_RID"
#
# Look for a local mount with the rid to fence. Typically we'll at
# least find the mount with the server that requested the fence that
# we're processing. But it's possible that mounts are unmounted
# before, or while, we're running.
#
mnts=$(findmnt -l -n -t scoutfs -o TARGET) || \
echo_fail "findmnt -t scoutfs failed" > /dev/stderr
for mnt in $mnts; do
mnt_rid=$(scoutfs statfs -p "$mnt" -s rid) || \
echo_fail "scoutfs statfs $mnt failed"
if [ "$mnt_rid" == "$rid" ]; then
umount -f "$mnt" || \
echo_fail "umout -f $mnt"
exit 0
fi
done
#
# If the mount doesn't exist on this host then it can't access the
# devices by definition and can be considered fenced.
#
exit 0
+139
View File
@@ -0,0 +1,139 @@
#!/usr/bin/bash
# /usr/libexec/scoutfs-fenced/run/powerman-remote-host
# powerman configuration
SCOUTFS_PM_CONFIG_FILE=${SCOUTFS_PM_CONFIG_FILE:-/etc/scoutfs/scoutfs-pm.conf}
SCOUTFS_PM_HOSTS_FILE=${SCOUTFS_PM_HOSTS_FILE:-/etc/scoutfs/scoutfs-pm-hosts.conf}
## hosts file format
## SCOUTFS_HOST_IP POWERMAN_NODE_NAME
## ex:
# 192.168.1.1 dm1
# command setup
PM_CMD="/usr/bin/pm"
SSH_CMD="ssh -o ConnectTimeout=3 -o BatchMode=yes -o StrictHostKeyChecking=no"
LOGGER="/bin/logger -p local3.crit -t scoutfs-fenced"
$LOGGER "ipmi fence script invoked: IP: $SCOUTFS_FENCED_REQ_IP RID: $SCOUTFS_FENCED_REQ_RID TEST: $IPMITEST"
echo_fail() {
echo "$@" >&2
$LOGGER "fence failed: $@"
exit 1
}
echo_log() {
echo "$@" >&2
$LOGGER "fence info: $@"
}
echo_test_pass() {
echo -e "\xE2\x9C\x94 $@"
}
echo_test_fail() {
echo -e "\xE2\x9D\x8C $@"
}
test -n "$SCOUTFS_PM_CONFIG_FILE" || \
echo_fail "SCOUTFS_PM_CONFIG_FILE isn't set"
test -r "$SCOUTFS_PM_CONFIG_FILE" || \
echo_fail "$SCOUTFS_PM_CONFIG_FILE isn't readable file"
. "$SCOUTFS_PM_CONFIG_FILE"
test -n "$SCOUTFS_PM_HOSTS_FILE" || \
echo_fail "SCOUTFS_PM_HOSTS_FILE isn't set"
test -r "$SCOUTFS_PM_HOSTS_FILE" || \
echo_fail "$SCOUTFS_PM_HOSTS_FILE isn't readable file"
test -x "$PM_CMD" || \
echo_fail "$PMCMD not found, need to install powerman?"
export ip="$SCOUTFS_FENCED_REQ_IP"
fence_rid="$SCOUTFS_FENCED_REQ_RID"
getPMhost () {
host=$(awk -v ip="$1" '$1 == ip {print $2}' "$SCOUTFS_PM_HOSTS_FILE") || \
echo_fail "lookup pm host failed"
echo "$host"
}
powerOffHost() {
$PM_CMD $PM_OPTS "$1" -0 || \
echo_fail "pm power off $host failed"
pmoutput=$($PM_CMD $PM_OPTS "$1" -q | grep "$1") || \
echo_fail "powerman power stat $1 failed"
if [[ ! "$pmoutput" =~ off ]]; then
echo_fail "powerman stat $1 not off"
fi
$LOGGER "powerman fence power down $1 success"
exit 0
}
if [ -n "$PMTEST" ]; then
for i in $(awk '!/^($|[[:space:]]*#)/ {print $1}' "$SCOUTFS_PM_HOSTS_FILE"); do
if ! $SSH_CMD "$i" /bin/true; then
echo_test_fail "ssh $i"
else
echo_test_pass "ssh $i"
fi
host=$(getPMhost "$i")
if [ -z "$host" ]; then
echo_test_fail "pm config $i $host"
else
if ! $PM_CMD $PM_OPTS "$host" -q; then
echo_test_fail "pm $i"
else
echo_test_pass "pm $i"
fi
fi
done
exit 0
fi
if [ -z "$ip" ]; then
echo_fail "no IP given for fencing"
fi
host=$(getPMhost "$ip")
if [ -z "$host" ]; then
echo_fail "no host found for fence IP"
fi
# first check via ssh if the mount still exists
# if ssh succeeds, we will only power down the node if mounted
if ! output=$($SSH_CMD "$ip" "echo BEGIN; LC_ALL=C egrep -m 1 '(^0x*|^$rid$)' /sys/kernel/boot_params/version /sys/fs/scoutfs/f*r*/rid; echo END"); then
# ssh not working, just power down host
powerOffHost "$host"
fi
if [[ ! "$output" =~ BEGIN ]]; then
# ssh failure
echo_log "no BEGIN"
powerOffHost "$host"
fi
if [[ ! "$output" =~ \/boot_params\/ ]]; then
# ssh failure
echo_log "no boot params"
powerOffHost "$host"
fi
if [[ ! "$output" =~ END ]]; then
# ssh failure
echo_log "no END"
powerOffHost "$host"
fi
if [[ "$output" =~ "rid:$rid" ]]; then
# rid still mounted, power down
echo_log "rid $rid still mounted"
powerOffHost "$host"
fi
$LOGGER "powerman fence host $ip/$host success (rid $rid not mounted)"
exit 0
+11
View File
@@ -0,0 +1,11 @@
# /etc/scoutfs/scoutfs-ipmi-hosts.conf
## config file format
##
## SCOUTFS_HOST_IP must match the interface used for scoutfs
## leader/follower communications
##
## SCOUTFS_HOST_IP IPMI_ADDRESS
## ex:
#192.168.1.1 192.168.10.1
+10
View File
@@ -0,0 +1,10 @@
#!/usr/bin/bash
# /etc/scoutfs/scoutfs-ipmi.conf
IPMI_USER="admin"
IPMI_PASSWORD="password"
IPMI_OPTS="-D LAN_2_0 -u $IPMI_USER -p $IPMI_PASSWORD"
# some Intel BMCs need -I 17
# IPMI_OPTS="-D LAN_2_0 -u $IPMI_USER -p $IPMI_PASSWORD -I 17"
+11
View File
@@ -0,0 +1,11 @@
# /etc/scoutfs/scoutfs-ipmi-hosts.conf
## config file format
##
## SCOUTFS_HOST_IP must match the interface used for scoutfs
## leader/follower communications
##
## SCOUTFS_HOST_IP POWERMAN_NODE_NAME
## ex:
#192.168.1.1 node1
+8
View File
@@ -0,0 +1,8 @@
#!/usr/bin/bash
# /etc/scoutfs/scoutfs-pm.conf
PM_OPTS=""
# optionally specify remote powerman server
#PM_OPTS="-h pm-server.localdomain"
+1 -15
View File
@@ -597,7 +597,7 @@ format.
.PD
.TP
.BI "print {-S|--skip-likely-huge} META-DEVICE"
.BI "print META-DEVICE"
.sp
Prints out all of the metadata in the file system. This makes no effort
to ensure that the structures are consistent as they're traversed and
@@ -607,20 +607,6 @@ output.
.PD 0
.TP
.sp
.B "-S, --skip-likely-huge"
Skip printing structures that are likely to be very large. The
structures that are skipped tend to be global and whose size tends to be
related to the size of the volume. Examples of skipped structures include
the global fs items, srch files, and metadata and data
allocators. Similar structures that are not skipped are related to the
number of mounts and are maintained at a relatively reasonable size.
These include per-mount log trees, srch files, allocators, and the
metadata allocators used by server commits.
.sp
Skipping the larger structures limits the print output to a relatively
constant size rather than being a large multiple of the used metadata
space of the volume making the output much more useful for inspection.
.TP
.B "META-DEVICE"
The path to the metadata device for the filesystem whose metadata will be
printed. Since this command reads via the host's buffer cache, it may not
+8 -1
View File
@@ -55,14 +55,21 @@ install -m 755 -D src/scoutfs $RPM_BUILD_ROOT%{_sbindir}/scoutfs
install -m 644 -D src/ioctl.h $RPM_BUILD_ROOT%{_includedir}/scoutfs/ioctl.h
install -m 644 -D src/format.h $RPM_BUILD_ROOT%{_includedir}/scoutfs/format.h
install -m 755 -D fenced/scoutfs-fenced $RPM_BUILD_ROOT%{_libexecdir}/scoutfs-fenced/scoutfs-fenced
install -m 755 -D fenced/local-force-unmount $RPM_BUILD_ROOT%{_libexecdir}/scoutfs-fenced/run/local-force-unmount
install -m 755 -D fenced/ipmi-remote-host $RPM_BUILD_ROOT%{_libexecdir}/scoutfs-fenced/run/ipmi-remote-host
install -m 755 -D fenced/powerman-remote-host $RPM_BUILD_ROOT%{_libexecdir}/scoutfs-fenced/run/powerman-remote-host
install -m 644 -D fenced/scoutfs-fenced.service $RPM_BUILD_ROOT%{_unitdir}/scoutfs-fenced.service
install -m 644 -D fenced/scoutfs-fenced.conf.example $RPM_BUILD_ROOT%{_sysconfdir}/scoutfs/scoutfs-fenced.conf.example
install -m 644 -D fenced/scoutfs-ipmi.conf $RPM_BUILD_ROOT%{_sysconfdir}/scoutfs/scoutfs-ipmi.conf
install -m 644 -D fenced/scoutfs-ipmi-hosts.conf $RPM_BUILD_ROOT%{_sysconfdir}/scoutfs/scoutfs-ipmi-hosts.conf
install -m 644 -D fenced/scoutfs-pm.conf $RPM_BUILD_ROOT%{_sysconfdir}/scoutfs/scoutfs-pm.conf
install -m 644 -D fenced/scoutfs-pm-hosts.conf $RPM_BUILD_ROOT%{_sysconfdir}/scoutfs/scoutfs-pm-hosts.conf
%files
%defattr(644,root,root,755)
%{_mandir}/man*/scoutfs*.gz
%{_unitdir}/scoutfs-fenced.service
%{_sysconfdir}/scoutfs
%config(noreplace) %{_sysconfdir}/scoutfs
%defattr(755,root,root,755)
%{_sbindir}/scoutfs
%{_libexecdir}/scoutfs-fenced
+19 -36
View File
@@ -8,7 +8,6 @@
#include <errno.h>
#include <string.h>
#include <stdarg.h>
#include <stdbool.h>
#include <ctype.h>
#include <uuid/uuid.h>
#include <sys/socket.h>
@@ -990,10 +989,9 @@ static void print_super_block(struct scoutfs_super_block *super, u64 blkno)
struct print_args {
char *meta_device;
bool skip_likely_huge;
};
static int print_volume(int fd, struct print_args *args)
static int print_volume(int fd)
{
struct scoutfs_super_block *super = NULL;
struct print_recursion_args pa;
@@ -1043,26 +1041,23 @@ static int print_volume(int fd, struct print_args *args)
ret = err;
}
if (!args->skip_likely_huge) {
for (i = 0; i < array_size(super->meta_alloc); i++) {
snprintf(str, sizeof(str), "meta_alloc[%u]", i);
err = print_btree(fd, super, str, &super->meta_alloc[i].root,
print_alloc_item, NULL);
if (err && !ret)
ret = err;
}
err = print_btree(fd, super, "data_alloc", &super->data_alloc.root,
for (i = 0; i < array_size(super->meta_alloc); i++) {
snprintf(str, sizeof(str), "meta_alloc[%u]", i);
err = print_btree(fd, super, str, &super->meta_alloc[i].root,
print_alloc_item, NULL);
if (err && !ret)
ret = err;
}
err = print_btree(fd, super, "data_alloc", &super->data_alloc.root,
print_alloc_item, NULL);
if (err && !ret)
ret = err;
err = print_btree(fd, super, "srch_root", &super->srch_root,
print_srch_root_item, NULL);
if (err && !ret)
ret = err;
err = print_btree(fd, super, "logs_root", &super->logs_root,
print_log_trees_item, NULL);
if (err && !ret)
@@ -1070,23 +1065,19 @@ static int print_volume(int fd, struct print_args *args)
pa.super = super;
pa.fd = fd;
if (!args->skip_likely_huge) {
err = print_btree_leaf_items(fd, super, &super->srch_root.ref,
print_srch_root_files, &pa);
if (err && !ret)
ret = err;
}
err = print_btree_leaf_items(fd, super, &super->srch_root.ref,
print_srch_root_files, &pa);
if (err && !ret)
ret = err;
err = print_btree_leaf_items(fd, super, &super->logs_root.ref,
print_log_trees_roots, &pa);
if (err && !ret)
ret = err;
if (!args->skip_likely_huge) {
err = print_btree(fd, super, "fs_root", &super->fs_root,
print_fs_item, NULL);
if (err && !ret)
ret = err;
}
err = print_btree(fd, super, "fs_root", &super->fs_root,
print_fs_item, NULL);
if (err && !ret)
ret = err;
out:
free(super);
@@ -1107,7 +1098,7 @@ static int do_print(struct print_args *args)
return ret;
}
ret = print_volume(fd, args);
ret = print_volume(fd);
close(fd);
return ret;
};
@@ -1117,9 +1108,6 @@ static int parse_opt(int key, char *arg, struct argp_state *state)
struct print_args *args = state->input;
switch (key) {
case 'S':
args->skip_likely_huge = true;
break;
case ARGP_KEY_ARG:
if (!args->meta_device)
args->meta_device = strdup_or_error(state, arg);
@@ -1137,13 +1125,8 @@ static int parse_opt(int key, char *arg, struct argp_state *state)
return 0;
}
static struct argp_option options[] = {
{ "skip-likely-huge", 'S', NULL, 0, "Skip large structures to minimize output size"},
{ NULL }
};
static struct argp argp = {
options,
NULL,
parse_opt,
"META-DEV",
"Print metadata structures"