diff --git a/kmod/src/Makefile b/kmod/src/Makefile index db5f5576..ba8dc505 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -30,6 +30,7 @@ scoutfs-y += \ options.o \ per_task.o \ quorum.o \ + recov.o \ scoutfs_trace.o \ server.o \ sort_priv.o \ diff --git a/kmod/src/recov.c b/kmod/src/recov.c new file mode 100644 index 00000000..b0d894c2 --- /dev/null +++ b/kmod/src/recov.c @@ -0,0 +1,280 @@ +/* + * Copyright (C) 2021 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include + +#include "super.h" +#include "recov.h" + +/* + * There are a few server messages which can't be processed until they + * know that they have state for all possibly active clients. These + * little helpers track which clients have recovered what state and give + * those message handlers a call to check if recovery has completed. We + * track the timeout here, but all we do is call back into the server to + * take steps to evict timed out clients and then let us know that their + * recovery has finished. + */ + +struct recov_info { + struct super_block *sb; + spinlock_t lock; + struct list_head pending; + struct timer_list timer; + void (*timeout_fn)(struct super_block *); +}; + +#define DECLARE_RECOV_INFO(sb, name) \ + struct recov_info *name = SCOUTFS_SB(sb)->recov_info + +struct recov_pending { + struct list_head head; + u64 rid; + int which; +}; + +static struct recov_pending *find_pending(struct recov_info *recinf, u64 rid, int which) +{ + struct recov_pending *pend; + + list_for_each_entry(pend, &recinf->pending, head) { + if ((rid == 0 || pend->rid == rid) && (pend->which & which)) + return pend; + } + + return NULL; +} + +/* + * Record that we'll be waiting for a client to recover something. + * _finished will eventually be called for every _prepare, either + * because recovery naturally finished or because it timed out and the + * server evicted the client. + */ +int scoutfs_recov_prepare(struct super_block *sb, u64 rid, int which) +{ + DECLARE_RECOV_INFO(sb, recinf); + struct recov_pending *alloc; + struct recov_pending *pend; + + if (WARN_ON_ONCE(which & SCOUTFS_RECOV_INVALID)) + return -EINVAL; + + alloc = kmalloc(sizeof(*pend), GFP_NOFS); + if (!alloc) + return -ENOMEM; + + spin_lock(&recinf->lock); + + pend = find_pending(recinf, rid, SCOUTFS_RECOV_ALL); + if (pend) { + pend->which |= which; + } else { + swap(pend, alloc); + pend->rid = rid; + pend->which = which; + list_add(&pend->head, &recinf->pending); + } + + spin_unlock(&recinf->lock); + + kfree(alloc); + return 0; +} + +/* + * Recovery is only finished once we've begun (which sets the timer) and + * all clients have finished. If we didn't test the timer we could + * claim it finished prematurely as clients are being prepared. + */ +static int recov_finished(struct recov_info *recinf) +{ + return !!(recinf->timeout_fn != NULL && list_empty(&recinf->pending)); +} + +static void timer_callback(struct timer_list *timer) +{ + struct recov_info *recinf = from_timer(recinf, timer, timer); + + recinf->timeout_fn(recinf->sb); +} + +/* + * Begin waiting for recovery once we've prepared all the clients. If + * the timeout period elapses before _finish is called on all prepared + * clients then the timer will call the callback. + * + * Returns > 0 if all the prepared clients finish recovery before begin + * is called. + */ +int scoutfs_recov_begin(struct super_block *sb, void (*timeout_fn)(struct super_block *), + unsigned int timeout_ms) +{ + DECLARE_RECOV_INFO(sb, recinf); + int ret; + + spin_lock(&recinf->lock); + + recinf->timeout_fn = timeout_fn; + recinf->timer.expires = jiffies + msecs_to_jiffies(timeout_ms); + add_timer(&recinf->timer); + + ret = recov_finished(recinf); + + spin_unlock(&recinf->lock); + + if (ret > 0) + del_timer_sync(&recinf->timer); + + return ret; +} + +/* + * A given client has recovered the given state. If it's finished all + * recovery then we free it, and if all clients have finished recovery + * then we cancel the timeout timer. + * + * Returns > 0 if _begin has been called and all clients have finished. + * The caller will only see > 0 returned once. + */ +int scoutfs_recov_finish(struct super_block *sb, u64 rid, int which) +{ + DECLARE_RECOV_INFO(sb, recinf); + struct recov_pending *pend; + int ret = 0; + + spin_lock(&recinf->lock); + + pend = find_pending(recinf, rid, which); + if (pend) { + pend->which &= ~which; + if (pend->which) { + pend = NULL; + } else { + list_del(&pend->head); + ret = recov_finished(recinf); + } + } + + spin_unlock(&recinf->lock); + + if (ret > 0) + del_timer_sync(&recinf->timer); + + kfree(pend); + + return ret; +} + +/* + * Returns true if the given client is still trying to recover + * the given state. + */ +bool scoutfs_recov_is_pending(struct super_block *sb, u64 rid, int which) +{ + DECLARE_RECOV_INFO(sb, recinf); + bool is_pending; + + spin_lock(&recinf->lock); + is_pending = find_pending(recinf, rid, which) != NULL; + spin_unlock(&recinf->lock); + + return is_pending; +} + +/* + * Returns 0 if there are no rids waiting for the given state to be + * recovered. Returns the rid of a client still waiting if there are + * any, in no specified order. + * + * This is inherently racey. Callers are responsible for resolving any + * actions taken based on pending with the recovery finishing, perhaps + * before we return. + */ +u64 scoutfs_recov_next_pending(struct super_block *sb, int which) +{ + DECLARE_RECOV_INFO(sb, recinf); + struct recov_pending *pend; + u64 rid; + + spin_lock(&recinf->lock); + pend = find_pending(recinf, 0, which); + rid = pend ? pend->rid : 0; + spin_unlock(&recinf->lock); + + return rid; +} + +/* + * The server is shutting down and doesn't need to worry about recovery + * anymore. It'll be built up again by the next server, if needed. + */ +void scoutfs_recov_shutdown(struct super_block *sb) +{ + DECLARE_RECOV_INFO(sb, recinf); + struct recov_pending *pend; + struct recov_pending *tmp; + LIST_HEAD(list); + + del_timer_sync(&recinf->timer); + + spin_lock(&recinf->lock); + list_splice_init(&recinf->pending, &list); + recinf->timeout_fn = NULL; + spin_unlock(&recinf->lock); + + list_for_each_entry_safe(pend, tmp, &recinf->pending, head) { + list_del(&pend->head); + kfree(pend); + } +} + +int scoutfs_recov_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct recov_info *recinf; + int ret; + + recinf = kzalloc(sizeof(struct recov_info), GFP_KERNEL); + if (!recinf) { + ret = -ENOMEM; + goto out; + } + + recinf->sb = sb; + spin_lock_init(&recinf->lock); + INIT_LIST_HEAD(&recinf->pending); + timer_setup(&recinf->timer, timer_callback, 0); + + sbi->recov_info = recinf; + ret = 0; +out: + return ret; +} + +void scoutfs_recov_destroy(struct super_block *sb) +{ + DECLARE_RECOV_INFO(sb, recinf); + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + if (recinf) { + scoutfs_recov_shutdown(sb); + + kfree(recinf); + sbi->recov_info = NULL; + } +} diff --git a/kmod/src/recov.h b/kmod/src/recov.h new file mode 100644 index 00000000..cfdca30e --- /dev/null +++ b/kmod/src/recov.h @@ -0,0 +1,23 @@ +#ifndef _SCOUTFS_RECOV_H_ +#define _SCOUTFS_RECOV_H_ + +enum { + SCOUTFS_RECOV_GREETING = ( 1 << 0), + SCOUTFS_RECOV_LOCKS = ( 1 << 1), + + SCOUTFS_RECOV_INVALID = (~0 << 2), + SCOUTFS_RECOV_ALL = (~SCOUTFS_RECOV_INVALID), +}; + +int scoutfs_recov_prepare(struct super_block *sb, u64 rid, int which); +int scoutfs_recov_begin(struct super_block *sb, void (*timeout_fn)(struct super_block *), + unsigned int timeout_ms); +int scoutfs_recov_finish(struct super_block *sb, u64 rid, int which); +bool scoutfs_recov_is_pending(struct super_block *sb, u64 rid, int which); +u64 scoutfs_recov_next_pending(struct super_block *sb, int which); +void scoutfs_recov_shutdown(struct super_block *sb); + +int scoutfs_recov_setup(struct super_block *sb); +void scoutfs_recov_destroy(struct super_block *sb); + +#endif diff --git a/kmod/src/super.h b/kmod/src/super.h index 13912bdc..8d004db3 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -26,6 +26,7 @@ struct net_info; struct block_info; struct forest_info; struct srch_info; +struct recov_info; struct scoutfs_sb_info { struct super_block *sb; @@ -70,6 +71,7 @@ struct scoutfs_sb_info { struct lock_server_info *lock_server_info; struct client_info *client_info; struct server_info *server_info; + struct recov_info *recov_info; struct sysfs_info *sfsinfo; struct scoutfs_counters *counters;