diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 5fc62f19..8859e646 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -85,6 +85,7 @@ EXPAND_COUNTER(quorum_write_block) \ EXPAND_COUNTER(quorum_write_block_error) \ EXPAND_COUNTER(quorum_fenced) \ + EXPAND_COUNTER(trans_commit_data_alloc_low) \ EXPAND_COUNTER(trans_commit_fsync) \ EXPAND_COUNTER(trans_commit_full) \ EXPAND_COUNTER(trans_commit_sync_fs) \ diff --git a/kmod/src/data.c b/kmod/src/data.c index 39f368ce..f83089e9 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -2017,6 +2017,16 @@ void scoutfs_data_get_btrees(struct super_block *sb, up_read(&datinf->alloc_rwsem); } +/* + * This isn't serializing with allocators so it can be a bit racey. + */ +u64 scoutfs_data_alloc_free_bytes(struct super_block *sb) +{ + DECLARE_DATA_INFO(sb, datinf); + + return scoutfs_radix_root_free_bytes(sb, &datinf->data_avail); +} + int scoutfs_data_setup(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); diff --git a/kmod/src/data.h b/kmod/src/data.h index 8b58d4f6..9e0a9c87 100644 --- a/kmod/src/data.h +++ b/kmod/src/data.h @@ -78,6 +78,7 @@ void scoutfs_data_init_btrees(struct super_block *sb, struct scoutfs_log_trees *lt); void scoutfs_data_get_btrees(struct super_block *sb, struct scoutfs_log_trees *lt); +u64 scoutfs_data_alloc_free_bytes(struct super_block *sb); int scoutfs_data_setup(struct super_block *sb); void scoutfs_data_destroy(struct super_block *sb); diff --git a/kmod/src/radix.c b/kmod/src/radix.c index c9d51a73..15d80e19 100644 --- a/kmod/src/radix.c +++ b/kmod/src/radix.c @@ -1526,6 +1526,12 @@ void scoutfs_radix_root_init(struct super_block *sb, init_ref(&root->ref, 0, false); } +u64 scoutfs_radix_root_free_bytes(struct super_block *sb, + struct scoutfs_radix_root *root) +{ + return le64_to_cpu(root->ref.sm_total) << SCOUTFS_BLOCK_SHIFT; +} + /* * The first bit nr in a leaf containing the bit, used by callers to * identify regions that span leafs and would need to be freed in diff --git a/kmod/src/radix.h b/kmod/src/radix.h index 982797e7..0ca79431 100644 --- a/kmod/src/radix.h +++ b/kmod/src/radix.h @@ -38,6 +38,8 @@ void scoutfs_radix_init_alloc(struct scoutfs_radix_allocator *alloc, struct scoutfs_radix_root *freed); void scoutfs_radix_root_init(struct super_block *sb, struct scoutfs_radix_root *root, bool meta); +u64 scoutfs_radix_root_free_bytes(struct super_block *sb, + struct scoutfs_radix_root *root); u64 scoutfs_radix_bit_leaf_nr(u64 bit); #endif diff --git a/kmod/src/server.c b/kmod/src/server.c index 0bdbedd3..a371748f 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -35,6 +35,7 @@ #include "lock_server.h" #include "endian_swap.h" #include "quorum.h" +#include "trans.h" /* * Every active mount can act as the server that listens on a net @@ -412,7 +413,7 @@ static int server_get_log_trees(struct super_block *sb, } /* ensure client has enough free data blocks for a transaction */ - target = (2ULL*1024*1024*1024) / SCOUTFS_BLOCK_SIZE; + target = SCOUTFS_TRANS_DATA_ALLOC_HWM / SCOUTFS_BLOCK_SIZE; if (le64_to_cpu(ltv.data_avail.ref.sm_total) < target) { count = target - le64_to_cpu(ltv.data_avail.ref.sm_total); diff --git a/kmod/src/trans.c b/kmod/src/trans.c index a5017821..a3467aa6 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -315,6 +315,13 @@ struct scoutfs_reservation { * we piggy back on their hold. We wait if the writer is trying to * write out the transation. And if our items won't fit then we kick off * a write. + * + * This is called as a condition for wait_event. It is very limited in + * the locking (blocking) it can do because the caller has set the task + * state before testing the condition safely race with waking after + * setting the condition. Our checking the amount of dirty metadata + * blocks and free data blocks is racy, but we don't mind the risk of + * delaying or prematurely forcing commits. */ static bool acquired_hold(struct super_block *sb, struct scoutfs_reservation *rsv, @@ -354,6 +361,13 @@ static bool acquired_hold(struct super_block *sb, goto out; } + /* Try to refill data allocator before premature enospc */ + if (scoutfs_data_alloc_free_bytes(sb) <= SCOUTFS_TRANS_DATA_ALLOC_LWM) { + scoutfs_inc_counter(sb, trans_commit_data_alloc_low); + queue_trans_work(sbi); + goto out; + } + tri->reserved_items = items; tri->reserved_vals = vals; diff --git a/kmod/src/trans.h b/kmod/src/trans.h index e5f3228e..014a35e8 100644 --- a/kmod/src/trans.h +++ b/kmod/src/trans.h @@ -1,6 +1,11 @@ #ifndef _SCOUTFS_TRANS_H_ #define _SCOUTFS_TRANS_H_ +/* the server will attempt to fill data allocs for each trans */ +#define SCOUTFS_TRANS_DATA_ALLOC_HWM (2ULL * 1024 * 1024 * 1024) +/* the client will force commits if data allocators get too low */ +#define SCOUTFS_TRANS_DATA_ALLOC_LWM (256ULL * 1024 * 1024) + #include "count.h" void scoutfs_trans_write_func(struct work_struct *work);