From acafb869e7312302a45eb943fffc3e256befbdf4 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 15 Jun 2023 14:45:26 -0700 Subject: [PATCH 1/3] Avoid deadlock from block reclaim in rht resize The RCU hash table uses deferred work to resize the hash table. There's a time during resize when hash table iteration will return EAGAIN until resize makes more progress. During this time resize can perform GFP_KERNEL allocations. Our shrinker tries to iterate over its RCU hash table to find blocks to reclaim. It tries to restart iteration if it gets EAGAIN on the assumption that it will be usable again soon. Combine the two and our shrinker can get stuck retrying iteration indefinitely because it's shrinking on behalf of the hash table resizing that is trying to allocate the next table before making iteration work again. We have to stop shrinking in this case so that the resizing caller can proceed. Signed-off-by: Zach Brown --- kmod/src/block.c | 24 +++++++++++++++--------- kmod/src/counters.h | 2 +- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/kmod/src/block.c b/kmod/src/block.c index 6849fe5c..89cda9c0 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -1096,6 +1096,7 @@ static int block_shrink(struct shrinker *shrink, struct shrink_control *sc) struct super_block *sb = binf->sb; struct rhashtable_iter iter; struct block_private *bp; + bool stop = false; unsigned long nr; u64 recently; @@ -1107,7 +1108,6 @@ static int block_shrink(struct shrinker *shrink, struct shrink_control *sc) nr = DIV_ROUND_UP(nr, SCOUTFS_BLOCK_LG_PAGES_PER); -restart: recently = accessed_recently(binf); rhashtable_walk_enter(&binf->ht, &iter); rhashtable_walk_start(&iter); @@ -1129,12 +1129,15 @@ restart: if (bp == NULL) break; if (bp == ERR_PTR(-EAGAIN)) { - /* hard exit to wait for rcu rebalance to finish */ - rhashtable_walk_stop(&iter); - rhashtable_walk_exit(&iter); - scoutfs_inc_counter(sb, block_cache_shrink_restart); - synchronize_rcu(); - goto restart; + /* + * We can be called from reclaim in the allocation + * to resize the hash table itself. We have to + * return so that the caller can proceed and + * enable hash table iteration again. + */ + scoutfs_inc_counter(sb, block_cache_shrink_stop); + stop = true; + break; } scoutfs_inc_counter(sb, block_cache_shrink_next); @@ -1157,8 +1160,11 @@ restart: rhashtable_walk_stop(&iter); rhashtable_walk_exit(&iter); out: - return min_t(u64, (u64)atomic_read(&binf->total_inserted) * SCOUTFS_BLOCK_LG_PAGES_PER, - INT_MAX); + if (stop) + return -1; + else + return min_t(u64, INT_MAX, + (u64)atomic_read(&binf->total_inserted) * SCOUTFS_BLOCK_LG_PAGES_PER); } struct sm_block_completion { diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 4784e73d..e681e07a 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -34,7 +34,7 @@ EXPAND_COUNTER(block_cache_shrink_next) \ EXPAND_COUNTER(block_cache_shrink_recent) \ EXPAND_COUNTER(block_cache_shrink_remove) \ - EXPAND_COUNTER(block_cache_shrink_restart) \ + EXPAND_COUNTER(block_cache_shrink_stop) \ EXPAND_COUNTER(btree_compact_values) \ EXPAND_COUNTER(btree_compact_values_enomem) \ EXPAND_COUNTER(btree_delete) \ From 05371b83f025801fe6bde35eb4e72abb5f8f1b8f Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 16 Jun 2023 09:37:37 -0700 Subject: [PATCH 2/3] Update expected console messages during testing Signed-off-by: Zach Brown --- tests/funcs/filter.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/funcs/filter.sh b/tests/funcs/filter.sh index 121b51ca..766ab562 100644 --- a/tests/funcs/filter.sh +++ b/tests/funcs/filter.sh @@ -18,6 +18,7 @@ t_filter_dmesg() # the kernel can just be noisy re=" used greatest stack depth: " + re="$re|sched: RT throttling activated" # mkfs/mount checks partition tables re="$re|unknown partition table" From 89b238a5c454e2cbb45b23f1e03cc9b7945973d6 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 16 Jun 2023 09:38:58 -0700 Subject: [PATCH 3/3] Add more acceptable quorum delay during testing Loaded VMs can see a few more seconds delay. Signed-off-by: Zach Brown --- tests/tests/quorum-heartbeat-timeout.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests/quorum-heartbeat-timeout.sh b/tests/tests/quorum-heartbeat-timeout.sh index 30563192..e34a6246 100644 --- a/tests/tests/quorum-heartbeat-timeout.sh +++ b/tests/tests/quorum-heartbeat-timeout.sh @@ -77,7 +77,7 @@ test_timeout() # make sure the new leader delay was reasonable, allowing for some slack low=$((to - 1000)) - high=$((to + 3000)) + high=$((to + 5000)) # make sure the new leader delay was reasonable test "$delay" -lt "$low" && t_fail "delay $delay < low $low (to $to)"