diff --git a/kmod/src/Makefile b/kmod/src/Makefile index d96c4967..2666e6df 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -25,6 +25,7 @@ scoutfs-y += \ inode.o \ ioctl.o \ item.o \ + kernelcompat.o \ lock.o \ lock_server.o \ msg.o \ diff --git a/kmod/src/Makefile.kernelcompat b/kmod/src/Makefile.kernelcompat index 99615f18..409bdc27 100644 --- a/kmod/src/Makefile.kernelcompat +++ b/kmod/src/Makefile.kernelcompat @@ -130,3 +130,12 @@ endif ifneq (,$(shell grep 'bi_status' include/linux/blk_types.h)) ccflags-y += -DKC_BIO_BI_STATUS endif + +# +# v3.11-8765-ga0b02131c5fc +# +# Remove the old ->shrink() API, ->{scan,count}_objects is preferred. +# +ifneq (,$(shell grep '(*shrink)' include/linux/shrinker.h)) +ccflags-y += -DKC_SHRINKER_SHRINK +endif diff --git a/kmod/src/block.c b/kmod/src/block.c index 7016a4b4..c23764d3 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -31,6 +31,7 @@ #include "scoutfs_trace.h" #include "alloc.h" #include "triggers.h" +#include "util.h" /* * The scoutfs block cache manages metadata blocks that can be larger @@ -58,7 +59,7 @@ struct block_info { atomic64_t access_counter; struct rhashtable ht; wait_queue_head_t waitq; - struct shrinker shrinker; + KC_DEFINE_SHRINKER(shrinker); struct work_struct free_work; struct llist_head free_llist; }; @@ -1070,6 +1071,16 @@ u64 scoutfs_block_writer_dirty_bytes(struct super_block *sb, return wri->nr_dirty_blocks * SCOUTFS_BLOCK_LG_SIZE; } +static unsigned long block_count_objects(struct shrinker *shrink, struct shrink_control *sc) +{ + struct block_info *binf = KC_SHRINKER_CONTAINER_OF(shrink, struct block_info); + struct super_block *sb = binf->sb; + + scoutfs_inc_counter(sb, block_cache_count_objects); + + return shrinker_min_long(atomic_read(&binf->total_inserted)); +} + /* * Remove a number of cached blocks that haven't been used recently. * @@ -1090,24 +1101,18 @@ u64 scoutfs_block_writer_dirty_bytes(struct super_block *sb, * atomically remove blocks when the only references are ours and the * hash table. */ -static int block_shrink(struct shrinker *shrink, struct shrink_control *sc) +static unsigned long block_scan_objects(struct shrinker *shrink, struct shrink_control *sc) { - struct block_info *binf = container_of(shrink, struct block_info, - shrinker); + struct block_info *binf = KC_SHRINKER_CONTAINER_OF(shrink, struct block_info); struct super_block *sb = binf->sb; struct rhashtable_iter iter; struct block_private *bp; bool stop = false; - unsigned long nr; + unsigned long freed = 0; + unsigned long nr = sc->nr_to_scan; u64 recently; - nr = sc->nr_to_scan; - if (nr == 0) - goto out; - - scoutfs_inc_counter(sb, block_cache_shrink); - - nr = DIV_ROUND_UP(nr, SCOUTFS_BLOCK_LG_PAGES_PER); + scoutfs_inc_counter(sb, block_cache_scan_objects); recently = accessed_recently(binf); rhashtable_walk_enter(&binf->ht, &iter); @@ -1152,6 +1157,7 @@ static int block_shrink(struct shrinker *shrink, struct shrink_control *sc) if (block_remove_solo(sb, bp)) { scoutfs_inc_counter(sb, block_cache_shrink_remove); TRACE_BLOCK(shrink, bp); + freed++; nr--; } block_put(sb, bp); @@ -1160,12 +1166,11 @@ static int block_shrink(struct shrinker *shrink, struct shrink_control *sc) rhashtable_walk_stop(&iter); rhashtable_walk_exit(&iter); -out: + if (stop) - return -1; + return SHRINK_STOP; else - return min_t(u64, INT_MAX, - (u64)atomic_read(&binf->total_inserted) * SCOUTFS_BLOCK_LG_PAGES_PER); + return freed; } struct sm_block_completion { @@ -1291,9 +1296,9 @@ int scoutfs_block_setup(struct super_block *sb) atomic_set(&binf->total_inserted, 0); atomic64_set(&binf->access_counter, 0); init_waitqueue_head(&binf->waitq); - binf->shrinker.shrink = block_shrink; - binf->shrinker.seeks = DEFAULT_SEEKS; - register_shrinker(&binf->shrinker); + KC_INIT_SHRINKER_FUNCS(&binf->shrinker, block_count_objects, + block_scan_objects); + KC_REGISTER_SHRINKER(&binf->shrinker); INIT_WORK(&binf->free_work, block_free_work); init_llist_head(&binf->free_llist); @@ -1313,7 +1318,7 @@ void scoutfs_block_destroy(struct super_block *sb) struct block_info *binf = SCOUTFS_SB(sb)->block_info; if (binf) { - unregister_shrinker(&binf->shrinker); + KC_UNREGISTER_SHRINKER(&binf->shrinker); block_remove_all(sb); flush_work(&binf->free_work); rhashtable_destroy(&binf->ht); diff --git a/kmod/src/counters.h b/kmod/src/counters.h index ccfa1f6f..dd291816 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -30,6 +30,8 @@ EXPAND_COUNTER(block_cache_free) \ EXPAND_COUNTER(block_cache_free_work) \ EXPAND_COUNTER(block_cache_remove_stale) \ + EXPAND_COUNTER(block_cache_count_objects) \ + EXPAND_COUNTER(block_cache_scan_objects) \ EXPAND_COUNTER(block_cache_shrink) \ EXPAND_COUNTER(block_cache_shrink_next) \ EXPAND_COUNTER(block_cache_shrink_recent) \ @@ -88,6 +90,8 @@ EXPAND_COUNTER(forest_read_items) \ EXPAND_COUNTER(forest_roots_next_hint) \ EXPAND_COUNTER(forest_set_bloom_bits) \ + EXPAND_COUNTER(item_cache_count_objects) \ + EXPAND_COUNTER(item_cache_scan_objects) \ EXPAND_COUNTER(item_clear_dirty) \ EXPAND_COUNTER(item_create) \ EXPAND_COUNTER(item_delete) \ @@ -121,6 +125,7 @@ EXPAND_COUNTER(item_update) \ EXPAND_COUNTER(item_write_dirty) \ EXPAND_COUNTER(lock_alloc) \ + EXPAND_COUNTER(lock_count_objects) \ EXPAND_COUNTER(lock_free) \ EXPAND_COUNTER(lock_grant_request) \ EXPAND_COUNTER(lock_grant_response) \ @@ -134,6 +139,7 @@ EXPAND_COUNTER(lock_lock_error) \ EXPAND_COUNTER(lock_nonblock_eagain) \ EXPAND_COUNTER(lock_recover_request) \ + EXPAND_COUNTER(lock_scan_objects) \ EXPAND_COUNTER(lock_shrink_attempted) \ EXPAND_COUNTER(lock_shrink_aborted) \ EXPAND_COUNTER(lock_shrink_work) \ diff --git a/kmod/src/item.c b/kmod/src/item.c index 948fcd70..6d7bd376 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -27,6 +27,7 @@ #include "trans.h" #include "counters.h" #include "scoutfs_trace.h" +#include "util.h" /* * The item cache maintains a consistent view of items that are read @@ -76,7 +77,7 @@ struct item_cache_info { /* almost always read, barely written */ struct super_block *sb; struct item_percpu_pages __percpu *pcpu_pages; - struct shrinker shrinker; + KC_DEFINE_SHRINKER(shrinker); struct notifier_block notifier; /* often walked, but per-cpu refs are fast path */ @@ -2530,27 +2531,35 @@ retry: put_pg(sb, right); } +static unsigned long item_cache_count_objects(struct shrinker *shrink, + struct shrink_control *sc) +{ + struct item_cache_info *cinf = KC_SHRINKER_CONTAINER_OF(shrink, struct item_cache_info); + struct super_block *sb = cinf->sb; + + scoutfs_inc_counter(sb, item_cache_count_objects); + + return shrinker_min_long(cinf->lru_pages); +} + /* * Shrink the size the item cache. We're operating against the fast * path lock ordering and we skip pages if we can't acquire locks. We * can run into dirty pages or pages with items that weren't visible to * the earliest active reader which must be skipped. */ -static int item_lru_shrink(struct shrinker *shrink, - struct shrink_control *sc) +static unsigned long item_cache_scan_objects(struct shrinker *shrink, + struct shrink_control *sc) { - struct item_cache_info *cinf = container_of(shrink, - struct item_cache_info, - shrinker); + struct item_cache_info *cinf = KC_SHRINKER_CONTAINER_OF(shrink, struct item_cache_info); struct super_block *sb = cinf->sb; struct cached_page *tmp; struct cached_page *pg; + unsigned long freed = 0; u64 first_reader_seq; - int nr; + int nr = sc->nr_to_scan; - if (sc->nr_to_scan == 0) - goto out; - nr = sc->nr_to_scan; + scoutfs_inc_counter(sb, item_cache_scan_objects); /* can't invalidate pages with items that weren't visible to first reader */ first_reader_seq = first_active_reader_seq(cinf); @@ -2582,6 +2591,7 @@ static int item_lru_shrink(struct shrinker *shrink, rbtree_erase(&pg->node, &cinf->pg_root); invalidate_pcpu_page(pg); write_unlock(&pg->rwlock); + freed++; put_pg(sb, pg); @@ -2591,8 +2601,8 @@ static int item_lru_shrink(struct shrinker *shrink, write_unlock(&cinf->rwlock); spin_unlock(&cinf->lru_lock); -out: - return min_t(unsigned long, cinf->lru_pages, INT_MAX); + + return freed; } static int item_cpu_callback(struct notifier_block *nfb, @@ -2638,9 +2648,9 @@ int scoutfs_item_setup(struct super_block *sb) for_each_possible_cpu(cpu) init_pcpu_pages(cinf, cpu); - cinf->shrinker.shrink = item_lru_shrink; - cinf->shrinker.seeks = DEFAULT_SEEKS; - register_shrinker(&cinf->shrinker); + KC_INIT_SHRINKER_FUNCS(&cinf->shrinker, item_cache_count_objects, + item_cache_scan_objects); + KC_REGISTER_SHRINKER(&cinf->shrinker); cinf->notifier.notifier_call = item_cpu_callback; register_hotcpu_notifier(&cinf->notifier); @@ -2663,7 +2673,7 @@ void scoutfs_item_destroy(struct super_block *sb) BUG_ON(!list_empty(&cinf->active_list)); unregister_hotcpu_notifier(&cinf->notifier); - unregister_shrinker(&cinf->shrinker); + KC_UNREGISTER_SHRINKER(&cinf->shrinker); for_each_possible_cpu(cpu) drop_pcpu_pages(sb, cinf, cpu); diff --git a/kmod/src/kernelcompat.c b/kmod/src/kernelcompat.c new file mode 100644 index 00000000..bb11a803 --- /dev/null +++ b/kmod/src/kernelcompat.c @@ -0,0 +1,30 @@ + +#include "kernelcompat.h" + +#ifdef KC_SHRINKER_SHRINK +#include +/* + * If a target doesn't have that .{count,scan}_objects() interface then + * we have a .shrink() helper that performs the shrink work in terms of + * count/scan. + */ +int kc_shrink_wrapper_fn(struct shrinker *shrink, struct shrink_control *sc) +{ + struct kc_shrinker_wrapper *wrapper = container_of(shrink, struct kc_shrinker_wrapper, shrink); + unsigned long nr; + unsigned long rc; + + if (sc->nr_to_scan != 0) { + rc = wrapper->scan_objects(shrink, sc); + /* translate magic values to the equivalent for older kernels */ + if (rc == SHRINK_STOP) + return -1; + else if (rc == SHRINK_EMPTY) + return 0; + } + + nr = wrapper->count_objects(shrink, sc); + + return min_t(unsigned long, nr, INT_MAX); +} +#endif diff --git a/kmod/src/kernelcompat.h b/kmod/src/kernelcompat.h index 93d1669d..b24d5241 100644 --- a/kmod/src/kernelcompat.h +++ b/kmod/src/kernelcompat.h @@ -167,4 +167,48 @@ do { \ #define kc_bio_get_errno(bio) ({ (int)((void)(bio), _error_arg); }) #endif +#ifndef KC_SHRINKER_SHRINK + +#define KC_DEFINE_SHRINKER(name) struct shrinker name +#define KC_INIT_SHRINKER_FUNCS(name, countfn, scanfn) do { \ + __typeof__(name) _shrink = (name); \ + _shrink->count_objects = (countfn); \ + _shrink->scan_objects = (scanfn); \ + _shrink->seeks = DEFAULT_SEEKS; \ +} while (0) + +#define KC_SHRINKER_CONTAINER_OF(ptr, type) container_of(ptr, type, shrinker) +#define KC_REGISTER_SHRINKER(ptr) (register_shrinker(ptr)) +#define KC_UNREGISTER_SHRINKER(ptr) (unregister_shrinker(ptr)) +#define KC_SHRINKER_FN(ptr) (ptr) +#else + +#include +#ifndef SHRINK_STOP +#define SHRINK_STOP (~0UL) +#define SHRINK_EMPTY (~0UL - 1) +#endif + +int kc_shrink_wrapper_fn(struct shrinker *shrink, struct shrink_control *sc); +struct kc_shrinker_wrapper { + unsigned long (*count_objects)(struct shrinker *, struct shrink_control *sc); + unsigned long (*scan_objects)(struct shrinker *, struct shrink_control *sc); + struct shrinker shrink; +}; + +#define KC_DEFINE_SHRINKER(name) struct kc_shrinker_wrapper name; +#define KC_INIT_SHRINKER_FUNCS(name, countfn, scanfn) do { \ + struct kc_shrinker_wrapper *_wrap = (name); \ + _wrap->count_objects = (countfn); \ + _wrap->scan_objects = (scanfn); \ + _wrap->shrink.shrink = kc_shrink_wrapper_fn; \ + _wrap->shrink.seeks = DEFAULT_SEEKS; \ +} while (0) +#define KC_SHRINKER_CONTAINER_OF(ptr, type) container_of(container_of(ptr, struct kc_shrinker_wrapper, shrink), type, shrinker) +#define KC_REGISTER_SHRINKER(ptr) (register_shrinker(ptr.shrink)) +#define KC_UNREGISTER_SHRINKER(ptr) (unregister_shrinker(ptr.shrink)) +#define KC_SHRINKER_FN(ptr) (ptr.shrink) + +#endif /* KC_SHRINKER_SHRINK */ + #endif diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 2ed75b9f..db4c384f 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -35,6 +35,7 @@ #include "xattr.h" #include "item.h" #include "omap.h" +#include "util.h" /* * scoutfs uses a lock service to manage item cache consistency between @@ -76,7 +77,7 @@ struct lock_info { bool unmounting; struct rb_root lock_tree; struct rb_root lock_range_tree; - struct shrinker shrinker; + KC_DEFINE_SHRINKER(shrinker); struct list_head lru_list; unsigned long long lru_nr; struct workqueue_struct *workq; @@ -1400,6 +1401,17 @@ static void lock_shrink_worker(struct work_struct *work) } } +static unsigned long lock_count_objects(struct shrinker *shrink, + struct shrink_control *sc) +{ + struct lock_info *linfo = KC_SHRINKER_CONTAINER_OF(shrink, struct lock_info); + struct super_block *sb = linfo->sb; + + scoutfs_inc_counter(sb, lock_count_objects); + + return shrinker_min_long(linfo->lru_nr); +} + /* * Start the shrinking process for locks on the lru. If a lock is on * the lru then it can't have any active users. We don't want to block @@ -1412,21 +1424,18 @@ static void lock_shrink_worker(struct work_struct *work) * mode which will prevent the lock from being freed when the null * response arrives. */ -static int scoutfs_lock_shrink(struct shrinker *shrink, - struct shrink_control *sc) +static unsigned long lock_scan_objects(struct shrinker *shrink, + struct shrink_control *sc) { - struct lock_info *linfo = container_of(shrink, struct lock_info, - shrinker); + struct lock_info *linfo = KC_SHRINKER_CONTAINER_OF(shrink, struct lock_info); struct super_block *sb = linfo->sb; struct scoutfs_lock *lock; struct scoutfs_lock *tmp; - unsigned long nr; + unsigned long freed = 0; + unsigned long nr = sc->nr_to_scan; bool added = false; - int ret; - nr = sc->nr_to_scan; - if (nr == 0) - goto out; + scoutfs_inc_counter(sb, lock_scan_objects); spin_lock(&linfo->lock); @@ -1444,6 +1453,7 @@ restart: lock->request_pending = 1; list_add_tail(&lock->shrink_head, &linfo->shrink_list); added = true; + freed++; scoutfs_inc_counter(sb, lock_shrink_attempted); trace_scoutfs_lock_shrink(sb, lock); @@ -1458,10 +1468,8 @@ restart: if (added) queue_work(linfo->workq, &linfo->shrink_work); -out: - ret = min_t(unsigned long, linfo->lru_nr, INT_MAX); - trace_scoutfs_lock_shrink_exit(sb, sc->nr_to_scan, ret); - return ret; + trace_scoutfs_lock_shrink_exit(sb, sc->nr_to_scan, freed); + return freed; } void scoutfs_free_unused_locks(struct super_block *sb) @@ -1472,7 +1480,7 @@ void scoutfs_free_unused_locks(struct super_block *sb) .nr_to_scan = INT_MAX, }; - linfo->shrinker.shrink(&linfo->shrinker, &sc); + lock_scan_objects(KC_SHRINKER_FN(&linfo->shrinker), &sc); } static void lock_tseq_show(struct seq_file *m, struct scoutfs_tseq_entry *ent) @@ -1579,7 +1587,7 @@ void scoutfs_lock_shutdown(struct super_block *sb) trace_scoutfs_lock_shutdown(sb, linfo); /* stop the shrinker from queueing work */ - unregister_shrinker(&linfo->shrinker); + KC_UNREGISTER_SHRINKER(&linfo->shrinker); flush_work(&linfo->shrink_work); /* cause current and future lock calls to return errors */ @@ -1698,9 +1706,9 @@ int scoutfs_lock_setup(struct super_block *sb) spin_lock_init(&linfo->lock); linfo->lock_tree = RB_ROOT; linfo->lock_range_tree = RB_ROOT; - linfo->shrinker.shrink = scoutfs_lock_shrink; - linfo->shrinker.seeks = DEFAULT_SEEKS; - register_shrinker(&linfo->shrinker); + KC_INIT_SHRINKER_FUNCS(&linfo->shrinker, lock_count_objects, + lock_scan_objects); + KC_REGISTER_SHRINKER(&linfo->shrinker); INIT_LIST_HEAD(&linfo->lru_list); INIT_WORK(&linfo->inv_work, lock_invalidate_worker); INIT_LIST_HEAD(&linfo->inv_list); diff --git a/kmod/src/util.h b/kmod/src/util.h index 20d9db79..7ca9d3eb 100644 --- a/kmod/src/util.h +++ b/kmod/src/util.h @@ -17,4 +17,15 @@ static inline void down_write_two(struct rw_semaphore *a, down_write_nested(b, SINGLE_DEPTH_NESTING); } +/* + * When returning shrinker counts from scan_objects, we should steer + * clear of the magic SHRINK_STOP and SHRINK_EMPTY values, which are near + * ~0UL values. Hence, we cap count to ~0L, which is arbitarily high + * enough to avoid it. + */ +static inline long shrinker_min_long(long count) +{ + return min(count, LONG_MAX); +} + #endif