From 0204e092e4c02e3e5ac7baadb945a91ab14fc916 Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Thu, 19 Oct 2023 10:30:17 -0700 Subject: [PATCH 01/42] FIELD_SIZEOF was deprecated. We could use sizeof_field as a direct replacement (which is the same) except that this entire thing can directly use offsetofend(). Signed-off-by: Auke Kok --- kmod/src/block.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kmod/src/block.c b/kmod/src/block.c index 7a17aa96..50f16d99 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -120,8 +120,7 @@ do { \ static __le32 block_calc_crc(struct scoutfs_block_header *hdr, u32 size) { - int off = offsetof(struct scoutfs_block_header, crc) + - FIELD_SIZEOF(struct scoutfs_block_header, crc); + int off = offsetofend(struct scoutfs_block_header, crc); u32 calc = crc32c(~0, (char *)hdr + off, size - off); return cpu_to_le32(calc); From 8885486bc89aa0b4b42359260782b4c5755d2140 Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Mon, 16 Oct 2023 15:48:06 -0700 Subject: [PATCH 02/42] Add several low level includes. Newer kernels include less header dependencies by default, so we have to add these. Signed-off-by: Auke Kok --- kmod/src/alloc.c | 1 + kmod/src/data.c | 1 + kmod/src/srch.c | 1 + kmod/src/sysfs.c | 1 + 4 files changed, 4 insertions(+) diff --git a/kmod/src/alloc.c b/kmod/src/alloc.c index 26af17f8..894b92ed 100644 --- a/kmod/src/alloc.c +++ b/kmod/src/alloc.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include diff --git a/kmod/src/data.c b/kmod/src/data.c index dd10a06f..64859706 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include "format.h" diff --git a/kmod/src/srch.c b/kmod/src/srch.c index e05f707c..bd98aa6b 100644 --- a/kmod/src/srch.c +++ b/kmod/src/srch.c @@ -18,6 +18,7 @@ #include #include #include +#include #include "super.h" #include "format.h" diff --git a/kmod/src/sysfs.c b/kmod/src/sysfs.c index 34fb4a64..b3c7ddbf 100644 --- a/kmod/src/sysfs.c +++ b/kmod/src/sysfs.c @@ -13,6 +13,7 @@ #include #include #include +#include #include "super.h" #include "sysfs.h" From 19af6e28fba3ec72c2bbf9ec28bab8bb0b2a001a Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Mon, 16 Oct 2023 15:58:10 -0700 Subject: [PATCH 03/42] "unaligned/access_ok.h" is not needed, and removed. Signed-off-by: Auke Kok --- kmod/src/scoutfs_trace.h | 1 - 1 file changed, 1 deletion(-) diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 3fd4821f..ceb538ba 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -24,7 +24,6 @@ #include #include -#include #include "key.h" #include "format.h" From c30172210ff335f3b3410a8fee241ce543698a18 Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Tue, 17 Oct 2023 11:53:17 -0700 Subject: [PATCH 04/42] Use blk_opf_t to pass bio op flags Compat is back to unsigned int. Signed-off-by: Auke Kok --- kmod/src/Makefile.kernelcompat | 8 ++++++++ kmod/src/block.c | 6 +++--- kmod/src/kernelcompat.h | 5 +++++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/kmod/src/Makefile.kernelcompat b/kmod/src/Makefile.kernelcompat index 352f2671..75058633 100644 --- a/kmod/src/Makefile.kernelcompat +++ b/kmod/src/Makefile.kernelcompat @@ -258,3 +258,11 @@ endif ifneq (,$(shell grep 'static inline const char .xattr_prefix' include/linux/xattr.h)) ccflags-y += -DKC_XATTR_HANDLER_NAME=1 endif + +# +# v5.19-rc4-96-g342a72a33407 +# +# Adds `typedef __u32 __bitwise blk_opf_t` to aid flag checking +ifneq (,$(shell grep 'typedef __u32 __bitwise blk_opf_t' include/linux/blk_types.h)) +ccflags-y += -DKC_HAVE_BLK_OPF_T=1 +endif diff --git a/kmod/src/block.c b/kmod/src/block.c index 50f16d99..08acc05e 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -437,7 +437,7 @@ static void block_remove_all(struct super_block *sb) * possible. Final freeing, verifying checksums, and unlinking errored * blocks are all done by future users of the blocks. */ -static void block_end_io(struct super_block *sb, unsigned int opf, +static void block_end_io(struct super_block *sb, blk_opf_t opf, struct block_private *bp, int err) { DECLARE_BLOCK_INFO(sb, binf); @@ -477,7 +477,7 @@ static void KC_DECLARE_BIO_END_IO(block_bio_end_io, struct bio *bio) * Kick off IO for a single block. */ static int block_submit_bio(struct super_block *sb, struct block_private *bp, - unsigned int opf) + blk_opf_t opf) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct bio *bio = NULL; @@ -1200,7 +1200,7 @@ static void KC_DECLARE_BIO_END_IO(sm_block_bio_end_io, struct bio *bio) * only layer that sees the full block buffer so we pass the calculated * crc to the caller for them to check in their context. */ -static int sm_block_io(struct super_block *sb, struct block_device *bdev, unsigned int opf, +static int sm_block_io(struct super_block *sb, struct block_device *bdev, blk_opf_t opf, u64 blkno, struct scoutfs_block_header *hdr, size_t len, __le32 *blk_crc) { struct scoutfs_block_header *pg_hdr; diff --git a/kmod/src/kernelcompat.h b/kmod/src/kernelcompat.h index 32124829..6f2a8b07 100644 --- a/kmod/src/kernelcompat.h +++ b/kmod/src/kernelcompat.h @@ -273,4 +273,9 @@ ssize_t kc_generic_file_buffered_write(struct kiocb *iocb, const struct iovec *i #define generic_file_buffered_write kc_generic_file_buffered_write #endif +#ifndef KC_HAVE_BLK_OPF_T +/* typedef __u32 __bitwise blk_opf_t; */ +typedef unsigned int blk_opf_t; +#endif + #endif From d3c53289091daec28c0ba720622d82ef7d9211e8 Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Mon, 2 Oct 2023 13:39:38 -0700 Subject: [PATCH 05/42] setattr_prepare no longer extern in fs.h v5.11-rc4-7-g2f221d6f7b88 Changes setattr_prepare from an extern to plain int. There's no impact further to the compat to keep it working except for the detection regex. Signed-off-by: Auke Kok --- kmod/src/Makefile.kernelcompat | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kmod/src/Makefile.kernelcompat b/kmod/src/Makefile.kernelcompat index 75058633..39210f32 100644 --- a/kmod/src/Makefile.kernelcompat +++ b/kmod/src/Makefile.kernelcompat @@ -78,8 +78,9 @@ endif # v4.8-rc1-29-g31051c85b5e2 # # inode_change_ok() removed - replace with setattr_prepare() +# v5.11-rc4-7-g2f221d6f7b88 removes extern attribute # -ifneq (,$(shell grep 'extern int setattr_prepare' include/linux/fs.h)) +ifneq (,$(shell grep 'int setattr_prepare' include/linux/fs.h)) ccflags-y += -DKC_SETATTR_PREPARE endif From ad82a5e52a5fca01fab87fbd55e0aab230e2573c Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Tue, 3 Oct 2023 11:11:42 -0700 Subject: [PATCH 06/42] Squelch warning from bpf_iter.c. v5.7-rc2-1174-gfd4f12bc38c3 significantly rewrites the bpf iterator which hits this _next() function. It also adds a check that verifies that the *pos is incremented after every call, even if it goes beyond the last member (in which case it's not used). Signed-off-by: Auke Kok --- kmod/src/tseq.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/kmod/src/tseq.c b/kmod/src/tseq.c index b4b07f34..2621406f 100644 --- a/kmod/src/tseq.c +++ b/kmod/src/tseq.c @@ -183,6 +183,13 @@ static void *scoutfs_tseq_seq_next(struct seq_file *m, void *v, loff_t *pos) ent = tseq_rb_next(ent); if (ent) *pos = ent->pos; + else + /* + * once we hit the end, *pos is never used, but it has to + * be updated to avoid an error in bpf_seq_read() + */ + (*pos)++; + return ent; } From c951713ab2ac1d5c04ca98e36fa6efd24483d500 Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Mon, 16 Oct 2023 16:39:30 -0700 Subject: [PATCH 07/42] list_cmp_func_t introduced, using const. v5.12-rc6-9-g4f0f586bf0c8 All list_sort functions use the list_cmp_func_t type, which compares list_head member types. These are now required to be `const` as the compiler will now check them. This propagates into our callers. Signed-off-by: Auke Kok --- kmod/src/Makefile.kernelcompat | 8 ++++++++ kmod/src/inode.c | 6 +++--- kmod/src/item.c | 12 ++++++------ kmod/src/kernelcompat.h | 6 ++++++ kmod/src/key.h | 12 ++++++------ kmod/src/recov.c | 6 +++--- 6 files changed, 32 insertions(+), 18 deletions(-) diff --git a/kmod/src/Makefile.kernelcompat b/kmod/src/Makefile.kernelcompat index 39210f32..849db059 100644 --- a/kmod/src/Makefile.kernelcompat +++ b/kmod/src/Makefile.kernelcompat @@ -267,3 +267,11 @@ endif ifneq (,$(shell grep 'typedef __u32 __bitwise blk_opf_t' include/linux/blk_types.h)) ccflags-y += -DKC_HAVE_BLK_OPF_T=1 endif + +# +# v5.12-rc6-9-g4f0f586bf0c8 +# +# list_sort cmp function takes const list_head args +ifneq (,$(shell grep 'const struct list_head ., const struct list_head .' include/linux/list_sort.h)) +ccflags-y += -DKC_LIST_CMP_CONST_ARG_LIST_HEAD +endif diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 2203a8e9..04c088c2 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -979,10 +979,10 @@ static bool inode_has_index(umode_t mode, u8 type) } } -static int cmp_index_lock(void *priv, struct list_head *A, struct list_head *B) +static int cmp_index_lock(void *priv, KC_LIST_CMP_CONST struct list_head *A, KC_LIST_CMP_CONST struct list_head *B) { - struct index_lock *a = list_entry(A, struct index_lock, head); - struct index_lock *b = list_entry(B, struct index_lock, head); + KC_LIST_CMP_CONST struct index_lock *a = list_entry(A, KC_LIST_CMP_CONST struct index_lock, head); + KC_LIST_CMP_CONST struct index_lock *b = list_entry(B, KC_LIST_CMP_CONST struct index_lock, head); return ((int)a->type - (int)b->type) ?: scoutfs_cmp_u64s(a->major, b->major) ?: diff --git a/kmod/src/item.c b/kmod/src/item.c index 9ec5aad0..f7691e6e 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -2241,18 +2241,18 @@ u64 scoutfs_item_dirty_pages(struct super_block *sb) return (u64)atomic_read(&cinf->dirty_pages); } -static int cmp_pg_start(void *priv, struct list_head *A, struct list_head *B) +static int cmp_pg_start(void *priv, KC_LIST_CMP_CONST struct list_head *A, KC_LIST_CMP_CONST struct list_head *B) { - struct cached_page *a = list_entry(A, struct cached_page, dirty_head); - struct cached_page *b = list_entry(B, struct cached_page, dirty_head); + KC_LIST_CMP_CONST struct cached_page *a = list_entry(A, KC_LIST_CMP_CONST struct cached_page, dirty_head); + KC_LIST_CMP_CONST struct cached_page *b = list_entry(B, KC_LIST_CMP_CONST struct cached_page, dirty_head); return scoutfs_key_compare(&a->start, &b->start); } -static int cmp_item_key(void *priv, struct list_head *A, struct list_head *B) +static int cmp_item_key(void *priv, KC_LIST_CMP_CONST struct list_head *A, KC_LIST_CMP_CONST struct list_head *B) { - struct cached_item *a = list_entry(A, struct cached_item, dirty_head); - struct cached_item *b = list_entry(B, struct cached_item, dirty_head); + KC_LIST_CMP_CONST struct cached_item *a = list_entry(A, KC_LIST_CMP_CONST struct cached_item, dirty_head); + KC_LIST_CMP_CONST struct cached_item *b = list_entry(B, KC_LIST_CMP_CONST struct cached_item, dirty_head); return scoutfs_key_compare(&a->key, &b->key); } diff --git a/kmod/src/kernelcompat.h b/kmod/src/kernelcompat.h index 6f2a8b07..54377d56 100644 --- a/kmod/src/kernelcompat.h +++ b/kmod/src/kernelcompat.h @@ -278,4 +278,10 @@ ssize_t kc_generic_file_buffered_write(struct kiocb *iocb, const struct iovec *i typedef unsigned int blk_opf_t; #endif +#ifdef KC_LIST_CMP_CONST_ARG_LIST_HEAD +#define KC_LIST_CMP_CONST const +#else +#define KC_LIST_CMP_CONST +#endif + #endif diff --git a/kmod/src/key.h b/kmod/src/key.h index 66a4c84a..cd1e7710 100644 --- a/kmod/src/key.h +++ b/kmod/src/key.h @@ -125,8 +125,8 @@ static inline bool scoutfs_key_is_ones(struct scoutfs_key *key) * other alternatives across keys that first differ in any of the * values. Say maybe 20% faster than memcmp. */ -static inline int scoutfs_key_compare(struct scoutfs_key *a, - struct scoutfs_key *b) +static inline int scoutfs_key_compare(const struct scoutfs_key *a, + const struct scoutfs_key *b) { return scoutfs_cmp(a->sk_zone, b->sk_zone) ?: scoutfs_cmp(le64_to_cpu(a->_sk_first), le64_to_cpu(b->_sk_first)) ?: @@ -142,10 +142,10 @@ static inline int scoutfs_key_compare(struct scoutfs_key *a, * 1: a_start > b_end * else 0: ranges overlap */ -static inline int scoutfs_key_compare_ranges(struct scoutfs_key *a_start, - struct scoutfs_key *a_end, - struct scoutfs_key *b_start, - struct scoutfs_key *b_end) +static inline int scoutfs_key_compare_ranges(const struct scoutfs_key *a_start, + const struct scoutfs_key *a_end, + const struct scoutfs_key *b_start, + const struct scoutfs_key *b_end) { return scoutfs_key_compare(a_end, b_start) < 0 ? -1 : scoutfs_key_compare(a_start, b_end) > 0 ? 1 : diff --git a/kmod/src/recov.c b/kmod/src/recov.c index 78123e27..d77e39cf 100644 --- a/kmod/src/recov.c +++ b/kmod/src/recov.c @@ -76,10 +76,10 @@ static struct recov_pending *lookup_pending(struct recov_info *recinf, u64 rid, * We keep the pending list sorted by rid so that we can iterate over * them. The list should be small and shouldn't be used often. */ -static int cmp_pending_rid(void *priv, struct list_head *A, struct list_head *B) +static int cmp_pending_rid(void *priv, KC_LIST_CMP_CONST struct list_head *A, KC_LIST_CMP_CONST struct list_head *B) { - struct recov_pending *a = list_entry(A, struct recov_pending, head); - struct recov_pending *b = list_entry(B, struct recov_pending, head); + KC_LIST_CMP_CONST struct recov_pending *a = list_entry(A, KC_LIST_CMP_CONST struct recov_pending, head); + KC_LIST_CMP_CONST struct recov_pending *b = list_entry(B, KC_LIST_CMP_CONST struct recov_pending, head); return scoutfs_cmp_u64s(a->rid, b->rid); } From 077468ac1e2be7124a7ea5074bc44385882fb189 Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Tue, 17 Oct 2023 11:15:04 -0700 Subject: [PATCH 08/42] debugfs_create_atomic_t now returns void, don't check result Greg KH tells us to do just this in v5.4-rc5-31-g9927c6fa3e1d: No one checks the return value of debugfs_create_atomic_t(), as it's not needed, so make the return value void, so that no one tries to do so in the future. Signed-off-by: Auke Kok --- kmod/src/triggers.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/kmod/src/triggers.c b/kmod/src/triggers.c index 750a86b1..8e46236d 100644 --- a/kmod/src/triggers.c +++ b/kmod/src/triggers.c @@ -93,13 +93,9 @@ int scoutfs_setup_triggers(struct super_block *sb) goto out; } - for (i = 0; i < ARRAY_SIZE(triggers->atomics); i++) { - if (!debugfs_create_atomic_t(names[i], 0644, triggers->dir, - &triggers->atomics[i])) { - ret = -ENOMEM; - goto out; - } - } + for (i = 0; i < ARRAY_SIZE(triggers->atomics); i++) + debugfs_create_atomic_t(names[i], 0644, triggers->dir, + &triggers->atomics[i]); ret = 0; out: From 1f0dd7f0256adb6861b8a0409bcc48929c540077 Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Mon, 16 Oct 2023 16:07:19 -0700 Subject: [PATCH 09/42] __vmalloc defaults to PAGE_KERNEL everywhere, so the arg was removed. v5.7-523-g88dca4ca5a93 __vmalloc no longer has the 3rd argument. Signed-off-by: Auke Kok --- kmod/src/Makefile.kernelcompat | 7 +++++++ kmod/src/block.c | 2 +- kmod/src/kernelcompat.h | 6 ++++++ kmod/src/quorum.c | 4 ++-- kmod/src/srch.c | 3 +-- 5 files changed, 17 insertions(+), 5 deletions(-) diff --git a/kmod/src/Makefile.kernelcompat b/kmod/src/Makefile.kernelcompat index 849db059..efa89b4e 100644 --- a/kmod/src/Makefile.kernelcompat +++ b/kmod/src/Makefile.kernelcompat @@ -275,3 +275,10 @@ endif ifneq (,$(shell grep 'const struct list_head ., const struct list_head .' include/linux/list_sort.h)) ccflags-y += -DKC_LIST_CMP_CONST_ARG_LIST_HEAD endif + +# v5.7-523-g88dca4ca5a93 +# +# The pgprot argument to vmalloc is always PAGE_KERNEL, so it is removed. +ifneq (,$(shell grep 'extern void .__vmalloc.unsigned long size, gfp_t gfp_mask, pgprot_t prot' include/linux/vmalloc.h)) +ccflags-y += -DKC_VMALLOC_PGPROT_T +endif diff --git a/kmod/src/block.c b/kmod/src/block.c index 08acc05e..ca051967 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -158,7 +158,7 @@ static struct block_private *block_alloc(struct super_block *sb, u64 blkno) */ lockdep_off(); nofs_flags = memalloc_nofs_save(); - bp->virt = __vmalloc(SCOUTFS_BLOCK_LG_SIZE, GFP_NOFS | __GFP_HIGHMEM, PAGE_KERNEL); + bp->virt = kc__vmalloc(SCOUTFS_BLOCK_LG_SIZE, GFP_NOFS | __GFP_HIGHMEM); memalloc_nofs_restore(nofs_flags); lockdep_on(); diff --git a/kmod/src/kernelcompat.h b/kmod/src/kernelcompat.h index 54377d56..361f5380 100644 --- a/kmod/src/kernelcompat.h +++ b/kmod/src/kernelcompat.h @@ -284,4 +284,10 @@ typedef unsigned int blk_opf_t; #define KC_LIST_CMP_CONST #endif +#ifdef KC_VMALLOC_PGPROT_T +#define kc__vmalloc(size, gfp_mask) __vmalloc(size, gfp_mask, PAGE_KERNEL) +#else +#define kc__vmalloc __vmalloc +#endif + #endif diff --git a/kmod/src/quorum.c b/kmod/src/quorum.c index 88e5c07a..da850cc7 100644 --- a/kmod/src/quorum.c +++ b/kmod/src/quorum.c @@ -1325,8 +1325,8 @@ int scoutfs_quorum_setup(struct super_block *sb) qinf = kzalloc(sizeof(struct quorum_info), GFP_KERNEL); super = kmalloc(sizeof(struct scoutfs_super_block), GFP_KERNEL); if (qinf) - qinf->hb_delay = __vmalloc(HB_DELAY_NR * sizeof(struct count_recent), - GFP_KERNEL | __GFP_ZERO, PAGE_KERNEL); + qinf->hb_delay = kc__vmalloc(HB_DELAY_NR * sizeof(struct count_recent), + GFP_KERNEL | __GFP_ZERO); if (!qinf || !super || !qinf->hb_delay) { if (qinf) vfree(qinf->hb_delay); diff --git a/kmod/src/srch.c b/kmod/src/srch.c index bd98aa6b..99ea6448 100644 --- a/kmod/src/srch.c +++ b/kmod/src/srch.c @@ -1589,8 +1589,7 @@ static int kway_merge(struct super_block *sb, nr_parents = max_t(unsigned long, 1, roundup_pow_of_two(nr) - 1); /* root at [1] for easy sib/parent index calc, final pad for odd sib */ nr_nodes = 1 + nr_parents + nr + 1; - tnodes = __vmalloc(nr_nodes * sizeof(struct tourn_node), - GFP_NOFS, PAGE_KERNEL); + tnodes = kc__vmalloc(nr_nodes * sizeof(struct tourn_node), GFP_NOFS); if (!tnodes) return -ENOMEM; From 2d58ee2a37011fae7b1d0612a825ed16ed6ad28d Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Mon, 16 Oct 2023 16:26:09 -0700 Subject: [PATCH 10/42] Account for new bio_alloc() args. Block device and opf are now passed through and set. We mimic compat code to do the same. Signed-off-by: Auke Kok --- kmod/src/Makefile.kernelcompat | 8 ++++++++ kmod/src/block.c | 8 ++------ kmod/src/kernelcompat.h | 16 ++++++++++++++++ 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/kmod/src/Makefile.kernelcompat b/kmod/src/Makefile.kernelcompat index efa89b4e..68812985 100644 --- a/kmod/src/Makefile.kernelcompat +++ b/kmod/src/Makefile.kernelcompat @@ -282,3 +282,11 @@ endif ifneq (,$(shell grep 'extern void .__vmalloc.unsigned long size, gfp_t gfp_mask, pgprot_t prot' include/linux/vmalloc.h)) ccflags-y += -DKC_VMALLOC_PGPROT_T endif + +# +# v5.17-rc2-21-g07888c665b40 +# +# Detect new style bio_alloc - pass bdev and opf. +ifneq (,$(shell grep 'struct bio .bio_alloc.struct block_device .bdev' include/linux/bio.h)) +ccflags-y += -DKC_BIO_ALLOC_DEV_OPF_ARGS +endif diff --git a/kmod/src/block.c b/kmod/src/block.c index ca051967..23a37b98 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -504,15 +504,13 @@ static int block_submit_bio(struct super_block *sb, struct block_private *bp, for (off = 0; off < SCOUTFS_BLOCK_LG_SIZE; off += PAGE_SIZE) { if (!bio) { - bio = bio_alloc(GFP_NOFS, SCOUTFS_BLOCK_LG_PAGES_PER); + bio = kc_bio_alloc(sbi->meta_bdev, SCOUTFS_BLOCK_LG_PAGES_PER, opf, GFP_NOFS); if (!bio) { ret = -ENOMEM; break; } - kc_bio_set_opf(bio, opf); kc_bio_set_sector(bio, sector + (off >> 9)); - bio_set_dev(bio, sbi->meta_bdev); bio->bi_end_io = block_bio_end_io; bio->bi_private = bp; @@ -1232,15 +1230,13 @@ static int sm_block_io(struct super_block *sb, struct block_device *bdev, blk_op pg_hdr->crc = block_calc_crc(pg_hdr, SCOUTFS_BLOCK_SM_SIZE); } - bio = bio_alloc(GFP_NOFS, 1); + bio = kc_bio_alloc(bdev, 1, opf, GFP_NOFS); if (!bio) { ret = -ENOMEM; goto out; } - kc_bio_set_opf(bio, opf | REQ_SYNC); kc_bio_set_sector(bio, blkno << (SCOUTFS_BLOCK_SM_SHIFT - 9)); - bio_set_dev(bio, bdev); bio->bi_end_io = sm_block_bio_end_io; bio->bi_private = &sbc; bio_add_page(bio, page, SCOUTFS_BLOCK_SM_SIZE, 0); diff --git a/kmod/src/kernelcompat.h b/kmod/src/kernelcompat.h index 361f5380..1d7acee7 100644 --- a/kmod/src/kernelcompat.h +++ b/kmod/src/kernelcompat.h @@ -290,4 +290,20 @@ typedef unsigned int blk_opf_t; #define kc__vmalloc __vmalloc #endif +#ifdef KC_BIO_ALLOC_DEV_OPF_ARGS +#define kc_bio_alloc bio_alloc +#else +#include +static inline struct bio *kc_bio_alloc(struct block_device *bdev, unsigned short nr_vecs, + blk_opf_t opf, gfp_t gfp_mask) +{ + struct bio *b = bio_alloc(gfp_mask, nr_vecs); + if (b) { + kc_bio_set_opf(b, opf); + bio_set_dev(b, bdev); + } + return b; +} +#endif + #endif From 4ef64c6fcf6bdd41c6de8ed884443224c1948766 Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Mon, 16 Oct 2023 16:15:04 -0700 Subject: [PATCH 11/42] Vfs methods become user namespace mount aware. v5.11-rc4-24-g549c7297717c All of these VFS methods are now passed a user_namespace. Signed-off-by: Auke Kok --- kmod/src/Makefile.kernelcompat | 24 +++++++++++++++++++++ kmod/src/acl.c | 2 +- kmod/src/data.c | 4 ++-- kmod/src/dir.c | 38 ++++++++++++++++++++++------------ kmod/src/file.c | 6 ++++-- kmod/src/file.h | 3 ++- kmod/src/inode.c | 18 ++++++++++------ kmod/src/inode.h | 6 ++++-- kmod/src/ioctl.c | 3 ++- kmod/src/kernelcompat.h | 10 +++++++++ 10 files changed, 86 insertions(+), 28 deletions(-) diff --git a/kmod/src/Makefile.kernelcompat b/kmod/src/Makefile.kernelcompat index 68812985..1a9c0da1 100644 --- a/kmod/src/Makefile.kernelcompat +++ b/kmod/src/Makefile.kernelcompat @@ -283,6 +283,30 @@ ifneq (,$(shell grep 'extern void .__vmalloc.unsigned long size, gfp_t gfp_mask, ccflags-y += -DKC_VMALLOC_PGPROT_T endif +# v6.2-rc1-18-g01beba7957a2 +# +# fs: port inode_owner_or_capable() to mnt_idmap +ifneq (,$(shell grep 'bool inode_owner_or_capable.struct user_namespace .mnt_userns' include/linux/fs.h)) +ccflags-y += -DKC_INODE_OWNER_OR_CAPABLE_USERNS +endif + +# +# v5.11-rc4-5-g47291baa8ddf +# +# namei: make permission helpers idmapped mount aware +ifneq (,$(shell grep 'int inode_permission.struct user_namespace' include/linux/fs.h)) +ccflags-y += -DKC_INODE_PERMISSION_USERNS +endif + +# +# v5.11-rc4-24-g549c7297717c +# +# fs: make helpers idmap mount aware +# Enlarges the VFS API methods to include user namespace argument. +ifneq (,$(shell grep 'int ..mknod. .struct user_namespace' include/linux/fs.h)) +ccflags-y += -DKC_VFS_METHOD_USER_NAMESPACE_ARG +endif + # # v5.17-rc2-21-g07888c665b40 # diff --git a/kmod/src/acl.c b/kmod/src/acl.c index 86d398ab..8e371fb2 100644 --- a/kmod/src/acl.c +++ b/kmod/src/acl.c @@ -265,7 +265,7 @@ int scoutfs_acl_set_xattr(struct dentry *dentry, const char *name, const void *v struct posix_acl *acl = NULL; int ret; - if (!inode_owner_or_capable(dentry->d_inode)) + if (!inode_owner_or_capable(KC_VFS_INIT_NS dentry->d_inode)) return -EPERM; if (!IS_POSIXACL(dentry->d_inode)) diff --git a/kmod/src/data.c b/kmod/src/data.c index 64859706..49ab801d 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -1305,8 +1305,8 @@ int scoutfs_data_move_blocks(struct inode *from, u64 from_off, goto out; } - ret = inode_permission(from, MAY_WRITE) ?: - inode_permission(to, MAY_WRITE); + ret = inode_permission(KC_VFS_INIT_NS from, MAY_WRITE) ?: + inode_permission(KC_VFS_INIT_NS to, MAY_WRITE); if (ret < 0) goto out; diff --git a/kmod/src/dir.c b/kmod/src/dir.c index ca23d3a6..6634cbf4 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -703,8 +703,9 @@ out_unlock: return inode; } -static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, - dev_t rdev) +static int scoutfs_mknod(KC_VFS_NS_DEF + struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t rdev) { struct super_block *sb = dir->i_sb; struct inode *inode = NULL; @@ -773,15 +774,20 @@ out: } /* XXX hmm, do something with excl? */ -static int scoutfs_create(struct inode *dir, struct dentry *dentry, - umode_t mode, bool excl) +static int scoutfs_create(KC_VFS_NS_DEF + struct inode *dir, + struct dentry *dentry, umode_t mode, bool excl) { - return scoutfs_mknod(dir, dentry, mode | S_IFREG, 0); + return scoutfs_mknod(KC_VFS_NS + dir, dentry, mode | S_IFREG, 0); } -static int scoutfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) +static int scoutfs_mkdir(KC_VFS_NS_DEF + struct inode *dir, + struct dentry *dentry, umode_t mode) { - return scoutfs_mknod(dir, dentry, mode | S_IFDIR, 0); + return scoutfs_mknod(KC_VFS_NS + dir, dentry, mode | S_IFDIR, 0); } static int scoutfs_link(struct dentry *old_dentry, @@ -1176,7 +1182,8 @@ static const char *scoutfs_get_link(struct dentry *dentry, struct inode *inode, * Symlink target paths can be annoyingly large. We store relatively * rare large paths in multiple items. */ -static int scoutfs_symlink(struct inode *dir, struct dentry *dentry, +static int scoutfs_symlink(KC_VFS_NS_DEF + struct inode *dir, struct dentry *dentry, const char *symname) { struct super_block *sb = dir->i_sb; @@ -1563,7 +1570,8 @@ static int verify_ancestors(struct super_block *sb, u64 p1, u64 p2, * from using parent/child locking orders as two groups can have both * parent and child relationships to each other. */ -static int scoutfs_rename_common(struct inode *old_dir, +static int scoutfs_rename_common(KC_VFS_NS_DEF + struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry, unsigned int flags) { @@ -1840,18 +1848,21 @@ static int scoutfs_rename(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry) { - return scoutfs_rename_common(old_dir, old_dentry, new_dir, new_dentry, 0); + return scoutfs_rename_common(KC_VFS_INIT_NS + old_dir, old_dentry, new_dir, new_dentry, 0); } #endif -static int scoutfs_rename2(struct inode *old_dir, +static int scoutfs_rename2(KC_VFS_NS_DEF + struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry, unsigned int flags) { if (flags & ~RENAME_NOREPLACE) return -EINVAL; - return scoutfs_rename_common(old_dir, old_dentry, new_dir, new_dentry, flags); + return scoutfs_rename_common(KC_VFS_NS + old_dir, old_dentry, new_dir, new_dentry, flags); } #ifdef KC_FMODE_KABI_ITERATE @@ -1863,7 +1874,8 @@ static int scoutfs_dir_open(struct inode *inode, struct file *file) } #endif -static int scoutfs_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode) +static int scoutfs_tmpfile(KC_VFS_NS_DEF + struct inode *dir, struct dentry *dentry, umode_t mode) { struct super_block *sb = dir->i_sb; struct inode *inode = NULL; diff --git a/kmod/src/file.c b/kmod/src/file.c index d44cd822..15158a2b 100644 --- a/kmod/src/file.c +++ b/kmod/src/file.c @@ -267,7 +267,8 @@ out: } #endif -int scoutfs_permission(struct inode *inode, int mask) +int scoutfs_permission(KC_VFS_NS_DEF + struct inode *inode, int mask) { struct super_block *sb = inode->i_sb; struct scoutfs_lock *inode_lock = NULL; @@ -281,7 +282,8 @@ int scoutfs_permission(struct inode *inode, int mask) if (ret) return ret; - ret = generic_permission(inode, mask); + ret = generic_permission(KC_VFS_INIT_NS + inode, mask); scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_READ); diff --git a/kmod/src/file.h b/kmod/src/file.h index 82829ef5..1009b11f 100644 --- a/kmod/src/file.h +++ b/kmod/src/file.h @@ -10,7 +10,8 @@ ssize_t scoutfs_file_aio_write(struct kiocb *iocb, const struct iovec *iov, ssize_t scoutfs_file_read_iter(struct kiocb *, struct iov_iter *); ssize_t scoutfs_file_write_iter(struct kiocb *, struct iov_iter *); #endif -int scoutfs_permission(struct inode *inode, int mask); +int scoutfs_permission(KC_VFS_NS_DEF + struct inode *inode, int mask); loff_t scoutfs_file_llseek(struct file *file, loff_t offset, int whence); #endif /* _SCOUTFS_FILE_H_ */ diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 04c088c2..5e3f724f 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -373,7 +373,8 @@ int scoutfs_getattr(struct vfsmount *mnt, struct dentry *dentry, { struct inode *inode = dentry->d_inode; #else -int scoutfs_getattr(const struct path *path, struct kstat *stat, +int scoutfs_getattr(KC_VFS_NS_DEF + const struct path *path, struct kstat *stat, u32 request_mask, unsigned int query_flags) { struct inode *inode = d_inode(path->dentry); @@ -385,7 +386,8 @@ int scoutfs_getattr(const struct path *path, struct kstat *stat, ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, SCOUTFS_LKF_REFRESH_INODE, inode, &lock); if (ret == 0) { - generic_fillattr(inode, stat); + generic_fillattr(KC_VFS_INIT_NS + inode, stat); scoutfs_unlock(sb, lock, SCOUTFS_LOCK_READ); } return ret; @@ -483,7 +485,8 @@ int scoutfs_complete_truncate(struct inode *inode, struct scoutfs_lock *lock) * re-acquire it. Ideally we'd fix this so that we can acquire the lock * instead of the caller. */ -int scoutfs_setattr(struct dentry *dentry, struct iattr *attr) +int scoutfs_setattr(KC_VFS_NS_DEF + struct dentry *dentry, struct iattr *attr) { struct inode *inode = dentry->d_inode; struct super_block *sb = inode->i_sb; @@ -501,7 +504,8 @@ retry: SCOUTFS_LKF_REFRESH_INODE, inode, &lock); if (ret) return ret; - ret = setattr_prepare(dentry, attr); + ret = setattr_prepare(KC_VFS_INIT_NS + dentry, attr); if (ret) goto out; @@ -565,7 +569,8 @@ retry: if (ret < 0) goto release; - setattr_copy(inode, attr); + setattr_copy(KC_VFS_INIT_NS + inode, attr); inode_inc_iversion(inode); scoutfs_update_inode_item(inode, lock, &ind_locks); @@ -1562,7 +1567,8 @@ int scoutfs_new_inode(struct super_block *sb, struct inode *dir, umode_t mode, d scoutfs_inode_set_data_seq(inode); inode->i_ino = ino; /* XXX overflow */ - inode_init_owner(inode, dir, mode); + inode_init_owner(KC_VFS_INIT_NS + inode, dir, mode); inode_set_bytes(inode, 0); inode->i_mtime = inode->i_atime = inode->i_ctime = current_time(inode); inode->i_rdev = rdev; diff --git a/kmod/src/inode.h b/kmod/src/inode.h index e068cee5..fcead8fb 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -135,10 +135,12 @@ int scoutfs_inode_refresh(struct inode *inode, struct scoutfs_lock *lock); int scoutfs_getattr(struct vfsmount *mnt, struct dentry *dentry, struct kstat *stat); #else -int scoutfs_getattr(const struct path *path, struct kstat *stat, +int scoutfs_getattr(KC_VFS_NS_DEF + const struct path *path, struct kstat *stat, u32 request_mask, unsigned int query_flags); #endif -int scoutfs_setattr(struct dentry *dentry, struct iattr *attr); +int scoutfs_setattr(KC_VFS_NS_DEF + struct dentry *dentry, struct iattr *attr); int scoutfs_inode_orphan_create(struct super_block *sb, u64 ino, struct scoutfs_lock *lock, struct scoutfs_lock *primary); diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 45fc0e51..9480d339 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -716,7 +716,8 @@ static long scoutfs_ioc_listxattr_hidden(struct file *file, unsigned long arg) int total = 0; int ret; - ret = inode_permission(inode, MAY_READ); + ret = inode_permission(KC_VFS_INIT_NS + inode, MAY_READ); if (ret < 0) goto out; diff --git a/kmod/src/kernelcompat.h b/kmod/src/kernelcompat.h index 1d7acee7..247aa4e1 100644 --- a/kmod/src/kernelcompat.h +++ b/kmod/src/kernelcompat.h @@ -290,6 +290,16 @@ typedef unsigned int blk_opf_t; #define kc__vmalloc __vmalloc #endif +#ifdef KC_VFS_METHOD_USER_NAMESPACE_ARG +#define KC_VFS_NS_DEF struct user_namespace *mnt_user_ns, +#define KC_VFS_NS mnt_user_ns, +#define KC_VFS_INIT_NS &init_user_ns, +#else +#define KC_VFS_NS_DEF +#define KC_VFS_NS +#define KC_VFS_INIT_NS +#endif + #ifdef KC_BIO_ALLOC_DEV_OPF_ARGS #define kc_bio_alloc bio_alloc #else From e4721366ff39b53c863654394e3cc776ce5efe3c Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Tue, 3 Oct 2023 11:28:51 -0700 Subject: [PATCH 12/42] Added user_ns argument to posix_acl_update_mode, set_posix_acl v5.11-rc4-8-ge65ce2a50cf6 adds idmap support to these calls. Signed-off-by: Auke Kok --- kmod/src/acl.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kmod/src/acl.c b/kmod/src/acl.c index 8e371fb2..d0fef58e 100644 --- a/kmod/src/acl.c +++ b/kmod/src/acl.c @@ -153,7 +153,8 @@ int scoutfs_set_acl_locked(struct inode *inode, struct posix_acl *acl, int type, switch (type) { case ACL_TYPE_ACCESS: if (acl) { - ret = posix_acl_update_mode(inode, &new_mode, &acl); + ret = posix_acl_update_mode(KC_VFS_INIT_NS + inode, &new_mode, &acl); if (ret < 0) goto out; set_mode = true; From b718cf09de12f02fef19de93014a79132c0d5b4b Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Mon, 16 Oct 2023 16:04:19 -0700 Subject: [PATCH 13/42] Handle idmapped mounts in xattr_handler In v5.11-rc4-8-ge65ce2a50cf6 the *set handler is passed a user_namespace struct pointing to the map from the mount. Signed-off-by: Auke Kok --- kmod/src/acl.c | 4 +++- kmod/src/acl.h | 4 +++- kmod/src/xattr.c | 4 +++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/kmod/src/acl.c b/kmod/src/acl.c index d0fef58e..0fdb9d93 100644 --- a/kmod/src/acl.c +++ b/kmod/src/acl.c @@ -253,7 +253,9 @@ int scoutfs_acl_get_xattr(struct dentry *dentry, const char *name, void *value, } #ifdef KC_XATTR_STRUCT_XATTR_HANDLER -int scoutfs_acl_set_xattr(const struct xattr_handler *handler, struct dentry *dentry, +int scoutfs_acl_set_xattr(const struct xattr_handler *handler, + KC_VFS_NS_DEF + struct dentry *dentry, struct inode *inode, const char *name, const void *value, size_t size, int flags) { diff --git a/kmod/src/acl.h b/kmod/src/acl.h index 1712b7e8..e7f983e6 100644 --- a/kmod/src/acl.h +++ b/kmod/src/acl.h @@ -10,7 +10,9 @@ int scoutfs_set_acl_locked(struct inode *inode, struct posix_acl *acl, int type, int scoutfs_acl_get_xattr(const struct xattr_handler *, struct dentry *dentry, struct inode *inode, const char *name, void *value, size_t size); -int scoutfs_acl_set_xattr(const struct xattr_handler *, struct dentry *dentry, +int scoutfs_acl_set_xattr(const struct xattr_handler *, + KC_VFS_NS_DEF + struct dentry *dentry, struct inode *inode, const char *name, const void *value, size_t size, int flags); #else diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index a13f3d19..09db9927 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -1026,7 +1026,9 @@ static int scoutfs_xattr_get_handler static int scoutfs_xattr_set_handler #ifdef KC_XATTR_STRUCT_XATTR_HANDLER - (const struct xattr_handler *handler, struct dentry *dentry, + (const struct xattr_handler *handler, + KC_VFS_NS_DEF + struct dentry *dentry, struct inode *inode, const char *name, const void *value, size_t size, int flags) { From 345ebd08761927a95cd3f70a47d9b0561c71cad1 Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Mon, 16 Oct 2023 16:36:22 -0700 Subject: [PATCH 14/42] fiemap_prep replaces fiemap_check_flags. v5.7-rc4-53-gcddf8a2c4a82 The prep helper replaces the sanity checks. Signed-off-by: Auke Kok --- kmod/src/Makefile.kernelcompat | 8 ++++++++ kmod/src/data.c | 2 +- kmod/src/kernelcompat.h | 4 ++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/kmod/src/Makefile.kernelcompat b/kmod/src/Makefile.kernelcompat index 1a9c0da1..5c8d519b 100644 --- a/kmod/src/Makefile.kernelcompat +++ b/kmod/src/Makefile.kernelcompat @@ -314,3 +314,11 @@ endif ifneq (,$(shell grep 'struct bio .bio_alloc.struct block_device .bdev' include/linux/bio.h)) ccflags-y += -DKC_BIO_ALLOC_DEV_OPF_ARGS endif + +# +# v5.7-rc4-53-gcddf8a2c4a82 +# +# fiemap_prep() replaces fiemap_check_flags() +ifneq (,$(shell grep -s 'int fiemap_prep.struct inode' include/linux/fiemap.h)) +ccflags-y += -DKC_FIEMAP_PREP +endif diff --git a/kmod/src/data.c b/kmod/src/data.c index 49ab801d..8a1c65e0 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -1544,7 +1544,7 @@ int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, goto out; } - ret = fiemap_check_flags(fieinfo, FIEMAP_FLAG_SYNC); + ret = fiemap_prep(inode, fieinfo, start, &len, FIEMAP_FLAG_SYNC); if (ret) goto out; diff --git a/kmod/src/kernelcompat.h b/kmod/src/kernelcompat.h index 247aa4e1..665d28fa 100644 --- a/kmod/src/kernelcompat.h +++ b/kmod/src/kernelcompat.h @@ -316,4 +316,8 @@ static inline struct bio *kc_bio_alloc(struct block_device *bdev, unsigned short } #endif +#ifndef KC_FIEMAP_PREP +#define fiemap_prep(inode, fieinfo, start, len, flags) fiemap_check_flags(fieinfo, flags) +#endif + #endif From 8a45c2baff677fd41978691693dd71d0f84256d5 Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Tue, 17 Oct 2023 11:09:39 -0700 Subject: [PATCH 15/42] Deprecate struct timeval. We switch to using 64bit usec structs and recommended replacement functions from Documentation/core-api/timekeeping.rst. Signed-off-by: Auke Kok --- kmod/src/fence.c | 6 +++--- kmod/src/quorum.c | 2 +- kmod/src/server.c | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/kmod/src/fence.c b/kmod/src/fence.c index ff617634..3669dc2d 100644 --- a/kmod/src/fence.c +++ b/kmod/src/fence.c @@ -105,12 +105,12 @@ static ssize_t elapsed_secs_show(struct kobject *kobj, { DECLARE_FENCE_FROM_KOBJ(fence, kobj); ktime_t now = ktime_get(); - struct timeval tv = { 0, }; + ktime_t t = ns_to_ktime(0); if (ktime_after(now, fence->start_kt)) - tv = ktime_to_timeval(ktime_sub(now, fence->start_kt)); + t = ktime_sub(now, fence->start_kt); - return snprintf(buf, PAGE_SIZE, "%llu", (long long)tv.tv_sec); + return snprintf(buf, PAGE_SIZE, "%llu", (long long)ktime_divns(t, NSEC_PER_SEC)); } SCOUTFS_ATTR_RO(elapsed_secs); diff --git a/kmod/src/quorum.c b/kmod/src/quorum.c index da850cc7..ede23494 100644 --- a/kmod/src/quorum.c +++ b/kmod/src/quorum.c @@ -486,7 +486,7 @@ static void set_quorum_block_event(struct super_block *sb, struct scoutfs_quorum if (WARN_ON_ONCE(event < 0 || event >= SCOUTFS_QUORUM_EVENT_NR)) return; - getnstimeofday64(&ts); + ktime_get_ts64(&ts); le64_add_cpu(&blk->write_nr, 1); ev = &blk->events[event]; diff --git a/kmod/src/server.c b/kmod/src/server.c index b49b6cd8..65b5d116 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -298,7 +298,7 @@ static void check_holder_budget(struct super_block *sb, struct server_info *serv { static bool exceeded_once = false; struct commit_hold *hold; - struct timespec ts; + struct timespec64 ts; u32 avail_used; u32 freed_used; u32 avail_now; @@ -330,7 +330,7 @@ static void check_holder_budget(struct super_block *sb, struct server_info *serv cusers->freed_before, freed_now); list_for_each_entry(hold, &cusers->holding, entry) { - ts = ktime_to_timespec(hold->start); + ts = ktime_to_timespec64(hold->start); scoutfs_err(sb, "exceeding hold start %llu.%09llu av %u fr %u", (u64)ts.tv_sec, (u64)ts.tv_nsec, hold->avail, hold->freed); hold->exceeded = true; @@ -445,7 +445,7 @@ static int server_apply_commit(struct super_block *sb, struct commit_hold *hold, { DECLARE_SERVER_INFO(sb, server); struct commit_users *cusers = &server->cusers; - struct timespec ts; + struct timespec64 ts; spin_lock(&cusers->lock); @@ -454,7 +454,7 @@ static int server_apply_commit(struct super_block *sb, struct commit_hold *hold, check_holder_budget(sb, server, cusers); if (hold->exceeded) { - ts = ktime_to_timespec(hold->start); + ts = ktime_to_timespec64(hold->start); scoutfs_err(sb, "exceeding hold start %llu.%09llu stack:", (u64)ts.tv_sec, (u64)ts.tv_nsec); dump_stack(); From f3abf9710bc93bfed457ac1d7b3e163e33e2c3ef Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Tue, 17 Oct 2023 11:14:03 -0700 Subject: [PATCH 16/42] generic_perform_write signature changed It now only needs the iocb and no longer the flip. Signed-off-by: Auke Kok --- kmod/src/Makefile.kernelcompat | 8 ++++++++ kmod/src/kernelcompat.c | 3 +-- kmod/src/kernelcompat.h | 13 +++++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/kmod/src/Makefile.kernelcompat b/kmod/src/Makefile.kernelcompat index 5c8d519b..61a74b8d 100644 --- a/kmod/src/Makefile.kernelcompat +++ b/kmod/src/Makefile.kernelcompat @@ -322,3 +322,11 @@ endif ifneq (,$(shell grep -s 'int fiemap_prep.struct inode' include/linux/fiemap.h)) ccflags-y += -DKC_FIEMAP_PREP endif + +# +# v5.17-13043-g800ba29547e1 +# +# generic_perform_write args use kiocb for passing filp and pos +ifneq (,$(shell grep 'ssize_t generic_perform_write.struct kiocb ., struct iov_iter' include/linux/fs.h)) +ccflags-y += -DKC_GENERIC_PERFORM_WRITE_KIOCB_IOV_ITER +endif diff --git a/kmod/src/kernelcompat.c b/kmod/src/kernelcompat.c index 7f8da413..cf1de599 100644 --- a/kmod/src/kernelcompat.c +++ b/kmod/src/kernelcompat.c @@ -67,12 +67,11 @@ kc_generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov, unsigned long nr_segs, loff_t pos, loff_t *ppos, size_t count, ssize_t written) { - struct file *file = iocb->ki_filp; ssize_t status; struct iov_iter i; iov_iter_init(&i, WRITE, iov, nr_segs, count); - status = generic_perform_write(file, &i, pos); + status = kc_generic_perform_write(iocb, &i, pos); if (likely(status >= 0)) { written += status; diff --git a/kmod/src/kernelcompat.h b/kmod/src/kernelcompat.h index 665d28fa..b79e9ef4 100644 --- a/kmod/src/kernelcompat.h +++ b/kmod/src/kernelcompat.h @@ -271,7 +271,20 @@ ssize_t kc_generic_file_buffered_write(struct kiocb *iocb, const struct iovec *i unsigned long nr_segs, loff_t pos, loff_t *ppos, size_t count, ssize_t written); #define generic_file_buffered_write kc_generic_file_buffered_write +#ifdef KC_GENERIC_PERFORM_WRITE_KIOCB_IOV_ITER +static inline int kc_generic_perform_write(struct kiocb *iocb, struct iov_iter *iter, loff_t pos) +{ + iocb->ki_pos = pos; + return generic_perform_write(iocb, iter); +} +#else +static inline int kc_generic_perform_write(struct kiocb *iocb, struct iov_iter *iter, loff_t pos) +{ + struct file *file = iocb->ki_filp; + return generic_perform_write(file, iter, pos); +} #endif +#endif // KC_GENERIC_FILE_BUFFERED_WRITE #ifndef KC_HAVE_BLK_OPF_T /* typedef __u32 __bitwise blk_opf_t; */ From 418a441604da6a79d0074bc2007e79d7f6f263e6 Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Tue, 17 Oct 2023 11:15:40 -0700 Subject: [PATCH 17/42] kernel_setsockopt no longer available. We instead opt to use sock_setsockopt which is generally exactly the same and can be easily converted to map to kernel_setsockopt without impacting the code significantly. There are 3 methods we're calling with usec timeval's, and that is significantly different now that this requires a bit more compat code so we split these out to separate compat functions to handle them. Some of the TCP sock functions also have a slightly different signature that we want to split them out (struct socket vs. sock). Some further no longer return success, either. Signed-off-by: Auke Kok --- kmod/src/Makefile.kernelcompat | 32 +++++++++++ kmod/src/kernelcompat.h | 101 +++++++++++++++++++++++++++++++++ kmod/src/net.c | 39 +++++-------- kmod/src/quorum.c | 9 +-- 4 files changed, 148 insertions(+), 33 deletions(-) diff --git a/kmod/src/Makefile.kernelcompat b/kmod/src/Makefile.kernelcompat index 61a74b8d..5c1eae89 100644 --- a/kmod/src/Makefile.kernelcompat +++ b/kmod/src/Makefile.kernelcompat @@ -330,3 +330,35 @@ endif ifneq (,$(shell grep 'ssize_t generic_perform_write.struct kiocb ., struct iov_iter' include/linux/fs.h)) ccflags-y += -DKC_GENERIC_PERFORM_WRITE_KIOCB_IOV_ITER endif + +# +# v5.7-rc6-2496-g76ee0785f42a +# +# net: add sock_set_sndtimeo +ifneq (,$(shell grep 'void sock_set_sndtimeo.struct sock' include/net/sock.h)) +ccflags-y += -DKC_SOCK_SET_SNDTIMEO +endif + +# +# v5.8-rc4-1931-gba423fdaa589 +# +# setsockopt functions are now passed a sockptr_t value instead of char* +ifneq (,$(shell grep -s 'include .linux/sockptr.h.' include/linux/net.h)) +ccflags-y += -DKC_SETSOCKOPT_SOCKPTR_T +endif + +# +# v5.7-rc6-2507-g71c48eb81c9e +# +# Adds a bunch of low level TCP sock parameter functions that we want to use. +ifneq (,$(shell grep 'int tcp_sock_set_keepintvl' include/linux/tcp.h)) +ccflags-y += -DKC_HAVE_TCP_SET_SOCKFN +endif + +# +# v4.16-rc3-13-ga84d1169164b +# +# Fixes y2038 issues with struct timeval. +ifneq (,$(shell grep -s '^struct __kernel_old_timeval .' include/uapi/linux/time_types.h)) +ccflags-y += -DKC_KERNEL_OLD_TIMEVAL_STRUCT +endif diff --git a/kmod/src/kernelcompat.h b/kmod/src/kernelcompat.h index b79e9ef4..6e4cdc02 100644 --- a/kmod/src/kernelcompat.h +++ b/kmod/src/kernelcompat.h @@ -333,4 +333,105 @@ static inline struct bio *kc_bio_alloc(struct block_device *bdev, unsigned short #define fiemap_prep(inode, fieinfo, start, len, flags) fiemap_check_flags(fieinfo, flags) #endif +#ifndef KC_KERNEL_OLD_TIMEVAL_STRUCT +#define __kernel_old_timeval timeval +#define ns_to_kernel_old_timeval(ktime) ns_to_timeval(ktime.tv64) +#endif + +#ifdef KC_SOCK_SET_SNDTIMEO +#include +static inline int kc_sock_set_sndtimeo(struct socket *sock, s64 secs) +{ + sock_set_sndtimeo(sock->sk, secs); + return 0; +} +static inline int kc_tcp_sock_set_rcvtimeo(struct socket *sock, ktime_t to) +{ + struct __kernel_old_timeval tv; + sockptr_t kopt; + + tv = ns_to_kernel_old_timeval(to); + + kopt = KERNEL_SOCKPTR(&tv); + + return sock_setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO_NEW, + kopt, sizeof(tv)); +} +#else +#include +static inline int kc_sock_set_sndtimeo(struct socket *sock, s64 secs) +{ + struct timeval tv = { .tv_sec = secs, .tv_usec = 0 }; + return kernel_setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, + (char *)&tv, sizeof(tv)); +} +static inline int kc_tcp_sock_set_rcvtimeo(struct socket *sock, ktime_t to) +{ + struct __kernel_old_timeval tv; + + tv = ns_to_kernel_old_timeval(to); + return kernel_setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, + (char *)&tv, sizeof(tv)); +} +#endif + +#ifdef KC_SETSOCKOPT_SOCKPTR_T +static inline int kc_sock_setsockopt(struct socket *sock, int level, int op, int *optval, unsigned int optlen) +{ + sockptr_t kopt = KERNEL_SOCKPTR(optval); + return sock_setsockopt(sock, level, op, kopt, sizeof(optval)); +} +#else +static inline int kc_sock_setsockopt(struct socket *sock, int level, int op, int *optval, unsigned int optlen) +{ + return kernel_setsockopt(sock, level, op, (char *)optval, sizeof(optval)); +} +#endif + +#ifdef KC_HAVE_TCP_SET_SOCKFN +#include +#include +static inline int kc_tcp_sock_set_keepintvl(struct socket *sock, int val) +{ + return tcp_sock_set_keepintvl(sock->sk, val); +} +static inline int kc_tcp_sock_set_keepidle(struct socket *sock, int val) +{ + return tcp_sock_set_keepidle(sock->sk, val); +} +static inline int kc_tcp_sock_set_user_timeout(struct socket *sock, int val) +{ + tcp_sock_set_user_timeout(sock->sk, val); + return 0; +} +static inline int kc_tcp_sock_set_nodelay(struct socket *sock) +{ + tcp_sock_set_nodelay(sock->sk); + return 0; +} +#else +#include +#include +static inline int kc_tcp_sock_set_keepintvl(struct socket *sock, int val) +{ + int optval = val; + return kernel_setsockopt(sock, SOL_TCP, TCP_KEEPINTVL, (char *)&optval, sizeof(optval)); +} +static inline int kc_tcp_sock_set_keepidle(struct socket *sock, int val) +{ + int optval = val; + return kernel_setsockopt(sock, SOL_TCP, TCP_KEEPIDLE, (char *)&optval, sizeof(optval)); +} +static inline int kc_tcp_sock_set_user_timeout(struct socket *sock, int val) +{ + int optval = val; + return kernel_setsockopt(sock, SOL_TCP, TCP_USER_TIMEOUT, (char *)&optval, sizeof(optval)); +} +static inline int kc_tcp_sock_set_nodelay(struct socket *sock) +{ + int optval = 1; + return kernel_setsockopt(sock, SOL_TCP, TCP_NODELAY, (char *)&optval, sizeof(optval)); +} +#endif + #endif diff --git a/kmod/src/net.c b/kmod/src/net.c index bbcff7db..d7c68743 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -904,53 +904,44 @@ static void destroy_conn(struct scoutfs_net_connection *conn) static int sock_opts_and_names(struct scoutfs_net_connection *conn, struct socket *sock) { - struct timeval tv; int optval; int ret; /* we use a keepalive timeout instead of send timeout */ - tv.tv_sec = 0; - tv.tv_usec = 0; - ret = kernel_setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, - (char *)&tv, sizeof(tv)); + ret = kc_sock_set_sndtimeo(sock, 0); if (ret) goto out; /* not checked when user_timeout != 0, but for clarity */ optval = UNRESPONSIVE_PROBES; - ret = kernel_setsockopt(sock, SOL_TCP, TCP_KEEPCNT, - (char *)&optval, sizeof(optval)); + ret = kc_sock_setsockopt(sock, SOL_TCP, TCP_KEEPCNT, + &optval, sizeof(optval)); if (ret) goto out; BUILD_BUG_ON(UNRESPONSIVE_PROBES >= UNRESPONSIVE_TIMEOUT_SECS); optval = UNRESPONSIVE_TIMEOUT_SECS - (UNRESPONSIVE_PROBES); - ret = kernel_setsockopt(sock, SOL_TCP, TCP_KEEPIDLE, - (char *)&optval, sizeof(optval)); + ret = kc_tcp_sock_set_keepidle(sock, optval); if (ret) goto out; optval = 1; - ret = kernel_setsockopt(sock, SOL_TCP, TCP_KEEPINTVL, - (char *)&optval, sizeof(optval)); + ret = kc_tcp_sock_set_keepintvl(sock, optval); if (ret) goto out; optval = UNRESPONSIVE_TIMEOUT_SECS * MSEC_PER_SEC; - ret = kernel_setsockopt(sock, SOL_TCP, TCP_USER_TIMEOUT, - (char *)&optval, sizeof(optval)); + ret = kc_tcp_sock_set_user_timeout(sock, optval); if (ret) goto out; optval = 1; - ret = kernel_setsockopt(sock, SOL_SOCKET, SO_KEEPALIVE, - (char *)&optval, sizeof(optval)); + ret = kc_sock_setsockopt(sock, SOL_SOCKET, SO_KEEPALIVE, + &optval, sizeof(optval)); if (ret) goto out; - optval = 1; - ret = kernel_setsockopt(sock, SOL_TCP, TCP_NODELAY, - (char *)&optval, sizeof(optval)); + ret = kc_tcp_sock_set_nodelay(sock); if (ret) goto out; @@ -1049,7 +1040,6 @@ static void scoutfs_net_connect_worker(struct work_struct *work) DEFINE_CONN_FROM_WORK(conn, work, connect_work); struct super_block *sb = conn->sb; struct socket *sock; - struct timeval tv; int ret; trace_scoutfs_net_connect_work_enter(sb, 0, 0); @@ -1060,11 +1050,8 @@ static void scoutfs_net_connect_worker(struct work_struct *work) sock->sk->sk_allocation = GFP_NOFS; - /* caller specified connect timeout */ - tv.tv_sec = conn->connect_timeout_ms / MSEC_PER_SEC; - tv.tv_usec = (conn->connect_timeout_ms % MSEC_PER_SEC) * USEC_PER_MSEC; - ret = kernel_setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, - (char *)&tv, sizeof(tv)); + /* caller specified connect timeout, defaults to 1 sec */ + ret = kc_sock_set_sndtimeo(sock, conn->connect_timeout_ms / MSEC_PER_SEC); if (ret) { sock_release(sock); goto out; @@ -1462,8 +1449,8 @@ int scoutfs_net_bind(struct super_block *sb, sock->sk->sk_allocation = GFP_NOFS; optval = 1; - ret = kernel_setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, - (char *)&optval, sizeof(optval)); + ret = kc_sock_setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, + &optval, sizeof(optval)); if (ret) goto out; diff --git a/kmod/src/quorum.c b/kmod/src/quorum.c index ede23494..7308b1dc 100644 --- a/kmod/src/quorum.c +++ b/kmod/src/quorum.c @@ -303,7 +303,6 @@ static int recv_msg(struct super_block *sb, struct quorum_host_msg *msg, DECLARE_QUORUM_INFO(sb, qinf); struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_quorum_message qmes; - struct timeval tv; ktime_t rel_to; ktime_t now; int ret; @@ -328,14 +327,10 @@ static int recv_msg(struct super_block *sb, struct quorum_host_msg *msg, else rel_to = ns_to_ktime(0); - tv = ktime_to_timeval(rel_to); - if (tv.tv_sec == 0 && tv.tv_usec == 0) { + if (ktime_compare(rel_to, ns_to_ktime(NSEC_PER_USEC)) <= 0) { mh.msg_flags |= MSG_DONTWAIT; } else { - ret = kernel_setsockopt(qinf->sock, SOL_SOCKET, SO_RCVTIMEO, - (char *)&tv, sizeof(tv)); - if (ret < 0) - return ret; + ret = kc_tcp_sock_set_rcvtimeo(qinf->sock, rel_to); } #ifdef KC_MSGHDR_STRUCT_IOV_ITER From 381f4543b7502f5919b94ca292405ebe4dffdc9e Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Wed, 29 Nov 2023 10:25:16 -0800 Subject: [PATCH 18/42] Use iter based read/write to support splice and thus sendfile(). The iter based read/write calls can support splice in el9 if we hook up these calls, otherwise splice will stop working. ->write() similar to: v3.15-rc4-330-g8d0207652cbe. ->read() to generic implementation. Signed-off-by: Auke Kok --- kmod/src/data.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kmod/src/data.c b/kmod/src/data.c index 8a1c65e0..eac86c5c 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -1912,6 +1912,8 @@ const struct file_operations scoutfs_file_fops = { #else .read_iter = scoutfs_file_read_iter, .write_iter = scoutfs_file_write_iter, + .splice_read = generic_file_splice_read, + .splice_write = iter_file_splice_write, #endif .unlocked_ioctl = scoutfs_ioctl, .fsync = scoutfs_file_fsync, From 546b437df78ab5713028a065eed4feb8140b6a3a Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Fri, 19 Jan 2024 12:10:10 -0800 Subject: [PATCH 19/42] Shrinkers are now registered with a name. v5.19-rc4-52-ge33c267ab70d Adds shrinker names to the registration call to aid with shrinker debugging, which is highly opaque. To enable you'll have to recompile the kernel with CONFIG_SHRINKER_DEBUG=y though, since it's disabled by default in OSV kernels. Signed-off-by: Auke Kok --- kmod/src/Makefile.kernelcompat | 8 ++++++++ kmod/src/block.c | 2 +- kmod/src/item.c | 2 +- kmod/src/kernelcompat.h | 8 ++++++-- kmod/src/lock.c | 2 +- kmod/src/quota.c | 2 +- kmod/src/wkic.c | 2 +- 7 files changed, 19 insertions(+), 7 deletions(-) diff --git a/kmod/src/Makefile.kernelcompat b/kmod/src/Makefile.kernelcompat index 5c1eae89..31b58708 100644 --- a/kmod/src/Makefile.kernelcompat +++ b/kmod/src/Makefile.kernelcompat @@ -362,3 +362,11 @@ endif ifneq (,$(shell grep -s '^struct __kernel_old_timeval .' include/uapi/linux/time_types.h)) ccflags-y += -DKC_KERNEL_OLD_TIMEVAL_STRUCT endif + +# +# v5.19-rc4-52-ge33c267ab70d +# +# register_shrinker now requires a name, used for debug stats etc. +ifneq (,$(shell grep 'int __printf.*register_shrinker.struct shrinker .shrinker,' include/linux/shrinker.h)) +ccflags-y += -DKC_SHRINKER_NAME +endif diff --git a/kmod/src/block.c b/kmod/src/block.c index 23a37b98..9eaba500 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -1297,7 +1297,7 @@ int scoutfs_block_setup(struct super_block *sb) init_waitqueue_head(&binf->waitq); KC_INIT_SHRINKER_FUNCS(&binf->shrinker, block_count_objects, block_scan_objects); - KC_REGISTER_SHRINKER(&binf->shrinker); + KC_REGISTER_SHRINKER(&binf->shrinker, "scoutfs-block:" SCSBF, SCSB_ARGS(sb)); INIT_WORK(&binf->free_work, block_free_work); init_llist_head(&binf->free_llist); diff --git a/kmod/src/item.c b/kmod/src/item.c index f7691e6e..44357e6d 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -2693,7 +2693,7 @@ int scoutfs_item_setup(struct super_block *sb) KC_INIT_SHRINKER_FUNCS(&cinf->shrinker, item_cache_count_objects, item_cache_scan_objects); - KC_REGISTER_SHRINKER(&cinf->shrinker); + KC_REGISTER_SHRINKER(&cinf->shrinker, "scoutfs-item:" SCSBF, SCSB_ARGS(sb)); #ifdef KC_CPU_NOTIFIER cinf->notifier.notifier_call = item_cpu_callback; register_hotcpu_notifier(&cinf->notifier); diff --git a/kmod/src/kernelcompat.h b/kmod/src/kernelcompat.h index 6e4cdc02..8c17d31e 100644 --- a/kmod/src/kernelcompat.h +++ b/kmod/src/kernelcompat.h @@ -197,7 +197,11 @@ struct timespec64 kc_current_time(struct inode *inode); } while (0) #define KC_SHRINKER_CONTAINER_OF(ptr, type) container_of(ptr, type, shrinker) -#define KC_REGISTER_SHRINKER(ptr) (register_shrinker(ptr)) +#ifdef KC_SHRINKER_NAME +#define KC_REGISTER_SHRINKER register_shrinker +#else +#define KC_REGISTER_SHRINKER(ptr, fmt, ...) (register_shrinker(ptr)) +#endif /* KC_SHRINKER_NAME */ #define KC_UNREGISTER_SHRINKER(ptr) (unregister_shrinker(ptr)) #define KC_SHRINKER_FN(ptr) (ptr) #else @@ -224,7 +228,7 @@ struct kc_shrinker_wrapper { _wrap->shrink.seeks = DEFAULT_SEEKS; \ } while (0) #define KC_SHRINKER_CONTAINER_OF(ptr, type) container_of(container_of(ptr, struct kc_shrinker_wrapper, shrink), type, shrinker) -#define KC_REGISTER_SHRINKER(ptr) (register_shrinker(ptr.shrink)) +#define KC_REGISTER_SHRINKER(ptr, fmt, ...) (register_shrinker(ptr.shrink)) #define KC_UNREGISTER_SHRINKER(ptr) (unregister_shrinker(ptr.shrink)) #define KC_SHRINKER_FN(ptr) (ptr.shrink) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index a6ae9a6e..5f280054 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -1732,7 +1732,7 @@ int scoutfs_lock_setup(struct super_block *sb) linfo->lock_range_tree = RB_ROOT; KC_INIT_SHRINKER_FUNCS(&linfo->shrinker, lock_count_objects, lock_scan_objects); - KC_REGISTER_SHRINKER(&linfo->shrinker); + KC_REGISTER_SHRINKER(&linfo->shrinker, "scoutfs-lock:" SCSBF, SCSB_ARGS(sb)); INIT_LIST_HEAD(&linfo->lru_list); INIT_WORK(&linfo->inv_work, lock_invalidate_worker); INIT_LIST_HEAD(&linfo->inv_list); diff --git a/kmod/src/quota.c b/kmod/src/quota.c index 04ec7244..3163ee8c 100644 --- a/kmod/src/quota.c +++ b/kmod/src/quota.c @@ -1226,7 +1226,7 @@ int scoutfs_quota_setup(struct super_block *sb) init_waitqueue_head(&qtinf->waitq); KC_INIT_SHRINKER_FUNCS(&qtinf->shrinker, count_cached_checks, scan_cached_checks); - KC_REGISTER_SHRINKER(&qtinf->shrinker); + KC_REGISTER_SHRINKER(&qtinf->shrinker, "scoutfs-quota:" SCSBF, SCSB_ARGS(sb)); sbi->squota_info = qtinf; diff --git a/kmod/src/wkic.c b/kmod/src/wkic.c index ebd1b19c..44eef494 100644 --- a/kmod/src/wkic.c +++ b/kmod/src/wkic.c @@ -1113,7 +1113,7 @@ int scoutfs_wkic_setup(struct super_block *sb) winf->sb = sb; KC_INIT_SHRINKER_FUNCS(&winf->shrinker, wkic_shrink_count, wkic_shrink_scan); - KC_REGISTER_SHRINKER(&winf->shrinker); + KC_REGISTER_SHRINKER(&winf->shrinker, "scoutfs-weak_item:" SCSBF, SCSB_ARGS(sb)); sbi->wkic_info = winf; return 0; From e2b06f2c921d20debf82e284c822cd60fc27629a Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Fri, 19 Jan 2024 12:13:28 -0800 Subject: [PATCH 20/42] mpage_readpage() is now replaced with mpage_read_folio. Folios are the new data types used for passing pages. For now, folios only appear to have a single page. Future kernels will change that. Signed-off-by: Auke Kok --- kmod/src/Makefile.kernelcompat | 8 ++++++++ kmod/src/data.c | 14 ++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/kmod/src/Makefile.kernelcompat b/kmod/src/Makefile.kernelcompat index 31b58708..7a579e22 100644 --- a/kmod/src/Makefile.kernelcompat +++ b/kmod/src/Makefile.kernelcompat @@ -370,3 +370,11 @@ endif ifneq (,$(shell grep 'int __printf.*register_shrinker.struct shrinker .shrinker,' include/linux/shrinker.h)) ccflags-y += -DKC_SHRINKER_NAME endif + +# +# v5.18-rc5-246-gf132ab7d3ab0 +# +# mpage_readpage() is now replaced with mpage_read_folio. +ifneq (,$(shell grep 'int mpage_read_folio.struct folio .folio' include/linux/mpage.h)) +ccflags-y += -DKC_MPAGE_READ_FOLIO +endif diff --git a/kmod/src/data.c b/kmod/src/data.c index eac86c5c..6949fbea 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -680,8 +680,14 @@ int scoutfs_get_block_write(struct inode *inode, sector_t iblock, struct buffer_ * We can return errors from locking and checking offline extents. The * page is unlocked if we return an error. */ +#ifdef KC_MPAGE_READ_FOLIO +static int scoutfs_read_folio(struct file *file, struct folio *folio) +{ + struct page *page = &folio->page; +#else static int scoutfs_readpage(struct file *file, struct page *page) { +#endif struct inode *inode = file->f_inode; struct scoutfs_inode_info *si = SCOUTFS_I(inode); struct super_block *sb = inode->i_sb; @@ -728,7 +734,11 @@ static int scoutfs_readpage(struct file *file, struct page *page) return ret; } +#ifdef KC_MPAGE_READ_FOLIO + ret = mpage_read_folio(folio, scoutfs_get_block_read); +#else ret = mpage_readpage(page, scoutfs_get_block_read); +#endif scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_READ); scoutfs_per_task_del(&si->pt_data_lock, &pt_ent); @@ -1891,7 +1901,11 @@ int scoutfs_data_waiting(struct super_block *sb, u64 ino, u64 iblock, } const struct address_space_operations scoutfs_file_aops = { +#ifdef KC_MPAGE_READ_FOLIO + .read_folio = scoutfs_read_folio, +#else .readpage = scoutfs_readpage, +#endif #ifndef KC_FILE_AOPS_READAHEAD .readpages = scoutfs_readpages, #else From 5b260e6b5483412fa782bec0c6b2c5edb55709b9 Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Fri, 19 Jan 2024 12:15:50 -0800 Subject: [PATCH 21/42] block_write_begin() no longer is being passed aop_flags. The flag is now obsolete, we don't need to set flags here or pass them. Signed-off-by: Auke Kok --- kmod/src/Makefile.kernelcompat | 8 ++++++++ kmod/src/data.c | 14 +++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/kmod/src/Makefile.kernelcompat b/kmod/src/Makefile.kernelcompat index 7a579e22..54364a58 100644 --- a/kmod/src/Makefile.kernelcompat +++ b/kmod/src/Makefile.kernelcompat @@ -378,3 +378,11 @@ endif ifneq (,$(shell grep 'int mpage_read_folio.struct folio .folio' include/linux/mpage.h)) ccflags-y += -DKC_MPAGE_READ_FOLIO endif + +# +# v5.18-rc5-219-gb3992d1e2ebc +# +# block_write_begin() no longer is being passed aop_flags +ifneq (,$(shell grep -C1 'int block_write_begin' include/linux/buffer_head.h | tail -n 2 | grep 'unsigned flags')) +ccflags-y += -DKC_BLOCK_WRITE_BEGIN_AOP_FLAGS +endif diff --git a/kmod/src/data.c b/kmod/src/data.c index 6949fbea..990c9029 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -836,7 +836,10 @@ struct write_begin_data { static int scoutfs_write_begin(struct file *file, struct address_space *mapping, loff_t pos, - unsigned len, unsigned flags, + unsigned len, +#ifdef KC_BLOCK_WRITE_BEGIN_AOP_FLAGS + unsigned flags, +#endif struct page **pagep, void **fsdata) { struct inode *inode = mapping->host; @@ -871,13 +874,18 @@ retry: if (ret < 0) goto out; +#ifdef KC_BLOCK_WRITE_BEGIN_AOP_FLAGS /* can't re-enter fs, have trans */ flags |= AOP_FLAG_NOFS; +#endif /* generic write_end updates i_size and calls dirty_inode */ ret = scoutfs_dirty_inode_item(inode, wbd->lock) ?: - block_write_begin(mapping, pos, len, flags, pagep, - scoutfs_get_block_write); + block_write_begin(mapping, pos, len, +#ifdef KC_BLOCK_WRITE_BEGIN_AOP_FLAGS + flags, +#endif + pagep, scoutfs_get_block_write); if (ret < 0) { scoutfs_release_trans(sb); scoutfs_inode_index_unlock(sb, &wbd->ind_locks); From 676d42926404477dac1b1ee4a6f0f26243cbbc9a Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Wed, 24 Jan 2024 11:30:48 -0800 Subject: [PATCH 22/42] Assume el9 is the same as el8 for rpmbuild purposes. The current spec template can't handle future major el releases gracefully and fails to build entirely. We isolate all changes so that they are either "el7 specific" or generic. This rids us entirely of el8 specific conditionals. Signed-off-by: Auke Kok --- kmod/scoutfs-kmod.spec.in | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/kmod/scoutfs-kmod.spec.in b/kmod/scoutfs-kmod.spec.in index b22df01c..23b76eff 100644 --- a/kmod/scoutfs-kmod.spec.in +++ b/kmod/scoutfs-kmod.spec.in @@ -13,8 +13,7 @@ %if 0%{?el7} %global kernel_source() /usr/src/kernels/%{kernel_version}.$(arch) -%endif -%if 0%{?el8} +%else %global kernel_source() /usr/src/kernels/%{kernel_version} %endif @@ -22,8 +21,7 @@ %if 0%{?el7} Name: %{kmod_name} -%endif -%if 0%{?el8} +%else Name: kmod-%{kmod_name} %endif Summary: %{kmod_name} kernel module @@ -35,8 +33,7 @@ URL: http://scoutfs.org/ %if 0%{?el7} BuildRequires: %{kernel_module_package_buildreqs} -%endif -%if 0%{?el8} +%else BuildRequires: elfutils-libelf-devel %endif BuildRequires: kernel-devel-uname-r = %{kernel_version} @@ -54,7 +51,8 @@ Source: %{kmod_name}-kmod-%{kmod_version}.tar %endif %global install_mod_dir extra/%{kmod_name} -%if 0%{?el8} + +%if ! 0%{?el7} %global flavors_to_build x86_64 %endif @@ -93,7 +91,7 @@ done # mark modules executable so that strip-to-file can strip them find %{buildroot} -type f -name \*.ko -exec %{__chmod} u+x \{\} \; -%if 0%{?el8} +%if ! 0%{?el7} %files /lib/modules From d5c2768f04f93897b13be05a4cd2354577726d5f Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Thu, 16 May 2024 11:04:09 -0700 Subject: [PATCH 23/42] .tmpfile method now passed a struct file, which must be opened. v6.0-rc6-9-g863f144f12ad changes the VFS method to pass in a struct file and not a dentry in preperation for tmpfile support in fuse. Signed-off-by: Auke Kok --- kmod/src/Makefile.kernelcompat | 10 ++++++++++ kmod/src/dir.c | 19 ++++++++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/kmod/src/Makefile.kernelcompat b/kmod/src/Makefile.kernelcompat index 54364a58..7e3235b5 100644 --- a/kmod/src/Makefile.kernelcompat +++ b/kmod/src/Makefile.kernelcompat @@ -386,3 +386,13 @@ endif ifneq (,$(shell grep -C1 'int block_write_begin' include/linux/buffer_head.h | tail -n 2 | grep 'unsigned flags')) ccflags-y += -DKC_BLOCK_WRITE_BEGIN_AOP_FLAGS endif + +# +# v6.0-rc6-9-g863f144f12ad +# +# the .tmpfile() vfs method calling convention changed and now a struct +# file* is passed to this metiond instead of a dentry. The function also +# should open the created file and call finish_open_simple() before returning. +ifneq (,$(shell grep 'extern void d_tmpfile.struct dentry' include/linux/dcache.h)) +ccflags-y += -DKC_D_TMPFILE_DENTRY +endif diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 6634cbf4..21c20274 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -1875,8 +1875,17 @@ static int scoutfs_dir_open(struct inode *inode, struct file *file) #endif static int scoutfs_tmpfile(KC_VFS_NS_DEF - struct inode *dir, struct dentry *dentry, umode_t mode) + struct inode *dir, +#ifdef KC_D_TMPFILE_DENTRY + struct dentry *dentry, +#else + struct file *file, +#endif + umode_t mode) { +#ifndef KC_D_TMPFILE_DENTRY + struct dentry *dentry = file->f_path.dentry; +#endif struct super_block *sb = dir->i_sb; struct inode *inode = NULL; struct scoutfs_lock *dir_lock = NULL; @@ -1903,7 +1912,11 @@ static int scoutfs_tmpfile(KC_VFS_NS_DEF si->crtime = inode->i_mtime; insert_inode_hash(inode); ihold(inode); /* need to update inode modifications in d_tmpfile */ +#ifdef KC_D_TMPFILE_DENTRY d_tmpfile(dentry, inode); +#else + d_tmpfile(file, inode); +#endif inode_inc_iversion(inode); scoutfs_forest_inc_inode_count(sb); @@ -1911,6 +1924,10 @@ static int scoutfs_tmpfile(KC_VFS_NS_DEF scoutfs_update_inode_item(dir, dir_lock, &ind_locks); scoutfs_inode_index_unlock(sb, &ind_locks); +#ifndef KC_D_TMPFILE_DENTRY + ret = finish_open_simple(file, 0); +#endif + out: scoutfs_release_trans(sb); scoutfs_inode_index_unlock(sb, &ind_locks); From 95f4e56546afa7f4cbe928658b2f2ad27817dc7d Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Fri, 17 May 2024 14:20:47 -0700 Subject: [PATCH 24/42] Introduce blk_mode_t instead of abuse of fmode_t v6.4-rc2-198-g05bdb9965305 adds a new type for passing flags instead of abusing fmode_t flags. They are essentially the same flags just in a new type. Signed-off-by: Auke Kok --- kmod/src/Makefile.kernelcompat | 8 ++++++++ kmod/src/super.h | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/kmod/src/Makefile.kernelcompat b/kmod/src/Makefile.kernelcompat index 7e3235b5..ee3de8f9 100644 --- a/kmod/src/Makefile.kernelcompat +++ b/kmod/src/Makefile.kernelcompat @@ -396,3 +396,11 @@ endif ifneq (,$(shell grep 'extern void d_tmpfile.struct dentry' include/linux/dcache.h)) ccflags-y += -DKC_D_TMPFILE_DENTRY endif + +# +# v6.4-rc2-201-g0733ad800291 +# +# New blk_mode_t replaces abuse of fmode_t +ifneq (,$(shell grep 'typedef unsigned int __bitwise blk_mode_t' include/linux/blkdev.h)) +ccflags-y += -DKC_HAVE_BLK_MODE_T +endif diff --git a/kmod/src/super.h b/kmod/src/super.h index 03c6a6ea..45ec8450 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -101,7 +101,11 @@ static inline bool SCOUTFS_IS_META_BDEV(struct scoutfs_super_block *super_block) return !!(le64_to_cpu(super_block->flags) & SCOUTFS_FLAG_IS_META_BDEV); } +#ifdef KC_HAVE_BLK_MODE_T +#define SCOUTFS_META_BDEV_MODE (BLK_OPEN_READ | BLK_OPEN_WRITE | BLK_OPEN_EXCL) +#else #define SCOUTFS_META_BDEV_MODE (FMODE_READ | FMODE_WRITE | FMODE_EXCL) +#endif static inline bool scoutfs_forcing_unmount(struct super_block *sb) { From c298360a49ea1b0ab8fd766bcf60355816b4d24b Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Mon, 20 May 2024 10:42:21 -0700 Subject: [PATCH 25/42] blkdev api changes - pass holder to replace FMODE_EXCL Passing a holder ptr to these functions now replaces the FMODE_EXCL flag. _put no longer needs flags for this reason, but the holder instead. Signed-off-by: Auke Kok --- kmod/src/Makefile.kernelcompat | 9 +++++++++ kmod/src/super.c | 8 ++++++++ 2 files changed, 17 insertions(+) diff --git a/kmod/src/Makefile.kernelcompat b/kmod/src/Makefile.kernelcompat index ee3de8f9..d1a21b11 100644 --- a/kmod/src/Makefile.kernelcompat +++ b/kmod/src/Makefile.kernelcompat @@ -404,3 +404,12 @@ endif ifneq (,$(shell grep 'typedef unsigned int __bitwise blk_mode_t' include/linux/blkdev.h)) ccflags-y += -DKC_HAVE_BLK_MODE_T endif + +# +# v6.4-rc2-186-g2736e8eeb0cc +# +# Reworks FMODE_EXCL kludge and instead modifies the blkdev_put() call to pass in +# the (exclusive) holder to implement FMODE_EXCL handling. +ifneq (,$(shell grep 'blkdev_put.struct block_device .bdev, void .holder' include/linux/blkdev.h)) +ccflags-y += -DKC_BLKDEV_PUT_HOLDER_ARG +endif diff --git a/kmod/src/super.c b/kmod/src/super.c index 0086d7fb..b9428ca3 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -160,7 +160,11 @@ static void scoutfs_metadev_close(struct super_block *sb) * from kill_sb->put_super. */ lockdep_off(); +#ifdef KC_BLKDEV_PUT_HOLDER_ARG + blkdev_put(sbi->meta_bdev, sb); +#else blkdev_put(sbi->meta_bdev, SCOUTFS_META_BDEV_MODE); +#endif lockdep_on(); sbi->meta_bdev = NULL; } @@ -523,7 +527,11 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) goto out; } +#ifdef KC_BLKDEV_PUT_HOLDER_ARG + meta_bdev = blkdev_get_by_path(opts.metadev_path, SCOUTFS_META_BDEV_MODE, sb, NULL); +#else meta_bdev = blkdev_get_by_path(opts.metadev_path, SCOUTFS_META_BDEV_MODE, sb); +#endif if (IS_ERR(meta_bdev)) { scoutfs_err(sb, "could not open metadev: error %ld", PTR_ERR(meta_bdev)); From 570c05898ceb6056f5227214c08b44b6a9278b81 Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Thu, 23 May 2024 10:48:56 -0700 Subject: [PATCH 26/42] Correct endian conversion length (blkno is le64) Trivial correction of wrong bitlength conversion. Signed-off-by: Auke Kok --- utils/src/util.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/src/util.c b/utils/src/util.c index 765103eb..f7acccd0 100644 --- a/utils/src/util.c +++ b/utils/src/util.c @@ -145,7 +145,7 @@ int read_block_verify(int fd, u32 magic, u64 fsid, u64 blkno, int shift, void ** else if (fsid != 0 && le64_to_cpu(hdr->fsid) != fsid) fprintf(stderr, "read blkno %llu has bad fsid %016llx != expected %016llx\n", blkno, le64_to_cpu(hdr->fsid), fsid); - else if (le32_to_cpu(hdr->blkno) != blkno) + else if (le64_to_cpu(hdr->blkno) != blkno) fprintf(stderr, "read blkno %llu has bad blkno %llu != expected %llu\n", blkno, le64_to_cpu(hdr->blkno), blkno); else From 00ebe92186204bdab83d914ca25e4aa2dc3affac Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Thu, 23 May 2024 10:50:14 -0700 Subject: [PATCH 27/42] Add stddef.h to util.h to avoid duplicate offsetof() def. In el9 releases, our includes declare offsetof() before our header chain includes stddef.h, which doesn't properly check if offsetof is already defined, leading to a redefinition. Just include stddef at all times here. Signed-off-by: Auke Kok --- utils/src/util.h | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/src/util.h b/utils/src/util.h index ce8f0ebc..2fafc0e9 100644 --- a/utils/src/util.h +++ b/utils/src/util.h @@ -2,6 +2,7 @@ #define _UTIL_H_ #include +#include #include #include #include From 6d42d260cf96def925ff5ed0cf307d03c5cb10d2 Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Tue, 13 Feb 2024 13:01:04 -0800 Subject: [PATCH 28/42] xargs option conflict now a warning in el9 The warnings thrown by el9's version of xargs are unexpected output and cause this test to fail. When using the -I option (replace) the -n 1 arguments are always assumed. In el7/8 no warnings were printed. We can just remove `-n 1` since the argument is never needed. Signed-off-by: Auke Kok --- tests/tests/quota.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests/quota.sh b/tests/tests/quota.sh index 394c9d05..00004042 100644 --- a/tests/tests/quota.sh +++ b/tests/tests/quota.sh @@ -22,7 +22,7 @@ reset_all() getfattr --absolute-names -d -m - "$T_D0" | \ grep "^scoutfs.totl." | \ cut -d '=' -f 1 | \ - xargs -n 1 -I'{}' setfattr -x '{}' "$T_D0" + xargs -I'{}' setfattr -x '{}' "$T_D0" } echo "== prepare dir with write perm for test ids" From ac00f5cedb3d29d8def0384137e52952462d0a80 Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Tue, 13 Feb 2024 13:02:55 -0800 Subject: [PATCH 29/42] Free after getline(), even if fail, and catch eof() on el9 getline() allocates the space for the return value even if there is an error, so when it returns an error, we still have to free() it. In el9, when reading stdin we will get errno=0 returned (no error) when we hit the end of stdin. This behavior is different from el7/8. We don't want to throw an error here to avoid failing the test, since it doesn't. Signed-off-by: Auke Kok --- utils/src/quota.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utils/src/quota.c b/utils/src/quota.c index 37d69f79..e9db420f 100644 --- a/utils/src/quota.c +++ b/utils/src/quota.c @@ -385,7 +385,8 @@ static int parse_stdin_in_fn(int fd, struct scoutfs_ioctl_quota_rule *irules, si ret = getline(&line, &size, stdin); if (ret < 0) { - if (errno == ENOENT) + free(line); + if ((errno == ENOENT) || (errno == 0)) return 0; ret = -errno; From 69de6d7a745a319413f38c6659c4f75ffe20537b Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Tue, 14 Nov 2023 15:06:52 -0500 Subject: [PATCH 30/42] Check for zero len in scoutfs_data_wait_check We consistently enter scoutfs_data_wait_check when len == 0 from scoutfs_aio_write() which directly passes the i_size_read() value, and for cases where we `echo >> $FILE` this is always reached. This can cause the wrapping check to fail since `0 + (0 - 1) < 0` which triggers the WARN_ON_ONCE wrap check that needs updating to allow certain operations on huge files. More importantly we can just omit all these checks if `len == 0` anyway, since they should always succeed and never should require taking all the locks. Signed-off-by: Auke Kok --- kmod/src/data.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kmod/src/data.c b/kmod/src/data.c index 990c9029..16639f95 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -1730,6 +1730,9 @@ int scoutfs_data_wait_check(struct inode *inode, loff_t pos, loff_t len, u64 off; int ret = 0; + if (len == 0) + goto out; + if (WARN_ON_ONCE(sef & SEF_UNKNOWN) || WARN_ON_ONCE(op & SCOUTFS_IOC_DWO_UNKNOWN) || WARN_ON_ONCE(dw && !RB_EMPTY_NODE(&dw->node)) || From 7d0e7e29f81fb4afe1a0105815c304672e8edc26 Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Fri, 23 Aug 2024 12:42:08 -0700 Subject: [PATCH 31/42] Avoid integer wrapping pitfalls for (off, len) pairs. We use check_add_overflow(a, b, d) here to validate that (off, len) pairs do not exceed the max value type. The kernel conveniently has several macros to sort out the problems with signed or unsigned types. However, we're not interested in purely seeing whether (a + b) overflows, because we're using this for (off, len) overflow checks, where the bytes we read are from 0 to len -1. We must therefore call this check with (b) being "len - 1". I've made sure that we don't accidentally fail when (len == 0) in all cases by making sure we've already checked this condition before, and moving code around as needed to ensure that (len > 0) in all cases where we check. The macro check_add_overflow requires a (d) argument in which temporarily the result of the addition is stored and then checked to see if an overflow occurred. We put a `tmp` variable on the stack of the correct type as needed to make the checks function. simple-release-extents test mistakenly relied on this buggy wrap code, so it needs fixing. The move-blocks test also got it wrong. Signed-off-by: Auke Kok --- kmod/src/client.c | 4 +++- kmod/src/data.c | 15 +++++++++------ kmod/src/ioctl.c | 11 +++++++---- tests/tests/move-blocks.sh | 6 +++--- tests/tests/simple-release-extents.sh | 5 ++++- 5 files changed, 26 insertions(+), 15 deletions(-) diff --git a/kmod/src/client.c b/kmod/src/client.c index 643b5693..9706d0b6 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -20,6 +20,7 @@ #include #include #include +#include #include "format.h" #include "counters.h" @@ -68,6 +69,7 @@ int scoutfs_client_alloc_inodes(struct super_block *sb, u64 count, struct client_info *client = SCOUTFS_SB(sb)->client_info; struct scoutfs_net_inode_alloc ial; __le64 lecount = cpu_to_le64(count); + u64 tmp; int ret; ret = scoutfs_net_sync_request(sb, client->conn, @@ -80,7 +82,7 @@ int scoutfs_client_alloc_inodes(struct super_block *sb, u64 count, if (*nr == 0) ret = -ENOSPC; - else if (*ino + *nr < *ino) + else if (check_add_overflow(*ino, *nr - 1, &tmp)) ret = -EINVAL; } diff --git a/kmod/src/data.c b/kmod/src/data.c index 16639f95..7045d547 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -22,6 +22,7 @@ #include #include #include +#include #include "format.h" #include "super.h" @@ -1087,6 +1088,7 @@ long scoutfs_fallocate(struct file *file, int mode, loff_t offset, loff_t len) loff_t end; u64 iblock; u64 last; + loff_t tmp; s64 ret; /* XXX support more flags */ @@ -1095,14 +1097,14 @@ long scoutfs_fallocate(struct file *file, int mode, loff_t offset, loff_t len) goto out; } - /* catch wrapping */ - if (offset + len < offset) { - ret = -EINVAL; + if (len == 0) { + ret = 0; goto out; } - if (len == 0) { - ret = 0; + /* catch wrapping */ + if (check_add_overflow(offset, len - 1, &tmp)) { + ret = -EINVAL; goto out; } @@ -1728,6 +1730,7 @@ int scoutfs_data_wait_check(struct inode *inode, loff_t pos, loff_t len, u64 last_block; u64 on; u64 off; + loff_t tmp; int ret = 0; if (len == 0) @@ -1736,7 +1739,7 @@ int scoutfs_data_wait_check(struct inode *inode, loff_t pos, loff_t len, if (WARN_ON_ONCE(sef & SEF_UNKNOWN) || WARN_ON_ONCE(op & SCOUTFS_IOC_DWO_UNKNOWN) || WARN_ON_ONCE(dw && !RB_EMPTY_NODE(&dw->node)) || - WARN_ON_ONCE(pos + len < pos)) { + WARN_ON_ONCE(check_add_overflow(pos, len - 1, &tmp))) { ret = -EINVAL; goto out; } diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 9480d339..a844a9b6 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -23,6 +23,7 @@ #include #include #include +#include #include "format.h" #include "key.h" @@ -47,6 +48,7 @@ #include "wkic.h" #include "quota.h" #include "scoutfs_trace.h" +#include "util.h" /* * We make inode index items coherent by locking fixed size regions of @@ -288,6 +290,7 @@ static long scoutfs_ioc_release(struct file *file, unsigned long arg) u64 online; u64 offline; u64 isize; + __u64 tmp; int ret; if (copy_from_user(&args, (void __user *)arg, sizeof(args))) @@ -297,12 +300,11 @@ static long scoutfs_ioc_release(struct file *file, unsigned long arg) if (args.length == 0) return 0; - if (((args.offset + args.length) < args.offset) || + if ((check_add_overflow(args.offset, args.length - 1, &tmp)) || (args.offset & SCOUTFS_BLOCK_SM_MASK) || (args.length & SCOUTFS_BLOCK_SM_MASK)) return -EINVAL; - ret = mnt_want_write_file(file); if (ret) return ret; @@ -955,6 +957,7 @@ static long scoutfs_ioc_move_blocks(struct file *file, unsigned long arg) struct scoutfs_ioctl_move_blocks mb; struct file *from_file; struct inode *from; + u64 tmp; int ret; if (copy_from_user(&mb, umb, sizeof(mb))) @@ -963,8 +966,8 @@ static long scoutfs_ioc_move_blocks(struct file *file, unsigned long arg) if (mb.len == 0) return 0; - if (mb.from_off + mb.len < mb.from_off || - mb.to_off + mb.len < mb.to_off) + if ((check_add_overflow(mb.from_off, mb.len - 1, &tmp)) || + (check_add_overflow(mb.to_off, mb.len - 1, &tmp))) return -EOVERFLOW; from_file = fget(mb.from_fd); diff --git a/tests/tests/move-blocks.sh b/tests/tests/move-blocks.sh index c930589c..b26669e8 100644 --- a/tests/tests/move-blocks.sh +++ b/tests/tests/move-blocks.sh @@ -83,9 +83,9 @@ touch "$OTHER" ln "$FROM" "$HARD" echo "== wrapped offsets should fail" -HUGE=0x8000000000000000 -scoutfs move-blocks "$FROM" -f "$HUGE" -l "$HUGE" "$TO" -t 0 2>&1 | t_filter_fs -scoutfs move-blocks "$FROM" -f 0 -l "$HUGE" "$TO" -t "$HUGE" 2>&1 | t_filter_fs +HUGE=0xfffffffffffff000 +scoutfs move-blocks "$FROM" -f "$HUGE" -l "8192" "$TO" -t 0 2>&1 | t_filter_fs +scoutfs move-blocks "$FROM" -f 0 -l "$HUGE" "$TO" -t "8192" 2>&1 | t_filter_fs echo "== specifying same file fails" scoutfs move-blocks "$FROM" -f 0 -l "$BS" "$HARD" -t 0 2>&1 | t_filter_fs diff --git a/tests/tests/simple-release-extents.sh b/tests/tests/simple-release-extents.sh index da3eee3d..51dfeb1b 100644 --- a/tests/tests/simple-release-extents.sh +++ b/tests/tests/simple-release-extents.sh @@ -61,7 +61,10 @@ echo "== release past i_size is fine" release_vers "$FILE" stat 400K 4K echo "== wrapped blocks fails" -release_vers "$FILE" stat $vers 0x8000000000000000 0x8000000000000000 +# just under! +release_vers "$FILE" stat $vers 0xfffffffffffff000 4096 +# this goes over +release_vers "$FILE" stat $vers 0xfffffffffffff000 8192 echo "== releasing non-file fails" mknod "$CHAR" c 1 3 From 606c519e96ea984134313135f83f76020503541c Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Fri, 23 Aug 2024 15:46:26 -0700 Subject: [PATCH 32/42] Simple-staging doesn't actually test overflow. This isn't a simple case where we can use u64_region_wraps because length is s32. Let's actually test an overflow case instead of a case that doesn't overflow, though. We still should properly add an overflow test here as well. Signed-off-by: Auke Kok --- tests/golden/simple-staging | 2 +- tests/tests/simple-staging.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/golden/simple-staging b/tests/golden/simple-staging index d05c8496..508732ca 100644 --- a/tests/golden/simple-staging +++ b/tests/golden/simple-staging @@ -7,7 +7,7 @@ stage: must provide file version with --data-version Try `stage --help' or `stage --usage' for more information. == wrapped region fails -stage returned -1, not 4096: error Invalid argument (22) +stage returned -1, not 8192: error Invalid argument (22) scoutfs: stage failed: Input/output error (5) == non-block aligned offset fails stage returned -1, not 4095: error Invalid argument (22) diff --git a/tests/tests/simple-staging.sh b/tests/tests/simple-staging.sh index ad059bd0..471cf7a0 100644 --- a/tests/tests/simple-staging.sh +++ b/tests/tests/simple-staging.sh @@ -143,8 +143,8 @@ hexdump -C "$FILE" rm -f "$FILE" echo "== wrapped region fails" -create_file "$FILE" 4096 -stage_vers "$FILE" stat 0xFFFFFFFFFFFFF000 4096 /dev/zero +create_file "$FILE" 8192 +stage_vers "$FILE" stat 0xFFFFFFFFFFFFF000 8192 /dev/zero rm -f "$FILE" echo "== non-block aligned offset fails" From 8a4b0967cb33464ebae1cf5254ec98070be19f44 Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Fri, 30 Aug 2024 17:27:07 -0700 Subject: [PATCH 33/42] Add fiemap output through scoutfs util. There's filefrag already, and that works, but, it's output is very inconsistent between various OS release versions, and it has already meant that we'd needed to adjust tests to account for these little but insignificant changes. A lot more work than useful. It's even more changed in el9. This adds `scoutfs get-fiemap FILE` and prints out block extent info with flags that we care about as an abbreviated letter: U for Unwritten, L for Last, and O for Unknown (as in, "offline"). The -P/--physical and -L/--logical options turn off logical or physical offset display, in case you only want to see the offsets in either units. You can pass -b/--byte to display offsets and lengths in byte values. The block size will then be obtained from fstat() of the queried file (4096 for scoutfs). I've removed all uses of filefrag from our scoutfs tests. Xfstests still calls it but their internal diff takes care of that issue. Where needed and appropriate, the tests are adjusted so that the output of `scoutfs get-fiemap` is as close as it can to what it used to be, so that reading the test results allows the quick view of what might have been going wrong. There are some output strings I have not bothered to update because there's no real value to updating every output string to match, and we just adjust the golden file accordingly. Signed-off-by: Auke Kok --- tests/golden/data-prealloc | 40 +++--- tests/golden/setattr_more | 7 +- tests/golden/simple-staging | 10 ++ tests/tests/basic-posix-consistency.sh | 8 +- tests/tests/data-prealloc.sh | 38 ++--- tests/tests/setattr_more.sh | 11 +- tests/tests/simple-release-extents.sh | 22 ++- tests/tests/simple-staging.sh | 16 +-- utils/src/fiemap.c | 191 +++++++++++++++++++++++++ 9 files changed, 259 insertions(+), 84 deletions(-) create mode 100644 utils/src/fiemap.c diff --git a/tests/golden/data-prealloc b/tests/golden/data-prealloc index ddea23cf..19efd20a 100644 --- a/tests/golden/data-prealloc +++ b/tests/golden/data-prealloc @@ -1,29 +1,29 @@ == initial writes smaller than prealloc grow to prealloc size -/mnt/test/test/data-prealloc/file-1: 7 extents found -/mnt/test/test/data-prealloc/file-2: 7 extents found +/mnt/test/test/data-prealloc/file-1: extents: 7 +/mnt/test/test/data-prealloc/file-2: extents: 7 == larger files get full prealloc extents -/mnt/test/test/data-prealloc/file-1: 9 extents found -/mnt/test/test/data-prealloc/file-2: 9 extents found +/mnt/test/test/data-prealloc/file-1: extents: 9 +/mnt/test/test/data-prealloc/file-2: extents: 9 == non-streaming writes with contig have per-block extents -/mnt/test/test/data-prealloc/file-1: 32 extents found -/mnt/test/test/data-prealloc/file-2: 32 extents found +/mnt/test/test/data-prealloc/file-1: extents: 32 +/mnt/test/test/data-prealloc/file-2: extents: 32 == any writes to region prealloc get full extents -/mnt/test/test/data-prealloc/file-1: 4 extents found -/mnt/test/test/data-prealloc/file-2: 4 extents found -/mnt/test/test/data-prealloc/file-1: 4 extents found -/mnt/test/test/data-prealloc/file-2: 4 extents found +/mnt/test/test/data-prealloc/file-1: extents: 4 +/mnt/test/test/data-prealloc/file-2: extents: 4 +/mnt/test/test/data-prealloc/file-1: extents: 4 +/mnt/test/test/data-prealloc/file-2: extents: 4 == streaming offline writes get full extents either way -/mnt/test/test/data-prealloc/file-1: 4 extents found -/mnt/test/test/data-prealloc/file-2: 4 extents found -/mnt/test/test/data-prealloc/file-1: 4 extents found -/mnt/test/test/data-prealloc/file-2: 4 extents found +/mnt/test/test/data-prealloc/file-1: extents: 4 +/mnt/test/test/data-prealloc/file-2: extents: 4 +/mnt/test/test/data-prealloc/file-1: extents: 4 +/mnt/test/test/data-prealloc/file-2: extents: 4 == goofy preallocation amounts work -/mnt/test/test/data-prealloc/file-1: 5 extents found -/mnt/test/test/data-prealloc/file-2: 5 extents found -/mnt/test/test/data-prealloc/file-1: 5 extents found -/mnt/test/test/data-prealloc/file-2: 5 extents found -/mnt/test/test/data-prealloc/file-1: 3 extents found -/mnt/test/test/data-prealloc/file-2: 3 extents found +/mnt/test/test/data-prealloc/file-1: extents: 6 +/mnt/test/test/data-prealloc/file-2: extents: 6 +/mnt/test/test/data-prealloc/file-1: extents: 6 +/mnt/test/test/data-prealloc/file-2: extents: 6 +/mnt/test/test/data-prealloc/file-1: extents: 3 +/mnt/test/test/data-prealloc/file-2: extents: 3 == block writes into region allocs hole wrote blk 24 wrote blk 32 diff --git a/tests/golden/setattr_more b/tests/golden/setattr_more index 7a8bfbe1..7ec57e8c 100644 --- a/tests/golden/setattr_more +++ b/tests/golden/setattr_more @@ -22,11 +22,8 @@ scoutfs: setattr failed: Invalid argument (22) == large ctime is set 1972-02-19 00:06:25.999999999 +0000 == large offline extents are created -Filesystem type is: 554f4353 -File size of /mnt/test/test/setattr_more/file is 40988672 (10007 blocks of 4096 bytes) - ext: logical_offset: physical_offset: length: expected: flags: - 0: 0.. 10006: 0.. 10006: 10007: unknown,eof -/mnt/test/test/setattr_more/file: 1 extent found +0: offset: 0 0 length: 10007 flags: O.L +extents: 1 == correct offline extent length 976563 == omitting data_version should not fail diff --git a/tests/golden/simple-staging b/tests/golden/simple-staging index 508732ca..b0cfeceb 100644 --- a/tests/golden/simple-staging +++ b/tests/golden/simple-staging @@ -1,5 +1,9 @@ == create/release/stage single block file +0: offset: 0 0 length: 1 flags: O.L +extents: 1 == create/release/stage larger file +0: offset: 0 0 length: 4096 flags: O.L +extents: 1 == multiple release,drop_cache,stage cycles == release+stage shouldn't change stat, data seq or vers == stage does change meta_seq @@ -12,11 +16,17 @@ scoutfs: stage failed: Input/output error (5) == non-block aligned offset fails stage returned -1, not 4095: error Invalid argument (22) scoutfs: stage failed: Input/output error (5) +0: offset: 0 0 length: 1 flags: O.L +extents: 1 == non-block aligned len within block fails stage returned -1, not 1024: error Invalid argument (22) scoutfs: stage failed: Input/output error (5) +0: offset: 0 0 length: 1 flags: O.L +extents: 1 == partial final block that writes to i_size does work == zero length stage doesn't bring blocks online +0: offset: 0 0 length: 100 flags: O.L +extents: 1 == stage of non-regular file fails ioctl failed: Inappropriate ioctl for device (25) stage: must provide file version with --data-version diff --git a/tests/tests/basic-posix-consistency.sh b/tests/tests/basic-posix-consistency.sh index 17643a61..6abe4e46 100644 --- a/tests/tests/basic-posix-consistency.sh +++ b/tests/tests/basic-posix-consistency.sh @@ -3,13 +3,13 @@ # operations in one mount and verify the results in another. # -t_require_commands getfattr setfattr dd filefrag diff touch stat scoutfs +t_require_commands getfattr setfattr dd diff touch stat scoutfs t_require_mounts 2 GETFATTR="getfattr --absolute-names" SETFATTR="setfattr" DD="dd status=none" -FILEFRAG="filefrag -v -b4096" +FIEMAP="scoutfs get-fiemap" echo "== root inode updates flow back and forth" sleep 1 @@ -55,8 +55,8 @@ for i in $(seq 1 10); do conv=notrunc oflag=append & wait done -$FILEFRAG "$T_D0/file" | t_filter_fs > "$T_TMP.0" -$FILEFRAG "$T_D1/file" | t_filter_fs > "$T_TMP.1" +$FIEMAP "$T_D0/file" > "$T_TMP.0" +$FIEMAP "$T_D1/file" > "$T_TMP.1" diff -u "$T_TMP.0" "$T_TMP.1" echo "== unlinked file isn't found" diff --git a/tests/tests/data-prealloc.sh b/tests/tests/data-prealloc.sh index 6c798246..a76909d7 100644 --- a/tests/tests/data-prealloc.sh +++ b/tests/tests/data-prealloc.sh @@ -4,7 +4,16 @@ # merge adjacent consecutive allocations. (we don't have multiple # allocation cursors) # -t_require_commands scoutfs stat filefrag dd touch truncate +t_require_commands scoutfs stat dd touch truncate + +get_fiemap() +{ + scoutfs get-fiemap "$1" | awk '($1 != "extents:") { + unwritten = (substr($8, 2, 1) == "U") ? "unwritten" : ""; + eof = (substr($8, 3, 1) == "L") ? "eof" : ""; + print $3 ".. " $6 ": " unwritten eof; + };' +} write_block() { @@ -76,26 +85,9 @@ print_extents_found() { local prefix="$1" - filefrag "$prefix"* 2>&1 | grep "extent.*found" | t_filter_fs -} - -# -# print the logical start, len, and flags if they're there. -# -print_logical_extents() -{ - local file="$1" - - filefrag -v -b4096 "$file" 2>&1 | t_filter_fs | awk ' - ($1 ~ /[0-9]+:/) { - if ($NF !~ /[0-9]+:/) { - flags=$NF - } else { - flags="" - } - print $2, $6, flags - } - ' | sed 's/last,eof/eof/' + for f in "$prefix"-*; do + echo "$f: $(scoutfs get-fiemap "$f" | tail -n 1)" | t_filter_fs + done } t_save_all_sysfs_mount_options data_prealloc_blocks @@ -197,7 +189,7 @@ for sides in 0 1 2 3; do done echo before: -print_logical_extents "$prefix" +get_fiemap "$prefix" # now write into the first, middle, and last empty block of each t_set_sysfs_mount_option 0 data_prealloc_contig_only 0 @@ -223,7 +215,7 @@ for sides in 0 1 2 3; do # mid (both has 6 blocks internally) 2) write_block $prefix $((left + 3)) ;; esac - print_logical_extents "$prefix" + get_fiemap "$prefix" ((base+=8)) done done diff --git a/tests/tests/setattr_more.sh b/tests/tests/setattr_more.sh index 846118ab..74d02def 100644 --- a/tests/tests/setattr_more.sh +++ b/tests/tests/setattr_more.sh @@ -2,7 +2,7 @@ # Test correctness of the setattr_more ioctl. # -t_require_commands filefrag scoutfs touch mkdir rm stat mknod +t_require_commands scoutfs touch mkdir rm stat mknod FILE="$T_D0/file" @@ -55,17 +55,10 @@ scoutfs setattr -t 67305985.999999999 -V 1 -s 1 "$FILE" 2>&1 | t_filter_fs TZ=GMT stat -c "%z" "$FILE" rm "$FILE" -# -# With e2fsprogs-v1.42.10-10-g29758d2f, the output of filefrag 'flags' changes -# significantly. First, the _LAST flag is now output. Second, the 'unknown' -# flag is now printed out as 'unknown_loc'. To compensate for this, we check -# and replace the "correct" output for new versions here with the expected -# value. -# echo "== large offline extents are created" touch "$FILE" scoutfs setattr -V 1 -o -s $((10007 * 4096)) "$FILE" 2>&1 | t_filter_fs -filefrag -v -b4096 "$FILE" 2>&1 | sed 's/last,unknown_loc,eof$/unknown,eof/' | t_filter_fs +scoutfs get-fiemap "$FILE" rm "$FILE" # had a bug where we were creating extents that were too long diff --git a/tests/tests/simple-release-extents.sh b/tests/tests/simple-release-extents.sh index 51dfeb1b..cd880a6c 100644 --- a/tests/tests/simple-release-extents.sh +++ b/tests/tests/simple-release-extents.sh @@ -7,6 +7,7 @@ t_require_commands xfs_io filefrag scoutfs mknod # this test wants to ignore unwritten extents fiemap_file() { filefrag -v -b4096 "$1" | grep -v "unwritten" + scoutfs get-fiemap "$1" | grep -v 'flags:.*U' } create_file() { @@ -108,25 +109,20 @@ for c in $(seq 0 4); do fi done - start=$(fiemap_file "$FILE" | \ - awk '($1 == "0:"){print substr($4, 0, length($4)- 2)}') - release_vers "$FILE" stat $(($a * 4))K 4K release_vers "$FILE" stat $(($b * 4))K 4K release_vers "$FILE" stat $(($c * 4))K 4K echo -n "$a $b $c:" - fiemap_file "$FILE" | \ - awk 'BEGIN{ORS=""}($1 == (NR - 4)":") { - off=substr($2, 0, length($2)- 2); - phys=substr($4, 0, length($4)- 2); - if (phys > 100) { - phys = phys - phys + 100 + off; - } - len=substr($6, 0, length($6)- 1); - print " (" off, phys, len ")"; - }' + scoutfs get-fiemap "$FILE" | \ + awk 'BEGIN{ORS=""}($1 != "extents:") { + off=$3; + len=$6; + phys=substr($8, 0, 1); + phys = (phys == ".") ? off + 100 : 0; + print " (" off, phys, len ")" + };' echo rm "$FILE" diff --git a/tests/tests/simple-staging.sh b/tests/tests/simple-staging.sh index 471cf7a0..9024661e 100644 --- a/tests/tests/simple-staging.sh +++ b/tests/tests/simple-staging.sh @@ -2,11 +2,7 @@ # Test correctness of the staging operation # -t_require_commands filefrag dd scoutfs cp cmp rm - -fiemap_file() { - filefrag -v -b4096 "$1" -} +t_require_commands dd scoutfs cp cmp rm create_file() { local file="$1" @@ -62,7 +58,7 @@ create_file "$FILE" 4096 cp "$FILE" "$T_TMP" release_vers "$FILE" stat 0 4K # make sure there only offline extents -fiemap_file "$FILE" | grep "^[ 0-9]*:" | grep -v "unknown" +scoutfs get-fiemap "$FILE" stage_vers "$FILE" stat 0 4096 "$T_TMP" cmp "$FILE" "$T_TMP" rm -f "$FILE" @@ -72,7 +68,7 @@ create_file "$FILE" $((4096 * 4096)) cp "$FILE" "$T_TMP" release_vers "$FILE" stat 0 16M # make sure there only offline extents -fiemap_file "$FILE" | grep "^[ 0-9]*:" | grep -v "unknown" +scoutfs get-fiemap "$FILE" stage_vers "$FILE" stat 0 $((4096 * 4096)) "$T_TMP" cmp "$FILE" "$T_TMP" rm -f "$FILE" @@ -152,7 +148,7 @@ create_file "$FILE" 4096 cp "$FILE" "$T_TMP" release_vers "$FILE" stat 0 4K stage_vers "$FILE" stat 1 4095 "$T_TMP" -fiemap_file "$FILE" | grep "^[ 0-9]*:" | grep -v "unknown" +scoutfs get-fiemap "$FILE" rm -f "$FILE" echo "== non-block aligned len within block fails" @@ -160,7 +156,7 @@ create_file "$FILE" 4096 cp "$FILE" "$T_TMP" release_vers "$FILE" stat 0 4K stage_vers "$FILE" stat 0 1024 "$T_TMP" -fiemap_file "$FILE" | grep "^[ 0-9]*:" | grep -v "unknown" +scoutfs get-fiemap "$FILE" rm -f "$FILE" echo "== partial final block that writes to i_size does work" @@ -175,7 +171,7 @@ echo "== zero length stage doesn't bring blocks online" create_file "$FILE" $((4096 * 100)) release_vers "$FILE" stat 0 400K stage_vers "$FILE" stat 4096 0 /dev/zero -fiemap_file "$FILE" | grep "^[ 0-9]*:" | grep -v "unknown" +scoutfs get-fiemap "$FILE" rm -f "$FILE" # XXX yup, needs to be updated for demand staging diff --git a/utils/src/fiemap.c b/utils/src/fiemap.c new file mode 100644 index 00000000..6b932460 --- /dev/null +++ b/utils/src/fiemap.c @@ -0,0 +1,191 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sparse.h" +#include "parse.h" +#include "util.h" +#include "format.h" +#include "ioctl.h" +#include "cmd.h" + +/* + * This is wholly modeled after e2fsprogs/filefrag.c from tso + */ + +struct get_fiemap_args { + char *filename; + bool phys; + bool logical; + bool byte; +}; + +static int do_get_fiemap(struct get_fiemap_args *args) +{ + __u64 buf[2048]; /* __u64 for proper field alignment */ + struct stat st; + struct fiemap *fiemap = (struct fiemap *)buf; + struct fiemap_extent *fm_ext = &fiemap->fm_extents[0]; + int count = (sizeof(buf) - sizeof(*fiemap)) / + sizeof(struct fiemap_extent); + int fd; + int ret; + int i; + u64 nr = 0; /* XXX we could put this in fm_start to make start/count an option */ + int last = 0; + u64 off_p, off_l; + u64 len; + + memset(fiemap, 0, sizeof(struct fiemap)); + + fd = open(args->filename, O_RDONLY); + if (fd < 0) { + ret = -errno; + fprintf(stderr, "failed to open '%s': %s (%d)\n", + args->filename, strerror(errno), errno); + goto out; + } + + /* get block size from stat */ + if (fstat(fd, &st) != 0) { + ret = -errno; + fprintf(stderr, "stat failed on '%s': %s (%d)\n", + args->filename, strerror(errno), errno); + goto out; + }; + + do { + fiemap->fm_length = ~0ULL; + fiemap->fm_extent_count = count; + + ret = ioctl(fd, FS_IOC_FIEMAP, (unsigned long) fiemap); + if (ret < 0) { + ret = -errno; + fprintf(stderr, "get_fiemap ioctl failed: " + "%s (%d)\n", strerror(errno), errno); + goto out; + } + + /* nothing returned, so exit */ + if (fiemap->fm_mapped_extents == 0) + break; + + for (i = 0; i < fiemap->fm_mapped_extents; i++) { + off_p = fm_ext[i].fe_physical; + off_l = fm_ext[i].fe_logical; + len = fm_ext[i].fe_length; + + if (!args->byte) { + off_p /= st.st_blksize; + off_l /= st.st_blksize; + len /= st.st_blksize; + } + + printf("%llu: offset: ", nr++); + + if (!args->phys) + printf("%llu ", off_l); + else if (!args->logical) + printf("%llu ", off_p); + else + printf("%llu %llu ", off_l, off_p); + + printf("length: %llu flags: %c%c%c\n", + len, + (fm_ext[i].fe_flags & FIEMAP_EXTENT_UNKNOWN) ? 'O' : '.', + (fm_ext[i].fe_flags & FIEMAP_EXTENT_UNWRITTEN) ? 'U' : '.', + (fm_ext[i].fe_flags & FIEMAP_EXTENT_LAST) ? 'L' : '.'); + + if (fm_ext[i].fe_flags & FIEMAP_EXTENT_LAST) + last = 1; + } + + /* fm_start from the next logical extent */ + fiemap->fm_start = fm_ext[i-1].fe_logical + fm_ext[i-1].fe_length; + } while (last == 0); + + printf("extents: %llu\n", nr); + +out: + if (fd >= 0) + close(fd); + + return ret; +}; + +static int parse_opt(int key, char *arg, struct argp_state *state) +{ + struct get_fiemap_args *args = state->input; + + switch (key) { + case 'P': + args->logical = false; + break; + case 'L': + args->phys = false; + break; + case 'b': + args->byte = true; + break; + case ARGP_KEY_ARG: + if (!args->filename) + args->filename = strdup_or_error(state, arg); + else + argp_error(state, "more than one argument given"); + break; + case ARGP_KEY_FINI: + if ((!args->logical) && (!args->phys)) + argp_error(state, "can't pass both -P and -L options"); + if (!args->filename) + argp_error(state, "no filename given"); + break; + default: + break; + } + + return 0; +} + +static struct argp_option options[] = { + { "physical", 'P', NULL, 0, "Output physical offsets only"}, + { "logical", 'L', NULL, 0, "Output logical offsets only"}, + { "byte", 'b', NULL, 0, "Output byte values instead of blocks"}, + { NULL } +}; + +static struct argp argp = { + options, + parse_opt, + "FILE", + "Print fiemap extent mapping" +}; + +static int get_fiemap_cmd(int argc, char **argv) +{ + struct get_fiemap_args get_fiemap_args = {NULL}; + int ret; + + get_fiemap_args.phys = true; + get_fiemap_args.logical = true; + + ret = argp_parse(&argp, argc, argv, 0, NULL, &get_fiemap_args); + if (ret) + return ret; + + return do_get_fiemap(&get_fiemap_args); +} + +static void __attribute__((constructor)) get_fiemap_ctor(void) +{ + cmd_register_argp("get-fiemap", &argp, GROUP_DEBUG, get_fiemap_cmd); +} From 0a8b3f4e94cb042977a228ea29a75e18d5bbd70c Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Fri, 30 Aug 2024 22:34:36 -0700 Subject: [PATCH 34/42] Fix basic-posix-acl test output on el9 It turns out that on el9, `bash -c` prints out `bash: line 1: cd..` instead of `line 0:` on el7 or el8. So discard all the stderr from these `cd` lines entirely and just rely on the expected echo output to stdout. Signed-off-by: Auke Kok --- tests/golden/basic-posix-acl | 2 -- tests/tests/basic-posix-acl.sh | 12 ++++++------ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/tests/golden/basic-posix-acl b/tests/golden/basic-posix-acl index b0415759..017060b2 100644 --- a/tests/golden/basic-posix-acl +++ b/tests/golden/basic-posix-acl @@ -41,9 +41,7 @@ group::r-x mask::rwx other::r-x -bash: line 0: cd: dir-root: Permission denied Failed -bash: line 0: cd: symlinkdir-root: Permission denied Failed # file: dir-root # owner: root diff --git a/tests/tests/basic-posix-acl.sh b/tests/tests/basic-posix-acl.sh index 6b1e2064..b80e8088 100644 --- a/tests/tests/basic-posix-acl.sh +++ b/tests/tests/basic-posix-acl.sh @@ -55,16 +55,16 @@ L dir-root/file-group-write L symlinkdir-root/file-group-write echo "== directory exec" -setpriv $SET_UID bash -c "cd dir-root && echo Success" -setpriv $SET_UID bash -c "cd symlinkdir-root && echo Success" +setpriv $SET_UID bash -c "cd dir-root 2>&- && echo Success" +setpriv $SET_UID bash -c "cd symlinkdir-root 2>&- && echo Success" setfacl -m u:22222:rw dir-root getfacl dir-root -setpriv $SET_UID bash -c "cd dir-root || echo Failed" -setpriv $SET_UID bash -c "cd symlinkdir-root || echo Failed" +setpriv $SET_UID bash -c "cd dir-root 2>&- || echo Failed" +setpriv $SET_UID bash -c "cd symlinkdir-root 2>&- || echo Failed" setfacl -m g:44444:rwx dir-root getfacl dir-root -setpriv $SET_GID bash -c "cd dir-root && echo Success" -setpriv $SET_GID bash -c "cd symlinkdir-root && echo Success" +setpriv $SET_GID bash -c "cd dir-root 2>&- && echo Success" +setpriv $SET_GID bash -c "cd symlinkdir-root 2>&- && echo Success" echo "== get/set attr" rm -rf file-root From 4723f4f9aba03b3aa3caaecb4b68727a4380c03c Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Tue, 3 Sep 2024 11:34:14 -0700 Subject: [PATCH 35/42] Disable format-version-forward-back test on el9+. Using t_skip, we just skip this test on el9. If we ever want to add a formatversion 2->3 test, perhaps we should just add a separate test script, instead of going over a static array. But let's not worry about this too much right now. Signed-off-by: Auke Kok --- tests/tests/format-version-forward-back.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/tests/format-version-forward-back.sh b/tests/tests/format-version-forward-back.sh index 403d0aaf..404660b5 100644 --- a/tests/tests/format-version-forward-back.sh +++ b/tests/tests/format-version-forward-back.sh @@ -11,6 +11,11 @@ # format version. # +# not supported on el9! +if [ $(source /etc/os-release ; echo ${VERSION_ID:0:1}) -gt 8 ]; then + t_skip "Unsupported OS version" +fi + mount_has_format_version() { local mnt="$1" From 21b5032365a4c5cbfe17a39f55bbfafb6cbe3a47 Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Tue, 26 Sep 2023 14:40:09 -0400 Subject: [PATCH 36/42] Add new xfstests that we won't support or don't pass The new version of xfstests adds a _lot_ more tests to our mix. Many of the new ones will auto enable or auto skip as needed. There are tests we can't or won't support that will be in future xfstests. Disable them now so we can avoid dealing with them later. Quite a few fall into "we don't support these types of mounting yet", mostly bind-mount or dm-mapper things. We disable all the swapfile tests flatout. A few tests fail on el7 but not el8/9 but we don't have a way to run them without failing yet, so disable them as well. Update golden with the proper new array of tests. This all requires the `auke/scoutfs-el9` branch in `versity/scoutfs-xfstests-dev`. Signed-off-by: Auke Kok --- tests/funcs/filter.sh | 5 + tests/golden/xfstests | 593 +++++++++++++++++++++++++++++++++++++++- tests/tests/xfstests.sh | 39 +++ 3 files changed, 629 insertions(+), 8 deletions(-) diff --git a/tests/funcs/filter.sh b/tests/funcs/filter.sh index dd840109..547b19e5 100644 --- a/tests/funcs/filter.sh +++ b/tests/funcs/filter.sh @@ -151,6 +151,11 @@ t_filter_dmesg() re="$re|scoutfs .* error.*outside of supported version.*" re="$re|scoutfs .* error.*could not get .*super.*" + # ignore "unsafe core pattern" when xfstests tries to disable cores" + re="$re|Unsafe core_pattern used with fs.suid_dumpable=2.*" + re="$re|Pipe handler or fully qualified core dump path required.*" + re="$re|Set kernel.core_pattern before fs.suid_dumpable.*" + egrep -v "($re)" | \ ignore_harmless_unwind_kasan_stack_oob } diff --git a/tests/golden/xfstests b/tests/golden/xfstests index 16824a05..8247aa14 100644 --- a/tests/golden/xfstests +++ b/tests/golden/xfstests @@ -5,24 +5,44 @@ generic/004 generic/005 generic/006 generic/007 +generic/008 +generic/009 generic/011 +generic/012 generic/013 generic/014 +generic/015 +generic/016 +generic/018 generic/020 +generic/021 +generic/022 generic/023 generic/024 +generic/025 +generic/026 generic/028 +generic/031 generic/032 +generic/033 generic/034 generic/035 generic/037 generic/039 generic/040 generic/041 +generic/050 +generic/052 generic/053 generic/056 generic/057 +generic/058 +generic/059 +generic/060 +generic/061 generic/062 +generic/063 +generic/064 generic/065 generic/066 generic/067 @@ -31,42 +51,193 @@ generic/070 generic/071 generic/073 generic/076 +generic/078 +generic/079 +generic/081 +generic/082 generic/084 generic/086 generic/087 generic/088 generic/090 +generic/091 generic/092 +generic/094 +generic/096 +generic/097 generic/098 +generic/099 generic/101 generic/104 generic/105 generic/106 generic/107 +generic/110 +generic/111 +generic/113 +generic/114 +generic/115 +generic/116 generic/117 +generic/118 +generic/119 +generic/121 +generic/122 +generic/123 generic/124 +generic/128 generic/129 +generic/130 generic/131 +generic/134 +generic/135 +generic/136 +generic/138 +generic/139 +generic/140 +generic/142 +generic/143 +generic/144 +generic/145 +generic/146 +generic/147 +generic/148 +generic/149 +generic/150 +generic/151 +generic/152 +generic/153 +generic/154 +generic/155 +generic/156 +generic/157 +generic/158 +generic/159 +generic/160 +generic/161 +generic/162 +generic/163 generic/169 +generic/171 +generic/172 +generic/173 +generic/174 +generic/177 +generic/178 +generic/179 +generic/180 +generic/181 +generic/182 +generic/183 generic/184 +generic/185 +generic/188 +generic/189 +generic/190 +generic/191 +generic/193 +generic/194 +generic/195 +generic/196 +generic/197 +generic/198 +generic/199 +generic/200 +generic/201 +generic/202 +generic/203 +generic/205 +generic/206 +generic/207 +generic/210 +generic/211 +generic/212 +generic/214 +generic/216 +generic/217 +generic/218 +generic/219 +generic/220 generic/221 +generic/222 +generic/223 +generic/225 +generic/227 generic/228 +generic/229 +generic/230 +generic/235 generic/236 generic/237 +generic/238 +generic/240 +generic/244 generic/245 generic/249 +generic/250 +generic/252 +generic/253 +generic/254 +generic/255 +generic/256 generic/257 generic/258 +generic/259 +generic/260 +generic/261 +generic/262 +generic/263 +generic/264 +generic/265 +generic/266 +generic/267 +generic/268 +generic/271 +generic/272 +generic/276 +generic/277 +generic/278 +generic/279 +generic/281 +generic/282 +generic/283 +generic/284 generic/286 +generic/287 +generic/288 +generic/289 +generic/290 +generic/291 +generic/292 +generic/293 generic/294 +generic/295 +generic/296 +generic/301 +generic/302 +generic/303 +generic/304 +generic/305 generic/306 generic/307 generic/308 generic/309 +generic/312 generic/313 +generic/314 generic/315 +generic/316 +generic/317 generic/319 generic/322 +generic/324 +generic/326 +generic/327 +generic/328 +generic/329 +generic/330 +generic/331 +generic/332 generic/335 generic/336 generic/337 @@ -74,10 +245,255 @@ generic/341 generic/342 generic/343 generic/348 +generic/353 +generic/355 +generic/358 +generic/359 generic/360 +generic/361 +generic/362 +generic/363 +generic/364 +generic/365 +generic/366 +generic/367 +generic/368 +generic/369 +generic/370 +generic/371 +generic/372 +generic/373 +generic/374 generic/375 generic/376 generic/377 +generic/378 +generic/379 +generic/380 +generic/381 +generic/382 +generic/383 +generic/384 +generic/385 +generic/386 +generic/389 +generic/391 +generic/392 +generic/393 +generic/394 +generic/395 +generic/396 +generic/397 +generic/398 +generic/400 +generic/401 +generic/402 +generic/403 +generic/404 +generic/406 +generic/407 +generic/408 +generic/412 +generic/413 +generic/414 +generic/417 +generic/419 +generic/420 +generic/421 +generic/422 +generic/424 +generic/425 +generic/426 +generic/427 +generic/436 +generic/439 +generic/440 +generic/443 +generic/445 +generic/446 +generic/448 +generic/449 +generic/450 +generic/451 +generic/453 +generic/454 +generic/456 +generic/458 +generic/460 +generic/462 +generic/463 +generic/465 +generic/466 +generic/468 +generic/469 +generic/470 +generic/471 +generic/474 +generic/477 +generic/478 +generic/479 +generic/480 +generic/481 +generic/483 +generic/485 +generic/486 +generic/487 +generic/488 +generic/489 +generic/490 +generic/491 +generic/492 +generic/498 +generic/499 +generic/501 +generic/502 +generic/503 +generic/504 +generic/505 +generic/506 +generic/507 +generic/508 +generic/509 +generic/510 +generic/511 +generic/512 +generic/513 +generic/514 +generic/515 +generic/516 +generic/517 +generic/518 +generic/519 +generic/520 +generic/523 +generic/524 +generic/525 +generic/526 +generic/527 +generic/528 +generic/529 +generic/530 +generic/531 +generic/533 +generic/534 +generic/535 +generic/536 +generic/537 +generic/538 +generic/539 +generic/540 +generic/541 +generic/542 +generic/543 +generic/544 +generic/545 +generic/546 +generic/547 +generic/548 +generic/549 +generic/550 +generic/552 +generic/553 +generic/555 +generic/556 +generic/557 +generic/566 +generic/567 +generic/571 +generic/572 +generic/573 +generic/574 +generic/575 +generic/576 +generic/577 +generic/578 +generic/580 +generic/581 +generic/582 +generic/583 +generic/584 +generic/586 +generic/587 +generic/588 +generic/591 +generic/592 +generic/593 +generic/594 +generic/595 +generic/596 +generic/597 +generic/598 +generic/599 +generic/600 +generic/601 +generic/602 +generic/603 +generic/604 +generic/605 +generic/606 +generic/607 +generic/608 +generic/609 +generic/610 +generic/611 +generic/612 +generic/613 +generic/618 +generic/621 +generic/623 +generic/624 +generic/625 +generic/626 +generic/628 +generic/629 +generic/630 +generic/632 +generic/634 +generic/635 +generic/637 +generic/639 +generic/640 +generic/644 +generic/645 +generic/646 +generic/647 +generic/651 +generic/652 +generic/653 +generic/654 +generic/655 +generic/657 +generic/658 +generic/659 +generic/660 +generic/661 +generic/662 +generic/663 +generic/664 +generic/665 +generic/666 +generic/667 +generic/668 +generic/669 +generic/673 +generic/674 +generic/675 +generic/676 +generic/677 +generic/678 +generic/679 +generic/680 +generic/681 +generic/682 +generic/683 +generic/684 +generic/685 +generic/686 +generic/687 +generic/688 +generic/689 +shared/002 +shared/032 Not run: generic/008 @@ -251,8 +667,6 @@ generic/331 generic/332 generic/353 generic/355 -generic/356 -generic/357 generic/358 generic/359 generic/361 @@ -278,11 +692,174 @@ generic/383 generic/384 generic/385 generic/386 -shared/001 +generic/391 +generic/392 +generic/395 +generic/396 +generic/397 +generic/398 +generic/400 +generic/402 +generic/404 +generic/406 +generic/407 +generic/408 +generic/412 +generic/413 +generic/414 +generic/417 +generic/419 +generic/420 +generic/421 +generic/422 +generic/424 +generic/425 +generic/427 +generic/439 +generic/440 +generic/446 +generic/449 +generic/450 +generic/451 +generic/453 +generic/454 +generic/456 +generic/458 +generic/462 +generic/463 +generic/465 +generic/466 +generic/468 +generic/469 +generic/470 +generic/471 +generic/474 +generic/485 +generic/487 +generic/488 +generic/491 +generic/492 +generic/499 +generic/501 +generic/503 +generic/505 +generic/506 +generic/507 +generic/508 +generic/511 +generic/513 +generic/514 +generic/515 +generic/516 +generic/517 +generic/518 +generic/519 +generic/520 +generic/528 +generic/530 +generic/536 +generic/537 +generic/538 +generic/539 +generic/540 +generic/541 +generic/542 +generic/543 +generic/544 +generic/545 +generic/546 +generic/548 +generic/549 +generic/550 +generic/552 +generic/553 +generic/555 +generic/556 +generic/566 +generic/567 +generic/572 +generic/573 +generic/574 +generic/575 +generic/576 +generic/577 +generic/578 +generic/580 +generic/581 +generic/582 +generic/583 +generic/584 +generic/586 +generic/587 +generic/588 +generic/591 +generic/592 +generic/593 +generic/594 +generic/595 +generic/596 +generic/597 +generic/598 +generic/599 +generic/600 +generic/601 +generic/602 +generic/603 +generic/605 +generic/606 +generic/607 +generic/608 +generic/609 +generic/610 +generic/612 +generic/613 +generic/621 +generic/623 +generic/624 +generic/625 +generic/626 +generic/628 +generic/629 +generic/630 +generic/635 +generic/644 +generic/645 +generic/646 +generic/647 +generic/651 +generic/652 +generic/653 +generic/654 +generic/655 +generic/657 +generic/658 +generic/659 +generic/660 +generic/661 +generic/662 +generic/663 +generic/664 +generic/665 +generic/666 +generic/667 +generic/668 +generic/669 +generic/673 +generic/674 +generic/675 +generic/677 +generic/678 +generic/679 +generic/680 +generic/681 +generic/682 +generic/683 +generic/684 +generic/685 +generic/686 +generic/687 +generic/688 +generic/689 shared/002 -shared/003 -shared/004 shared/032 -shared/051 -shared/289 -Passed all 79 tests +Passed all 495 tests diff --git a/tests/tests/xfstests.sh b/tests/tests/xfstests.sh index 25e66bbe..53dbc0f6 100644 --- a/tests/tests/xfstests.sh +++ b/tests/tests/xfstests.sh @@ -81,6 +81,45 @@ generic/325 # mmap missing generic/338 # BUG_ON update inode error handling generic/346 # mmap missing generic/347 # _dmthin_mount doesn't work? +generic/356 # swap +generic/357 # swap +generic/409 # bind mounts not scripted yet +generic/410 # bind mounts not scripted yet +generic/411 # bind mounts not scripted yet +generic/423 # symlink inode size is strlen() + 1 on scoutfs +generic/428 # mmap missing +generic/430 # xfs_io copy_range missing in el7 +generic/431 # xfs_io copy_range missing in el7 +generic/432 # xfs_io copy_range missing in el7 +generic/433 # xfs_io copy_range missing in el7 +generic/434 # xfs_io copy_range missing in el7 +generic/437 # mmap missing +generic/441 # dm-mapper +generic/444 # el9's posix_acl_update_mode is buggy ? +generic/452 # exec test - no mmap +generic/467 # open_by_handle ESTALE +generic/472 # swap +generic/484 # dm-mapper +generic/493 # swap +generic/494 # swap +generic/495 # swap +generic/496 # swap +generic/497 # swap +generic/532 # xfs_io statx attrib_mask missing in el7 +generic/554 # swap +generic/563 # cgroup+loopdev +generic/564 # xfs_io copy_range missing in el7 +generic/565 # xfs_io copy_range missing in el7 +generic/568 # falloc not resulting in block count increase +generic/569 # swap +generic/570 # swap +generic/614 # mmap missing +generic/620 # dm-hugedisk +generic/633 # mmap, id-mapped mounts missing in el7 +generic/636 # swap +generic/638 # mmap missing +generic/641 # swap +generic/643 # swap EOF t_restore_output From 97b081de3f52193ecf3a3d793f53a8a98518fde7 Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Wed, 4 Sep 2024 18:06:34 -0400 Subject: [PATCH 37/42] Switch xfstests tag over in CI jobs using this marker file. CI testing needs to know which xfstests branch to use on all OSs. We can't just use the el9 xfstests branch on el9 only, because we need to run the same el9 xfstests on el8 and el7 as well, otherwise testing will just fail. So, we put a marker file in our git repo that tells us that we're not going to use the default `scoutfs` branch from scoutfs-xfstests-dev but our own special tag or branch. The CI job then should pass the proper -x {branch} flag to the run-tests.sh script. Signed-off-by: Auke Kok --- tests/.xfstests-branch | 1 + 1 file changed, 1 insertion(+) create mode 100644 tests/.xfstests-branch diff --git a/tests/.xfstests-branch b/tests/.xfstests-branch new file mode 100644 index 00000000..31a466f6 --- /dev/null +++ b/tests/.xfstests-branch @@ -0,0 +1 @@ +v2022.05.01-2-g787cd20 From 9335d2eb86a13f5b40d0e01c702d52c24d4af062 Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Wed, 4 Sep 2024 19:52:40 -0400 Subject: [PATCH 38/42] Don't --track when checking out a tag. I've pushed a tag/release to scoutfs-xfstests-dev instead of a full blown branch. This seems simpler and cleaner than using branches, because we're going to end up rebasing these things a lot. However, we can't --track tags, so, if the branch name passed to -x is actually a tag instead of a branch, we have to omit the --track option here. Signed-off-by: Auke Kok --- tests/tests/xfstests.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/tests/xfstests.sh b/tests/tests/xfstests.sh index 53dbc0f6..f2850a62 100644 --- a/tests/tests/xfstests.sh +++ b/tests/tests/xfstests.sh @@ -30,8 +30,13 @@ t_quiet mkdir -p "$T_TMPDIR/mnt.scratch" t_quiet cd "$T_XFSTESTS_REPO" if [ -z "$T_SKIP_CHECKOUT" ]; then t_quiet git fetch + # if we're passed a tag instead of a branch, we can't --track + TRACK="--track" + if git tag -l | grep -q "$T_XFSTESTS_BRANCH" ; then + TRACK="" + fi # this remote use is bad, do better - t_quiet git checkout -B "$T_XFSTESTS_BRANCH" --track "origin/$T_XFSTESTS_BRANCH" + t_quiet git checkout -B "$T_XFSTESTS_BRANCH" ${TRACK} "origin/$T_XFSTESTS_BRANCH" fi t_quiet make t_quiet sync From 235ab133a77523c3877473fc92f97f0cf9621109 Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Thu, 12 Sep 2024 12:20:20 -0700 Subject: [PATCH 39/42] We must provide a_ops->dirty_folio and invalidate_folio. In v5.17-rc4-53-g3a3bae50af5d, we can no longer omit having this method unhooked as the mm caller blindly calls it now. In-kernel filesystems all were fixed in this change. aops->invalidatepage was the old aops method that would free pages with private attached data. This method is replaced with the new invalidate_folio method. If this method is NULL, the memory will become orphaned. (v5.17-rc4-29-gf50015a596fa) Signed-off-by: Auke Kok --- kmod/src/data.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kmod/src/data.c b/kmod/src/data.c index 7045d547..909167bc 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -1916,6 +1916,8 @@ int scoutfs_data_waiting(struct super_block *sb, u64 ino, u64 iblock, const struct address_space_operations scoutfs_file_aops = { #ifdef KC_MPAGE_READ_FOLIO + .dirty_folio = block_dirty_folio, + .invalidate_folio = block_invalidate_folio, .read_folio = scoutfs_read_folio, #else .readpage = scoutfs_readpage, From 8a22bdd3664f5427bf9c53fce71ad90244a00c87 Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Tue, 17 Sep 2024 09:20:21 -0700 Subject: [PATCH 40/42] Ignore device mapper size change dmesg output. In v1.18-10-g5507ee5, we changed the test code away from loopback to device-mapper, which simplified our DUT setup code. However, this results in the occasional `device changed size` messages now being emitted by the `dm` driver instead of the `loop` kernel module. We have to additionally ignore these kernel messages from now as well. Signed-off-by: Auke Kok --- tests/funcs/filter.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/funcs/filter.sh b/tests/funcs/filter.sh index 547b19e5..6e63e80c 100644 --- a/tests/funcs/filter.sh +++ b/tests/funcs/filter.sh @@ -143,6 +143,7 @@ t_filter_dmesg() # change-devices causes loop device resizing re="$re|loop: module loaded" re="$re|loop[0-9].* detected capacity change from.*" + re="$re|dm-[0-9].* detected capacity change from.*" # ignore systemd-journal rotating re="$re|systemd-journald.*" From 5337b9e221e6ae8627d767890fc3b5f920e49887 Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Tue, 17 Sep 2024 09:26:51 -0700 Subject: [PATCH 41/42] Ingore Process accounting resumed dmesg. I'm seeing more and more of these as audit is enabled in el8 and el9 images I am using for testing, and during ENOSPC tests this has a chance of triggering process accounting suspension, and subsequent resume. Signed-off-by: Auke Kok --- tests/funcs/filter.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/funcs/filter.sh b/tests/funcs/filter.sh index 6e63e80c..8f5d8c36 100644 --- a/tests/funcs/filter.sh +++ b/tests/funcs/filter.sh @@ -148,6 +148,9 @@ t_filter_dmesg() # ignore systemd-journal rotating re="$re|systemd-journald.*" + # process accounting can be noisy + re="$re|Process accounting resumed.*" + # format vers back/compat tries bad mounts re="$re|scoutfs .* error.*outside of supported version.*" re="$re|scoutfs .* error.*could not get .*super.*" From fc7876e844d530713aa3586749c88679516b7587 Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Fri, 20 Sep 2024 11:49:34 -0700 Subject: [PATCH 42/42] Allow certain tests to skip, but not fail exit condition. Previously, any t_skip would cause the final test result to be a failure because up until now no test should have been skipped. However, with format-version-forward-back not being compatible with el9, we are going to rely on el7/8 testing for that test soleley, and therefore we have to allow skipping of this test on el9 and newer OS versions. We add `t_skip_permitted` to signal this from the test case to the run-tests.sh script. A new exit code is passed, and all accounting is updated to reflect that a test was skipped, but this was permitted. We modify format-version-forward-back to use this new exit path. Signed-off-by: Auke Kok --- tests/funcs/exec.sh | 14 +++++++++++++- tests/run-tests.sh | 7 ++++++- tests/tests/format-version-forward-back.sh | 2 +- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/tests/funcs/exec.sh b/tests/funcs/exec.sh index 0aa595eb..a53e72e8 100644 --- a/tests/funcs/exec.sh +++ b/tests/funcs/exec.sh @@ -7,8 +7,9 @@ t_status_msg() export T_PASS_STATUS=100 export T_SKIP_STATUS=101 export T_FAIL_STATUS=102 +export T_SKIP_PERMITTED_STATUS=103 export T_FIRST_STATUS="$T_PASS_STATUS" -export T_LAST_STATUS="$T_FAIL_STATUS" +export T_LAST_STATUS="$T_SKIP_PERMITTED_STATUS" t_pass() { @@ -21,6 +22,17 @@ t_skip() exit $T_SKIP_STATUS } +# +# This exit code is *reserved* for tests that are up-front never going to work +# in certain cases. This should be expressly documented per-case and made +# abundantly clear before merging. The test itself should document its case. +# +t_skip_permitted() +{ + t_status_msg "$@" + exit $T_SKIP_PERMITTED_STATUS +} + t_fail() { t_status_msg "$@" diff --git a/tests/run-tests.sh b/tests/run-tests.sh index fe28a8a6..891f7dfb 100755 --- a/tests/run-tests.sh +++ b/tests/run-tests.sh @@ -515,6 +515,7 @@ msg "running tests" passed=0 skipped=0 failed=0 +skipped_permitted=0 for t in $tests; do # tests has basenames from sequence, get path and name t="tests/$t" @@ -621,6 +622,10 @@ for t in $tests; do grep -s -v "^$test_name " "$last" > "$last.tmp" echo "$test_name $stats" >> "$last.tmp" mv -f "$last.tmp" "$last" + elif [ "$sts" == "$T_SKIP_PERMITTED_STATUS" ]; then + echo " [ skipped (permitted): $message ]" + echo "$test_name skipped (permitted) $message " >> "$T_RESULTS/skip.log" + ((skipped_permitted++)) elif [ "$sts" == "$T_SKIP_STATUS" ]; then echo " [ skipped: $message ]" echo "$test_name $message" >> "$T_RESULTS/skip.log" @@ -634,7 +639,7 @@ for t in $tests; do fi done -msg "all tests run: $passed passed, $skipped skipped, $failed failed" +msg "all tests run: $passed passed, $skipped skipped, $skipped_permitted skipped (permitted), $failed failed" if [ -n "$T_TRACE_GLOB" -o -n "$T_TRACE_PRINTK" ]; then diff --git a/tests/tests/format-version-forward-back.sh b/tests/tests/format-version-forward-back.sh index 404660b5..a5dde26b 100644 --- a/tests/tests/format-version-forward-back.sh +++ b/tests/tests/format-version-forward-back.sh @@ -13,7 +13,7 @@ # not supported on el9! if [ $(source /etc/os-release ; echo ${VERSION_ID:0:1}) -gt 8 ]; then - t_skip "Unsupported OS version" + t_skip_permitted "Unsupported OS version" fi mount_has_format_version()