diff --git a/scripts/kunit/test_clean.py b/scripts/kunit/test_clean.py index cdc60b6bc..d43a8d715 100644 --- a/scripts/kunit/test_clean.py +++ b/scripts/kunit/test_clean.py @@ -35,10 +35,13 @@ class CleanTests(unittest.TestCase): (target / 'visited.o').touch() kernel = root / 'kernel' kernel.mkdir() + # iSCSI cleans kernel/ and kernel/isert-scst/ concurrently. Like + # Kbuild's rm -f, tolerate artifacts already removed by a peer. (kernel / 'Makefile').write_text( 'clean:\n' '\t@test "$(FAIL_CLEAN)" != y\n' - '\t@find "$(M)" -type f \\( -name "*.o" -o -name "*.ko" \\) -delete\n') + '\t@find "$(M)" -ignore_readdir_race -type f ' + '\\( -name "*.o" -o -name "*.ko" \\) -delete\n') artifacts = ['scst/src/scst.ko', 'scst/src/kunit/lib/suites/test.o', 'scst/src/dev_handlers/scst_vdisk.ko', 'scst/src/dev_handlers/kunit/vdisk/suites/test.o', diff --git a/scst/include/backport.h b/scst/include/backport.h index b9a96d97f..15418a17b 100644 --- a/scst/include/backport.h +++ b/scst/include/backport.h @@ -53,6 +53,7 @@ #endif #include #include +#include #include /* struct scatterlist */ #include #include /* kmalloc() */ @@ -1325,18 +1326,72 @@ typedef unsigned long percpu_count_t; #define READ_REF_COUNT(ref) atomic_long_read(&(ref)->data->count) #endif -#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 20, 0) && \ - (!defined(RHEL_MAJOR) || RHEL_MAJOR -0 < 8) +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 20, 0) && \ + (!defined(RHEL_MAJOR) || RHEL_MAJOR - 0 < 8)) || \ + defined(SCST_KUNIT_ENABLED) /* * See also commit 18c9a6bbe064 ("percpu-refcount: Introduce * percpu_ref_resurrect()") # v4.20. + * + * The caller must serialize kill, resurrection, mode switches and exit, and + * keep ref alive even if its release callback runs. This helper may sleep. + * SCST holds scst_suspend_mutex and uses persistent command counters. Gets + * and puts may run concurrently. + * + * Keep this implementation available to KUnit on newer kernels so the tests + * exercise the backport with real kernel counters and RCU callbacks. */ -static inline void percpu_ref_resurrect(struct percpu_ref *ref) +static inline void scst_percpu_ref_resurrect_backport(struct percpu_ref *ref) { - percpu_ref_reinit(ref); +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0) + struct percpu_ref_data *data = ref->data; +#else + struct percpu_ref *data = ref; +#endif + unsigned long __percpu *percpu_count; + unsigned long ptr; + int cpu; + + /* + * The kill confirmation can wake the caller before confirm_switch is + * cleared. The kernel's switch waitqueue is private, and older kernels + * do not export switch_to_percpu(). Drain a pending callback before + * reusing the per-CPU counters. The supported pre-4.20 implementations, + * including EL7 and UEK4/UEK5, use call_rcu_sched(). + */ + if (READ_ONCE(data->confirm_switch)) { +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 20, 0) + rcu_barrier_sched(); +#else + rcu_barrier(); +#endif + } + + ptr = READ_ONCE(ref->percpu_count_ptr); + percpu_count = (unsigned long __percpu *)(ptr & ~__PERCPU_REF_ATOMIC_DEAD); + if (WARN_ON_ONCE((ptr & __PERCPU_REF_ATOMIC_DEAD) != __PERCPU_REF_ATOMIC_DEAD || + !percpu_count)) + return; + + /* Restore the initial reference without discarding outstanding refs. */ + atomic_long_inc(&data->count); + if (!data->force_atomic) { + atomic_long_add(1UL << (BITS_PER_LONG - 1), &data->count); + for_each_possible_cpu(cpu) + *per_cpu_ptr(percpu_count, cpu) = 0; + ptr &= ~__PERCPU_REF_ATOMIC; + } + + /* Publish initialized counters before allowing new live acquisitions. */ + smp_store_release(&ref->percpu_count_ptr, ptr & ~__PERCPU_REF_DEAD); } #endif +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 20, 0) && \ + (!defined(RHEL_MAJOR) || RHEL_MAJOR - 0 < 8) +#define percpu_ref_resurrect scst_percpu_ref_resurrect_backport +#endif + #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0) && !defined(RHEL_MAJOR) /* * See also commit 4c907baf36d8 ("percpu_ref: implement percpu_ref_is_dying()") diff --git a/scst/src/kunit/lib/suites/scst_kunit_percpu_ref.c b/scst/src/kunit/lib/suites/scst_kunit_percpu_ref.c new file mode 100644 index 000000000..94932c9f9 --- /dev/null +++ b/scst/src/kunit/lib/suites/scst_kunit_percpu_ref.c @@ -0,0 +1,215 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include "../../scst_kunit.h" + +#ifdef INSIDE_KERNEL_TREE +#include +#else +#include "scst.h" +#endif + +struct percpu_ref_fixture { + struct percpu_ref ref; + atomic_t releases; + atomic_t confirmations; + unsigned int held; + bool initialized; + bool dead; +}; + +static void percpu_test_release(struct percpu_ref *ref) +{ + struct percpu_ref_fixture *f = container_of(ref, struct percpu_ref_fixture, ref); + + /* Like the SCST command counters, the fixture outlives zero references. */ + atomic_inc(&f->releases); +} + +static void percpu_test_confirm(struct percpu_ref *ref) +{ + struct percpu_ref_fixture *f = container_of(ref, struct percpu_ref_fixture, ref); + + atomic_inc(&f->confirmations); +} + +static int percpu_test_init(struct kunit *test) +{ + struct percpu_ref_fixture *f; + + f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + if (!f) + return -ENOMEM; + atomic_set(&f->releases, 0); + atomic_set(&f->confirmations, 0); + test->priv = f; + return 0; +} + +static int percpu_test_start(struct kunit *test, unsigned int flags) +{ + struct percpu_ref_fixture *f = test->priv; + int ret; + + ret = percpu_ref_init(&f->ref, percpu_test_release, + flags | PERCPU_REF_ALLOW_REINIT, GFP_KERNEL); + if (!ret) + f->initialized = true; + return ret; +} + +static void percpu_test_get(struct percpu_ref_fixture *f) +{ + percpu_ref_get(&f->ref); + f->held++; +} + +static void percpu_test_put(struct percpu_ref_fixture *f) +{ + f->held--; + percpu_ref_put(&f->ref); +} + +static void percpu_test_kill(struct percpu_ref_fixture *f) +{ + f->dead = true; + percpu_ref_kill_and_confirm(&f->ref, percpu_test_confirm); +} + +static void percpu_test_resurrect(struct percpu_ref_fixture *f) +{ + /* Exercise the backport even when the running kernel has resurrect(). */ + scst_percpu_ref_resurrect_backport(&f->ref); + f->dead = percpu_ref_is_dying(&f->ref); +} + +static void percpu_test_exit(struct kunit *test) +{ + struct percpu_ref_fixture *f = test->priv; + + if (!f || !f->initialized) + return; + if (!f->dead) + percpu_test_kill(f); + /* KUnit runs on kernels using the unified RCU callback API. */ + rcu_barrier(); + while (f->held) + percpu_test_put(f); + percpu_ref_exit(&f->ref); +} + +static void percpu_resurrect_preserves_active_refs(struct kunit *test) +{ + struct percpu_ref_fixture *f = test->priv; + unsigned long allocation; + bool acquired; + + KUNIT_ASSERT_EQ(test, percpu_test_start(test, 0), 0); + allocation = f->ref.percpu_count_ptr & ~__PERCPU_REF_ATOMIC_DEAD; + percpu_test_get(f); + percpu_test_get(f); + percpu_test_get(f); + percpu_test_kill(f); + acquired = percpu_ref_tryget_live(&f->ref); + KUNIT_EXPECT_FALSE(test, acquired); + if (acquired) + percpu_ref_put(&f->ref); + + /* Do not drain RCU first: the backport must wait for a pending switch. */ + percpu_test_resurrect(f); + KUNIT_ASSERT_FALSE(test, f->dead); + KUNIT_EXPECT_EQ(test, atomic_read(&f->confirmations), 1); + KUNIT_EXPECT_EQ(test, atomic_read(&f->releases), 0); + KUNIT_EXPECT_EQ(test, f->ref.percpu_count_ptr, allocation); + acquired = percpu_ref_tryget_live(&f->ref); + KUNIT_EXPECT_TRUE(test, acquired); + if (acquired) + percpu_ref_put(&f->ref); + + /* Regather the counters to detect lost refs or stale per-CPU values. */ + percpu_test_kill(f); + rcu_barrier(); + KUNIT_EXPECT_EQ(test, READ_REF_COUNT(&f->ref), 3L); + KUNIT_EXPECT_EQ(test, atomic_read(&f->releases), 0); + while (f->held) + percpu_test_put(f); + KUNIT_EXPECT_TRUE(test, percpu_ref_is_zero(&f->ref)); + KUNIT_EXPECT_EQ(test, atomic_read(&f->releases), 1); +} + +static void percpu_resurrect_after_drain(struct kunit *test) +{ + struct percpu_ref_fixture *f = test->priv; + unsigned int cycle; + + KUNIT_ASSERT_EQ(test, percpu_test_start(test, 0), 0); + for (cycle = 0; cycle < 3; cycle++) { + percpu_test_kill(f); + rcu_barrier(); + KUNIT_EXPECT_TRUE(test, percpu_ref_is_zero(&f->ref)); + KUNIT_EXPECT_EQ(test, atomic_read(&f->releases), (int)cycle + 1); + percpu_test_resurrect(f); + KUNIT_ASSERT_FALSE(test, f->dead); + KUNIT_EXPECT_FALSE(test, percpu_ref_is_zero(&f->ref)); + } +} + +static void percpu_resurrect_keeps_atomic_mode(struct kunit *test) +{ + struct percpu_ref_fixture *f = test->priv; + + KUNIT_ASSERT_EQ(test, percpu_test_start(test, PERCPU_REF_INIT_ATOMIC), 0); + percpu_test_get(f); + percpu_test_get(f); + percpu_test_kill(f); + percpu_test_resurrect(f); + KUNIT_ASSERT_FALSE(test, f->dead); + KUNIT_EXPECT_TRUE(test, f->ref.percpu_count_ptr & __PERCPU_REF_ATOMIC); + KUNIT_EXPECT_EQ(test, READ_REF_COUNT(&f->ref), 3L); + KUNIT_EXPECT_EQ(test, atomic_read(&f->releases), 0); + percpu_test_kill(f); + rcu_barrier(); + KUNIT_EXPECT_EQ(test, READ_REF_COUNT(&f->ref), 2L); + while (f->held) + percpu_test_put(f); + KUNIT_EXPECT_EQ(test, atomic_read(&f->releases), 1); +} + +static void percpu_resurrect_repeated_cancellations(struct kunit *test) +{ + struct percpu_ref_fixture *f = test->priv; + unsigned int cycle; + + KUNIT_ASSERT_EQ(test, percpu_test_start(test, 0), 0); + percpu_test_get(f); + percpu_test_get(f); + for (cycle = 0; cycle < 5; cycle++) { + percpu_test_kill(f); + rcu_barrier(); + KUNIT_EXPECT_EQ(test, READ_REF_COUNT(&f->ref), 2L); + /* One command completes while suspended; the other survives resume. */ + percpu_test_put(f); + percpu_test_resurrect(f); + KUNIT_ASSERT_FALSE(test, f->dead); + KUNIT_EXPECT_EQ(test, atomic_read(&f->releases), 0); + percpu_test_get(f); + } + percpu_test_kill(f); + rcu_barrier(); + KUNIT_EXPECT_EQ(test, READ_REF_COUNT(&f->ref), 2L); + while (f->held) + percpu_test_put(f); + KUNIT_EXPECT_EQ(test, atomic_read(&f->releases), 1); +} + +static struct kunit_case percpu_ref_cases[] = { + KUNIT_CASE(percpu_resurrect_preserves_active_refs), + KUNIT_CASE(percpu_resurrect_after_drain), + KUNIT_CASE(percpu_resurrect_keeps_atomic_mode), + KUNIT_CASE(percpu_resurrect_repeated_cancellations), + {} +}; + +scst_kunit_suite(percpu_ref_suite, "LIB", "-percpu-ref", + .init = percpu_test_init, + .exit = percpu_test_exit, + .test_cases = percpu_ref_cases, +); diff --git a/scst/src/kunit/objects.mk b/scst/src/kunit/objects.mk index ceef218ab..efc30db11 100644 --- a/scst/src/kunit/objects.mk +++ b/scst/src/kunit/objects.mk @@ -11,6 +11,7 @@ scst-kunit-scst := \ kunit/event/suites/scst_kunit_event_delivery.o \ kunit/lib/suites/scst_kunit_capacity.o \ kunit/lib/suites/scst_kunit_lib.o \ + kunit/lib/suites/scst_kunit_percpu_ref.o \ kunit/lib/suites/scst_kunit_sgv.o \ kunit/lib/suites/scst_kunit_write_same.o \ kunit/lifecycle/suites/scst_kunit_lifecycle.o \