From b63343207da21179d82df933f42a1725a53b9a63 Mon Sep 17 00:00:00 2001 From: John Stultz Date: Tue, 21 Feb 2023 19:02:35 +0000 Subject: [PATCH 1/5] locktorture: Add nested_[un]lock() hooks and nlocks parameter In order to extend locktorture to support lock nesting, add nested_lock() and nested_unlock() hooks to the torture ops. These take a 32bit lockset mask which is generated at random, so some number of locks will be taken before the main lock is taken and released afterwards. Additionally, add nested_locks module parameter to allow specifying the number of nested locks to be used. This has been helpful to uncover issues in the proxy-exec series development. This was inspired by locktorture extensions originally implemented by Connor O'Brien, for stress testing the proxy-execution series: https://lore.kernel.org/lkml/20221003214501.2050087-12-connoro@google.com/ Cc: Davidlohr Bueso Cc: "Paul E. McKenney" Cc: Josh Triplett Cc: Joel Fernandes Cc: Juri Lelli Cc: Valentin Schneider Cc: Dietmar Eggemann Cc: kernel-team@android.com Reviewed-by: Davidlohr Bueso Co-developed-by: Connor O'Brien Signed-off-by: Connor O'Brien Signed-off-by: John Stultz Signed-off-by: Paul E. McKenney --- kernel/locking/locktorture.c | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/kernel/locking/locktorture.c b/kernel/locking/locktorture.c index f04b1978899d..a8519af25ece 100644 --- a/kernel/locking/locktorture.c +++ b/kernel/locking/locktorture.c @@ -51,6 +51,9 @@ torture_param(int, rt_boost, 2, torture_param(int, rt_boost_factor, 50, "A factor determining how often rt-boost happens."); torture_param(int, verbose, 1, "Enable verbose debugging printk()s"); +torture_param(int, nested_locks, 0, "Number of nested locks (max = 8)"); +/* Going much higher trips "BUG: MAX_LOCKDEP_CHAIN_HLOCKS too low!" errors */ +#define MAX_NESTED_LOCKS 8 static char *torture_type = "spin_lock"; module_param(torture_type, charp, 0444); @@ -79,10 +82,12 @@ static void lock_torture_cleanup(void); struct lock_torture_ops { void (*init)(void); void (*exit)(void); + int (*nested_lock)(int tid, u32 lockset); int (*writelock)(int tid); void (*write_delay)(struct torture_random_state *trsp); void (*task_boost)(struct torture_random_state *trsp); void (*writeunlock)(int tid); + void (*nested_unlock)(int tid, u32 lockset); int (*readlock)(int tid); void (*read_delay)(struct torture_random_state *trsp); void (*readunlock)(int tid); @@ -684,6 +689,7 @@ static int lock_torture_writer(void *arg) struct lock_stress_stats *lwsp = arg; int tid = lwsp - cxt.lwsa; DEFINE_TORTURE_RANDOM(rand); + u32 lockset_mask; VERBOSE_TOROUT_STRING("lock_torture_writer task started"); set_user_nice(current, MAX_NICE); @@ -692,7 +698,10 @@ static int lock_torture_writer(void *arg) if ((torture_random(&rand) & 0xfffff) == 0) schedule_timeout_uninterruptible(1); + lockset_mask = torture_random(&rand); cxt.cur_ops->task_boost(&rand); + if (cxt.cur_ops->nested_lock) + cxt.cur_ops->nested_lock(tid, lockset_mask); cxt.cur_ops->writelock(tid); if (WARN_ON_ONCE(lock_is_write_held)) lwsp->n_lock_fail++; @@ -705,6 +714,8 @@ static int lock_torture_writer(void *arg) lock_is_write_held = false; WRITE_ONCE(last_lock_release, jiffies); cxt.cur_ops->writeunlock(tid); + if (cxt.cur_ops->nested_unlock) + cxt.cur_ops->nested_unlock(tid, lockset_mask); stutter_wait("lock_torture_writer"); } while (!torture_must_stop()); @@ -845,11 +856,11 @@ lock_torture_print_module_parms(struct lock_torture_ops *cur_ops, const char *tag) { pr_alert("%s" TORTURE_FLAG - "--- %s%s: nwriters_stress=%d nreaders_stress=%d stat_interval=%d verbose=%d shuffle_interval=%d stutter=%d shutdown_secs=%d onoff_interval=%d onoff_holdoff=%d\n", + "--- %s%s: nwriters_stress=%d nreaders_stress=%d nested_locks=%d stat_interval=%d verbose=%d shuffle_interval=%d stutter=%d shutdown_secs=%d onoff_interval=%d onoff_holdoff=%d\n", torture_type, tag, cxt.debug_lock ? " [debug]": "", - cxt.nrealwriters_stress, cxt.nrealreaders_stress, stat_interval, - verbose, shuffle_interval, stutter, shutdown_secs, - onoff_interval, onoff_holdoff); + cxt.nrealwriters_stress, cxt.nrealreaders_stress, + nested_locks, stat_interval, verbose, shuffle_interval, + stutter, shutdown_secs, onoff_interval, onoff_holdoff); } static void lock_torture_cleanup(void) @@ -1068,6 +1079,10 @@ static int __init lock_torture_init(void) } } + /* cap nested_locks to MAX_NESTED_LOCKS */ + if (nested_locks > MAX_NESTED_LOCKS) + nested_locks = MAX_NESTED_LOCKS; + if (cxt.cur_ops->readlock) { reader_tasks = kcalloc(cxt.nrealreaders_stress, sizeof(reader_tasks[0]), From 3e5aeaf53422bdd79fecf89aa160f048114e013c Mon Sep 17 00:00:00 2001 From: John Stultz Date: Tue, 21 Feb 2023 19:02:36 +0000 Subject: [PATCH 2/5] locktorture: Add nested locking to mutex torture tests This patch adds randomized nested locking to the mutex torture tests, as well as new LOCK08 config files for testing mutexes with nested locking Cc: Davidlohr Bueso Cc: "Paul E. McKenney" Cc: Josh Triplett Cc: Joel Fernandes Cc: Juri Lelli Cc: Valentin Schneider Cc: Dietmar Eggemann Cc: kernel-team@android.com Reviewed-by: Davidlohr Bueso Co-developed-by: Connor O'Brien Signed-off-by: Connor O'Brien Signed-off-by: John Stultz Signed-off-by: Paul E. McKenney --- kernel/locking/locktorture.c | 35 +++++++++++++++++++ .../selftests/rcutorture/configs/lock/CFLIST | 1 + .../selftests/rcutorture/configs/lock/LOCK08 | 6 ++++ .../rcutorture/configs/lock/LOCK08.boot | 1 + 4 files changed, 43 insertions(+) create mode 100644 tools/testing/selftests/rcutorture/configs/lock/LOCK08 create mode 100644 tools/testing/selftests/rcutorture/configs/lock/LOCK08.boot diff --git a/kernel/locking/locktorture.c b/kernel/locking/locktorture.c index a8519af25ece..a4d15a9a9d7f 100644 --- a/kernel/locking/locktorture.c +++ b/kernel/locking/locktorture.c @@ -370,6 +370,28 @@ static struct lock_torture_ops rw_lock_irq_ops = { }; static DEFINE_MUTEX(torture_mutex); +static struct mutex torture_nested_mutexes[MAX_NESTED_LOCKS]; +static struct lock_class_key nested_mutex_keys[MAX_NESTED_LOCKS]; + +static void torture_mutex_init(void) +{ + int i; + + for (i = 0; i < MAX_NESTED_LOCKS; i++) + __mutex_init(&torture_nested_mutexes[i], __func__, + &nested_mutex_keys[i]); +} + +static int torture_mutex_nested_lock(int tid __maybe_unused, + u32 lockset) +{ + int i; + + for (i = 0; i < nested_locks; i++) + if (lockset & (1 << i)) + mutex_lock(&torture_nested_mutexes[i]); + return 0; +} static int torture_mutex_lock(int tid __maybe_unused) __acquires(torture_mutex) @@ -398,11 +420,24 @@ __releases(torture_mutex) mutex_unlock(&torture_mutex); } +static void torture_mutex_nested_unlock(int tid __maybe_unused, + u32 lockset) +{ + int i; + + for (i = nested_locks - 1; i >= 0; i--) + if (lockset & (1 << i)) + mutex_unlock(&torture_nested_mutexes[i]); +} + static struct lock_torture_ops mutex_lock_ops = { + .init = torture_mutex_init, + .nested_lock = torture_mutex_nested_lock, .writelock = torture_mutex_lock, .write_delay = torture_mutex_delay, .task_boost = torture_rt_boost, .writeunlock = torture_mutex_unlock, + .nested_unlock = torture_mutex_nested_unlock, .readlock = NULL, .read_delay = NULL, .readunlock = NULL, diff --git a/tools/testing/selftests/rcutorture/configs/lock/CFLIST b/tools/testing/selftests/rcutorture/configs/lock/CFLIST index 41bae5824339..a48bba0d35a6 100644 --- a/tools/testing/selftests/rcutorture/configs/lock/CFLIST +++ b/tools/testing/selftests/rcutorture/configs/lock/CFLIST @@ -5,3 +5,4 @@ LOCK04 LOCK05 LOCK06 LOCK07 +LOCK08 diff --git a/tools/testing/selftests/rcutorture/configs/lock/LOCK08 b/tools/testing/selftests/rcutorture/configs/lock/LOCK08 new file mode 100644 index 000000000000..1d1da1477fc3 --- /dev/null +++ b/tools/testing/selftests/rcutorture/configs/lock/LOCK08 @@ -0,0 +1,6 @@ +CONFIG_SMP=y +CONFIG_NR_CPUS=4 +CONFIG_HOTPLUG_CPU=y +CONFIG_PREEMPT_NONE=n +CONFIG_PREEMPT_VOLUNTARY=n +CONFIG_PREEMPT=y diff --git a/tools/testing/selftests/rcutorture/configs/lock/LOCK08.boot b/tools/testing/selftests/rcutorture/configs/lock/LOCK08.boot new file mode 100644 index 000000000000..b8b6caebb89e --- /dev/null +++ b/tools/testing/selftests/rcutorture/configs/lock/LOCK08.boot @@ -0,0 +1 @@ +locktorture.torture_type=mutex_lock locktorture.nested_locks=8 From ae4823e427954d30ab393888a334f9d1fd8cd597 Mon Sep 17 00:00:00 2001 From: John Stultz Date: Tue, 21 Feb 2023 19:02:37 +0000 Subject: [PATCH 3/5] locktorture: Add nested locking to rtmutex torture tests This patch adds randomized nested locking to the rtmutex torture tests. Additionally it adds LOCK09 config files for testing rtmutexes with nested locking. Cc: Davidlohr Bueso Cc: "Paul E. McKenney" Cc: Josh Triplett Cc: Joel Fernandes Cc: Juri Lelli Cc: Valentin Schneider Cc: Dietmar Eggemann Cc: kernel-team@android.com Reviewed-by: Davidlohr Bueso Co-developed-by: Connor O'Brien Signed-off-by: Connor O'Brien Signed-off-by: John Stultz Signed-off-by: Paul E. McKenney --- kernel/locking/locktorture.c | 35 +++++++++++++++++++ .../selftests/rcutorture/configs/lock/CFLIST | 1 + .../selftests/rcutorture/configs/lock/LOCK09 | 6 ++++ .../rcutorture/configs/lock/LOCK09.boot | 1 + 4 files changed, 43 insertions(+) create mode 100644 tools/testing/selftests/rcutorture/configs/lock/LOCK09 create mode 100644 tools/testing/selftests/rcutorture/configs/lock/LOCK09.boot diff --git a/kernel/locking/locktorture.c b/kernel/locking/locktorture.c index a4d15a9a9d7f..c7f264aed5b7 100644 --- a/kernel/locking/locktorture.c +++ b/kernel/locking/locktorture.c @@ -544,6 +544,28 @@ static struct lock_torture_ops ww_mutex_lock_ops = { #ifdef CONFIG_RT_MUTEXES static DEFINE_RT_MUTEX(torture_rtmutex); +static struct rt_mutex torture_nested_rtmutexes[MAX_NESTED_LOCKS]; +static struct lock_class_key nested_rtmutex_keys[MAX_NESTED_LOCKS]; + +static void torture_rtmutex_init(void) +{ + int i; + + for (i = 0; i < MAX_NESTED_LOCKS; i++) + __rt_mutex_init(&torture_nested_rtmutexes[i], __func__, + &nested_rtmutex_keys[i]); +} + +static int torture_rtmutex_nested_lock(int tid __maybe_unused, + u32 lockset) +{ + int i; + + for (i = 0; i < nested_locks; i++) + if (lockset & (1 << i)) + rt_mutex_lock(&torture_nested_rtmutexes[i]); + return 0; +} static int torture_rtmutex_lock(int tid __maybe_unused) __acquires(torture_rtmutex) @@ -585,11 +607,24 @@ static void torture_rt_boost_rtmutex(struct torture_random_state *trsp) __torture_rt_boost(trsp); } +static void torture_rtmutex_nested_unlock(int tid __maybe_unused, + u32 lockset) +{ + int i; + + for (i = nested_locks - 1; i >= 0; i--) + if (lockset & (1 << i)) + rt_mutex_unlock(&torture_nested_rtmutexes[i]); +} + static struct lock_torture_ops rtmutex_lock_ops = { + .init = torture_rtmutex_init, + .nested_lock = torture_rtmutex_nested_lock, .writelock = torture_rtmutex_lock, .write_delay = torture_rtmutex_delay, .task_boost = torture_rt_boost_rtmutex, .writeunlock = torture_rtmutex_unlock, + .nested_unlock = torture_rtmutex_nested_unlock, .readlock = NULL, .read_delay = NULL, .readunlock = NULL, diff --git a/tools/testing/selftests/rcutorture/configs/lock/CFLIST b/tools/testing/selftests/rcutorture/configs/lock/CFLIST index a48bba0d35a6..28e23d05d5a5 100644 --- a/tools/testing/selftests/rcutorture/configs/lock/CFLIST +++ b/tools/testing/selftests/rcutorture/configs/lock/CFLIST @@ -6,3 +6,4 @@ LOCK05 LOCK06 LOCK07 LOCK08 +LOCK09 diff --git a/tools/testing/selftests/rcutorture/configs/lock/LOCK09 b/tools/testing/selftests/rcutorture/configs/lock/LOCK09 new file mode 100644 index 000000000000..1d1da1477fc3 --- /dev/null +++ b/tools/testing/selftests/rcutorture/configs/lock/LOCK09 @@ -0,0 +1,6 @@ +CONFIG_SMP=y +CONFIG_NR_CPUS=4 +CONFIG_HOTPLUG_CPU=y +CONFIG_PREEMPT_NONE=n +CONFIG_PREEMPT_VOLUNTARY=n +CONFIG_PREEMPT=y diff --git a/tools/testing/selftests/rcutorture/configs/lock/LOCK09.boot b/tools/testing/selftests/rcutorture/configs/lock/LOCK09.boot new file mode 100644 index 000000000000..fd5eff148a93 --- /dev/null +++ b/tools/testing/selftests/rcutorture/configs/lock/LOCK09.boot @@ -0,0 +1 @@ +locktorture.torture_type=rtmutex_lock locktorture.nested_locks=8 From 45bcf0bd8cbe163c5aec18570b6befd2193f1a57 Mon Sep 17 00:00:00 2001 From: John Stultz Date: Tue, 21 Feb 2023 19:02:38 +0000 Subject: [PATCH 4/5] locktorture: With nested locks, occasionally skip main lock If we're using nested locking to stress things, occasionally skip taking the main lock, so that we can get some different contention patterns between the writers (to hopefully get two disjoint blocked trees) Cc: Davidlohr Bueso Cc: "Paul E. McKenney" Cc: Josh Triplett Cc: Joel Fernandes Cc: Juri Lelli Cc: Valentin Schneider Cc: Dietmar Eggemann Cc: kernel-team@android.com Co-developed-by: Connor O'Brien Signed-off-by: Connor O'Brien Signed-off-by: John Stultz Signed-off-by: Paul E. McKenney --- kernel/locking/locktorture.c | 37 ++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/kernel/locking/locktorture.c b/kernel/locking/locktorture.c index c7f264aed5b7..9425aff08936 100644 --- a/kernel/locking/locktorture.c +++ b/kernel/locking/locktorture.c @@ -760,6 +760,7 @@ static int lock_torture_writer(void *arg) int tid = lwsp - cxt.lwsa; DEFINE_TORTURE_RANDOM(rand); u32 lockset_mask; + bool skip_main_lock; VERBOSE_TOROUT_STRING("lock_torture_writer task started"); set_user_nice(current, MAX_NICE); @@ -769,21 +770,37 @@ static int lock_torture_writer(void *arg) schedule_timeout_uninterruptible(1); lockset_mask = torture_random(&rand); + /* + * When using nested_locks, we want to occasionally + * skip the main lock so we can avoid always serializing + * the lock chains on that central lock. By skipping the + * main lock occasionally, we can create different + * contention patterns (allowing for multiple disjoint + * blocked trees) + */ + skip_main_lock = (nested_locks && + !(torture_random(&rand) % 100)); + cxt.cur_ops->task_boost(&rand); if (cxt.cur_ops->nested_lock) cxt.cur_ops->nested_lock(tid, lockset_mask); - cxt.cur_ops->writelock(tid); - if (WARN_ON_ONCE(lock_is_write_held)) - lwsp->n_lock_fail++; - lock_is_write_held = true; - if (WARN_ON_ONCE(atomic_read(&lock_is_read_held))) - lwsp->n_lock_fail++; /* rare, but... */ - lwsp->n_lock_acquired++; + if (!skip_main_lock) { + cxt.cur_ops->writelock(tid); + if (WARN_ON_ONCE(lock_is_write_held)) + lwsp->n_lock_fail++; + lock_is_write_held = true; + if (WARN_ON_ONCE(atomic_read(&lock_is_read_held))) + lwsp->n_lock_fail++; /* rare, but... */ + + lwsp->n_lock_acquired++; + } cxt.cur_ops->write_delay(&rand); - lock_is_write_held = false; - WRITE_ONCE(last_lock_release, jiffies); - cxt.cur_ops->writeunlock(tid); + if (!skip_main_lock) { + lock_is_write_held = false; + WRITE_ONCE(last_lock_release, jiffies); + cxt.cur_ops->writeunlock(tid); + } if (cxt.cur_ops->nested_unlock) cxt.cur_ops->nested_unlock(tid, lockset_mask); From 5d65cf6ae6aea1a8d533d4499201a13d0068a0dc Mon Sep 17 00:00:00 2001 From: Zqiang Date: Fri, 24 Feb 2023 09:20:35 +0800 Subject: [PATCH 5/5] locktorture: Add raw_spinlock* torture tests for PREEMPT_RT kernels In PREEMPT_RT kernels, both spin_lock() and spin_lock_irq() are converted to sleepable rt_spin_lock(). This means that the interrupt related suffixes for spin_lock/unlock(_irq, irqsave/irqrestore) do not affect the CPU's interrupt state. This commit therefore adds raw spin-lock torture tests. This in turn permits pure spin locks to be tested in PREEMPT_RT kernels. Suggested-by: Paul E. McKenney Suggested-by: Davidlohr Bueso Signed-off-by: Zqiang Reviewed-by: Davidlohr Bueso Signed-off-by: Paul E. McKenney --- kernel/locking/locktorture.c | 56 +++++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/kernel/locking/locktorture.c b/kernel/locking/locktorture.c index 9425aff08936..153ddc4c47ef 100644 --- a/kernel/locking/locktorture.c +++ b/kernel/locking/locktorture.c @@ -55,7 +55,7 @@ torture_param(int, nested_locks, 0, "Number of nested locks (max = 8)"); /* Going much higher trips "BUG: MAX_LOCKDEP_CHAIN_HLOCKS too low!" errors */ #define MAX_NESTED_LOCKS 8 -static char *torture_type = "spin_lock"; +static char *torture_type = IS_ENABLED(CONFIG_PREEMPT_RT) ? "raw_spin_lock" : "spin_lock"; module_param(torture_type, charp, 0444); MODULE_PARM_DESC(torture_type, "Type of lock to torture (spin_lock, spin_lock_irq, mutex_lock, ...)"); @@ -257,6 +257,59 @@ static struct lock_torture_ops spin_lock_irq_ops = { .name = "spin_lock_irq" }; +static DEFINE_RAW_SPINLOCK(torture_raw_spinlock); + +static int torture_raw_spin_lock_write_lock(int tid __maybe_unused) +__acquires(torture_raw_spinlock) +{ + raw_spin_lock(&torture_raw_spinlock); + return 0; +} + +static void torture_raw_spin_lock_write_unlock(int tid __maybe_unused) +__releases(torture_raw_spinlock) +{ + raw_spin_unlock(&torture_raw_spinlock); +} + +static struct lock_torture_ops raw_spin_lock_ops = { + .writelock = torture_raw_spin_lock_write_lock, + .write_delay = torture_spin_lock_write_delay, + .task_boost = torture_rt_boost, + .writeunlock = torture_raw_spin_lock_write_unlock, + .readlock = NULL, + .read_delay = NULL, + .readunlock = NULL, + .name = "raw_spin_lock" +}; + +static int torture_raw_spin_lock_write_lock_irq(int tid __maybe_unused) +__acquires(torture_raw_spinlock) +{ + unsigned long flags; + + raw_spin_lock_irqsave(&torture_raw_spinlock, flags); + cxt.cur_ops->flags = flags; + return 0; +} + +static void torture_raw_spin_lock_write_unlock_irq(int tid __maybe_unused) +__releases(torture_raw_spinlock) +{ + raw_spin_unlock_irqrestore(&torture_raw_spinlock, cxt.cur_ops->flags); +} + +static struct lock_torture_ops raw_spin_lock_irq_ops = { + .writelock = torture_raw_spin_lock_write_lock_irq, + .write_delay = torture_spin_lock_write_delay, + .task_boost = torture_rt_boost, + .writeunlock = torture_raw_spin_lock_write_unlock_irq, + .readlock = NULL, + .read_delay = NULL, + .readunlock = NULL, + .name = "raw_spin_lock_irq" +}; + static DEFINE_RWLOCK(torture_rwlock); static int torture_rwlock_write_lock(int tid __maybe_unused) @@ -1017,6 +1070,7 @@ static int __init lock_torture_init(void) static struct lock_torture_ops *torture_ops[] = { &lock_busted_ops, &spin_lock_ops, &spin_lock_irq_ops, + &raw_spin_lock_ops, &raw_spin_lock_irq_ops, &rw_lock_ops, &rw_lock_irq_ops, &mutex_lock_ops, &ww_mutex_lock_ops,