iv/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

srcu: Add contention check to call_srcu() srcu_data ->lock acquisition

Browse Source 
This commit increases the sensitivity of contention detection by adding
checks to the acquisition of the srcu_data structure's lock on the
call_srcu() code path.

Co-developed-by: Neeraj Upadhyay <quic_neeraju@quicinc.com>
Signed-off-by: Neeraj Upadhyay <quic_neeraju@quicinc.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
This commit is contained in:
Paul E. McKenney 2022-01-31 13:27:15 -08:00
parent a57ffb3c6b
commit c2445d3878
1 changed files with 36 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
kernel/rcu/srcutree.c
View File

@ -330,18 +330,13 @@ static void srcu_transition_to_big(struct srcu_struct *ssp)
}
/*
* Acquire the specified srcu_struct structure's ->lock, but check for
* excessive contention, which results in initiation of a transition
* to SRCU_SIZE_BIG. But only if the srcutree.convert_to_big module
* parameter permits this.
* Check to see if the just-encountered contention event justifies
* a transition to SRCU_SIZE_BIG.
*/
static void spin_lock_irqsave_ssp_contention(struct srcu_struct *ssp, unsigned long *flags)
static void spin_lock_irqsave_check_contention(struct srcu_struct *ssp)
{
unsigned long j;
if (spin_trylock_irqsave_rcu_node(ssp, *flags))
return;
spin_lock_irqsave_rcu_node(ssp, *flags);
if (!SRCU_SIZING_IS_CONTEND() || ssp->srcu_size_state)
return;
j = jiffies;
@ -354,6 +349,38 @@ static void spin_lock_irqsave_ssp_contention(struct srcu_struct *ssp, unsigned l
__srcu_transition_to_big(ssp);
}
/*
* Acquire the specified srcu_data structure's ->lock, but check for
* excessive contention, which results in initiation of a transition
* to SRCU_SIZE_BIG. But only if the srcutree.convert_to_big module
* parameter permits this.
*/
static void spin_lock_irqsave_sdp_contention(struct srcu_data *sdp, unsigned long *flags)
{
struct srcu_struct *ssp = sdp->ssp;
if (spin_trylock_irqsave_rcu_node(sdp, *flags))
return;
spin_lock_irqsave_rcu_node(ssp, *flags);
spin_lock_irqsave_check_contention(ssp);
spin_unlock_irqrestore_rcu_node(ssp, *flags);
spin_lock_irqsave_rcu_node(sdp, *flags);
}
/*
* Acquire the specified srcu_struct structure's ->lock, but check for
* excessive contention, which results in initiation of a transition
* to SRCU_SIZE_BIG. But only if the srcutree.convert_to_big module
* parameter permits this.
*/
static void spin_lock_irqsave_ssp_contention(struct srcu_struct *ssp, unsigned long *flags)
{
if (spin_trylock_irqsave_rcu_node(ssp, *flags))
return;
spin_lock_irqsave_rcu_node(ssp, *flags);
spin_lock_irqsave_check_contention(ssp);
}
/*
* First-use initialization of statically allocated srcu_struct
* structure. Wiring up the combining tree is more than can be
@ -1010,7 +1037,7 @@ static unsigned long srcu_gp_start_if_needed(struct srcu_struct *ssp,
sdp = per_cpu_ptr(ssp->sda, 0);
else
sdp = raw_cpu_ptr(ssp->sda);
spin_lock_irqsave_rcu_node(sdp, flags);
spin_lock_irqsave_sdp_contention(sdp, &flags);
if (rhp)
rcu_segcblist_enqueue(&sdp->srcu_cblist, rhp);
rcu_segcblist_advance(&sdp->srcu_cblist,