2019-01-17 21:23:39 +03:00
/* SPDX-License-Identifier: GPL-2.0+ */
2016-04-16 02:35:29 +03:00
/*
* RCU expedited grace periods
*
* Copyright IBM Corporation , 2016
*
2019-01-17 21:23:39 +03:00
* Authors : Paul E . McKenney < paulmck @ linux . ibm . com >
2016-04-16 02:35:29 +03:00
*/
2018-03-09 04:14:51 +03:00
# include <linux/lockdep.h>
2018-11-29 20:15:54 +03:00
static void rcu_exp_handler ( void * unused ) ;
2019-01-12 07:51:49 +03:00
static int rcu_print_task_exp_stall ( struct rcu_node * rnp ) ;
2018-11-29 20:15:54 +03:00
2016-12-19 00:31:02 +03:00
/*
* Record the start of an expedited grace period .
*/
2018-07-04 03:22:34 +03:00
static void rcu_exp_gp_seq_start ( void )
2016-04-16 02:35:29 +03:00
{
2018-07-04 03:22:34 +03:00
rcu_seq_start ( & rcu_state . expedited_sequence ) ;
2016-04-16 02:35:29 +03:00
}
2016-12-19 00:31:02 +03:00
2018-02-01 06:23:24 +03:00
/*
2019-11-28 01:20:41 +03:00
* Return the value that the expedited - grace - period counter will have
2018-02-01 06:23:24 +03:00
* at the end of the current grace period .
*/
2018-07-04 03:22:34 +03:00
static __maybe_unused unsigned long rcu_exp_gp_seq_endval ( void )
2018-02-01 06:23:24 +03:00
{
2018-07-04 03:22:34 +03:00
return rcu_seq_endval ( & rcu_state . expedited_sequence ) ;
2018-02-01 06:23:24 +03:00
}
2016-12-19 00:31:02 +03:00
/*
* Record the end of an expedited grace period .
*/
2018-07-04 03:22:34 +03:00
static void rcu_exp_gp_seq_end ( void )
2016-04-16 02:35:29 +03:00
{
2018-07-04 03:22:34 +03:00
rcu_seq_end ( & rcu_state . expedited_sequence ) ;
2016-04-16 02:35:29 +03:00
smp_mb ( ) ; /* Ensure that consecutive grace periods serialize. */
}
2016-12-19 00:31:02 +03:00
/*
2019-11-28 01:20:41 +03:00
* Take a snapshot of the expedited - grace - period counter , which is the
* earliest value that will indicate that a full grace period has
* elapsed since the current time .
2016-12-19 00:31:02 +03:00
*/
2018-07-04 03:22:34 +03:00
static unsigned long rcu_exp_gp_seq_snap ( void )
2016-04-16 02:35:29 +03:00
{
unsigned long s ;
smp_mb ( ) ; /* Caller's modifications seen first by other CPUs. */
2018-07-04 03:22:34 +03:00
s = rcu_seq_snap ( & rcu_state . expedited_sequence ) ;
trace_rcu_exp_grace_period ( rcu_state . name , s , TPS ( " snap " ) ) ;
2016-04-16 02:35:29 +03:00
return s ;
}
2016-12-19 00:31:02 +03:00
/*
* Given a counter snapshot from rcu_exp_gp_seq_snap ( ) , return true
* if a full expedited grace period has elapsed since that snapshot
* was taken .
*/
2018-07-04 03:22:34 +03:00
static bool rcu_exp_gp_seq_done ( unsigned long s )
2016-04-16 02:35:29 +03:00
{
2018-07-04 03:22:34 +03:00
return rcu_seq_done ( & rcu_state . expedited_sequence , s ) ;
2016-04-16 02:35:29 +03:00
}
/*
* Reset the - > expmaskinit values in the rcu_node tree to reflect any
* recent CPU - online activity . Note that these masks are not cleared
* when CPUs go offline , so they reflect the union of all CPUs that have
* ever been online . This means that this function normally takes its
* no - work - to - do fastpath .
*/
2018-07-04 03:22:34 +03:00
static void sync_exp_reset_tree_hotplug ( void )
2016-04-16 02:35:29 +03:00
{
bool done ;
unsigned long flags ;
unsigned long mask ;
unsigned long oldmask ;
2018-07-04 03:22:34 +03:00
int ncpus = smp_load_acquire ( & rcu_state . ncpus ) ; /* Order vs. locking. */
2016-04-16 02:35:29 +03:00
struct rcu_node * rnp ;
struct rcu_node * rnp_up ;
/* If no new CPUs onlined since last time, nothing to do. */
2018-07-04 03:22:34 +03:00
if ( likely ( ncpus = = rcu_state . ncpus_snap ) )
2016-04-16 02:35:29 +03:00
return ;
2018-07-04 03:22:34 +03:00
rcu_state . ncpus_snap = ncpus ;
2016-04-16 02:35:29 +03:00
/*
* Each pass through the following loop propagates newly onlined
* CPUs for the current rcu_node structure up the rcu_node tree .
*/
2018-07-05 00:33:59 +03:00
rcu_for_each_leaf_node ( rnp ) {
2016-04-16 02:35:29 +03:00
raw_spin_lock_irqsave_rcu_node ( rnp , flags ) ;
if ( rnp - > expmaskinit = = rnp - > expmaskinitnext ) {
raw_spin_unlock_irqrestore_rcu_node ( rnp , flags ) ;
continue ; /* No new CPUs, nothing to do. */
}
/* Update this node's mask, track old value for propagation. */
oldmask = rnp - > expmaskinit ;
rnp - > expmaskinit = rnp - > expmaskinitnext ;
raw_spin_unlock_irqrestore_rcu_node ( rnp , flags ) ;
/* If was already nonzero, nothing to propagate. */
if ( oldmask )
continue ;
/* Propagate the new CPU up the tree. */
mask = rnp - > grpmask ;
rnp_up = rnp - > parent ;
done = false ;
while ( rnp_up ) {
raw_spin_lock_irqsave_rcu_node ( rnp_up , flags ) ;
if ( rnp_up - > expmaskinit )
done = true ;
rnp_up - > expmaskinit | = mask ;
raw_spin_unlock_irqrestore_rcu_node ( rnp_up , flags ) ;
if ( done )
break ;
mask = rnp_up - > grpmask ;
rnp_up = rnp_up - > parent ;
}
}
}
/*
* Reset the - > expmask values in the rcu_node tree in preparation for
* a new expedited grace period .
*/
2018-07-04 03:22:34 +03:00
static void __maybe_unused sync_exp_reset_tree ( void )
2016-04-16 02:35:29 +03:00
{
unsigned long flags ;
struct rcu_node * rnp ;
2018-07-04 03:22:34 +03:00
sync_exp_reset_tree_hotplug ( ) ;
2018-07-05 00:33:59 +03:00
rcu_for_each_node_breadth_first ( rnp ) {
2016-04-16 02:35:29 +03:00
raw_spin_lock_irqsave_rcu_node ( rnp , flags ) ;
WARN_ON_ONCE ( rnp - > expmask ) ;
2019-10-08 04:53:18 +03:00
WRITE_ONCE ( rnp - > expmask , rnp - > expmaskinit ) ;
2016-04-16 02:35:29 +03:00
raw_spin_unlock_irqrestore_rcu_node ( rnp , flags ) ;
}
}
/*
* Return non - zero if there is no RCU expedited grace period in progress
* for the specified rcu_node structure , in other words , if all CPUs and
* tasks covered by the specified rcu_node structure have done their bit
2019-11-28 01:20:41 +03:00
* for the current expedited grace period .
2016-04-16 02:35:29 +03:00
*/
2019-11-28 00:59:37 +03:00
static bool sync_rcu_exp_done ( struct rcu_node * rnp )
2016-04-16 02:35:29 +03:00
{
2018-03-09 04:14:51 +03:00
raw_lockdep_assert_held_rcu_node ( rnp ) ;
2020-01-04 01:18:12 +03:00
return READ_ONCE ( rnp - > exp_tasks ) = = NULL & &
2016-04-16 02:35:29 +03:00
READ_ONCE ( rnp - > expmask ) = = 0 ;
}
2018-03-09 04:14:51 +03:00
/*
2019-11-28 01:20:41 +03:00
* Like sync_rcu_exp_done ( ) , but where the caller does not hold the
* rcu_node ' s - > lock .
2018-03-09 04:14:51 +03:00
*/
2019-11-28 00:59:37 +03:00
static bool sync_rcu_exp_done_unlocked ( struct rcu_node * rnp )
2018-03-09 04:14:51 +03:00
{
unsigned long flags ;
bool ret ;
raw_spin_lock_irqsave_rcu_node ( rnp , flags ) ;
2019-11-28 00:59:37 +03:00
ret = sync_rcu_exp_done ( rnp ) ;
2018-03-09 04:14:51 +03:00
raw_spin_unlock_irqrestore_rcu_node ( rnp , flags ) ;
return ret ;
}
2016-04-16 02:35:29 +03:00
/*
* Report the exit from RCU read - side critical section for the last task
* that queued itself during or before the current expedited preemptible - RCU
* grace period . This event is reported either to the rcu_node structure on
* which the task was queued or to one of that rcu_node structure ' s ancestors ,
* recursively up the tree . ( Calm down , calm down , we do the recursion
* iteratively ! )
*/
2018-07-04 03:22:34 +03:00
static void __rcu_report_exp_rnp ( struct rcu_node * rnp ,
2016-04-16 02:35:29 +03:00
bool wake , unsigned long flags )
__releases ( rnp - > lock )
{
unsigned long mask ;
2019-11-28 01:20:41 +03:00
raw_lockdep_assert_held_rcu_node ( rnp ) ;
2016-04-16 02:35:29 +03:00
for ( ; ; ) {
2019-11-28 00:59:37 +03:00
if ( ! sync_rcu_exp_done ( rnp ) ) {
2016-04-16 02:35:29 +03:00
if ( ! rnp - > expmask )
rcu_initiate_boost ( rnp , flags ) ;
else
raw_spin_unlock_irqrestore_rcu_node ( rnp , flags ) ;
break ;
}
if ( rnp - > parent = = NULL ) {
raw_spin_unlock_irqrestore_rcu_node ( rnp , flags ) ;
if ( wake ) {
smp_mb ( ) ; /* EGP done before wake_up(). */
2018-07-04 03:22:34 +03:00
swake_up_one ( & rcu_state . expedited_wq ) ;
2016-04-16 02:35:29 +03:00
}
break ;
}
mask = rnp - > grpmask ;
raw_spin_unlock_rcu_node ( rnp ) ; /* irqs remain disabled */
rnp = rnp - > parent ;
raw_spin_lock_rcu_node ( rnp ) ; /* irqs already disabled */
WARN_ON_ONCE ( ! ( rnp - > expmask & mask ) ) ;
2019-10-08 04:53:18 +03:00
WRITE_ONCE ( rnp - > expmask , rnp - > expmask & ~ mask ) ;
2016-04-16 02:35:29 +03:00
}
}
/*
* Report expedited quiescent state for specified node . This is a
* lock - acquisition wrapper function for __rcu_report_exp_rnp ( ) .
*/
2018-07-04 03:22:34 +03:00
static void __maybe_unused rcu_report_exp_rnp ( struct rcu_node * rnp , bool wake )
2016-04-16 02:35:29 +03:00
{
unsigned long flags ;
raw_spin_lock_irqsave_rcu_node ( rnp , flags ) ;
2018-07-04 03:22:34 +03:00
__rcu_report_exp_rnp ( rnp , wake , flags ) ;
2016-04-16 02:35:29 +03:00
}
/*
* Report expedited quiescent state for multiple CPUs , all covered by the
2018-03-07 11:49:39 +03:00
* specified leaf rcu_node structure .
2016-04-16 02:35:29 +03:00
*/
2018-07-04 03:22:34 +03:00
static void rcu_report_exp_cpu_mult ( struct rcu_node * rnp ,
2016-04-16 02:35:29 +03:00
unsigned long mask , bool wake )
{
2019-11-28 03:36:45 +03:00
int cpu ;
2016-04-16 02:35:29 +03:00
unsigned long flags ;
2019-11-28 03:36:45 +03:00
struct rcu_data * rdp ;
2016-04-16 02:35:29 +03:00
raw_spin_lock_irqsave_rcu_node ( rnp , flags ) ;
if ( ! ( rnp - > expmask & mask ) ) {
raw_spin_unlock_irqrestore_rcu_node ( rnp , flags ) ;
return ;
}
2019-10-08 04:53:18 +03:00
WRITE_ONCE ( rnp - > expmask , rnp - > expmask & ~ mask ) ;
2019-11-28 03:36:45 +03:00
for_each_leaf_node_cpu_mask ( rnp , cpu , mask ) {
rdp = per_cpu_ptr ( & rcu_data , cpu ) ;
if ( ! IS_ENABLED ( CONFIG_NO_HZ_FULL ) | | ! rdp - > rcu_forced_tick_exp )
continue ;
rdp - > rcu_forced_tick_exp = false ;
tick_dep_clear_cpu ( cpu , TICK_DEP_BIT_RCU_EXP ) ;
}
2018-07-04 03:22:34 +03:00
__rcu_report_exp_rnp ( rnp , wake , flags ) ; /* Releases rnp->lock. */
2016-04-16 02:35:29 +03:00
}
/*
* Report expedited quiescent state for specified rcu_data ( CPU ) .
*/
2018-07-04 03:22:34 +03:00
static void rcu_report_exp_rdp ( struct rcu_data * rdp )
2016-04-16 02:35:29 +03:00
{
2019-03-28 01:51:25 +03:00
WRITE_ONCE ( rdp - > exp_deferred_qs , false ) ;
2018-07-04 03:22:34 +03:00
rcu_report_exp_cpu_mult ( rdp - > mynode , rdp - > grpmask , true ) ;
2016-04-16 02:35:29 +03:00
}
2018-07-03 00:30:37 +03:00
/* Common code for work-done checking. */
2018-07-04 03:22:34 +03:00
static bool sync_exp_work_done ( unsigned long s )
2016-04-16 02:35:29 +03:00
{
2018-07-04 03:22:34 +03:00
if ( rcu_exp_gp_seq_done ( s ) ) {
trace_rcu_exp_grace_period ( rcu_state . name , s , TPS ( " done " ) ) ;
2019-04-20 11:40:54 +03:00
smp_mb ( ) ; /* Ensure test happens before caller kfree(). */
2016-04-16 02:35:29 +03:00
return true ;
}
return false ;
}
/*
* Funnel - lock acquisition for expedited grace periods . Returns true
* if some other task completed an expedited grace period that this task
* can piggy - back on , and with no mutex held . Otherwise , returns false
* with the mutex held , indicating that the caller must actually do the
* expedited grace period .
*/
2018-07-04 03:22:34 +03:00
static bool exp_funnel_lock ( unsigned long s )
2016-04-16 02:35:29 +03:00
{
2018-07-04 01:37:16 +03:00
struct rcu_data * rdp = per_cpu_ptr ( & rcu_data , raw_smp_processor_id ( ) ) ;
2016-04-16 02:35:29 +03:00
struct rcu_node * rnp = rdp - > mynode ;
2018-07-04 03:22:34 +03:00
struct rcu_node * rnp_root = rcu_get_root ( ) ;
2016-04-16 02:35:29 +03:00
/* Low-contention fastpath. */
if ( ULONG_CMP_LT ( READ_ONCE ( rnp - > exp_seq_rq ) , s ) & &
( rnp = = rnp_root | |
ULONG_CMP_LT ( READ_ONCE ( rnp_root - > exp_seq_rq ) , s ) ) & &
2018-07-04 03:22:34 +03:00
mutex_trylock ( & rcu_state . exp_mutex ) )
2016-04-16 02:35:29 +03:00
goto fastpath ;
/*
* Each pass through the following loop works its way up
* the rcu_node tree , returning if others have done the work or
2018-07-04 03:22:34 +03:00
* otherwise falls through to acquire - > exp_mutex . The mapping
2016-04-16 02:35:29 +03:00
* from CPU to rcu_node structure can be inexact , as it is just
* promoting locality and is not strictly needed for correctness .
*/
for ( ; rnp ! = NULL ; rnp = rnp - > parent ) {
2018-07-04 03:22:34 +03:00
if ( sync_exp_work_done ( s ) )
2016-04-16 02:35:29 +03:00
return true ;
/* Work not done, either wait here or go up. */
spin_lock ( & rnp - > exp_lock ) ;
if ( ULONG_CMP_GE ( rnp - > exp_seq_rq , s ) ) {
/* Someone else doing GP, so wait for them. */
spin_unlock ( & rnp - > exp_lock ) ;
2018-07-04 03:22:34 +03:00
trace_rcu_exp_funnel_lock ( rcu_state . name , rnp - > level ,
2016-04-16 02:35:29 +03:00
rnp - > grplo , rnp - > grphi ,
TPS ( " wait " ) ) ;
2017-03-21 17:28:14 +03:00
wait_event ( rnp - > exp_wq [ rcu_seq_ctr ( s ) & 0x3 ] ,
2018-07-04 03:22:34 +03:00
sync_exp_work_done ( s ) ) ;
2016-04-16 02:35:29 +03:00
return true ;
}
2020-01-03 23:12:06 +03:00
WRITE_ONCE ( rnp - > exp_seq_rq , s ) ; /* Followers can wait on us. */
2016-04-16 02:35:29 +03:00
spin_unlock ( & rnp - > exp_lock ) ;
2018-07-04 03:22:34 +03:00
trace_rcu_exp_funnel_lock ( rcu_state . name , rnp - > level ,
rnp - > grplo , rnp - > grphi , TPS ( " nxtlvl " ) ) ;
2016-04-16 02:35:29 +03:00
}
2018-07-04 03:22:34 +03:00
mutex_lock ( & rcu_state . exp_mutex ) ;
2016-04-16 02:35:29 +03:00
fastpath :
2018-07-04 03:22:34 +03:00
if ( sync_exp_work_done ( s ) ) {
mutex_unlock ( & rcu_state . exp_mutex ) ;
2016-04-16 02:35:29 +03:00
return true ;
}
2018-07-04 03:22:34 +03:00
rcu_exp_gp_seq_start ( ) ;
trace_rcu_exp_grace_period ( rcu_state . name , s , TPS ( " start " ) ) ;
2016-04-16 02:35:29 +03:00
return false ;
}
/*
2018-02-02 09:05:38 +03:00
* Select the CPUs within the specified rcu_node that the upcoming
* expedited grace period needs to wait for .
2016-04-16 02:35:29 +03:00
*/
2018-02-02 09:05:38 +03:00
static void sync_rcu_exp_select_node_cpus ( struct work_struct * wp )
2016-04-16 02:35:29 +03:00
{
int cpu ;
unsigned long flags ;
unsigned long mask_ofl_test ;
unsigned long mask_ofl_ipi ;
int ret ;
2018-02-02 09:05:38 +03:00
struct rcu_exp_work * rewp =
container_of ( wp , struct rcu_exp_work , rew_work ) ;
struct rcu_node * rnp = container_of ( rewp , struct rcu_node , rew ) ;
2016-04-16 02:35:29 +03:00
2018-02-02 09:05:38 +03:00
raw_spin_lock_irqsave_rcu_node ( rnp , flags ) ;
2016-04-16 02:35:29 +03:00
2018-02-02 09:05:38 +03:00
/* Each pass checks a CPU for identity, offline, and idle. */
mask_ofl_test = 0 ;
for_each_leaf_node_cpu_mask ( rnp , cpu , rnp - > expmask ) {
2018-07-04 01:37:16 +03:00
struct rcu_data * rdp = per_cpu_ptr ( & rcu_data , cpu ) ;
2019-10-30 16:51:57 +03:00
unsigned long mask = rdp - > grpmask ;
2018-02-02 09:05:38 +03:00
int snap ;
if ( raw_smp_processor_id ( ) = = cpu | |
! ( rnp - > qsmaskinitnext & mask ) ) {
mask_ofl_test | = mask ;
} else {
2018-08-04 07:00:38 +03:00
snap = rcu_dynticks_snap ( rdp ) ;
2018-02-02 09:05:38 +03:00
if ( rcu_dynticks_in_eqs ( snap ) )
2018-02-01 07:24:15 +03:00
mask_ofl_test | = mask ;
2018-02-02 09:05:38 +03:00
else
rdp - > exp_dynticks_snap = snap ;
2016-04-16 02:35:29 +03:00
}
2018-02-02 09:05:38 +03:00
}
mask_ofl_ipi = rnp - > expmask & ~ mask_ofl_test ;
2016-04-16 02:35:29 +03:00
2018-02-02 09:05:38 +03:00
/*
* Need to wait for any blocked tasks as well . Note that
* additional blocking tasks will also block the expedited GP
* until such time as the - > expmask bits are cleared .
*/
if ( rcu_preempt_has_tasks ( rnp ) )
2020-01-04 01:18:12 +03:00
WRITE_ONCE ( rnp - > exp_tasks , rnp - > blkd_tasks . next ) ;
2018-02-02 09:05:38 +03:00
raw_spin_unlock_irqrestore_rcu_node ( rnp , flags ) ;
/* IPI the remaining CPUs for expedited quiescent state. */
2019-10-08 04:53:18 +03:00
for_each_leaf_node_cpu_mask ( rnp , cpu , mask_ofl_ipi ) {
2018-07-04 01:37:16 +03:00
struct rcu_data * rdp = per_cpu_ptr ( & rcu_data , cpu ) ;
2019-10-30 16:51:57 +03:00
unsigned long mask = rdp - > grpmask ;
2016-10-11 16:09:59 +03:00
2016-04-16 02:35:29 +03:00
retry_ipi :
2018-08-04 07:00:38 +03:00
if ( rcu_dynticks_in_eqs_since ( rdp , rdp - > exp_dynticks_snap ) ) {
2018-02-02 09:05:38 +03:00
mask_ofl_test | = mask ;
continue ;
}
2019-03-27 19:09:47 +03:00
if ( get_cpu ( ) = = cpu ) {
put_cpu ( ) ;
continue ;
}
2018-11-29 20:15:54 +03:00
ret = smp_call_function_single ( cpu , rcu_exp_handler , NULL , 0 ) ;
2019-03-27 19:09:47 +03:00
put_cpu ( ) ;
2019-10-08 08:01:40 +03:00
/* The CPU will report the QS in response to the IPI. */
if ( ! ret )
2018-02-02 09:05:38 +03:00
continue ;
2019-10-08 08:01:40 +03:00
2018-02-02 09:05:38 +03:00
/* Failed, raced with CPU hotplug operation. */
raw_spin_lock_irqsave_rcu_node ( rnp , flags ) ;
if ( ( rnp - > qsmaskinitnext & mask ) & &
( rnp - > expmask & mask ) ) {
/* Online, so delay for a bit and try again. */
2016-04-16 02:35:29 +03:00
raw_spin_unlock_irqrestore_rcu_node ( rnp , flags ) ;
2018-07-04 03:22:34 +03:00
trace_rcu_exp_grace_period ( rcu_state . name , rcu_exp_gp_seq_endval ( ) , TPS ( " selectofl " ) ) ;
2020-05-08 02:38:29 +03:00
schedule_timeout_idle ( 1 ) ;
2018-02-02 09:05:38 +03:00
goto retry_ipi ;
}
2019-10-08 08:01:40 +03:00
/* CPU really is offline, so we must report its QS. */
if ( rnp - > expmask & mask )
mask_ofl_test | = mask ;
2018-02-02 09:05:38 +03:00
raw_spin_unlock_irqrestore_rcu_node ( rnp , flags ) ;
}
/* Report quiescent states for those that went offline. */
if ( mask_ofl_test )
2018-07-04 03:22:34 +03:00
rcu_report_exp_cpu_mult ( rnp , mask_ofl_test , false ) ;
2018-02-02 09:05:38 +03:00
}
/*
* Select the nodes that the upcoming expedited grace period needs
* to wait for .
*/
2018-11-29 20:15:54 +03:00
static void sync_rcu_exp_select_cpus ( void )
2018-02-02 09:05:38 +03:00
{
2018-06-15 22:06:31 +03:00
int cpu ;
2018-02-02 09:05:38 +03:00
struct rcu_node * rnp ;
2018-07-04 03:22:34 +03:00
trace_rcu_exp_grace_period ( rcu_state . name , rcu_exp_gp_seq_endval ( ) , TPS ( " reset " ) ) ;
sync_exp_reset_tree ( ) ;
trace_rcu_exp_grace_period ( rcu_state . name , rcu_exp_gp_seq_endval ( ) , TPS ( " select " ) ) ;
2018-02-02 09:05:38 +03:00
/* Schedule work for each leaf rcu_node structure. */
2018-07-05 00:33:59 +03:00
rcu_for_each_leaf_node ( rnp ) {
2018-02-02 09:05:38 +03:00
rnp - > exp_need_flush = false ;
if ( ! READ_ONCE ( rnp - > expmask ) )
continue ; /* Avoid early boot non-existent wq. */
if ( ! READ_ONCE ( rcu_par_gp_wq ) | |
2018-04-24 21:03:39 +03:00
rcu_scheduler_active ! = RCU_SCHEDULER_RUNNING | |
2018-07-05 00:33:59 +03:00
rcu_is_last_leaf_node ( rnp ) ) {
2018-04-24 21:03:39 +03:00
/* No workqueues yet or last leaf, do direct call. */
2018-02-02 09:05:38 +03:00
sync_rcu_exp_select_node_cpus ( & rnp - > rew . rew_work ) ;
continue ;
2016-04-16 02:35:29 +03:00
}
2018-02-02 09:05:38 +03:00
INIT_WORK ( & rnp - > rew . rew_work , sync_rcu_exp_select_node_cpus ) ;
2018-09-11 18:57:48 +03:00
cpu = find_next_bit ( & rnp - > ffmask , BITS_PER_LONG , - 1 ) ;
2018-06-15 22:06:31 +03:00
/* If all offline, queue the work on an unbound CPU. */
2018-09-11 18:57:48 +03:00
if ( unlikely ( cpu > rnp - > grphi - rnp - > grplo ) )
2018-06-15 22:06:31 +03:00
cpu = WORK_CPU_UNBOUND ;
2018-09-11 18:57:48 +03:00
else
cpu + = rnp - > grplo ;
2018-06-15 22:06:31 +03:00
queue_work_on ( cpu , rcu_par_gp_wq , & rnp - > rew . rew_work ) ;
2018-02-02 09:05:38 +03:00
rnp - > exp_need_flush = true ;
2016-04-16 02:35:29 +03:00
}
2018-02-02 09:05:38 +03:00
/* Wait for workqueue jobs (if any) to complete. */
2018-07-05 00:33:59 +03:00
rcu_for_each_leaf_node ( rnp )
2018-02-02 09:05:38 +03:00
if ( rnp - > exp_need_flush )
flush_work ( & rnp - > rew . rew_work ) ;
2016-04-16 02:35:29 +03:00
}
2019-11-28 03:36:45 +03:00
/*
* Wait for the expedited grace period to elapse , within time limit .
* If the time limit is exceeded without the grace period elapsing ,
* return false , otherwise return true .
*/
static bool synchronize_rcu_expedited_wait_once ( long tlimit )
{
int t ;
struct rcu_node * rnp_root = rcu_get_root ( ) ;
t = swait_event_timeout_exclusive ( rcu_state . expedited_wq ,
sync_rcu_exp_done_unlocked ( rnp_root ) ,
tlimit ) ;
// Workqueues should not be signaled.
if ( t > 0 | | sync_rcu_exp_done_unlocked ( rnp_root ) )
return true ;
WARN_ON ( t < 0 ) ; /* workqueues should not be signaled. */
return false ;
}
2019-11-28 01:20:41 +03:00
/*
* Wait for the expedited grace period to elapse , issuing any needed
* RCU CPU stall warnings along the way .
*/
2019-11-28 01:24:58 +03:00
static void synchronize_rcu_expedited_wait ( void )
2016-04-16 02:35:29 +03:00
{
int cpu ;
2019-11-29 05:54:06 +03:00
unsigned long j ;
2016-04-16 02:35:29 +03:00
unsigned long jiffies_stall ;
unsigned long jiffies_start ;
unsigned long mask ;
int ndetected ;
2019-11-28 03:36:45 +03:00
struct rcu_data * rdp ;
2016-04-16 02:35:29 +03:00
struct rcu_node * rnp ;
2018-07-04 03:22:34 +03:00
struct rcu_node * rnp_root = rcu_get_root ( ) ;
2016-04-16 02:35:29 +03:00
2018-07-04 03:22:34 +03:00
trace_rcu_exp_grace_period ( rcu_state . name , rcu_exp_gp_seq_endval ( ) , TPS ( " startwait " ) ) ;
2016-04-16 02:35:29 +03:00
jiffies_stall = rcu_jiffies_till_stall_check ( ) ;
jiffies_start = jiffies ;
2019-11-29 05:54:06 +03:00
if ( tick_nohz_full_enabled ( ) & & rcu_inkernel_boot_has_ended ( ) ) {
2019-11-28 03:36:45 +03:00
if ( synchronize_rcu_expedited_wait_once ( 1 ) )
return ;
rcu_for_each_leaf_node ( rnp ) {
for_each_leaf_node_cpu_mask ( rnp , cpu , rnp - > expmask ) {
rdp = per_cpu_ptr ( & rcu_data , cpu ) ;
if ( rdp - > rcu_forced_tick_exp )
continue ;
rdp - > rcu_forced_tick_exp = true ;
tick_dep_set_cpu ( cpu , TICK_DEP_BIT_RCU_EXP ) ;
}
}
2019-11-29 05:54:06 +03:00
j = READ_ONCE ( jiffies_till_first_fqs ) ;
if ( synchronize_rcu_expedited_wait_once ( j + HZ ) )
return ;
WARN_ON_ONCE ( IS_ENABLED ( CONFIG_PREEMPT_RT ) ) ;
2019-11-28 03:36:45 +03:00
}
2016-04-16 02:35:29 +03:00
for ( ; ; ) {
2019-11-28 03:36:45 +03:00
if ( synchronize_rcu_expedited_wait_once ( jiffies_stall ) )
2016-04-16 02:35:29 +03:00
return ;
2019-12-05 22:29:01 +03:00
if ( rcu_stall_is_suppressed ( ) )
2016-06-30 00:49:29 +03:00
continue ;
panic_on_rcu_stall ( ) ;
2021-03-02 14:55:15 +03:00
trace_rcu_stall_warning ( rcu_state . name , TPS ( " ExpeditedStall " ) ) ;
2016-04-16 02:35:29 +03:00
pr_err ( " INFO: %s detected expedited stalls on CPUs/tasks: { " ,
2018-07-04 03:22:34 +03:00
rcu_state . name ) ;
2016-04-16 02:35:29 +03:00
ndetected = 0 ;
2018-07-05 00:33:59 +03:00
rcu_for_each_leaf_node ( rnp ) {
2016-04-16 02:35:29 +03:00
ndetected + = rcu_print_task_exp_stall ( rnp ) ;
rcu: Correctly handle sparse possible cpus
In many cases in the RCU tree code, we iterate over the set of cpus for
a leaf node described by rcu_node::grplo and rcu_node::grphi, checking
per-cpu data for each cpu in this range. However, if the set of possible
cpus is sparse, some cpus described in this range are not possible, and
thus no per-cpu region will have been allocated (or initialised) for
them by the generic percpu code.
Erroneous accesses to a per-cpu area for these !possible cpus may fault
or may hit other data depending on the addressed generated when the
erroneous per cpu offset is applied. In practice, both cases have been
observed on arm64 hardware (the former being silent, but detectable with
additional patches).
To avoid issues resulting from this, we must iterate over the set of
*possible* cpus for a given leaf node. This patch add a new helper,
for_each_leaf_node_possible_cpu, to enable this. As iteration is often
intertwined with rcu_node local bitmask manipulation, a new
leaf_node_cpu_bit helper is added to make this simpler and more
consistent. The RCU tree code is made to use both of these where
appropriate.
Without this patch, running reboot at a shell can result in an oops
like:
[ 3369.075979] Unable to handle kernel paging request at virtual address ffffff8008b21b4c
[ 3369.083881] pgd = ffffffc3ecdda000
[ 3369.087270] [ffffff8008b21b4c] *pgd=00000083eca48003, *pud=00000083eca48003, *pmd=0000000000000000
[ 3369.096222] Internal error: Oops: 96000007 [#1] PREEMPT SMP
[ 3369.101781] Modules linked in:
[ 3369.104825] CPU: 2 PID: 1817 Comm: NetworkManager Tainted: G W 4.6.0+ #3
[ 3369.121239] task: ffffffc0fa13e000 ti: ffffffc3eb940000 task.ti: ffffffc3eb940000
[ 3369.128708] PC is at sync_rcu_exp_select_cpus+0x188/0x510
[ 3369.134094] LR is at sync_rcu_exp_select_cpus+0x104/0x510
[ 3369.139479] pc : [<ffffff80081109a8>] lr : [<ffffff8008110924>] pstate: 200001c5
[ 3369.146860] sp : ffffffc3eb9435a0
[ 3369.150162] x29: ffffffc3eb9435a0 x28: ffffff8008be4f88
[ 3369.155465] x27: ffffff8008b66c80 x26: ffffffc3eceb2600
[ 3369.160767] x25: 0000000000000001 x24: ffffff8008be4f88
[ 3369.166070] x23: ffffff8008b51c3c x22: ffffff8008b66c80
[ 3369.171371] x21: 0000000000000001 x20: ffffff8008b21b40
[ 3369.176673] x19: ffffff8008b66c80 x18: 0000000000000000
[ 3369.181975] x17: 0000007fa951a010 x16: ffffff80086a30f0
[ 3369.187278] x15: 0000007fa9505590 x14: 0000000000000000
[ 3369.192580] x13: ffffff8008b51000 x12: ffffffc3eb940000
[ 3369.197882] x11: 0000000000000006 x10: ffffff8008b51b78
[ 3369.203184] x9 : 0000000000000001 x8 : ffffff8008be4000
[ 3369.208486] x7 : ffffff8008b21b40 x6 : 0000000000001003
[ 3369.213788] x5 : 0000000000000000 x4 : ffffff8008b27280
[ 3369.219090] x3 : ffffff8008b21b4c x2 : 0000000000000001
[ 3369.224406] x1 : 0000000000000001 x0 : 0000000000000140
...
[ 3369.972257] [<ffffff80081109a8>] sync_rcu_exp_select_cpus+0x188/0x510
[ 3369.978685] [<ffffff80081128b4>] synchronize_rcu_expedited+0x64/0xa8
[ 3369.985026] [<ffffff80086b987c>] synchronize_net+0x24/0x30
[ 3369.990499] [<ffffff80086ddb54>] dev_deactivate_many+0x28c/0x298
[ 3369.996493] [<ffffff80086b6bb8>] __dev_close_many+0x60/0xd0
[ 3370.002052] [<ffffff80086b6d48>] __dev_close+0x28/0x40
[ 3370.007178] [<ffffff80086bf62c>] __dev_change_flags+0x8c/0x158
[ 3370.012999] [<ffffff80086bf718>] dev_change_flags+0x20/0x60
[ 3370.018558] [<ffffff80086cf7f0>] do_setlink+0x288/0x918
[ 3370.023771] [<ffffff80086d0798>] rtnl_newlink+0x398/0x6a8
[ 3370.029158] [<ffffff80086cee84>] rtnetlink_rcv_msg+0xe4/0x220
[ 3370.034891] [<ffffff80086e274c>] netlink_rcv_skb+0xc4/0xf8
[ 3370.040364] [<ffffff80086ced8c>] rtnetlink_rcv+0x2c/0x40
[ 3370.045663] [<ffffff80086e1fe8>] netlink_unicast+0x160/0x238
[ 3370.051309] [<ffffff80086e24b8>] netlink_sendmsg+0x2f0/0x358
[ 3370.056956] [<ffffff80086a0070>] sock_sendmsg+0x18/0x30
[ 3370.062168] [<ffffff80086a21cc>] ___sys_sendmsg+0x26c/0x280
[ 3370.067728] [<ffffff80086a30ac>] __sys_sendmsg+0x44/0x88
[ 3370.073027] [<ffffff80086a3100>] SyS_sendmsg+0x10/0x20
[ 3370.078153] [<ffffff8008085e70>] el0_svc_naked+0x24/0x28
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Reported-by: Dennis Chen <dennis.chen@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Josh Triplett <josh@joshtriplett.org>
Cc: Lai Jiangshan <jiangshanlai@gmail.com>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: Steve Capper <steve.capper@arm.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Will Deacon <will.deacon@arm.com>
Cc: linux-kernel@vger.kernel.org
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
2016-06-03 17:20:04 +03:00
for_each_leaf_node_possible_cpu ( rnp , cpu ) {
2016-04-16 02:35:29 +03:00
struct rcu_data * rdp ;
rcu: Correctly handle sparse possible cpus
In many cases in the RCU tree code, we iterate over the set of cpus for
a leaf node described by rcu_node::grplo and rcu_node::grphi, checking
per-cpu data for each cpu in this range. However, if the set of possible
cpus is sparse, some cpus described in this range are not possible, and
thus no per-cpu region will have been allocated (or initialised) for
them by the generic percpu code.
Erroneous accesses to a per-cpu area for these !possible cpus may fault
or may hit other data depending on the addressed generated when the
erroneous per cpu offset is applied. In practice, both cases have been
observed on arm64 hardware (the former being silent, but detectable with
additional patches).
To avoid issues resulting from this, we must iterate over the set of
*possible* cpus for a given leaf node. This patch add a new helper,
for_each_leaf_node_possible_cpu, to enable this. As iteration is often
intertwined with rcu_node local bitmask manipulation, a new
leaf_node_cpu_bit helper is added to make this simpler and more
consistent. The RCU tree code is made to use both of these where
appropriate.
Without this patch, running reboot at a shell can result in an oops
like:
[ 3369.075979] Unable to handle kernel paging request at virtual address ffffff8008b21b4c
[ 3369.083881] pgd = ffffffc3ecdda000
[ 3369.087270] [ffffff8008b21b4c] *pgd=00000083eca48003, *pud=00000083eca48003, *pmd=0000000000000000
[ 3369.096222] Internal error: Oops: 96000007 [#1] PREEMPT SMP
[ 3369.101781] Modules linked in:
[ 3369.104825] CPU: 2 PID: 1817 Comm: NetworkManager Tainted: G W 4.6.0+ #3
[ 3369.121239] task: ffffffc0fa13e000 ti: ffffffc3eb940000 task.ti: ffffffc3eb940000
[ 3369.128708] PC is at sync_rcu_exp_select_cpus+0x188/0x510
[ 3369.134094] LR is at sync_rcu_exp_select_cpus+0x104/0x510
[ 3369.139479] pc : [<ffffff80081109a8>] lr : [<ffffff8008110924>] pstate: 200001c5
[ 3369.146860] sp : ffffffc3eb9435a0
[ 3369.150162] x29: ffffffc3eb9435a0 x28: ffffff8008be4f88
[ 3369.155465] x27: ffffff8008b66c80 x26: ffffffc3eceb2600
[ 3369.160767] x25: 0000000000000001 x24: ffffff8008be4f88
[ 3369.166070] x23: ffffff8008b51c3c x22: ffffff8008b66c80
[ 3369.171371] x21: 0000000000000001 x20: ffffff8008b21b40
[ 3369.176673] x19: ffffff8008b66c80 x18: 0000000000000000
[ 3369.181975] x17: 0000007fa951a010 x16: ffffff80086a30f0
[ 3369.187278] x15: 0000007fa9505590 x14: 0000000000000000
[ 3369.192580] x13: ffffff8008b51000 x12: ffffffc3eb940000
[ 3369.197882] x11: 0000000000000006 x10: ffffff8008b51b78
[ 3369.203184] x9 : 0000000000000001 x8 : ffffff8008be4000
[ 3369.208486] x7 : ffffff8008b21b40 x6 : 0000000000001003
[ 3369.213788] x5 : 0000000000000000 x4 : ffffff8008b27280
[ 3369.219090] x3 : ffffff8008b21b4c x2 : 0000000000000001
[ 3369.224406] x1 : 0000000000000001 x0 : 0000000000000140
...
[ 3369.972257] [<ffffff80081109a8>] sync_rcu_exp_select_cpus+0x188/0x510
[ 3369.978685] [<ffffff80081128b4>] synchronize_rcu_expedited+0x64/0xa8
[ 3369.985026] [<ffffff80086b987c>] synchronize_net+0x24/0x30
[ 3369.990499] [<ffffff80086ddb54>] dev_deactivate_many+0x28c/0x298
[ 3369.996493] [<ffffff80086b6bb8>] __dev_close_many+0x60/0xd0
[ 3370.002052] [<ffffff80086b6d48>] __dev_close+0x28/0x40
[ 3370.007178] [<ffffff80086bf62c>] __dev_change_flags+0x8c/0x158
[ 3370.012999] [<ffffff80086bf718>] dev_change_flags+0x20/0x60
[ 3370.018558] [<ffffff80086cf7f0>] do_setlink+0x288/0x918
[ 3370.023771] [<ffffff80086d0798>] rtnl_newlink+0x398/0x6a8
[ 3370.029158] [<ffffff80086cee84>] rtnetlink_rcv_msg+0xe4/0x220
[ 3370.034891] [<ffffff80086e274c>] netlink_rcv_skb+0xc4/0xf8
[ 3370.040364] [<ffffff80086ced8c>] rtnetlink_rcv+0x2c/0x40
[ 3370.045663] [<ffffff80086e1fe8>] netlink_unicast+0x160/0x238
[ 3370.051309] [<ffffff80086e24b8>] netlink_sendmsg+0x2f0/0x358
[ 3370.056956] [<ffffff80086a0070>] sock_sendmsg+0x18/0x30
[ 3370.062168] [<ffffff80086a21cc>] ___sys_sendmsg+0x26c/0x280
[ 3370.067728] [<ffffff80086a30ac>] __sys_sendmsg+0x44/0x88
[ 3370.073027] [<ffffff80086a3100>] SyS_sendmsg+0x10/0x20
[ 3370.078153] [<ffffff8008085e70>] el0_svc_naked+0x24/0x28
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Reported-by: Dennis Chen <dennis.chen@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Josh Triplett <josh@joshtriplett.org>
Cc: Lai Jiangshan <jiangshanlai@gmail.com>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: Steve Capper <steve.capper@arm.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Will Deacon <will.deacon@arm.com>
Cc: linux-kernel@vger.kernel.org
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
2016-06-03 17:20:04 +03:00
mask = leaf_node_cpu_bit ( rnp , cpu ) ;
2019-10-08 04:53:18 +03:00
if ( ! ( READ_ONCE ( rnp - > expmask ) & mask ) )
2016-04-16 02:35:29 +03:00
continue ;
ndetected + + ;
2018-07-04 01:37:16 +03:00
rdp = per_cpu_ptr ( & rcu_data , cpu ) ;
2016-04-16 02:35:29 +03:00
pr_cont ( " %d-%c%c%c " , cpu ,
" O. " [ ! ! cpu_online ( cpu ) ] ,
" o. " [ ! ! ( rdp - > grpmask & rnp - > expmaskinit ) ] ,
" N. " [ ! ! ( rdp - > grpmask & rnp - > expmaskinitnext ) ] ) ;
}
}
pr_cont ( " } %lu jiffies s: %lu root: %#lx/%c \n " ,
2018-07-04 03:22:34 +03:00
jiffies - jiffies_start , rcu_state . expedited_sequence ,
2020-01-04 01:18:12 +03:00
data_race ( rnp_root - > expmask ) ,
" .T " [ ! ! data_race ( rnp_root - > exp_tasks ) ] ) ;
2016-04-16 02:35:29 +03:00
if ( ndetected ) {
2020-11-07 00:52:31 +03:00
pr_err ( " blocking rcu_node structures (internal RCU debug): " ) ;
2018-07-05 00:33:59 +03:00
rcu_for_each_node_breadth_first ( rnp ) {
2016-04-16 02:35:29 +03:00
if ( rnp = = rnp_root )
continue ; /* printed unconditionally */
2019-11-28 00:59:37 +03:00
if ( sync_rcu_exp_done_unlocked ( rnp ) )
2016-04-16 02:35:29 +03:00
continue ;
pr_cont ( " l=%u:%d-%d:%#lx/%c " ,
rnp - > level , rnp - > grplo , rnp - > grphi ,
2020-01-04 01:18:12 +03:00
data_race ( rnp - > expmask ) ,
" .T " [ ! ! data_race ( rnp - > exp_tasks ) ] ) ;
2016-04-16 02:35:29 +03:00
}
pr_cont ( " \n " ) ;
}
2018-07-05 00:33:59 +03:00
rcu_for_each_leaf_node ( rnp ) {
rcu: Correctly handle sparse possible cpus
In many cases in the RCU tree code, we iterate over the set of cpus for
a leaf node described by rcu_node::grplo and rcu_node::grphi, checking
per-cpu data for each cpu in this range. However, if the set of possible
cpus is sparse, some cpus described in this range are not possible, and
thus no per-cpu region will have been allocated (or initialised) for
them by the generic percpu code.
Erroneous accesses to a per-cpu area for these !possible cpus may fault
or may hit other data depending on the addressed generated when the
erroneous per cpu offset is applied. In practice, both cases have been
observed on arm64 hardware (the former being silent, but detectable with
additional patches).
To avoid issues resulting from this, we must iterate over the set of
*possible* cpus for a given leaf node. This patch add a new helper,
for_each_leaf_node_possible_cpu, to enable this. As iteration is often
intertwined with rcu_node local bitmask manipulation, a new
leaf_node_cpu_bit helper is added to make this simpler and more
consistent. The RCU tree code is made to use both of these where
appropriate.
Without this patch, running reboot at a shell can result in an oops
like:
[ 3369.075979] Unable to handle kernel paging request at virtual address ffffff8008b21b4c
[ 3369.083881] pgd = ffffffc3ecdda000
[ 3369.087270] [ffffff8008b21b4c] *pgd=00000083eca48003, *pud=00000083eca48003, *pmd=0000000000000000
[ 3369.096222] Internal error: Oops: 96000007 [#1] PREEMPT SMP
[ 3369.101781] Modules linked in:
[ 3369.104825] CPU: 2 PID: 1817 Comm: NetworkManager Tainted: G W 4.6.0+ #3
[ 3369.121239] task: ffffffc0fa13e000 ti: ffffffc3eb940000 task.ti: ffffffc3eb940000
[ 3369.128708] PC is at sync_rcu_exp_select_cpus+0x188/0x510
[ 3369.134094] LR is at sync_rcu_exp_select_cpus+0x104/0x510
[ 3369.139479] pc : [<ffffff80081109a8>] lr : [<ffffff8008110924>] pstate: 200001c5
[ 3369.146860] sp : ffffffc3eb9435a0
[ 3369.150162] x29: ffffffc3eb9435a0 x28: ffffff8008be4f88
[ 3369.155465] x27: ffffff8008b66c80 x26: ffffffc3eceb2600
[ 3369.160767] x25: 0000000000000001 x24: ffffff8008be4f88
[ 3369.166070] x23: ffffff8008b51c3c x22: ffffff8008b66c80
[ 3369.171371] x21: 0000000000000001 x20: ffffff8008b21b40
[ 3369.176673] x19: ffffff8008b66c80 x18: 0000000000000000
[ 3369.181975] x17: 0000007fa951a010 x16: ffffff80086a30f0
[ 3369.187278] x15: 0000007fa9505590 x14: 0000000000000000
[ 3369.192580] x13: ffffff8008b51000 x12: ffffffc3eb940000
[ 3369.197882] x11: 0000000000000006 x10: ffffff8008b51b78
[ 3369.203184] x9 : 0000000000000001 x8 : ffffff8008be4000
[ 3369.208486] x7 : ffffff8008b21b40 x6 : 0000000000001003
[ 3369.213788] x5 : 0000000000000000 x4 : ffffff8008b27280
[ 3369.219090] x3 : ffffff8008b21b4c x2 : 0000000000000001
[ 3369.224406] x1 : 0000000000000001 x0 : 0000000000000140
...
[ 3369.972257] [<ffffff80081109a8>] sync_rcu_exp_select_cpus+0x188/0x510
[ 3369.978685] [<ffffff80081128b4>] synchronize_rcu_expedited+0x64/0xa8
[ 3369.985026] [<ffffff80086b987c>] synchronize_net+0x24/0x30
[ 3369.990499] [<ffffff80086ddb54>] dev_deactivate_many+0x28c/0x298
[ 3369.996493] [<ffffff80086b6bb8>] __dev_close_many+0x60/0xd0
[ 3370.002052] [<ffffff80086b6d48>] __dev_close+0x28/0x40
[ 3370.007178] [<ffffff80086bf62c>] __dev_change_flags+0x8c/0x158
[ 3370.012999] [<ffffff80086bf718>] dev_change_flags+0x20/0x60
[ 3370.018558] [<ffffff80086cf7f0>] do_setlink+0x288/0x918
[ 3370.023771] [<ffffff80086d0798>] rtnl_newlink+0x398/0x6a8
[ 3370.029158] [<ffffff80086cee84>] rtnetlink_rcv_msg+0xe4/0x220
[ 3370.034891] [<ffffff80086e274c>] netlink_rcv_skb+0xc4/0xf8
[ 3370.040364] [<ffffff80086ced8c>] rtnetlink_rcv+0x2c/0x40
[ 3370.045663] [<ffffff80086e1fe8>] netlink_unicast+0x160/0x238
[ 3370.051309] [<ffffff80086e24b8>] netlink_sendmsg+0x2f0/0x358
[ 3370.056956] [<ffffff80086a0070>] sock_sendmsg+0x18/0x30
[ 3370.062168] [<ffffff80086a21cc>] ___sys_sendmsg+0x26c/0x280
[ 3370.067728] [<ffffff80086a30ac>] __sys_sendmsg+0x44/0x88
[ 3370.073027] [<ffffff80086a3100>] SyS_sendmsg+0x10/0x20
[ 3370.078153] [<ffffff8008085e70>] el0_svc_naked+0x24/0x28
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Reported-by: Dennis Chen <dennis.chen@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Josh Triplett <josh@joshtriplett.org>
Cc: Lai Jiangshan <jiangshanlai@gmail.com>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: Steve Capper <steve.capper@arm.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Will Deacon <will.deacon@arm.com>
Cc: linux-kernel@vger.kernel.org
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
2016-06-03 17:20:04 +03:00
for_each_leaf_node_possible_cpu ( rnp , cpu ) {
mask = leaf_node_cpu_bit ( rnp , cpu ) ;
2019-10-08 04:53:18 +03:00
if ( ! ( READ_ONCE ( rnp - > expmask ) & mask ) )
2016-04-16 02:35:29 +03:00
continue ;
dump_cpu_task ( cpu ) ;
}
}
jiffies_stall = 3 * rcu_jiffies_till_stall_check ( ) + 3 ;
}
}
/*
* Wait for the current expedited grace period to complete , and then
* wake up everyone who piggybacked on the just - completed expedited
* grace period . Also update all the - > exp_seq_rq counters as needed
* in order to avoid counter - wrap problems .
*/
2018-07-04 03:22:34 +03:00
static void rcu_exp_wait_wake ( unsigned long s )
2016-04-16 02:35:29 +03:00
{
struct rcu_node * rnp ;
2019-11-28 01:24:58 +03:00
synchronize_rcu_expedited_wait ( ) ;
2016-04-16 02:35:29 +03:00
rcu: Allow only one expedited GP to run concurrently with wakeups
The current expedited RCU grace-period code expects that a task
requesting an expedited grace period cannot awaken until that grace
period has reached the wakeup phase. However, it is possible for a long
preemption to result in the waiting task never sleeping. For example,
consider the following sequence of events:
1. Task A starts an expedited grace period by invoking
synchronize_rcu_expedited(). It proceeds normally up to the
wait_event() near the end of that function, and is then preempted
(or interrupted or whatever).
2. The expedited grace period completes, and a kworker task starts
the awaken phase, having incremented the counter and acquired
the rcu_state structure's .exp_wake_mutex. This kworker task
is then preempted or interrupted or whatever.
3. Task A resumes and enters wait_event(), which notes that the
expedited grace period has completed, and thus doesn't sleep.
4. Task B starts an expedited grace period exactly as did Task A,
complete with the preemption (or whatever delay) just before
the call to wait_event().
5. The expedited grace period completes, and another kworker
task starts the awaken phase, having incremented the counter.
However, it blocks when attempting to acquire the rcu_state
structure's .exp_wake_mutex because step 2's kworker task has
not yet released it.
6. Steps 4 and 5 repeat, resulting in overflow of the rcu_node
structure's ->exp_wq[] array.
In theory, this is harmless. Tasks waiting on the various ->exp_wq[]
array will just be spuriously awakened, but they will just sleep again
on noting that the rcu_state structure's ->expedited_sequence value has
not advanced far enough.
In practice, this wastes CPU time and is an accident waiting to happen.
This commit therefore moves the rcu_exp_gp_seq_end() call that officially
ends the expedited grace period (along with associate tracing) until
after the ->exp_wake_mutex has been acquired. This prevents Task A from
awakening prematurely, thus preventing more than one expedited grace
period from being in flight during a previous expedited grace period's
wakeup phase.
Fixes: 3b5f668e715b ("rcu: Overlap wakeups with next expedited grace period")
Signed-off-by: Neeraj Upadhyay <neeraju@codeaurora.org>
[ paulmck: Added updated comment. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2019-11-19 22:50:52 +03:00
// Switch over to wakeup mode, allowing the next GP to proceed.
// End the previous grace period only after acquiring the mutex
// to ensure that only one GP runs concurrently with wakeups.
2018-07-04 03:22:34 +03:00
mutex_lock ( & rcu_state . exp_wake_mutex ) ;
rcu: Allow only one expedited GP to run concurrently with wakeups
The current expedited RCU grace-period code expects that a task
requesting an expedited grace period cannot awaken until that grace
period has reached the wakeup phase. However, it is possible for a long
preemption to result in the waiting task never sleeping. For example,
consider the following sequence of events:
1. Task A starts an expedited grace period by invoking
synchronize_rcu_expedited(). It proceeds normally up to the
wait_event() near the end of that function, and is then preempted
(or interrupted or whatever).
2. The expedited grace period completes, and a kworker task starts
the awaken phase, having incremented the counter and acquired
the rcu_state structure's .exp_wake_mutex. This kworker task
is then preempted or interrupted or whatever.
3. Task A resumes and enters wait_event(), which notes that the
expedited grace period has completed, and thus doesn't sleep.
4. Task B starts an expedited grace period exactly as did Task A,
complete with the preemption (or whatever delay) just before
the call to wait_event().
5. The expedited grace period completes, and another kworker
task starts the awaken phase, having incremented the counter.
However, it blocks when attempting to acquire the rcu_state
structure's .exp_wake_mutex because step 2's kworker task has
not yet released it.
6. Steps 4 and 5 repeat, resulting in overflow of the rcu_node
structure's ->exp_wq[] array.
In theory, this is harmless. Tasks waiting on the various ->exp_wq[]
array will just be spuriously awakened, but they will just sleep again
on noting that the rcu_state structure's ->expedited_sequence value has
not advanced far enough.
In practice, this wastes CPU time and is an accident waiting to happen.
This commit therefore moves the rcu_exp_gp_seq_end() call that officially
ends the expedited grace period (along with associate tracing) until
after the ->exp_wake_mutex has been acquired. This prevents Task A from
awakening prematurely, thus preventing more than one expedited grace
period from being in flight during a previous expedited grace period's
wakeup phase.
Fixes: 3b5f668e715b ("rcu: Overlap wakeups with next expedited grace period")
Signed-off-by: Neeraj Upadhyay <neeraju@codeaurora.org>
[ paulmck: Added updated comment. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2019-11-19 22:50:52 +03:00
rcu_exp_gp_seq_end ( ) ;
trace_rcu_exp_grace_period ( rcu_state . name , s , TPS ( " end " ) ) ;
2016-04-16 02:35:29 +03:00
2018-07-05 00:33:59 +03:00
rcu_for_each_node_breadth_first ( rnp ) {
2016-04-16 02:35:29 +03:00
if ( ULONG_CMP_LT ( READ_ONCE ( rnp - > exp_seq_rq ) , s ) ) {
spin_lock ( & rnp - > exp_lock ) ;
/* Recheck, avoid hang in case someone just arrived. */
if ( ULONG_CMP_LT ( rnp - > exp_seq_rq , s ) )
2019-12-23 06:55:50 +03:00
WRITE_ONCE ( rnp - > exp_seq_rq , s ) ;
2016-04-16 02:35:29 +03:00
spin_unlock ( & rnp - > exp_lock ) ;
}
2017-03-04 23:33:53 +03:00
smp_mb ( ) ; /* All above changes before wakeup. */
rcu: Fix missed wakeup of exp_wq waiters
Tasks waiting within exp_funnel_lock() for an expedited grace period to
elapse can be starved due to the following sequence of events:
1. Tasks A and B both attempt to start an expedited grace
period at about the same time. This grace period will have
completed when the lower four bits of the rcu_state structure's
->expedited_sequence field are 0b'0100', for example, when the
initial value of this counter is zero. Task A wins, and thus
does the actual work of starting the grace period, including
acquiring the rcu_state structure's .exp_mutex and sets the
counter to 0b'0001'.
2. Because task B lost the race to start the grace period, it
waits on ->expedited_sequence to reach 0b'0100' inside of
exp_funnel_lock(). This task therefore blocks on the rcu_node
structure's ->exp_wq[1] field, keeping in mind that the
end-of-grace-period value of ->expedited_sequence (0b'0100')
is shifted down two bits before indexing the ->exp_wq[] field.
3. Task C attempts to start another expedited grace period,
but blocks on ->exp_mutex, which is still held by Task A.
4. The aforementioned expedited grace period completes, so that
->expedited_sequence now has the value 0b'0100'. A kworker task
therefore acquires the rcu_state structure's ->exp_wake_mutex
and starts awakening any tasks waiting for this grace period.
5. One of the first tasks awakened happens to be Task A. Task A
therefore releases the rcu_state structure's ->exp_mutex,
which allows Task C to start the next expedited grace period,
which causes the lower four bits of the rcu_state structure's
->expedited_sequence field to become 0b'0101'.
6. Task C's expedited grace period completes, so that the lower four
bits of the rcu_state structure's ->expedited_sequence field now
become 0b'1000'.
7. The kworker task from step 4 above continues its wakeups.
Unfortunately, the wake_up_all() refetches the rcu_state
structure's .expedited_sequence field:
wake_up_all(&rnp->exp_wq[rcu_seq_ctr(rcu_state.expedited_sequence) & 0x3]);
This results in the wakeup being applied to the rcu_node
structure's ->exp_wq[2] field, which is unfortunate given that
Task B is instead waiting on ->exp_wq[1].
On a busy system, no harm is done (or at least no permanent harm is done).
Some later expedited grace period will redo the wakeup. But on a quiet
system, such as many embedded systems, it might be a good long time before
there was another expedited grace period. On such embedded systems,
this situation could therefore result in a system hang.
This issue manifested as DPM device timeout during suspend (which
usually qualifies as a quiet time) due to a SCSI device being stuck in
_synchronize_rcu_expedited(), with the following stack trace:
schedule()
synchronize_rcu_expedited()
synchronize_rcu()
scsi_device_quiesce()
scsi_bus_suspend()
dpm_run_callback()
__device_suspend()
This commit therefore prevents such delays, timeouts, and hangs by
making rcu_exp_wait_wake() use its "s" argument consistently instead of
refetching from rcu_state.expedited_sequence.
Fixes: 3b5f668e715b ("rcu: Overlap wakeups with next expedited grace period")
Signed-off-by: Neeraj Upadhyay <neeraju@codeaurora.org>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2019-11-19 06:17:07 +03:00
wake_up_all ( & rnp - > exp_wq [ rcu_seq_ctr ( s ) & 0x3 ] ) ;
2016-04-16 02:35:29 +03:00
}
2018-07-04 03:22:34 +03:00
trace_rcu_exp_grace_period ( rcu_state . name , s , TPS ( " endwake " ) ) ;
mutex_unlock ( & rcu_state . exp_wake_mutex ) ;
2016-04-16 02:35:29 +03:00
}
rcu: Narrow early boot window of illegal synchronous grace periods
The current preemptible RCU implementation goes through three phases
during bootup. In the first phase, there is only one CPU that is running
with preemption disabled, so that a no-op is a synchronous grace period.
In the second mid-boot phase, the scheduler is running, but RCU has
not yet gotten its kthreads spawned (and, for expedited grace periods,
workqueues are not yet running. During this time, any attempt to do
a synchronous grace period will hang the system (or complain bitterly,
depending). In the third and final phase, RCU is fully operational and
everything works normally.
This has been OK for some time, but there has recently been some
synchronous grace periods showing up during the second mid-boot phase.
This code worked "by accident" for awhile, but started failing as soon
as expedited RCU grace periods switched over to workqueues in commit
8b355e3bc140 ("rcu: Drive expedited grace periods from workqueue").
Note that the code was buggy even before this commit, as it was subject
to failure on real-time systems that forced all expedited grace periods
to run as normal grace periods (for example, using the rcu_normal ksysfs
parameter). The callchain from the failure case is as follows:
early_amd_iommu_init()
|-> acpi_put_table(ivrs_base);
|-> acpi_tb_put_table(table_desc);
|-> acpi_tb_invalidate_table(table_desc);
|-> acpi_tb_release_table(...)
|-> acpi_os_unmap_memory
|-> acpi_os_unmap_iomem
|-> acpi_os_map_cleanup
|-> synchronize_rcu_expedited
The kernel showing this callchain was built with CONFIG_PREEMPT_RCU=y,
which caused the code to try using workqueues before they were
initialized, which did not go well.
This commit therefore reworks RCU to permit synchronous grace periods
to proceed during this mid-boot phase. This commit is therefore a
fix to a regression introduced in v4.9, and is therefore being put
forward post-merge-window in v4.10.
This commit sets a flag from the existing rcu_scheduler_starting()
function which causes all synchronous grace periods to take the expedited
path. The expedited path now checks this flag, using the requesting task
to drive the expedited grace period forward during the mid-boot phase.
Finally, this flag is updated by a core_initcall() function named
rcu_exp_runtime_mode(), which causes the runtime codepaths to be used.
Note that this arrangement assumes that tasks are not sent POSIX signals
(or anything similar) from the time that the first task is spawned
through core_initcall() time.
Fixes: 8b355e3bc140 ("rcu: Drive expedited grace periods from workqueue")
Reported-by: "Zheng, Lv" <lv.zheng@intel.com>
Reported-by: Borislav Petkov <bp@alien8.de>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Tested-by: Stan Kain <stan.kain@gmail.com>
Tested-by: Ivan <waffolz@hotmail.com>
Tested-by: Emanuel Castelo <emanuel.castelo@gmail.com>
Tested-by: Bruno Pesavento <bpesavento@infinito.it>
Tested-by: Borislav Petkov <bp@suse.de>
Tested-by: Frederic Bezies <fredbezies@gmail.com>
Cc: <stable@vger.kernel.org> # 4.9.0-
2017-01-10 13:28:26 +03:00
/*
* Common code to drive an expedited grace period forward , used by
* workqueues and mid - boot - time tasks .
*/
2018-11-29 20:15:54 +03:00
static void rcu_exp_sel_wait_wake ( unsigned long s )
rcu: Narrow early boot window of illegal synchronous grace periods
The current preemptible RCU implementation goes through three phases
during bootup. In the first phase, there is only one CPU that is running
with preemption disabled, so that a no-op is a synchronous grace period.
In the second mid-boot phase, the scheduler is running, but RCU has
not yet gotten its kthreads spawned (and, for expedited grace periods,
workqueues are not yet running. During this time, any attempt to do
a synchronous grace period will hang the system (or complain bitterly,
depending). In the third and final phase, RCU is fully operational and
everything works normally.
This has been OK for some time, but there has recently been some
synchronous grace periods showing up during the second mid-boot phase.
This code worked "by accident" for awhile, but started failing as soon
as expedited RCU grace periods switched over to workqueues in commit
8b355e3bc140 ("rcu: Drive expedited grace periods from workqueue").
Note that the code was buggy even before this commit, as it was subject
to failure on real-time systems that forced all expedited grace periods
to run as normal grace periods (for example, using the rcu_normal ksysfs
parameter). The callchain from the failure case is as follows:
early_amd_iommu_init()
|-> acpi_put_table(ivrs_base);
|-> acpi_tb_put_table(table_desc);
|-> acpi_tb_invalidate_table(table_desc);
|-> acpi_tb_release_table(...)
|-> acpi_os_unmap_memory
|-> acpi_os_unmap_iomem
|-> acpi_os_map_cleanup
|-> synchronize_rcu_expedited
The kernel showing this callchain was built with CONFIG_PREEMPT_RCU=y,
which caused the code to try using workqueues before they were
initialized, which did not go well.
This commit therefore reworks RCU to permit synchronous grace periods
to proceed during this mid-boot phase. This commit is therefore a
fix to a regression introduced in v4.9, and is therefore being put
forward post-merge-window in v4.10.
This commit sets a flag from the existing rcu_scheduler_starting()
function which causes all synchronous grace periods to take the expedited
path. The expedited path now checks this flag, using the requesting task
to drive the expedited grace period forward during the mid-boot phase.
Finally, this flag is updated by a core_initcall() function named
rcu_exp_runtime_mode(), which causes the runtime codepaths to be used.
Note that this arrangement assumes that tasks are not sent POSIX signals
(or anything similar) from the time that the first task is spawned
through core_initcall() time.
Fixes: 8b355e3bc140 ("rcu: Drive expedited grace periods from workqueue")
Reported-by: "Zheng, Lv" <lv.zheng@intel.com>
Reported-by: Borislav Petkov <bp@alien8.de>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Tested-by: Stan Kain <stan.kain@gmail.com>
Tested-by: Ivan <waffolz@hotmail.com>
Tested-by: Emanuel Castelo <emanuel.castelo@gmail.com>
Tested-by: Bruno Pesavento <bpesavento@infinito.it>
Tested-by: Borislav Petkov <bp@suse.de>
Tested-by: Frederic Bezies <fredbezies@gmail.com>
Cc: <stable@vger.kernel.org> # 4.9.0-
2017-01-10 13:28:26 +03:00
{
/* Initialize the rcu_node tree in preparation for the wait. */
2018-11-29 20:15:54 +03:00
sync_rcu_exp_select_cpus ( ) ;
rcu: Narrow early boot window of illegal synchronous grace periods
The current preemptible RCU implementation goes through three phases
during bootup. In the first phase, there is only one CPU that is running
with preemption disabled, so that a no-op is a synchronous grace period.
In the second mid-boot phase, the scheduler is running, but RCU has
not yet gotten its kthreads spawned (and, for expedited grace periods,
workqueues are not yet running. During this time, any attempt to do
a synchronous grace period will hang the system (or complain bitterly,
depending). In the third and final phase, RCU is fully operational and
everything works normally.
This has been OK for some time, but there has recently been some
synchronous grace periods showing up during the second mid-boot phase.
This code worked "by accident" for awhile, but started failing as soon
as expedited RCU grace periods switched over to workqueues in commit
8b355e3bc140 ("rcu: Drive expedited grace periods from workqueue").
Note that the code was buggy even before this commit, as it was subject
to failure on real-time systems that forced all expedited grace periods
to run as normal grace periods (for example, using the rcu_normal ksysfs
parameter). The callchain from the failure case is as follows:
early_amd_iommu_init()
|-> acpi_put_table(ivrs_base);
|-> acpi_tb_put_table(table_desc);
|-> acpi_tb_invalidate_table(table_desc);
|-> acpi_tb_release_table(...)
|-> acpi_os_unmap_memory
|-> acpi_os_unmap_iomem
|-> acpi_os_map_cleanup
|-> synchronize_rcu_expedited
The kernel showing this callchain was built with CONFIG_PREEMPT_RCU=y,
which caused the code to try using workqueues before they were
initialized, which did not go well.
This commit therefore reworks RCU to permit synchronous grace periods
to proceed during this mid-boot phase. This commit is therefore a
fix to a regression introduced in v4.9, and is therefore being put
forward post-merge-window in v4.10.
This commit sets a flag from the existing rcu_scheduler_starting()
function which causes all synchronous grace periods to take the expedited
path. The expedited path now checks this flag, using the requesting task
to drive the expedited grace period forward during the mid-boot phase.
Finally, this flag is updated by a core_initcall() function named
rcu_exp_runtime_mode(), which causes the runtime codepaths to be used.
Note that this arrangement assumes that tasks are not sent POSIX signals
(or anything similar) from the time that the first task is spawned
through core_initcall() time.
Fixes: 8b355e3bc140 ("rcu: Drive expedited grace periods from workqueue")
Reported-by: "Zheng, Lv" <lv.zheng@intel.com>
Reported-by: Borislav Petkov <bp@alien8.de>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Tested-by: Stan Kain <stan.kain@gmail.com>
Tested-by: Ivan <waffolz@hotmail.com>
Tested-by: Emanuel Castelo <emanuel.castelo@gmail.com>
Tested-by: Bruno Pesavento <bpesavento@infinito.it>
Tested-by: Borislav Petkov <bp@suse.de>
Tested-by: Frederic Bezies <fredbezies@gmail.com>
Cc: <stable@vger.kernel.org> # 4.9.0-
2017-01-10 13:28:26 +03:00
/* Wait and clean up, including waking everyone. */
2018-07-04 03:22:34 +03:00
rcu_exp_wait_wake ( s ) ;
rcu: Narrow early boot window of illegal synchronous grace periods
The current preemptible RCU implementation goes through three phases
during bootup. In the first phase, there is only one CPU that is running
with preemption disabled, so that a no-op is a synchronous grace period.
In the second mid-boot phase, the scheduler is running, but RCU has
not yet gotten its kthreads spawned (and, for expedited grace periods,
workqueues are not yet running. During this time, any attempt to do
a synchronous grace period will hang the system (or complain bitterly,
depending). In the third and final phase, RCU is fully operational and
everything works normally.
This has been OK for some time, but there has recently been some
synchronous grace periods showing up during the second mid-boot phase.
This code worked "by accident" for awhile, but started failing as soon
as expedited RCU grace periods switched over to workqueues in commit
8b355e3bc140 ("rcu: Drive expedited grace periods from workqueue").
Note that the code was buggy even before this commit, as it was subject
to failure on real-time systems that forced all expedited grace periods
to run as normal grace periods (for example, using the rcu_normal ksysfs
parameter). The callchain from the failure case is as follows:
early_amd_iommu_init()
|-> acpi_put_table(ivrs_base);
|-> acpi_tb_put_table(table_desc);
|-> acpi_tb_invalidate_table(table_desc);
|-> acpi_tb_release_table(...)
|-> acpi_os_unmap_memory
|-> acpi_os_unmap_iomem
|-> acpi_os_map_cleanup
|-> synchronize_rcu_expedited
The kernel showing this callchain was built with CONFIG_PREEMPT_RCU=y,
which caused the code to try using workqueues before they were
initialized, which did not go well.
This commit therefore reworks RCU to permit synchronous grace periods
to proceed during this mid-boot phase. This commit is therefore a
fix to a regression introduced in v4.9, and is therefore being put
forward post-merge-window in v4.10.
This commit sets a flag from the existing rcu_scheduler_starting()
function which causes all synchronous grace periods to take the expedited
path. The expedited path now checks this flag, using the requesting task
to drive the expedited grace period forward during the mid-boot phase.
Finally, this flag is updated by a core_initcall() function named
rcu_exp_runtime_mode(), which causes the runtime codepaths to be used.
Note that this arrangement assumes that tasks are not sent POSIX signals
(or anything similar) from the time that the first task is spawned
through core_initcall() time.
Fixes: 8b355e3bc140 ("rcu: Drive expedited grace periods from workqueue")
Reported-by: "Zheng, Lv" <lv.zheng@intel.com>
Reported-by: Borislav Petkov <bp@alien8.de>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Tested-by: Stan Kain <stan.kain@gmail.com>
Tested-by: Ivan <waffolz@hotmail.com>
Tested-by: Emanuel Castelo <emanuel.castelo@gmail.com>
Tested-by: Bruno Pesavento <bpesavento@infinito.it>
Tested-by: Borislav Petkov <bp@suse.de>
Tested-by: Frederic Bezies <fredbezies@gmail.com>
Cc: <stable@vger.kernel.org> # 4.9.0-
2017-01-10 13:28:26 +03:00
}
2016-06-29 23:46:25 +03:00
/*
* Work - queue handler to drive an expedited grace period forward .
*/
static void wait_rcu_exp_gp ( struct work_struct * wp )
{
struct rcu_exp_work * rewp ;
rewp = container_of ( wp , struct rcu_exp_work , rew_work ) ;
2018-11-29 20:15:54 +03:00
rcu_exp_sel_wait_wake ( rewp - > rew_s ) ;
2016-06-29 23:46:25 +03:00
}
2016-04-16 02:44:07 +03:00
# ifdef CONFIG_PREEMPT_RCU
/*
* Remote handler for smp_call_function_single ( ) . If there is an
* RCU read - side critical section in effect , request that the
* next rcu_read_unlock ( ) record the quiescent state up the
* - > expmask fields in the rcu_node tree . Otherwise , immediately
* report the quiescent state .
*/
2018-11-29 20:15:54 +03:00
static void rcu_exp_handler ( void * unused )
2016-04-16 02:44:07 +03:00
{
2020-02-16 02:23:26 +03:00
int depth = rcu_preempt_depth ( ) ;
rcu: Defer reporting RCU-preempt quiescent states when disabled
This commit defers reporting of RCU-preempt quiescent states at
rcu_read_unlock_special() time when any of interrupts, softirq, or
preemption are disabled. These deferred quiescent states are reported
at a later RCU_SOFTIRQ, context switch, idle entry, or CPU-hotplug
offline operation. Of course, if another RCU read-side critical
section has started in the meantime, the reporting of the quiescent
state will be further deferred.
This also means that disabling preemption, interrupts, and/or
softirqs will act as an RCU-preempt read-side critical section.
This is enforced by checking preempt_count() as needed.
Some special cases must be handled on an ad-hoc basis, for example,
context switch is a quiescent state even though both the scheduler and
do_exit() disable preemption. In these cases, additional calls to
rcu_preempt_deferred_qs() override the preemption disabling. Similar
logic overrides disabled interrupts in rcu_preempt_check_callbacks()
because in this case the quiescent state happened just before the
corresponding scheduling-clock interrupt.
In theory, this change lifts a long-standing restriction that required
that if interrupts were disabled across a call to rcu_read_unlock()
that the matching rcu_read_lock() also be contained within that
interrupts-disabled region of code. Because the reporting of the
corresponding RCU-preempt quiescent state is now deferred until
after interrupts have been enabled, it is no longer possible for this
situation to result in deadlocks involving the scheduler's runqueue and
priority-inheritance locks. This may allow some code simplification that
might reduce interrupt latency a bit. Unfortunately, in practice this
would also defer deboosting a low-priority task that had been subjected
to RCU priority boosting, so real-time-response considerations might
well force this restriction to remain in place.
Because RCU-preempt grace periods are now blocked not only by RCU
read-side critical sections, but also by disabling of interrupts,
preemption, and softirqs, it will be possible to eliminate RCU-bh and
RCU-sched in favor of RCU-preempt in CONFIG_PREEMPT=y kernels. This may
require some additional plumbing to provide the network denial-of-service
guarantees that have been traditionally provided by RCU-bh. Once these
are in place, CONFIG_PREEMPT=n kernels will be able to fold RCU-bh
into RCU-sched. This would mean that all kernels would have but
one flavor of RCU, which would open the door to significant code
cleanup.
Moving to a single flavor of RCU would also have the beneficial effect
of reducing the NOCB kthreads by at least a factor of two.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
[ paulmck: Apply rcu_read_unlock_special() preempt_count() feedback
from Joel Fernandes. ]
[ paulmck: Adjust rcu_eqs_enter() call to rcu_preempt_deferred_qs() in
response to bug reports from kbuild test robot. ]
[ paulmck: Fix bug located by kbuild test robot involving recursion
via rcu_preempt_deferred_qs(). ]
2018-06-21 22:50:01 +03:00
unsigned long flags ;
2018-07-04 01:37:16 +03:00
struct rcu_data * rdp = this_cpu_ptr ( & rcu_data ) ;
rcu: Defer reporting RCU-preempt quiescent states when disabled
This commit defers reporting of RCU-preempt quiescent states at
rcu_read_unlock_special() time when any of interrupts, softirq, or
preemption are disabled. These deferred quiescent states are reported
at a later RCU_SOFTIRQ, context switch, idle entry, or CPU-hotplug
offline operation. Of course, if another RCU read-side critical
section has started in the meantime, the reporting of the quiescent
state will be further deferred.
This also means that disabling preemption, interrupts, and/or
softirqs will act as an RCU-preempt read-side critical section.
This is enforced by checking preempt_count() as needed.
Some special cases must be handled on an ad-hoc basis, for example,
context switch is a quiescent state even though both the scheduler and
do_exit() disable preemption. In these cases, additional calls to
rcu_preempt_deferred_qs() override the preemption disabling. Similar
logic overrides disabled interrupts in rcu_preempt_check_callbacks()
because in this case the quiescent state happened just before the
corresponding scheduling-clock interrupt.
In theory, this change lifts a long-standing restriction that required
that if interrupts were disabled across a call to rcu_read_unlock()
that the matching rcu_read_lock() also be contained within that
interrupts-disabled region of code. Because the reporting of the
corresponding RCU-preempt quiescent state is now deferred until
after interrupts have been enabled, it is no longer possible for this
situation to result in deadlocks involving the scheduler's runqueue and
priority-inheritance locks. This may allow some code simplification that
might reduce interrupt latency a bit. Unfortunately, in practice this
would also defer deboosting a low-priority task that had been subjected
to RCU priority boosting, so real-time-response considerations might
well force this restriction to remain in place.
Because RCU-preempt grace periods are now blocked not only by RCU
read-side critical sections, but also by disabling of interrupts,
preemption, and softirqs, it will be possible to eliminate RCU-bh and
RCU-sched in favor of RCU-preempt in CONFIG_PREEMPT=y kernels. This may
require some additional plumbing to provide the network denial-of-service
guarantees that have been traditionally provided by RCU-bh. Once these
are in place, CONFIG_PREEMPT=n kernels will be able to fold RCU-bh
into RCU-sched. This would mean that all kernels would have but
one flavor of RCU, which would open the door to significant code
cleanup.
Moving to a single flavor of RCU would also have the beneficial effect
of reducing the NOCB kthreads by at least a factor of two.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
[ paulmck: Apply rcu_read_unlock_special() preempt_count() feedback
from Joel Fernandes. ]
[ paulmck: Adjust rcu_eqs_enter() call to rcu_preempt_deferred_qs() in
response to bug reports from kbuild test robot. ]
[ paulmck: Fix bug located by kbuild test robot involving recursion
via rcu_preempt_deferred_qs(). ]
2018-06-21 22:50:01 +03:00
struct rcu_node * rnp = rdp - > mynode ;
2016-04-16 02:44:07 +03:00
struct task_struct * t = current ;
/*
rcu: Defer reporting RCU-preempt quiescent states when disabled
This commit defers reporting of RCU-preempt quiescent states at
rcu_read_unlock_special() time when any of interrupts, softirq, or
preemption are disabled. These deferred quiescent states are reported
at a later RCU_SOFTIRQ, context switch, idle entry, or CPU-hotplug
offline operation. Of course, if another RCU read-side critical
section has started in the meantime, the reporting of the quiescent
state will be further deferred.
This also means that disabling preemption, interrupts, and/or
softirqs will act as an RCU-preempt read-side critical section.
This is enforced by checking preempt_count() as needed.
Some special cases must be handled on an ad-hoc basis, for example,
context switch is a quiescent state even though both the scheduler and
do_exit() disable preemption. In these cases, additional calls to
rcu_preempt_deferred_qs() override the preemption disabling. Similar
logic overrides disabled interrupts in rcu_preempt_check_callbacks()
because in this case the quiescent state happened just before the
corresponding scheduling-clock interrupt.
In theory, this change lifts a long-standing restriction that required
that if interrupts were disabled across a call to rcu_read_unlock()
that the matching rcu_read_lock() also be contained within that
interrupts-disabled region of code. Because the reporting of the
corresponding RCU-preempt quiescent state is now deferred until
after interrupts have been enabled, it is no longer possible for this
situation to result in deadlocks involving the scheduler's runqueue and
priority-inheritance locks. This may allow some code simplification that
might reduce interrupt latency a bit. Unfortunately, in practice this
would also defer deboosting a low-priority task that had been subjected
to RCU priority boosting, so real-time-response considerations might
well force this restriction to remain in place.
Because RCU-preempt grace periods are now blocked not only by RCU
read-side critical sections, but also by disabling of interrupts,
preemption, and softirqs, it will be possible to eliminate RCU-bh and
RCU-sched in favor of RCU-preempt in CONFIG_PREEMPT=y kernels. This may
require some additional plumbing to provide the network denial-of-service
guarantees that have been traditionally provided by RCU-bh. Once these
are in place, CONFIG_PREEMPT=n kernels will be able to fold RCU-bh
into RCU-sched. This would mean that all kernels would have but
one flavor of RCU, which would open the door to significant code
cleanup.
Moving to a single flavor of RCU would also have the beneficial effect
of reducing the NOCB kthreads by at least a factor of two.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
[ paulmck: Apply rcu_read_unlock_special() preempt_count() feedback
from Joel Fernandes. ]
[ paulmck: Adjust rcu_eqs_enter() call to rcu_preempt_deferred_qs() in
response to bug reports from kbuild test robot. ]
[ paulmck: Fix bug located by kbuild test robot involving recursion
via rcu_preempt_deferred_qs(). ]
2018-06-21 22:50:01 +03:00
* First , the common case of not being in an RCU read - side
* critical section . If also enabled or idle , immediately
* report the quiescent state , otherwise defer .
2016-04-16 02:44:07 +03:00
*/
2020-02-16 02:23:26 +03:00
if ( ! depth ) {
rcu: Defer reporting RCU-preempt quiescent states when disabled
This commit defers reporting of RCU-preempt quiescent states at
rcu_read_unlock_special() time when any of interrupts, softirq, or
preemption are disabled. These deferred quiescent states are reported
at a later RCU_SOFTIRQ, context switch, idle entry, or CPU-hotplug
offline operation. Of course, if another RCU read-side critical
section has started in the meantime, the reporting of the quiescent
state will be further deferred.
This also means that disabling preemption, interrupts, and/or
softirqs will act as an RCU-preempt read-side critical section.
This is enforced by checking preempt_count() as needed.
Some special cases must be handled on an ad-hoc basis, for example,
context switch is a quiescent state even though both the scheduler and
do_exit() disable preemption. In these cases, additional calls to
rcu_preempt_deferred_qs() override the preemption disabling. Similar
logic overrides disabled interrupts in rcu_preempt_check_callbacks()
because in this case the quiescent state happened just before the
corresponding scheduling-clock interrupt.
In theory, this change lifts a long-standing restriction that required
that if interrupts were disabled across a call to rcu_read_unlock()
that the matching rcu_read_lock() also be contained within that
interrupts-disabled region of code. Because the reporting of the
corresponding RCU-preempt quiescent state is now deferred until
after interrupts have been enabled, it is no longer possible for this
situation to result in deadlocks involving the scheduler's runqueue and
priority-inheritance locks. This may allow some code simplification that
might reduce interrupt latency a bit. Unfortunately, in practice this
would also defer deboosting a low-priority task that had been subjected
to RCU priority boosting, so real-time-response considerations might
well force this restriction to remain in place.
Because RCU-preempt grace periods are now blocked not only by RCU
read-side critical sections, but also by disabling of interrupts,
preemption, and softirqs, it will be possible to eliminate RCU-bh and
RCU-sched in favor of RCU-preempt in CONFIG_PREEMPT=y kernels. This may
require some additional plumbing to provide the network denial-of-service
guarantees that have been traditionally provided by RCU-bh. Once these
are in place, CONFIG_PREEMPT=n kernels will be able to fold RCU-bh
into RCU-sched. This would mean that all kernels would have but
one flavor of RCU, which would open the door to significant code
cleanup.
Moving to a single flavor of RCU would also have the beneficial effect
of reducing the NOCB kthreads by at least a factor of two.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
[ paulmck: Apply rcu_read_unlock_special() preempt_count() feedback
from Joel Fernandes. ]
[ paulmck: Adjust rcu_eqs_enter() call to rcu_preempt_deferred_qs() in
response to bug reports from kbuild test robot. ]
[ paulmck: Fix bug located by kbuild test robot involving recursion
via rcu_preempt_deferred_qs(). ]
2018-06-21 22:50:01 +03:00
if ( ! ( preempt_count ( ) & ( PREEMPT_MASK | SOFTIRQ_MASK ) ) | |
rcu_dynticks_curr_cpu_in_eqs ( ) ) {
2018-07-04 03:22:34 +03:00
rcu_report_exp_rdp ( rdp ) ;
rcu: Defer reporting RCU-preempt quiescent states when disabled
This commit defers reporting of RCU-preempt quiescent states at
rcu_read_unlock_special() time when any of interrupts, softirq, or
preemption are disabled. These deferred quiescent states are reported
at a later RCU_SOFTIRQ, context switch, idle entry, or CPU-hotplug
offline operation. Of course, if another RCU read-side critical
section has started in the meantime, the reporting of the quiescent
state will be further deferred.
This also means that disabling preemption, interrupts, and/or
softirqs will act as an RCU-preempt read-side critical section.
This is enforced by checking preempt_count() as needed.
Some special cases must be handled on an ad-hoc basis, for example,
context switch is a quiescent state even though both the scheduler and
do_exit() disable preemption. In these cases, additional calls to
rcu_preempt_deferred_qs() override the preemption disabling. Similar
logic overrides disabled interrupts in rcu_preempt_check_callbacks()
because in this case the quiescent state happened just before the
corresponding scheduling-clock interrupt.
In theory, this change lifts a long-standing restriction that required
that if interrupts were disabled across a call to rcu_read_unlock()
that the matching rcu_read_lock() also be contained within that
interrupts-disabled region of code. Because the reporting of the
corresponding RCU-preempt quiescent state is now deferred until
after interrupts have been enabled, it is no longer possible for this
situation to result in deadlocks involving the scheduler's runqueue and
priority-inheritance locks. This may allow some code simplification that
might reduce interrupt latency a bit. Unfortunately, in practice this
would also defer deboosting a low-priority task that had been subjected
to RCU priority boosting, so real-time-response considerations might
well force this restriction to remain in place.
Because RCU-preempt grace periods are now blocked not only by RCU
read-side critical sections, but also by disabling of interrupts,
preemption, and softirqs, it will be possible to eliminate RCU-bh and
RCU-sched in favor of RCU-preempt in CONFIG_PREEMPT=y kernels. This may
require some additional plumbing to provide the network denial-of-service
guarantees that have been traditionally provided by RCU-bh. Once these
are in place, CONFIG_PREEMPT=n kernels will be able to fold RCU-bh
into RCU-sched. This would mean that all kernels would have but
one flavor of RCU, which would open the door to significant code
cleanup.
Moving to a single flavor of RCU would also have the beneficial effect
of reducing the NOCB kthreads by at least a factor of two.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
[ paulmck: Apply rcu_read_unlock_special() preempt_count() feedback
from Joel Fernandes. ]
[ paulmck: Adjust rcu_eqs_enter() call to rcu_preempt_deferred_qs() in
response to bug reports from kbuild test robot. ]
[ paulmck: Fix bug located by kbuild test robot involving recursion
via rcu_preempt_deferred_qs(). ]
2018-06-21 22:50:01 +03:00
} else {
2019-03-28 01:51:25 +03:00
rdp - > exp_deferred_qs = true ;
2018-07-26 23:44:00 +03:00
set_tsk_need_resched ( t ) ;
set_preempt_need_resched ( ) ;
rcu: Defer reporting RCU-preempt quiescent states when disabled
This commit defers reporting of RCU-preempt quiescent states at
rcu_read_unlock_special() time when any of interrupts, softirq, or
preemption are disabled. These deferred quiescent states are reported
at a later RCU_SOFTIRQ, context switch, idle entry, or CPU-hotplug
offline operation. Of course, if another RCU read-side critical
section has started in the meantime, the reporting of the quiescent
state will be further deferred.
This also means that disabling preemption, interrupts, and/or
softirqs will act as an RCU-preempt read-side critical section.
This is enforced by checking preempt_count() as needed.
Some special cases must be handled on an ad-hoc basis, for example,
context switch is a quiescent state even though both the scheduler and
do_exit() disable preemption. In these cases, additional calls to
rcu_preempt_deferred_qs() override the preemption disabling. Similar
logic overrides disabled interrupts in rcu_preempt_check_callbacks()
because in this case the quiescent state happened just before the
corresponding scheduling-clock interrupt.
In theory, this change lifts a long-standing restriction that required
that if interrupts were disabled across a call to rcu_read_unlock()
that the matching rcu_read_lock() also be contained within that
interrupts-disabled region of code. Because the reporting of the
corresponding RCU-preempt quiescent state is now deferred until
after interrupts have been enabled, it is no longer possible for this
situation to result in deadlocks involving the scheduler's runqueue and
priority-inheritance locks. This may allow some code simplification that
might reduce interrupt latency a bit. Unfortunately, in practice this
would also defer deboosting a low-priority task that had been subjected
to RCU priority boosting, so real-time-response considerations might
well force this restriction to remain in place.
Because RCU-preempt grace periods are now blocked not only by RCU
read-side critical sections, but also by disabling of interrupts,
preemption, and softirqs, it will be possible to eliminate RCU-bh and
RCU-sched in favor of RCU-preempt in CONFIG_PREEMPT=y kernels. This may
require some additional plumbing to provide the network denial-of-service
guarantees that have been traditionally provided by RCU-bh. Once these
are in place, CONFIG_PREEMPT=n kernels will be able to fold RCU-bh
into RCU-sched. This would mean that all kernels would have but
one flavor of RCU, which would open the door to significant code
cleanup.
Moving to a single flavor of RCU would also have the beneficial effect
of reducing the NOCB kthreads by at least a factor of two.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
[ paulmck: Apply rcu_read_unlock_special() preempt_count() feedback
from Joel Fernandes. ]
[ paulmck: Adjust rcu_eqs_enter() call to rcu_preempt_deferred_qs() in
response to bug reports from kbuild test robot. ]
[ paulmck: Fix bug located by kbuild test robot involving recursion
via rcu_preempt_deferred_qs(). ]
2018-06-21 22:50:01 +03:00
}
2016-04-16 02:44:07 +03:00
return ;
}
/*
rcu: Defer reporting RCU-preempt quiescent states when disabled
This commit defers reporting of RCU-preempt quiescent states at
rcu_read_unlock_special() time when any of interrupts, softirq, or
preemption are disabled. These deferred quiescent states are reported
at a later RCU_SOFTIRQ, context switch, idle entry, or CPU-hotplug
offline operation. Of course, if another RCU read-side critical
section has started in the meantime, the reporting of the quiescent
state will be further deferred.
This also means that disabling preemption, interrupts, and/or
softirqs will act as an RCU-preempt read-side critical section.
This is enforced by checking preempt_count() as needed.
Some special cases must be handled on an ad-hoc basis, for example,
context switch is a quiescent state even though both the scheduler and
do_exit() disable preemption. In these cases, additional calls to
rcu_preempt_deferred_qs() override the preemption disabling. Similar
logic overrides disabled interrupts in rcu_preempt_check_callbacks()
because in this case the quiescent state happened just before the
corresponding scheduling-clock interrupt.
In theory, this change lifts a long-standing restriction that required
that if interrupts were disabled across a call to rcu_read_unlock()
that the matching rcu_read_lock() also be contained within that
interrupts-disabled region of code. Because the reporting of the
corresponding RCU-preempt quiescent state is now deferred until
after interrupts have been enabled, it is no longer possible for this
situation to result in deadlocks involving the scheduler's runqueue and
priority-inheritance locks. This may allow some code simplification that
might reduce interrupt latency a bit. Unfortunately, in practice this
would also defer deboosting a low-priority task that had been subjected
to RCU priority boosting, so real-time-response considerations might
well force this restriction to remain in place.
Because RCU-preempt grace periods are now blocked not only by RCU
read-side critical sections, but also by disabling of interrupts,
preemption, and softirqs, it will be possible to eliminate RCU-bh and
RCU-sched in favor of RCU-preempt in CONFIG_PREEMPT=y kernels. This may
require some additional plumbing to provide the network denial-of-service
guarantees that have been traditionally provided by RCU-bh. Once these
are in place, CONFIG_PREEMPT=n kernels will be able to fold RCU-bh
into RCU-sched. This would mean that all kernels would have but
one flavor of RCU, which would open the door to significant code
cleanup.
Moving to a single flavor of RCU would also have the beneficial effect
of reducing the NOCB kthreads by at least a factor of two.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
[ paulmck: Apply rcu_read_unlock_special() preempt_count() feedback
from Joel Fernandes. ]
[ paulmck: Adjust rcu_eqs_enter() call to rcu_preempt_deferred_qs() in
response to bug reports from kbuild test robot. ]
[ paulmck: Fix bug located by kbuild test robot involving recursion
via rcu_preempt_deferred_qs(). ]
2018-06-21 22:50:01 +03:00
* Second , the less - common case of being in an RCU read - side
* critical section . In this case we can count on a future
* rcu_read_unlock ( ) . However , this rcu_read_unlock ( ) might
* execute on some other CPU , but in that case there will be
* a future context switch . Either way , if the expedited
* grace period is still waiting on this CPU , set - > deferred_qs
* so that the eventual quiescent state will be reported .
* Note that there is a large group of race conditions that
* can have caused this quiescent state to already have been
* reported , so we really do need to check - > expmask .
2016-04-16 02:44:07 +03:00
*/
2020-02-16 02:23:26 +03:00
if ( depth > 0 ) {
rcu: Defer reporting RCU-preempt quiescent states when disabled
This commit defers reporting of RCU-preempt quiescent states at
rcu_read_unlock_special() time when any of interrupts, softirq, or
preemption are disabled. These deferred quiescent states are reported
at a later RCU_SOFTIRQ, context switch, idle entry, or CPU-hotplug
offline operation. Of course, if another RCU read-side critical
section has started in the meantime, the reporting of the quiescent
state will be further deferred.
This also means that disabling preemption, interrupts, and/or
softirqs will act as an RCU-preempt read-side critical section.
This is enforced by checking preempt_count() as needed.
Some special cases must be handled on an ad-hoc basis, for example,
context switch is a quiescent state even though both the scheduler and
do_exit() disable preemption. In these cases, additional calls to
rcu_preempt_deferred_qs() override the preemption disabling. Similar
logic overrides disabled interrupts in rcu_preempt_check_callbacks()
because in this case the quiescent state happened just before the
corresponding scheduling-clock interrupt.
In theory, this change lifts a long-standing restriction that required
that if interrupts were disabled across a call to rcu_read_unlock()
that the matching rcu_read_lock() also be contained within that
interrupts-disabled region of code. Because the reporting of the
corresponding RCU-preempt quiescent state is now deferred until
after interrupts have been enabled, it is no longer possible for this
situation to result in deadlocks involving the scheduler's runqueue and
priority-inheritance locks. This may allow some code simplification that
might reduce interrupt latency a bit. Unfortunately, in practice this
would also defer deboosting a low-priority task that had been subjected
to RCU priority boosting, so real-time-response considerations might
well force this restriction to remain in place.
Because RCU-preempt grace periods are now blocked not only by RCU
read-side critical sections, but also by disabling of interrupts,
preemption, and softirqs, it will be possible to eliminate RCU-bh and
RCU-sched in favor of RCU-preempt in CONFIG_PREEMPT=y kernels. This may
require some additional plumbing to provide the network denial-of-service
guarantees that have been traditionally provided by RCU-bh. Once these
are in place, CONFIG_PREEMPT=n kernels will be able to fold RCU-bh
into RCU-sched. This would mean that all kernels would have but
one flavor of RCU, which would open the door to significant code
cleanup.
Moving to a single flavor of RCU would also have the beneficial effect
of reducing the NOCB kthreads by at least a factor of two.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
[ paulmck: Apply rcu_read_unlock_special() preempt_count() feedback
from Joel Fernandes. ]
[ paulmck: Adjust rcu_eqs_enter() call to rcu_preempt_deferred_qs() in
response to bug reports from kbuild test robot. ]
[ paulmck: Fix bug located by kbuild test robot involving recursion
via rcu_preempt_deferred_qs(). ]
2018-06-21 22:50:01 +03:00
raw_spin_lock_irqsave_rcu_node ( rnp , flags ) ;
rcu: Speed up expedited GPs when interrupting RCU reader
In PREEMPT kernels, an expedited grace period might send an IPI to a
CPU that is executing an RCU read-side critical section. In that case,
it would be nice if the rcu_read_unlock() directly interacted with the
RCU core code to immediately report the quiescent state. And this does
happen in the case where the reader has been preempted. But it would
also be a nice performance optimization if immediate reporting also
happened in the preemption-free case.
This commit therefore adds an ->exp_hint field to the task_struct structure's
->rcu_read_unlock_special field. The IPI handler sets this hint when
it has interrupted an RCU read-side critical section, and this causes
the outermost rcu_read_unlock() call to invoke rcu_read_unlock_special(),
which, if preemption is enabled, reports the quiescent state immediately.
If preemption is disabled, then the report is required to be deferred
until preemption (or bottom halves or interrupts or whatever) is re-enabled.
Because this is a hint, it does nothing for more complicated cases. For
example, if the IPI interrupts an RCU reader, but interrupts are disabled
across the rcu_read_unlock(), but another rcu_read_lock() is executed
before interrupts are re-enabled, the hint will already have been cleared.
If you do crazy things like this, reporting will be deferred until some
later RCU_SOFTIRQ handler, context switch, cond_resched(), or similar.
Reported-by: Joel Fernandes <joel@joelfernandes.org>
Signed-off-by: Paul E. McKenney <paulmck@linux.ibm.com>
Acked-by: Joel Fernandes (Google) <joel@joelfernandes.org>
2018-10-16 14:12:58 +03:00
if ( rnp - > expmask & rdp - > grpmask ) {
2019-03-28 01:51:25 +03:00
rdp - > exp_deferred_qs = true ;
2019-03-26 20:22:22 +03:00
t - > rcu_read_unlock_special . b . exp_hint = true ;
rcu: Speed up expedited GPs when interrupting RCU reader
In PREEMPT kernels, an expedited grace period might send an IPI to a
CPU that is executing an RCU read-side critical section. In that case,
it would be nice if the rcu_read_unlock() directly interacted with the
RCU core code to immediately report the quiescent state. And this does
happen in the case where the reader has been preempted. But it would
also be a nice performance optimization if immediate reporting also
happened in the preemption-free case.
This commit therefore adds an ->exp_hint field to the task_struct structure's
->rcu_read_unlock_special field. The IPI handler sets this hint when
it has interrupted an RCU read-side critical section, and this causes
the outermost rcu_read_unlock() call to invoke rcu_read_unlock_special(),
which, if preemption is enabled, reports the quiescent state immediately.
If preemption is disabled, then the report is required to be deferred
until preemption (or bottom halves or interrupts or whatever) is re-enabled.
Because this is a hint, it does nothing for more complicated cases. For
example, if the IPI interrupts an RCU reader, but interrupts are disabled
across the rcu_read_unlock(), but another rcu_read_lock() is executed
before interrupts are re-enabled, the hint will already have been cleared.
If you do crazy things like this, reporting will be deferred until some
later RCU_SOFTIRQ handler, context switch, cond_resched(), or similar.
Reported-by: Joel Fernandes <joel@joelfernandes.org>
Signed-off-by: Paul E. McKenney <paulmck@linux.ibm.com>
Acked-by: Joel Fernandes (Google) <joel@joelfernandes.org>
2018-10-16 14:12:58 +03:00
}
rcu: Defer reporting RCU-preempt quiescent states when disabled
This commit defers reporting of RCU-preempt quiescent states at
rcu_read_unlock_special() time when any of interrupts, softirq, or
preemption are disabled. These deferred quiescent states are reported
at a later RCU_SOFTIRQ, context switch, idle entry, or CPU-hotplug
offline operation. Of course, if another RCU read-side critical
section has started in the meantime, the reporting of the quiescent
state will be further deferred.
This also means that disabling preemption, interrupts, and/or
softirqs will act as an RCU-preempt read-side critical section.
This is enforced by checking preempt_count() as needed.
Some special cases must be handled on an ad-hoc basis, for example,
context switch is a quiescent state even though both the scheduler and
do_exit() disable preemption. In these cases, additional calls to
rcu_preempt_deferred_qs() override the preemption disabling. Similar
logic overrides disabled interrupts in rcu_preempt_check_callbacks()
because in this case the quiescent state happened just before the
corresponding scheduling-clock interrupt.
In theory, this change lifts a long-standing restriction that required
that if interrupts were disabled across a call to rcu_read_unlock()
that the matching rcu_read_lock() also be contained within that
interrupts-disabled region of code. Because the reporting of the
corresponding RCU-preempt quiescent state is now deferred until
after interrupts have been enabled, it is no longer possible for this
situation to result in deadlocks involving the scheduler's runqueue and
priority-inheritance locks. This may allow some code simplification that
might reduce interrupt latency a bit. Unfortunately, in practice this
would also defer deboosting a low-priority task that had been subjected
to RCU priority boosting, so real-time-response considerations might
well force this restriction to remain in place.
Because RCU-preempt grace periods are now blocked not only by RCU
read-side critical sections, but also by disabling of interrupts,
preemption, and softirqs, it will be possible to eliminate RCU-bh and
RCU-sched in favor of RCU-preempt in CONFIG_PREEMPT=y kernels. This may
require some additional plumbing to provide the network denial-of-service
guarantees that have been traditionally provided by RCU-bh. Once these
are in place, CONFIG_PREEMPT=n kernels will be able to fold RCU-bh
into RCU-sched. This would mean that all kernels would have but
one flavor of RCU, which would open the door to significant code
cleanup.
Moving to a single flavor of RCU would also have the beneficial effect
of reducing the NOCB kthreads by at least a factor of two.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
[ paulmck: Apply rcu_read_unlock_special() preempt_count() feedback
from Joel Fernandes. ]
[ paulmck: Adjust rcu_eqs_enter() call to rcu_preempt_deferred_qs() in
response to bug reports from kbuild test robot. ]
[ paulmck: Fix bug located by kbuild test robot involving recursion
via rcu_preempt_deferred_qs(). ]
2018-06-21 22:50:01 +03:00
raw_spin_unlock_irqrestore_rcu_node ( rnp , flags ) ;
2018-11-28 21:37:42 +03:00
return ;
rcu: Defer reporting RCU-preempt quiescent states when disabled
This commit defers reporting of RCU-preempt quiescent states at
rcu_read_unlock_special() time when any of interrupts, softirq, or
preemption are disabled. These deferred quiescent states are reported
at a later RCU_SOFTIRQ, context switch, idle entry, or CPU-hotplug
offline operation. Of course, if another RCU read-side critical
section has started in the meantime, the reporting of the quiescent
state will be further deferred.
This also means that disabling preemption, interrupts, and/or
softirqs will act as an RCU-preempt read-side critical section.
This is enforced by checking preempt_count() as needed.
Some special cases must be handled on an ad-hoc basis, for example,
context switch is a quiescent state even though both the scheduler and
do_exit() disable preemption. In these cases, additional calls to
rcu_preempt_deferred_qs() override the preemption disabling. Similar
logic overrides disabled interrupts in rcu_preempt_check_callbacks()
because in this case the quiescent state happened just before the
corresponding scheduling-clock interrupt.
In theory, this change lifts a long-standing restriction that required
that if interrupts were disabled across a call to rcu_read_unlock()
that the matching rcu_read_lock() also be contained within that
interrupts-disabled region of code. Because the reporting of the
corresponding RCU-preempt quiescent state is now deferred until
after interrupts have been enabled, it is no longer possible for this
situation to result in deadlocks involving the scheduler's runqueue and
priority-inheritance locks. This may allow some code simplification that
might reduce interrupt latency a bit. Unfortunately, in practice this
would also defer deboosting a low-priority task that had been subjected
to RCU priority boosting, so real-time-response considerations might
well force this restriction to remain in place.
Because RCU-preempt grace periods are now blocked not only by RCU
read-side critical sections, but also by disabling of interrupts,
preemption, and softirqs, it will be possible to eliminate RCU-bh and
RCU-sched in favor of RCU-preempt in CONFIG_PREEMPT=y kernels. This may
require some additional plumbing to provide the network denial-of-service
guarantees that have been traditionally provided by RCU-bh. Once these
are in place, CONFIG_PREEMPT=n kernels will be able to fold RCU-bh
into RCU-sched. This would mean that all kernels would have but
one flavor of RCU, which would open the door to significant code
cleanup.
Moving to a single flavor of RCU would also have the beneficial effect
of reducing the NOCB kthreads by at least a factor of two.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
[ paulmck: Apply rcu_read_unlock_special() preempt_count() feedback
from Joel Fernandes. ]
[ paulmck: Adjust rcu_eqs_enter() call to rcu_preempt_deferred_qs() in
response to bug reports from kbuild test robot. ]
[ paulmck: Fix bug located by kbuild test robot involving recursion
via rcu_preempt_deferred_qs(). ]
2018-06-21 22:50:01 +03:00
}
2020-02-16 02:23:26 +03:00
// Finally, negative nesting depth should not happen.
WARN_ON_ONCE ( 1 ) ;
2016-04-16 02:44:07 +03:00
}
2019-10-15 22:18:14 +03:00
/* PREEMPTION=y, so no PREEMPTION=n expedited grace period to clean up after. */
2018-07-03 00:30:37 +03:00
static void sync_sched_exp_online_cleanup ( int cpu )
{
}
2019-01-12 07:51:49 +03:00
/*
* Scan the current list of tasks blocked within RCU read - side critical
* sections , printing out the tid of each that is blocking the current
* expedited grace period .
*/
static int rcu_print_task_exp_stall ( struct rcu_node * rnp )
{
2020-01-04 01:18:12 +03:00
unsigned long flags ;
2019-01-12 07:51:49 +03:00
int ndetected = 0 ;
2020-01-04 01:18:12 +03:00
struct task_struct * t ;
2019-01-12 07:51:49 +03:00
2020-01-04 01:18:12 +03:00
if ( ! READ_ONCE ( rnp - > exp_tasks ) )
2019-01-12 07:51:49 +03:00
return 0 ;
2020-01-04 01:18:12 +03:00
raw_spin_lock_irqsave_rcu_node ( rnp , flags ) ;
2019-01-12 07:51:49 +03:00
t = list_entry ( rnp - > exp_tasks - > prev ,
struct task_struct , rcu_node_entry ) ;
list_for_each_entry_continue ( t , & rnp - > blkd_tasks , rcu_node_entry ) {
pr_cont ( " P%d " , t - > pid ) ;
ndetected + + ;
}
2020-01-04 01:18:12 +03:00
raw_spin_unlock_irqrestore_rcu_node ( rnp , flags ) ;
2019-01-12 07:51:49 +03:00
return ndetected ;
}
2016-04-16 02:44:07 +03:00
# else /* #ifdef CONFIG_PREEMPT_RCU */
2019-03-27 20:03:12 +03:00
/* Request an expedited quiescent state. */
static void rcu_exp_need_qs ( void )
{
__this_cpu_write ( rcu_data . cpu_no_qs . b . exp , true ) ;
/* Store .exp before .rcu_urgent_qs. */
smp_store_release ( this_cpu_ptr ( & rcu_data . rcu_urgent_qs ) , true ) ;
set_tsk_need_resched ( current ) ;
set_preempt_need_resched ( ) ;
}
2018-07-03 00:30:37 +03:00
/* Invoked on each online non-idle CPU for expedited quiescent state. */
2018-11-29 20:15:54 +03:00
static void rcu_exp_handler ( void * unused )
2018-07-03 00:30:37 +03:00
{
2020-06-18 19:51:12 +03:00
struct rcu_data * rdp = this_cpu_ptr ( & rcu_data ) ;
struct rcu_node * rnp = rdp - > mynode ;
2018-07-03 00:30:37 +03:00
if ( ! ( READ_ONCE ( rnp - > expmask ) & rdp - > grpmask ) | |
__this_cpu_read ( rcu_data . cpu_no_qs . b . exp ) )
return ;
if ( rcu_is_cpu_rrupt_from_idle ( ) ) {
2018-07-04 03:22:34 +03:00
rcu_report_exp_rdp ( this_cpu_ptr ( & rcu_data ) ) ;
2018-07-03 00:30:37 +03:00
return ;
}
2019-03-27 20:03:12 +03:00
rcu_exp_need_qs ( ) ;
2018-07-03 00:30:37 +03:00
}
/* Send IPI for expedited cleanup if needed at end of CPU-hotplug operation. */
static void sync_sched_exp_online_cleanup ( int cpu )
{
2019-03-27 20:03:12 +03:00
unsigned long flags ;
int my_cpu ;
2018-07-03 00:30:37 +03:00
struct rcu_data * rdp ;
int ret ;
struct rcu_node * rnp ;
2018-07-04 01:37:16 +03:00
rdp = per_cpu_ptr ( & rcu_data , cpu ) ;
2018-07-03 00:30:37 +03:00
rnp = rdp - > mynode ;
2019-03-27 20:03:12 +03:00
my_cpu = get_cpu ( ) ;
/* Quiescent state either not needed or already requested, leave. */
if ( ! ( READ_ONCE ( rnp - > expmask ) & rdp - > grpmask ) | |
__this_cpu_read ( rcu_data . cpu_no_qs . b . exp ) ) {
put_cpu ( ) ;
2018-07-03 00:30:37 +03:00
return ;
2019-03-27 20:03:12 +03:00
}
/* Quiescent state needed on current CPU, so set it up locally. */
if ( my_cpu = = cpu ) {
local_irq_save ( flags ) ;
rcu_exp_need_qs ( ) ;
local_irq_restore ( flags ) ;
put_cpu ( ) ;
return ;
}
/* Quiescent state needed on some other CPU, send IPI. */
2018-11-29 20:15:54 +03:00
ret = smp_call_function_single ( cpu , rcu_exp_handler , NULL , 0 ) ;
2019-03-27 20:03:12 +03:00
put_cpu ( ) ;
2018-07-03 00:30:37 +03:00
WARN_ON_ONCE ( ret ) ;
}
2019-01-12 07:51:49 +03:00
/*
* Because preemptible RCU does not exist , we never have to check for
* tasks blocked within RCU read - side critical sections that are
* blocking the current expedited grace period .
*/
static int rcu_print_task_exp_stall ( struct rcu_node * rnp )
{
return 0 ;
}
2018-11-29 21:01:52 +03:00
# endif /* #else #ifdef CONFIG_PREEMPT_RCU */
2018-07-03 00:30:37 +03:00
2018-11-29 21:01:52 +03:00
/**
* synchronize_rcu_expedited - Brute - force RCU grace period
*
* Wait for an RCU grace period , but expedite it . The basic idea is to
* IPI all non - idle non - nohz online CPUs . The IPI handler checks whether
* the CPU is in an RCU critical section , and if so , it sets a flag that
* causes the outermost rcu_read_unlock ( ) to report the quiescent state
* for RCU - preempt or asks the scheduler for help for RCU - sched . On the
* other hand , if the CPU is not in an RCU read - side critical section ,
* the IPI handler reports the quiescent state immediately .
*
2019-07-29 10:55:57 +03:00
* Although this is a great improvement over previous expedited
2018-11-29 21:01:52 +03:00
* implementations , it is still unfriendly to real - time workloads , so is
* thus not recommended for any sort of common - case code . In fact , if
* you are using synchronize_rcu_expedited ( ) in a loop , please restructure
2019-11-28 01:20:41 +03:00
* your code to batch your updates , and then use a single synchronize_rcu ( )
2018-11-29 21:01:52 +03:00
* instead .
*
* This has the same semantics as ( but is more brutal than ) synchronize_rcu ( ) .
*/
2016-04-16 02:44:07 +03:00
void synchronize_rcu_expedited ( void )
{
2019-06-20 01:42:51 +03:00
bool boottime = ( rcu_scheduler_active = = RCU_SCHEDULER_INIT ) ;
2018-11-29 22:50:04 +03:00
struct rcu_exp_work rew ;
struct rcu_node * rnp ;
unsigned long s ;
2018-07-03 00:30:37 +03:00
RCU_LOCKDEP_WARN ( lock_is_held ( & rcu_bh_lock_map ) | |
lock_is_held ( & rcu_lock_map ) | |
lock_is_held ( & rcu_sched_lock_map ) ,
2018-07-08 04:12:26 +03:00
" Illegal synchronize_rcu_expedited() in RCU read-side critical section " ) ;
2018-07-03 00:30:37 +03:00
2018-11-29 21:01:52 +03:00
/* Is the state is such that the call is a grace period? */
2018-07-03 00:30:37 +03:00
if ( rcu_blocking_is_gp ( ) )
return ;
2018-11-29 22:50:04 +03:00
/* If expedited grace periods are prohibited, fall back to normal. */
if ( rcu_gp_is_normal ( ) ) {
wait_rcu_gp ( call_rcu ) ;
return ;
}
/* Take a snapshot of the sequence number. */
s = rcu_exp_gp_seq_snap ( ) ;
if ( exp_funnel_lock ( s ) )
return ; /* Someone else did our work for us. */
/* Ensure that load happens before action based on it. */
2019-06-20 01:42:51 +03:00
if ( unlikely ( boottime ) ) {
2018-11-29 22:50:04 +03:00
/* Direct call during scheduler init and early_initcalls(). */
rcu_exp_sel_wait_wake ( s ) ;
} else {
/* Marshall arguments & schedule the expedited grace period. */
rew . rew_s = s ;
INIT_WORK_ONSTACK ( & rew . rew_work , wait_rcu_exp_gp ) ;
queue_work ( rcu_gp_wq , & rew . rew_work ) ;
}
/* Wait for expedited grace period to complete. */
rnp = rcu_get_root ( ) ;
wait_event ( rnp - > exp_wq [ rcu_seq_ctr ( s ) & 0x3 ] ,
sync_exp_work_done ( s ) ) ;
smp_mb ( ) ; /* Workqueue actions happen before return. */
/* Let the next expedited grace period start. */
mutex_unlock ( & rcu_state . exp_mutex ) ;
2019-06-20 01:42:51 +03:00
if ( likely ( ! boottime ) )
destroy_work_on_stack ( & rew . rew_work ) ;
2016-04-16 02:44:07 +03:00
}
EXPORT_SYMBOL_GPL ( synchronize_rcu_expedited ) ;