2019-01-17 10:13:19 -08:00
/* SPDX-License-Identifier: GPL-2.0+ */
2017-05-02 10:31:18 +02:00
/*
2017-05-02 06:30:12 -07:00
* RCU segmented callback lists , internal - to - rcu header file
2017-05-02 10:31:18 +02:00
*
* Copyright IBM Corporation , 2017
*
2019-01-17 10:13:19 -08:00
* Authors : Paul E . McKenney < paulmck @ linux . ibm . com >
2017-05-02 10:31:18 +02:00
*/
# include <linux/rcu_segcblist.h>
2019-07-01 17:36:53 -07:00
/* Return number of callbacks in the specified callback list. */
static inline long rcu_cblist_n_cbs ( struct rcu_cblist * rclp )
{
return READ_ONCE ( rclp - > len ) ;
}
2017-05-02 06:30:12 -07:00
void rcu_cblist_init ( struct rcu_cblist * rclp ) ;
rcu/nocb: Add bypass callback queueing
Use of the rcu_data structure's segmented ->cblist for no-CBs CPUs
takes advantage of unrelated grace periods, thus reducing the memory
footprint in the face of floods of call_rcu() invocations. However,
the ->cblist field is a more-complex rcu_segcblist structure which must
be protected via locking. Even though there are only three entities
which can acquire this lock (the CPU invoking call_rcu(), the no-CBs
grace-period kthread, and the no-CBs callbacks kthread), the contention
on this lock is excessive under heavy stress.
This commit therefore greatly reduces contention by provisioning
an rcu_cblist structure field named ->nocb_bypass within the
rcu_data structure. Each no-CBs CPU is permitted only a limited
number of enqueues onto the ->cblist per jiffy, controlled by a new
nocb_nobypass_lim_per_jiffy kernel boot parameter that defaults to
about 16 enqueues per millisecond (16 * 1000 / HZ). When that limit is
exceeded, the CPU instead enqueues onto the new ->nocb_bypass.
The ->nocb_bypass is flushed into the ->cblist every jiffy or when
the number of callbacks on ->nocb_bypass exceeds qhimark, whichever
happens first. During call_rcu() floods, this flushing is carried out
by the CPU during the course of its call_rcu() invocations. However,
a CPU could simply stop invoking call_rcu() at any time. The no-CBs
grace-period kthread therefore carries out less-aggressive flushing
(every few jiffies or when the number of callbacks on ->nocb_bypass
exceeds (2 * qhimark), whichever comes first). This means that the
no-CBs grace-period kthread cannot be permitted to do unbounded waits
while there are callbacks on ->nocb_bypass. A ->nocb_bypass_timer is
used to provide the needed wakeups.
[ paulmck: Apply Coverity feedback reported by Colin Ian King. ]
Signed-off-by: Paul E. McKenney <paulmck@linux.ibm.com>
2019-07-02 16:03:33 -07:00
void rcu_cblist_enqueue ( struct rcu_cblist * rclp , struct rcu_head * rhp ) ;
void rcu_cblist_flush_enqueue ( struct rcu_cblist * drclp ,
struct rcu_cblist * srclp ,
struct rcu_head * rhp ) ;
2017-05-02 06:30:12 -07:00
struct rcu_head * rcu_cblist_dequeue ( struct rcu_cblist * rclp ) ;
2017-05-02 10:31:18 +02:00
/*
* Is the specified rcu_segcblist structure empty ?
*
* But careful ! The fact that the - > head field is NULL does not
* necessarily imply that there are no callbacks associated with
* this structure . When callbacks are being invoked , they are
* removed as a group . If callback invocation must be preempted ,
* the remaining callbacks will be added back to the list . Either
* way , the counts are updated later .
*
* So it is often the case that rcu_segcblist_n_cbs ( ) should be used
* instead .
*/
static inline bool rcu_segcblist_empty ( struct rcu_segcblist * rsclp )
{
2019-05-13 15:57:50 -07:00
return ! READ_ONCE ( rsclp - > head ) ;
2017-05-02 10:31:18 +02:00
}
/* Return number of callbacks in segmented callback list. */
static inline long rcu_segcblist_n_cbs ( struct rcu_segcblist * rsclp )
{
2019-07-01 17:36:53 -07:00
# ifdef CONFIG_RCU_NOCB_CPU
return atomic_long_read ( & rsclp - > len ) ;
# else
2017-05-02 10:31:18 +02:00
return READ_ONCE ( rsclp - > len ) ;
2019-07-01 17:36:53 -07:00
# endif
2017-05-02 10:31:18 +02:00
}
/*
* Is the specified rcu_segcblist enabled , for example , not corresponding
2019-05-14 09:50:49 -07:00
* to an offline CPU ?
2017-05-02 10:31:18 +02:00
*/
static inline bool rcu_segcblist_is_enabled ( struct rcu_segcblist * rsclp )
{
2019-04-12 12:34:41 -07:00
return rsclp - > enabled ;
2017-05-02 10:31:18 +02:00
}
2019-04-12 15:58:34 -07:00
/* Is the specified rcu_segcblist offloaded? */
static inline bool rcu_segcblist_is_offloaded ( struct rcu_segcblist * rsclp )
{
return rsclp - > offloaded ;
}
2017-05-02 10:31:18 +02:00
/*
* Are all segments following the specified segment of the specified
* rcu_segcblist structure empty of callbacks ? ( The specified
* segment might well contain callbacks . )
*/
static inline bool rcu_segcblist_restempty ( struct rcu_segcblist * rsclp , int seg )
{
2019-05-13 14:36:11 -07:00
return ! READ_ONCE ( * READ_ONCE ( rsclp - > tails [ seg ] ) ) ;
2017-05-02 10:31:18 +02:00
}
rcu/nocb: Add bypass callback queueing
Use of the rcu_data structure's segmented ->cblist for no-CBs CPUs
takes advantage of unrelated grace periods, thus reducing the memory
footprint in the face of floods of call_rcu() invocations. However,
the ->cblist field is a more-complex rcu_segcblist structure which must
be protected via locking. Even though there are only three entities
which can acquire this lock (the CPU invoking call_rcu(), the no-CBs
grace-period kthread, and the no-CBs callbacks kthread), the contention
on this lock is excessive under heavy stress.
This commit therefore greatly reduces contention by provisioning
an rcu_cblist structure field named ->nocb_bypass within the
rcu_data structure. Each no-CBs CPU is permitted only a limited
number of enqueues onto the ->cblist per jiffy, controlled by a new
nocb_nobypass_lim_per_jiffy kernel boot parameter that defaults to
about 16 enqueues per millisecond (16 * 1000 / HZ). When that limit is
exceeded, the CPU instead enqueues onto the new ->nocb_bypass.
The ->nocb_bypass is flushed into the ->cblist every jiffy or when
the number of callbacks on ->nocb_bypass exceeds qhimark, whichever
happens first. During call_rcu() floods, this flushing is carried out
by the CPU during the course of its call_rcu() invocations. However,
a CPU could simply stop invoking call_rcu() at any time. The no-CBs
grace-period kthread therefore carries out less-aggressive flushing
(every few jiffies or when the number of callbacks on ->nocb_bypass
exceeds (2 * qhimark), whichever comes first). This means that the
no-CBs grace-period kthread cannot be permitted to do unbounded waits
while there are callbacks on ->nocb_bypass. A ->nocb_bypass_timer is
used to provide the needed wakeups.
[ paulmck: Apply Coverity feedback reported by Colin Ian King. ]
Signed-off-by: Paul E. McKenney <paulmck@linux.ibm.com>
2019-07-02 16:03:33 -07:00
void rcu_segcblist_inc_len ( struct rcu_segcblist * rsclp ) ;
2017-05-02 06:30:12 -07:00
void rcu_segcblist_init ( struct rcu_segcblist * rsclp ) ;
void rcu_segcblist_disable ( struct rcu_segcblist * rsclp ) ;
2019-04-12 15:58:34 -07:00
void rcu_segcblist_offload ( struct rcu_segcblist * rsclp ) ;
2017-05-02 06:30:12 -07:00
bool rcu_segcblist_ready_cbs ( struct rcu_segcblist * rsclp ) ;
bool rcu_segcblist_pend_cbs ( struct rcu_segcblist * rsclp ) ;
struct rcu_head * rcu_segcblist_first_cb ( struct rcu_segcblist * rsclp ) ;
struct rcu_head * rcu_segcblist_first_pend_cb ( struct rcu_segcblist * rsclp ) ;
2019-05-15 09:56:40 -07:00
bool rcu_segcblist_nextgp ( struct rcu_segcblist * rsclp , unsigned long * lp ) ;
2017-05-02 06:30:12 -07:00
void rcu_segcblist_enqueue ( struct rcu_segcblist * rsclp ,
2019-08-30 12:36:32 -04:00
struct rcu_head * rhp ) ;
2017-05-02 06:30:12 -07:00
bool rcu_segcblist_entrain ( struct rcu_segcblist * rsclp ,
2019-08-30 12:36:32 -04:00
struct rcu_head * rhp ) ;
2017-05-02 06:30:12 -07:00
void rcu_segcblist_extract_count ( struct rcu_segcblist * rsclp ,
struct rcu_cblist * rclp ) ;
void rcu_segcblist_extract_done_cbs ( struct rcu_segcblist * rsclp ,
struct rcu_cblist * rclp ) ;
void rcu_segcblist_extract_pend_cbs ( struct rcu_segcblist * rsclp ,
struct rcu_cblist * rclp ) ;
void rcu_segcblist_insert_count ( struct rcu_segcblist * rsclp ,
struct rcu_cblist * rclp ) ;
void rcu_segcblist_insert_done_cbs ( struct rcu_segcblist * rsclp ,
struct rcu_cblist * rclp ) ;
void rcu_segcblist_insert_pend_cbs ( struct rcu_segcblist * rsclp ,
struct rcu_cblist * rclp ) ;
void rcu_segcblist_advance ( struct rcu_segcblist * rsclp , unsigned long seq ) ;
bool rcu_segcblist_accelerate ( struct rcu_segcblist * rsclp , unsigned long seq ) ;
2017-06-27 07:44:06 -07:00
void rcu_segcblist_merge ( struct rcu_segcblist * dst_rsclp ,
struct rcu_segcblist * src_rsclp ) ;
