iv/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rcu: Make __call_rcu() handle invocation from idle

Browse Source 
Although __call_rcu() is handled correctly when called from a momentary
non-idle period, if it is called on a CPU that RCU believes to be idle
on RCU_FAST_NO_HZ kernels, the callback might be indefinitely postponed.
This commit therefore ensures that RCU is aware of the new callback and
has a chance to force the CPU out of dyntick-idle mode when a new callback
is posted.

Reported-by: Frederic Weisbecker <fweisbec@gmail.com>
Signed-off-by: Paul E. McKenney <paul.mckenney@linaro.org>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
This commit is contained in:
Paul E. McKenney 2012-05-22 22:10:24 -07:00 committed by Paul E. McKenney
parent 28f5c693d0
commit 62fde6edf1
2 changed files with 13 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
include/linux/rcupdate.h
View File

@ -256,6 +256,10 @@ static inline void destroy_rcu_head_on_stack(struct rcu_head *head)
} }
#endif /* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */ #endif /* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
#if defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_SMP)
extern int rcu_is_cpu_idle(void);
#endif /* #if defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_SMP) */
#if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_PROVE_RCU) #if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_PROVE_RCU)
bool rcu_lockdep_current_cpu_online(void); bool rcu_lockdep_current_cpu_online(void);
#else /* #if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_PROVE_RCU) */ #else /* #if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_PROVE_RCU) */
@ -267,15 +271,6 @@ static inline bool rcu_lockdep_current_cpu_online(void)
#ifdef CONFIG_DEBUG_LOCK_ALLOC #ifdef CONFIG_DEBUG_LOCK_ALLOC
#ifdef CONFIG_PROVE_RCU
extern int rcu_is_cpu_idle(void);
#else /* !CONFIG_PROVE_RCU */
static inline int rcu_is_cpu_idle(void)
{
return 0;
}
#endif /* else !CONFIG_PROVE_RCU */
static inline void rcu_lock_acquire(struct lockdep_map *map) static inline void rcu_lock_acquire(struct lockdep_map *map)
{ {
lock_acquire(map, 0, 0, 2, 1, NULL, _THIS_IP_); lock_acquire(map, 0, 0, 2, 1, NULL, _THIS_IP_);

15
kernel/rcutree.c
View File

@ -585,8 +585,6 @@ void rcu_nmi_exit(void)
WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1); WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1);
} }
#ifdef CONFIG_PROVE_RCU
/** /**
* rcu_is_cpu_idle - see if RCU thinks that the current CPU is idle * rcu_is_cpu_idle - see if RCU thinks that the current CPU is idle
* *
@ -604,7 +602,7 @@ int rcu_is_cpu_idle(void)
} }
EXPORT_SYMBOL(rcu_is_cpu_idle); EXPORT_SYMBOL(rcu_is_cpu_idle);
#ifdef CONFIG_HOTPLUG_CPU #if defined(CONFIG_PROVE_RCU) && defined(CONFIG_HOTPLUG_CPU)
/* /*
* Is the current CPU online? Disable preemption to avoid false positives * Is the current CPU online? Disable preemption to avoid false positives
@ -645,9 +643,7 @@ bool rcu_lockdep_current_cpu_online(void)
} }
EXPORT_SYMBOL_GPL(rcu_lockdep_current_cpu_online); EXPORT_SYMBOL_GPL(rcu_lockdep_current_cpu_online);
#endif /* #ifdef CONFIG_HOTPLUG_CPU */ #endif /* #if defined(CONFIG_PROVE_RCU) && defined(CONFIG_HOTPLUG_CPU) */
#endif /* #ifdef CONFIG_PROVE_RCU */
/** /**
* rcu_is_cpu_rrupt_from_idle - see if idle or immediately interrupted from idle * rcu_is_cpu_rrupt_from_idle - see if idle or immediately interrupted from idle
@ -1904,6 +1900,13 @@ __call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu),
else else
trace_rcu_callback(rsp->name, head, rdp->qlen_lazy, rdp->qlen); trace_rcu_callback(rsp->name, head, rdp->qlen_lazy, rdp->qlen);
/*
* If called from an extended quiescent state, invoke the RCU
* core in order to force a re-evaluation of RCU's idleness.
*/
if (rcu_is_cpu_idle())
invoke_rcu_core();
/* If interrupts were disabled, don't dive into RCU core. */ /* If interrupts were disabled, don't dive into RCU core. */
if (irqs_disabled_flags(flags)) { if (irqs_disabled_flags(flags)) {
local_irq_restore(flags); local_irq_restore(flags);