2019-05-19 15:08:55 +03:00
// SPDX-License-Identifier: GPL-2.0-only
2010-10-14 10:01:34 +04:00
/*
2015-11-16 13:08:45 +03:00
* Copyright ( C ) 2010 Red Hat , Inc . , Peter Zijlstra
2010-10-14 10:01:34 +04:00
*
* Provides a framework for enqueueing and running callbacks from hardirq
* context . The enqueueing is NMI - safe .
*/
2012-04-02 00:38:37 +04:00
# include <linux/bug.h>
2010-10-14 10:01:34 +04:00
# include <linux/kernel.h>
2011-05-23 22:51:41 +04:00
# include <linux/export.h>
2010-10-14 10:01:34 +04:00
# include <linux/irq_work.h>
2011-07-18 21:03:04 +04:00
# include <linux/percpu.h>
2010-10-14 10:01:34 +04:00
# include <linux/hardirq.h>
2012-04-11 20:21:39 +04:00
# include <linux/irqflags.h>
2012-10-20 00:43:41 +04:00
# include <linux/sched.h>
# include <linux/tick.h>
2012-11-15 20:34:21 +04:00
# include <linux/cpu.h>
# include <linux/notifier.h>
2014-05-08 03:37:48 +04:00
# include <linux/smp.h>
2011-07-18 21:03:04 +04:00
# include <asm/processor.h>
2010-10-14 10:01:34 +04:00
2014-05-23 20:10:21 +04:00
static DEFINE_PER_CPU ( struct llist_head , raised_list ) ;
static DEFINE_PER_CPU ( struct llist_head , lazy_list ) ;
2010-10-14 10:01:34 +04:00
/*
* Claim the entry so that no one else will poke at it .
*/
2011-09-08 10:00:46 +04:00
static bool irq_work_claim ( struct irq_work * work )
2010-10-14 10:01:34 +04:00
{
2019-11-08 19:08:56 +03:00
int oflags ;
2010-10-14 10:01:34 +04:00
2019-11-08 19:08:56 +03:00
oflags = atomic_fetch_or ( IRQ_WORK_CLAIMED , & work - > flags ) ;
irq_work: Fix racy check on work pending flag
Work claiming wants to be SMP-safe.
And by the time we try to claim a work, if it is already executing
concurrently on another CPU, we want to succeed the claiming and queue
the work again because the other CPU may have missed the data we wanted
to handle in our work if it's about to complete there.
This scenario is summarized below:
CPU 1 CPU 2
----- -----
(flags = 0)
cmpxchg(flags, 0, IRQ_WORK_FLAGS)
(flags = 3)
[...]
xchg(flags, IRQ_WORK_BUSY)
(flags = 2)
func()
if (flags & IRQ_WORK_PENDING)
(not true)
cmpxchg(flags, flags, IRQ_WORK_FLAGS)
(flags = 3)
[...]
cmpxchg(flags, IRQ_WORK_BUSY, 0);
(fail, pending on CPU 2)
This state machine is synchronized using [cmp]xchg() on the flags.
As such, the early IRQ_WORK_PENDING check in CPU 2 above is racy.
By the time we check it, we may be dealing with a stale value because
we aren't using an atomic accessor. As a result, CPU 2 may "see"
that the work is still pending on another CPU while it may be
actually completing the work function exection already, leaving
our data unprocessed.
To fix this, we start by speculating about the value we wish to be
in the work->flags but we only make any conclusion after the value
returned by the cmpxchg() call that either claims the work or let
the current owner handle the pending work for us.
Changelog-heavily-inspired-by: Steven Rostedt <rostedt@goodmis.org>
Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com>
Acked-by: Steven Rostedt <rostedt@goodmis.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: Anish Kumar <anish198519851985@gmail.com>
2012-10-27 17:21:36 +04:00
/*
2019-11-08 19:08:56 +03:00
* If the work is already pending , no need to raise the IPI .
2019-11-08 19:08:57 +03:00
* The pairing atomic_fetch_andnot ( ) in irq_work_run ( ) makes sure
2019-11-08 19:08:56 +03:00
* everything we did before is visible .
irq_work: Fix racy check on work pending flag
Work claiming wants to be SMP-safe.
And by the time we try to claim a work, if it is already executing
concurrently on another CPU, we want to succeed the claiming and queue
the work again because the other CPU may have missed the data we wanted
to handle in our work if it's about to complete there.
This scenario is summarized below:
CPU 1 CPU 2
----- -----
(flags = 0)
cmpxchg(flags, 0, IRQ_WORK_FLAGS)
(flags = 3)
[...]
xchg(flags, IRQ_WORK_BUSY)
(flags = 2)
func()
if (flags & IRQ_WORK_PENDING)
(not true)
cmpxchg(flags, flags, IRQ_WORK_FLAGS)
(flags = 3)
[...]
cmpxchg(flags, IRQ_WORK_BUSY, 0);
(fail, pending on CPU 2)
This state machine is synchronized using [cmp]xchg() on the flags.
As such, the early IRQ_WORK_PENDING check in CPU 2 above is racy.
By the time we check it, we may be dealing with a stale value because
we aren't using an atomic accessor. As a result, CPU 2 may "see"
that the work is still pending on another CPU while it may be
actually completing the work function exection already, leaving
our data unprocessed.
To fix this, we start by speculating about the value we wish to be
in the work->flags but we only make any conclusion after the value
returned by the cmpxchg() call that either claims the work or let
the current owner handle the pending work for us.
Changelog-heavily-inspired-by: Steven Rostedt <rostedt@goodmis.org>
Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com>
Acked-by: Steven Rostedt <rostedt@goodmis.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: Anish Kumar <anish198519851985@gmail.com>
2012-10-27 17:21:36 +04:00
*/
2019-11-08 19:08:56 +03:00
if ( oflags & IRQ_WORK_PENDING )
return false ;
2010-10-14 10:01:34 +04:00
return true ;
}
void __weak arch_irq_work_raise ( void )
{
/*
* Lame architectures will get the timer tick callback
*/
}
2019-04-09 12:34:03 +03:00
/* Enqueue on current CPU, work must already be claimed and preempt disabled */
static void __irq_work_queue_local ( struct irq_work * work )
2014-05-08 03:37:48 +04:00
{
2019-04-09 12:34:03 +03:00
/* If the work is "lazy", handle it from next tick if any */
2019-11-08 19:08:55 +03:00
if ( atomic_read ( & work - > flags ) & IRQ_WORK_LAZY ) {
2019-04-09 12:34:03 +03:00
if ( llist_add ( & work - > llnode , this_cpu_ptr ( & lazy_list ) ) & &
tick_nohz_tick_stopped ( ) )
arch_irq_work_raise ( ) ;
} else {
if ( llist_add ( & work - > llnode , this_cpu_ptr ( & raised_list ) ) )
arch_irq_work_raise ( ) ;
}
}
2014-05-08 03:37:48 +04:00
2019-04-09 12:34:03 +03:00
/* Enqueue the irq work @work on the current CPU */
bool irq_work_queue ( struct irq_work * work )
{
2014-05-08 03:37:48 +04:00
/* Only queue if not already pending */
if ( ! irq_work_claim ( work ) )
return false ;
2019-04-09 12:34:03 +03:00
/* Queue the entry and raise the IPI if needed. */
preempt_disable ( ) ;
__irq_work_queue_local ( work ) ;
preempt_enable ( ) ;
2017-08-18 20:59:16 +03:00
2014-05-08 03:37:48 +04:00
return true ;
}
2019-04-09 12:34:03 +03:00
EXPORT_SYMBOL_GPL ( irq_work_queue ) ;
2014-05-08 03:37:48 +04:00
2019-04-09 12:34:03 +03:00
/*
* Enqueue the irq_work @ work on @ cpu unless it ' s already pending
* somewhere .
*
* Can be re - enqueued while the callback is still in progress .
*/
bool irq_work_queue_on ( struct irq_work * work , int cpu )
2010-10-14 10:01:34 +04:00
{
2019-04-09 12:34:03 +03:00
# ifndef CONFIG_SMP
return irq_work_queue ( work ) ;
# else /* CONFIG_SMP: */
/* All work should have been flushed before going offline */
WARN_ON_ONCE ( cpu_is_offline ( cpu ) ) ;
2013-02-04 01:08:23 +04:00
/* Only queue if not already pending */
if ( ! irq_work_claim ( work ) )
2014-02-11 19:01:16 +04:00
return false ;
2013-02-04 01:08:23 +04:00
2010-12-14 19:28:45 +03:00
preempt_disable ( ) ;
2019-04-09 12:34:03 +03:00
if ( cpu ! = smp_processor_id ( ) ) {
/* Arch remote IPI send/receive backend aren't NMI safe */
WARN_ON_ONCE ( in_nmi ( ) ) ;
if ( llist_add ( & work - > llnode , & per_cpu ( raised_list , cpu ) ) )
arch_send_call_function_single_ipi ( cpu ) ;
2014-05-23 20:10:21 +04:00
} else {
2019-04-09 12:34:03 +03:00
__irq_work_queue_local ( work ) ;
2012-10-20 00:43:41 +04:00
}
2010-12-14 19:28:45 +03:00
preempt_enable ( ) ;
2014-02-11 19:01:16 +04:00
return true ;
2019-04-09 12:34:03 +03:00
# endif /* CONFIG_SMP */
2010-10-14 10:01:34 +04:00
}
2019-04-09 12:34:03 +03:00
2010-10-14 10:01:34 +04:00
2012-11-08 00:03:07 +04:00
bool irq_work_needs_cpu ( void )
{
2014-05-23 20:10:21 +04:00
struct llist_head * raised , * lazy ;
2012-11-08 00:03:07 +04:00
2014-08-17 21:30:25 +04:00
raised = this_cpu_ptr ( & raised_list ) ;
lazy = this_cpu_ptr ( & lazy_list ) ;
2014-08-16 20:37:19 +04:00
if ( llist_empty ( raised ) | | arch_irq_work_has_interrupt ( ) )
if ( llist_empty ( lazy ) )
return false ;
2012-11-08 00:03:07 +04:00
2012-11-15 21:52:44 +04:00
/* All work should have been flushed before going offline */
WARN_ON_ONCE ( cpu_is_offline ( smp_processor_id ( ) ) ) ;
2012-11-08 00:03:07 +04:00
return true ;
}
2014-05-23 20:10:21 +04:00
static void irq_work_run_list ( struct llist_head * list )
2010-10-14 10:01:34 +04:00
{
2017-11-12 15:02:51 +03:00
struct irq_work * work , * tmp ;
2011-09-08 10:00:46 +04:00
struct llist_node * llnode ;
2010-10-14 10:01:34 +04:00
2014-05-23 20:10:21 +04:00
BUG_ON ( ! irqs_disabled ( ) ) ;
2012-10-20 00:43:41 +04:00
2014-05-23 20:10:21 +04:00
if ( llist_empty ( list ) )
2010-10-14 10:01:34 +04:00
return ;
2014-05-23 20:10:21 +04:00
llnode = llist_del_all ( list ) ;
2017-11-12 15:02:51 +03:00
llist_for_each_entry_safe ( work , tmp , llnode , llnode ) {
2019-11-08 19:08:57 +03:00
int flags ;
2010-10-14 10:01:34 +04:00
/*
2011-09-08 10:00:46 +04:00
* Clear the PENDING bit , after this point the @ work
2010-10-14 10:01:34 +04:00
* can be re - used .
irq_work: Fix racy IRQ_WORK_BUSY flag setting
The IRQ_WORK_BUSY flag is set right before we execute the
work. Once this flag value is set, the work enters a
claimable state again.
So if we have specific data to compute in our work, we ensure it's
either handled by another CPU or locally by enqueuing the work again.
This state machine is guanranteed by atomic operations on the flags.
So when we set IRQ_WORK_BUSY without using an xchg-like operation,
we break this guarantee as in the following summarized scenario:
CPU 1 CPU 2
----- -----
(flags = 0)
old_flags = flags;
(flags = 0)
cmpxchg(flags, old_flags,
old_flags | IRQ_WORK_FLAGS)
(flags = 3)
[...]
flags = IRQ_WORK_BUSY
(flags = 2)
func()
(sees flags = 3)
cmpxchg(flags, old_flags,
old_flags | IRQ_WORK_FLAGS)
(give up)
cmpxchg(flags, 2, 0);
(flags = 0)
CPU 1 claims a work and executes it, so it sets IRQ_WORK_BUSY and
the work is again in a claimable state. Now CPU 2 has new data to process
and try to claim that work but it may see a stale value of the flags
and think the work is still pending somewhere that will handle our data.
This is because CPU 1 doesn't set IRQ_WORK_BUSY atomically.
As a result, the data expected to be handle by CPU 2 won't get handled.
To fix this, use xchg() to set IRQ_WORK_BUSY, this way we ensure the CPU 2
will see the correct value with cmpxchg() using the expected ordering.
Changelog-heavily-inspired-by: Steven Rostedt <rostedt@goodmis.org>
Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com>
Acked-by: Steven Rostedt <rostedt@goodmis.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: Anish Kumar <anish198519851985@gmail.com>
2012-10-30 16:33:54 +04:00
* Make it immediately visible so that other CPUs trying
* to claim that work don ' t rely on us to handle their data
* while we are in the middle of the func .
2010-10-14 10:01:34 +04:00
*/
2019-11-08 19:08:57 +03:00
flags = atomic_fetch_andnot ( IRQ_WORK_PENDING , & work - > flags ) ;
2012-10-20 00:43:41 +04:00
2011-09-08 10:00:46 +04:00
work - > func ( work ) ;
2010-10-14 10:01:34 +04:00
/*
* Clear the BUSY bit and return to the free state if
* no - one else claimed it meanwhile .
*/
2019-11-08 19:08:55 +03:00
( void ) atomic_cmpxchg ( & work - > flags , flags , flags & ~ IRQ_WORK_BUSY ) ;
2010-10-14 10:01:34 +04:00
}
}
2012-11-15 20:34:21 +04:00
/*
2014-06-25 09:13:07 +04:00
* hotplug calls this through :
* hotplug_cfd ( ) - > flush_smp_call_function_queue ( )
2012-11-15 20:34:21 +04:00
*/
void irq_work_run ( void )
{
2014-08-17 21:30:25 +04:00
irq_work_run_list ( this_cpu_ptr ( & raised_list ) ) ;
irq_work_run_list ( this_cpu_ptr ( & lazy_list ) ) ;
2012-11-15 20:34:21 +04:00
}
2010-10-14 10:01:34 +04:00
EXPORT_SYMBOL_GPL ( irq_work_run ) ;
2014-08-16 20:37:19 +04:00
void irq_work_tick ( void )
{
2014-10-27 18:49:45 +03:00
struct llist_head * raised = this_cpu_ptr ( & raised_list ) ;
2014-08-16 20:37:19 +04:00
if ( ! llist_empty ( raised ) & & ! arch_irq_work_has_interrupt ( ) )
irq_work_run_list ( raised ) ;
2014-10-27 18:49:45 +03:00
irq_work_run_list ( this_cpu_ptr ( & lazy_list ) ) ;
2014-08-16 20:37:19 +04:00
}
2010-10-14 10:01:34 +04:00
/*
* Synchronize against the irq_work @ entry , ensures the entry is not
* currently in use .
*/
2011-09-08 10:00:46 +04:00
void irq_work_sync ( struct irq_work * work )
2010-10-14 10:01:34 +04:00
{
2017-11-06 18:01:26 +03:00
lockdep_assert_irqs_enabled ( ) ;
2010-10-14 10:01:34 +04:00
2019-11-08 19:08:55 +03:00
while ( atomic_read ( & work - > flags ) & IRQ_WORK_BUSY )
2010-10-14 10:01:34 +04:00
cpu_relax ( ) ;
}
EXPORT_SYMBOL_GPL ( irq_work_sync ) ;