2005-04-17 02:20:36 +04:00
/*
2010-09-10 18:51:36 +04:00
* kernel / workqueue . c - generic async execution with shared worker pool
2005-04-17 02:20:36 +04:00
*
2010-09-10 18:51:36 +04:00
* Copyright ( C ) 2002 Ingo Molnar
2005-04-17 02:20:36 +04:00
*
2010-09-10 18:51:36 +04:00
* Derived from the taskqueue / keventd code by :
* David Woodhouse < dwmw2 @ infradead . org >
* Andrew Morton
* Kai Petzke < wpp @ marie . physik . tu - berlin . de >
* Theodore Ts ' o < tytso @ mit . edu >
2005-04-17 02:20:36 +04:00
*
2010-09-10 18:51:36 +04:00
* Made to use alloc_percpu by Christoph Lameter .
2005-04-17 02:20:36 +04:00
*
2010-09-10 18:51:36 +04:00
* Copyright ( C ) 2010 SUSE Linux Products GmbH
* Copyright ( C ) 2010 Tejun Heo < tj @ kernel . org >
2005-10-31 02:01:59 +03:00
*
2010-09-10 18:51:36 +04:00
* This is the generic async execution mechanism . Work items as are
* executed in process context . The worker pool is shared and
* automatically managed . There is one worker pool for each CPU and
* one extra for works which are better served by workers which are
* not bound to any specific CPU .
*
* Please read Documentation / workqueue . txt for details .
2005-04-17 02:20:36 +04:00
*/
2011-05-23 22:51:41 +04:00
# include <linux/export.h>
2005-04-17 02:20:36 +04:00
# include <linux/kernel.h>
# include <linux/sched.h>
# include <linux/init.h>
# include <linux/signal.h>
# include <linux/completion.h>
# include <linux/workqueue.h>
# include <linux/slab.h>
# include <linux/cpu.h>
# include <linux/notifier.h>
# include <linux/kthread.h>
2006-02-23 21:43:43 +03:00
# include <linux/hardirq.h>
2006-10-11 12:21:26 +04:00
# include <linux/mempolicy.h>
2006-12-07 07:34:49 +03:00
# include <linux/freezer.h>
2006-12-07 07:37:26 +03:00
# include <linux/kallsyms.h>
# include <linux/debug_locks.h>
2007-10-19 10:39:55 +04:00
# include <linux/lockdep.h>
2010-06-29 12:07:11 +04:00
# include <linux/idr.h>
2010-06-29 12:07:14 +04:00
# include "workqueue_sched.h"
2005-04-17 02:20:36 +04:00
2010-06-29 12:07:12 +04:00
enum {
2010-06-29 12:07:12 +04:00
/* global_cwq flags */
2010-06-29 12:07:14 +04:00
GCWQ_MANAGE_WORKERS = 1 < < 0 , /* need to manage workers */
GCWQ_MANAGING_WORKERS = 1 < < 1 , /* managing workers */
GCWQ_DISASSOCIATED = 1 < < 2 , /* cpu can't serve workers */
2010-06-29 12:07:12 +04:00
GCWQ_FREEZING = 1 < < 3 , /* freeze in progress */
2010-06-29 12:07:14 +04:00
GCWQ_HIGHPRI_PENDING = 1 < < 4 , /* highpri works on queue */
2010-06-29 12:07:12 +04:00
2010-06-29 12:07:12 +04:00
/* worker flags */
WORKER_STARTED = 1 < < 0 , /* started */
WORKER_DIE = 1 < < 1 , /* die die die */
WORKER_IDLE = 1 < < 2 , /* is idle */
2010-06-29 12:07:14 +04:00
WORKER_PREP = 1 < < 3 , /* preparing to run works */
2010-06-29 12:07:12 +04:00
WORKER_ROGUE = 1 < < 4 , /* not bound to any cpu */
2010-06-29 12:07:14 +04:00
WORKER_REBIND = 1 < < 5 , /* mom is home, come back */
2010-06-29 12:07:15 +04:00
WORKER_CPU_INTENSIVE = 1 < < 6 , /* cpu intensive */
2010-07-02 12:03:51 +04:00
WORKER_UNBOUND = 1 < < 7 , /* worker is unbound */
2010-06-29 12:07:14 +04:00
2010-06-29 12:07:15 +04:00
WORKER_NOT_RUNNING = WORKER_PREP | WORKER_ROGUE | WORKER_REBIND |
2010-07-02 12:03:51 +04:00
WORKER_CPU_INTENSIVE | WORKER_UNBOUND ,
2010-06-29 12:07:12 +04:00
/* gcwq->trustee_state */
TRUSTEE_START = 0 , /* start */
TRUSTEE_IN_CHARGE = 1 , /* trustee in charge of gcwq */
TRUSTEE_BUTCHER = 2 , /* butcher workers */
TRUSTEE_RELEASE = 3 , /* release workers */
TRUSTEE_DONE = 4 , /* trustee is done */
2010-06-29 12:07:12 +04:00
BUSY_WORKER_HASH_ORDER = 6 , /* 64 pointers */
BUSY_WORKER_HASH_SIZE = 1 < < BUSY_WORKER_HASH_ORDER ,
BUSY_WORKER_HASH_MASK = BUSY_WORKER_HASH_SIZE - 1 ,
2010-06-29 12:07:12 +04:00
2010-06-29 12:07:14 +04:00
MAX_IDLE_WORKERS_RATIO = 4 , /* 1/4 of busy can be idle */
IDLE_WORKER_TIMEOUT = 300 * HZ , /* keep idle ones for 5 mins */
2011-02-16 20:10:19 +03:00
MAYDAY_INITIAL_TIMEOUT = HZ / 100 > = 2 ? HZ / 100 : 2 ,
/* call for help after 10ms
( min two ticks ) */
2010-06-29 12:07:14 +04:00
MAYDAY_INTERVAL = HZ / 10 , /* and then every 100ms */
CREATE_COOLDOWN = HZ , /* time to breath after fail */
2010-06-29 12:07:12 +04:00
TRUSTEE_COOLDOWN = HZ / 10 , /* for trustee draining */
2010-06-29 12:07:14 +04:00
/*
* Rescue workers are used only on emergencies and shared by
* all cpus . Give - 20.
*/
RESCUER_NICE_LEVEL = - 20 ,
2010-06-29 12:07:12 +04:00
} ;
2005-04-17 02:20:36 +04:00
/*
2010-06-29 12:07:10 +04:00
* Structure fields follow one of the following exclusion rules .
*
2010-08-24 16:22:47 +04:00
* I : Modifiable by initialization / destruction paths and read - only for
* everyone else .
2010-06-29 12:07:10 +04:00
*
2010-06-29 12:07:14 +04:00
* P : Preemption protected . Disabling preemption is enough and should
* only be modified and accessed from the local cpu .
*
2010-06-29 12:07:12 +04:00
* L : gcwq - > lock protected . Access with gcwq - > lock held .
2010-06-29 12:07:10 +04:00
*
2010-06-29 12:07:14 +04:00
* X : During normal operation , modification requires gcwq - > lock and
* should be done only from local cpu . Either disabling preemption
* on local cpu or grabbing gcwq - > lock is enough for read access .
2010-07-02 12:03:51 +04:00
* If GCWQ_DISASSOCIATED is set , it ' s identical to L .
2010-06-29 12:07:14 +04:00
*
2010-06-29 12:07:11 +04:00
* F : wq - > flush_mutex protected .
*
2010-06-29 12:07:10 +04:00
* W : workqueue_lock protected .
2005-04-17 02:20:36 +04:00
*/
2010-06-29 12:07:12 +04:00
struct global_cwq ;
2005-04-17 02:20:36 +04:00
2010-06-29 12:07:14 +04:00
/*
* The poor guys doing the actual heavy lifting . All on - duty workers
* are either serving the manager role , on idle list or on busy hash .
*/
2010-06-29 12:07:11 +04:00
struct worker {
2010-06-29 12:07:12 +04:00
/* on idle list while idle, on busy hash table while busy */
union {
struct list_head entry ; /* L: while idle */
struct hlist_node hentry ; /* L: while busy */
} ;
2005-04-17 02:20:36 +04:00
2010-06-29 12:07:11 +04:00
struct work_struct * current_work ; /* L: work being processed */
2010-06-29 12:07:13 +04:00
struct cpu_workqueue_struct * current_cwq ; /* L: current_work's cwq */
2010-06-29 12:07:12 +04:00
struct list_head scheduled ; /* L: scheduled works */
2010-06-29 12:07:11 +04:00
struct task_struct * task ; /* I: worker task */
2010-06-29 12:07:12 +04:00
struct global_cwq * gcwq ; /* I: the associated gcwq */
2010-06-29 12:07:14 +04:00
/* 64 bytes boundary on 64bit, 32 on 32bit */
unsigned long last_active ; /* L: last active timestamp */
unsigned int flags ; /* X: flags */
2010-06-29 12:07:11 +04:00
int id ; /* I: worker id */
2010-06-29 12:07:14 +04:00
struct work_struct rebind_work ; /* L: rebind worker to cpu */
2010-06-29 12:07:11 +04:00
} ;
2010-06-29 12:07:12 +04:00
/*
2010-06-29 12:07:14 +04:00
* Global per - cpu workqueue . There ' s one and only one for each cpu
* and all works are queued and processed here regardless of their
* target workqueues .
2010-06-29 12:07:12 +04:00
*/
struct global_cwq {
spinlock_t lock ; /* the gcwq lock */
2010-06-29 12:07:13 +04:00
struct list_head worklist ; /* L: list of pending works */
2010-06-29 12:07:12 +04:00
unsigned int cpu ; /* I: the associated cpu */
2010-06-29 12:07:12 +04:00
unsigned int flags ; /* L: GCWQ_* flags */
2010-06-29 12:07:12 +04:00
int nr_workers ; /* L: total number of workers */
int nr_idle ; /* L: currently idle ones */
/* workers are chained either in the idle_list or busy_hash */
2010-06-29 12:07:14 +04:00
struct list_head idle_list ; /* X: list of idle workers */
2010-06-29 12:07:12 +04:00
struct hlist_head busy_hash [ BUSY_WORKER_HASH_SIZE ] ;
/* L: hash of busy workers */
2010-06-29 12:07:14 +04:00
struct timer_list idle_timer ; /* L: worker idle timeout */
struct timer_list mayday_timer ; /* L: SOS timer for dworkers */
2010-06-29 12:07:12 +04:00
struct ida worker_ida ; /* L: for worker IDs */
2010-06-29 12:07:12 +04:00
struct task_struct * trustee ; /* L: for gcwq shutdown */
unsigned int trustee_state ; /* L: trustee state */
wait_queue_head_t trustee_wait ; /* trustee wait */
2010-06-29 12:07:14 +04:00
struct worker * first_idle ; /* L: first idle worker */
2010-06-29 12:07:12 +04:00
} ____cacheline_aligned_in_smp ;
2005-04-17 02:20:36 +04:00
/*
2010-06-29 12:07:13 +04:00
* The per - CPU workqueue . The lower WORK_STRUCT_FLAG_BITS of
2010-06-29 12:07:11 +04:00
* work_struct - > data are used for flags and thus cwqs need to be
* aligned at two ' s power of the number of flag bits .
2005-04-17 02:20:36 +04:00
*/
struct cpu_workqueue_struct {
2010-06-29 12:07:12 +04:00
struct global_cwq * gcwq ; /* I: the associated gcwq */
2010-06-29 12:07:10 +04:00
struct workqueue_struct * wq ; /* I: the owning workqueue */
2010-06-29 12:07:11 +04:00
int work_color ; /* L: current color */
int flush_color ; /* L: flushing color */
int nr_in_flight [ WORK_NR_COLORS ] ;
/* L: nr of in_flight works */
2010-06-29 12:07:12 +04:00
int nr_active ; /* L: nr of active works */
2010-06-29 12:07:12 +04:00
int max_active ; /* L: max active works */
2010-06-29 12:07:12 +04:00
struct list_head delayed_works ; /* L: delayed works */
2010-06-29 12:07:11 +04:00
} ;
2005-04-17 02:20:36 +04:00
2010-06-29 12:07:11 +04:00
/*
* Structure used to wait for workqueue flush .
*/
struct wq_flusher {
struct list_head list ; /* F: list of flushers */
int flush_color ; /* F: flush color waiting for */
struct completion done ; /* flush completion */
} ;
2010-07-20 17:59:09 +04:00
/*
* All cpumasks are assumed to be always set on UP and thus can ' t be
* used to determine whether there ' s something to be done .
*/
# ifdef CONFIG_SMP
typedef cpumask_var_t mayday_mask_t ;
# define mayday_test_and_set_cpu(cpu, mask) \
cpumask_test_and_set_cpu ( ( cpu ) , ( mask ) )
# define mayday_clear_cpu(cpu, mask) cpumask_clear_cpu((cpu), (mask))
# define for_each_mayday_cpu(cpu, mask) for_each_cpu((cpu), (mask))
2010-08-31 13:18:34 +04:00
# define alloc_mayday_mask(maskp, gfp) zalloc_cpumask_var((maskp), (gfp))
2010-07-20 17:59:09 +04:00
# define free_mayday_mask(mask) free_cpumask_var((mask))
# else
typedef unsigned long mayday_mask_t ;
# define mayday_test_and_set_cpu(cpu, mask) test_and_set_bit(0, &(mask))
# define mayday_clear_cpu(cpu, mask) clear_bit(0, &(mask))
# define for_each_mayday_cpu(cpu, mask) if ((cpu) = 0, (mask))
# define alloc_mayday_mask(maskp, gfp) true
# define free_mayday_mask(mask) do { } while (0)
# endif
2005-04-17 02:20:36 +04:00
/*
* The externally visible workqueue abstraction is an array of
* per - CPU workqueues :
*/
struct workqueue_struct {
2011-04-05 20:01:44 +04:00
unsigned int flags ; /* W: WQ_* flags */
2010-07-02 12:03:51 +04:00
union {
struct cpu_workqueue_struct __percpu * pcpu ;
struct cpu_workqueue_struct * single ;
unsigned long v ;
} cpu_wq ; /* I: cwq's */
2010-06-29 12:07:10 +04:00
struct list_head list ; /* W: list of all workqueues */
2010-06-29 12:07:11 +04:00
struct mutex flush_mutex ; /* protects wq flushing */
int work_color ; /* F: current work color */
int flush_color ; /* F: current flush color */
atomic_t nr_cwqs_to_flush ; /* flush in progress */
struct wq_flusher * first_flusher ; /* F: first flusher */
struct list_head flusher_queue ; /* F: flush waiters */
struct list_head flusher_overflow ; /* F: flush overflow list */
2010-07-20 17:59:09 +04:00
mayday_mask_t mayday_mask ; /* cpus requesting rescue */
2010-06-29 12:07:14 +04:00
struct worker * rescuer ; /* I: rescue worker */
2011-04-05 20:01:44 +04:00
int nr_drainers ; /* W: drain in progress */
2010-06-29 12:07:14 +04:00
int saved_max_active ; /* W: saved cwq max_active */
2007-10-19 10:39:55 +04:00
# ifdef CONFIG_LOCKDEP
2010-06-29 12:07:10 +04:00
struct lockdep_map lockdep_map ;
2007-10-19 10:39:55 +04:00
# endif
2012-01-11 03:11:35 +04:00
char name [ ] ; /* I: workqueue name */
2005-04-17 02:20:36 +04:00
} ;
2010-06-29 12:07:14 +04:00
struct workqueue_struct * system_wq __read_mostly ;
struct workqueue_struct * system_long_wq __read_mostly ;
struct workqueue_struct * system_nrt_wq __read_mostly ;
2010-07-02 12:03:51 +04:00
struct workqueue_struct * system_unbound_wq __read_mostly ;
2011-02-21 11:52:50 +03:00
struct workqueue_struct * system_freezable_wq __read_mostly ;
2012-03-02 13:51:00 +04:00
struct workqueue_struct * system_nrt_freezable_wq __read_mostly ;
2010-06-29 12:07:14 +04:00
EXPORT_SYMBOL_GPL ( system_wq ) ;
EXPORT_SYMBOL_GPL ( system_long_wq ) ;
EXPORT_SYMBOL_GPL ( system_nrt_wq ) ;
2010-07-02 12:03:51 +04:00
EXPORT_SYMBOL_GPL ( system_unbound_wq ) ;
2011-02-21 11:52:50 +03:00
EXPORT_SYMBOL_GPL ( system_freezable_wq ) ;
2012-03-02 13:51:00 +04:00
EXPORT_SYMBOL_GPL ( system_nrt_freezable_wq ) ;
2010-06-29 12:07:14 +04:00
2010-10-05 12:41:14 +04:00
# define CREATE_TRACE_POINTS
# include <trace/events/workqueue.h>
2010-06-29 12:07:12 +04:00
# define for_each_busy_worker(worker, i, pos, gcwq) \
for ( i = 0 ; i < BUSY_WORKER_HASH_SIZE ; i + + ) \
hlist_for_each_entry ( worker , pos , & gcwq - > busy_hash [ i ] , hentry )
2010-07-02 12:03:51 +04:00
static inline int __next_gcwq_cpu ( int cpu , const struct cpumask * mask ,
unsigned int sw )
{
if ( cpu < nr_cpu_ids ) {
if ( sw & 1 ) {
cpu = cpumask_next ( cpu , mask ) ;
if ( cpu < nr_cpu_ids )
return cpu ;
}
if ( sw & 2 )
return WORK_CPU_UNBOUND ;
}
return WORK_CPU_NONE ;
}
static inline int __next_wq_cpu ( int cpu , const struct cpumask * mask ,
struct workqueue_struct * wq )
{
return __next_gcwq_cpu ( cpu , mask , ! ( wq - > flags & WQ_UNBOUND ) ? 1 : 2 ) ;
}
2010-08-01 13:50:12 +04:00
/*
* CPU iterators
*
* An extra gcwq is defined for an invalid cpu number
* ( WORK_CPU_UNBOUND ) to host workqueues which are not bound to any
* specific CPU . The following iterators are similar to
* for_each_ * _cpu ( ) iterators but also considers the unbound gcwq .
*
* for_each_gcwq_cpu ( ) : possible CPUs + WORK_CPU_UNBOUND
* for_each_online_gcwq_cpu ( ) : online CPUs + WORK_CPU_UNBOUND
* for_each_cwq_cpu ( ) : possible CPUs for bound workqueues ,
* WORK_CPU_UNBOUND for unbound workqueues
*/
2010-07-02 12:03:51 +04:00
# define for_each_gcwq_cpu(cpu) \
for ( ( cpu ) = __next_gcwq_cpu ( - 1 , cpu_possible_mask , 3 ) ; \
( cpu ) < WORK_CPU_NONE ; \
( cpu ) = __next_gcwq_cpu ( ( cpu ) , cpu_possible_mask , 3 ) )
# define for_each_online_gcwq_cpu(cpu) \
for ( ( cpu ) = __next_gcwq_cpu ( - 1 , cpu_online_mask , 3 ) ; \
( cpu ) < WORK_CPU_NONE ; \
( cpu ) = __next_gcwq_cpu ( ( cpu ) , cpu_online_mask , 3 ) )
# define for_each_cwq_cpu(cpu, wq) \
for ( ( cpu ) = __next_wq_cpu ( - 1 , cpu_possible_mask , ( wq ) ) ; \
( cpu ) < WORK_CPU_NONE ; \
( cpu ) = __next_wq_cpu ( ( cpu ) , cpu_possible_mask , ( wq ) ) )
2009-11-15 19:09:48 +03:00
# ifdef CONFIG_DEBUG_OBJECTS_WORK
static struct debug_obj_descr work_debug_descr ;
2011-03-07 11:58:33 +03:00
static void * work_debug_hint ( void * addr )
{
return ( ( struct work_struct * ) addr ) - > func ;
}
2009-11-15 19:09:48 +03:00
/*
* fixup_init is called when :
* - an active object is initialized
*/
static int work_fixup_init ( void * addr , enum debug_obj_state state )
{
struct work_struct * work = addr ;
switch ( state ) {
case ODEBUG_STATE_ACTIVE :
cancel_work_sync ( work ) ;
debug_object_init ( work , & work_debug_descr ) ;
return 1 ;
default :
return 0 ;
}
}
/*
* fixup_activate is called when :
* - an active object is activated
* - an unknown object is activated ( might be a statically initialized object )
*/
static int work_fixup_activate ( void * addr , enum debug_obj_state state )
{
struct work_struct * work = addr ;
switch ( state ) {
case ODEBUG_STATE_NOTAVAILABLE :
/*
* This is not really a fixup . The work struct was
* statically initialized . We just make sure that it
* is tracked in the object tracker .
*/
2010-06-29 12:07:10 +04:00
if ( test_bit ( WORK_STRUCT_STATIC_BIT , work_data_bits ( work ) ) ) {
2009-11-15 19:09:48 +03:00
debug_object_init ( work , & work_debug_descr ) ;
debug_object_activate ( work , & work_debug_descr ) ;
return 0 ;
}
WARN_ON_ONCE ( 1 ) ;
return 0 ;
case ODEBUG_STATE_ACTIVE :
WARN_ON ( 1 ) ;
default :
return 0 ;
}
}
/*
* fixup_free is called when :
* - an active object is freed
*/
static int work_fixup_free ( void * addr , enum debug_obj_state state )
{
struct work_struct * work = addr ;
switch ( state ) {
case ODEBUG_STATE_ACTIVE :
cancel_work_sync ( work ) ;
debug_object_free ( work , & work_debug_descr ) ;
return 1 ;
default :
return 0 ;
}
}
static struct debug_obj_descr work_debug_descr = {
. name = " work_struct " ,
2011-03-07 11:58:33 +03:00
. debug_hint = work_debug_hint ,
2009-11-15 19:09:48 +03:00
. fixup_init = work_fixup_init ,
. fixup_activate = work_fixup_activate ,
. fixup_free = work_fixup_free ,
} ;
static inline void debug_work_activate ( struct work_struct * work )
{
debug_object_activate ( work , & work_debug_descr ) ;
}
static inline void debug_work_deactivate ( struct work_struct * work )
{
debug_object_deactivate ( work , & work_debug_descr ) ;
}
void __init_work ( struct work_struct * work , int onstack )
{
if ( onstack )
debug_object_init_on_stack ( work , & work_debug_descr ) ;
else
debug_object_init ( work , & work_debug_descr ) ;
}
EXPORT_SYMBOL_GPL ( __init_work ) ;
void destroy_work_on_stack ( struct work_struct * work )
{
debug_object_free ( work , & work_debug_descr ) ;
}
EXPORT_SYMBOL_GPL ( destroy_work_on_stack ) ;
# else
static inline void debug_work_activate ( struct work_struct * work ) { }
static inline void debug_work_deactivate ( struct work_struct * work ) { }
# endif
2008-01-25 23:08:02 +03:00
/* Serializes the accesses to the list of workqueues. */
static DEFINE_SPINLOCK ( workqueue_lock ) ;
2005-04-17 02:20:36 +04:00
static LIST_HEAD ( workqueues ) ;
2010-06-29 12:07:12 +04:00
static bool workqueue_freezing ; /* W: have wqs started freezing? */
2010-06-29 12:07:11 +04:00
2010-06-29 12:07:14 +04:00
/*
* The almighty global cpu workqueues . nr_running is the only field
* which is expected to be used frequently by other cpus via
* try_to_wake_up ( ) . Put it in a separate cacheline .
*/
2010-06-29 12:07:12 +04:00
static DEFINE_PER_CPU ( struct global_cwq , global_cwq ) ;
2010-06-29 12:07:14 +04:00
static DEFINE_PER_CPU_SHARED_ALIGNED ( atomic_t , gcwq_nr_running ) ;
2010-06-29 12:07:12 +04:00
2010-07-02 12:03:51 +04:00
/*
* Global cpu workqueue and nr_running counter for unbound gcwq . The
* gcwq is always online , has GCWQ_DISASSOCIATED set , and all its
* workers have WORKER_UNBOUND set .
*/
static struct global_cwq unbound_global_cwq ;
static atomic_t unbound_gcwq_nr_running = ATOMIC_INIT ( 0 ) ; /* always 0 */
2010-06-29 12:07:11 +04:00
static int worker_thread ( void * __worker ) ;
2005-04-17 02:20:36 +04:00
2010-06-29 12:07:12 +04:00
static struct global_cwq * get_gcwq ( unsigned int cpu )
{
2010-07-02 12:03:51 +04:00
if ( cpu ! = WORK_CPU_UNBOUND )
return & per_cpu ( global_cwq , cpu ) ;
else
return & unbound_global_cwq ;
2010-06-29 12:07:12 +04:00
}
2010-06-29 12:07:14 +04:00
static atomic_t * get_gcwq_nr_running ( unsigned int cpu )
{
2010-07-02 12:03:51 +04:00
if ( cpu ! = WORK_CPU_UNBOUND )
return & per_cpu ( gcwq_nr_running , cpu ) ;
else
return & unbound_gcwq_nr_running ;
2010-06-29 12:07:14 +04:00
}
2010-06-29 12:07:11 +04:00
static struct cpu_workqueue_struct * get_cwq ( unsigned int cpu ,
struct workqueue_struct * wq )
2007-05-09 13:34:12 +04:00
{
2010-07-02 12:03:51 +04:00
if ( ! ( wq - > flags & WQ_UNBOUND ) ) {
2012-03-09 14:03:20 +04:00
if ( likely ( cpu < nr_cpu_ids ) )
2010-07-02 12:03:51 +04:00
return per_cpu_ptr ( wq - > cpu_wq . pcpu , cpu ) ;
} else if ( likely ( cpu = = WORK_CPU_UNBOUND ) )
return wq - > cpu_wq . single ;
return NULL ;
2007-05-09 13:34:12 +04:00
}
2010-06-29 12:07:11 +04:00
static unsigned int work_color_to_flags ( int color )
{
return color < < WORK_STRUCT_COLOR_SHIFT ;
}
static int get_work_color ( struct work_struct * work )
{
return ( * work_data_bits ( work ) > > WORK_STRUCT_COLOR_SHIFT ) &
( ( 1 < < WORK_STRUCT_COLOR_BITS ) - 1 ) ;
}
static int work_next_color ( int color )
{
return ( color + 1 ) % WORK_NR_COLORS ;
}
2005-04-17 02:20:36 +04:00
2007-05-24 00:57:57 +04:00
/*
2010-07-22 16:14:25 +04:00
* A work ' s data points to the cwq with WORK_STRUCT_CWQ set while the
* work is on queue . Once execution starts , WORK_STRUCT_CWQ is
* cleared and the work data contains the cpu number it was last on .
2010-06-29 12:07:13 +04:00
*
* set_work_ { cwq | cpu } ( ) and clear_work_data ( ) can be used to set the
* cwq , cpu or clear work - > data . These functions should only be
* called while the work is owned - ie . while the PENDING bit is set .
*
* get_work_ [ g ] cwq ( ) can be used to obtain the gcwq or cwq
* corresponding to a work . gcwq is available once the work has been
* queued anywhere after initialization . cwq is available only from
* queueing until execution starts .
2007-05-24 00:57:57 +04:00
*/
2010-06-29 12:07:13 +04:00
static inline void set_work_data ( struct work_struct * work , unsigned long data ,
unsigned long flags )
2006-11-22 17:54:49 +03:00
{
[PATCH] WorkStruct: Use direct assignment rather than cmpxchg()
Use direct assignment rather than cmpxchg() as the latter is unavailable
and unimplementable on some platforms and is actually unnecessary.
The use of cmpxchg() was to guard against two possibilities, neither of
which can actually occur:
(1) The pending flag may have been unset or may be cleared. However, given
where it's called, the pending flag is _always_ set. I don't think it
can be unset whilst we're in set_wq_data().
Once the work is enqueued to be actually run, the only way off the queue
is for it to be actually run.
If it's a delayed work item, then the bit can't be cleared by the timer
because we haven't started the timer yet. Also, the pending bit can't be
cleared by cancelling the delayed work _until_ the work item has had its
timer started.
(2) The workqueue pointer might change. This can only happen in two cases:
(a) The work item has just been queued to actually run, and so we're
protected by the appropriate workqueue spinlock.
(b) A delayed work item is being queued, and so the timer hasn't been
started yet, and so no one else knows about the work item or can
access it (the pending bit protects us).
Besides, set_wq_data() _sets_ the workqueue pointer unconditionally, so
it can be assigned instead.
So, replacing the set_wq_data() with a straight assignment would be okay
in most cases.
The problem is where we end up tangling with test_and_set_bit() emulated
using spinlocks, and even then it's not a problem _provided_
test_and_set_bit() doesn't attempt to modify the word if the bit was
set.
If that's a problem, then a bitops-proofed assignment will be required -
equivalent to atomic_set() vs other atomic_xxx() ops.
Signed-off-by: David Howells <dhowells@redhat.com>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-12-07 14:33:26 +03:00
BUG_ON ( ! work_pending ( work ) ) ;
2010-06-29 12:07:13 +04:00
atomic_long_set ( & work - > data , data | flags | work_static ( work ) ) ;
}
2006-11-22 17:54:49 +03:00
2010-06-29 12:07:13 +04:00
static void set_work_cwq ( struct work_struct * work ,
struct cpu_workqueue_struct * cwq ,
unsigned long extra_flags )
{
set_work_data ( work , ( unsigned long ) cwq ,
2010-07-22 16:14:25 +04:00
WORK_STRUCT_PENDING | WORK_STRUCT_CWQ | extra_flags ) ;
2006-11-22 17:54:49 +03:00
}
2010-06-29 12:07:13 +04:00
static void set_work_cpu ( struct work_struct * work , unsigned int cpu )
{
set_work_data ( work , cpu < < WORK_STRUCT_FLAG_BITS , WORK_STRUCT_PENDING ) ;
}
2006-01-08 12:05:12 +03:00
2010-06-29 12:07:13 +04:00
static void clear_work_data ( struct work_struct * work )
2005-04-17 02:20:36 +04:00
{
2010-06-29 12:07:13 +04:00
set_work_data ( work , WORK_STRUCT_NO_CPU , 0 ) ;
2005-04-17 02:20:36 +04:00
}
2010-06-29 12:07:13 +04:00
static struct cpu_workqueue_struct * get_work_cwq ( struct work_struct * work )
2007-05-09 13:34:12 +04:00
{
2010-07-22 16:14:25 +04:00
unsigned long data = atomic_long_read ( & work - > data ) ;
2010-06-29 12:07:13 +04:00
2010-07-22 16:14:25 +04:00
if ( data & WORK_STRUCT_CWQ )
return ( void * ) ( data & WORK_STRUCT_WQ_DATA_MASK ) ;
else
return NULL ;
2010-04-23 19:40:40 +04:00
}
2010-06-29 12:07:13 +04:00
static struct global_cwq * get_work_gcwq ( struct work_struct * work )
2006-11-22 17:54:49 +03:00
{
2010-07-22 16:14:25 +04:00
unsigned long data = atomic_long_read ( & work - > data ) ;
2010-06-29 12:07:13 +04:00
unsigned int cpu ;
2010-07-22 16:14:25 +04:00
if ( data & WORK_STRUCT_CWQ )
return ( ( struct cpu_workqueue_struct * )
( data & WORK_STRUCT_WQ_DATA_MASK ) ) - > gcwq ;
2010-06-29 12:07:13 +04:00
cpu = data > > WORK_STRUCT_FLAG_BITS ;
2010-07-02 12:03:51 +04:00
if ( cpu = = WORK_CPU_NONE )
2010-06-29 12:07:13 +04:00
return NULL ;
2010-07-02 12:03:51 +04:00
BUG_ON ( cpu > = nr_cpu_ids & & cpu ! = WORK_CPU_UNBOUND ) ;
2010-06-29 12:07:13 +04:00
return get_gcwq ( cpu ) ;
2007-05-09 13:34:12 +04:00
}
2010-06-29 12:07:14 +04:00
/*
* Policy functions . These define the policies on how the global
* worker pool is managed . Unless noted otherwise , these functions
* assume that they ' re being called with gcwq - > lock held .
*/
2010-06-29 12:07:14 +04:00
static bool __need_more_worker ( struct global_cwq * gcwq )
2007-05-09 13:34:17 +04:00
{
2010-06-29 12:07:14 +04:00
return ! atomic_read ( get_gcwq_nr_running ( gcwq - > cpu ) ) | |
gcwq - > flags & GCWQ_HIGHPRI_PENDING ;
2007-05-09 13:34:17 +04:00
}
[PATCH] WorkStruct: Use direct assignment rather than cmpxchg()
Use direct assignment rather than cmpxchg() as the latter is unavailable
and unimplementable on some platforms and is actually unnecessary.
The use of cmpxchg() was to guard against two possibilities, neither of
which can actually occur:
(1) The pending flag may have been unset or may be cleared. However, given
where it's called, the pending flag is _always_ set. I don't think it
can be unset whilst we're in set_wq_data().
Once the work is enqueued to be actually run, the only way off the queue
is for it to be actually run.
If it's a delayed work item, then the bit can't be cleared by the timer
because we haven't started the timer yet. Also, the pending bit can't be
cleared by cancelling the delayed work _until_ the work item has had its
timer started.
(2) The workqueue pointer might change. This can only happen in two cases:
(a) The work item has just been queued to actually run, and so we're
protected by the appropriate workqueue spinlock.
(b) A delayed work item is being queued, and so the timer hasn't been
started yet, and so no one else knows about the work item or can
access it (the pending bit protects us).
Besides, set_wq_data() _sets_ the workqueue pointer unconditionally, so
it can be assigned instead.
So, replacing the set_wq_data() with a straight assignment would be okay
in most cases.
The problem is where we end up tangling with test_and_set_bit() emulated
using spinlocks, and even then it's not a problem _provided_
test_and_set_bit() doesn't attempt to modify the word if the bit was
set.
If that's a problem, then a bitops-proofed assignment will be required -
equivalent to atomic_set() vs other atomic_xxx() ops.
Signed-off-by: David Howells <dhowells@redhat.com>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-12-07 14:33:26 +03:00
/*
2010-06-29 12:07:14 +04:00
* Need to wake up a worker ? Called from anything but currently
* running workers .
2012-07-13 01:46:37 +04:00
*
* Note that , because unbound workers never contribute to nr_running , this
* function will always return % true for unbound gcwq as long as the
* worklist isn ' t empty .
[PATCH] WorkStruct: Use direct assignment rather than cmpxchg()
Use direct assignment rather than cmpxchg() as the latter is unavailable
and unimplementable on some platforms and is actually unnecessary.
The use of cmpxchg() was to guard against two possibilities, neither of
which can actually occur:
(1) The pending flag may have been unset or may be cleared. However, given
where it's called, the pending flag is _always_ set. I don't think it
can be unset whilst we're in set_wq_data().
Once the work is enqueued to be actually run, the only way off the queue
is for it to be actually run.
If it's a delayed work item, then the bit can't be cleared by the timer
because we haven't started the timer yet. Also, the pending bit can't be
cleared by cancelling the delayed work _until_ the work item has had its
timer started.
(2) The workqueue pointer might change. This can only happen in two cases:
(a) The work item has just been queued to actually run, and so we're
protected by the appropriate workqueue spinlock.
(b) A delayed work item is being queued, and so the timer hasn't been
started yet, and so no one else knows about the work item or can
access it (the pending bit protects us).
Besides, set_wq_data() _sets_ the workqueue pointer unconditionally, so
it can be assigned instead.
So, replacing the set_wq_data() with a straight assignment would be okay
in most cases.
The problem is where we end up tangling with test_and_set_bit() emulated
using spinlocks, and even then it's not a problem _provided_
test_and_set_bit() doesn't attempt to modify the word if the bit was
set.
If that's a problem, then a bitops-proofed assignment will be required -
equivalent to atomic_set() vs other atomic_xxx() ops.
Signed-off-by: David Howells <dhowells@redhat.com>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-12-07 14:33:26 +03:00
*/
2010-06-29 12:07:14 +04:00
static bool need_more_worker ( struct global_cwq * gcwq )
2006-11-22 17:54:49 +03:00
{
2010-06-29 12:07:14 +04:00
return ! list_empty ( & gcwq - > worklist ) & & __need_more_worker ( gcwq ) ;
2010-06-29 12:07:14 +04:00
}
[PATCH] WorkStruct: Use direct assignment rather than cmpxchg()
Use direct assignment rather than cmpxchg() as the latter is unavailable
and unimplementable on some platforms and is actually unnecessary.
The use of cmpxchg() was to guard against two possibilities, neither of
which can actually occur:
(1) The pending flag may have been unset or may be cleared. However, given
where it's called, the pending flag is _always_ set. I don't think it
can be unset whilst we're in set_wq_data().
Once the work is enqueued to be actually run, the only way off the queue
is for it to be actually run.
If it's a delayed work item, then the bit can't be cleared by the timer
because we haven't started the timer yet. Also, the pending bit can't be
cleared by cancelling the delayed work _until_ the work item has had its
timer started.
(2) The workqueue pointer might change. This can only happen in two cases:
(a) The work item has just been queued to actually run, and so we're
protected by the appropriate workqueue spinlock.
(b) A delayed work item is being queued, and so the timer hasn't been
started yet, and so no one else knows about the work item or can
access it (the pending bit protects us).
Besides, set_wq_data() _sets_ the workqueue pointer unconditionally, so
it can be assigned instead.
So, replacing the set_wq_data() with a straight assignment would be okay
in most cases.
The problem is where we end up tangling with test_and_set_bit() emulated
using spinlocks, and even then it's not a problem _provided_
test_and_set_bit() doesn't attempt to modify the word if the bit was
set.
If that's a problem, then a bitops-proofed assignment will be required -
equivalent to atomic_set() vs other atomic_xxx() ops.
Signed-off-by: David Howells <dhowells@redhat.com>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-12-07 14:33:26 +03:00
2010-06-29 12:07:14 +04:00
/* Can I start working? Called from busy but !running workers. */
static bool may_start_working ( struct global_cwq * gcwq )
{
return gcwq - > nr_idle ;
}
/* Do I need to keep working? Called from currently running workers. */
static bool keep_working ( struct global_cwq * gcwq )
{
atomic_t * nr_running = get_gcwq_nr_running ( gcwq - > cpu ) ;
2010-10-11 13:51:57 +04:00
return ! list_empty ( & gcwq - > worklist ) & &
( atomic_read ( nr_running ) < = 1 | |
gcwq - > flags & GCWQ_HIGHPRI_PENDING ) ;
2010-06-29 12:07:14 +04:00
}
/* Do we need a new worker? Called from manager. */
static bool need_to_create_worker ( struct global_cwq * gcwq )
{
return need_more_worker ( gcwq ) & & ! may_start_working ( gcwq ) ;
}
2006-11-22 17:54:49 +03:00
2010-06-29 12:07:14 +04:00
/* Do I need to be the manager? */
static bool need_to_manage_workers ( struct global_cwq * gcwq )
{
return need_to_create_worker ( gcwq ) | | gcwq - > flags & GCWQ_MANAGE_WORKERS ;
}
/* Do we have too many workers and should some go away? */
static bool too_many_workers ( struct global_cwq * gcwq )
{
bool managing = gcwq - > flags & GCWQ_MANAGING_WORKERS ;
int nr_idle = gcwq - > nr_idle + managing ; /* manager is considered idle */
int nr_busy = gcwq - > nr_workers - nr_idle ;
return nr_idle > 2 & & ( nr_idle - 2 ) * MAX_IDLE_WORKERS_RATIO > = nr_busy ;
2006-11-22 17:54:49 +03:00
}
2010-04-23 19:40:40 +04:00
/*
2010-06-29 12:07:14 +04:00
* Wake up functions .
*/
2010-06-29 12:07:13 +04:00
/* Return the first worker. Safe with preemption disabled */
static struct worker * first_worker ( struct global_cwq * gcwq )
{
if ( unlikely ( list_empty ( & gcwq - > idle_list ) ) )
return NULL ;
return list_first_entry ( & gcwq - > idle_list , struct worker , entry ) ;
}
/**
* wake_up_worker - wake up an idle worker
* @ gcwq : gcwq to wake worker for
*
* Wake up the first idle worker of @ gcwq .
*
* CONTEXT :
* spin_lock_irq ( gcwq - > lock ) .
*/
static void wake_up_worker ( struct global_cwq * gcwq )
{
struct worker * worker = first_worker ( gcwq ) ;
if ( likely ( worker ) )
wake_up_process ( worker - > task ) ;
}
2010-06-29 12:07:13 +04:00
/**
2010-06-29 12:07:14 +04:00
* wq_worker_waking_up - a worker is waking up
* @ task : task waking up
* @ cpu : CPU @ task is waking up to
*
* This function is called during try_to_wake_up ( ) when a worker is
* being awoken .
*
* CONTEXT :
* spin_lock_irq ( rq - > lock )
*/
void wq_worker_waking_up ( struct task_struct * task , unsigned int cpu )
{
struct worker * worker = kthread_data ( task ) ;
workqueue: It is likely that WORKER_NOT_RUNNING is true
Running the annotate branch profiler on three boxes, including my
main box that runs firefox, evolution, xchat, and is part of the distcc farm,
showed this with the likelys in the workqueue code:
correct incorrect % Function File Line
------- --------- - -------- ---- ----
96 996253 99 wq_worker_sleeping workqueue.c 703
96 996247 99 wq_worker_waking_up workqueue.c 677
The likely()s in this case were assuming that WORKER_NOT_RUNNING will
most likely be false. But this is not the case. The reason is
(and shown by adding trace_printks and testing it) that most of the time
WORKER_PREP is set.
In worker_thread() we have:
worker_clr_flags(worker, WORKER_PREP);
[ do work stuff ]
worker_set_flags(worker, WORKER_PREP, false);
(that 'false' means not to wake up an idle worker)
The wq_worker_sleeping() is called from schedule when a worker thread
is putting itself to sleep. Which happens most of the time outside
of that [ do work stuff ].
The wq_worker_waking_up is called by the wakeup worker code, which
is also callod outside that [ do work stuff ].
Thus, the likely and unlikely used by those two functions are actually
backwards.
Remove the annotation and let gcc figure it out.
Acked-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
Signed-off-by: Tejun Heo <tj@kernel.org>
2010-12-04 07:12:33 +03:00
if ( ! ( worker - > flags & WORKER_NOT_RUNNING ) )
2010-06-29 12:07:14 +04:00
atomic_inc ( get_gcwq_nr_running ( cpu ) ) ;
}
/**
* wq_worker_sleeping - a worker is going to sleep
* @ task : task going to sleep
* @ cpu : CPU in question , must be the current CPU number
*
* This function is called during schedule ( ) when a busy worker is
* going to sleep . Worker on the same cpu can be woken up by
* returning pointer to its task .
*
* CONTEXT :
* spin_lock_irq ( rq - > lock )
*
* RETURNS :
* Worker task on @ cpu to wake up , % NULL if none .
*/
struct task_struct * wq_worker_sleeping ( struct task_struct * task ,
unsigned int cpu )
{
struct worker * worker = kthread_data ( task ) , * to_wakeup = NULL ;
struct global_cwq * gcwq = get_gcwq ( cpu ) ;
atomic_t * nr_running = get_gcwq_nr_running ( cpu ) ;
workqueue: It is likely that WORKER_NOT_RUNNING is true
Running the annotate branch profiler on three boxes, including my
main box that runs firefox, evolution, xchat, and is part of the distcc farm,
showed this with the likelys in the workqueue code:
correct incorrect % Function File Line
------- --------- - -------- ---- ----
96 996253 99 wq_worker_sleeping workqueue.c 703
96 996247 99 wq_worker_waking_up workqueue.c 677
The likely()s in this case were assuming that WORKER_NOT_RUNNING will
most likely be false. But this is not the case. The reason is
(and shown by adding trace_printks and testing it) that most of the time
WORKER_PREP is set.
In worker_thread() we have:
worker_clr_flags(worker, WORKER_PREP);
[ do work stuff ]
worker_set_flags(worker, WORKER_PREP, false);
(that 'false' means not to wake up an idle worker)
The wq_worker_sleeping() is called from schedule when a worker thread
is putting itself to sleep. Which happens most of the time outside
of that [ do work stuff ].
The wq_worker_waking_up is called by the wakeup worker code, which
is also callod outside that [ do work stuff ].
Thus, the likely and unlikely used by those two functions are actually
backwards.
Remove the annotation and let gcc figure it out.
Acked-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
Signed-off-by: Tejun Heo <tj@kernel.org>
2010-12-04 07:12:33 +03:00
if ( worker - > flags & WORKER_NOT_RUNNING )
2010-06-29 12:07:14 +04:00
return NULL ;
/* this can only happen on the local cpu */
BUG_ON ( cpu ! = raw_smp_processor_id ( ) ) ;
/*
* The counterpart of the following dec_and_test , implied mb ,
* worklist not empty test sequence is in insert_work ( ) .
* Please read comment there .
*
* NOT_RUNNING is clear . This means that trustee is not in
* charge and we ' re running on the local cpu w / rq lock held
* and preemption disabled , which in turn means that none else
* could be manipulating idle_list , so dereferencing idle_list
* without gcwq lock is safe .
*/
if ( atomic_dec_and_test ( nr_running ) & & ! list_empty ( & gcwq - > worklist ) )
to_wakeup = first_worker ( gcwq ) ;
return to_wakeup ? to_wakeup - > task : NULL ;
}
/**
* worker_set_flags - set worker flags and adjust nr_running accordingly
2010-07-02 12:03:50 +04:00
* @ worker : self
2010-06-29 12:07:13 +04:00
* @ flags : flags to set
* @ wakeup : wakeup an idle worker if necessary
*
2010-06-29 12:07:14 +04:00
* Set @ flags in @ worker - > flags and adjust nr_running accordingly . If
* nr_running becomes zero and @ wakeup is % true , an idle worker is
* woken up .
2010-06-29 12:07:13 +04:00
*
2010-07-02 12:03:50 +04:00
* CONTEXT :
* spin_lock_irq ( gcwq - > lock )
2010-06-29 12:07:13 +04:00
*/
static inline void worker_set_flags ( struct worker * worker , unsigned int flags ,
bool wakeup )
{
2010-06-29 12:07:14 +04:00
struct global_cwq * gcwq = worker - > gcwq ;
2010-07-02 12:03:50 +04:00
WARN_ON_ONCE ( worker - > task ! = current ) ;
2010-06-29 12:07:14 +04:00
/*
* If transitioning into NOT_RUNNING , adjust nr_running and
* wake up an idle worker as necessary if requested by
* @ wakeup .
*/
if ( ( flags & WORKER_NOT_RUNNING ) & &
! ( worker - > flags & WORKER_NOT_RUNNING ) ) {
atomic_t * nr_running = get_gcwq_nr_running ( gcwq - > cpu ) ;
if ( wakeup ) {
if ( atomic_dec_and_test ( nr_running ) & &
! list_empty ( & gcwq - > worklist ) )
wake_up_worker ( gcwq ) ;
} else
atomic_dec ( nr_running ) ;
}
2010-06-29 12:07:13 +04:00
worker - > flags | = flags ;
}
/**
2010-06-29 12:07:14 +04:00
* worker_clr_flags - clear worker flags and adjust nr_running accordingly
2010-07-02 12:03:50 +04:00
* @ worker : self
2010-06-29 12:07:13 +04:00
* @ flags : flags to clear
*
2010-06-29 12:07:14 +04:00
* Clear @ flags in @ worker - > flags and adjust nr_running accordingly .
2010-06-29 12:07:13 +04:00
*
2010-07-02 12:03:50 +04:00
* CONTEXT :
* spin_lock_irq ( gcwq - > lock )
2010-06-29 12:07:13 +04:00
*/
static inline void worker_clr_flags ( struct worker * worker , unsigned int flags )
{
2010-06-29 12:07:14 +04:00
struct global_cwq * gcwq = worker - > gcwq ;
unsigned int oflags = worker - > flags ;
2010-07-02 12:03:50 +04:00
WARN_ON_ONCE ( worker - > task ! = current ) ;
2010-06-29 12:07:13 +04:00
worker - > flags & = ~ flags ;
2010-06-29 12:07:14 +04:00
2011-01-11 17:58:49 +03:00
/*
* If transitioning out of NOT_RUNNING , increment nr_running . Note
* that the nested NOT_RUNNING is not a noop . NOT_RUNNING is mask
* of multiple flags , not a single flag .
*/
2010-06-29 12:07:14 +04:00
if ( ( flags & WORKER_NOT_RUNNING ) & & ( oflags & WORKER_NOT_RUNNING ) )
if ( ! ( worker - > flags & WORKER_NOT_RUNNING ) )
atomic_inc ( get_gcwq_nr_running ( gcwq - > cpu ) ) ;
2010-06-29 12:07:13 +04:00
}
2010-06-29 12:07:12 +04:00
/**
* busy_worker_head - return the busy hash head for a work
* @ gcwq : gcwq of interest
* @ work : work to be hashed
*
* Return hash head of @ gcwq for @ work .
*
* CONTEXT :
* spin_lock_irq ( gcwq - > lock ) .
*
* RETURNS :
* Pointer to the hash head .
*/
static struct hlist_head * busy_worker_head ( struct global_cwq * gcwq ,
struct work_struct * work )
{
const int base_shift = ilog2 ( sizeof ( struct work_struct ) ) ;
unsigned long v = ( unsigned long ) work ;
/* simple shift and fold hash, do we need something better? */
v > > = base_shift ;
v + = v > > BUSY_WORKER_HASH_ORDER ;
v & = BUSY_WORKER_HASH_MASK ;
return & gcwq - > busy_hash [ v ] ;
}
2010-06-29 12:07:13 +04:00
/**
* __find_worker_executing_work - find worker which is executing a work
* @ gcwq : gcwq of interest
* @ bwh : hash head as returned by busy_worker_head ( )
* @ work : work to find worker for
*
* Find a worker which is executing @ work on @ gcwq . @ bwh should be
* the hash head obtained by calling busy_worker_head ( ) with the same
* work .
*
* CONTEXT :
* spin_lock_irq ( gcwq - > lock ) .
*
* RETURNS :
* Pointer to worker which is executing @ work if found , NULL
* otherwise .
*/
static struct worker * __find_worker_executing_work ( struct global_cwq * gcwq ,
struct hlist_head * bwh ,
struct work_struct * work )
{
struct worker * worker ;
struct hlist_node * tmp ;
hlist_for_each_entry ( worker , tmp , bwh , hentry )
if ( worker - > current_work = = work )
return worker ;
return NULL ;
}
/**
* find_worker_executing_work - find worker which is executing a work
* @ gcwq : gcwq of interest
* @ work : work to find worker for
*
* Find a worker which is executing @ work on @ gcwq . This function is
* identical to __find_worker_executing_work ( ) except that this
* function calculates @ bwh itself .
*
* CONTEXT :
* spin_lock_irq ( gcwq - > lock ) .
*
* RETURNS :
* Pointer to worker which is executing @ work if found , NULL
* otherwise .
2010-04-23 19:40:40 +04:00
*/
2010-06-29 12:07:13 +04:00
static struct worker * find_worker_executing_work ( struct global_cwq * gcwq ,
struct work_struct * work )
2010-04-23 19:40:40 +04:00
{
2010-06-29 12:07:13 +04:00
return __find_worker_executing_work ( gcwq , busy_worker_head ( gcwq , work ) ,
work ) ;
2010-04-23 19:40:40 +04:00
}
2010-06-29 12:07:14 +04:00
/**
* gcwq_determine_ins_pos - find insertion position
* @ gcwq : gcwq of interest
* @ cwq : cwq a work is being queued for
*
* A work for @ cwq is about to be queued on @ gcwq , determine insertion
* position for the work . If @ cwq is for HIGHPRI wq , the work is
* queued at the head of the queue but in FIFO order with respect to
* other HIGHPRI works ; otherwise , at the end of the queue . This
* function also sets GCWQ_HIGHPRI_PENDING flag to hint @ gcwq that
* there are HIGHPRI works pending .
*
* CONTEXT :
* spin_lock_irq ( gcwq - > lock ) .
*
* RETURNS :
* Pointer to inserstion position .
*/
static inline struct list_head * gcwq_determine_ins_pos ( struct global_cwq * gcwq ,
struct cpu_workqueue_struct * cwq )
2006-11-22 17:54:49 +03:00
{
2010-06-29 12:07:14 +04:00
struct work_struct * twork ;
if ( likely ( ! ( cwq - > wq - > flags & WQ_HIGHPRI ) ) )
return & gcwq - > worklist ;
list_for_each_entry ( twork , & gcwq - > worklist , entry ) {
struct cpu_workqueue_struct * tcwq = get_work_cwq ( twork ) ;
if ( ! ( tcwq - > wq - > flags & WQ_HIGHPRI ) )
break ;
}
gcwq - > flags | = GCWQ_HIGHPRI_PENDING ;
return & twork - > entry ;
2006-11-22 17:54:49 +03:00
}
2010-06-29 12:07:10 +04:00
/**
2010-06-29 12:07:13 +04:00
* insert_work - insert a work into gcwq
2010-06-29 12:07:10 +04:00
* @ cwq : cwq @ work belongs to
* @ work : work to insert
* @ head : insertion point
* @ extra_flags : extra WORK_STRUCT_ * flags to set
*
2010-06-29 12:07:13 +04:00
* Insert @ work which belongs to @ cwq into @ gcwq after @ head .
* @ extra_flags is or ' d to work_struct flags .
2010-06-29 12:07:10 +04:00
*
* CONTEXT :
2010-06-29 12:07:12 +04:00
* spin_lock_irq ( gcwq - > lock ) .
2010-06-29 12:07:10 +04:00
*/
implement flush_work()
A basic problem with flush_scheduled_work() is that it blocks behind _all_
presently-queued works, rather than just the work whcih the caller wants to
flush. If the caller holds some lock, and if one of the queued work happens
to want that lock as well then accidental deadlocks can occur.
One example of this is the phy layer: it wants to flush work while holding
rtnl_lock(). But if a linkwatch event happens to be queued, the phy code will
deadlock because the linkwatch callback function takes rtnl_lock.
So we implement a new function which will flush a *single* work - just the one
which the caller wants to free up. Thus we avoid the accidental deadlocks
which can arise from unrelated subsystems' callbacks taking shared locks.
flush_work() non-blockingly dequeues the work_struct which we want to kill,
then it waits for its handler to complete on all CPUs.
Add ->current_work to the "struct cpu_workqueue_struct", it points to
currently running "struct work_struct". When flush_work(work) detects
->current_work == work, it inserts a barrier at the _head_ of ->worklist
(and thus right _after_ that work) and waits for completition. This means
that the next work fired on that CPU will be this barrier, or another
barrier queued by concurrent flush_work(), so the caller of flush_work()
will be woken before any "regular" work has a chance to run.
When wait_on_work() unlocks workqueue_mutex (or whatever we choose to protect
against CPU hotplug), CPU may go away. But in that case take_over_work() will
move a barrier we queued to another CPU, it will be fired sometime, and
wait_on_work() will be woken.
Actually, we are doing cleanup_workqueue_thread()->kthread_stop() before
take_over_work(), so cwq->thread should complete its ->worklist (and thus
the barrier), because currently we don't check kthread_should_stop() in
run_workqueue(). But even if we did, everything should be ok.
[akpm@osdl.org: cleanup]
[akpm@osdl.org: add flush_work_keventd() wrapper]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 13:33:52 +04:00
static void insert_work ( struct cpu_workqueue_struct * cwq ,
2010-06-29 12:07:10 +04:00
struct work_struct * work , struct list_head * head ,
unsigned int extra_flags )
implement flush_work()
A basic problem with flush_scheduled_work() is that it blocks behind _all_
presently-queued works, rather than just the work whcih the caller wants to
flush. If the caller holds some lock, and if one of the queued work happens
to want that lock as well then accidental deadlocks can occur.
One example of this is the phy layer: it wants to flush work while holding
rtnl_lock(). But if a linkwatch event happens to be queued, the phy code will
deadlock because the linkwatch callback function takes rtnl_lock.
So we implement a new function which will flush a *single* work - just the one
which the caller wants to free up. Thus we avoid the accidental deadlocks
which can arise from unrelated subsystems' callbacks taking shared locks.
flush_work() non-blockingly dequeues the work_struct which we want to kill,
then it waits for its handler to complete on all CPUs.
Add ->current_work to the "struct cpu_workqueue_struct", it points to
currently running "struct work_struct". When flush_work(work) detects
->current_work == work, it inserts a barrier at the _head_ of ->worklist
(and thus right _after_ that work) and waits for completition. This means
that the next work fired on that CPU will be this barrier, or another
barrier queued by concurrent flush_work(), so the caller of flush_work()
will be woken before any "regular" work has a chance to run.
When wait_on_work() unlocks workqueue_mutex (or whatever we choose to protect
against CPU hotplug), CPU may go away. But in that case take_over_work() will
move a barrier we queued to another CPU, it will be fired sometime, and
wait_on_work() will be woken.
Actually, we are doing cleanup_workqueue_thread()->kthread_stop() before
take_over_work(), so cwq->thread should complete its ->worklist (and thus
the barrier), because currently we don't check kthread_should_stop() in
run_workqueue(). But even if we did, everything should be ok.
[akpm@osdl.org: cleanup]
[akpm@osdl.org: add flush_work_keventd() wrapper]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 13:33:52 +04:00
{
2010-06-29 12:07:14 +04:00
struct global_cwq * gcwq = cwq - > gcwq ;
2010-06-29 12:07:10 +04:00
/* we own @work, set data and link */
2010-06-29 12:07:13 +04:00
set_work_cwq ( work , cwq , extra_flags ) ;
2009-01-13 01:15:46 +03:00
make cancel_rearming_delayed_work() reliable
Thanks to Jarek Poplawski for the ideas and for spotting the bug in the
initial draft patch.
cancel_rearming_delayed_work() currently has many limitations, because it
requires that dwork always re-arms itself via queue_delayed_work(). So it
hangs forever if dwork doesn't do this, or cancel_rearming_delayed_work/
cancel_delayed_work was already called. It uses flush_workqueue() in a
loop, so it can't be used if workqueue was freezed, and it is potentially
live- lockable on busy system if delay is small.
With this patch cancel_rearming_delayed_work() doesn't make any assumptions
about dwork, it can re-arm itself via queue_delayed_work(), or
queue_work(), or do nothing.
As a "side effect", cancel_work_sync() was changed to handle re-arming works
as well.
Disadvantages:
- this patch adds wmb() to insert_work().
- slowdowns the fast path (when del_timer() succeeds on entry) of
cancel_rearming_delayed_work(), because wait_on_work() is called
unconditionally. In that case, compared to the old version, we are
doing "unneeded" lock/unlock for each online CPU.
On the other hand, this means we don't need to use cancel_work_sync()
after cancel_rearming_delayed_work().
- complicates the code (.text grows by 130 bytes).
[akpm@linux-foundation.org: fix speling]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Cc: David Chinner <dgc@sgi.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Gautham Shenoy <ego@in.ibm.com>
Acked-by: Jarek Poplawski <jarkao2@o2.pl>
Cc: Srivatsa Vaddagiri <vatsa@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 13:34:46 +04:00
/*
* Ensure that we get the right work - > data if we see the
* result of list_add ( ) below , see try_to_grab_pending ( ) .
*/
smp_wmb ( ) ;
2010-06-29 12:07:10 +04:00
2008-07-25 12:47:47 +04:00
list_add_tail ( & work - > entry , head ) ;
2010-06-29 12:07:14 +04:00
/*
* Ensure either worker_sched_deactivated ( ) sees the above
* list_add_tail ( ) or we see zero nr_running to avoid workers
* lying around lazily while there are works to be processed .
*/
smp_mb ( ) ;
2010-06-29 12:07:14 +04:00
if ( __need_more_worker ( gcwq ) )
2010-06-29 12:07:14 +04:00
wake_up_worker ( gcwq ) ;
implement flush_work()
A basic problem with flush_scheduled_work() is that it blocks behind _all_
presently-queued works, rather than just the work whcih the caller wants to
flush. If the caller holds some lock, and if one of the queued work happens
to want that lock as well then accidental deadlocks can occur.
One example of this is the phy layer: it wants to flush work while holding
rtnl_lock(). But if a linkwatch event happens to be queued, the phy code will
deadlock because the linkwatch callback function takes rtnl_lock.
So we implement a new function which will flush a *single* work - just the one
which the caller wants to free up. Thus we avoid the accidental deadlocks
which can arise from unrelated subsystems' callbacks taking shared locks.
flush_work() non-blockingly dequeues the work_struct which we want to kill,
then it waits for its handler to complete on all CPUs.
Add ->current_work to the "struct cpu_workqueue_struct", it points to
currently running "struct work_struct". When flush_work(work) detects
->current_work == work, it inserts a barrier at the _head_ of ->worklist
(and thus right _after_ that work) and waits for completition. This means
that the next work fired on that CPU will be this barrier, or another
barrier queued by concurrent flush_work(), so the caller of flush_work()
will be woken before any "regular" work has a chance to run.
When wait_on_work() unlocks workqueue_mutex (or whatever we choose to protect
against CPU hotplug), CPU may go away. But in that case take_over_work() will
move a barrier we queued to another CPU, it will be fired sometime, and
wait_on_work() will be woken.
Actually, we are doing cleanup_workqueue_thread()->kthread_stop() before
take_over_work(), so cwq->thread should complete its ->worklist (and thus
the barrier), because currently we don't check kthread_should_stop() in
run_workqueue(). But even if we did, everything should be ok.
[akpm@osdl.org: cleanup]
[akpm@osdl.org: add flush_work_keventd() wrapper]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 13:33:52 +04:00
}
2010-12-20 21:32:04 +03:00
/*
* Test whether @ work is being queued from another work executing on the
* same workqueue . This is rather expensive and should only be used from
* cold paths .
*/
static bool is_chained_work ( struct workqueue_struct * wq )
{
unsigned long flags ;
unsigned int cpu ;
for_each_gcwq_cpu ( cpu ) {
struct global_cwq * gcwq = get_gcwq ( cpu ) ;
struct worker * worker ;
struct hlist_node * pos ;
int i ;
spin_lock_irqsave ( & gcwq - > lock , flags ) ;
for_each_busy_worker ( worker , i , pos , gcwq ) {
if ( worker - > task ! = current )
continue ;
spin_unlock_irqrestore ( & gcwq - > lock , flags ) ;
/*
* I ' m @ worker , no locking necessary . See if @ work
* is headed to the same workqueue .
*/
return worker - > current_cwq - > wq = = wq ;
}
spin_unlock_irqrestore ( & gcwq - > lock , flags ) ;
}
return false ;
}
2010-06-29 12:07:10 +04:00
static void __queue_work ( unsigned int cpu , struct workqueue_struct * wq ,
2005-04-17 02:20:36 +04:00
struct work_struct * work )
{
2010-06-29 12:07:13 +04:00
struct global_cwq * gcwq ;
struct cpu_workqueue_struct * cwq ;
2010-06-29 12:07:12 +04:00
struct list_head * worklist ;
2010-08-25 12:33:56 +04:00
unsigned int work_flags ;
2005-04-17 02:20:36 +04:00
unsigned long flags ;
2009-11-15 19:09:48 +03:00
debug_work_activate ( work ) ;
2010-06-29 12:07:12 +04:00
2010-12-20 21:32:04 +03:00
/* if dying, only works from the same workqueue are allowed */
2011-04-05 20:01:44 +04:00
if ( unlikely ( wq - > flags & WQ_DRAINING ) & &
2010-12-20 21:32:04 +03:00
WARN_ON_ONCE ( ! is_chained_work ( wq ) ) )
2010-08-24 16:22:47 +04:00
return ;
2010-07-02 12:03:51 +04:00
/* determine gcwq to use */
if ( ! ( wq - > flags & WQ_UNBOUND ) ) {
2010-06-29 12:07:13 +04:00
struct global_cwq * last_gcwq ;
2010-07-02 12:03:51 +04:00
if ( unlikely ( cpu = = WORK_CPU_UNBOUND ) )
cpu = raw_smp_processor_id ( ) ;
2010-06-29 12:07:13 +04:00
/*
* It ' s multi cpu . If @ wq is non - reentrant and @ work
* was previously on a different cpu , it might still
* be running there , in which case the work needs to
* be queued on that cpu to guarantee non - reentrance .
*/
2010-06-29 12:07:13 +04:00
gcwq = get_gcwq ( cpu ) ;
2010-06-29 12:07:13 +04:00
if ( wq - > flags & WQ_NON_REENTRANT & &
( last_gcwq = get_work_gcwq ( work ) ) & & last_gcwq ! = gcwq ) {
struct worker * worker ;
spin_lock_irqsave ( & last_gcwq - > lock , flags ) ;
worker = find_worker_executing_work ( last_gcwq , work ) ;
if ( worker & & worker - > current_cwq - > wq = = wq )
gcwq = last_gcwq ;
else {
/* meh... not running there, queue here */
spin_unlock_irqrestore ( & last_gcwq - > lock , flags ) ;
spin_lock_irqsave ( & gcwq - > lock , flags ) ;
}
} else
spin_lock_irqsave ( & gcwq - > lock , flags ) ;
2010-07-02 12:03:51 +04:00
} else {
gcwq = get_gcwq ( WORK_CPU_UNBOUND ) ;
spin_lock_irqsave ( & gcwq - > lock , flags ) ;
2010-06-29 12:07:13 +04:00
}
/* gcwq determined, get cwq and queue */
cwq = get_cwq ( gcwq - > cpu , wq ) ;
2010-10-05 12:49:55 +04:00
trace_workqueue_queue_work ( cpu , cwq , work ) ;
2010-06-29 12:07:13 +04:00
2012-04-13 23:06:58 +04:00
if ( WARN_ON ( ! list_empty ( & work - > entry ) ) ) {
spin_unlock_irqrestore ( & gcwq - > lock , flags ) ;
return ;
}
2010-06-29 12:07:12 +04:00
2010-06-29 12:07:11 +04:00
cwq - > nr_in_flight [ cwq - > work_color ] + + ;
2010-08-25 12:33:56 +04:00
work_flags = work_color_to_flags ( cwq - > work_color ) ;
2010-06-29 12:07:12 +04:00
if ( likely ( cwq - > nr_active < cwq - > max_active ) ) {
2010-10-05 12:49:55 +04:00
trace_workqueue_activate_work ( work ) ;
2010-06-29 12:07:12 +04:00
cwq - > nr_active + + ;
2010-06-29 12:07:14 +04:00
worklist = gcwq_determine_ins_pos ( gcwq , cwq ) ;
2010-08-25 12:33:56 +04:00
} else {
work_flags | = WORK_STRUCT_DELAYED ;
2010-06-29 12:07:12 +04:00
worklist = & cwq - > delayed_works ;
2010-08-25 12:33:56 +04:00
}
2010-06-29 12:07:12 +04:00
2010-08-25 12:33:56 +04:00
insert_work ( cwq , work , worklist , work_flags ) ;
2010-06-29 12:07:12 +04:00
2010-06-29 12:07:12 +04:00
spin_unlock_irqrestore ( & gcwq - > lock , flags ) ;
2005-04-17 02:20:36 +04:00
}
2006-07-30 14:03:42 +04:00
/**
* queue_work - queue work on a workqueue
* @ wq : workqueue to use
* @ work : work to queue
*
2006-10-28 21:38:58 +04:00
* Returns 0 if @ work was already on a queue , non - zero otherwise .
2005-04-17 02:20:36 +04:00
*
2008-04-29 12:00:27 +04:00
* We queue the work to the CPU on which it was submitted , but if the CPU dies
* it can be processed by another CPU .
2005-04-17 02:20:36 +04:00
*/
2008-02-08 15:19:53 +03:00
int queue_work ( struct workqueue_struct * wq , struct work_struct * work )
2005-04-17 02:20:36 +04:00
{
2008-07-25 12:47:53 +04:00
int ret ;
ret = queue_work_on ( get_cpu ( ) , wq , work ) ;
put_cpu ( ) ;
2005-04-17 02:20:36 +04:00
return ret ;
}
2006-06-30 09:40:45 +04:00
EXPORT_SYMBOL_GPL ( queue_work ) ;
2005-04-17 02:20:36 +04:00
2008-07-24 08:28:39 +04:00
/**
* queue_work_on - queue work on specific cpu
* @ cpu : CPU number to execute work on
* @ wq : workqueue to use
* @ work : work to queue
*
* Returns 0 if @ work was already on a queue , non - zero otherwise .
*
* We queue the work to a specific CPU , the caller must ensure it
* can ' t go away .
*/
int
queue_work_on ( int cpu , struct workqueue_struct * wq , struct work_struct * work )
{
int ret = 0 ;
2010-06-29 12:07:10 +04:00
if ( ! test_and_set_bit ( WORK_STRUCT_PENDING_BIT , work_data_bits ( work ) ) ) {
2010-06-29 12:07:10 +04:00
__queue_work ( cpu , wq , work ) ;
2008-07-24 08:28:39 +04:00
ret = 1 ;
}
return ret ;
}
EXPORT_SYMBOL_GPL ( queue_work_on ) ;
2008-02-08 15:21:09 +03:00
static void delayed_work_timer_fn ( unsigned long __data )
2005-04-17 02:20:36 +04:00
{
2006-11-22 17:54:01 +03:00
struct delayed_work * dwork = ( struct delayed_work * ) __data ;
2010-06-29 12:07:13 +04:00
struct cpu_workqueue_struct * cwq = get_work_cwq ( & dwork - > work ) ;
2005-04-17 02:20:36 +04:00
2010-06-29 12:07:10 +04:00
__queue_work ( smp_processor_id ( ) , cwq - > wq , & dwork - > work ) ;
2005-04-17 02:20:36 +04:00
}
2006-07-30 14:03:42 +04:00
/**
* queue_delayed_work - queue work on a workqueue after delay
* @ wq : workqueue to use
2006-12-22 12:06:52 +03:00
* @ dwork : delayable work to queue
2006-07-30 14:03:42 +04:00
* @ delay : number of jiffies to wait before queueing
*
2006-10-28 21:38:58 +04:00
* Returns 0 if @ work was already on a queue , non - zero otherwise .
2006-07-30 14:03:42 +04:00
*/
2008-02-08 15:19:53 +03:00
int queue_delayed_work ( struct workqueue_struct * wq ,
2006-11-22 17:54:01 +03:00
struct delayed_work * dwork , unsigned long delay )
2005-04-17 02:20:36 +04:00
{
2006-11-22 17:54:01 +03:00
if ( delay = = 0 )
2007-05-09 13:34:16 +04:00
return queue_work ( wq , & dwork - > work ) ;
2005-04-17 02:20:36 +04:00
2007-05-09 13:34:16 +04:00
return queue_delayed_work_on ( - 1 , wq , dwork , delay ) ;
2005-04-17 02:20:36 +04:00
}
2006-06-30 09:40:45 +04:00
EXPORT_SYMBOL_GPL ( queue_delayed_work ) ;
2005-04-17 02:20:36 +04:00
2006-07-30 14:03:42 +04:00
/**
* queue_delayed_work_on - queue work on specific CPU after delay
* @ cpu : CPU number to execute work on
* @ wq : workqueue to use
2006-12-22 12:06:52 +03:00
* @ dwork : work to queue
2006-07-30 14:03:42 +04:00
* @ delay : number of jiffies to wait before queueing
*
2006-10-28 21:38:58 +04:00
* Returns 0 if @ work was already on a queue , non - zero otherwise .
2006-07-30 14:03:42 +04:00
*/
2006-06-29 00:50:33 +04:00
int queue_delayed_work_on ( int cpu , struct workqueue_struct * wq ,
2006-11-22 17:54:01 +03:00
struct delayed_work * dwork , unsigned long delay )
2006-06-29 00:50:33 +04:00
{
int ret = 0 ;
2006-11-22 17:54:01 +03:00
struct timer_list * timer = & dwork - > timer ;
struct work_struct * work = & dwork - > work ;
2006-06-29 00:50:33 +04:00
2010-06-29 12:07:10 +04:00
if ( ! test_and_set_bit ( WORK_STRUCT_PENDING_BIT , work_data_bits ( work ) ) ) {
2010-07-02 12:03:51 +04:00
unsigned int lcpu ;
2010-06-29 12:07:13 +04:00
2006-06-29 00:50:33 +04:00
BUG_ON ( timer_pending ( timer ) ) ;
BUG_ON ( ! list_empty ( & work - > entry ) ) ;
2008-05-01 15:35:14 +04:00
timer_stats_timer_set_start_info ( & dwork - > timer ) ;
2010-06-29 12:07:13 +04:00
/*
* This stores cwq for the moment , for the timer_fn .
* Note that the work ' s gcwq is preserved to allow
* reentrance detection for delayed works .
*/
2010-07-02 12:03:51 +04:00
if ( ! ( wq - > flags & WQ_UNBOUND ) ) {
struct global_cwq * gcwq = get_work_gcwq ( work ) ;
if ( gcwq & & gcwq - > cpu ! = WORK_CPU_UNBOUND )
lcpu = gcwq - > cpu ;
else
lcpu = raw_smp_processor_id ( ) ;
} else
lcpu = WORK_CPU_UNBOUND ;
2010-06-29 12:07:13 +04:00
set_work_cwq ( work , get_cwq ( lcpu , wq ) , 0 ) ;
2010-07-02 12:03:51 +04:00
2006-06-29 00:50:33 +04:00
timer - > expires = jiffies + delay ;
2006-11-22 17:54:01 +03:00
timer - > data = ( unsigned long ) dwork ;
2006-06-29 00:50:33 +04:00
timer - > function = delayed_work_timer_fn ;
2007-05-09 13:34:16 +04:00
if ( unlikely ( cpu > = 0 ) )
add_timer_on ( timer , cpu ) ;
else
add_timer ( timer ) ;
2006-06-29 00:50:33 +04:00
ret = 1 ;
}
return ret ;
}
2006-06-30 09:40:45 +04:00
EXPORT_SYMBOL_GPL ( queue_delayed_work_on ) ;
2005-04-17 02:20:36 +04:00
2010-06-29 12:07:12 +04:00
/**
* worker_enter_idle - enter idle state
* @ worker : worker which is entering idle state
*
* @ worker is entering idle state . Update stats and idle timer if
* necessary .
*
* LOCKING :
* spin_lock_irq ( gcwq - > lock ) .
*/
static void worker_enter_idle ( struct worker * worker )
2005-04-17 02:20:36 +04:00
{
2010-06-29 12:07:12 +04:00
struct global_cwq * gcwq = worker - > gcwq ;
BUG_ON ( worker - > flags & WORKER_IDLE ) ;
BUG_ON ( ! list_empty ( & worker - > entry ) & &
( worker - > hentry . next | | worker - > hentry . pprev ) ) ;
2010-07-02 12:03:50 +04:00
/* can't use worker_set_flags(), also called from start_worker() */
worker - > flags | = WORKER_IDLE ;
2010-06-29 12:07:12 +04:00
gcwq - > nr_idle + + ;
2010-06-29 12:07:14 +04:00
worker - > last_active = jiffies ;
2010-06-29 12:07:12 +04:00
/* idle_list is LIFO */
list_add ( & worker - > entry , & gcwq - > idle_list ) ;
2010-06-29 12:07:12 +04:00
2010-06-29 12:07:14 +04:00
if ( likely ( ! ( worker - > flags & WORKER_ROGUE ) ) ) {
if ( too_many_workers ( gcwq ) & & ! timer_pending ( & gcwq - > idle_timer ) )
mod_timer ( & gcwq - > idle_timer ,
jiffies + IDLE_WORKER_TIMEOUT ) ;
} else
2010-06-29 12:07:12 +04:00
wake_up_all ( & gcwq - > trustee_wait ) ;
2010-07-02 12:03:50 +04:00
2012-05-15 02:04:50 +04:00
/*
* Sanity check nr_running . Because trustee releases gcwq - > lock
* between setting % WORKER_ROGUE and zapping nr_running , the
* warning may trigger spuriously . Check iff trustee is idle .
*/
WARN_ON_ONCE ( gcwq - > trustee_state = = TRUSTEE_DONE & &
gcwq - > nr_workers = = gcwq - > nr_idle & &
2010-07-02 12:03:50 +04:00
atomic_read ( get_gcwq_nr_running ( gcwq - > cpu ) ) ) ;
2010-06-29 12:07:12 +04:00
}
/**
* worker_leave_idle - leave idle state
* @ worker : worker which is leaving idle state
*
* @ worker is leaving idle state . Update stats .
*
* LOCKING :
* spin_lock_irq ( gcwq - > lock ) .
*/
static void worker_leave_idle ( struct worker * worker )
{
struct global_cwq * gcwq = worker - > gcwq ;
BUG_ON ( ! ( worker - > flags & WORKER_IDLE ) ) ;
2010-06-29 12:07:13 +04:00
worker_clr_flags ( worker , WORKER_IDLE ) ;
2010-06-29 12:07:12 +04:00
gcwq - > nr_idle - - ;
list_del_init ( & worker - > entry ) ;
}
2010-06-29 12:07:14 +04:00
/**
* worker_maybe_bind_and_lock - bind worker to its cpu if possible and lock gcwq
* @ worker : self
*
* Works which are scheduled while the cpu is online must at least be
* scheduled to a worker which is bound to the cpu so that if they are
* flushed from cpu callbacks while cpu is going down , they are
* guaranteed to execute on the cpu .
*
* This function is to be used by rogue workers and rescuers to bind
* themselves to the target cpu and may race with cpu going down or
* coming online . kthread_bind ( ) can ' t be used because it may put the
* worker to already dead cpu and set_cpus_allowed_ptr ( ) can ' t be used
* verbatim as it ' s best effort and blocking and gcwq may be
* [ dis ] associated in the meantime .
*
* This function tries set_cpus_allowed ( ) and locks gcwq and verifies
* the binding against GCWQ_DISASSOCIATED which is set during
* CPU_DYING and cleared during CPU_ONLINE , so if the worker enters
* idle state or fetches works without dropping lock , it can guarantee
* the scheduling requirement described in the first paragraph .
*
* CONTEXT :
* Might sleep . Called without any lock but returns with gcwq - > lock
* held .
*
* RETURNS :
* % true if the associated gcwq is online ( @ worker is successfully
* bound ) , % false if offline .
*/
static bool worker_maybe_bind_and_lock ( struct worker * worker )
2010-08-22 18:19:43 +04:00
__acquires ( & gcwq - > lock )
2010-06-29 12:07:14 +04:00
{
struct global_cwq * gcwq = worker - > gcwq ;
struct task_struct * task = worker - > task ;
while ( true ) {
2007-10-19 10:39:55 +04:00
/*
2010-06-29 12:07:14 +04:00
* The following call may fail , succeed or succeed
* without actually migrating the task to the cpu if
* it races with cpu hotunplug operation . Verify
* against GCWQ_DISASSOCIATED .
2007-10-19 10:39:55 +04:00
*/
2010-07-02 12:03:51 +04:00
if ( ! ( gcwq - > flags & GCWQ_DISASSOCIATED ) )
set_cpus_allowed_ptr ( task , get_cpu_mask ( gcwq - > cpu ) ) ;
2010-06-29 12:07:14 +04:00
spin_lock_irq ( & gcwq - > lock ) ;
if ( gcwq - > flags & GCWQ_DISASSOCIATED )
return false ;
if ( task_cpu ( task ) = = gcwq - > cpu & &
cpumask_equal ( & current - > cpus_allowed ,
get_cpu_mask ( gcwq - > cpu ) ) )
return true ;
spin_unlock_irq ( & gcwq - > lock ) ;
2011-04-29 20:08:37 +04:00
/*
* We ' ve raced with CPU hot [ un ] plug . Give it a breather
* and retry migration . cond_resched ( ) is required here ;
* otherwise , we might deadlock against cpu_stop trying to
* bring down the CPU on non - preemptive kernel .
*/
2010-06-29 12:07:14 +04:00
cpu_relax ( ) ;
2011-04-29 20:08:37 +04:00
cond_resched ( ) ;
2010-06-29 12:07:14 +04:00
}
}
/*
* Function for worker - > rebind_work used to rebind rogue busy workers
* to the associated cpu which is coming back online . This is
* scheduled by cpu up but can race with other cpu hotplug operations
* and may be executed twice without intervening cpu down .
*/
static void worker_rebind_fn ( struct work_struct * work )
{
struct worker * worker = container_of ( work , struct worker , rebind_work ) ;
struct global_cwq * gcwq = worker - > gcwq ;
if ( worker_maybe_bind_and_lock ( worker ) )
worker_clr_flags ( worker , WORKER_REBIND ) ;
spin_unlock_irq ( & gcwq - > lock ) ;
}
2010-06-29 12:07:11 +04:00
static struct worker * alloc_worker ( void )
{
struct worker * worker ;
worker = kzalloc ( sizeof ( * worker ) , GFP_KERNEL ) ;
2010-06-29 12:07:12 +04:00
if ( worker ) {
INIT_LIST_HEAD ( & worker - > entry ) ;
2010-06-29 12:07:12 +04:00
INIT_LIST_HEAD ( & worker - > scheduled ) ;
2010-06-29 12:07:14 +04:00
INIT_WORK ( & worker - > rebind_work , worker_rebind_fn ) ;
/* on creation a worker is in !idle && prep state */
worker - > flags = WORKER_PREP ;
2010-06-29 12:07:12 +04:00
}
2010-06-29 12:07:11 +04:00
return worker ;
}
/**
* create_worker - create a new workqueue worker
2010-06-29 12:07:13 +04:00
* @ gcwq : gcwq the new worker will belong to
2010-06-29 12:07:11 +04:00
* @ bind : whether to set affinity to @ cpu or not
*
2010-06-29 12:07:13 +04:00
* Create a new worker which is bound to @ gcwq . The returned worker
2010-06-29 12:07:11 +04:00
* can be started by calling start_worker ( ) or destroyed using
* destroy_worker ( ) .
*
* CONTEXT :
* Might sleep . Does GFP_KERNEL allocations .
*
* RETURNS :
* Pointer to the newly created worker .
*/
2010-06-29 12:07:13 +04:00
static struct worker * create_worker ( struct global_cwq * gcwq , bool bind )
2010-06-29 12:07:11 +04:00
{
2010-07-02 12:03:51 +04:00
bool on_unbound_cpu = gcwq - > cpu = = WORK_CPU_UNBOUND ;
2010-06-29 12:07:11 +04:00
struct worker * worker = NULL ;
2010-07-02 12:03:51 +04:00
int id = - 1 ;
2010-06-29 12:07:11 +04:00
2010-06-29 12:07:12 +04:00
spin_lock_irq ( & gcwq - > lock ) ;
while ( ida_get_new ( & gcwq - > worker_ida , & id ) ) {
spin_unlock_irq ( & gcwq - > lock ) ;
if ( ! ida_pre_get ( & gcwq - > worker_ida , GFP_KERNEL ) )
2010-06-29 12:07:11 +04:00
goto fail ;
2010-06-29 12:07:12 +04:00
spin_lock_irq ( & gcwq - > lock ) ;
2010-06-29 12:07:11 +04:00
}
2010-06-29 12:07:12 +04:00
spin_unlock_irq ( & gcwq - > lock ) ;
2010-06-29 12:07:11 +04:00
worker = alloc_worker ( ) ;
if ( ! worker )
goto fail ;
2010-06-29 12:07:12 +04:00
worker - > gcwq = gcwq ;
2010-06-29 12:07:11 +04:00
worker - > id = id ;
2010-07-02 12:03:51 +04:00
if ( ! on_unbound_cpu )
2011-03-23 02:30:45 +03:00
worker - > task = kthread_create_on_node ( worker_thread ,
worker ,
cpu_to_node ( gcwq - > cpu ) ,
" kworker/%u:%d " , gcwq - > cpu , id ) ;
2010-07-02 12:03:51 +04:00
else
worker - > task = kthread_create ( worker_thread , worker ,
" kworker/u:%d " , id ) ;
2010-06-29 12:07:11 +04:00
if ( IS_ERR ( worker - > task ) )
goto fail ;
2010-06-29 12:07:12 +04:00
/*
* A rogue worker will become a regular one if CPU comes
* online later on . Make sure every worker has
* PF_THREAD_BOUND set .
*/
2010-07-02 12:03:51 +04:00
if ( bind & & ! on_unbound_cpu )
2010-06-29 12:07:12 +04:00
kthread_bind ( worker - > task , gcwq - > cpu ) ;
2010-07-02 12:03:51 +04:00
else {
2010-06-29 12:07:12 +04:00
worker - > task - > flags | = PF_THREAD_BOUND ;
2010-07-02 12:03:51 +04:00
if ( on_unbound_cpu )
worker - > flags | = WORKER_UNBOUND ;
}
2010-06-29 12:07:11 +04:00
return worker ;
fail :
if ( id > = 0 ) {
2010-06-29 12:07:12 +04:00
spin_lock_irq ( & gcwq - > lock ) ;
ida_remove ( & gcwq - > worker_ida , id ) ;
spin_unlock_irq ( & gcwq - > lock ) ;
2010-06-29 12:07:11 +04:00
}
kfree ( worker ) ;
return NULL ;
}
/**
* start_worker - start a newly created worker
* @ worker : worker to start
*
2010-06-29 12:07:12 +04:00
* Make the gcwq aware of @ worker and start it .
2010-06-29 12:07:11 +04:00
*
* CONTEXT :
2010-06-29 12:07:12 +04:00
* spin_lock_irq ( gcwq - > lock ) .
2010-06-29 12:07:11 +04:00
*/
static void start_worker ( struct worker * worker )
{
2010-07-02 12:03:50 +04:00
worker - > flags | = WORKER_STARTED ;
2010-06-29 12:07:12 +04:00
worker - > gcwq - > nr_workers + + ;
worker_enter_idle ( worker ) ;
2010-06-29 12:07:11 +04:00
wake_up_process ( worker - > task ) ;
}
/**
* destroy_worker - destroy a workqueue worker
* @ worker : worker to be destroyed
*
2010-06-29 12:07:12 +04:00
* Destroy @ worker and adjust @ gcwq stats accordingly .
*
* CONTEXT :
* spin_lock_irq ( gcwq - > lock ) which is released and regrabbed .
2010-06-29 12:07:11 +04:00
*/
static void destroy_worker ( struct worker * worker )
{
2010-06-29 12:07:12 +04:00
struct global_cwq * gcwq = worker - > gcwq ;
2010-06-29 12:07:11 +04:00
int id = worker - > id ;
/* sanity check frenzy */
BUG_ON ( worker - > current_work ) ;
2010-06-29 12:07:12 +04:00
BUG_ON ( ! list_empty ( & worker - > scheduled ) ) ;
2010-06-29 12:07:11 +04:00
2010-06-29 12:07:12 +04:00
if ( worker - > flags & WORKER_STARTED )
gcwq - > nr_workers - - ;
if ( worker - > flags & WORKER_IDLE )
gcwq - > nr_idle - - ;
list_del_init ( & worker - > entry ) ;
2010-07-02 12:03:50 +04:00
worker - > flags | = WORKER_DIE ;
2010-06-29 12:07:12 +04:00
spin_unlock_irq ( & gcwq - > lock ) ;
2010-06-29 12:07:11 +04:00
kthread_stop ( worker - > task ) ;
kfree ( worker ) ;
2010-06-29 12:07:12 +04:00
spin_lock_irq ( & gcwq - > lock ) ;
ida_remove ( & gcwq - > worker_ida , id ) ;
2010-06-29 12:07:11 +04:00
}
2010-06-29 12:07:14 +04:00
static void idle_worker_timeout ( unsigned long __gcwq )
{
struct global_cwq * gcwq = ( void * ) __gcwq ;
spin_lock_irq ( & gcwq - > lock ) ;
if ( too_many_workers ( gcwq ) ) {
struct worker * worker ;
unsigned long expires ;
/* idle_list is kept in LIFO order, check the last one */
worker = list_entry ( gcwq - > idle_list . prev , struct worker , entry ) ;
expires = worker - > last_active + IDLE_WORKER_TIMEOUT ;
if ( time_before ( jiffies , expires ) )
mod_timer ( & gcwq - > idle_timer , expires ) ;
else {
/* it's been idle for too long, wake up manager */
gcwq - > flags | = GCWQ_MANAGE_WORKERS ;
wake_up_worker ( gcwq ) ;
2006-12-07 07:37:26 +03:00
}
2010-06-29 12:07:14 +04:00
}
spin_unlock_irq ( & gcwq - > lock ) ;
}
2006-12-07 07:37:26 +03:00
2010-06-29 12:07:14 +04:00
static bool send_mayday ( struct work_struct * work )
{
struct cpu_workqueue_struct * cwq = get_work_cwq ( work ) ;
struct workqueue_struct * wq = cwq - > wq ;
2010-07-02 12:03:51 +04:00
unsigned int cpu ;
2010-06-29 12:07:14 +04:00
if ( ! ( wq - > flags & WQ_RESCUER ) )
return false ;
/* mayday mayday mayday */
2010-07-02 12:03:51 +04:00
cpu = cwq - > gcwq - > cpu ;
/* WORK_CPU_UNBOUND can't be set in cpumask, use cpu 0 instead */
if ( cpu = = WORK_CPU_UNBOUND )
cpu = 0 ;
2010-07-20 17:59:09 +04:00
if ( ! mayday_test_and_set_cpu ( cpu , wq - > mayday_mask ) )
2010-06-29 12:07:14 +04:00
wake_up_process ( wq - > rescuer - > task ) ;
return true ;
}
static void gcwq_mayday_timeout ( unsigned long __gcwq )
{
struct global_cwq * gcwq = ( void * ) __gcwq ;
struct work_struct * work ;
spin_lock_irq ( & gcwq - > lock ) ;
if ( need_to_create_worker ( gcwq ) ) {
/*
* We ' ve been trying to create a new worker but
* haven ' t been successful . We might be hitting an
* allocation deadlock . Send distress signals to
* rescuers .
*/
list_for_each_entry ( work , & gcwq - > worklist , entry )
send_mayday ( work ) ;
2005-04-17 02:20:36 +04:00
}
2010-06-29 12:07:14 +04:00
spin_unlock_irq ( & gcwq - > lock ) ;
mod_timer ( & gcwq - > mayday_timer , jiffies + MAYDAY_INTERVAL ) ;
2005-04-17 02:20:36 +04:00
}
2010-06-29 12:07:14 +04:00
/**
* maybe_create_worker - create a new worker if necessary
* @ gcwq : gcwq to create a new worker for
*
* Create a new worker for @ gcwq if necessary . @ gcwq is guaranteed to
* have at least one idle worker on return from this function . If
* creating a new worker takes longer than MAYDAY_INTERVAL , mayday is
* sent to all rescuers with works scheduled on @ gcwq to resolve
* possible allocation deadlock .
*
* On return , need_to_create_worker ( ) is guaranteed to be false and
* may_start_working ( ) true .
*
* LOCKING :
* spin_lock_irq ( gcwq - > lock ) which may be released and regrabbed
* multiple times . Does GFP_KERNEL allocations . Called only from
* manager .
*
* RETURNS :
* false if no action was taken and gcwq - > lock stayed locked , true
* otherwise .
*/
static bool maybe_create_worker ( struct global_cwq * gcwq )
2010-08-22 18:19:42 +04:00
__releases ( & gcwq - > lock )
__acquires ( & gcwq - > lock )
2005-04-17 02:20:36 +04:00
{
2010-06-29 12:07:14 +04:00
if ( ! need_to_create_worker ( gcwq ) )
return false ;
restart :
2010-07-14 13:31:20 +04:00
spin_unlock_irq ( & gcwq - > lock ) ;
2010-06-29 12:07:14 +04:00
/* if we don't make progress in MAYDAY_INITIAL_TIMEOUT, call for help */
mod_timer ( & gcwq - > mayday_timer , jiffies + MAYDAY_INITIAL_TIMEOUT ) ;
while ( true ) {
struct worker * worker ;
worker = create_worker ( gcwq , true ) ;
if ( worker ) {
del_timer_sync ( & gcwq - > mayday_timer ) ;
spin_lock_irq ( & gcwq - > lock ) ;
start_worker ( worker ) ;
BUG_ON ( need_to_create_worker ( gcwq ) ) ;
return true ;
}
if ( ! need_to_create_worker ( gcwq ) )
break ;
2005-04-17 02:20:36 +04:00
2010-06-29 12:07:14 +04:00
__set_current_state ( TASK_INTERRUPTIBLE ) ;
schedule_timeout ( CREATE_COOLDOWN ) ;
2010-07-14 13:31:20 +04:00
2010-06-29 12:07:14 +04:00
if ( ! need_to_create_worker ( gcwq ) )
break ;
}
del_timer_sync ( & gcwq - > mayday_timer ) ;
spin_lock_irq ( & gcwq - > lock ) ;
if ( need_to_create_worker ( gcwq ) )
goto restart ;
return true ;
}
/**
* maybe_destroy_worker - destroy workers which have been idle for a while
* @ gcwq : gcwq to destroy workers for
*
* Destroy @ gcwq workers which have been idle for longer than
* IDLE_WORKER_TIMEOUT .
*
* LOCKING :
* spin_lock_irq ( gcwq - > lock ) which may be released and regrabbed
* multiple times . Called only from manager .
*
* RETURNS :
* false if no action was taken and gcwq - > lock stayed locked , true
* otherwise .
*/
static bool maybe_destroy_workers ( struct global_cwq * gcwq )
{
bool ret = false ;
2005-04-17 02:20:36 +04:00
2010-06-29 12:07:14 +04:00
while ( too_many_workers ( gcwq ) ) {
struct worker * worker ;
unsigned long expires ;
2007-05-09 13:34:09 +04:00
2010-06-29 12:07:14 +04:00
worker = list_entry ( gcwq - > idle_list . prev , struct worker , entry ) ;
expires = worker - > last_active + IDLE_WORKER_TIMEOUT ;
2007-05-09 13:34:20 +04:00
2010-06-29 12:07:14 +04:00
if ( time_before ( jiffies , expires ) ) {
mod_timer ( & gcwq - > idle_timer , expires ) ;
2007-05-09 13:34:09 +04:00
break ;
2010-06-29 12:07:14 +04:00
}
2005-04-17 02:20:36 +04:00
2010-06-29 12:07:14 +04:00
destroy_worker ( worker ) ;
ret = true ;
2005-04-17 02:20:36 +04:00
}
2007-05-09 13:34:09 +04:00
2010-06-29 12:07:14 +04:00
return ret ;
}
/**
* manage_workers - manage worker pool
* @ worker : self
*
* Assume the manager role and manage gcwq worker pool @ worker belongs
* to . At any given time , there can be only zero or one manager per
* gcwq . The exclusion is handled automatically by this function .
*
* The caller can safely start processing works on false return . On
* true return , it ' s guaranteed that need_to_create_worker ( ) is false
* and may_start_working ( ) is true .
*
* CONTEXT :
* spin_lock_irq ( gcwq - > lock ) which may be released and regrabbed
* multiple times . Does GFP_KERNEL allocations .
*
* RETURNS :
* false if no action was taken and gcwq - > lock stayed locked , true if
* some action was taken .
*/
static bool manage_workers ( struct worker * worker )
{
struct global_cwq * gcwq = worker - > gcwq ;
bool ret = false ;
if ( gcwq - > flags & GCWQ_MANAGING_WORKERS )
return ret ;
gcwq - > flags & = ~ GCWQ_MANAGE_WORKERS ;
gcwq - > flags | = GCWQ_MANAGING_WORKERS ;
/*
* Destroy and then create so that may_start_working ( ) is true
* on return .
*/
ret | = maybe_destroy_workers ( gcwq ) ;
ret | = maybe_create_worker ( gcwq ) ;
gcwq - > flags & = ~ GCWQ_MANAGING_WORKERS ;
/*
* The trustee might be waiting to take over the manager
* position , tell it we ' re done .
*/
if ( unlikely ( gcwq - > trustee ) )
wake_up_all ( & gcwq - > trustee_wait ) ;
return ret ;
}
2010-06-29 12:07:12 +04:00
/**
* move_linked_works - move linked works to a list
* @ work : start of series of works to be scheduled
* @ head : target list to append @ work to
* @ nextp : out paramter for nested worklist walking
*
* Schedule linked works starting from @ work to @ head . Work series to
* be scheduled starts at @ work and includes any consecutive work with
* WORK_STRUCT_LINKED set in its predecessor .
*
* If @ nextp is not NULL , it ' s updated to point to the next work of
* the last scheduled work . This allows move_linked_works ( ) to be
* nested inside outer list_for_each_entry_safe ( ) .
*
* CONTEXT :
2010-06-29 12:07:12 +04:00
* spin_lock_irq ( gcwq - > lock ) .
2010-06-29 12:07:12 +04:00
*/
static void move_linked_works ( struct work_struct * work , struct list_head * head ,
struct work_struct * * nextp )
{
struct work_struct * n ;
/*
* Linked worklist will always end before the end of the list ,
* use NULL for list head .
*/
list_for_each_entry_safe_from ( work , n , NULL , entry ) {
list_move_tail ( & work - > entry , head ) ;
if ( ! ( * work_data_bits ( work ) & WORK_STRUCT_LINKED ) )
break ;
}
/*
* If we ' re already inside safe list traversal and have moved
* multiple works to the scheduled queue , the next position
* needs to be updated .
*/
if ( nextp )
* nextp = n ;
}
2010-06-29 12:07:12 +04:00
static void cwq_activate_first_delayed ( struct cpu_workqueue_struct * cwq )
{
struct work_struct * work = list_first_entry ( & cwq - > delayed_works ,
struct work_struct , entry ) ;
2010-06-29 12:07:14 +04:00
struct list_head * pos = gcwq_determine_ins_pos ( cwq - > gcwq , cwq ) ;
2010-06-29 12:07:12 +04:00
2010-10-05 12:49:55 +04:00
trace_workqueue_activate_work ( work ) ;
2010-06-29 12:07:14 +04:00
move_linked_works ( work , pos , NULL ) ;
2010-08-25 12:33:56 +04:00
__clear_bit ( WORK_STRUCT_DELAYED_BIT , work_data_bits ( work ) ) ;
2010-06-29 12:07:12 +04:00
cwq - > nr_active + + ;
}
2010-06-29 12:07:11 +04:00
/**
* cwq_dec_nr_in_flight - decrement cwq ' s nr_in_flight
* @ cwq : cwq of interest
* @ color : color of work which left the queue
2010-08-25 12:33:56 +04:00
* @ delayed : for a delayed work
2010-06-29 12:07:11 +04:00
*
* A work either has completed or is removed from pending queue ,
* decrement nr_in_flight of its cwq and handle workqueue flushing .
*
* CONTEXT :
2010-06-29 12:07:12 +04:00
* spin_lock_irq ( gcwq - > lock ) .
2010-06-29 12:07:11 +04:00
*/
2010-08-25 12:33:56 +04:00
static void cwq_dec_nr_in_flight ( struct cpu_workqueue_struct * cwq , int color ,
bool delayed )
2010-06-29 12:07:11 +04:00
{
/* ignore uncolored works */
if ( color = = WORK_NO_COLOR )
return ;
cwq - > nr_in_flight [ color ] - - ;
2010-06-29 12:07:12 +04:00
2010-08-25 12:33:56 +04:00
if ( ! delayed ) {
cwq - > nr_active - - ;
if ( ! list_empty ( & cwq - > delayed_works ) ) {
/* one down, submit a delayed one */
if ( cwq - > nr_active < cwq - > max_active )
cwq_activate_first_delayed ( cwq ) ;
}
2010-06-29 12:07:13 +04:00
}
2010-06-29 12:07:11 +04:00
/* is flush in progress and are we at the flushing tip? */
if ( likely ( cwq - > flush_color ! = color ) )
return ;
/* are there still in-flight works? */
if ( cwq - > nr_in_flight [ color ] )
return ;
/* this cwq is done, clear flush_color */
cwq - > flush_color = - 1 ;
/*
* If this was the last cwq , wake up the first flusher . It
* will handle the rest .
*/
if ( atomic_dec_and_test ( & cwq - > wq - > nr_cwqs_to_flush ) )
complete ( & cwq - > wq - > first_flusher - > done ) ;
}
2010-06-29 12:07:10 +04:00
/**
* process_one_work - process single work
2010-06-29 12:07:11 +04:00
* @ worker : self
2010-06-29 12:07:10 +04:00
* @ work : work to process
*
* Process @ work . This function contains all the logics necessary to
* process a single work including synchronization against and
* interaction with other workers on the same cpu , queueing and
* flushing . As long as context requirement is met , any worker can
* call this function to process a work .
*
* CONTEXT :
2010-06-29 12:07:12 +04:00
* spin_lock_irq ( gcwq - > lock ) which is released and regrabbed .
2010-06-29 12:07:10 +04:00
*/
2010-06-29 12:07:11 +04:00
static void process_one_work ( struct worker * worker , struct work_struct * work )
2010-08-22 18:19:42 +04:00
__releases ( & gcwq - > lock )
__acquires ( & gcwq - > lock )
2010-06-29 12:07:10 +04:00
{
2010-06-29 12:07:13 +04:00
struct cpu_workqueue_struct * cwq = get_work_cwq ( work ) ;
2010-06-29 12:07:12 +04:00
struct global_cwq * gcwq = cwq - > gcwq ;
2010-06-29 12:07:12 +04:00
struct hlist_head * bwh = busy_worker_head ( gcwq , work ) ;
2010-06-29 12:07:15 +04:00
bool cpu_intensive = cwq - > wq - > flags & WQ_CPU_INTENSIVE ;
2010-06-29 12:07:10 +04:00
work_func_t f = work - > func ;
2010-06-29 12:07:11 +04:00
int work_color ;
2010-06-29 12:07:13 +04:00
struct worker * collision ;
2010-06-29 12:07:10 +04:00
# ifdef CONFIG_LOCKDEP
/*
* It is permissible to free the struct work_struct from
* inside the function that is called from it , this we need to
* take into account for lockdep too . To avoid bogus " held
* lock freed " warnings as well as problems when looking into
* work - > lockdep_map , make a copy and use that here .
*/
lockdep: fix oops in processing workqueue
Under memory load, on x86_64, with lockdep enabled, the workqueue's
process_one_work() has been seen to oops in __lock_acquire(), barfing
on a 0xffffffff00000000 pointer in the lockdep_map's class_cache[].
Because it's permissible to free a work_struct from its callout function,
the map used is an onstack copy of the map given in the work_struct: and
that copy is made without any locking.
Surprisingly, gcc (4.5.1 in Hugh's case) uses "rep movsl" rather than
"rep movsq" for that structure copy: which might race with a workqueue
user's wait_on_work() doing lock_map_acquire() on the source of the
copy, putting a pointer into the class_cache[], but only in time for
the top half of that pointer to be copied to the destination map.
Boom when process_one_work() subsequently does lock_map_acquire()
on its onstack copy of the lockdep_map.
Fix this, and a similar instance in call_timer_fn(), with a
lockdep_copy_map() function which additionally NULLs the class_cache[].
Note: this oops was actually seen on 3.4-next, where flush_work() newly
does the racing lock_map_acquire(); but Tejun points out that 3.4 and
earlier are already vulnerable to the same through wait_on_work().
* Patch orginally from Peter. Hugh modified it a bit and wrote the
description.
Signed-off-by: Peter Zijlstra <peterz@infradead.org>
Reported-by: Hugh Dickins <hughd@google.com>
LKML-Reference: <alpine.LSU.2.00.1205070951170.1544@eggly.anvils>
Signed-off-by: Tejun Heo <tj@kernel.org>
2012-05-15 19:06:19 +04:00
struct lockdep_map lockdep_map ;
lockdep_copy_map ( & lockdep_map , & work - > lockdep_map ) ;
2010-06-29 12:07:10 +04:00
# endif
2010-06-29 12:07:13 +04:00
/*
* A single work shouldn ' t be executed concurrently by
* multiple workers on a single cpu . Check whether anyone is
* already processing the work . If so , defer the work to the
* currently executing one .
*/
collision = __find_worker_executing_work ( gcwq , bwh , work ) ;
if ( unlikely ( collision ) ) {
move_linked_works ( work , & collision - > scheduled , NULL ) ;
return ;
}
2010-06-29 12:07:10 +04:00
/* claim and process */
debug_work_deactivate ( work ) ;
2010-06-29 12:07:12 +04:00
hlist_add_head ( & worker - > hentry , bwh ) ;
2010-06-29 12:07:11 +04:00
worker - > current_work = work ;
2010-06-29 12:07:13 +04:00
worker - > current_cwq = cwq ;
2010-06-29 12:07:11 +04:00
work_color = get_work_color ( work ) ;
2010-06-29 12:07:13 +04:00
/* record the current cpu number in the work data and dequeue */
set_work_cpu ( work , gcwq - > cpu ) ;
2010-06-29 12:07:10 +04:00
list_del_init ( & work - > entry ) ;
2010-06-29 12:07:14 +04:00
/*
* If HIGHPRI_PENDING , check the next work , and , if HIGHPRI ,
* wake up another worker ; otherwise , clear HIGHPRI_PENDING .
*/
if ( unlikely ( gcwq - > flags & GCWQ_HIGHPRI_PENDING ) ) {
struct work_struct * nwork = list_first_entry ( & gcwq - > worklist ,
struct work_struct , entry ) ;
if ( ! list_empty ( & gcwq - > worklist ) & &
get_work_cwq ( nwork ) - > wq - > flags & WQ_HIGHPRI )
wake_up_worker ( gcwq ) ;
else
gcwq - > flags & = ~ GCWQ_HIGHPRI_PENDING ;
}
2010-06-29 12:07:15 +04:00
/*
* CPU intensive works don ' t participate in concurrency
* management . They ' re the scheduler ' s responsibility .
*/
if ( unlikely ( cpu_intensive ) )
worker_set_flags ( worker , WORKER_CPU_INTENSIVE , true ) ;
2012-07-13 01:46:37 +04:00
/*
* Unbound gcwq isn ' t concurrency managed and work items should be
* executed ASAP . Wake up another worker if necessary .
*/
if ( ( worker - > flags & WORKER_UNBOUND ) & & need_more_worker ( gcwq ) )
wake_up_worker ( gcwq ) ;
2010-06-29 12:07:12 +04:00
spin_unlock_irq ( & gcwq - > lock ) ;
2010-06-29 12:07:10 +04:00
work_clear_pending ( work ) ;
2011-01-10 01:32:15 +03:00
lock_map_acquire_read ( & cwq - > wq - > lockdep_map ) ;
2010-06-29 12:07:10 +04:00
lock_map_acquire ( & lockdep_map ) ;
2010-08-22 00:07:26 +04:00
trace_workqueue_execute_start ( work ) ;
2010-06-29 12:07:10 +04:00
f ( work ) ;
2010-08-22 00:07:26 +04:00
/*
* While we must be careful to not use " work " after this , the trace
* point will only record its address .
*/
trace_workqueue_execute_end ( work ) ;
2010-06-29 12:07:10 +04:00
lock_map_release ( & lockdep_map ) ;
lock_map_release ( & cwq - > wq - > lockdep_map ) ;
if ( unlikely ( in_atomic ( ) | | lockdep_depth ( current ) > 0 ) ) {
printk ( KERN_ERR " BUG: workqueue leaked lock or atomic: "
" %s/0x%08x/%d \n " ,
current - > comm , preempt_count ( ) , task_pid_nr ( current ) ) ;
printk ( KERN_ERR " last function: " ) ;
print_symbol ( " %s \n " , ( unsigned long ) f ) ;
debug_show_held_locks ( current ) ;
dump_stack ( ) ;
}
2010-06-29 12:07:12 +04:00
spin_lock_irq ( & gcwq - > lock ) ;
2010-06-29 12:07:10 +04:00
2010-06-29 12:07:15 +04:00
/* clear cpu intensive status */
if ( unlikely ( cpu_intensive ) )
worker_clr_flags ( worker , WORKER_CPU_INTENSIVE ) ;
2010-06-29 12:07:10 +04:00
/* we're done with it, release */
2010-06-29 12:07:12 +04:00
hlist_del_init ( & worker - > hentry ) ;
2010-06-29 12:07:11 +04:00
worker - > current_work = NULL ;
2010-06-29 12:07:13 +04:00
worker - > current_cwq = NULL ;
2010-08-25 12:33:56 +04:00
cwq_dec_nr_in_flight ( cwq , work_color , false ) ;
2010-06-29 12:07:10 +04:00
}
2010-06-29 12:07:12 +04:00
/**
* process_scheduled_works - process scheduled works
* @ worker : self
*
* Process all scheduled works . Please note that the scheduled list
* may change while processing a work , so this function repeatedly
* fetches a work from the top and executes it .
*
* CONTEXT :
2010-06-29 12:07:12 +04:00
* spin_lock_irq ( gcwq - > lock ) which may be released and regrabbed
2010-06-29 12:07:12 +04:00
* multiple times .
*/
static void process_scheduled_works ( struct worker * worker )
2005-04-17 02:20:36 +04:00
{
2010-06-29 12:07:12 +04:00
while ( ! list_empty ( & worker - > scheduled ) ) {
struct work_struct * work = list_first_entry ( & worker - > scheduled ,
2005-04-17 02:20:36 +04:00
struct work_struct , entry ) ;
2010-06-29 12:07:11 +04:00
process_one_work ( worker , work ) ;
2005-04-17 02:20:36 +04:00
}
}
2010-06-29 12:07:10 +04:00
/**
* worker_thread - the worker thread function
2010-06-29 12:07:11 +04:00
* @ __worker : self
2010-06-29 12:07:10 +04:00
*
2010-06-29 12:07:14 +04:00
* The gcwq worker thread function . There ' s a single dynamic pool of
* these per each cpu . These workers process all works regardless of
* their specific target workqueue . The only exception is works which
* belong to workqueues with a rescuer which will be explained in
* rescuer_thread ( ) .
2010-06-29 12:07:10 +04:00
*/
2010-06-29 12:07:11 +04:00
static int worker_thread ( void * __worker )
2005-04-17 02:20:36 +04:00
{
2010-06-29 12:07:11 +04:00
struct worker * worker = __worker ;
2010-06-29 12:07:12 +04:00
struct global_cwq * gcwq = worker - > gcwq ;
2005-04-17 02:20:36 +04:00
2010-06-29 12:07:14 +04:00
/* tell the scheduler that this is a workqueue worker */
worker - > task - > flags | = PF_WQ_WORKER ;
2010-06-29 12:07:12 +04:00
woke_up :
spin_lock_irq ( & gcwq - > lock ) ;
2005-04-17 02:20:36 +04:00
2010-06-29 12:07:12 +04:00
/* DIE can be set only while we're idle, checking here is enough */
if ( worker - > flags & WORKER_DIE ) {
spin_unlock_irq ( & gcwq - > lock ) ;
2010-06-29 12:07:14 +04:00
worker - > task - > flags & = ~ PF_WQ_WORKER ;
2010-06-29 12:07:12 +04:00
return 0 ;
}
2010-06-29 12:07:12 +04:00
2010-06-29 12:07:12 +04:00
worker_leave_idle ( worker ) ;
2010-06-29 12:07:12 +04:00
recheck :
2010-06-29 12:07:14 +04:00
/* no more worker necessary? */
if ( ! need_more_worker ( gcwq ) )
goto sleep ;
/* do we need to manage? */
if ( unlikely ( ! may_start_working ( gcwq ) ) & & manage_workers ( worker ) )
goto recheck ;
2010-06-29 12:07:12 +04:00
/*
* - > scheduled list can only be filled while a worker is
* preparing to process a work or actually processing it .
* Make sure nobody diddled with it while I was sleeping .
*/
BUG_ON ( ! list_empty ( & worker - > scheduled ) ) ;
2010-06-29 12:07:14 +04:00
/*
* When control reaches this point , we ' re guaranteed to have
* at least one idle worker or that someone else has already
* assumed the manager role .
*/
worker_clr_flags ( worker , WORKER_PREP ) ;
do {
2010-06-29 12:07:12 +04:00
struct work_struct * work =
2010-06-29 12:07:13 +04:00
list_first_entry ( & gcwq - > worklist ,
2010-06-29 12:07:12 +04:00
struct work_struct , entry ) ;
if ( likely ( ! ( * work_data_bits ( work ) & WORK_STRUCT_LINKED ) ) ) {
/* optimization path, not strictly necessary */
process_one_work ( worker , work ) ;
if ( unlikely ( ! list_empty ( & worker - > scheduled ) ) )
2010-06-29 12:07:12 +04:00
process_scheduled_works ( worker ) ;
2010-06-29 12:07:12 +04:00
} else {
move_linked_works ( work , & worker - > scheduled , NULL ) ;
process_scheduled_works ( worker ) ;
2010-06-29 12:07:12 +04:00
}
2010-06-29 12:07:14 +04:00
} while ( keep_working ( gcwq ) ) ;
worker_set_flags ( worker , WORKER_PREP , false ) ;
2010-07-02 12:03:51 +04:00
sleep :
2010-06-29 12:07:14 +04:00
if ( unlikely ( need_to_manage_workers ( gcwq ) ) & & manage_workers ( worker ) )
goto recheck ;
2010-07-02 12:03:51 +04:00
2010-06-29 12:07:12 +04:00
/*
2010-06-29 12:07:14 +04:00
* gcwq - > lock is held and there ' s no work to process and no
* need to manage , sleep . Workers are woken up only while
* holding gcwq - > lock or from local cpu , so setting the
* current state before releasing gcwq - > lock is enough to
* prevent losing any event .
2010-06-29 12:07:12 +04:00
*/
worker_enter_idle ( worker ) ;
__set_current_state ( TASK_INTERRUPTIBLE ) ;
spin_unlock_irq ( & gcwq - > lock ) ;
schedule ( ) ;
goto woke_up ;
2005-04-17 02:20:36 +04:00
}
2010-06-29 12:07:14 +04:00
/**
* rescuer_thread - the rescuer thread function
* @ __wq : the associated workqueue
*
* Workqueue rescuer thread function . There ' s one rescuer for each
* workqueue which has WQ_RESCUER set .
*
* Regular work processing on a gcwq may block trying to create a new
* worker which uses GFP_KERNEL allocation which has slight chance of
* developing into deadlock if some works currently on the same queue
* need to be processed to satisfy the GFP_KERNEL allocation . This is
* the problem rescuer solves .
*
* When such condition is possible , the gcwq summons rescuers of all
* workqueues which have works queued on the gcwq and let them process
* those works so that forward progress can be guaranteed .
*
* This should happen rarely .
*/
static int rescuer_thread ( void * __wq )
{
struct workqueue_struct * wq = __wq ;
struct worker * rescuer = wq - > rescuer ;
struct list_head * scheduled = & rescuer - > scheduled ;
2010-07-02 12:03:51 +04:00
bool is_unbound = wq - > flags & WQ_UNBOUND ;
2010-06-29 12:07:14 +04:00
unsigned int cpu ;
set_user_nice ( current , RESCUER_NICE_LEVEL ) ;
repeat :
set_current_state ( TASK_INTERRUPTIBLE ) ;
if ( kthread_should_stop ( ) )
return 0 ;
2010-07-02 12:03:51 +04:00
/*
* See whether any cpu is asking for help . Unbounded
* workqueues use cpu 0 in mayday_mask for CPU_UNBOUND .
*/
2010-07-20 17:59:09 +04:00
for_each_mayday_cpu ( cpu , wq - > mayday_mask ) {
2010-07-02 12:03:51 +04:00
unsigned int tcpu = is_unbound ? WORK_CPU_UNBOUND : cpu ;
struct cpu_workqueue_struct * cwq = get_cwq ( tcpu , wq ) ;
2010-06-29 12:07:14 +04:00
struct global_cwq * gcwq = cwq - > gcwq ;
struct work_struct * work , * n ;
__set_current_state ( TASK_RUNNING ) ;
2010-07-20 17:59:09 +04:00
mayday_clear_cpu ( cpu , wq - > mayday_mask ) ;
2010-06-29 12:07:14 +04:00
/* migrate to the target cpu if possible */
rescuer - > gcwq = gcwq ;
worker_maybe_bind_and_lock ( rescuer ) ;
/*
* Slurp in all works issued via this workqueue and
* process ' em .
*/
BUG_ON ( ! list_empty ( & rescuer - > scheduled ) ) ;
list_for_each_entry_safe ( work , n , & gcwq - > worklist , entry )
if ( get_work_cwq ( work ) = = cwq )
move_linked_works ( work , scheduled , & n ) ;
process_scheduled_works ( rescuer ) ;
2011-02-14 16:04:46 +03:00
/*
* Leave this gcwq . If keep_working ( ) is % true , notify a
* regular worker ; otherwise , we end up with 0 concurrency
* and stalling the execution .
*/
if ( keep_working ( gcwq ) )
wake_up_worker ( gcwq ) ;
2010-06-29 12:07:14 +04:00
spin_unlock_irq ( & gcwq - > lock ) ;
}
schedule ( ) ;
goto repeat ;
2005-04-17 02:20:36 +04:00
}
2007-05-09 13:33:51 +04:00
struct wq_barrier {
struct work_struct work ;
struct completion done ;
} ;
static void wq_barrier_func ( struct work_struct * work )
{
struct wq_barrier * barr = container_of ( work , struct wq_barrier , work ) ;
complete ( & barr - > done ) ;
}
2010-06-29 12:07:10 +04:00
/**
* insert_wq_barrier - insert a barrier work
* @ cwq : cwq to insert barrier into
* @ barr : wq_barrier to insert
2010-06-29 12:07:12 +04:00
* @ target : target work to attach @ barr to
* @ worker : worker currently executing @ target , NULL if @ target is not executing
2010-06-29 12:07:10 +04:00
*
2010-06-29 12:07:12 +04:00
* @ barr is linked to @ target such that @ barr is completed only after
* @ target finishes execution . Please note that the ordering
* guarantee is observed only with respect to @ target and on the local
* cpu .
*
* Currently , a queued barrier can ' t be canceled . This is because
* try_to_grab_pending ( ) can ' t determine whether the work to be
* grabbed is at the head of the queue and thus can ' t clear LINKED
* flag of the previous work while there must be a valid next work
* after a work with LINKED flag set .
*
* Note that when @ worker is non - NULL , @ target may be modified
* underneath us , so we can ' t reliably determine cwq from @ target .
2010-06-29 12:07:10 +04:00
*
* CONTEXT :
2010-06-29 12:07:12 +04:00
* spin_lock_irq ( gcwq - > lock ) .
2010-06-29 12:07:10 +04:00
*/
2007-05-09 13:33:54 +04:00
static void insert_wq_barrier ( struct cpu_workqueue_struct * cwq ,
2010-06-29 12:07:12 +04:00
struct wq_barrier * barr ,
struct work_struct * target , struct worker * worker )
2007-05-09 13:33:51 +04:00
{
2010-06-29 12:07:12 +04:00
struct list_head * head ;
unsigned int linked = 0 ;
2009-11-15 19:09:48 +03:00
/*
2010-06-29 12:07:12 +04:00
* debugobject calls are safe here even with gcwq - > lock locked
2009-11-15 19:09:48 +03:00
* as we know for sure that this will not trigger any of the
* checks and call back into the fixup functions where we
* might deadlock .
*/
2010-10-27 01:22:34 +04:00
INIT_WORK_ONSTACK ( & barr - > work , wq_barrier_func ) ;
2010-06-29 12:07:10 +04:00
__set_bit ( WORK_STRUCT_PENDING_BIT , work_data_bits ( & barr - > work ) ) ;
2007-05-09 13:33:51 +04:00
init_completion ( & barr - > done ) ;
2007-05-09 13:33:54 +04:00
2010-06-29 12:07:12 +04:00
/*
* If @ target is currently being executed , schedule the
* barrier to the worker ; otherwise , put it after @ target .
*/
if ( worker )
head = worker - > scheduled . next ;
else {
unsigned long * bits = work_data_bits ( target ) ;
head = target - > entry . next ;
/* there can already be other linked works, inherit and set */
linked = * bits & WORK_STRUCT_LINKED ;
__set_bit ( WORK_STRUCT_LINKED_BIT , bits ) ;
}
2009-11-15 19:09:48 +03:00
debug_work_activate ( & barr - > work ) ;
2010-06-29 12:07:12 +04:00
insert_work ( cwq , & barr - > work , head ,
work_color_to_flags ( WORK_NO_COLOR ) | linked ) ;
2007-05-09 13:33:51 +04:00
}
2010-06-29 12:07:11 +04:00
/**
* flush_workqueue_prep_cwqs - prepare cwqs for workqueue flushing
* @ wq : workqueue being flushed
* @ flush_color : new flush color , < 0 for no - op
* @ work_color : new work color , < 0 for no - op
*
* Prepare cwqs for workqueue flushing .
*
* If @ flush_color is non - negative , flush_color on all cwqs should be
* - 1. If no cwq has in - flight commands at the specified color , all
* cwq - > flush_color ' s stay at - 1 and % false is returned . If any cwq
* has in flight commands , its cwq - > flush_color is set to
* @ flush_color , @ wq - > nr_cwqs_to_flush is updated accordingly , cwq
* wakeup logic is armed and % true is returned .
*
* The caller should have initialized @ wq - > first_flusher prior to
* calling this function with non - negative @ flush_color . If
* @ flush_color is negative , no flush color update is done and % false
* is returned .
*
* If @ work_color is non - negative , all cwqs should have the same
* work_color which is previous to @ work_color and all will be
* advanced to @ work_color .
*
* CONTEXT :
* mutex_lock ( wq - > flush_mutex ) .
*
* RETURNS :
* % true if @ flush_color > = 0 and there ' s something to flush . % false
* otherwise .
*/
static bool flush_workqueue_prep_cwqs ( struct workqueue_struct * wq ,
int flush_color , int work_color )
2005-04-17 02:20:36 +04:00
{
2010-06-29 12:07:11 +04:00
bool wait = false ;
unsigned int cpu ;
2005-04-17 02:20:36 +04:00
2010-06-29 12:07:11 +04:00
if ( flush_color > = 0 ) {
BUG_ON ( atomic_read ( & wq - > nr_cwqs_to_flush ) ) ;
atomic_set ( & wq - > nr_cwqs_to_flush , 1 ) ;
2005-04-17 02:20:36 +04:00
}
2009-04-03 03:58:24 +04:00
2010-07-02 12:03:51 +04:00
for_each_cwq_cpu ( cpu , wq ) {
2010-06-29 12:07:11 +04:00
struct cpu_workqueue_struct * cwq = get_cwq ( cpu , wq ) ;
2010-06-29 12:07:12 +04:00
struct global_cwq * gcwq = cwq - > gcwq ;
2007-05-09 13:33:51 +04:00
2010-06-29 12:07:12 +04:00
spin_lock_irq ( & gcwq - > lock ) ;
2007-05-09 13:33:54 +04:00
2010-06-29 12:07:11 +04:00
if ( flush_color > = 0 ) {
BUG_ON ( cwq - > flush_color ! = - 1 ) ;
2007-05-09 13:33:51 +04:00
2010-06-29 12:07:11 +04:00
if ( cwq - > nr_in_flight [ flush_color ] ) {
cwq - > flush_color = flush_color ;
atomic_inc ( & wq - > nr_cwqs_to_flush ) ;
wait = true ;
}
}
2005-04-17 02:20:36 +04:00
2010-06-29 12:07:11 +04:00
if ( work_color > = 0 ) {
BUG_ON ( work_color ! = work_next_color ( cwq - > work_color ) ) ;
cwq - > work_color = work_color ;
}
2005-04-17 02:20:36 +04:00
2010-06-29 12:07:12 +04:00
spin_unlock_irq ( & gcwq - > lock ) ;
2005-04-17 02:20:36 +04:00
}
2009-04-03 03:58:24 +04:00
2010-06-29 12:07:11 +04:00
if ( flush_color > = 0 & & atomic_dec_and_test ( & wq - > nr_cwqs_to_flush ) )
complete ( & wq - > first_flusher - > done ) ;
2007-05-24 00:57:57 +04:00
2010-06-29 12:07:11 +04:00
return wait ;
2005-04-17 02:20:36 +04:00
}
2006-07-30 14:03:42 +04:00
/**
2005-04-17 02:20:36 +04:00
* flush_workqueue - ensure that any scheduled work has run to completion .
2006-07-30 14:03:42 +04:00
* @ wq : workqueue to flush
2005-04-17 02:20:36 +04:00
*
* Forces execution of the workqueue and blocks until its completion .
* This is typically used in driver shutdown handlers .
*
2007-05-09 13:33:51 +04:00
* We sleep until all works which were queued on entry have been handled ,
* but we are not livelocked by new incoming ones .
2005-04-17 02:20:36 +04:00
*/
2008-02-08 15:19:53 +03:00
void flush_workqueue ( struct workqueue_struct * wq )
2005-04-17 02:20:36 +04:00
{
2010-06-29 12:07:11 +04:00
struct wq_flusher this_flusher = {
. list = LIST_HEAD_INIT ( this_flusher . list ) ,
. flush_color = - 1 ,
. done = COMPLETION_INITIALIZER_ONSTACK ( this_flusher . done ) ,
} ;
int next_color ;
2005-04-17 02:20:36 +04:00
2008-08-11 12:30:30 +04:00
lock_map_acquire ( & wq - > lockdep_map ) ;
lock_map_release ( & wq - > lockdep_map ) ;
2010-06-29 12:07:11 +04:00
mutex_lock ( & wq - > flush_mutex ) ;
/*
* Start - to - wait phase
*/
next_color = work_next_color ( wq - > work_color ) ;
if ( next_color ! = wq - > flush_color ) {
/*
* Color space is not full . The current work_color
* becomes our flush_color and work_color is advanced
* by one .
*/
BUG_ON ( ! list_empty ( & wq - > flusher_overflow ) ) ;
this_flusher . flush_color = wq - > work_color ;
wq - > work_color = next_color ;
if ( ! wq - > first_flusher ) {
/* no flush in progress, become the first flusher */
BUG_ON ( wq - > flush_color ! = this_flusher . flush_color ) ;
wq - > first_flusher = & this_flusher ;
if ( ! flush_workqueue_prep_cwqs ( wq , wq - > flush_color ,
wq - > work_color ) ) {
/* nothing to flush, done */
wq - > flush_color = next_color ;
wq - > first_flusher = NULL ;
goto out_unlock ;
}
} else {
/* wait in queue */
BUG_ON ( wq - > flush_color = = this_flusher . flush_color ) ;
list_add_tail ( & this_flusher . list , & wq - > flusher_queue ) ;
flush_workqueue_prep_cwqs ( wq , - 1 , wq - > work_color ) ;
}
} else {
/*
* Oops , color space is full , wait on overflow queue .
* The next flush completion will assign us
* flush_color and transfer to flusher_queue .
*/
list_add_tail ( & this_flusher . list , & wq - > flusher_overflow ) ;
}
mutex_unlock ( & wq - > flush_mutex ) ;
wait_for_completion ( & this_flusher . done ) ;
/*
* Wake - up - and - cascade phase
*
* First flushers are responsible for cascading flushes and
* handling overflow . Non - first flushers can simply return .
*/
if ( wq - > first_flusher ! = & this_flusher )
return ;
mutex_lock ( & wq - > flush_mutex ) ;
2010-07-02 12:03:51 +04:00
/* we might have raced, check again with mutex held */
if ( wq - > first_flusher ! = & this_flusher )
goto out_unlock ;
2010-06-29 12:07:11 +04:00
wq - > first_flusher = NULL ;
BUG_ON ( ! list_empty ( & this_flusher . list ) ) ;
BUG_ON ( wq - > flush_color ! = this_flusher . flush_color ) ;
while ( true ) {
struct wq_flusher * next , * tmp ;
/* complete all the flushers sharing the current flush color */
list_for_each_entry_safe ( next , tmp , & wq - > flusher_queue , list ) {
if ( next - > flush_color ! = wq - > flush_color )
break ;
list_del_init ( & next - > list ) ;
complete ( & next - > done ) ;
}
BUG_ON ( ! list_empty ( & wq - > flusher_overflow ) & &
wq - > flush_color ! = work_next_color ( wq - > work_color ) ) ;
/* this flush_color is finished, advance by one */
wq - > flush_color = work_next_color ( wq - > flush_color ) ;
/* one color has been freed, handle overflow queue */
if ( ! list_empty ( & wq - > flusher_overflow ) ) {
/*
* Assign the same color to all overflowed
* flushers , advance work_color and append to
* flusher_queue . This is the start - to - wait
* phase for these overflowed flushers .
*/
list_for_each_entry ( tmp , & wq - > flusher_overflow , list )
tmp - > flush_color = wq - > work_color ;
wq - > work_color = work_next_color ( wq - > work_color ) ;
list_splice_tail_init ( & wq - > flusher_overflow ,
& wq - > flusher_queue ) ;
flush_workqueue_prep_cwqs ( wq , - 1 , wq - > work_color ) ;
}
if ( list_empty ( & wq - > flusher_queue ) ) {
BUG_ON ( wq - > flush_color ! = wq - > work_color ) ;
break ;
}
/*
* Need to flush more colors . Make the next flusher
* the new first flusher and arm cwqs .
*/
BUG_ON ( wq - > flush_color = = wq - > work_color ) ;
BUG_ON ( wq - > flush_color ! = next - > flush_color ) ;
list_del_init ( & next - > list ) ;
wq - > first_flusher = next ;
if ( flush_workqueue_prep_cwqs ( wq , wq - > flush_color , - 1 ) )
break ;
/*
* Meh . . . this color is already done , clear first
* flusher and repeat cascading .
*/
wq - > first_flusher = NULL ;
}
out_unlock :
mutex_unlock ( & wq - > flush_mutex ) ;
2005-04-17 02:20:36 +04:00
}
2006-06-30 09:40:45 +04:00
EXPORT_SYMBOL_GPL ( flush_workqueue ) ;
2005-04-17 02:20:36 +04:00
2011-04-05 20:01:44 +04:00
/**
* drain_workqueue - drain a workqueue
* @ wq : workqueue to drain
*
* Wait until the workqueue becomes empty . While draining is in progress ,
* only chain queueing is allowed . IOW , only currently pending or running
* work items on @ wq can queue further work items on it . @ wq is flushed
* repeatedly until it becomes empty . The number of flushing is detemined
* by the depth of chaining and should be relatively short . Whine if it
* takes too long .
*/
void drain_workqueue ( struct workqueue_struct * wq )
{
unsigned int flush_cnt = 0 ;
unsigned int cpu ;
/*
* __queue_work ( ) needs to test whether there are drainers , is much
* hotter than drain_workqueue ( ) and already looks at @ wq - > flags .
* Use WQ_DRAINING so that queue doesn ' t have to check nr_drainers .
*/
spin_lock ( & workqueue_lock ) ;
if ( ! wq - > nr_drainers + + )
wq - > flags | = WQ_DRAINING ;
spin_unlock ( & workqueue_lock ) ;
reflush :
flush_workqueue ( wq ) ;
for_each_cwq_cpu ( cpu , wq ) {
struct cpu_workqueue_struct * cwq = get_cwq ( cpu , wq ) ;
2011-09-15 03:22:28 +04:00
bool drained ;
2011-04-05 20:01:44 +04:00
2011-09-15 03:22:28 +04:00
spin_lock_irq ( & cwq - > gcwq - > lock ) ;
drained = ! cwq - > nr_active & & list_empty ( & cwq - > delayed_works ) ;
spin_unlock_irq ( & cwq - > gcwq - > lock ) ;
if ( drained )
2011-04-05 20:01:44 +04:00
continue ;
if ( + + flush_cnt = = 10 | |
( flush_cnt % 100 = = 0 & & flush_cnt < = 1000 ) )
pr_warning ( " workqueue %s: flush on destruction isn't complete after %u tries \n " ,
wq - > name , flush_cnt ) ;
goto reflush ;
}
spin_lock ( & workqueue_lock ) ;
if ( ! - - wq - > nr_drainers )
wq - > flags & = ~ WQ_DRAINING ;
spin_unlock ( & workqueue_lock ) ;
}
EXPORT_SYMBOL_GPL ( drain_workqueue ) ;
2010-09-16 12:42:16 +04:00
static bool start_flush_work ( struct work_struct * work , struct wq_barrier * barr ,
bool wait_executing )
workqueues: implement flush_work()
Most of users of flush_workqueue() can be changed to use cancel_work_sync(),
but sometimes we really need to wait for the completion and cancelling is not
an option. schedule_on_each_cpu() is good example.
Add the new helper, flush_work(work), which waits for the completion of the
specific work_struct. More precisely, it "flushes" the result of of the last
queue_work() which is visible to the caller.
For example, this code
queue_work(wq, work);
/* WINDOW */
queue_work(wq, work);
flush_work(work);
doesn't necessary work "as expected". What can happen in the WINDOW above is
- wq starts the execution of work->func()
- the caller migrates to another CPU
now, after the 2nd queue_work() this work is active on the previous CPU, and
at the same time it is queued on another. In this case flush_work(work) may
return before the first work->func() completes.
It is trivial to add another helper
int flush_work_sync(struct work_struct *work)
{
return flush_work(work) || wait_on_work(work);
}
which works "more correctly", but it has to iterate over all CPUs and thus
it much slower than flush_work().
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Acked-by: Max Krasnyansky <maxk@qualcomm.com>
Acked-by: Jarek Poplawski <jarkao2@gmail.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-07-25 12:47:49 +04:00
{
2010-06-29 12:07:12 +04:00
struct worker * worker = NULL ;
2010-06-29 12:07:12 +04:00
struct global_cwq * gcwq ;
workqueues: implement flush_work()
Most of users of flush_workqueue() can be changed to use cancel_work_sync(),
but sometimes we really need to wait for the completion and cancelling is not
an option. schedule_on_each_cpu() is good example.
Add the new helper, flush_work(work), which waits for the completion of the
specific work_struct. More precisely, it "flushes" the result of of the last
queue_work() which is visible to the caller.
For example, this code
queue_work(wq, work);
/* WINDOW */
queue_work(wq, work);
flush_work(work);
doesn't necessary work "as expected". What can happen in the WINDOW above is
- wq starts the execution of work->func()
- the caller migrates to another CPU
now, after the 2nd queue_work() this work is active on the previous CPU, and
at the same time it is queued on another. In this case flush_work(work) may
return before the first work->func() completes.
It is trivial to add another helper
int flush_work_sync(struct work_struct *work)
{
return flush_work(work) || wait_on_work(work);
}
which works "more correctly", but it has to iterate over all CPUs and thus
it much slower than flush_work().
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Acked-by: Max Krasnyansky <maxk@qualcomm.com>
Acked-by: Jarek Poplawski <jarkao2@gmail.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-07-25 12:47:49 +04:00
struct cpu_workqueue_struct * cwq ;
might_sleep ( ) ;
2010-06-29 12:07:13 +04:00
gcwq = get_work_gcwq ( work ) ;
if ( ! gcwq )
2010-09-16 12:42:16 +04:00
return false ;
workqueues: implement flush_work()
Most of users of flush_workqueue() can be changed to use cancel_work_sync(),
but sometimes we really need to wait for the completion and cancelling is not
an option. schedule_on_each_cpu() is good example.
Add the new helper, flush_work(work), which waits for the completion of the
specific work_struct. More precisely, it "flushes" the result of of the last
queue_work() which is visible to the caller.
For example, this code
queue_work(wq, work);
/* WINDOW */
queue_work(wq, work);
flush_work(work);
doesn't necessary work "as expected". What can happen in the WINDOW above is
- wq starts the execution of work->func()
- the caller migrates to another CPU
now, after the 2nd queue_work() this work is active on the previous CPU, and
at the same time it is queued on another. In this case flush_work(work) may
return before the first work->func() completes.
It is trivial to add another helper
int flush_work_sync(struct work_struct *work)
{
return flush_work(work) || wait_on_work(work);
}
which works "more correctly", but it has to iterate over all CPUs and thus
it much slower than flush_work().
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Acked-by: Max Krasnyansky <maxk@qualcomm.com>
Acked-by: Jarek Poplawski <jarkao2@gmail.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-07-25 12:47:49 +04:00
2010-06-29 12:07:12 +04:00
spin_lock_irq ( & gcwq - > lock ) ;
workqueues: implement flush_work()
Most of users of flush_workqueue() can be changed to use cancel_work_sync(),
but sometimes we really need to wait for the completion and cancelling is not
an option. schedule_on_each_cpu() is good example.
Add the new helper, flush_work(work), which waits for the completion of the
specific work_struct. More precisely, it "flushes" the result of of the last
queue_work() which is visible to the caller.
For example, this code
queue_work(wq, work);
/* WINDOW */
queue_work(wq, work);
flush_work(work);
doesn't necessary work "as expected". What can happen in the WINDOW above is
- wq starts the execution of work->func()
- the caller migrates to another CPU
now, after the 2nd queue_work() this work is active on the previous CPU, and
at the same time it is queued on another. In this case flush_work(work) may
return before the first work->func() completes.
It is trivial to add another helper
int flush_work_sync(struct work_struct *work)
{
return flush_work(work) || wait_on_work(work);
}
which works "more correctly", but it has to iterate over all CPUs and thus
it much slower than flush_work().
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Acked-by: Max Krasnyansky <maxk@qualcomm.com>
Acked-by: Jarek Poplawski <jarkao2@gmail.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-07-25 12:47:49 +04:00
if ( ! list_empty ( & work - > entry ) ) {
/*
* See the comment near try_to_grab_pending ( ) - > smp_rmb ( ) .
2010-06-29 12:07:13 +04:00
* If it was re - queued to a different gcwq under us , we
* are not going to wait .
workqueues: implement flush_work()
Most of users of flush_workqueue() can be changed to use cancel_work_sync(),
but sometimes we really need to wait for the completion and cancelling is not
an option. schedule_on_each_cpu() is good example.
Add the new helper, flush_work(work), which waits for the completion of the
specific work_struct. More precisely, it "flushes" the result of of the last
queue_work() which is visible to the caller.
For example, this code
queue_work(wq, work);
/* WINDOW */
queue_work(wq, work);
flush_work(work);
doesn't necessary work "as expected". What can happen in the WINDOW above is
- wq starts the execution of work->func()
- the caller migrates to another CPU
now, after the 2nd queue_work() this work is active on the previous CPU, and
at the same time it is queued on another. In this case flush_work(work) may
return before the first work->func() completes.
It is trivial to add another helper
int flush_work_sync(struct work_struct *work)
{
return flush_work(work) || wait_on_work(work);
}
which works "more correctly", but it has to iterate over all CPUs and thus
it much slower than flush_work().
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Acked-by: Max Krasnyansky <maxk@qualcomm.com>
Acked-by: Jarek Poplawski <jarkao2@gmail.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-07-25 12:47:49 +04:00
*/
smp_rmb ( ) ;
2010-06-29 12:07:13 +04:00
cwq = get_work_cwq ( work ) ;
if ( unlikely ( ! cwq | | gcwq ! = cwq - > gcwq ) )
2010-06-29 12:07:10 +04:00
goto already_gone ;
2010-09-16 12:42:16 +04:00
} else if ( wait_executing ) {
2010-06-29 12:07:13 +04:00
worker = find_worker_executing_work ( gcwq , work ) ;
2010-06-29 12:07:12 +04:00
if ( ! worker )
2010-06-29 12:07:10 +04:00
goto already_gone ;
2010-06-29 12:07:13 +04:00
cwq = worker - > current_cwq ;
2010-09-16 12:42:16 +04:00
} else
goto already_gone ;
workqueues: implement flush_work()
Most of users of flush_workqueue() can be changed to use cancel_work_sync(),
but sometimes we really need to wait for the completion and cancelling is not
an option. schedule_on_each_cpu() is good example.
Add the new helper, flush_work(work), which waits for the completion of the
specific work_struct. More precisely, it "flushes" the result of of the last
queue_work() which is visible to the caller.
For example, this code
queue_work(wq, work);
/* WINDOW */
queue_work(wq, work);
flush_work(work);
doesn't necessary work "as expected". What can happen in the WINDOW above is
- wq starts the execution of work->func()
- the caller migrates to another CPU
now, after the 2nd queue_work() this work is active on the previous CPU, and
at the same time it is queued on another. In this case flush_work(work) may
return before the first work->func() completes.
It is trivial to add another helper
int flush_work_sync(struct work_struct *work)
{
return flush_work(work) || wait_on_work(work);
}
which works "more correctly", but it has to iterate over all CPUs and thus
it much slower than flush_work().
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Acked-by: Max Krasnyansky <maxk@qualcomm.com>
Acked-by: Jarek Poplawski <jarkao2@gmail.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-07-25 12:47:49 +04:00
2010-09-16 12:42:16 +04:00
insert_wq_barrier ( cwq , barr , work , worker ) ;
2010-06-29 12:07:12 +04:00
spin_unlock_irq ( & gcwq - > lock ) ;
2010-06-29 12:07:13 +04:00
2011-01-10 01:32:15 +03:00
/*
* If @ max_active is 1 or rescuer is in use , flushing another work
* item on the same workqueue may lead to deadlock . Make sure the
* flusher is not running on the same workqueue by verifying write
* access .
*/
if ( cwq - > wq - > saved_max_active = = 1 | | cwq - > wq - > flags & WQ_RESCUER )
lock_map_acquire ( & cwq - > wq - > lockdep_map ) ;
else
lock_map_acquire_read ( & cwq - > wq - > lockdep_map ) ;
2010-06-29 12:07:13 +04:00
lock_map_release ( & cwq - > wq - > lockdep_map ) ;
2011-01-10 01:32:15 +03:00
2010-09-16 12:36:00 +04:00
return true ;
2010-06-29 12:07:10 +04:00
already_gone :
2010-06-29 12:07:12 +04:00
spin_unlock_irq ( & gcwq - > lock ) ;
2010-09-16 12:36:00 +04:00
return false ;
workqueues: implement flush_work()
Most of users of flush_workqueue() can be changed to use cancel_work_sync(),
but sometimes we really need to wait for the completion and cancelling is not
an option. schedule_on_each_cpu() is good example.
Add the new helper, flush_work(work), which waits for the completion of the
specific work_struct. More precisely, it "flushes" the result of of the last
queue_work() which is visible to the caller.
For example, this code
queue_work(wq, work);
/* WINDOW */
queue_work(wq, work);
flush_work(work);
doesn't necessary work "as expected". What can happen in the WINDOW above is
- wq starts the execution of work->func()
- the caller migrates to another CPU
now, after the 2nd queue_work() this work is active on the previous CPU, and
at the same time it is queued on another. In this case flush_work(work) may
return before the first work->func() completes.
It is trivial to add another helper
int flush_work_sync(struct work_struct *work)
{
return flush_work(work) || wait_on_work(work);
}
which works "more correctly", but it has to iterate over all CPUs and thus
it much slower than flush_work().
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Acked-by: Max Krasnyansky <maxk@qualcomm.com>
Acked-by: Jarek Poplawski <jarkao2@gmail.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-07-25 12:47:49 +04:00
}
2010-09-16 12:42:16 +04:00
/**
* flush_work - wait for a work to finish executing the last queueing instance
* @ work : the work to flush
*
* Wait until @ work has finished execution . This function considers
* only the last queueing instance of @ work . If @ work has been
* enqueued across different CPUs on a non - reentrant workqueue or on
* multiple workqueues , @ work might still be executing on return on
* some of the CPUs from earlier queueing .
*
* If @ work was queued only on a non - reentrant , ordered or unbound
* workqueue , @ work is guaranteed to be idle on return if it hasn ' t
* been requeued since flush started .
*
* RETURNS :
* % true if flush_work ( ) waited for the work to finish execution ,
* % false if it was already idle .
*/
bool flush_work ( struct work_struct * work )
{
struct wq_barrier barr ;
2012-04-21 04:28:50 +04:00
lock_map_acquire ( & work - > lockdep_map ) ;
lock_map_release ( & work - > lockdep_map ) ;
2010-09-16 12:42:16 +04:00
if ( start_flush_work ( work , & barr , true ) ) {
wait_for_completion ( & barr . done ) ;
destroy_work_on_stack ( & barr . work ) ;
return true ;
} else
return false ;
}
workqueues: implement flush_work()
Most of users of flush_workqueue() can be changed to use cancel_work_sync(),
but sometimes we really need to wait for the completion and cancelling is not
an option. schedule_on_each_cpu() is good example.
Add the new helper, flush_work(work), which waits for the completion of the
specific work_struct. More precisely, it "flushes" the result of of the last
queue_work() which is visible to the caller.
For example, this code
queue_work(wq, work);
/* WINDOW */
queue_work(wq, work);
flush_work(work);
doesn't necessary work "as expected". What can happen in the WINDOW above is
- wq starts the execution of work->func()
- the caller migrates to another CPU
now, after the 2nd queue_work() this work is active on the previous CPU, and
at the same time it is queued on another. In this case flush_work(work) may
return before the first work->func() completes.
It is trivial to add another helper
int flush_work_sync(struct work_struct *work)
{
return flush_work(work) || wait_on_work(work);
}
which works "more correctly", but it has to iterate over all CPUs and thus
it much slower than flush_work().
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Acked-by: Max Krasnyansky <maxk@qualcomm.com>
Acked-by: Jarek Poplawski <jarkao2@gmail.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-07-25 12:47:49 +04:00
EXPORT_SYMBOL_GPL ( flush_work ) ;
2010-09-16 12:36:00 +04:00
static bool wait_on_cpu_work ( struct global_cwq * gcwq , struct work_struct * work )
{
struct wq_barrier barr ;
struct worker * worker ;
spin_lock_irq ( & gcwq - > lock ) ;
worker = find_worker_executing_work ( gcwq , work ) ;
if ( unlikely ( worker ) )
insert_wq_barrier ( worker - > current_cwq , & barr , work , worker ) ;
spin_unlock_irq ( & gcwq - > lock ) ;
if ( unlikely ( worker ) ) {
wait_for_completion ( & barr . done ) ;
destroy_work_on_stack ( & barr . work ) ;
return true ;
} else
return false ;
}
static bool wait_on_work ( struct work_struct * work )
{
bool ret = false ;
int cpu ;
might_sleep ( ) ;
lock_map_acquire ( & work - > lockdep_map ) ;
lock_map_release ( & work - > lockdep_map ) ;
for_each_gcwq_cpu ( cpu )
ret | = wait_on_cpu_work ( get_gcwq ( cpu ) , work ) ;
return ret ;
}
2010-09-16 12:48:29 +04:00
/**
* flush_work_sync - wait until a work has finished execution
* @ work : the work to flush
*
* Wait until @ work has finished execution . On return , it ' s
* guaranteed that all queueing instances of @ work which happened
* before this function is called are finished . In other words , if
* @ work hasn ' t been requeued since this function was called , @ work is
* guaranteed to be idle on return .
*
* RETURNS :
* % true if flush_work_sync ( ) waited for the work to finish execution ,
* % false if it was already idle .
*/
bool flush_work_sync ( struct work_struct * work )
{
struct wq_barrier barr ;
bool pending , waited ;
/* we'll wait for executions separately, queue barr only if pending */
pending = start_flush_work ( work , & barr , false ) ;
/* wait for executions to finish */
waited = wait_on_work ( work ) ;
/* wait for the pending one */
if ( pending ) {
wait_for_completion ( & barr . done ) ;
destroy_work_on_stack ( & barr . work ) ;
}
return pending | | waited ;
}
EXPORT_SYMBOL_GPL ( flush_work_sync ) ;
make cancel_rearming_delayed_work() reliable
Thanks to Jarek Poplawski for the ideas and for spotting the bug in the
initial draft patch.
cancel_rearming_delayed_work() currently has many limitations, because it
requires that dwork always re-arms itself via queue_delayed_work(). So it
hangs forever if dwork doesn't do this, or cancel_rearming_delayed_work/
cancel_delayed_work was already called. It uses flush_workqueue() in a
loop, so it can't be used if workqueue was freezed, and it is potentially
live- lockable on busy system if delay is small.
With this patch cancel_rearming_delayed_work() doesn't make any assumptions
about dwork, it can re-arm itself via queue_delayed_work(), or
queue_work(), or do nothing.
As a "side effect", cancel_work_sync() was changed to handle re-arming works
as well.
Disadvantages:
- this patch adds wmb() to insert_work().
- slowdowns the fast path (when del_timer() succeeds on entry) of
cancel_rearming_delayed_work(), because wait_on_work() is called
unconditionally. In that case, compared to the old version, we are
doing "unneeded" lock/unlock for each online CPU.
On the other hand, this means we don't need to use cancel_work_sync()
after cancel_rearming_delayed_work().
- complicates the code (.text grows by 130 bytes).
[akpm@linux-foundation.org: fix speling]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Cc: David Chinner <dgc@sgi.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Gautham Shenoy <ego@in.ibm.com>
Acked-by: Jarek Poplawski <jarkao2@o2.pl>
Cc: Srivatsa Vaddagiri <vatsa@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 13:34:46 +04:00
/*
2007-07-16 10:41:44 +04:00
* Upon a successful return ( > = 0 ) , the caller " owns " WORK_STRUCT_PENDING bit ,
make cancel_rearming_delayed_work() reliable
Thanks to Jarek Poplawski for the ideas and for spotting the bug in the
initial draft patch.
cancel_rearming_delayed_work() currently has many limitations, because it
requires that dwork always re-arms itself via queue_delayed_work(). So it
hangs forever if dwork doesn't do this, or cancel_rearming_delayed_work/
cancel_delayed_work was already called. It uses flush_workqueue() in a
loop, so it can't be used if workqueue was freezed, and it is potentially
live- lockable on busy system if delay is small.
With this patch cancel_rearming_delayed_work() doesn't make any assumptions
about dwork, it can re-arm itself via queue_delayed_work(), or
queue_work(), or do nothing.
As a "side effect", cancel_work_sync() was changed to handle re-arming works
as well.
Disadvantages:
- this patch adds wmb() to insert_work().
- slowdowns the fast path (when del_timer() succeeds on entry) of
cancel_rearming_delayed_work(), because wait_on_work() is called
unconditionally. In that case, compared to the old version, we are
doing "unneeded" lock/unlock for each online CPU.
On the other hand, this means we don't need to use cancel_work_sync()
after cancel_rearming_delayed_work().
- complicates the code (.text grows by 130 bytes).
[akpm@linux-foundation.org: fix speling]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Cc: David Chinner <dgc@sgi.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Gautham Shenoy <ego@in.ibm.com>
Acked-by: Jarek Poplawski <jarkao2@o2.pl>
Cc: Srivatsa Vaddagiri <vatsa@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 13:34:46 +04:00
* so this work can ' t be re - armed in any way .
*/
static int try_to_grab_pending ( struct work_struct * work )
{
2010-06-29 12:07:12 +04:00
struct global_cwq * gcwq ;
2007-07-16 10:41:44 +04:00
int ret = - 1 ;
make cancel_rearming_delayed_work() reliable
Thanks to Jarek Poplawski for the ideas and for spotting the bug in the
initial draft patch.
cancel_rearming_delayed_work() currently has many limitations, because it
requires that dwork always re-arms itself via queue_delayed_work(). So it
hangs forever if dwork doesn't do this, or cancel_rearming_delayed_work/
cancel_delayed_work was already called. It uses flush_workqueue() in a
loop, so it can't be used if workqueue was freezed, and it is potentially
live- lockable on busy system if delay is small.
With this patch cancel_rearming_delayed_work() doesn't make any assumptions
about dwork, it can re-arm itself via queue_delayed_work(), or
queue_work(), or do nothing.
As a "side effect", cancel_work_sync() was changed to handle re-arming works
as well.
Disadvantages:
- this patch adds wmb() to insert_work().
- slowdowns the fast path (when del_timer() succeeds on entry) of
cancel_rearming_delayed_work(), because wait_on_work() is called
unconditionally. In that case, compared to the old version, we are
doing "unneeded" lock/unlock for each online CPU.
On the other hand, this means we don't need to use cancel_work_sync()
after cancel_rearming_delayed_work().
- complicates the code (.text grows by 130 bytes).
[akpm@linux-foundation.org: fix speling]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Cc: David Chinner <dgc@sgi.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Gautham Shenoy <ego@in.ibm.com>
Acked-by: Jarek Poplawski <jarkao2@o2.pl>
Cc: Srivatsa Vaddagiri <vatsa@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 13:34:46 +04:00
2010-06-29 12:07:10 +04:00
if ( ! test_and_set_bit ( WORK_STRUCT_PENDING_BIT , work_data_bits ( work ) ) )
2007-07-16 10:41:44 +04:00
return 0 ;
make cancel_rearming_delayed_work() reliable
Thanks to Jarek Poplawski for the ideas and for spotting the bug in the
initial draft patch.
cancel_rearming_delayed_work() currently has many limitations, because it
requires that dwork always re-arms itself via queue_delayed_work(). So it
hangs forever if dwork doesn't do this, or cancel_rearming_delayed_work/
cancel_delayed_work was already called. It uses flush_workqueue() in a
loop, so it can't be used if workqueue was freezed, and it is potentially
live- lockable on busy system if delay is small.
With this patch cancel_rearming_delayed_work() doesn't make any assumptions
about dwork, it can re-arm itself via queue_delayed_work(), or
queue_work(), or do nothing.
As a "side effect", cancel_work_sync() was changed to handle re-arming works
as well.
Disadvantages:
- this patch adds wmb() to insert_work().
- slowdowns the fast path (when del_timer() succeeds on entry) of
cancel_rearming_delayed_work(), because wait_on_work() is called
unconditionally. In that case, compared to the old version, we are
doing "unneeded" lock/unlock for each online CPU.
On the other hand, this means we don't need to use cancel_work_sync()
after cancel_rearming_delayed_work().
- complicates the code (.text grows by 130 bytes).
[akpm@linux-foundation.org: fix speling]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Cc: David Chinner <dgc@sgi.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Gautham Shenoy <ego@in.ibm.com>
Acked-by: Jarek Poplawski <jarkao2@o2.pl>
Cc: Srivatsa Vaddagiri <vatsa@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 13:34:46 +04:00
/*
* The queueing is in progress , or it is already queued . Try to
* steal it from - > worklist without clearing WORK_STRUCT_PENDING .
*/
2010-06-29 12:07:13 +04:00
gcwq = get_work_gcwq ( work ) ;
if ( ! gcwq )
make cancel_rearming_delayed_work() reliable
Thanks to Jarek Poplawski for the ideas and for spotting the bug in the
initial draft patch.
cancel_rearming_delayed_work() currently has many limitations, because it
requires that dwork always re-arms itself via queue_delayed_work(). So it
hangs forever if dwork doesn't do this, or cancel_rearming_delayed_work/
cancel_delayed_work was already called. It uses flush_workqueue() in a
loop, so it can't be used if workqueue was freezed, and it is potentially
live- lockable on busy system if delay is small.
With this patch cancel_rearming_delayed_work() doesn't make any assumptions
about dwork, it can re-arm itself via queue_delayed_work(), or
queue_work(), or do nothing.
As a "side effect", cancel_work_sync() was changed to handle re-arming works
as well.
Disadvantages:
- this patch adds wmb() to insert_work().
- slowdowns the fast path (when del_timer() succeeds on entry) of
cancel_rearming_delayed_work(), because wait_on_work() is called
unconditionally. In that case, compared to the old version, we are
doing "unneeded" lock/unlock for each online CPU.
On the other hand, this means we don't need to use cancel_work_sync()
after cancel_rearming_delayed_work().
- complicates the code (.text grows by 130 bytes).
[akpm@linux-foundation.org: fix speling]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Cc: David Chinner <dgc@sgi.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Gautham Shenoy <ego@in.ibm.com>
Acked-by: Jarek Poplawski <jarkao2@o2.pl>
Cc: Srivatsa Vaddagiri <vatsa@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 13:34:46 +04:00
return ret ;
2010-06-29 12:07:12 +04:00
spin_lock_irq ( & gcwq - > lock ) ;
make cancel_rearming_delayed_work() reliable
Thanks to Jarek Poplawski for the ideas and for spotting the bug in the
initial draft patch.
cancel_rearming_delayed_work() currently has many limitations, because it
requires that dwork always re-arms itself via queue_delayed_work(). So it
hangs forever if dwork doesn't do this, or cancel_rearming_delayed_work/
cancel_delayed_work was already called. It uses flush_workqueue() in a
loop, so it can't be used if workqueue was freezed, and it is potentially
live- lockable on busy system if delay is small.
With this patch cancel_rearming_delayed_work() doesn't make any assumptions
about dwork, it can re-arm itself via queue_delayed_work(), or
queue_work(), or do nothing.
As a "side effect", cancel_work_sync() was changed to handle re-arming works
as well.
Disadvantages:
- this patch adds wmb() to insert_work().
- slowdowns the fast path (when del_timer() succeeds on entry) of
cancel_rearming_delayed_work(), because wait_on_work() is called
unconditionally. In that case, compared to the old version, we are
doing "unneeded" lock/unlock for each online CPU.
On the other hand, this means we don't need to use cancel_work_sync()
after cancel_rearming_delayed_work().
- complicates the code (.text grows by 130 bytes).
[akpm@linux-foundation.org: fix speling]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Cc: David Chinner <dgc@sgi.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Gautham Shenoy <ego@in.ibm.com>
Acked-by: Jarek Poplawski <jarkao2@o2.pl>
Cc: Srivatsa Vaddagiri <vatsa@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 13:34:46 +04:00
if ( ! list_empty ( & work - > entry ) ) {
/*
2010-06-29 12:07:13 +04:00
* This work is queued , but perhaps we locked the wrong gcwq .
make cancel_rearming_delayed_work() reliable
Thanks to Jarek Poplawski for the ideas and for spotting the bug in the
initial draft patch.
cancel_rearming_delayed_work() currently has many limitations, because it
requires that dwork always re-arms itself via queue_delayed_work(). So it
hangs forever if dwork doesn't do this, or cancel_rearming_delayed_work/
cancel_delayed_work was already called. It uses flush_workqueue() in a
loop, so it can't be used if workqueue was freezed, and it is potentially
live- lockable on busy system if delay is small.
With this patch cancel_rearming_delayed_work() doesn't make any assumptions
about dwork, it can re-arm itself via queue_delayed_work(), or
queue_work(), or do nothing.
As a "side effect", cancel_work_sync() was changed to handle re-arming works
as well.
Disadvantages:
- this patch adds wmb() to insert_work().
- slowdowns the fast path (when del_timer() succeeds on entry) of
cancel_rearming_delayed_work(), because wait_on_work() is called
unconditionally. In that case, compared to the old version, we are
doing "unneeded" lock/unlock for each online CPU.
On the other hand, this means we don't need to use cancel_work_sync()
after cancel_rearming_delayed_work().
- complicates the code (.text grows by 130 bytes).
[akpm@linux-foundation.org: fix speling]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Cc: David Chinner <dgc@sgi.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Gautham Shenoy <ego@in.ibm.com>
Acked-by: Jarek Poplawski <jarkao2@o2.pl>
Cc: Srivatsa Vaddagiri <vatsa@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 13:34:46 +04:00
* In that case we must see the new value after rmb ( ) , see
* insert_work ( ) - > wmb ( ) .
*/
smp_rmb ( ) ;
2010-06-29 12:07:13 +04:00
if ( gcwq = = get_work_gcwq ( work ) ) {
2009-11-15 19:09:48 +03:00
debug_work_deactivate ( work ) ;
make cancel_rearming_delayed_work() reliable
Thanks to Jarek Poplawski for the ideas and for spotting the bug in the
initial draft patch.
cancel_rearming_delayed_work() currently has many limitations, because it
requires that dwork always re-arms itself via queue_delayed_work(). So it
hangs forever if dwork doesn't do this, or cancel_rearming_delayed_work/
cancel_delayed_work was already called. It uses flush_workqueue() in a
loop, so it can't be used if workqueue was freezed, and it is potentially
live- lockable on busy system if delay is small.
With this patch cancel_rearming_delayed_work() doesn't make any assumptions
about dwork, it can re-arm itself via queue_delayed_work(), or
queue_work(), or do nothing.
As a "side effect", cancel_work_sync() was changed to handle re-arming works
as well.
Disadvantages:
- this patch adds wmb() to insert_work().
- slowdowns the fast path (when del_timer() succeeds on entry) of
cancel_rearming_delayed_work(), because wait_on_work() is called
unconditionally. In that case, compared to the old version, we are
doing "unneeded" lock/unlock for each online CPU.
On the other hand, this means we don't need to use cancel_work_sync()
after cancel_rearming_delayed_work().
- complicates the code (.text grows by 130 bytes).
[akpm@linux-foundation.org: fix speling]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Cc: David Chinner <dgc@sgi.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Gautham Shenoy <ego@in.ibm.com>
Acked-by: Jarek Poplawski <jarkao2@o2.pl>
Cc: Srivatsa Vaddagiri <vatsa@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 13:34:46 +04:00
list_del_init ( & work - > entry ) ;
2010-06-29 12:07:13 +04:00
cwq_dec_nr_in_flight ( get_work_cwq ( work ) ,
2010-08-25 12:33:56 +04:00
get_work_color ( work ) ,
* work_data_bits ( work ) & WORK_STRUCT_DELAYED ) ;
make cancel_rearming_delayed_work() reliable
Thanks to Jarek Poplawski for the ideas and for spotting the bug in the
initial draft patch.
cancel_rearming_delayed_work() currently has many limitations, because it
requires that dwork always re-arms itself via queue_delayed_work(). So it
hangs forever if dwork doesn't do this, or cancel_rearming_delayed_work/
cancel_delayed_work was already called. It uses flush_workqueue() in a
loop, so it can't be used if workqueue was freezed, and it is potentially
live- lockable on busy system if delay is small.
With this patch cancel_rearming_delayed_work() doesn't make any assumptions
about dwork, it can re-arm itself via queue_delayed_work(), or
queue_work(), or do nothing.
As a "side effect", cancel_work_sync() was changed to handle re-arming works
as well.
Disadvantages:
- this patch adds wmb() to insert_work().
- slowdowns the fast path (when del_timer() succeeds on entry) of
cancel_rearming_delayed_work(), because wait_on_work() is called
unconditionally. In that case, compared to the old version, we are
doing "unneeded" lock/unlock for each online CPU.
On the other hand, this means we don't need to use cancel_work_sync()
after cancel_rearming_delayed_work().
- complicates the code (.text grows by 130 bytes).
[akpm@linux-foundation.org: fix speling]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Cc: David Chinner <dgc@sgi.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Gautham Shenoy <ego@in.ibm.com>
Acked-by: Jarek Poplawski <jarkao2@o2.pl>
Cc: Srivatsa Vaddagiri <vatsa@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 13:34:46 +04:00
ret = 1 ;
}
}
2010-06-29 12:07:12 +04:00
spin_unlock_irq ( & gcwq - > lock ) ;
make cancel_rearming_delayed_work() reliable
Thanks to Jarek Poplawski for the ideas and for spotting the bug in the
initial draft patch.
cancel_rearming_delayed_work() currently has many limitations, because it
requires that dwork always re-arms itself via queue_delayed_work(). So it
hangs forever if dwork doesn't do this, or cancel_rearming_delayed_work/
cancel_delayed_work was already called. It uses flush_workqueue() in a
loop, so it can't be used if workqueue was freezed, and it is potentially
live- lockable on busy system if delay is small.
With this patch cancel_rearming_delayed_work() doesn't make any assumptions
about dwork, it can re-arm itself via queue_delayed_work(), or
queue_work(), or do nothing.
As a "side effect", cancel_work_sync() was changed to handle re-arming works
as well.
Disadvantages:
- this patch adds wmb() to insert_work().
- slowdowns the fast path (when del_timer() succeeds on entry) of
cancel_rearming_delayed_work(), because wait_on_work() is called
unconditionally. In that case, compared to the old version, we are
doing "unneeded" lock/unlock for each online CPU.
On the other hand, this means we don't need to use cancel_work_sync()
after cancel_rearming_delayed_work().
- complicates the code (.text grows by 130 bytes).
[akpm@linux-foundation.org: fix speling]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Cc: David Chinner <dgc@sgi.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Gautham Shenoy <ego@in.ibm.com>
Acked-by: Jarek Poplawski <jarkao2@o2.pl>
Cc: Srivatsa Vaddagiri <vatsa@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 13:34:46 +04:00
return ret ;
}
2010-09-16 12:36:00 +04:00
static bool __cancel_work_timer ( struct work_struct * work ,
2007-07-16 10:41:44 +04:00
struct timer_list * timer )
{
int ret ;
do {
ret = ( timer & & likely ( del_timer ( timer ) ) ) ;
if ( ! ret )
ret = try_to_grab_pending ( work ) ;
wait_on_work ( work ) ;
} while ( unlikely ( ret < 0 ) ) ;
2010-06-29 12:07:13 +04:00
clear_work_data ( work ) ;
2007-07-16 10:41:44 +04:00
return ret ;
}
make cancel_rearming_delayed_work() reliable
Thanks to Jarek Poplawski for the ideas and for spotting the bug in the
initial draft patch.
cancel_rearming_delayed_work() currently has many limitations, because it
requires that dwork always re-arms itself via queue_delayed_work(). So it
hangs forever if dwork doesn't do this, or cancel_rearming_delayed_work/
cancel_delayed_work was already called. It uses flush_workqueue() in a
loop, so it can't be used if workqueue was freezed, and it is potentially
live- lockable on busy system if delay is small.
With this patch cancel_rearming_delayed_work() doesn't make any assumptions
about dwork, it can re-arm itself via queue_delayed_work(), or
queue_work(), or do nothing.
As a "side effect", cancel_work_sync() was changed to handle re-arming works
as well.
Disadvantages:
- this patch adds wmb() to insert_work().
- slowdowns the fast path (when del_timer() succeeds on entry) of
cancel_rearming_delayed_work(), because wait_on_work() is called
unconditionally. In that case, compared to the old version, we are
doing "unneeded" lock/unlock for each online CPU.
On the other hand, this means we don't need to use cancel_work_sync()
after cancel_rearming_delayed_work().
- complicates the code (.text grows by 130 bytes).
[akpm@linux-foundation.org: fix speling]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Cc: David Chinner <dgc@sgi.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Gautham Shenoy <ego@in.ibm.com>
Acked-by: Jarek Poplawski <jarkao2@o2.pl>
Cc: Srivatsa Vaddagiri <vatsa@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 13:34:46 +04:00
/**
2010-09-16 12:36:00 +04:00
* cancel_work_sync - cancel a work and wait for it to finish
* @ work : the work to cancel
make cancel_rearming_delayed_work() reliable
Thanks to Jarek Poplawski for the ideas and for spotting the bug in the
initial draft patch.
cancel_rearming_delayed_work() currently has many limitations, because it
requires that dwork always re-arms itself via queue_delayed_work(). So it
hangs forever if dwork doesn't do this, or cancel_rearming_delayed_work/
cancel_delayed_work was already called. It uses flush_workqueue() in a
loop, so it can't be used if workqueue was freezed, and it is potentially
live- lockable on busy system if delay is small.
With this patch cancel_rearming_delayed_work() doesn't make any assumptions
about dwork, it can re-arm itself via queue_delayed_work(), or
queue_work(), or do nothing.
As a "side effect", cancel_work_sync() was changed to handle re-arming works
as well.
Disadvantages:
- this patch adds wmb() to insert_work().
- slowdowns the fast path (when del_timer() succeeds on entry) of
cancel_rearming_delayed_work(), because wait_on_work() is called
unconditionally. In that case, compared to the old version, we are
doing "unneeded" lock/unlock for each online CPU.
On the other hand, this means we don't need to use cancel_work_sync()
after cancel_rearming_delayed_work().
- complicates the code (.text grows by 130 bytes).
[akpm@linux-foundation.org: fix speling]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Cc: David Chinner <dgc@sgi.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Gautham Shenoy <ego@in.ibm.com>
Acked-by: Jarek Poplawski <jarkao2@o2.pl>
Cc: Srivatsa Vaddagiri <vatsa@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 13:34:46 +04:00
*
2010-09-16 12:36:00 +04:00
* Cancel @ work and wait for its execution to finish . This function
* can be used even if the work re - queues itself or migrates to
* another workqueue . On return from this function , @ work is
* guaranteed to be not pending or executing on any CPU .
2007-07-16 10:41:44 +04:00
*
2010-09-16 12:36:00 +04:00
* cancel_work_sync ( & delayed_work - > work ) must not be used for
* delayed_work ' s . Use cancel_delayed_work_sync ( ) instead .
make cancel_rearming_delayed_work() reliable
Thanks to Jarek Poplawski for the ideas and for spotting the bug in the
initial draft patch.
cancel_rearming_delayed_work() currently has many limitations, because it
requires that dwork always re-arms itself via queue_delayed_work(). So it
hangs forever if dwork doesn't do this, or cancel_rearming_delayed_work/
cancel_delayed_work was already called. It uses flush_workqueue() in a
loop, so it can't be used if workqueue was freezed, and it is potentially
live- lockable on busy system if delay is small.
With this patch cancel_rearming_delayed_work() doesn't make any assumptions
about dwork, it can re-arm itself via queue_delayed_work(), or
queue_work(), or do nothing.
As a "side effect", cancel_work_sync() was changed to handle re-arming works
as well.
Disadvantages:
- this patch adds wmb() to insert_work().
- slowdowns the fast path (when del_timer() succeeds on entry) of
cancel_rearming_delayed_work(), because wait_on_work() is called
unconditionally. In that case, compared to the old version, we are
doing "unneeded" lock/unlock for each online CPU.
On the other hand, this means we don't need to use cancel_work_sync()
after cancel_rearming_delayed_work().
- complicates the code (.text grows by 130 bytes).
[akpm@linux-foundation.org: fix speling]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Cc: David Chinner <dgc@sgi.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Gautham Shenoy <ego@in.ibm.com>
Acked-by: Jarek Poplawski <jarkao2@o2.pl>
Cc: Srivatsa Vaddagiri <vatsa@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 13:34:46 +04:00
*
2010-09-16 12:36:00 +04:00
* The caller must ensure that the workqueue on which @ work was last
make cancel_rearming_delayed_work() reliable
Thanks to Jarek Poplawski for the ideas and for spotting the bug in the
initial draft patch.
cancel_rearming_delayed_work() currently has many limitations, because it
requires that dwork always re-arms itself via queue_delayed_work(). So it
hangs forever if dwork doesn't do this, or cancel_rearming_delayed_work/
cancel_delayed_work was already called. It uses flush_workqueue() in a
loop, so it can't be used if workqueue was freezed, and it is potentially
live- lockable on busy system if delay is small.
With this patch cancel_rearming_delayed_work() doesn't make any assumptions
about dwork, it can re-arm itself via queue_delayed_work(), or
queue_work(), or do nothing.
As a "side effect", cancel_work_sync() was changed to handle re-arming works
as well.
Disadvantages:
- this patch adds wmb() to insert_work().
- slowdowns the fast path (when del_timer() succeeds on entry) of
cancel_rearming_delayed_work(), because wait_on_work() is called
unconditionally. In that case, compared to the old version, we are
doing "unneeded" lock/unlock for each online CPU.
On the other hand, this means we don't need to use cancel_work_sync()
after cancel_rearming_delayed_work().
- complicates the code (.text grows by 130 bytes).
[akpm@linux-foundation.org: fix speling]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Cc: David Chinner <dgc@sgi.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Gautham Shenoy <ego@in.ibm.com>
Acked-by: Jarek Poplawski <jarkao2@o2.pl>
Cc: Srivatsa Vaddagiri <vatsa@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 13:34:46 +04:00
* queued can ' t be destroyed before this function returns .
2010-09-16 12:36:00 +04:00
*
* RETURNS :
* % true if @ work was pending , % false otherwise .
make cancel_rearming_delayed_work() reliable
Thanks to Jarek Poplawski for the ideas and for spotting the bug in the
initial draft patch.
cancel_rearming_delayed_work() currently has many limitations, because it
requires that dwork always re-arms itself via queue_delayed_work(). So it
hangs forever if dwork doesn't do this, or cancel_rearming_delayed_work/
cancel_delayed_work was already called. It uses flush_workqueue() in a
loop, so it can't be used if workqueue was freezed, and it is potentially
live- lockable on busy system if delay is small.
With this patch cancel_rearming_delayed_work() doesn't make any assumptions
about dwork, it can re-arm itself via queue_delayed_work(), or
queue_work(), or do nothing.
As a "side effect", cancel_work_sync() was changed to handle re-arming works
as well.
Disadvantages:
- this patch adds wmb() to insert_work().
- slowdowns the fast path (when del_timer() succeeds on entry) of
cancel_rearming_delayed_work(), because wait_on_work() is called
unconditionally. In that case, compared to the old version, we are
doing "unneeded" lock/unlock for each online CPU.
On the other hand, this means we don't need to use cancel_work_sync()
after cancel_rearming_delayed_work().
- complicates the code (.text grows by 130 bytes).
[akpm@linux-foundation.org: fix speling]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Cc: David Chinner <dgc@sgi.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Gautham Shenoy <ego@in.ibm.com>
Acked-by: Jarek Poplawski <jarkao2@o2.pl>
Cc: Srivatsa Vaddagiri <vatsa@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 13:34:46 +04:00
*/
2010-09-16 12:36:00 +04:00
bool cancel_work_sync ( struct work_struct * work )
make cancel_rearming_delayed_work() reliable
Thanks to Jarek Poplawski for the ideas and for spotting the bug in the
initial draft patch.
cancel_rearming_delayed_work() currently has many limitations, because it
requires that dwork always re-arms itself via queue_delayed_work(). So it
hangs forever if dwork doesn't do this, or cancel_rearming_delayed_work/
cancel_delayed_work was already called. It uses flush_workqueue() in a
loop, so it can't be used if workqueue was freezed, and it is potentially
live- lockable on busy system if delay is small.
With this patch cancel_rearming_delayed_work() doesn't make any assumptions
about dwork, it can re-arm itself via queue_delayed_work(), or
queue_work(), or do nothing.
As a "side effect", cancel_work_sync() was changed to handle re-arming works
as well.
Disadvantages:
- this patch adds wmb() to insert_work().
- slowdowns the fast path (when del_timer() succeeds on entry) of
cancel_rearming_delayed_work(), because wait_on_work() is called
unconditionally. In that case, compared to the old version, we are
doing "unneeded" lock/unlock for each online CPU.
On the other hand, this means we don't need to use cancel_work_sync()
after cancel_rearming_delayed_work().
- complicates the code (.text grows by 130 bytes).
[akpm@linux-foundation.org: fix speling]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Cc: David Chinner <dgc@sgi.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Gautham Shenoy <ego@in.ibm.com>
Acked-by: Jarek Poplawski <jarkao2@o2.pl>
Cc: Srivatsa Vaddagiri <vatsa@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 13:34:46 +04:00
{
2007-07-16 10:41:44 +04:00
return __cancel_work_timer ( work , NULL ) ;
implement flush_work()
A basic problem with flush_scheduled_work() is that it blocks behind _all_
presently-queued works, rather than just the work whcih the caller wants to
flush. If the caller holds some lock, and if one of the queued work happens
to want that lock as well then accidental deadlocks can occur.
One example of this is the phy layer: it wants to flush work while holding
rtnl_lock(). But if a linkwatch event happens to be queued, the phy code will
deadlock because the linkwatch callback function takes rtnl_lock.
So we implement a new function which will flush a *single* work - just the one
which the caller wants to free up. Thus we avoid the accidental deadlocks
which can arise from unrelated subsystems' callbacks taking shared locks.
flush_work() non-blockingly dequeues the work_struct which we want to kill,
then it waits for its handler to complete on all CPUs.
Add ->current_work to the "struct cpu_workqueue_struct", it points to
currently running "struct work_struct". When flush_work(work) detects
->current_work == work, it inserts a barrier at the _head_ of ->worklist
(and thus right _after_ that work) and waits for completition. This means
that the next work fired on that CPU will be this barrier, or another
barrier queued by concurrent flush_work(), so the caller of flush_work()
will be woken before any "regular" work has a chance to run.
When wait_on_work() unlocks workqueue_mutex (or whatever we choose to protect
against CPU hotplug), CPU may go away. But in that case take_over_work() will
move a barrier we queued to another CPU, it will be fired sometime, and
wait_on_work() will be woken.
Actually, we are doing cleanup_workqueue_thread()->kthread_stop() before
take_over_work(), so cwq->thread should complete its ->worklist (and thus
the barrier), because currently we don't check kthread_should_stop() in
run_workqueue(). But even if we did, everything should be ok.
[akpm@osdl.org: cleanup]
[akpm@osdl.org: add flush_work_keventd() wrapper]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 13:33:52 +04:00
}
2007-05-09 13:34:22 +04:00
EXPORT_SYMBOL_GPL ( cancel_work_sync ) ;
implement flush_work()
A basic problem with flush_scheduled_work() is that it blocks behind _all_
presently-queued works, rather than just the work whcih the caller wants to
flush. If the caller holds some lock, and if one of the queued work happens
to want that lock as well then accidental deadlocks can occur.
One example of this is the phy layer: it wants to flush work while holding
rtnl_lock(). But if a linkwatch event happens to be queued, the phy code will
deadlock because the linkwatch callback function takes rtnl_lock.
So we implement a new function which will flush a *single* work - just the one
which the caller wants to free up. Thus we avoid the accidental deadlocks
which can arise from unrelated subsystems' callbacks taking shared locks.
flush_work() non-blockingly dequeues the work_struct which we want to kill,
then it waits for its handler to complete on all CPUs.
Add ->current_work to the "struct cpu_workqueue_struct", it points to
currently running "struct work_struct". When flush_work(work) detects
->current_work == work, it inserts a barrier at the _head_ of ->worklist
(and thus right _after_ that work) and waits for completition. This means
that the next work fired on that CPU will be this barrier, or another
barrier queued by concurrent flush_work(), so the caller of flush_work()
will be woken before any "regular" work has a chance to run.
When wait_on_work() unlocks workqueue_mutex (or whatever we choose to protect
against CPU hotplug), CPU may go away. But in that case take_over_work() will
move a barrier we queued to another CPU, it will be fired sometime, and
wait_on_work() will be woken.
Actually, we are doing cleanup_workqueue_thread()->kthread_stop() before
take_over_work(), so cwq->thread should complete its ->worklist (and thus
the barrier), because currently we don't check kthread_should_stop() in
run_workqueue(). But even if we did, everything should be ok.
[akpm@osdl.org: cleanup]
[akpm@osdl.org: add flush_work_keventd() wrapper]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 13:33:52 +04:00
make cancel_rearming_delayed_work() reliable
Thanks to Jarek Poplawski for the ideas and for spotting the bug in the
initial draft patch.
cancel_rearming_delayed_work() currently has many limitations, because it
requires that dwork always re-arms itself via queue_delayed_work(). So it
hangs forever if dwork doesn't do this, or cancel_rearming_delayed_work/
cancel_delayed_work was already called. It uses flush_workqueue() in a
loop, so it can't be used if workqueue was freezed, and it is potentially
live- lockable on busy system if delay is small.
With this patch cancel_rearming_delayed_work() doesn't make any assumptions
about dwork, it can re-arm itself via queue_delayed_work(), or
queue_work(), or do nothing.
As a "side effect", cancel_work_sync() was changed to handle re-arming works
as well.
Disadvantages:
- this patch adds wmb() to insert_work().
- slowdowns the fast path (when del_timer() succeeds on entry) of
cancel_rearming_delayed_work(), because wait_on_work() is called
unconditionally. In that case, compared to the old version, we are
doing "unneeded" lock/unlock for each online CPU.
On the other hand, this means we don't need to use cancel_work_sync()
after cancel_rearming_delayed_work().
- complicates the code (.text grows by 130 bytes).
[akpm@linux-foundation.org: fix speling]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Cc: David Chinner <dgc@sgi.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Gautham Shenoy <ego@in.ibm.com>
Acked-by: Jarek Poplawski <jarkao2@o2.pl>
Cc: Srivatsa Vaddagiri <vatsa@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 13:34:46 +04:00
/**
2010-09-16 12:36:00 +04:00
* flush_delayed_work - wait for a dwork to finish executing the last queueing
* @ dwork : the delayed work to flush
make cancel_rearming_delayed_work() reliable
Thanks to Jarek Poplawski for the ideas and for spotting the bug in the
initial draft patch.
cancel_rearming_delayed_work() currently has many limitations, because it
requires that dwork always re-arms itself via queue_delayed_work(). So it
hangs forever if dwork doesn't do this, or cancel_rearming_delayed_work/
cancel_delayed_work was already called. It uses flush_workqueue() in a
loop, so it can't be used if workqueue was freezed, and it is potentially
live- lockable on busy system if delay is small.
With this patch cancel_rearming_delayed_work() doesn't make any assumptions
about dwork, it can re-arm itself via queue_delayed_work(), or
queue_work(), or do nothing.
As a "side effect", cancel_work_sync() was changed to handle re-arming works
as well.
Disadvantages:
- this patch adds wmb() to insert_work().
- slowdowns the fast path (when del_timer() succeeds on entry) of
cancel_rearming_delayed_work(), because wait_on_work() is called
unconditionally. In that case, compared to the old version, we are
doing "unneeded" lock/unlock for each online CPU.
On the other hand, this means we don't need to use cancel_work_sync()
after cancel_rearming_delayed_work().
- complicates the code (.text grows by 130 bytes).
[akpm@linux-foundation.org: fix speling]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Cc: David Chinner <dgc@sgi.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Gautham Shenoy <ego@in.ibm.com>
Acked-by: Jarek Poplawski <jarkao2@o2.pl>
Cc: Srivatsa Vaddagiri <vatsa@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 13:34:46 +04:00
*
2010-09-16 12:36:00 +04:00
* Delayed timer is cancelled and the pending work is queued for
* immediate execution . Like flush_work ( ) , this function only
* considers the last queueing instance of @ dwork .
2007-07-16 10:41:44 +04:00
*
2010-09-16 12:36:00 +04:00
* RETURNS :
* % true if flush_work ( ) waited for the work to finish execution ,
* % false if it was already idle .
make cancel_rearming_delayed_work() reliable
Thanks to Jarek Poplawski for the ideas and for spotting the bug in the
initial draft patch.
cancel_rearming_delayed_work() currently has many limitations, because it
requires that dwork always re-arms itself via queue_delayed_work(). So it
hangs forever if dwork doesn't do this, or cancel_rearming_delayed_work/
cancel_delayed_work was already called. It uses flush_workqueue() in a
loop, so it can't be used if workqueue was freezed, and it is potentially
live- lockable on busy system if delay is small.
With this patch cancel_rearming_delayed_work() doesn't make any assumptions
about dwork, it can re-arm itself via queue_delayed_work(), or
queue_work(), or do nothing.
As a "side effect", cancel_work_sync() was changed to handle re-arming works
as well.
Disadvantages:
- this patch adds wmb() to insert_work().
- slowdowns the fast path (when del_timer() succeeds on entry) of
cancel_rearming_delayed_work(), because wait_on_work() is called
unconditionally. In that case, compared to the old version, we are
doing "unneeded" lock/unlock for each online CPU.
On the other hand, this means we don't need to use cancel_work_sync()
after cancel_rearming_delayed_work().
- complicates the code (.text grows by 130 bytes).
[akpm@linux-foundation.org: fix speling]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Cc: David Chinner <dgc@sgi.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Gautham Shenoy <ego@in.ibm.com>
Acked-by: Jarek Poplawski <jarkao2@o2.pl>
Cc: Srivatsa Vaddagiri <vatsa@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 13:34:46 +04:00
*/
2010-09-16 12:36:00 +04:00
bool flush_delayed_work ( struct delayed_work * dwork )
{
if ( del_timer_sync ( & dwork - > timer ) )
__queue_work ( raw_smp_processor_id ( ) ,
get_work_cwq ( & dwork - > work ) - > wq , & dwork - > work ) ;
return flush_work ( & dwork - > work ) ;
}
EXPORT_SYMBOL ( flush_delayed_work ) ;
2010-09-16 12:48:29 +04:00
/**
* flush_delayed_work_sync - wait for a dwork to finish
* @ dwork : the delayed work to flush
*
* Delayed timer is cancelled and the pending work is queued for
* execution immediately . Other than timer handling , its behavior
* is identical to flush_work_sync ( ) .
*
* RETURNS :
* % true if flush_work_sync ( ) waited for the work to finish execution ,
* % false if it was already idle .
*/
bool flush_delayed_work_sync ( struct delayed_work * dwork )
{
if ( del_timer_sync ( & dwork - > timer ) )
__queue_work ( raw_smp_processor_id ( ) ,
get_work_cwq ( & dwork - > work ) - > wq , & dwork - > work ) ;
return flush_work_sync ( & dwork - > work ) ;
}
EXPORT_SYMBOL ( flush_delayed_work_sync ) ;
2010-09-16 12:36:00 +04:00
/**
* cancel_delayed_work_sync - cancel a delayed work and wait for it to finish
* @ dwork : the delayed work cancel
*
* This is cancel_work_sync ( ) for delayed works .
*
* RETURNS :
* % true if @ dwork was pending , % false otherwise .
*/
bool cancel_delayed_work_sync ( struct delayed_work * dwork )
make cancel_rearming_delayed_work() reliable
Thanks to Jarek Poplawski for the ideas and for spotting the bug in the
initial draft patch.
cancel_rearming_delayed_work() currently has many limitations, because it
requires that dwork always re-arms itself via queue_delayed_work(). So it
hangs forever if dwork doesn't do this, or cancel_rearming_delayed_work/
cancel_delayed_work was already called. It uses flush_workqueue() in a
loop, so it can't be used if workqueue was freezed, and it is potentially
live- lockable on busy system if delay is small.
With this patch cancel_rearming_delayed_work() doesn't make any assumptions
about dwork, it can re-arm itself via queue_delayed_work(), or
queue_work(), or do nothing.
As a "side effect", cancel_work_sync() was changed to handle re-arming works
as well.
Disadvantages:
- this patch adds wmb() to insert_work().
- slowdowns the fast path (when del_timer() succeeds on entry) of
cancel_rearming_delayed_work(), because wait_on_work() is called
unconditionally. In that case, compared to the old version, we are
doing "unneeded" lock/unlock for each online CPU.
On the other hand, this means we don't need to use cancel_work_sync()
after cancel_rearming_delayed_work().
- complicates the code (.text grows by 130 bytes).
[akpm@linux-foundation.org: fix speling]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Cc: David Chinner <dgc@sgi.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Gautham Shenoy <ego@in.ibm.com>
Acked-by: Jarek Poplawski <jarkao2@o2.pl>
Cc: Srivatsa Vaddagiri <vatsa@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 13:34:46 +04:00
{
2007-07-16 10:41:44 +04:00
return __cancel_work_timer ( & dwork - > work , & dwork - > timer ) ;
make cancel_rearming_delayed_work() reliable
Thanks to Jarek Poplawski for the ideas and for spotting the bug in the
initial draft patch.
cancel_rearming_delayed_work() currently has many limitations, because it
requires that dwork always re-arms itself via queue_delayed_work(). So it
hangs forever if dwork doesn't do this, or cancel_rearming_delayed_work/
cancel_delayed_work was already called. It uses flush_workqueue() in a
loop, so it can't be used if workqueue was freezed, and it is potentially
live- lockable on busy system if delay is small.
With this patch cancel_rearming_delayed_work() doesn't make any assumptions
about dwork, it can re-arm itself via queue_delayed_work(), or
queue_work(), or do nothing.
As a "side effect", cancel_work_sync() was changed to handle re-arming works
as well.
Disadvantages:
- this patch adds wmb() to insert_work().
- slowdowns the fast path (when del_timer() succeeds on entry) of
cancel_rearming_delayed_work(), because wait_on_work() is called
unconditionally. In that case, compared to the old version, we are
doing "unneeded" lock/unlock for each online CPU.
On the other hand, this means we don't need to use cancel_work_sync()
after cancel_rearming_delayed_work().
- complicates the code (.text grows by 130 bytes).
[akpm@linux-foundation.org: fix speling]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Cc: David Chinner <dgc@sgi.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Gautham Shenoy <ego@in.ibm.com>
Acked-by: Jarek Poplawski <jarkao2@o2.pl>
Cc: Srivatsa Vaddagiri <vatsa@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 13:34:46 +04:00
}
2007-07-16 10:41:44 +04:00
EXPORT_SYMBOL ( cancel_delayed_work_sync ) ;
2005-04-17 02:20:36 +04:00
2006-07-30 14:03:42 +04:00
/**
* schedule_work - put work task in global workqueue
* @ work : job to be done
*
2009-07-30 21:00:53 +04:00
* Returns zero if @ work was already on the kernel - global workqueue and
* non - zero otherwise .
*
* This puts a job in the kernel - global workqueue if it was not already
* queued and leaves it in the same position on the kernel - global
* workqueue otherwise .
2006-07-30 14:03:42 +04:00
*/
2008-02-08 15:19:53 +03:00
int schedule_work ( struct work_struct * work )
2005-04-17 02:20:36 +04:00
{
2010-06-29 12:07:14 +04:00
return queue_work ( system_wq , work ) ;
2005-04-17 02:20:36 +04:00
}
2006-06-30 09:40:45 +04:00
EXPORT_SYMBOL ( schedule_work ) ;
2005-04-17 02:20:36 +04:00
2008-07-24 08:28:39 +04:00
/*
* schedule_work_on - put work task on a specific cpu
* @ cpu : cpu to put the work task on
* @ work : job to be done
*
* This puts a job on a specific cpu
*/
int schedule_work_on ( int cpu , struct work_struct * work )
{
2010-06-29 12:07:14 +04:00
return queue_work_on ( cpu , system_wq , work ) ;
2008-07-24 08:28:39 +04:00
}
EXPORT_SYMBOL ( schedule_work_on ) ;
2006-07-30 14:03:42 +04:00
/**
* schedule_delayed_work - put work task in global workqueue after delay
2006-11-22 17:54:01 +03:00
* @ dwork : job to be done
* @ delay : number of jiffies to wait or 0 for immediate execution
2006-07-30 14:03:42 +04:00
*
* After waiting for a given time this puts a job in the kernel - global
* workqueue .
*/
2008-02-08 15:19:53 +03:00
int schedule_delayed_work ( struct delayed_work * dwork ,
[PATCH] Add debugging feature /proc/timer_stat
Add /proc/timer_stats support: debugging feature to profile timer expiration.
Both the starting site, process/PID and the expiration function is captured.
This allows the quick identification of timer event sources in a system.
Sample output:
# echo 1 > /proc/timer_stats
# cat /proc/timer_stats
Timer Stats Version: v0.1
Sample period: 4.010 s
24, 0 swapper hrtimer_stop_sched_tick (hrtimer_sched_tick)
11, 0 swapper sk_reset_timer (tcp_delack_timer)
6, 0 swapper hrtimer_stop_sched_tick (hrtimer_sched_tick)
2, 1 swapper queue_delayed_work_on (delayed_work_timer_fn)
17, 0 swapper hrtimer_restart_sched_tick (hrtimer_sched_tick)
2, 1 swapper queue_delayed_work_on (delayed_work_timer_fn)
4, 2050 pcscd do_nanosleep (hrtimer_wakeup)
5, 4179 sshd sk_reset_timer (tcp_write_timer)
4, 2248 yum-updatesd schedule_timeout (process_timeout)
18, 0 swapper hrtimer_restart_sched_tick (hrtimer_sched_tick)
3, 0 swapper sk_reset_timer (tcp_delack_timer)
1, 1 swapper neigh_table_init_no_netlink (neigh_periodic_timer)
2, 1 swapper e1000_up (e1000_watchdog)
1, 1 init schedule_timeout (process_timeout)
100 total events, 25.24 events/sec
[ cleanups and hrtimers support from Thomas Gleixner <tglx@linutronix.de> ]
[bunk@stusta.de: nr_entries can become static]
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: john stultz <johnstul@us.ibm.com>
Cc: Roman Zippel <zippel@linux-m68k.org>
Cc: Andi Kleen <ak@suse.de>
Signed-off-by: Adrian Bunk <bunk@stusta.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-02-16 12:28:13 +03:00
unsigned long delay )
2005-04-17 02:20:36 +04:00
{
2010-06-29 12:07:14 +04:00
return queue_delayed_work ( system_wq , dwork , delay ) ;
2005-04-17 02:20:36 +04:00
}
2006-06-30 09:40:45 +04:00
EXPORT_SYMBOL ( schedule_delayed_work ) ;
2005-04-17 02:20:36 +04:00
2006-07-30 14:03:42 +04:00
/**
* schedule_delayed_work_on - queue work in global workqueue on CPU after delay
* @ cpu : cpu to use
2006-11-22 17:54:01 +03:00
* @ dwork : job to be done
2006-07-30 14:03:42 +04:00
* @ delay : number of jiffies to wait
*
* After waiting for a given time this puts a job in the kernel - global
* workqueue on the specified CPU .
*/
2005-04-17 02:20:36 +04:00
int schedule_delayed_work_on ( int cpu ,
2006-11-22 17:54:01 +03:00
struct delayed_work * dwork , unsigned long delay )
2005-04-17 02:20:36 +04:00
{
2010-06-29 12:07:14 +04:00
return queue_delayed_work_on ( cpu , system_wq , dwork , delay ) ;
2005-04-17 02:20:36 +04:00
}
2006-06-30 09:40:45 +04:00
EXPORT_SYMBOL ( schedule_delayed_work_on ) ;
2005-04-17 02:20:36 +04:00
2006-06-25 16:47:49 +04:00
/**
2010-10-19 13:14:49 +04:00
* schedule_on_each_cpu - execute a function synchronously on each online CPU
2006-06-25 16:47:49 +04:00
* @ func : the function to call
*
2010-10-19 13:14:49 +04:00
* schedule_on_each_cpu ( ) executes @ func on each online CPU using the
* system workqueue and blocks until all CPUs have completed .
2006-06-25 16:47:49 +04:00
* schedule_on_each_cpu ( ) is very slow .
2010-10-19 13:14:49 +04:00
*
* RETURNS :
* 0 on success , - errno on failure .
2006-06-25 16:47:49 +04:00
*/
2006-11-22 17:55:48 +03:00
int schedule_on_each_cpu ( work_func_t func )
2006-01-08 12:00:43 +03:00
{
int cpu ;
2010-08-08 16:24:09 +04:00
struct work_struct __percpu * works ;
2006-01-08 12:00:43 +03:00
2006-06-25 16:47:49 +04:00
works = alloc_percpu ( struct work_struct ) ;
if ( ! works )
2006-01-08 12:00:43 +03:00
return - ENOMEM ;
2006-06-25 16:47:49 +04:00
2009-11-18 01:06:20 +03:00
get_online_cpus ( ) ;
2006-01-08 12:00:43 +03:00
for_each_online_cpu ( cpu ) {
2006-12-18 22:05:09 +03:00
struct work_struct * work = per_cpu_ptr ( works , cpu ) ;
INIT_WORK ( work , func ) ;
2010-06-29 12:07:14 +04:00
schedule_work_on ( cpu , work ) ;
2009-10-14 08:22:47 +04:00
}
2009-11-18 01:06:20 +03:00
for_each_online_cpu ( cpu )
flush_work ( per_cpu_ptr ( works , cpu ) ) ;
2008-01-25 23:08:02 +03:00
put_online_cpus ( ) ;
2006-06-25 16:47:49 +04:00
free_percpu ( works ) ;
2006-01-08 12:00:43 +03:00
return 0 ;
}
2010-02-12 11:39:21 +03:00
/**
* flush_scheduled_work - ensure that any scheduled work has run to completion .
*
* Forces execution of the kernel - global workqueue and blocks until its
* completion .
*
* Think twice before calling this function ! It ' s very easy to get into
* trouble if you don ' t take great care . Either of the following situations
* will lead to deadlock :
*
* One of the work items currently on the workqueue needs to acquire
* a lock held by your code or its caller .
*
* Your code is running in the context of a work routine .
*
* They will be detected by lockdep when they occur , but the first might not
* occur very often . It depends on what work items are on the workqueue and
* what locks they need , which you have no control over .
*
* In most situations flushing the entire workqueue is overkill ; you merely
* need to know that a particular work item isn ' t queued and isn ' t running .
* In such cases you should use cancel_delayed_work_sync ( ) or
* cancel_work_sync ( ) instead .
*/
2005-04-17 02:20:36 +04:00
void flush_scheduled_work ( void )
{
2010-06-29 12:07:14 +04:00
flush_workqueue ( system_wq ) ;
2005-04-17 02:20:36 +04:00
}
2006-06-30 09:40:45 +04:00
EXPORT_SYMBOL ( flush_scheduled_work ) ;
2005-04-17 02:20:36 +04:00
2006-02-23 21:43:43 +03:00
/**
* execute_in_process_context - reliably execute the routine with user context
* @ fn : the function to execute
* @ ew : guaranteed storage for the execute work structure ( must
* be available when the work executes )
*
* Executes the function immediately if process context is available ,
* otherwise schedules the function for delayed execution .
*
* Returns : 0 - function was executed
* 1 - function was scheduled for execution
*/
2006-11-22 17:55:48 +03:00
int execute_in_process_context ( work_func_t fn , struct execute_work * ew )
2006-02-23 21:43:43 +03:00
{
if ( ! in_interrupt ( ) ) {
2006-11-22 17:55:48 +03:00
fn ( & ew - > work ) ;
2006-02-23 21:43:43 +03:00
return 0 ;
}
2006-11-22 17:55:48 +03:00
INIT_WORK ( & ew - > work , fn ) ;
2006-02-23 21:43:43 +03:00
schedule_work ( & ew - > work ) ;
return 1 ;
}
EXPORT_SYMBOL_GPL ( execute_in_process_context ) ;
2005-04-17 02:20:36 +04:00
int keventd_up ( void )
{
2010-06-29 12:07:14 +04:00
return system_wq ! = NULL ;
2005-04-17 02:20:36 +04:00
}
2010-07-02 12:03:51 +04:00
static int alloc_cwqs ( struct workqueue_struct * wq )
2010-06-29 12:07:11 +04:00
{
2009-10-14 08:22:47 +04:00
/*
2010-06-29 12:07:11 +04:00
* cwqs are forced aligned according to WORK_STRUCT_FLAG_BITS .
* Make sure that the alignment isn ' t lower than that of
* unsigned long long .
2009-10-14 08:22:47 +04:00
*/
2010-06-29 12:07:11 +04:00
const size_t size = sizeof ( struct cpu_workqueue_struct ) ;
const size_t align = max_t ( size_t , 1 < < WORK_STRUCT_FLAG_BITS ,
__alignof__ ( unsigned long long ) ) ;
2009-10-14 08:22:47 +04:00
2012-03-09 14:03:20 +04:00
if ( ! ( wq - > flags & WQ_UNBOUND ) )
2010-07-02 12:03:51 +04:00
wq - > cpu_wq . pcpu = __alloc_percpu ( size , align ) ;
2010-07-20 13:07:48 +04:00
else {
2010-07-02 12:03:51 +04:00
void * ptr ;
/*
* Allocate enough room to align cwq and put an extra
* pointer at the end pointing back to the originally
* allocated pointer which will be used for free .
*/
ptr = kzalloc ( size + align + sizeof ( void * ) , GFP_KERNEL ) ;
if ( ptr ) {
wq - > cpu_wq . single = PTR_ALIGN ( ptr , align ) ;
* ( void * * ) ( wq - > cpu_wq . single + 1 ) = ptr ;
}
2010-07-02 12:03:51 +04:00
}
2010-07-02 12:03:51 +04:00
2011-03-24 20:50:09 +03:00
/* just in case, make sure it's actually aligned */
2010-07-02 12:03:51 +04:00
BUG_ON ( ! IS_ALIGNED ( wq - > cpu_wq . v , align ) ) ;
return wq - > cpu_wq . v ? 0 : - ENOMEM ;
2010-06-29 12:07:11 +04:00
}
2010-07-02 12:03:51 +04:00
static void free_cwqs ( struct workqueue_struct * wq )
2010-06-29 12:07:11 +04:00
{
2012-03-09 14:03:20 +04:00
if ( ! ( wq - > flags & WQ_UNBOUND ) )
2010-07-02 12:03:51 +04:00
free_percpu ( wq - > cpu_wq . pcpu ) ;
else if ( wq - > cpu_wq . single ) {
/* the pointer to free is stored right after the cwq */
2010-07-02 12:03:51 +04:00
kfree ( * ( void * * ) ( wq - > cpu_wq . single + 1 ) ) ;
2010-07-02 12:03:51 +04:00
}
2010-06-29 12:07:11 +04:00
}
2010-07-02 12:03:51 +04:00
static int wq_clamp_max_active ( int max_active , unsigned int flags ,
const char * name )
2010-06-29 12:07:14 +04:00
{
2010-07-02 12:03:51 +04:00
int lim = flags & WQ_UNBOUND ? WQ_UNBOUND_MAX_ACTIVE : WQ_MAX_ACTIVE ;
if ( max_active < 1 | | max_active > lim )
2010-06-29 12:07:14 +04:00
printk ( KERN_WARNING " workqueue: max_active %d requested for %s "
" is out of range, clamping between %d and %d \n " ,
2010-07-02 12:03:51 +04:00
max_active , name , 1 , lim ) ;
2010-06-29 12:07:14 +04:00
2010-07-02 12:03:51 +04:00
return clamp_val ( max_active , 1 , lim ) ;
2010-06-29 12:07:14 +04:00
}
2012-01-11 03:11:35 +04:00
struct workqueue_struct * __alloc_workqueue_key ( const char * fmt ,
2010-06-29 12:07:14 +04:00
unsigned int flags ,
int max_active ,
struct lock_class_key * key ,
2012-01-11 03:11:35 +04:00
const char * lock_name , . . . )
2005-04-17 02:20:36 +04:00
{
2012-01-11 03:11:35 +04:00
va_list args , args1 ;
2005-04-17 02:20:36 +04:00
struct workqueue_struct * wq ;
2010-06-29 12:07:11 +04:00
unsigned int cpu ;
2012-01-11 03:11:35 +04:00
size_t namelen ;
/* determine namelen, allocate wq and format name */
va_start ( args , lock_name ) ;
va_copy ( args1 , args ) ;
namelen = vsnprintf ( NULL , 0 , fmt , args ) + 1 ;
wq = kzalloc ( sizeof ( * wq ) + namelen , GFP_KERNEL ) ;
if ( ! wq )
goto err ;
vsnprintf ( wq - > name , namelen , fmt , args1 ) ;
va_end ( args ) ;
va_end ( args1 ) ;
2005-04-17 02:20:36 +04:00
2010-10-11 17:12:27 +04:00
/*
* Workqueues which may be used during memory reclaim should
* have a rescuer to guarantee forward progress .
*/
if ( flags & WQ_MEM_RECLAIM )
flags | = WQ_RESCUER ;
2010-06-29 12:07:14 +04:00
max_active = max_active ? : WQ_DFL_ACTIVE ;
2012-01-11 03:11:35 +04:00
max_active = wq_clamp_max_active ( max_active , flags , wq - > name ) ;
2007-05-09 13:34:09 +04:00
2012-01-11 03:11:35 +04:00
/* init wq */
2010-06-29 12:07:10 +04:00
wq - > flags = flags ;
2010-06-29 12:07:12 +04:00
wq - > saved_max_active = max_active ;
2010-06-29 12:07:11 +04:00
mutex_init ( & wq - > flush_mutex ) ;
atomic_set ( & wq - > nr_cwqs_to_flush , 0 ) ;
INIT_LIST_HEAD ( & wq - > flusher_queue ) ;
INIT_LIST_HEAD ( & wq - > flusher_overflow ) ;
2010-06-29 12:07:13 +04:00
2008-01-16 11:51:58 +03:00
lockdep_init_map ( & wq - > lockdep_map , lock_name , key , 0 ) ;
2007-05-09 13:34:13 +04:00
INIT_LIST_HEAD ( & wq - > list ) ;
2007-05-09 13:34:09 +04:00
2010-07-02 12:03:51 +04:00
if ( alloc_cwqs ( wq ) < 0 )
goto err ;
2010-07-02 12:03:51 +04:00
for_each_cwq_cpu ( cpu , wq ) {
2010-06-29 12:07:11 +04:00
struct cpu_workqueue_struct * cwq = get_cwq ( cpu , wq ) ;
2010-06-29 12:07:12 +04:00
struct global_cwq * gcwq = get_gcwq ( cpu ) ;
2010-06-29 12:07:11 +04:00
2010-06-29 12:07:11 +04:00
BUG_ON ( ( unsigned long ) cwq & WORK_STRUCT_FLAG_MASK ) ;
2010-06-29 12:07:12 +04:00
cwq - > gcwq = gcwq ;
2010-06-29 12:07:11 +04:00
cwq - > wq = wq ;
2010-06-29 12:07:11 +04:00
cwq - > flush_color = - 1 ;
2010-06-29 12:07:12 +04:00
cwq - > max_active = max_active ;
INIT_LIST_HEAD ( & cwq - > delayed_works ) ;
2010-06-29 12:07:14 +04:00
}
2010-06-29 12:07:11 +04:00
2010-06-29 12:07:14 +04:00
if ( flags & WQ_RESCUER ) {
struct worker * rescuer ;
2010-07-20 17:59:09 +04:00
if ( ! alloc_mayday_mask ( & wq - > mayday_mask , GFP_KERNEL ) )
2010-06-29 12:07:14 +04:00
goto err ;
wq - > rescuer = rescuer = alloc_worker ( ) ;
if ( ! rescuer )
goto err ;
2012-01-11 03:11:35 +04:00
rescuer - > task = kthread_create ( rescuer_thread , wq , " %s " ,
wq - > name ) ;
2010-06-29 12:07:14 +04:00
if ( IS_ERR ( rescuer - > task ) )
goto err ;
rescuer - > task - > flags | = PF_THREAD_BOUND ;
wake_up_process ( rescuer - > task ) ;
2007-05-09 13:34:09 +04:00
}
2010-06-29 12:07:12 +04:00
/*
* workqueue_lock protects global freeze state and workqueues
* list . Grab it , set max_active accordingly and add the new
* workqueue to workqueues list .
*/
2010-06-29 12:07:11 +04:00
spin_lock ( & workqueue_lock ) ;
2010-06-29 12:07:12 +04:00
2011-02-16 11:25:31 +03:00
if ( workqueue_freezing & & wq - > flags & WQ_FREEZABLE )
2010-07-02 12:03:51 +04:00
for_each_cwq_cpu ( cpu , wq )
2010-06-29 12:07:12 +04:00
get_cwq ( cpu , wq ) - > max_active = 0 ;
2010-06-29 12:07:11 +04:00
list_add ( & wq - > list , & workqueues ) ;
2010-06-29 12:07:12 +04:00
2010-06-29 12:07:11 +04:00
spin_unlock ( & workqueue_lock ) ;
2007-05-09 13:34:09 +04:00
return wq ;
2010-06-29 12:07:10 +04:00
err :
if ( wq ) {
2010-07-02 12:03:51 +04:00
free_cwqs ( wq ) ;
2010-07-20 17:59:09 +04:00
free_mayday_mask ( wq - > mayday_mask ) ;
2010-06-29 12:07:14 +04:00
kfree ( wq - > rescuer ) ;
2010-06-29 12:07:10 +04:00
kfree ( wq ) ;
}
return NULL ;
2007-05-09 13:34:09 +04:00
}
2010-06-29 12:07:14 +04:00
EXPORT_SYMBOL_GPL ( __alloc_workqueue_key ) ;
2005-04-17 02:20:36 +04:00
2007-05-09 13:34:09 +04:00
/**
* destroy_workqueue - safely terminate a workqueue
* @ wq : target workqueue
*
* Safely destroy a workqueue . All work currently pending will be done first .
*/
void destroy_workqueue ( struct workqueue_struct * wq )
{
2010-06-29 12:07:12 +04:00
unsigned int cpu ;
2007-05-09 13:34:09 +04:00
2011-04-05 20:01:44 +04:00
/* drain it before proceeding with destruction */
drain_workqueue ( wq ) ;
2010-12-20 21:32:04 +03:00
2010-06-29 12:07:12 +04:00
/*
* wq list is used to freeze wq , remove from list after
* flushing is complete in case freeze races us .
*/
2008-01-25 23:08:02 +03:00
spin_lock ( & workqueue_lock ) ;
2007-05-09 13:34:12 +04:00
list_del ( & wq - > list ) ;
2008-01-25 23:08:02 +03:00
spin_unlock ( & workqueue_lock ) ;
2007-05-09 13:34:09 +04:00
2010-06-29 12:07:14 +04:00
/* sanity check */
2010-07-02 12:03:51 +04:00
for_each_cwq_cpu ( cpu , wq ) {
2010-06-29 12:07:11 +04:00
struct cpu_workqueue_struct * cwq = get_cwq ( cpu , wq ) ;
int i ;
for ( i = 0 ; i < WORK_NR_COLORS ; i + + )
BUG_ON ( cwq - > nr_in_flight [ i ] ) ;
2010-06-29 12:07:12 +04:00
BUG_ON ( cwq - > nr_active ) ;
BUG_ON ( ! list_empty ( & cwq - > delayed_works ) ) ;
2010-06-29 12:07:11 +04:00
}
2006-08-14 10:24:26 +04:00
2010-06-29 12:07:14 +04:00
if ( wq - > flags & WQ_RESCUER ) {
kthread_stop ( wq - > rescuer - > task ) ;
2010-07-20 17:59:09 +04:00
free_mayday_mask ( wq - > mayday_mask ) ;
2010-08-16 11:54:28 +04:00
kfree ( wq - > rescuer ) ;
2010-06-29 12:07:14 +04:00
}
2010-07-02 12:03:51 +04:00
free_cwqs ( wq ) ;
2007-05-09 13:34:09 +04:00
kfree ( wq ) ;
}
EXPORT_SYMBOL_GPL ( destroy_workqueue ) ;
2010-06-29 12:07:14 +04:00
/**
* workqueue_set_max_active - adjust max_active of a workqueue
* @ wq : target workqueue
* @ max_active : new max_active value .
*
* Set max_active of @ wq to @ max_active .
*
* CONTEXT :
* Don ' t call from IRQ context .
*/
void workqueue_set_max_active ( struct workqueue_struct * wq , int max_active )
{
unsigned int cpu ;
2010-07-02 12:03:51 +04:00
max_active = wq_clamp_max_active ( max_active , wq - > flags , wq - > name ) ;
2010-06-29 12:07:14 +04:00
spin_lock ( & workqueue_lock ) ;
wq - > saved_max_active = max_active ;
2010-07-02 12:03:51 +04:00
for_each_cwq_cpu ( cpu , wq ) {
2010-06-29 12:07:14 +04:00
struct global_cwq * gcwq = get_gcwq ( cpu ) ;
spin_lock_irq ( & gcwq - > lock ) ;
2011-02-16 11:25:31 +03:00
if ( ! ( wq - > flags & WQ_FREEZABLE ) | |
2010-06-29 12:07:14 +04:00
! ( gcwq - > flags & GCWQ_FREEZING ) )
get_cwq ( gcwq - > cpu , wq ) - > max_active = max_active ;
2006-12-18 22:05:09 +03:00
2010-06-29 12:07:14 +04:00
spin_unlock_irq ( & gcwq - > lock ) ;
2009-10-14 08:22:47 +04:00
}
2009-11-18 01:06:20 +03:00
2010-06-29 12:07:14 +04:00
spin_unlock ( & workqueue_lock ) ;
2006-01-08 12:00:43 +03:00
}
2010-06-29 12:07:14 +04:00
EXPORT_SYMBOL_GPL ( workqueue_set_max_active ) ;
2006-01-08 12:00:43 +03:00
2010-02-12 11:39:21 +03:00
/**
2010-06-29 12:07:14 +04:00
* workqueue_congested - test whether a workqueue is congested
* @ cpu : CPU in question
* @ wq : target workqueue
2010-02-12 11:39:21 +03:00
*
2010-06-29 12:07:14 +04:00
* Test whether @ wq ' s cpu workqueue for @ cpu is congested . There is
* no synchronization around this function and the test result is
* unreliable and only useful as advisory hints or for debugging .
2010-02-12 11:39:21 +03:00
*
2010-06-29 12:07:14 +04:00
* RETURNS :
* % true if congested , % false otherwise .
2010-02-12 11:39:21 +03:00
*/
2010-06-29 12:07:14 +04:00
bool workqueue_congested ( unsigned int cpu , struct workqueue_struct * wq )
2005-04-17 02:20:36 +04:00
{
2010-06-29 12:07:14 +04:00
struct cpu_workqueue_struct * cwq = get_cwq ( cpu , wq ) ;
return ! list_empty ( & cwq - > delayed_works ) ;
2005-04-17 02:20:36 +04:00
}
2010-06-29 12:07:14 +04:00
EXPORT_SYMBOL_GPL ( workqueue_congested ) ;
2005-04-17 02:20:36 +04:00
2006-02-23 21:43:43 +03:00
/**
2010-06-29 12:07:14 +04:00
* work_cpu - return the last known associated cpu for @ work
* @ work : the work of interest
2006-02-23 21:43:43 +03:00
*
2010-06-29 12:07:14 +04:00
* RETURNS :
2010-07-02 12:03:51 +04:00
* CPU number if @ work was ever queued . WORK_CPU_NONE otherwise .
2006-02-23 21:43:43 +03:00
*/
2010-06-29 12:07:14 +04:00
unsigned int work_cpu ( struct work_struct * work )
2006-02-23 21:43:43 +03:00
{
2010-06-29 12:07:14 +04:00
struct global_cwq * gcwq = get_work_gcwq ( work ) ;
2006-02-23 21:43:43 +03:00
2010-07-02 12:03:51 +04:00
return gcwq ? gcwq - > cpu : WORK_CPU_NONE ;
2006-02-23 21:43:43 +03:00
}
2010-06-29 12:07:14 +04:00
EXPORT_SYMBOL_GPL ( work_cpu ) ;
2006-02-23 21:43:43 +03:00
2010-06-29 12:07:14 +04:00
/**
* work_busy - test whether a work is currently pending or running
* @ work : the work to be tested
*
* Test whether @ work is currently pending or running . There is no
* synchronization around this function and the test result is
* unreliable and only useful as advisory hints or for debugging .
* Especially for reentrant wqs , the pending state might hide the
* running state .
*
* RETURNS :
* OR ' d bitmask of WORK_BUSY_ * bits .
*/
unsigned int work_busy ( struct work_struct * work )
2005-04-17 02:20:36 +04:00
{
2010-06-29 12:07:14 +04:00
struct global_cwq * gcwq = get_work_gcwq ( work ) ;
unsigned long flags ;
unsigned int ret = 0 ;
2005-04-17 02:20:36 +04:00
2010-06-29 12:07:14 +04:00
if ( ! gcwq )
return false ;
2005-04-17 02:20:36 +04:00
2010-06-29 12:07:14 +04:00
spin_lock_irqsave ( & gcwq - > lock , flags ) ;
2005-04-17 02:20:36 +04:00
2010-06-29 12:07:14 +04:00
if ( work_pending ( work ) )
ret | = WORK_BUSY_PENDING ;
if ( find_worker_executing_work ( gcwq , work ) )
ret | = WORK_BUSY_RUNNING ;
2005-04-17 02:20:36 +04:00
2010-06-29 12:07:14 +04:00
spin_unlock_irqrestore ( & gcwq - > lock , flags ) ;
2005-04-17 02:20:36 +04:00
2010-06-29 12:07:14 +04:00
return ret ;
2005-04-17 02:20:36 +04:00
}
2010-06-29 12:07:14 +04:00
EXPORT_SYMBOL_GPL ( work_busy ) ;
2005-04-17 02:20:36 +04:00
2010-06-29 12:07:12 +04:00
/*
* CPU hotplug .
*
2010-06-29 12:07:14 +04:00
* There are two challenges in supporting CPU hotplug . Firstly , there
* are a lot of assumptions on strong associations among work , cwq and
* gcwq which make migrating pending and scheduled works very
* difficult to implement without impacting hot paths . Secondly ,
* gcwqs serve mix of short , long and very long running works making
* blocked draining impractical .
*
* This is solved by allowing a gcwq to be detached from CPU , running
* it with unbound ( rogue ) workers and allowing it to be reattached
* later if the cpu comes back online . A separate thread is created
* to govern a gcwq in such state and is called the trustee of the
* gcwq .
2010-06-29 12:07:12 +04:00
*
* Trustee states and their descriptions .
*
* START Command state used on startup . On CPU_DOWN_PREPARE , a
* new trustee is started with this state .
*
* IN_CHARGE Once started , trustee will enter this state after
2010-06-29 12:07:14 +04:00
* assuming the manager role and making all existing
* workers rogue . DOWN_PREPARE waits for trustee to
* enter this state . After reaching IN_CHARGE , trustee
* tries to execute the pending worklist until it ' s empty
* and the state is set to BUTCHER , or the state is set
* to RELEASE .
2010-06-29 12:07:12 +04:00
*
* BUTCHER Command state which is set by the cpu callback after
* the cpu has went down . Once this state is set trustee
* knows that there will be no new works on the worklist
* and once the worklist is empty it can proceed to
* killing idle workers .
*
* RELEASE Command state which is set by the cpu callback if the
* cpu down has been canceled or it has come online
* again . After recognizing this state , trustee stops
2010-06-29 12:07:14 +04:00
* trying to drain or butcher and clears ROGUE , rebinds
* all remaining workers back to the cpu and releases
* manager role .
2010-06-29 12:07:12 +04:00
*
* DONE Trustee will enter this state after BUTCHER or RELEASE
* is complete .
*
* trustee CPU draining
* took over down complete
* START - - - - - - - - - - - > IN_CHARGE - - - - - - - - - - - > BUTCHER - - - - - - - - - - - > DONE
* | | ^
* | CPU is back online v return workers |
* - - - - - - - - - - - - - - - - > RELEASE - - - - - - - - - - - - - -
*/
2005-04-17 02:20:36 +04:00
2010-06-29 12:07:12 +04:00
/**
* trustee_wait_event_timeout - timed event wait for trustee
* @ cond : condition to wait for
* @ timeout : timeout in jiffies
*
* wait_event_timeout ( ) for trustee to use . Handles locking and
* checks for RELEASE request .
*
* CONTEXT :
* spin_lock_irq ( gcwq - > lock ) which may be released and regrabbed
* multiple times . To be used by trustee .
*
* RETURNS :
* Positive indicating left time if @ cond is satisfied , 0 if timed
* out , - 1 if canceled .
*/
# define trustee_wait_event_timeout(cond, timeout) ({ \
long __ret = ( timeout ) ; \
while ( ! ( ( cond ) | | ( gcwq - > trustee_state = = TRUSTEE_RELEASE ) ) & & \
__ret ) { \
spin_unlock_irq ( & gcwq - > lock ) ; \
__wait_event_timeout ( gcwq - > trustee_wait , ( cond ) | | \
( gcwq - > trustee_state = = TRUSTEE_RELEASE ) , \
__ret ) ; \
spin_lock_irq ( & gcwq - > lock ) ; \
} \
gcwq - > trustee_state = = TRUSTEE_RELEASE ? - 1 : ( __ret ) ; \
} )
2007-05-09 13:34:09 +04:00
2010-06-29 12:07:12 +04:00
/**
* trustee_wait_event - event wait for trustee
* @ cond : condition to wait for
*
* wait_event ( ) for trustee to use . Automatically handles locking and
* checks for CANCEL request .
*
* CONTEXT :
* spin_lock_irq ( gcwq - > lock ) which may be released and regrabbed
* multiple times . To be used by trustee .
*
* RETURNS :
* 0 if @ cond is satisfied , - 1 if canceled .
*/
# define trustee_wait_event(cond) ({ \
long __ret1 ; \
__ret1 = trustee_wait_event_timeout ( cond , MAX_SCHEDULE_TIMEOUT ) ; \
__ret1 < 0 ? - 1 : 0 ; \
} )
2005-04-17 02:20:36 +04:00
2010-06-29 12:07:12 +04:00
static int __cpuinit trustee_thread ( void * __gcwq )
2007-05-09 13:34:09 +04:00
{
2010-06-29 12:07:12 +04:00
struct global_cwq * gcwq = __gcwq ;
struct worker * worker ;
2010-06-29 12:07:14 +04:00
struct work_struct * work ;
2010-06-29 12:07:12 +04:00
struct hlist_node * pos ;
2010-06-29 12:07:14 +04:00
long rc ;
2010-06-29 12:07:12 +04:00
int i ;
2007-05-09 13:34:09 +04:00
2010-06-29 12:07:12 +04:00
BUG_ON ( gcwq - > cpu ! = smp_processor_id ( ) ) ;
spin_lock_irq ( & gcwq - > lock ) ;
2007-05-09 13:34:09 +04:00
/*
2010-06-29 12:07:14 +04:00
* Claim the manager position and make all workers rogue .
* Trustee must be bound to the target cpu and can ' t be
* cancelled .
2007-05-09 13:34:09 +04:00
*/
2010-06-29 12:07:12 +04:00
BUG_ON ( gcwq - > cpu ! = smp_processor_id ( ) ) ;
2010-06-29 12:07:14 +04:00
rc = trustee_wait_event ( ! ( gcwq - > flags & GCWQ_MANAGING_WORKERS ) ) ;
BUG_ON ( rc < 0 ) ;
2007-05-09 13:34:09 +04:00
2010-06-29 12:07:14 +04:00
gcwq - > flags | = GCWQ_MANAGING_WORKERS ;
2009-01-13 01:15:46 +03:00
2010-06-29 12:07:12 +04:00
list_for_each_entry ( worker , & gcwq - > idle_list , entry )
2010-07-02 12:03:50 +04:00
worker - > flags | = WORKER_ROGUE ;
2007-05-09 13:34:09 +04:00
2010-06-29 12:07:12 +04:00
for_each_busy_worker ( worker , i , pos , gcwq )
2010-07-02 12:03:50 +04:00
worker - > flags | = WORKER_ROGUE ;
2007-05-09 13:34:15 +04:00
2010-06-29 12:07:14 +04:00
/*
* Call schedule ( ) so that we cross rq - > lock and thus can
* guarantee sched callbacks see the rogue flag . This is
* necessary as scheduler callbacks may be invoked from other
* cpus .
*/
spin_unlock_irq ( & gcwq - > lock ) ;
schedule ( ) ;
spin_lock_irq ( & gcwq - > lock ) ;
2007-05-09 13:34:15 +04:00
2010-06-29 12:07:14 +04:00
/*
2010-07-02 12:03:50 +04:00
* Sched callbacks are disabled now . Zap nr_running . After
* this , nr_running stays zero and need_more_worker ( ) and
* keep_working ( ) are always true as long as the worklist is
* not empty .
2010-06-29 12:07:14 +04:00
*/
2010-07-02 12:03:50 +04:00
atomic_set ( get_gcwq_nr_running ( gcwq - > cpu ) , 0 ) ;
2005-04-17 02:20:36 +04:00
2010-06-29 12:07:14 +04:00
spin_unlock_irq ( & gcwq - > lock ) ;
del_timer_sync ( & gcwq - > idle_timer ) ;
spin_lock_irq ( & gcwq - > lock ) ;
2007-05-09 13:34:09 +04:00
2010-06-29 12:07:12 +04:00
/*
* We ' re now in charge . Notify and proceed to drain . We need
* to keep the gcwq running during the whole CPU down
* procedure as other cpu hotunplug callbacks may need to
* flush currently running tasks .
*/
gcwq - > trustee_state = TRUSTEE_IN_CHARGE ;
wake_up_all ( & gcwq - > trustee_wait ) ;
2007-05-09 13:34:09 +04:00
2010-06-29 12:07:12 +04:00
/*
* The original cpu is in the process of dying and may go away
* anytime now . When that happens , we and all workers would
2010-06-29 12:07:14 +04:00
* be migrated to other cpus . Try draining any left work . We
* want to get it over with ASAP - spam rescuers , wake up as
* many idlers as necessary and create new ones till the
* worklist is empty . Note that if the gcwq is frozen , there
2011-02-16 11:25:31 +03:00
* may be frozen works in freezable cwqs . Don ' t declare
2010-06-29 12:07:14 +04:00
* completion while frozen .
2010-06-29 12:07:12 +04:00
*/
while ( gcwq - > nr_workers ! = gcwq - > nr_idle | |
gcwq - > flags & GCWQ_FREEZING | |
gcwq - > trustee_state = = TRUSTEE_IN_CHARGE ) {
2010-06-29 12:07:14 +04:00
int nr_works = 0 ;
list_for_each_entry ( work , & gcwq - > worklist , entry ) {
send_mayday ( work ) ;
nr_works + + ;
}
2007-05-09 13:34:09 +04:00
2010-06-29 12:07:14 +04:00
list_for_each_entry ( worker , & gcwq - > idle_list , entry ) {
if ( ! nr_works - - )
break ;
wake_up_process ( worker - > task ) ;
}
if ( need_to_create_worker ( gcwq ) ) {
spin_unlock_irq ( & gcwq - > lock ) ;
worker = create_worker ( gcwq , false ) ;
spin_lock_irq ( & gcwq - > lock ) ;
if ( worker ) {
2010-07-02 12:03:50 +04:00
worker - > flags | = WORKER_ROGUE ;
2010-06-29 12:07:14 +04:00
start_worker ( worker ) ;
}
2005-04-17 02:20:36 +04:00
}
2007-05-09 13:34:09 +04:00
2010-06-29 12:07:12 +04:00
/* give a breather */
if ( trustee_wait_event_timeout ( false , TRUSTEE_COOLDOWN ) < 0 )
break ;
2007-05-09 13:34:09 +04:00
}
2005-04-17 02:20:36 +04:00
2007-05-24 00:57:57 +04:00
/*
2010-06-29 12:07:14 +04:00
* Either all works have been scheduled and cpu is down , or
* cpu down has already been canceled . Wait for and butcher
* all workers till we ' re canceled .
2007-05-24 00:57:57 +04:00
*/
2010-06-29 12:07:14 +04:00
do {
rc = trustee_wait_event ( ! list_empty ( & gcwq - > idle_list ) ) ;
while ( ! list_empty ( & gcwq - > idle_list ) )
destroy_worker ( list_first_entry ( & gcwq - > idle_list ,
struct worker , entry ) ) ;
} while ( gcwq - > nr_workers & & rc > = 0 ) ;
2007-10-19 10:39:55 +04:00
2007-05-24 00:57:57 +04:00
/*
2010-06-29 12:07:14 +04:00
* At this point , either draining has completed and no worker
* is left , or cpu down has been canceled or the cpu is being
* brought back up . There shouldn ' t be any idle one left .
* Tell the remaining busy ones to rebind once it finishes the
* currently scheduled works by scheduling the rebind_work .
2007-05-24 00:57:57 +04:00
*/
2010-06-29 12:07:14 +04:00
WARN_ON ( ! list_empty ( & gcwq - > idle_list ) ) ;
for_each_busy_worker ( worker , i , pos , gcwq ) {
struct work_struct * rebind_work = & worker - > rebind_work ;
/*
* Rebind_work may race with future cpu hotplug
* operations . Use a separate flag to mark that
* rebinding is scheduled .
*/
2010-07-02 12:03:50 +04:00
worker - > flags | = WORKER_REBIND ;
worker - > flags & = ~ WORKER_ROGUE ;
2010-06-29 12:07:14 +04:00
/* queue rebind_work, wq doesn't matter, use the default one */
if ( test_and_set_bit ( WORK_STRUCT_PENDING_BIT ,
work_data_bits ( rebind_work ) ) )
continue ;
debug_work_activate ( rebind_work ) ;
2010-06-29 12:07:14 +04:00
insert_work ( get_cwq ( gcwq - > cpu , system_wq ) , rebind_work ,
2010-06-29 12:07:14 +04:00
worker - > scheduled . next ,
work_color_to_flags ( WORK_NO_COLOR ) ) ;
}
/* relinquish manager role */
gcwq - > flags & = ~ GCWQ_MANAGING_WORKERS ;
2010-06-29 12:07:12 +04:00
/* notify completion */
gcwq - > trustee = NULL ;
gcwq - > trustee_state = TRUSTEE_DONE ;
wake_up_all ( & gcwq - > trustee_wait ) ;
spin_unlock_irq ( & gcwq - > lock ) ;
return 0 ;
2007-05-09 13:34:09 +04:00
}
/**
2010-06-29 12:07:12 +04:00
* wait_trustee_state - wait for trustee to enter the specified state
* @ gcwq : gcwq the trustee of interest belongs to
* @ state : target state to wait for
2007-05-09 13:34:09 +04:00
*
2010-06-29 12:07:12 +04:00
* Wait for the trustee to reach @ state . DONE is already matched .
*
* CONTEXT :
* spin_lock_irq ( gcwq - > lock ) which may be released and regrabbed
* multiple times . To be used by cpu_callback .
2007-05-09 13:34:09 +04:00
*/
2010-06-29 12:07:12 +04:00
static void __cpuinit wait_trustee_state ( struct global_cwq * gcwq , int state )
2010-08-22 18:19:42 +04:00
__releases ( & gcwq - > lock )
__acquires ( & gcwq - > lock )
2007-05-09 13:34:09 +04:00
{
2010-06-29 12:07:12 +04:00
if ( ! ( gcwq - > trustee_state = = state | |
gcwq - > trustee_state = = TRUSTEE_DONE ) ) {
spin_unlock_irq ( & gcwq - > lock ) ;
__wait_event ( gcwq - > trustee_wait ,
gcwq - > trustee_state = = state | |
gcwq - > trustee_state = = TRUSTEE_DONE ) ;
spin_lock_irq ( & gcwq - > lock ) ;
}
2007-05-09 13:34:09 +04:00
}
static int __devinit workqueue_cpu_callback ( struct notifier_block * nfb ,
unsigned long action ,
void * hcpu )
{
unsigned int cpu = ( unsigned long ) hcpu ;
2010-06-29 12:07:12 +04:00
struct global_cwq * gcwq = get_gcwq ( cpu ) ;
struct task_struct * new_trustee = NULL ;
2010-06-29 12:07:14 +04:00
struct worker * uninitialized_var ( new_worker ) ;
2010-06-29 12:07:12 +04:00
unsigned long flags ;
2007-05-09 13:34:09 +04:00
2007-05-09 13:35:10 +04:00
action & = ~ CPU_TASKS_FROZEN ;
2007-05-09 13:34:09 +04:00
switch ( action ) {
2010-06-29 12:07:12 +04:00
case CPU_DOWN_PREPARE :
new_trustee = kthread_create ( trustee_thread , gcwq ,
" workqueue_trustee/%d \n " , cpu ) ;
if ( IS_ERR ( new_trustee ) )
return notifier_from_errno ( PTR_ERR ( new_trustee ) ) ;
kthread_bind ( new_trustee , cpu ) ;
2010-06-29 12:07:14 +04:00
/* fall through */
2007-05-09 13:34:09 +04:00
case CPU_UP_PREPARE :
2010-06-29 12:07:14 +04:00
BUG_ON ( gcwq - > first_idle ) ;
new_worker = create_worker ( gcwq , false ) ;
if ( ! new_worker ) {
if ( new_trustee )
kthread_stop ( new_trustee ) ;
return NOTIFY_BAD ;
2007-05-09 13:34:09 +04:00
}
2005-04-17 02:20:36 +04:00
}
2010-06-29 12:07:12 +04:00
/* some are called w/ irq disabled, don't disturb irq status */
spin_lock_irqsave ( & gcwq - > lock , flags ) ;
2007-05-09 13:34:09 +04:00
2008-04-29 12:00:27 +04:00
switch ( action ) {
2010-06-29 12:07:12 +04:00
case CPU_DOWN_PREPARE :
/* initialize trustee and tell it to acquire the gcwq */
BUG_ON ( gcwq - > trustee | | gcwq - > trustee_state ! = TRUSTEE_DONE ) ;
gcwq - > trustee = new_trustee ;
gcwq - > trustee_state = TRUSTEE_START ;
wake_up_process ( gcwq - > trustee ) ;
wait_trustee_state ( gcwq , TRUSTEE_IN_CHARGE ) ;
2010-06-29 12:07:14 +04:00
/* fall through */
case CPU_UP_PREPARE :
BUG_ON ( gcwq - > first_idle ) ;
gcwq - > first_idle = new_worker ;
break ;
case CPU_DYING :
/*
* Before this , the trustee and all workers except for
* the ones which are still executing works from
* before the last CPU down must be on the cpu . After
* this , they ' ll all be diasporas .
*/
gcwq - > flags | = GCWQ_DISASSOCIATED ;
2010-06-29 12:07:12 +04:00
break ;
2008-07-25 12:47:50 +04:00
case CPU_POST_DEAD :
2010-06-29 12:07:12 +04:00
gcwq - > trustee_state = TRUSTEE_BUTCHER ;
2010-06-29 12:07:14 +04:00
/* fall through */
case CPU_UP_CANCELED :
destroy_worker ( gcwq - > first_idle ) ;
gcwq - > first_idle = NULL ;
2010-06-29 12:07:12 +04:00
break ;
case CPU_DOWN_FAILED :
case CPU_ONLINE :
2010-06-29 12:07:14 +04:00
gcwq - > flags & = ~ GCWQ_DISASSOCIATED ;
2010-06-29 12:07:12 +04:00
if ( gcwq - > trustee_state ! = TRUSTEE_DONE ) {
gcwq - > trustee_state = TRUSTEE_RELEASE ;
wake_up_process ( gcwq - > trustee ) ;
wait_trustee_state ( gcwq , TRUSTEE_DONE ) ;
2007-05-09 13:34:09 +04:00
}
2010-06-29 12:07:12 +04:00
2010-06-29 12:07:14 +04:00
/*
* Trustee is done and there might be no worker left .
* Put the first_idle in and request a real manager to
* take a look .
*/
spin_unlock_irq ( & gcwq - > lock ) ;
kthread_bind ( gcwq - > first_idle - > task , cpu ) ;
spin_lock_irq ( & gcwq - > lock ) ;
gcwq - > flags | = GCWQ_MANAGE_WORKERS ;
start_worker ( gcwq - > first_idle ) ;
gcwq - > first_idle = NULL ;
2010-06-29 12:07:12 +04:00
break ;
2008-04-29 12:00:27 +04:00
}
2010-06-29 12:07:12 +04:00
spin_unlock_irqrestore ( & gcwq - > lock , flags ) ;
2010-06-29 12:07:11 +04:00
return notifier_from_errno ( 0 ) ;
2005-04-17 02:20:36 +04:00
}
2008-11-05 05:39:10 +03:00
# ifdef CONFIG_SMP
2009-01-17 02:31:15 +03:00
2008-11-05 05:39:10 +03:00
struct work_for_cpu {
2009-04-09 19:50:37 +04:00
struct completion completion ;
2008-11-05 05:39:10 +03:00
long ( * fn ) ( void * ) ;
void * arg ;
long ret ;
} ;
2009-04-09 19:50:37 +04:00
static int do_work_for_cpu ( void * _wfc )
2008-11-05 05:39:10 +03:00
{
2009-04-09 19:50:37 +04:00
struct work_for_cpu * wfc = _wfc ;
2008-11-05 05:39:10 +03:00
wfc - > ret = wfc - > fn ( wfc - > arg ) ;
2009-04-09 19:50:37 +04:00
complete ( & wfc - > completion ) ;
return 0 ;
2008-11-05 05:39:10 +03:00
}
/**
* work_on_cpu - run a function in user context on a particular cpu
* @ cpu : the cpu to run on
* @ fn : the function to run
* @ arg : the function arg
*
2009-01-17 02:31:15 +03:00
* This will return the value @ fn returns .
* It is up to the caller to ensure that the cpu doesn ' t go offline .
2009-04-09 19:50:37 +04:00
* The caller must not hold any locks which would prevent @ fn from completing .
2008-11-05 05:39:10 +03:00
*/
long work_on_cpu ( unsigned int cpu , long ( * fn ) ( void * ) , void * arg )
{
2009-04-09 19:50:37 +04:00
struct task_struct * sub_thread ;
struct work_for_cpu wfc = {
. completion = COMPLETION_INITIALIZER_ONSTACK ( wfc . completion ) ,
. fn = fn ,
. arg = arg ,
} ;
sub_thread = kthread_create ( do_work_for_cpu , & wfc , " work_for_cpu " ) ;
if ( IS_ERR ( sub_thread ) )
return PTR_ERR ( sub_thread ) ;
kthread_bind ( sub_thread , cpu ) ;
wake_up_process ( sub_thread ) ;
wait_for_completion ( & wfc . completion ) ;
2008-11-05 05:39:10 +03:00
return wfc . ret ;
}
EXPORT_SYMBOL_GPL ( work_on_cpu ) ;
# endif /* CONFIG_SMP */
2010-06-29 12:07:12 +04:00
# ifdef CONFIG_FREEZER
/**
* freeze_workqueues_begin - begin freezing workqueues
*
2011-02-16 11:25:31 +03:00
* Start freezing workqueues . After this function returns , all freezable
* workqueues will queue new works to their frozen_works list instead of
* gcwq - > worklist .
2010-06-29 12:07:12 +04:00
*
* CONTEXT :
2010-06-29 12:07:12 +04:00
* Grabs and releases workqueue_lock and gcwq - > lock ' s .
2010-06-29 12:07:12 +04:00
*/
void freeze_workqueues_begin ( void )
{
unsigned int cpu ;
spin_lock ( & workqueue_lock ) ;
BUG_ON ( workqueue_freezing ) ;
workqueue_freezing = true ;
2010-07-02 12:03:51 +04:00
for_each_gcwq_cpu ( cpu ) {
2010-06-29 12:07:12 +04:00
struct global_cwq * gcwq = get_gcwq ( cpu ) ;
2010-07-02 12:03:51 +04:00
struct workqueue_struct * wq ;
2010-06-29 12:07:12 +04:00
spin_lock_irq ( & gcwq - > lock ) ;
2010-06-29 12:07:12 +04:00
BUG_ON ( gcwq - > flags & GCWQ_FREEZING ) ;
gcwq - > flags | = GCWQ_FREEZING ;
2010-06-29 12:07:12 +04:00
list_for_each_entry ( wq , & workqueues , list ) {
struct cpu_workqueue_struct * cwq = get_cwq ( cpu , wq ) ;
2011-02-16 11:25:31 +03:00
if ( cwq & & wq - > flags & WQ_FREEZABLE )
2010-06-29 12:07:12 +04:00
cwq - > max_active = 0 ;
}
2010-06-29 12:07:12 +04:00
spin_unlock_irq ( & gcwq - > lock ) ;
2010-06-29 12:07:12 +04:00
}
spin_unlock ( & workqueue_lock ) ;
}
/**
2011-02-16 11:25:31 +03:00
* freeze_workqueues_busy - are freezable workqueues still busy ?
2010-06-29 12:07:12 +04:00
*
* Check whether freezing is complete . This function must be called
* between freeze_workqueues_begin ( ) and thaw_workqueues ( ) .
*
* CONTEXT :
* Grabs and releases workqueue_lock .
*
* RETURNS :
2011-02-16 11:25:31 +03:00
* % true if some freezable workqueues are still busy . % false if freezing
* is complete .
2010-06-29 12:07:12 +04:00
*/
bool freeze_workqueues_busy ( void )
{
unsigned int cpu ;
bool busy = false ;
spin_lock ( & workqueue_lock ) ;
BUG_ON ( ! workqueue_freezing ) ;
2010-07-02 12:03:51 +04:00
for_each_gcwq_cpu ( cpu ) {
2010-07-02 12:03:51 +04:00
struct workqueue_struct * wq ;
2010-06-29 12:07:12 +04:00
/*
* nr_active is monotonically decreasing . It ' s safe
* to peek without lock .
*/
list_for_each_entry ( wq , & workqueues , list ) {
struct cpu_workqueue_struct * cwq = get_cwq ( cpu , wq ) ;
2011-02-16 11:25:31 +03:00
if ( ! cwq | | ! ( wq - > flags & WQ_FREEZABLE ) )
2010-06-29 12:07:12 +04:00
continue ;
BUG_ON ( cwq - > nr_active < 0 ) ;
if ( cwq - > nr_active ) {
busy = true ;
goto out_unlock ;
}
}
}
out_unlock :
spin_unlock ( & workqueue_lock ) ;
return busy ;
}
/**
* thaw_workqueues - thaw workqueues
*
* Thaw workqueues . Normal queueing is restored and all collected
2010-06-29 12:07:13 +04:00
* frozen works are transferred to their respective gcwq worklists .
2010-06-29 12:07:12 +04:00
*
* CONTEXT :
2010-06-29 12:07:12 +04:00
* Grabs and releases workqueue_lock and gcwq - > lock ' s .
2010-06-29 12:07:12 +04:00
*/
void thaw_workqueues ( void )
{
unsigned int cpu ;
spin_lock ( & workqueue_lock ) ;
if ( ! workqueue_freezing )
goto out_unlock ;
2010-07-02 12:03:51 +04:00
for_each_gcwq_cpu ( cpu ) {
2010-06-29 12:07:12 +04:00
struct global_cwq * gcwq = get_gcwq ( cpu ) ;
2010-07-02 12:03:51 +04:00
struct workqueue_struct * wq ;
2010-06-29 12:07:12 +04:00
spin_lock_irq ( & gcwq - > lock ) ;
2010-06-29 12:07:12 +04:00
BUG_ON ( ! ( gcwq - > flags & GCWQ_FREEZING ) ) ;
gcwq - > flags & = ~ GCWQ_FREEZING ;
2010-06-29 12:07:12 +04:00
list_for_each_entry ( wq , & workqueues , list ) {
struct cpu_workqueue_struct * cwq = get_cwq ( cpu , wq ) ;
2011-02-16 11:25:31 +03:00
if ( ! cwq | | ! ( wq - > flags & WQ_FREEZABLE ) )
2010-06-29 12:07:12 +04:00
continue ;
/* restore max_active and repopulate worklist */
cwq - > max_active = wq - > saved_max_active ;
while ( ! list_empty ( & cwq - > delayed_works ) & &
cwq - > nr_active < cwq - > max_active )
cwq_activate_first_delayed ( cwq ) ;
}
2010-06-29 12:07:12 +04:00
2010-06-29 12:07:14 +04:00
wake_up_worker ( gcwq ) ;
2010-06-29 12:07:12 +04:00
spin_unlock_irq ( & gcwq - > lock ) ;
2010-06-29 12:07:12 +04:00
}
workqueue_freezing = false ;
out_unlock :
spin_unlock ( & workqueue_lock ) ;
}
# endif /* CONFIG_FREEZER */
2010-07-31 01:57:37 +04:00
static int __init init_workqueues ( void )
2005-04-17 02:20:36 +04:00
{
2010-06-29 12:07:11 +04:00
unsigned int cpu ;
2010-06-29 12:07:12 +04:00
int i ;
2010-06-29 12:07:11 +04:00
2010-08-09 13:50:34 +04:00
cpu_notifier ( workqueue_cpu_callback , CPU_PRI_WORKQUEUE ) ;
2010-06-29 12:07:12 +04:00
/* initialize gcwqs */
2010-07-02 12:03:51 +04:00
for_each_gcwq_cpu ( cpu ) {
2010-06-29 12:07:12 +04:00
struct global_cwq * gcwq = get_gcwq ( cpu ) ;
spin_lock_init ( & gcwq - > lock ) ;
2010-06-29 12:07:13 +04:00
INIT_LIST_HEAD ( & gcwq - > worklist ) ;
2010-06-29 12:07:12 +04:00
gcwq - > cpu = cpu ;
2010-08-31 12:54:35 +04:00
gcwq - > flags | = GCWQ_DISASSOCIATED ;
2010-06-29 12:07:12 +04:00
2010-06-29 12:07:12 +04:00
INIT_LIST_HEAD ( & gcwq - > idle_list ) ;
for ( i = 0 ; i < BUSY_WORKER_HASH_SIZE ; i + + )
INIT_HLIST_HEAD ( & gcwq - > busy_hash [ i ] ) ;
2010-06-29 12:07:14 +04:00
init_timer_deferrable ( & gcwq - > idle_timer ) ;
gcwq - > idle_timer . function = idle_worker_timeout ;
gcwq - > idle_timer . data = ( unsigned long ) gcwq ;
2009-01-01 02:42:25 +03:00
2010-06-29 12:07:14 +04:00
setup_timer ( & gcwq - > mayday_timer , gcwq_mayday_timeout ,
( unsigned long ) gcwq ) ;
2010-06-29 12:07:12 +04:00
ida_init ( & gcwq - > worker_ida ) ;
2010-06-29 12:07:12 +04:00
gcwq - > trustee_state = TRUSTEE_DONE ;
init_waitqueue_head ( & gcwq - > trustee_wait ) ;
2010-06-29 12:07:12 +04:00
}
2010-06-29 12:07:14 +04:00
/* create the initial worker */
2010-07-02 12:03:51 +04:00
for_each_online_gcwq_cpu ( cpu ) {
2010-06-29 12:07:14 +04:00
struct global_cwq * gcwq = get_gcwq ( cpu ) ;
struct worker * worker ;
2010-08-31 12:54:35 +04:00
if ( cpu ! = WORK_CPU_UNBOUND )
gcwq - > flags & = ~ GCWQ_DISASSOCIATED ;
2010-06-29 12:07:14 +04:00
worker = create_worker ( gcwq , true ) ;
BUG_ON ( ! worker ) ;
spin_lock_irq ( & gcwq - > lock ) ;
start_worker ( worker ) ;
spin_unlock_irq ( & gcwq - > lock ) ;
}
2010-06-29 12:07:14 +04:00
system_wq = alloc_workqueue ( " events " , 0 , 0 ) ;
system_long_wq = alloc_workqueue ( " events_long " , 0 , 0 ) ;
system_nrt_wq = alloc_workqueue ( " events_nrt " , WQ_NON_REENTRANT , 0 ) ;
2010-07-02 12:03:51 +04:00
system_unbound_wq = alloc_workqueue ( " events_unbound " , WQ_UNBOUND ,
WQ_UNBOUND_MAX_ACTIVE ) ;
2011-02-21 11:52:50 +03:00
system_freezable_wq = alloc_workqueue ( " events_freezable " ,
WQ_FREEZABLE , 0 ) ;
2012-03-02 13:51:00 +04:00
system_nrt_freezable_wq = alloc_workqueue ( " events_nrt_freezable " ,
WQ_NON_REENTRANT | WQ_FREEZABLE , 0 ) ;
2010-11-26 14:06:44 +03:00
BUG_ON ( ! system_wq | | ! system_long_wq | | ! system_nrt_wq | |
2012-03-02 13:51:00 +04:00
! system_unbound_wq | | ! system_freezable_wq | |
! system_nrt_freezable_wq ) ;
2010-07-31 01:57:37 +04:00
return 0 ;
2005-04-17 02:20:36 +04:00
}
2010-07-31 01:57:37 +04:00
early_initcall ( init_workqueues ) ;