47933ad41a
A number of situations currently require the heavyweight smp_mb(), even though there is no need to order prior stores against later loads. Many architectures have much cheaper ways to handle these situations, but the Linux kernel currently has no portable way to make use of them. This commit therefore supplies smp_load_acquire() and smp_store_release() to remedy this situation. The new smp_load_acquire() primitive orders the specified load against any subsequent reads or writes, while the new smp_store_release() primitive orders the specifed store against any prior reads or writes. These primitives allow array-based circular FIFOs to be implemented without an smp_mb(), and also allow a theoretical hole in rcu_assign_pointer() to be closed at no additional expense on most architectures. In addition, the RCU experience transitioning from explicit smp_read_barrier_depends() and smp_wmb() to rcu_dereference() and rcu_assign_pointer(), respectively resulted in substantial improvements in readability. It therefore seems likely that replacing other explicit barriers with smp_load_acquire() and smp_store_release() will provide similar benefits. It appears that roughly half of the explicit barriers in core kernel code might be so replaced. [Changelog by PaulMck] Reviewed-by: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> Signed-off-by: Peter Zijlstra <peterz@infradead.org> Acked-by: Will Deacon <will.deacon@arm.com> Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org> Cc: Frederic Weisbecker <fweisbec@gmail.com> Cc: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca> Cc: Michael Ellerman <michael@ellerman.id.au> Cc: Michael Neuling <mikey@neuling.org> Cc: Russell King <linux@arm.linux.org.uk> Cc: Geert Uytterhoeven <geert@linux-m68k.org> Cc: Heiko Carstens <heiko.carstens@de.ibm.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Martin Schwidefsky <schwidefsky@de.ibm.com> Cc: Victor Kaplansky <VICTORK@il.ibm.com> Cc: Tony Luck <tony.luck@intel.com> Cc: Oleg Nesterov <oleg@redhat.com> Link: http://lkml.kernel.org/r/20131213150640.908486364@infradead.org Signed-off-by: Ingo Molnar <mingo@kernel.org>
92 lines
2.7 KiB
C
92 lines
2.7 KiB
C
/*
|
|
* Memory barrier definitions. This is based on information published
|
|
* in the Processor Abstraction Layer and the System Abstraction Layer
|
|
* manual.
|
|
*
|
|
* Copyright (C) 1998-2003 Hewlett-Packard Co
|
|
* David Mosberger-Tang <davidm@hpl.hp.com>
|
|
* Copyright (C) 1999 Asit Mallick <asit.k.mallick@intel.com>
|
|
* Copyright (C) 1999 Don Dugger <don.dugger@intel.com>
|
|
*/
|
|
#ifndef _ASM_IA64_BARRIER_H
|
|
#define _ASM_IA64_BARRIER_H
|
|
|
|
#include <linux/compiler.h>
|
|
|
|
/*
|
|
* Macros to force memory ordering. In these descriptions, "previous"
|
|
* and "subsequent" refer to program order; "visible" means that all
|
|
* architecturally visible effects of a memory access have occurred
|
|
* (at a minimum, this means the memory has been read or written).
|
|
*
|
|
* wmb(): Guarantees that all preceding stores to memory-
|
|
* like regions are visible before any subsequent
|
|
* stores and that all following stores will be
|
|
* visible only after all previous stores.
|
|
* rmb(): Like wmb(), but for reads.
|
|
* mb(): wmb()/rmb() combo, i.e., all previous memory
|
|
* accesses are visible before all subsequent
|
|
* accesses and vice versa. This is also known as
|
|
* a "fence."
|
|
*
|
|
* Note: "mb()" and its variants cannot be used as a fence to order
|
|
* accesses to memory mapped I/O registers. For that, mf.a needs to
|
|
* be used. However, we don't want to always use mf.a because (a)
|
|
* it's (presumably) much slower than mf and (b) mf.a is supported for
|
|
* sequential memory pages only.
|
|
*/
|
|
#define mb() ia64_mf()
|
|
#define rmb() mb()
|
|
#define wmb() mb()
|
|
#define read_barrier_depends() do { } while(0)
|
|
|
|
#ifdef CONFIG_SMP
|
|
# define smp_mb() mb()
|
|
# define smp_rmb() rmb()
|
|
# define smp_wmb() wmb()
|
|
# define smp_read_barrier_depends() read_barrier_depends()
|
|
|
|
#else
|
|
|
|
# define smp_mb() barrier()
|
|
# define smp_rmb() barrier()
|
|
# define smp_wmb() barrier()
|
|
# define smp_read_barrier_depends() do { } while(0)
|
|
|
|
#endif
|
|
|
|
/*
|
|
* IA64 GCC turns volatile stores into st.rel and volatile loads into ld.acq no
|
|
* need for asm trickery!
|
|
*/
|
|
|
|
#define smp_store_release(p, v) \
|
|
do { \
|
|
compiletime_assert_atomic_type(*p); \
|
|
barrier(); \
|
|
ACCESS_ONCE(*p) = (v); \
|
|
} while (0)
|
|
|
|
#define smp_load_acquire(p) \
|
|
({ \
|
|
typeof(*p) ___p1 = ACCESS_ONCE(*p); \
|
|
compiletime_assert_atomic_type(*p); \
|
|
barrier(); \
|
|
___p1; \
|
|
})
|
|
|
|
/*
|
|
* XXX check on this ---I suspect what Linus really wants here is
|
|
* acquire vs release semantics but we can't discuss this stuff with
|
|
* Linus just yet. Grrr...
|
|
*/
|
|
#define set_mb(var, value) do { (var) = (value); mb(); } while (0)
|
|
|
|
/*
|
|
* The group barrier in front of the rsm & ssm are necessary to ensure
|
|
* that none of the previous instructions in the same group are
|
|
* affected by the rsm/ssm.
|
|
*/
|
|
|
|
#endif /* _ASM_IA64_BARRIER_H */
|