ff5b4f1ed5
We'd like all architectures to convert to ARCH_ATOMIC, as once all
architectures are converted it will be possible to make significant
cleanups to the atomics headers, and this will make it much easier to
generically enable atomic functionality (e.g. debug logic in the
instrumented wrappers).
As a step towards that, this patch migrates sparc to ARCH_ATOMIC. The
arch code provides arch_{atomic,atomic64,xchg,cmpxchg}*(), and common
code wraps these with optional instrumentation to provide the regular
functions.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: "David S. Miller" <davem@davemloft.net>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Will Deacon <will@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/20210525140232.53872-31-mark.rutland@arm.com
203 lines
4.3 KiB
C
203 lines
4.3 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
/*
|
|
* atomic32.c: 32-bit atomic_t implementation
|
|
*
|
|
* Copyright (C) 2004 Keith M Wesolowski
|
|
* Copyright (C) 2007 Kyle McMartin
|
|
*
|
|
* Based on asm-parisc/atomic.h Copyright (C) 2000 Philipp Rumpf
|
|
*/
|
|
|
|
#include <linux/atomic.h>
|
|
#include <linux/spinlock.h>
|
|
#include <linux/module.h>
|
|
|
|
#ifdef CONFIG_SMP
|
|
#define ATOMIC_HASH_SIZE 4
|
|
#define ATOMIC_HASH(a) (&__atomic_hash[(((unsigned long)a)>>8) & (ATOMIC_HASH_SIZE-1)])
|
|
|
|
spinlock_t __atomic_hash[ATOMIC_HASH_SIZE] = {
|
|
[0 ... (ATOMIC_HASH_SIZE-1)] = __SPIN_LOCK_UNLOCKED(__atomic_hash)
|
|
};
|
|
|
|
#else /* SMP */
|
|
|
|
static DEFINE_SPINLOCK(dummy);
|
|
#define ATOMIC_HASH_SIZE 1
|
|
#define ATOMIC_HASH(a) (&dummy)
|
|
|
|
#endif /* SMP */
|
|
|
|
#define ATOMIC_FETCH_OP(op, c_op) \
|
|
int arch_atomic_fetch_##op(int i, atomic_t *v) \
|
|
{ \
|
|
int ret; \
|
|
unsigned long flags; \
|
|
spin_lock_irqsave(ATOMIC_HASH(v), flags); \
|
|
\
|
|
ret = v->counter; \
|
|
v->counter c_op i; \
|
|
\
|
|
spin_unlock_irqrestore(ATOMIC_HASH(v), flags); \
|
|
return ret; \
|
|
} \
|
|
EXPORT_SYMBOL(arch_atomic_fetch_##op);
|
|
|
|
#define ATOMIC_OP_RETURN(op, c_op) \
|
|
int arch_atomic_##op##_return(int i, atomic_t *v) \
|
|
{ \
|
|
int ret; \
|
|
unsigned long flags; \
|
|
spin_lock_irqsave(ATOMIC_HASH(v), flags); \
|
|
\
|
|
ret = (v->counter c_op i); \
|
|
\
|
|
spin_unlock_irqrestore(ATOMIC_HASH(v), flags); \
|
|
return ret; \
|
|
} \
|
|
EXPORT_SYMBOL(arch_atomic_##op##_return);
|
|
|
|
ATOMIC_OP_RETURN(add, +=)
|
|
|
|
ATOMIC_FETCH_OP(add, +=)
|
|
ATOMIC_FETCH_OP(and, &=)
|
|
ATOMIC_FETCH_OP(or, |=)
|
|
ATOMIC_FETCH_OP(xor, ^=)
|
|
|
|
#undef ATOMIC_FETCH_OP
|
|
#undef ATOMIC_OP_RETURN
|
|
|
|
int arch_atomic_xchg(atomic_t *v, int new)
|
|
{
|
|
int ret;
|
|
unsigned long flags;
|
|
|
|
spin_lock_irqsave(ATOMIC_HASH(v), flags);
|
|
ret = v->counter;
|
|
v->counter = new;
|
|
spin_unlock_irqrestore(ATOMIC_HASH(v), flags);
|
|
return ret;
|
|
}
|
|
EXPORT_SYMBOL(arch_atomic_xchg);
|
|
|
|
int arch_atomic_cmpxchg(atomic_t *v, int old, int new)
|
|
{
|
|
int ret;
|
|
unsigned long flags;
|
|
|
|
spin_lock_irqsave(ATOMIC_HASH(v), flags);
|
|
ret = v->counter;
|
|
if (likely(ret == old))
|
|
v->counter = new;
|
|
|
|
spin_unlock_irqrestore(ATOMIC_HASH(v), flags);
|
|
return ret;
|
|
}
|
|
EXPORT_SYMBOL(arch_atomic_cmpxchg);
|
|
|
|
int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u)
|
|
{
|
|
int ret;
|
|
unsigned long flags;
|
|
|
|
spin_lock_irqsave(ATOMIC_HASH(v), flags);
|
|
ret = v->counter;
|
|
if (ret != u)
|
|
v->counter += a;
|
|
spin_unlock_irqrestore(ATOMIC_HASH(v), flags);
|
|
return ret;
|
|
}
|
|
EXPORT_SYMBOL(arch_atomic_fetch_add_unless);
|
|
|
|
/* Atomic operations are already serializing */
|
|
void arch_atomic_set(atomic_t *v, int i)
|
|
{
|
|
unsigned long flags;
|
|
|
|
spin_lock_irqsave(ATOMIC_HASH(v), flags);
|
|
v->counter = i;
|
|
spin_unlock_irqrestore(ATOMIC_HASH(v), flags);
|
|
}
|
|
EXPORT_SYMBOL(arch_atomic_set);
|
|
|
|
unsigned long ___set_bit(unsigned long *addr, unsigned long mask)
|
|
{
|
|
unsigned long old, flags;
|
|
|
|
spin_lock_irqsave(ATOMIC_HASH(addr), flags);
|
|
old = *addr;
|
|
*addr = old | mask;
|
|
spin_unlock_irqrestore(ATOMIC_HASH(addr), flags);
|
|
|
|
return old & mask;
|
|
}
|
|
EXPORT_SYMBOL(___set_bit);
|
|
|
|
unsigned long ___clear_bit(unsigned long *addr, unsigned long mask)
|
|
{
|
|
unsigned long old, flags;
|
|
|
|
spin_lock_irqsave(ATOMIC_HASH(addr), flags);
|
|
old = *addr;
|
|
*addr = old & ~mask;
|
|
spin_unlock_irqrestore(ATOMIC_HASH(addr), flags);
|
|
|
|
return old & mask;
|
|
}
|
|
EXPORT_SYMBOL(___clear_bit);
|
|
|
|
unsigned long ___change_bit(unsigned long *addr, unsigned long mask)
|
|
{
|
|
unsigned long old, flags;
|
|
|
|
spin_lock_irqsave(ATOMIC_HASH(addr), flags);
|
|
old = *addr;
|
|
*addr = old ^ mask;
|
|
spin_unlock_irqrestore(ATOMIC_HASH(addr), flags);
|
|
|
|
return old & mask;
|
|
}
|
|
EXPORT_SYMBOL(___change_bit);
|
|
|
|
unsigned long __cmpxchg_u32(volatile u32 *ptr, u32 old, u32 new)
|
|
{
|
|
unsigned long flags;
|
|
u32 prev;
|
|
|
|
spin_lock_irqsave(ATOMIC_HASH(ptr), flags);
|
|
if ((prev = *ptr) == old)
|
|
*ptr = new;
|
|
spin_unlock_irqrestore(ATOMIC_HASH(ptr), flags);
|
|
|
|
return (unsigned long)prev;
|
|
}
|
|
EXPORT_SYMBOL(__cmpxchg_u32);
|
|
|
|
u64 __cmpxchg_u64(u64 *ptr, u64 old, u64 new)
|
|
{
|
|
unsigned long flags;
|
|
u64 prev;
|
|
|
|
spin_lock_irqsave(ATOMIC_HASH(ptr), flags);
|
|
if ((prev = *ptr) == old)
|
|
*ptr = new;
|
|
spin_unlock_irqrestore(ATOMIC_HASH(ptr), flags);
|
|
|
|
return prev;
|
|
}
|
|
EXPORT_SYMBOL(__cmpxchg_u64);
|
|
|
|
unsigned long __xchg_u32(volatile u32 *ptr, u32 new)
|
|
{
|
|
unsigned long flags;
|
|
u32 prev;
|
|
|
|
spin_lock_irqsave(ATOMIC_HASH(ptr), flags);
|
|
prev = *ptr;
|
|
*ptr = new;
|
|
spin_unlock_irqrestore(ATOMIC_HASH(ptr), flags);
|
|
|
|
return (unsigned long)prev;
|
|
}
|
|
EXPORT_SYMBOL(__xchg_u32);
|