42f51fb24f
... to avoid unwanted gcc optimizations
SMP kernels fail to boot with commit 596ff4a09b89
("cpumask: re-introduce constant-sized cpumask optimizations").
|
| percpu: BUG: failure at mm/percpu.c:2981/pcpu_build_alloc_info()!
|
The write operation performed by the SCOND instruction in the atomic
inline asm code is not properly passed to the compiler. The compiler
cannot correctly optimize a nested loop that runs through the cpumask
in the pcpu_build_alloc_info() function.
Fix this by add a compiler barrier (memory clobber in inline asm).
Apparently atomic ops used to have memory clobber implicitly via
surrounding smp_mb(). However commit b64be6836993c431e
("ARC: atomics: implement relaxed variants") removed the smp_mb() for
the relaxed variants, but failed to add the explicit compiler barrier.
Link: https://github.com/foss-for-synopsys-dwc-arc-processors/linux/issues/135
Cc: <stable@vger.kernel.org> # v6.3+
Fixes: b64be6836993c43 ("ARC: atomics: implement relaxed variants")
Signed-off-by: Pavel Kozlov <pavel.kozlov@synopsys.com>
Signed-off-by: Vineet Gupta <vgupta@kernel.org>
[vgupta: tweaked the changelog and added Fixes tag]
98 lines
2.7 KiB
C
98 lines
2.7 KiB
C
/* SPDX-License-Identifier: GPL-2.0-only */
|
|
|
|
#ifndef _ASM_ARC_ATOMIC_LLSC_H
|
|
#define _ASM_ARC_ATOMIC_LLSC_H
|
|
|
|
#define arch_atomic_set(v, i) WRITE_ONCE(((v)->counter), (i))
|
|
|
|
#define ATOMIC_OP(op, asm_op) \
|
|
static inline void arch_atomic_##op(int i, atomic_t *v) \
|
|
{ \
|
|
unsigned int val; \
|
|
\
|
|
__asm__ __volatile__( \
|
|
"1: llock %[val], [%[ctr]] \n" \
|
|
" " #asm_op " %[val], %[val], %[i] \n" \
|
|
" scond %[val], [%[ctr]] \n" \
|
|
" bnz 1b \n" \
|
|
: [val] "=&r" (val) /* Early clobber to prevent reg reuse */ \
|
|
: [ctr] "r" (&v->counter), /* Not "m": llock only supports reg direct addr mode */ \
|
|
[i] "ir" (i) \
|
|
: "cc", "memory"); \
|
|
} \
|
|
|
|
#define ATOMIC_OP_RETURN(op, asm_op) \
|
|
static inline int arch_atomic_##op##_return_relaxed(int i, atomic_t *v) \
|
|
{ \
|
|
unsigned int val; \
|
|
\
|
|
__asm__ __volatile__( \
|
|
"1: llock %[val], [%[ctr]] \n" \
|
|
" " #asm_op " %[val], %[val], %[i] \n" \
|
|
" scond %[val], [%[ctr]] \n" \
|
|
" bnz 1b \n" \
|
|
: [val] "=&r" (val) \
|
|
: [ctr] "r" (&v->counter), \
|
|
[i] "ir" (i) \
|
|
: "cc", "memory"); \
|
|
\
|
|
return val; \
|
|
}
|
|
|
|
#define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed
|
|
#define arch_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed
|
|
|
|
#define ATOMIC_FETCH_OP(op, asm_op) \
|
|
static inline int arch_atomic_fetch_##op##_relaxed(int i, atomic_t *v) \
|
|
{ \
|
|
unsigned int val, orig; \
|
|
\
|
|
__asm__ __volatile__( \
|
|
"1: llock %[orig], [%[ctr]] \n" \
|
|
" " #asm_op " %[val], %[orig], %[i] \n" \
|
|
" scond %[val], [%[ctr]] \n" \
|
|
" bnz 1b \n" \
|
|
: [val] "=&r" (val), \
|
|
[orig] "=&r" (orig) \
|
|
: [ctr] "r" (&v->counter), \
|
|
[i] "ir" (i) \
|
|
: "cc", "memory"); \
|
|
\
|
|
return orig; \
|
|
}
|
|
|
|
#define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed
|
|
#define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed
|
|
|
|
#define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed
|
|
#define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot_relaxed
|
|
#define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed
|
|
#define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed
|
|
|
|
#define ATOMIC_OPS(op, asm_op) \
|
|
ATOMIC_OP(op, asm_op) \
|
|
ATOMIC_OP_RETURN(op, asm_op) \
|
|
ATOMIC_FETCH_OP(op, asm_op)
|
|
|
|
ATOMIC_OPS(add, add)
|
|
ATOMIC_OPS(sub, sub)
|
|
|
|
#undef ATOMIC_OPS
|
|
#define ATOMIC_OPS(op, asm_op) \
|
|
ATOMIC_OP(op, asm_op) \
|
|
ATOMIC_FETCH_OP(op, asm_op)
|
|
|
|
ATOMIC_OPS(and, and)
|
|
ATOMIC_OPS(andnot, bic)
|
|
ATOMIC_OPS(or, or)
|
|
ATOMIC_OPS(xor, xor)
|
|
|
|
#define arch_atomic_andnot arch_atomic_andnot
|
|
|
|
#undef ATOMIC_OPS
|
|
#undef ATOMIC_FETCH_OP
|
|
#undef ATOMIC_OP_RETURN
|
|
#undef ATOMIC_OP
|
|
|
|
#endif
|