11276d5306
There are various problems and short-comings with the current
static_key interface:
- static_key_{true,false}() read like a branch depending on the key
value, instead of the actual likely/unlikely branch depending on
init value.
- static_key_{true,false}() are, as stated above, tied to the
static_key init values STATIC_KEY_INIT_{TRUE,FALSE}.
- we're limited to the 2 (out of 4) possible options that compile to
a default NOP because that's what our arch_static_branch() assembly
emits.
So provide a new static_key interface:
DEFINE_STATIC_KEY_TRUE(name);
DEFINE_STATIC_KEY_FALSE(name);
Which define a key of different types with an initial true/false
value.
Then allow:
static_branch_likely()
static_branch_unlikely()
to take a key of either type and emit the right instruction for the
case.
This means adding a second arch_static_branch_jump() assembly helper
which emits a JMP per default.
In order to determine the right instruction for the right state,
encode the branch type in the LSB of jump_entry::key.
This is the final step in removing the naming confusion that has led to
a stream of avoidable bugs such as:
a833581e372a ("x86, perf: Fix static_key bug in load_mm_cr4()")
... but it also allows new static key combinations that will give us
performance enhancements in the subsequent patches.
Tested-by: Rabin Vincent <rabin@rab.in> # arm
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Michael Ellerman <mpe@ellerman.id.au> # ppc
Acked-by: Heiko Carstens <heiko.carstens@de.ibm.com> # s390
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel@vger.kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>
71 lines
1.4 KiB
C
71 lines
1.4 KiB
C
/*
|
|
* This file is subject to the terms and conditions of the GNU General Public
|
|
* License. See the file "COPYING" in the main directory of this archive
|
|
* for more details.
|
|
*
|
|
* Copyright (c) 2010 Cavium Networks, Inc.
|
|
*/
|
|
#ifndef _ASM_MIPS_JUMP_LABEL_H
|
|
#define _ASM_MIPS_JUMP_LABEL_H
|
|
|
|
#ifndef __ASSEMBLY__
|
|
|
|
#include <linux/types.h>
|
|
|
|
#define JUMP_LABEL_NOP_SIZE 4
|
|
|
|
#ifdef CONFIG_64BIT
|
|
#define WORD_INSN ".dword"
|
|
#else
|
|
#define WORD_INSN ".word"
|
|
#endif
|
|
|
|
#ifdef CONFIG_CPU_MICROMIPS
|
|
#define NOP_INSN "nop32"
|
|
#else
|
|
#define NOP_INSN "nop"
|
|
#endif
|
|
|
|
static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
|
|
{
|
|
asm_volatile_goto("1:\t" NOP_INSN "\n\t"
|
|
"nop\n\t"
|
|
".pushsection __jump_table, \"aw\"\n\t"
|
|
WORD_INSN " 1b, %l[l_yes], %0\n\t"
|
|
".popsection\n\t"
|
|
: : "i" (&((char *)key)[branch]) : : l_yes);
|
|
|
|
return false;
|
|
l_yes:
|
|
return true;
|
|
}
|
|
|
|
static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch)
|
|
{
|
|
asm_volatile_goto("1:\tj %l[l_yes]\n\t"
|
|
"nop\n\t"
|
|
".pushsection __jump_table, \"aw\"\n\t"
|
|
WORD_INSN " 1b, %l[l_yes], %0\n\t"
|
|
".popsection\n\t"
|
|
: : "i" (&((char *)key)[branch]) : : l_yes);
|
|
|
|
return false;
|
|
l_yes:
|
|
return true;
|
|
}
|
|
|
|
#ifdef CONFIG_64BIT
|
|
typedef u64 jump_label_t;
|
|
#else
|
|
typedef u32 jump_label_t;
|
|
#endif
|
|
|
|
struct jump_entry {
|
|
jump_label_t code;
|
|
jump_label_t target;
|
|
jump_label_t key;
|
|
};
|
|
|
|
#endif /* __ASSEMBLY__ */
|
|
#endif /* _ASM_MIPS_JUMP_LABEL_H */
|