iv/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

x86/percpu: Introduce %rip-relative addressing to PER_CPU_VAR()

Browse Source 
Introduce x86_64 %rip-relative addressing to the PER_CPU_VAR() macro.
Instructions using %rip-relative address operand are one byte shorter
than their absolute address counterparts and are also compatible with
position independent executable (-fpie) builds. The patch reduces
code size of a test kernel build by 150 bytes.

The PER_CPU_VAR() macro is intended to be applied to a symbol and should
not be used with register operands. Introduce the new __percpu macro and
use it in cmpxchg{8,16}b_emu.S instead.

Also add a missing function comment to this_cpu_cmpxchg8b_emu().

No functional changes intended.

Signed-off-by: Uros Bizjak <ubizjak@gmail.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Cc: linux-kernel@vger.kernel.org
Cc: Brian Gerst <brgerst@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Sean Christopherson <seanjc@google.com>
This commit is contained in:
Uros Bizjak 2023-10-17 18:27:34 +02:00 committed by Ingo Molnar
parent aa47f90cd4
commit 59bec00ace
3 changed files with 35 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
arch/x86/include/asm/percpu.h
View File

@ -4,17 +4,21 @@
#ifdef CONFIG_X86_64
#define __percpu_seg gs
#define __percpu_rel (%rip)
#else
#define __percpu_seg fs
#define __percpu_rel
#endif
#ifdef __ASSEMBLY__
#ifdef CONFIG_SMP
#define PER_CPU_VAR(var) %__percpu_seg:var
#else /* ! SMP */
#define PER_CPU_VAR(var) var
#endif /* SMP */
#define __percpu %__percpu_seg:
#else
#define __percpu
#endif
#define PER_CPU_VAR(var) __percpu(var)__percpu_rel
#ifdef CONFIG_X86_64_SMP
#define INIT_PER_CPU_VAR(var) init_per_cpu__##var

12
arch/x86/lib/cmpxchg16b_emu.S
View File

@ -23,14 +23,14 @@ SYM_FUNC_START(this_cpu_cmpxchg16b_emu)
cli
/* if (*ptr == old) */
cmpq PER_CPU_VAR(0(%rsi)), %rax
cmpq __percpu (%rsi), %rax
jne .Lnot_same
cmpq PER_CPU_VAR(8(%rsi)), %rdx
cmpq __percpu 8(%rsi), %rdx
jne .Lnot_same
/* *ptr = new */
movq %rbx, PER_CPU_VAR(0(%rsi))
movq %rcx, PER_CPU_VAR(8(%rsi))
movq %rbx, __percpu (%rsi)
movq %rcx, __percpu 8(%rsi)
/* set ZF in EFLAGS to indicate success */
orl $X86_EFLAGS_ZF, (%rsp)
@ -42,8 +42,8 @@ SYM_FUNC_START(this_cpu_cmpxchg16b_emu)
/* *ptr != old */
/* old = *ptr */
movq PER_CPU_VAR(0(%rsi)), %rax
movq PER_CPU_VAR(8(%rsi)), %rdx
movq __percpu (%rsi), %rax
movq __percpu 8(%rsi), %rdx
/* clear ZF in EFLAGS to indicate failure */
andl $(~X86_EFLAGS_ZF), (%rsp)

30
arch/x86/lib/cmpxchg8b_emu.S
View File

@ -24,12 +24,12 @@ SYM_FUNC_START(cmpxchg8b_emu)
pushfl
cli
cmpl 0(%esi), %eax
cmpl (%esi), %eax
jne .Lnot_same
cmpl 4(%esi), %edx
jne .Lnot_same
movl %ebx, 0(%esi)
movl %ebx, (%esi)
movl %ecx, 4(%esi)
orl $X86_EFLAGS_ZF, (%esp)
@ -38,7 +38,7 @@ SYM_FUNC_START(cmpxchg8b_emu)
RET
.Lnot_same:
movl 0(%esi), %eax
movl (%esi), %eax
movl 4(%esi), %edx
andl $(~X86_EFLAGS_ZF), (%esp)
@ -53,18 +53,30 @@ EXPORT_SYMBOL(cmpxchg8b_emu)
#ifndef CONFIG_UML
/*
* Emulate 'cmpxchg8b %fs:(%rsi)'
*
* Inputs:
* %esi : memory location to compare
* %eax : low 32 bits of old value
* %edx : high 32 bits of old value
* %ebx : low 32 bits of new value
* %ecx : high 32 bits of new value
*
* Notably this is not LOCK prefixed and is not safe against NMIs
*/
SYM_FUNC_START(this_cpu_cmpxchg8b_emu)
pushfl
cli
cmpl PER_CPU_VAR(0(%esi)), %eax
cmpl __percpu (%esi), %eax
jne .Lnot_same2
cmpl PER_CPU_VAR(4(%esi)), %edx
cmpl __percpu 4(%esi), %edx
jne .Lnot_same2
movl %ebx, PER_CPU_VAR(0(%esi))
movl %ecx, PER_CPU_VAR(4(%esi))
movl %ebx, __percpu (%esi)
movl %ecx, __percpu 4(%esi)
orl $X86_EFLAGS_ZF, (%esp)
@ -72,8 +84,8 @@ SYM_FUNC_START(this_cpu_cmpxchg8b_emu)
RET
.Lnot_same2:
movl PER_CPU_VAR(0(%esi)), %eax
movl PER_CPU_VAR(4(%esi)), %edx
movl __percpu (%esi), %eax
movl __percpu 4(%esi), %edx
andl $(~X86_EFLAGS_ZF), (%esp)