iv/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

powerpc/syscall: Avoid storing 'current' in another pointer

Browse Source 
By saving the pointer pointing to thread_info.flags, gcc copies r2
in a non-volatile register.

We know 'current' doesn't change, so avoid that intermediaite pointer.

Reduces null_syscall benchmark by 2 cycles (322 => 320 cycles)

On PPC64, gcc seems to know that 'current' is not changing, and it keeps
it in a non volatile register to avoid multiple read of 'current' in paca.

Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://lore.kernel.org/r/ad0363ff0ff8c125f40e1cdc589a85bbd7e31693.1612946484.git.christophe.leroy@csgroup.eu
This commit is contained in:
Christophe Leroy
2021-02-10 08:44:09 +00:00
committed by Michael Ellerman
parent d524dda719
commit 5b90b9661a
1 changed files with 9 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
arch/powerpc/kernel/interrupt.c
View File

@ -241,7 +241,6 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3,
struct pt_regs *regs, struct pt_regs *regs,
long scv) long scv)
{ {
unsigned long *ti_flagsp = &current_thread_info()->flags;
unsigned long ti_flags; unsigned long ti_flags;
unsigned long ret = 0; unsigned long ret = 0;
bool is_not_scv = !IS_ENABLED(CONFIG_PPC_BOOK3S_64) || !scv; bool is_not_scv = !IS_ENABLED(CONFIG_PPC_BOOK3S_64) || !scv;
@ -257,7 +256,7 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3,
/* Check whether the syscall is issued inside a restartable sequence */ /* Check whether the syscall is issued inside a restartable sequence */
rseq_syscall(regs); rseq_syscall(regs);
ti_flags = *ti_flagsp; ti_flags = current_thread_info()->flags;
if (unlikely(r3 >= (unsigned long)-MAX_ERRNO) && is_not_scv) { if (unlikely(r3 >= (unsigned long)-MAX_ERRNO) && is_not_scv) {
if (likely(!(ti_flags & (_TIF_NOERROR | _TIF_RESTOREALL)))) { if (likely(!(ti_flags & (_TIF_NOERROR | _TIF_RESTOREALL)))) {
@ -271,7 +270,7 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3,
ret = _TIF_RESTOREALL; ret = _TIF_RESTOREALL;
else else
regs->gpr[3] = r3; regs->gpr[3] = r3;
clear_bits(_TIF_PERSYSCALL_MASK, ti_flagsp); clear_bits(_TIF_PERSYSCALL_MASK, &current_thread_info()->flags);
} else { } else {
regs->gpr[3] = r3; regs->gpr[3] = r3;
} }
@ -284,7 +283,7 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3,
local_irq_disable(); local_irq_disable();
again: again:
ti_flags = READ_ONCE(*ti_flagsp); ti_flags = READ_ONCE(current_thread_info()->flags);
while (unlikely(ti_flags & (_TIF_USER_WORK_MASK & ~_TIF_RESTORE_TM))) { while (unlikely(ti_flags & (_TIF_USER_WORK_MASK & ~_TIF_RESTORE_TM))) {
local_irq_enable(); local_irq_enable();
if (ti_flags & _TIF_NEED_RESCHED) { if (ti_flags & _TIF_NEED_RESCHED) {
@ -300,7 +299,7 @@ again:
do_notify_resume(regs, ti_flags); do_notify_resume(regs, ti_flags);
} }
local_irq_disable(); local_irq_disable();
ti_flags = READ_ONCE(*ti_flagsp); ti_flags = READ_ONCE(current_thread_info()->flags);
} }
if (IS_ENABLED(CONFIG_PPC_BOOK3S) && IS_ENABLED(CONFIG_PPC_FPU)) { if (IS_ENABLED(CONFIG_PPC_BOOK3S) && IS_ENABLED(CONFIG_PPC_FPU)) {
@ -357,7 +356,6 @@ again:
#ifndef CONFIG_PPC_BOOK3E_64 /* BOOK3E not yet using this */ #ifndef CONFIG_PPC_BOOK3E_64 /* BOOK3E not yet using this */
notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned long msr) notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned long msr)
{ {
unsigned long *ti_flagsp = &current_thread_info()->flags;
unsigned long ti_flags; unsigned long ti_flags;
unsigned long flags; unsigned long flags;
unsigned long ret = 0; unsigned long ret = 0;
@ -380,7 +378,7 @@ notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned
local_irq_save(flags); local_irq_save(flags);
again: again:
ti_flags = READ_ONCE(*ti_flagsp); ti_flags = READ_ONCE(current_thread_info()->flags);
while (unlikely(ti_flags & (_TIF_USER_WORK_MASK & ~_TIF_RESTORE_TM))) { while (unlikely(ti_flags & (_TIF_USER_WORK_MASK & ~_TIF_RESTORE_TM))) {
local_irq_enable(); /* returning to user: may enable */ local_irq_enable(); /* returning to user: may enable */
if (ti_flags & _TIF_NEED_RESCHED) { if (ti_flags & _TIF_NEED_RESCHED) {
@ -391,7 +389,7 @@ again:
do_notify_resume(regs, ti_flags); do_notify_resume(regs, ti_flags);
} }
local_irq_disable(); local_irq_disable();
ti_flags = READ_ONCE(*ti_flagsp); ti_flags = READ_ONCE(current_thread_info()->flags);
} }
if (IS_ENABLED(CONFIG_PPC_BOOK3S) && IS_ENABLED(CONFIG_PPC_FPU)) { if (IS_ENABLED(CONFIG_PPC_BOOK3S) && IS_ENABLED(CONFIG_PPC_FPU)) {
@ -443,7 +441,6 @@ void preempt_schedule_irq(void);
notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsigned long msr) notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsigned long msr)
{ {
unsigned long *ti_flagsp = &current_thread_info()->flags;
unsigned long flags; unsigned long flags;
unsigned long ret = 0; unsigned long ret = 0;
#ifdef CONFIG_PPC64 #ifdef CONFIG_PPC64
@ -466,8 +463,8 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsign
amr = kuap_get_and_check_amr(); amr = kuap_get_and_check_amr();
#endif #endif
if (unlikely(*ti_flagsp & _TIF_EMULATE_STACK_STORE)) { if (unlikely(current_thread_info()->flags & _TIF_EMULATE_STACK_STORE)) {
clear_bits(_TIF_EMULATE_STACK_STORE, ti_flagsp); clear_bits(_TIF_EMULATE_STACK_STORE, &current_thread_info()->flags);
ret = 1; ret = 1;
} }
@ -479,7 +476,7 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsign
again: again:
if (IS_ENABLED(CONFIG_PREEMPT)) { if (IS_ENABLED(CONFIG_PREEMPT)) {
/* Return to preemptible kernel context */ /* Return to preemptible kernel context */
if (unlikely(*ti_flagsp & _TIF_NEED_RESCHED)) { if (unlikely(current_thread_info()->flags & _TIF_NEED_RESCHED)) {
if (preempt_count() == 0) if (preempt_count() == 0)
preempt_schedule_irq(); preempt_schedule_irq();
} }