iv/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

x86: Speed up ___preempt_schedule*() by using THUNK helpers

Browse Source 
___preempt_schedule() does SAVE_ALL/RESTORE_ALL but this is
suboptimal, we do not need to save/restore the callee-saved
register. And we already have arch/x86/lib/thunk_*.S which
implements the similar asm wrappers, so it makes sense to
redefine ___preempt_schedule() as "THUNK ..." and remove
preempt.S altogether.

Signed-off-by: Oleg Nesterov <oleg@redhat.com>
Reviewed-by: Andy Lutomirski <luto@amacapital.net>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Link: http://lkml.kernel.org/r/20140921184153.GA23727@redhat.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
This commit is contained in:
Oleg Nesterov 2014-09-21 20:41:53 +02:00 committed by Ingo Molnar
parent f3670394c2
commit 0ad6e3c519
4 changed files with 23 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
arch/x86/kernel/Makefile
View File

@ -39,8 +39,6 @@ obj-y += tsc.o tsc_msr.o io_delay.o rtc.o
obj-y += pci-iommu_table.o obj-y += pci-iommu_table.o
obj-y += resource.o obj-y += resource.o
obj-$(CONFIG_PREEMPT) += preempt.o
obj-y += process.o obj-y += process.o
obj-y += i387.o xsave.o obj-y += i387.o xsave.o
obj-y += ptrace.o obj-y += ptrace.o

25
arch/x86/kernel/preempt.S
View File

@ -1,25 +0,0 @@
#include <linux/linkage.h>
#include <asm/dwarf2.h>
#include <asm/asm.h>
#include <asm/calling.h>
ENTRY(___preempt_schedule)
CFI_STARTPROC
SAVE_ALL
call preempt_schedule
RESTORE_ALL
ret
CFI_ENDPROC
#ifdef CONFIG_CONTEXT_TRACKING
ENTRY(___preempt_schedule_context)
CFI_STARTPROC
SAVE_ALL
call preempt_schedule_context
RESTORE_ALL
ret
CFI_ENDPROC
#endif

20
arch/x86/lib/thunk_32.S
View File

@ -7,16 +7,19 @@
#include <linux/linkage.h> #include <linux/linkage.h>
#include <asm/asm.h> #include <asm/asm.h>
#ifdef CONFIG_TRACE_IRQFLAGS
/* put return address in eax (arg1) */ /* put return address in eax (arg1) */
.macro thunk_ra name,func .macro THUNK name, func, put_ret_addr_in_eax=0
.globl \name .globl \name
\name: \name:
pushl %eax pushl %eax
pushl %ecx pushl %ecx
pushl %edx pushl %edx
.if \put_ret_addr_in_eax
/* Place EIP in the arg1 */ /* Place EIP in the arg1 */
movl 3*4(%esp), %eax movl 3*4(%esp), %eax
.endif
call \func call \func
popl %edx popl %edx
popl %ecx popl %ecx
@ -25,6 +28,15 @@
_ASM_NOKPROBE(\name) _ASM_NOKPROBE(\name)
.endm .endm
thunk_ra trace_hardirqs_on_thunk,trace_hardirqs_on_caller #ifdef CONFIG_TRACE_IRQFLAGS
thunk_ra trace_hardirqs_off_thunk,trace_hardirqs_off_caller THUNK trace_hardirqs_on_thunk,trace_hardirqs_on_caller,1
THUNK trace_hardirqs_off_thunk,trace_hardirqs_off_caller,1
#endif #endif
#ifdef CONFIG_PREEMPT
THUNK ___preempt_schedule, preempt_schedule
#ifdef CONFIG_CONTEXT_TRACKING
THUNK ___preempt_schedule_context, preempt_schedule_context
#endif
#endif

7
arch/x86/lib/thunk_64.S
View File

@ -38,6 +38,13 @@
THUNK lockdep_sys_exit_thunk,lockdep_sys_exit THUNK lockdep_sys_exit_thunk,lockdep_sys_exit
#endif #endif
#ifdef CONFIG_PREEMPT
THUNK ___preempt_schedule, preempt_schedule
#ifdef CONFIG_CONTEXT_TRACKING
THUNK ___preempt_schedule_context, preempt_schedule_context
#endif
#endif
/* SAVE_ARGS below is used only for the .cfi directives it contains. */ /* SAVE_ARGS below is used only for the .cfi directives it contains. */
CFI_STARTPROC CFI_STARTPROC
SAVE_ARGS SAVE_ARGS