2005-04-17 02:20:36 +04:00
/ *
* Compatibility m o d e s y s t e m c a l l e n t r y p o i n t f o r x86 - 6 4 .
*
* Copyright 2 0 0 0 - 2 0 0 2 A n d i K l e e n , S u S E L a b s .
* /
# include < a s m / d w a r f2 . h >
# include < a s m / c a l l i n g . h >
2005-09-09 23:28:48 +04:00
# include < a s m / a s m - o f f s e t s . h >
2005-04-17 02:20:36 +04:00
# include < a s m / c u r r e n t . h >
# include < a s m / e r r n o . h >
# include < a s m / i a32 _ u n i s t d . h >
# include < a s m / t h r e a d _ i n f o . h >
# include < a s m / s e g m e n t . h >
2006-07-03 11:24:45 +04:00
# include < a s m / i r q f l a g s . h >
2012-04-20 23:19:50 +04:00
# include < a s m / a s m . h >
2012-09-21 23:43:12 +04:00
# include < a s m / s m a p . h >
2005-04-17 02:20:36 +04:00
# include < l i n u x / l i n k a g e . h >
2012-01-03 23:23:06 +04:00
# include < l i n u x / e r r . h >
2005-04-17 02:20:36 +04:00
2008-06-24 12:13:31 +04:00
/* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this. */
# include < l i n u x / e l f - e m . h >
# define A U D I T _ A R C H _ I 3 8 6 ( E M _ 3 8 6 | _ _ A U D I T _ A R C H _ L E )
# define _ _ A U D I T _ A R C H _ L E 0 x40 0 0 0 0 0 0
# ifndef C O N F I G _ A U D I T S Y S C A L L
2009-09-30 14:22:11 +04:00
# define s y s e x i t _ a u d i t i a32 _ r e t _ f r o m _ s y s _ c a l l
# define s y s r e t l _ a u d i t i a32 _ r e t _ f r o m _ s y s _ c a l l
2008-06-24 12:13:31 +04:00
# endif
2011-03-07 21:10:39 +03:00
.section .entry .text , " ax"
2005-04-17 02:20:36 +04:00
.macro IA32_ARG_FIXUP noebp=0
movl % e d i ,% r8 d
.if \ noebp
.else
movl % e b p ,% r9 d
.endif
xchg % e c x ,% e s i
movl % e b x ,% e d i
movl % e d x ,% e d x / * z e r o e x t e n s i o n * /
.endm
2015-02-27 01:40:30 +03:00
/* clobbers %rax */
.macro CLEAR_RREGS _ r9 =rax
2005-04-17 02:20:36 +04:00
xorl % e a x ,% e a x
2015-02-27 01:40:30 +03:00
movq % r a x ,R 1 1 ( % r s p )
movq % r a x ,R 1 0 ( % r s p )
movq % \ _ r9 ,R 9 ( % r s p )
movq % r a x ,R 8 ( % r s p )
2005-04-17 02:20:36 +04:00
.endm
2008-07-09 13:38:07 +04:00
/ *
* Reload a r g r e g i s t e r s f r o m s t a c k i n c a s e p t r a c e c h a n g e d t h e m .
* We d o n ' t r e l o a d % e a x b e c a u s e s y s c a l l _ t r a c e _ e n t e r ( ) r e t u r n e d
2010-09-14 23:22:58 +04:00
* the % r a x v a l u e w e s h o u l d s e e . I n s t e a d , w e j u s t t r u n c a t e t h a t
* value t o 3 2 b i t s a g a i n a s w e d i d o n e n t r y f r o m u s e r m o d e .
* If i t ' s a n e w v a l u e s e t b y u s e r _ r e g s e t d u r i n g e n t r y t r a c i n g ,
* this m a t c h e s t h e n o r m a l t r u n c a t i o n o f t h e u s e r - m o d e v a l u e .
* If i t ' s - 1 t o m a k e u s p u n t t h e s y s c a l l , t h e n ( u 3 2 ) - 1 i s s t i l l
* an a p p r o p r i a t e l y i n v a l i d v a l u e .
2008-07-09 13:38:07 +04:00
* /
2015-02-27 01:40:30 +03:00
.macro LOAD_ARGS32 _ r9 =0
2008-08-29 16:21:11 +04:00
.if \ _ r9
2015-02-27 01:40:30 +03:00
movl R 9 ( % r s p ) ,% r9 d
2008-08-29 16:21:11 +04:00
.endif
2015-02-27 01:40:30 +03:00
movl R C X ( % r s p ) ,% e c x
movl R D X ( % r s p ) ,% e d x
movl R S I ( % r s p ) ,% e s i
movl R D I ( % r s p ) ,% e d i
2010-09-14 23:22:58 +04:00
movl % e a x ,% e a x / * z e r o e x t e n s i o n * /
2007-09-21 18:16:18 +04:00
.endm
2006-01-12 00:41:59 +03:00
.macro CFI_STARTPROC32 simple
CFI_ S T A R T P R O C \ s i m p l e
CFI_ U N D E F I N E D r8
CFI_ U N D E F I N E D r9
CFI_ U N D E F I N E D r10
CFI_ U N D E F I N E D r11
CFI_ U N D E F I N E D r12
CFI_ U N D E F I N E D r13
CFI_ U N D E F I N E D r14
CFI_ U N D E F I N E D r15
.endm
2008-06-25 08:19:28 +04:00
# ifdef C O N F I G _ P A R A V I R T
ENTRY( n a t i v e _ u s e r g s _ s y s r e t 3 2 )
swapgs
sysretl
ENDPROC( n a t i v e _ u s e r g s _ s y s r e t 3 2 )
ENTRY( n a t i v e _ i r q _ e n a b l e _ s y s e x i t )
swapgs
sti
sysexit
ENDPROC( n a t i v e _ i r q _ e n a b l e _ s y s e x i t )
# endif
2005-04-17 02:20:36 +04:00
/ *
* 3 2 bit S Y S E N T E R i n s t r u c t i o n e n t r y .
*
* Arguments :
* % eax S y s t e m c a l l n u m b e r .
* % ebx A r g 1
* % ecx A r g 2
* % edx A r g 3
* % esi A r g 4
* % edi A r g 5
* % ebp u s e r s t a c k
* 0 ( % ebp) A r g 6
*
* Interrupts o f f .
*
* This i s p u r e l y a f a s t p a t h . F o r a n y t h i n g c o m p l i c a t e d w e u s e t h e i n t 0 x80
* path b e l o w . S e t u p a c o m p l e t e h a r d w a r e s t a c k f r a m e t o s h a r e c o d e
* with t h e i n t 0 x80 p a t h .
* /
ENTRY( i a32 _ s y s e n t e r _ t a r g e t )
2006-01-12 00:41:59 +03:00
CFI_ S T A R T P R O C 3 2 s i m p l e
2006-09-26 12:52:41 +04:00
CFI_ S I G N A L _ F R A M E
2005-09-12 20:49:24 +04:00
CFI_ D E F _ C F A r s p ,0
CFI_ R E G I S T E R r s p ,r b p
2008-06-26 18:28:51 +04:00
SWAPGS_ U N S A F E _ S T A C K
2009-01-18 18:38:58 +03:00
movq P E R _ C P U _ V A R ( k e r n e l _ s t a c k ) , % r s p
addq $ ( K E R N E L _ S T A C K _ O F F S E T ) ,% r s p
2006-07-03 11:24:45 +04:00
/ *
* No n e e d t o f o l l o w t h i s i r q s o n / o f f s e c t i o n : t h e s y s c a l l
* disabled i r q s , h e r e w e e n a b l e i t s t r a i g h t a f t e r e n t r y :
* /
2008-06-25 08:19:29 +04:00
ENABLE_ I N T E R R U P T S ( C L B R _ N O N E )
2005-04-17 02:20:36 +04:00
movl % e b p ,% e b p / * z e r o e x t e n s i o n * /
2011-02-28 18:54:40 +03:00
pushq_ c f i $ _ _ U S E R 3 2 _ D S
2005-09-12 20:49:24 +04:00
/*CFI_REL_OFFSET ss,0*/
2011-02-28 18:54:40 +03:00
pushq_ c f i % r b p
2005-09-12 20:49:24 +04:00
CFI_ R E L _ O F F S E T r s p ,0
2011-02-28 18:54:40 +03:00
pushfq_ c f i
2005-09-12 20:49:24 +04:00
/*CFI_REL_OFFSET rflags,0*/
2011-11-29 15:17:45 +04:00
movl T I _ s y s e n t e r _ r e t u r n + T H R E A D _ I N F O ( % r s p ,3 * 8 - K E R N E L _ S T A C K _ O F F S E T ) ,% r10 d
2005-09-12 20:49:24 +04:00
CFI_ R E G I S T E R r i p ,r10
2011-02-28 18:54:40 +03:00
pushq_ c f i $ _ _ U S E R 3 2 _ C S
2005-09-12 20:49:24 +04:00
/*CFI_REL_OFFSET cs,0*/
2005-04-17 02:20:36 +04:00
movl % e a x , % e a x
2011-02-28 18:54:40 +03:00
pushq_ c f i % r10
2005-09-12 20:49:24 +04:00
CFI_ R E L _ O F F S E T r i p ,0
2011-02-28 18:54:40 +03:00
pushq_ c f i % r a x
2005-04-17 02:20:36 +04:00
cld
x86/asm/entry/64: Always allocate a complete "struct pt_regs" on the kernel stack
The 64-bit entry code was using six stack slots less by not
saving/restoring registers which are callee-preserved according
to the C ABI, and was not allocating space for them.
Only when syscalls needed a complete "struct pt_regs" was
the complete area allocated and filled in.
As an additional twist, on interrupt entry a "slightly less
truncated pt_regs" trick is used, to make nested interrupt
stacks easier to unwind.
This proved to be a source of significant obfuscation and subtle
bugs. For example, 'stub_fork' had to pop the return address,
extend the struct, save registers, and push return address back.
Ugly. 'ia32_ptregs_common' pops return address and "returns" via
jmp insn, throwing a wrench into CPU return stack cache.
This patch changes the code to always allocate a complete
"struct pt_regs" on the kernel stack. The saving of registers
is still done lazily.
"Partial pt_regs" trick on interrupt stack is retained.
Macros which manipulate "struct pt_regs" on stack are reworked:
- ALLOC_PT_GPREGS_ON_STACK allocates the structure.
- SAVE_C_REGS saves to it those registers which are clobbered
by C code.
- SAVE_EXTRA_REGS saves to it all other registers.
- Corresponding RESTORE_* and REMOVE_PT_GPREGS_FROM_STACK macros
reverse it.
'ia32_ptregs_common', 'stub_fork' and friends lost their ugly dance
with the return pointer.
LOAD_ARGS32 in ia32entry.S now uses symbolic stack offsets
instead of magic numbers.
'error_entry' and 'save_paranoid' now use SAVE_C_REGS +
SAVE_EXTRA_REGS instead of having it open-coded yet again.
Patch was run-tested: 64-bit executables, 32-bit executables,
strace works.
Timing tests did not show measurable difference in 32-bit
and 64-bit syscalls.
Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Signed-off-by: Andy Lutomirski <luto@amacapital.net>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/1423778052-21038-2-git-send-email-dvlasenk@redhat.com
Link: http://lkml.kernel.org/r/b89763d354aa23e670b9bdf3a40ae320320a7c2e.1424989793.git.luto@amacapital.net
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-02-27 01:40:27 +03:00
ALLOC_ P T _ G P R E G S _ O N _ S T A C K
SAVE_ C _ R E G S _ E X C E P T _ R 8 9 1 0 1 1
2005-04-17 02:20:36 +04:00
/ * no n e e d t o d o a n a c c e s s _ o k c h e c k h e r e b e c a u s e r b p h a s b e e n
3 2 bit z e r o e x t e n d e d * /
2012-09-21 23:43:12 +04:00
ASM_ S T A C
2008-08-29 16:21:11 +04:00
1 : movl ( % r b p ) ,% e b p
2012-04-20 23:19:50 +04:00
_ ASM_ E X T A B L E ( 1 b ,i a32 _ b a d a r g )
2012-09-21 23:43:12 +04:00
ASM_ C L A C
x86_64, entry: Filter RFLAGS.NT on entry from userspace
The NT flag doesn't do anything in long mode other than causing IRET
to #GP. Oddly, CPL3 code can still set NT using popf.
Entry via hardware or software interrupt clears NT automatically, so
the only relevant entries are fast syscalls.
If user code causes kernel code to run with NT set, then there's at
least some (small) chance that it could cause trouble. For example,
user code could cause a call to EFI code with NT set, and who knows
what would happen? Apparently some games on Wine sometimes do
this (!), and, if an IRET return happens, they will segfault. That
segfault cannot be handled, because signal delivery fails, too.
This patch programs the CPU to clear NT on entry via SYSCALL (both
32-bit and 64-bit, by my reading of the AMD APM), and it clears NT
in software on entry via SYSENTER.
To save a few cycles, this borrows a trick from Jan Beulich in Xen:
it checks whether NT is set before trying to clear it. As a result,
it seems to have very little effect on SYSENTER performance on my
machine.
There's another minor bug fix in here: it looks like the CFI
annotations were wrong if CONFIG_AUDITSYSCALL=n.
Testers beware: on Xen, SYSENTER with NT set turns into a GPF.
I haven't touched anything on 32-bit kernels.
The syscall mask change comes from a variant of this patch by Anish
Bhatt.
Note to stable maintainers: there is no known security issue here.
A misguided program can set NT and cause the kernel to try and fail
to deliver SIGSEGV, crashing the program. This patch fixes Far Cry
on Wine: https://bugs.winehq.org/show_bug.cgi?id=33275
Cc: <stable@vger.kernel.org>
Reported-by: Anish Bhatt <anish@chelsio.com>
Signed-off-by: Andy Lutomirski <luto@amacapital.net>
Link: http://lkml.kernel.org/r/395749a5d39a29bd3e4b35899cf3a3c1340e5595.1412189265.git.luto@amacapital.net
Signed-off-by: H. Peter Anvin <hpa@zytor.com>
2014-10-01 22:49:04 +04:00
/ *
* Sysenter d o e s n ' t f i l t e r f l a g s , s o w e n e e d t o c l e a r N T
* ourselves. T o s a v e a f e w c y c l e s , w e c a n c h e c k w h e t h e r
* NT w a s s e t i n s t e a d o f d o i n g a n u n c o n d i t i o n a l p o p f q .
* /
2015-02-27 01:40:30 +03:00
testl $ X 8 6 _ E F L A G S _ N T ,E F L A G S ( % r s p )
x86_64, entry: Filter RFLAGS.NT on entry from userspace
The NT flag doesn't do anything in long mode other than causing IRET
to #GP. Oddly, CPL3 code can still set NT using popf.
Entry via hardware or software interrupt clears NT automatically, so
the only relevant entries are fast syscalls.
If user code causes kernel code to run with NT set, then there's at
least some (small) chance that it could cause trouble. For example,
user code could cause a call to EFI code with NT set, and who knows
what would happen? Apparently some games on Wine sometimes do
this (!), and, if an IRET return happens, they will segfault. That
segfault cannot be handled, because signal delivery fails, too.
This patch programs the CPU to clear NT on entry via SYSCALL (both
32-bit and 64-bit, by my reading of the AMD APM), and it clears NT
in software on entry via SYSENTER.
To save a few cycles, this borrows a trick from Jan Beulich in Xen:
it checks whether NT is set before trying to clear it. As a result,
it seems to have very little effect on SYSENTER performance on my
machine.
There's another minor bug fix in here: it looks like the CFI
annotations were wrong if CONFIG_AUDITSYSCALL=n.
Testers beware: on Xen, SYSENTER with NT set turns into a GPF.
I haven't touched anything on 32-bit kernels.
The syscall mask change comes from a variant of this patch by Anish
Bhatt.
Note to stable maintainers: there is no known security issue here.
A misguided program can set NT and cause the kernel to try and fail
to deliver SIGSEGV, crashing the program. This patch fixes Far Cry
on Wine: https://bugs.winehq.org/show_bug.cgi?id=33275
Cc: <stable@vger.kernel.org>
Reported-by: Anish Bhatt <anish@chelsio.com>
Signed-off-by: Andy Lutomirski <luto@amacapital.net>
Link: http://lkml.kernel.org/r/395749a5d39a29bd3e4b35899cf3a3c1340e5595.1412189265.git.luto@amacapital.net
Signed-off-by: H. Peter Anvin <hpa@zytor.com>
2014-10-01 22:49:04 +04:00
jnz s y s e n t e r _ f i x _ f l a g s
sysenter_flags_fixed :
2015-02-27 01:40:30 +03:00
orl $ T S _ C O M P A T ,T I _ s t a t u s + T H R E A D _ I N F O ( % r s p ,R I P )
testl $ _ T I F _ W O R K _ S Y S C A L L _ E N T R Y ,T I _ f l a g s + T H R E A D _ I N F O ( % r s p ,R I P )
2005-09-12 20:49:24 +04:00
CFI_ R E M E M B E R _ S T A T E
2005-04-17 02:20:36 +04:00
jnz s y s e n t e r _ t r a c e s y s
x86-64, compat: Test %rax for the syscall number, not %eax
On 64 bits, we always, by necessity, jump through the system call
table via %rax. For 32-bit system calls, in theory the system call
number is stored in %eax, and the code was testing %eax for a valid
system call number. At one point we loaded the stored value back from
the stack to enforce zero-extension, but that was removed in checkin
d4d67150165df8bf1cc05e532f6efca96f907cab. An actual 32-bit process
will not be able to introduce a non-zero-extended number, but it can
happen via ptrace.
Instead of re-introducing the zero-extension, test what we are
actually going to use, i.e. %rax. This only adds a handful of REX
prefixes to the code.
Reported-by: Ben Hawkes <hawkes@sota.gen.nz>
Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
Cc: <stable@kernel.org>
Cc: Roland McGrath <roland@redhat.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
2010-09-14 23:42:41 +04:00
cmpq $ ( I A 3 2 _ N R _ s y s c a l l s - 1 ) ,% r a x
2006-04-07 21:50:31 +04:00
ja i a32 _ b a d s y s
2008-07-09 13:38:07 +04:00
sysenter_do_call :
2008-08-29 16:21:11 +04:00
IA3 2 _ A R G _ F I X U P
2008-06-24 12:13:31 +04:00
sysenter_dispatch :
2005-04-17 02:20:36 +04:00
call * i a32 _ s y s _ c a l l _ t a b l e ( ,% r a x ,8 )
2015-02-27 01:40:30 +03:00
movq % r a x ,R A X ( % r s p )
2008-06-25 08:19:29 +04:00
DISABLE_ I N T E R R U P T S ( C L B R _ N O N E )
2006-07-03 11:24:45 +04:00
TRACE_ I R Q S _ O F F
2015-02-27 01:40:30 +03:00
testl $ _ T I F _ A L L W O R K _ M A S K ,T I _ f l a g s + T H R E A D _ I N F O ( % r s p ,R I P )
2008-06-24 12:13:31 +04:00
jnz s y s e x i t _ a u d i t
sysexit_from_sys_call :
2015-02-27 01:40:30 +03:00
andl $ ~ T S _ C O M P A T ,T I _ s t a t u s + T H R E A D _ I N F O ( % r s p ,R I P )
2005-04-17 02:20:36 +04:00
/* clear IF, that popfq doesn't enable interrupts early */
2015-02-27 01:40:30 +03:00
andl $ ~ 0 x20 0 ,E F L A G S ( % r s p )
movl R I P ( % r s p ) ,% e d x / * U s e r % e i p * /
2008-01-30 15:30:43 +03:00
CFI_ R E G I S T E R r i p ,r d x
x86/asm/entry/64: Always allocate a complete "struct pt_regs" on the kernel stack
The 64-bit entry code was using six stack slots less by not
saving/restoring registers which are callee-preserved according
to the C ABI, and was not allocating space for them.
Only when syscalls needed a complete "struct pt_regs" was
the complete area allocated and filled in.
As an additional twist, on interrupt entry a "slightly less
truncated pt_regs" trick is used, to make nested interrupt
stacks easier to unwind.
This proved to be a source of significant obfuscation and subtle
bugs. For example, 'stub_fork' had to pop the return address,
extend the struct, save registers, and push return address back.
Ugly. 'ia32_ptregs_common' pops return address and "returns" via
jmp insn, throwing a wrench into CPU return stack cache.
This patch changes the code to always allocate a complete
"struct pt_regs" on the kernel stack. The saving of registers
is still done lazily.
"Partial pt_regs" trick on interrupt stack is retained.
Macros which manipulate "struct pt_regs" on stack are reworked:
- ALLOC_PT_GPREGS_ON_STACK allocates the structure.
- SAVE_C_REGS saves to it those registers which are clobbered
by C code.
- SAVE_EXTRA_REGS saves to it all other registers.
- Corresponding RESTORE_* and REMOVE_PT_GPREGS_FROM_STACK macros
reverse it.
'ia32_ptregs_common', 'stub_fork' and friends lost their ugly dance
with the return pointer.
LOAD_ARGS32 in ia32entry.S now uses symbolic stack offsets
instead of magic numbers.
'error_entry' and 'save_paranoid' now use SAVE_C_REGS +
SAVE_EXTRA_REGS instead of having it open-coded yet again.
Patch was run-tested: 64-bit executables, 32-bit executables,
strace works.
Timing tests did not show measurable difference in 32-bit
and 64-bit syscalls.
Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Signed-off-by: Andy Lutomirski <luto@amacapital.net>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/1423778052-21038-2-git-send-email-dvlasenk@redhat.com
Link: http://lkml.kernel.org/r/b89763d354aa23e670b9bdf3a40ae320320a7c2e.1424989793.git.luto@amacapital.net
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-02-27 01:40:27 +03:00
RESTORE_ R S I _ R D I
REMOVE_ P T _ G P R E G S _ F R O M _ S T A C K 3 * 8
2009-09-30 14:22:11 +04:00
xorq % r8 ,% r8
xorq % r9 ,% r9
xorq % r10 ,% r10
xorq % r11 ,% r11
2011-02-28 18:54:40 +03:00
popfq_ c f i
2005-09-12 20:49:24 +04:00
/*CFI_RESTORE rflags*/
2011-02-28 18:54:40 +03:00
popq_ c f i % r c x / * U s e r % e s p * /
2005-09-12 20:49:24 +04:00
CFI_ R E G I S T E R r s p ,r c x
2006-07-03 11:24:45 +04:00
TRACE_ I R Q S _ O N
2008-06-25 08:19:28 +04:00
ENABLE_ I N T E R R U P T S _ S Y S E X I T 3 2
2005-04-17 02:20:36 +04:00
x86_64, entry: Filter RFLAGS.NT on entry from userspace
The NT flag doesn't do anything in long mode other than causing IRET
to #GP. Oddly, CPL3 code can still set NT using popf.
Entry via hardware or software interrupt clears NT automatically, so
the only relevant entries are fast syscalls.
If user code causes kernel code to run with NT set, then there's at
least some (small) chance that it could cause trouble. For example,
user code could cause a call to EFI code with NT set, and who knows
what would happen? Apparently some games on Wine sometimes do
this (!), and, if an IRET return happens, they will segfault. That
segfault cannot be handled, because signal delivery fails, too.
This patch programs the CPU to clear NT on entry via SYSCALL (both
32-bit and 64-bit, by my reading of the AMD APM), and it clears NT
in software on entry via SYSENTER.
To save a few cycles, this borrows a trick from Jan Beulich in Xen:
it checks whether NT is set before trying to clear it. As a result,
it seems to have very little effect on SYSENTER performance on my
machine.
There's another minor bug fix in here: it looks like the CFI
annotations were wrong if CONFIG_AUDITSYSCALL=n.
Testers beware: on Xen, SYSENTER with NT set turns into a GPF.
I haven't touched anything on 32-bit kernels.
The syscall mask change comes from a variant of this patch by Anish
Bhatt.
Note to stable maintainers: there is no known security issue here.
A misguided program can set NT and cause the kernel to try and fail
to deliver SIGSEGV, crashing the program. This patch fixes Far Cry
on Wine: https://bugs.winehq.org/show_bug.cgi?id=33275
Cc: <stable@vger.kernel.org>
Reported-by: Anish Bhatt <anish@chelsio.com>
Signed-off-by: Andy Lutomirski <luto@amacapital.net>
Link: http://lkml.kernel.org/r/395749a5d39a29bd3e4b35899cf3a3c1340e5595.1412189265.git.luto@amacapital.net
Signed-off-by: H. Peter Anvin <hpa@zytor.com>
2014-10-01 22:49:04 +04:00
CFI_ R E S T O R E _ S T A T E
2008-06-24 12:13:31 +04:00
# ifdef C O N F I G _ A U D I T S Y S C A L L
.macro auditsys_entry_common
2014-03-04 19:38:06 +04:00
movl % e s i ,% r8 d / * 5 t h a r g : 4 t h s y s c a l l a r g * /
movl % e c x ,% r9 d / * s w a p w i t h e d x * /
movl % e d x ,% e c x / * 4 t h a r g : 3 r d s y s c a l l a r g * /
movl % r9 d ,% e d x / * 3 r d a r g : 2 n d s y s c a l l a r g * /
movl % e b x ,% e s i / * 2 n d a r g : 1 s t s y s c a l l a r g * /
movl % e a x ,% e d i / * 1 s t a r g : s y s c a l l n u m b e r * /
2012-01-03 23:23:06 +04:00
call _ _ a u d i t _ s y s c a l l _ e n t r y
2015-02-27 01:40:30 +03:00
movl R A X ( % r s p ) ,% e a x / * r e l o a d s y s c a l l n u m b e r * /
x86-64, compat: Test %rax for the syscall number, not %eax
On 64 bits, we always, by necessity, jump through the system call
table via %rax. For 32-bit system calls, in theory the system call
number is stored in %eax, and the code was testing %eax for a valid
system call number. At one point we loaded the stored value back from
the stack to enforce zero-extension, but that was removed in checkin
d4d67150165df8bf1cc05e532f6efca96f907cab. An actual 32-bit process
will not be able to introduce a non-zero-extended number, but it can
happen via ptrace.
Instead of re-introducing the zero-extension, test what we are
actually going to use, i.e. %rax. This only adds a handful of REX
prefixes to the code.
Reported-by: Ben Hawkes <hawkes@sota.gen.nz>
Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
Cc: <stable@kernel.org>
Cc: Roland McGrath <roland@redhat.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
2010-09-14 23:42:41 +04:00
cmpq $ ( I A 3 2 _ N R _ s y s c a l l s - 1 ) ,% r a x
2008-06-24 12:13:31 +04:00
ja i a32 _ b a d s y s
movl % e b x ,% e d i / * r e l o a d 1 s t s y s c a l l a r g * /
2015-02-27 01:40:30 +03:00
movl R C X ( % r s p ) ,% e s i / * r e l o a d 2 n d s y s c a l l a r g * /
movl R D X ( % r s p ) ,% e d x / * r e l o a d 3 r d s y s c a l l a r g * /
movl R S I ( % r s p ) ,% e c x / * r e l o a d 4 t h s y s c a l l a r g * /
movl R D I ( % r s p ) ,% r8 d / * r e l o a d 5 t h s y s c a l l a r g * /
2008-06-24 12:13:31 +04:00
.endm
2009-10-26 18:20:29 +03:00
.macro auditsys_exit exit
2015-02-27 01:40:30 +03:00
testl $ ( _ T I F _ A L L W O R K _ M A S K & ~ _ T I F _ S Y S C A L L _ A U D I T ) ,T I _ f l a g s + T H R E A D _ I N F O ( % r s p ,R I P )
2009-09-30 14:22:11 +04:00
jnz i a32 _ r e t _ f r o m _ s y s _ c a l l
2008-06-24 12:13:31 +04:00
TRACE_ I R Q S _ O N
2013-01-30 11:55:53 +04:00
ENABLE_ I N T E R R U P T S ( C L B R _ N O N E )
2008-06-24 12:13:31 +04:00
movl % e a x ,% e s i / * s e c o n d a r g , s y s c a l l r e t u r n v a l u e * /
2012-01-03 23:23:06 +04:00
cmpl $ - M A X _ E R R N O ,% e a x / * i s i t a n e r r o r ? * /
2012-01-03 23:23:06 +04:00
jbe 1 f
movslq % e a x , % r s i / * i f e r r o r s i g n e x t e n d t o 6 4 b i t s * /
1 : setbe % a l / * 1 i f e r r o r , 0 i f n o t * /
2008-06-24 12:13:31 +04:00
movzbl % a l ,% e d i / * z e r o - e x t e n d t h a t i n t o % e d i * /
2012-01-03 23:23:06 +04:00
call _ _ a u d i t _ s y s c a l l _ e x i t
2015-02-27 01:40:30 +03:00
movq R A X ( % r s p ) ,% r a x / * r e l o a d s y s c a l l r e t u r n v a l u e * /
2008-06-24 12:13:31 +04:00
movl $ ( _ T I F _ A L L W O R K _ M A S K & ~ _ T I F _ S Y S C A L L _ A U D I T ) ,% e d i
2013-01-30 11:55:53 +04:00
DISABLE_ I N T E R R U P T S ( C L B R _ N O N E )
2008-06-24 12:13:31 +04:00
TRACE_ I R Q S _ O F F
2015-02-27 01:40:30 +03:00
testl % e d i ,T I _ f l a g s + T H R E A D _ I N F O ( % r s p ,R I P )
2009-09-30 14:22:11 +04:00
jz \ e x i t
2015-02-27 01:40:30 +03:00
CLEAR_ R R E G S
2009-09-30 14:22:11 +04:00
jmp i n t _ w i t h _ c h e c k
2008-06-24 12:13:31 +04:00
.endm
sysenter_auditsys :
auditsys_ e n t r y _ c o m m o n
movl % e b p ,% r9 d / * r e l o a d 6 t h s y s c a l l a r g * /
jmp s y s e n t e r _ d i s p a t c h
sysexit_audit :
auditsys_ e x i t s y s e x i t _ f r o m _ s y s _ c a l l
# endif
x86_64, entry: Filter RFLAGS.NT on entry from userspace
The NT flag doesn't do anything in long mode other than causing IRET
to #GP. Oddly, CPL3 code can still set NT using popf.
Entry via hardware or software interrupt clears NT automatically, so
the only relevant entries are fast syscalls.
If user code causes kernel code to run with NT set, then there's at
least some (small) chance that it could cause trouble. For example,
user code could cause a call to EFI code with NT set, and who knows
what would happen? Apparently some games on Wine sometimes do
this (!), and, if an IRET return happens, they will segfault. That
segfault cannot be handled, because signal delivery fails, too.
This patch programs the CPU to clear NT on entry via SYSCALL (both
32-bit and 64-bit, by my reading of the AMD APM), and it clears NT
in software on entry via SYSENTER.
To save a few cycles, this borrows a trick from Jan Beulich in Xen:
it checks whether NT is set before trying to clear it. As a result,
it seems to have very little effect on SYSENTER performance on my
machine.
There's another minor bug fix in here: it looks like the CFI
annotations were wrong if CONFIG_AUDITSYSCALL=n.
Testers beware: on Xen, SYSENTER with NT set turns into a GPF.
I haven't touched anything on 32-bit kernels.
The syscall mask change comes from a variant of this patch by Anish
Bhatt.
Note to stable maintainers: there is no known security issue here.
A misguided program can set NT and cause the kernel to try and fail
to deliver SIGSEGV, crashing the program. This patch fixes Far Cry
on Wine: https://bugs.winehq.org/show_bug.cgi?id=33275
Cc: <stable@vger.kernel.org>
Reported-by: Anish Bhatt <anish@chelsio.com>
Signed-off-by: Andy Lutomirski <luto@amacapital.net>
Link: http://lkml.kernel.org/r/395749a5d39a29bd3e4b35899cf3a3c1340e5595.1412189265.git.luto@amacapital.net
Signed-off-by: H. Peter Anvin <hpa@zytor.com>
2014-10-01 22:49:04 +04:00
sysenter_fix_flags :
pushq_ c f i $ ( X 8 6 _ E F L A G S _ I F | X 8 6 _ E F L A G S _ F I X E D )
popfq_ c f i
jmp s y s e n t e r _ f l a g s _ f i x e d
2008-06-24 12:13:31 +04:00
sysenter_tracesys :
# ifdef C O N F I G _ A U D I T S Y S C A L L
2015-02-27 01:40:30 +03:00
testl $ ( _ T I F _ W O R K _ S Y S C A L L _ E N T R Y & ~ _ T I F _ S Y S C A L L _ A U D I T ) ,T I _ f l a g s + T H R E A D _ I N F O ( % r s p ,R I P )
2008-06-24 12:13:31 +04:00
jz s y s e n t e r _ a u d i t s y s
# endif
x86/asm/entry/64: Always allocate a complete "struct pt_regs" on the kernel stack
The 64-bit entry code was using six stack slots less by not
saving/restoring registers which are callee-preserved according
to the C ABI, and was not allocating space for them.
Only when syscalls needed a complete "struct pt_regs" was
the complete area allocated and filled in.
As an additional twist, on interrupt entry a "slightly less
truncated pt_regs" trick is used, to make nested interrupt
stacks easier to unwind.
This proved to be a source of significant obfuscation and subtle
bugs. For example, 'stub_fork' had to pop the return address,
extend the struct, save registers, and push return address back.
Ugly. 'ia32_ptregs_common' pops return address and "returns" via
jmp insn, throwing a wrench into CPU return stack cache.
This patch changes the code to always allocate a complete
"struct pt_regs" on the kernel stack. The saving of registers
is still done lazily.
"Partial pt_regs" trick on interrupt stack is retained.
Macros which manipulate "struct pt_regs" on stack are reworked:
- ALLOC_PT_GPREGS_ON_STACK allocates the structure.
- SAVE_C_REGS saves to it those registers which are clobbered
by C code.
- SAVE_EXTRA_REGS saves to it all other registers.
- Corresponding RESTORE_* and REMOVE_PT_GPREGS_FROM_STACK macros
reverse it.
'ia32_ptregs_common', 'stub_fork' and friends lost their ugly dance
with the return pointer.
LOAD_ARGS32 in ia32entry.S now uses symbolic stack offsets
instead of magic numbers.
'error_entry' and 'save_paranoid' now use SAVE_C_REGS +
SAVE_EXTRA_REGS instead of having it open-coded yet again.
Patch was run-tested: 64-bit executables, 32-bit executables,
strace works.
Timing tests did not show measurable difference in 32-bit
and 64-bit syscalls.
Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Signed-off-by: Andy Lutomirski <luto@amacapital.net>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/1423778052-21038-2-git-send-email-dvlasenk@redhat.com
Link: http://lkml.kernel.org/r/b89763d354aa23e670b9bdf3a40ae320320a7c2e.1424989793.git.luto@amacapital.net
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-02-27 01:40:27 +03:00
SAVE_ E X T R A _ R E G S
2005-04-17 02:20:36 +04:00
CLEAR_ R R E G S
2008-03-19 04:23:50 +03:00
movq $ - E N O S Y S ,R A X ( % r s p ) / * p t r a c e c a n c h a n g e t h i s f o r a b a d s y s c a l l * /
2005-04-17 02:20:36 +04:00
movq % r s p ,% r d i / * & p t _ r e g s - > a r g 1 * /
call s y s c a l l _ t r a c e _ e n t e r
2015-02-27 01:40:30 +03:00
LOAD_ A R G S 3 2 / * r e l o a d a r g s f r o m s t a c k i n c a s e p t r a c e c h a n g e d i t * /
x86/asm/entry/64: Always allocate a complete "struct pt_regs" on the kernel stack
The 64-bit entry code was using six stack slots less by not
saving/restoring registers which are callee-preserved according
to the C ABI, and was not allocating space for them.
Only when syscalls needed a complete "struct pt_regs" was
the complete area allocated and filled in.
As an additional twist, on interrupt entry a "slightly less
truncated pt_regs" trick is used, to make nested interrupt
stacks easier to unwind.
This proved to be a source of significant obfuscation and subtle
bugs. For example, 'stub_fork' had to pop the return address,
extend the struct, save registers, and push return address back.
Ugly. 'ia32_ptregs_common' pops return address and "returns" via
jmp insn, throwing a wrench into CPU return stack cache.
This patch changes the code to always allocate a complete
"struct pt_regs" on the kernel stack. The saving of registers
is still done lazily.
"Partial pt_regs" trick on interrupt stack is retained.
Macros which manipulate "struct pt_regs" on stack are reworked:
- ALLOC_PT_GPREGS_ON_STACK allocates the structure.
- SAVE_C_REGS saves to it those registers which are clobbered
by C code.
- SAVE_EXTRA_REGS saves to it all other registers.
- Corresponding RESTORE_* and REMOVE_PT_GPREGS_FROM_STACK macros
reverse it.
'ia32_ptregs_common', 'stub_fork' and friends lost their ugly dance
with the return pointer.
LOAD_ARGS32 in ia32entry.S now uses symbolic stack offsets
instead of magic numbers.
'error_entry' and 'save_paranoid' now use SAVE_C_REGS +
SAVE_EXTRA_REGS instead of having it open-coded yet again.
Patch was run-tested: 64-bit executables, 32-bit executables,
strace works.
Timing tests did not show measurable difference in 32-bit
and 64-bit syscalls.
Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Signed-off-by: Andy Lutomirski <luto@amacapital.net>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/1423778052-21038-2-git-send-email-dvlasenk@redhat.com
Link: http://lkml.kernel.org/r/b89763d354aa23e670b9bdf3a40ae320320a7c2e.1424989793.git.luto@amacapital.net
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-02-27 01:40:27 +03:00
RESTORE_ E X T R A _ R E G S
x86-64, compat: Test %rax for the syscall number, not %eax
On 64 bits, we always, by necessity, jump through the system call
table via %rax. For 32-bit system calls, in theory the system call
number is stored in %eax, and the code was testing %eax for a valid
system call number. At one point we loaded the stored value back from
the stack to enforce zero-extension, but that was removed in checkin
d4d67150165df8bf1cc05e532f6efca96f907cab. An actual 32-bit process
will not be able to introduce a non-zero-extended number, but it can
happen via ptrace.
Instead of re-introducing the zero-extension, test what we are
actually going to use, i.e. %rax. This only adds a handful of REX
prefixes to the code.
Reported-by: Ben Hawkes <hawkes@sota.gen.nz>
Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
Cc: <stable@kernel.org>
Cc: Roland McGrath <roland@redhat.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
2010-09-14 23:42:41 +04:00
cmpq $ ( I A 3 2 _ N R _ s y s c a l l s - 1 ) ,% r a x
2008-03-19 04:23:50 +03:00
ja i n t _ r e t _ f r o m _ s y s _ c a l l / * s y s e n t e r _ t r a c e s y s h a s s e t R A X ( % r s p ) * /
2005-04-17 02:20:36 +04:00
jmp s y s e n t e r _ d o _ c a l l
CFI_ E N D P R O C
2006-06-26 15:56:55 +04:00
ENDPROC( i a32 _ s y s e n t e r _ t a r g e t )
2005-04-17 02:20:36 +04:00
/ *
* 3 2 bit S Y S C A L L i n s t r u c t i o n e n t r y .
*
* Arguments :
* % eax S y s t e m c a l l n u m b e r .
* % ebx A r g 1
* % ecx r e t u r n E I P
* % edx A r g 3
* % esi A r g 4
* % edi A r g 5
* % ebp A r g 2 [ n o t e : n o t s a v e d i n t h e s t a c k f r a m e , s h o u l d n o t b e t o u c h e d ]
* % esp u s e r s t a c k
* 0 ( % esp) A r g 6
*
* Interrupts o f f .
*
* This i s p u r e l y a f a s t p a t h . F o r a n y t h i n g c o m p l i c a t e d w e u s e t h e i n t 0 x80
* path b e l o w . S e t u p a c o m p l e t e h a r d w a r e s t a c k f r a m e t o s h a r e c o d e
* with t h e i n t 0 x80 p a t h .
* /
ENTRY( i a32 _ c s t a r _ t a r g e t )
2006-01-12 00:41:59 +03:00
CFI_ S T A R T P R O C 3 2 s i m p l e
2006-09-26 12:52:41 +04:00
CFI_ S I G N A L _ F R A M E
2009-01-18 18:38:58 +03:00
CFI_ D E F _ C F A r s p ,K E R N E L _ S T A C K _ O F F S E T
2005-09-12 20:49:24 +04:00
CFI_ R E G I S T E R r i p ,r c x
/*CFI_REGISTER rflags,r11*/
2008-06-26 18:28:51 +04:00
SWAPGS_ U N S A F E _ S T A C K
2005-04-17 02:20:36 +04:00
movl % e s p ,% r8 d
2005-09-12 20:49:24 +04:00
CFI_ R E G I S T E R r s p ,r8
2009-01-18 18:38:58 +03:00
movq P E R _ C P U _ V A R ( k e r n e l _ s t a c k ) ,% r s p
2006-07-03 11:24:45 +04:00
/ *
* No n e e d t o f o l l o w t h i s i r q s o n / o f f s e c t i o n : t h e s y s c a l l
* disabled i r q s a n d h e r e w e e n a b l e i t s t r a i g h t a f t e r e n t r y :
* /
2008-06-25 08:19:29 +04:00
ENABLE_ I N T E R R U P T S ( C L B R _ N O N E )
x86/asm/entry/64: Always allocate a complete "struct pt_regs" on the kernel stack
The 64-bit entry code was using six stack slots less by not
saving/restoring registers which are callee-preserved according
to the C ABI, and was not allocating space for them.
Only when syscalls needed a complete "struct pt_regs" was
the complete area allocated and filled in.
As an additional twist, on interrupt entry a "slightly less
truncated pt_regs" trick is used, to make nested interrupt
stacks easier to unwind.
This proved to be a source of significant obfuscation and subtle
bugs. For example, 'stub_fork' had to pop the return address,
extend the struct, save registers, and push return address back.
Ugly. 'ia32_ptregs_common' pops return address and "returns" via
jmp insn, throwing a wrench into CPU return stack cache.
This patch changes the code to always allocate a complete
"struct pt_regs" on the kernel stack. The saving of registers
is still done lazily.
"Partial pt_regs" trick on interrupt stack is retained.
Macros which manipulate "struct pt_regs" on stack are reworked:
- ALLOC_PT_GPREGS_ON_STACK allocates the structure.
- SAVE_C_REGS saves to it those registers which are clobbered
by C code.
- SAVE_EXTRA_REGS saves to it all other registers.
- Corresponding RESTORE_* and REMOVE_PT_GPREGS_FROM_STACK macros
reverse it.
'ia32_ptregs_common', 'stub_fork' and friends lost their ugly dance
with the return pointer.
LOAD_ARGS32 in ia32entry.S now uses symbolic stack offsets
instead of magic numbers.
'error_entry' and 'save_paranoid' now use SAVE_C_REGS +
SAVE_EXTRA_REGS instead of having it open-coded yet again.
Patch was run-tested: 64-bit executables, 32-bit executables,
strace works.
Timing tests did not show measurable difference in 32-bit
and 64-bit syscalls.
Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Signed-off-by: Andy Lutomirski <luto@amacapital.net>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/1423778052-21038-2-git-send-email-dvlasenk@redhat.com
Link: http://lkml.kernel.org/r/b89763d354aa23e670b9bdf3a40ae320320a7c2e.1424989793.git.luto@amacapital.net
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-02-27 01:40:27 +03:00
ALLOC_ P T _ G P R E G S _ O N _ S T A C K 8
SAVE_ C _ R E G S _ E X C E P T _ R C X _ R 8 9 1 0 1 1
2005-04-17 02:20:36 +04:00
movl % e a x ,% e a x / * z e r o e x t e n s i o n * /
2015-02-27 01:40:30 +03:00
movq % r a x ,O R I G _ R A X ( % r s p )
movq % r c x ,R I P ( % r s p )
CFI_ R E L _ O F F S E T r i p ,R I P
movq % r b p ,R C X ( % r s p ) / * t h i s l i e s s l i g h t l y t o p t r a c e * /
2005-04-17 02:20:36 +04:00
movl % e b p ,% e c x
2015-02-27 01:40:30 +03:00
movq $ _ _ U S E R 3 2 _ C S ,C S ( % r s p )
movq $ _ _ U S E R 3 2 _ D S ,S S ( % r s p )
movq % r11 ,E F L A G S ( % r s p )
/*CFI_REL_OFFSET rflags,EFLAGS*/
movq % r8 ,R S P ( % r s p )
CFI_ R E L _ O F F S E T r s p ,R S P
2005-04-17 02:20:36 +04:00
/ * no n e e d t o d o a n a c c e s s _ o k c h e c k h e r e b e c a u s e r8 h a s b e e n
3 2 bit z e r o e x t e n d e d * /
/* hardware stack frame is complete now */
2012-09-21 23:43:12 +04:00
ASM_ S T A C
2005-04-17 02:20:36 +04:00
1 : movl ( % r8 ) ,% r9 d
2012-04-21 03:51:50 +04:00
_ ASM_ E X T A B L E ( 1 b ,i a32 _ b a d a r g )
2012-09-21 23:43:12 +04:00
ASM_ C L A C
2015-02-27 01:40:30 +03:00
orl $ T S _ C O M P A T ,T I _ s t a t u s + T H R E A D _ I N F O ( % r s p ,R I P )
testl $ _ T I F _ W O R K _ S Y S C A L L _ E N T R Y ,T I _ f l a g s + T H R E A D _ I N F O ( % r s p ,R I P )
2005-09-12 20:49:24 +04:00
CFI_ R E M E M B E R _ S T A T E
2005-04-17 02:20:36 +04:00
jnz c s t a r _ t r a c e s y s
x86-64, compat: Test %rax for the syscall number, not %eax
On 64 bits, we always, by necessity, jump through the system call
table via %rax. For 32-bit system calls, in theory the system call
number is stored in %eax, and the code was testing %eax for a valid
system call number. At one point we loaded the stored value back from
the stack to enforce zero-extension, but that was removed in checkin
d4d67150165df8bf1cc05e532f6efca96f907cab. An actual 32-bit process
will not be able to introduce a non-zero-extended number, but it can
happen via ptrace.
Instead of re-introducing the zero-extension, test what we are
actually going to use, i.e. %rax. This only adds a handful of REX
prefixes to the code.
Reported-by: Ben Hawkes <hawkes@sota.gen.nz>
Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
Cc: <stable@kernel.org>
Cc: Roland McGrath <roland@redhat.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
2010-09-14 23:42:41 +04:00
cmpq $ I A 3 2 _ N R _ s y s c a l l s - 1 ,% r a x
2006-04-07 21:50:31 +04:00
ja i a32 _ b a d s y s
2008-08-29 16:21:11 +04:00
cstar_do_call :
2005-04-17 02:20:36 +04:00
IA3 2 _ A R G _ F I X U P 1
2008-06-24 12:13:31 +04:00
cstar_dispatch :
2005-04-17 02:20:36 +04:00
call * i a32 _ s y s _ c a l l _ t a b l e ( ,% r a x ,8 )
2015-02-27 01:40:30 +03:00
movq % r a x ,R A X ( % r s p )
2008-06-25 08:19:29 +04:00
DISABLE_ I N T E R R U P T S ( C L B R _ N O N E )
2006-07-03 11:24:45 +04:00
TRACE_ I R Q S _ O F F
2015-02-27 01:40:30 +03:00
testl $ _ T I F _ A L L W O R K _ M A S K ,T I _ f l a g s + T H R E A D _ I N F O ( % r s p ,R I P )
2008-06-24 12:13:31 +04:00
jnz s y s r e t l _ a u d i t
sysretl_from_sys_call :
2015-02-27 01:40:30 +03:00
andl $ ~ T S _ C O M P A T ,T I _ s t a t u s + T H R E A D _ I N F O ( % r s p ,R I P )
x86/asm/entry/64: Always allocate a complete "struct pt_regs" on the kernel stack
The 64-bit entry code was using six stack slots less by not
saving/restoring registers which are callee-preserved according
to the C ABI, and was not allocating space for them.
Only when syscalls needed a complete "struct pt_regs" was
the complete area allocated and filled in.
As an additional twist, on interrupt entry a "slightly less
truncated pt_regs" trick is used, to make nested interrupt
stacks easier to unwind.
This proved to be a source of significant obfuscation and subtle
bugs. For example, 'stub_fork' had to pop the return address,
extend the struct, save registers, and push return address back.
Ugly. 'ia32_ptregs_common' pops return address and "returns" via
jmp insn, throwing a wrench into CPU return stack cache.
This patch changes the code to always allocate a complete
"struct pt_regs" on the kernel stack. The saving of registers
is still done lazily.
"Partial pt_regs" trick on interrupt stack is retained.
Macros which manipulate "struct pt_regs" on stack are reworked:
- ALLOC_PT_GPREGS_ON_STACK allocates the structure.
- SAVE_C_REGS saves to it those registers which are clobbered
by C code.
- SAVE_EXTRA_REGS saves to it all other registers.
- Corresponding RESTORE_* and REMOVE_PT_GPREGS_FROM_STACK macros
reverse it.
'ia32_ptregs_common', 'stub_fork' and friends lost their ugly dance
with the return pointer.
LOAD_ARGS32 in ia32entry.S now uses symbolic stack offsets
instead of magic numbers.
'error_entry' and 'save_paranoid' now use SAVE_C_REGS +
SAVE_EXTRA_REGS instead of having it open-coded yet again.
Patch was run-tested: 64-bit executables, 32-bit executables,
strace works.
Timing tests did not show measurable difference in 32-bit
and 64-bit syscalls.
Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Signed-off-by: Andy Lutomirski <luto@amacapital.net>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/1423778052-21038-2-git-send-email-dvlasenk@redhat.com
Link: http://lkml.kernel.org/r/b89763d354aa23e670b9bdf3a40ae320320a7c2e.1424989793.git.luto@amacapital.net
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-02-27 01:40:27 +03:00
RESTORE_ R S I _ R D I _ R D X
2015-02-27 01:40:30 +03:00
movl R I P ( % r s p ) ,% e c x
2005-09-12 20:49:24 +04:00
CFI_ R E G I S T E R r i p ,r c x
2015-02-27 01:40:30 +03:00
movl E F L A G S ( % r s p ) ,% r11 d
2005-09-12 20:49:24 +04:00
/*CFI_REGISTER rflags,r11*/
2009-09-30 14:22:11 +04:00
xorq % r10 ,% r10
xorq % r9 ,% r9
xorq % r8 ,% r8
2006-07-03 11:24:45 +04:00
TRACE_ I R Q S _ O N
2015-02-27 01:40:30 +03:00
movl R S P ( % r s p ) ,% e s p
2005-09-12 20:49:24 +04:00
CFI_ R E S T O R E r s p
2008-06-25 08:19:28 +04:00
USERGS_ S Y S R E T 3 2
2005-04-17 02:20:36 +04:00
2008-06-24 12:13:31 +04:00
# ifdef C O N F I G _ A U D I T S Y S C A L L
cstar_auditsys :
2005-09-12 20:49:24 +04:00
CFI_ R E S T O R E _ S T A T E
2015-02-27 01:40:30 +03:00
movl % r9 d ,R 9 ( % r s p ) / * r e g i s t e r t o b e c l o b b e r e d b y c a l l * /
2008-06-24 12:13:31 +04:00
auditsys_ e n t r y _ c o m m o n
2015-02-27 01:40:30 +03:00
movl R 9 ( % r s p ) ,% r9 d / * r e l o a d 6 t h s y s c a l l a r g * /
2008-06-24 12:13:31 +04:00
jmp c s t a r _ d i s p a t c h
sysretl_audit :
2009-10-26 18:20:29 +03:00
auditsys_ e x i t s y s r e t l _ f r o m _ s y s _ c a l l
2008-06-24 12:13:31 +04:00
# endif
cstar_tracesys :
# ifdef C O N F I G _ A U D I T S Y S C A L L
2015-02-27 01:40:30 +03:00
testl $ ( _ T I F _ W O R K _ S Y S C A L L _ E N T R Y & ~ _ T I F _ S Y S C A L L _ A U D I T ) ,T I _ f l a g s + T H R E A D _ I N F O ( % r s p ,R I P )
2008-06-24 12:13:31 +04:00
jz c s t a r _ a u d i t s y s
# endif
2007-11-07 18:48:39 +03:00
xchgl % r9 d ,% e b p
x86/asm/entry/64: Always allocate a complete "struct pt_regs" on the kernel stack
The 64-bit entry code was using six stack slots less by not
saving/restoring registers which are callee-preserved according
to the C ABI, and was not allocating space for them.
Only when syscalls needed a complete "struct pt_regs" was
the complete area allocated and filled in.
As an additional twist, on interrupt entry a "slightly less
truncated pt_regs" trick is used, to make nested interrupt
stacks easier to unwind.
This proved to be a source of significant obfuscation and subtle
bugs. For example, 'stub_fork' had to pop the return address,
extend the struct, save registers, and push return address back.
Ugly. 'ia32_ptregs_common' pops return address and "returns" via
jmp insn, throwing a wrench into CPU return stack cache.
This patch changes the code to always allocate a complete
"struct pt_regs" on the kernel stack. The saving of registers
is still done lazily.
"Partial pt_regs" trick on interrupt stack is retained.
Macros which manipulate "struct pt_regs" on stack are reworked:
- ALLOC_PT_GPREGS_ON_STACK allocates the structure.
- SAVE_C_REGS saves to it those registers which are clobbered
by C code.
- SAVE_EXTRA_REGS saves to it all other registers.
- Corresponding RESTORE_* and REMOVE_PT_GPREGS_FROM_STACK macros
reverse it.
'ia32_ptregs_common', 'stub_fork' and friends lost their ugly dance
with the return pointer.
LOAD_ARGS32 in ia32entry.S now uses symbolic stack offsets
instead of magic numbers.
'error_entry' and 'save_paranoid' now use SAVE_C_REGS +
SAVE_EXTRA_REGS instead of having it open-coded yet again.
Patch was run-tested: 64-bit executables, 32-bit executables,
strace works.
Timing tests did not show measurable difference in 32-bit
and 64-bit syscalls.
Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Signed-off-by: Andy Lutomirski <luto@amacapital.net>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/1423778052-21038-2-git-send-email-dvlasenk@redhat.com
Link: http://lkml.kernel.org/r/b89763d354aa23e670b9bdf3a40ae320320a7c2e.1424989793.git.luto@amacapital.net
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-02-27 01:40:27 +03:00
SAVE_ E X T R A _ R E G S
2015-02-27 01:40:30 +03:00
CLEAR_ R R E G S r9
2008-03-19 04:23:50 +03:00
movq $ - E N O S Y S ,R A X ( % r s p ) / * p t r a c e c a n c h a n g e t h i s f o r a b a d s y s c a l l * /
2005-04-17 02:20:36 +04:00
movq % r s p ,% r d i / * & p t _ r e g s - > a r g 1 * /
call s y s c a l l _ t r a c e _ e n t e r
2015-02-27 01:40:30 +03:00
LOAD_ A R G S 3 2 1 / * r e l o a d a r g s f r o m s t a c k i n c a s e p t r a c e c h a n g e d i t * /
x86/asm/entry/64: Always allocate a complete "struct pt_regs" on the kernel stack
The 64-bit entry code was using six stack slots less by not
saving/restoring registers which are callee-preserved according
to the C ABI, and was not allocating space for them.
Only when syscalls needed a complete "struct pt_regs" was
the complete area allocated and filled in.
As an additional twist, on interrupt entry a "slightly less
truncated pt_regs" trick is used, to make nested interrupt
stacks easier to unwind.
This proved to be a source of significant obfuscation and subtle
bugs. For example, 'stub_fork' had to pop the return address,
extend the struct, save registers, and push return address back.
Ugly. 'ia32_ptregs_common' pops return address and "returns" via
jmp insn, throwing a wrench into CPU return stack cache.
This patch changes the code to always allocate a complete
"struct pt_regs" on the kernel stack. The saving of registers
is still done lazily.
"Partial pt_regs" trick on interrupt stack is retained.
Macros which manipulate "struct pt_regs" on stack are reworked:
- ALLOC_PT_GPREGS_ON_STACK allocates the structure.
- SAVE_C_REGS saves to it those registers which are clobbered
by C code.
- SAVE_EXTRA_REGS saves to it all other registers.
- Corresponding RESTORE_* and REMOVE_PT_GPREGS_FROM_STACK macros
reverse it.
'ia32_ptregs_common', 'stub_fork' and friends lost their ugly dance
with the return pointer.
LOAD_ARGS32 in ia32entry.S now uses symbolic stack offsets
instead of magic numbers.
'error_entry' and 'save_paranoid' now use SAVE_C_REGS +
SAVE_EXTRA_REGS instead of having it open-coded yet again.
Patch was run-tested: 64-bit executables, 32-bit executables,
strace works.
Timing tests did not show measurable difference in 32-bit
and 64-bit syscalls.
Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Signed-off-by: Andy Lutomirski <luto@amacapital.net>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/1423778052-21038-2-git-send-email-dvlasenk@redhat.com
Link: http://lkml.kernel.org/r/b89763d354aa23e670b9bdf3a40ae320320a7c2e.1424989793.git.luto@amacapital.net
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-02-27 01:40:27 +03:00
RESTORE_ E X T R A _ R E G S
2007-11-07 18:48:39 +03:00
xchgl % e b p ,% r9 d
x86-64, compat: Test %rax for the syscall number, not %eax
On 64 bits, we always, by necessity, jump through the system call
table via %rax. For 32-bit system calls, in theory the system call
number is stored in %eax, and the code was testing %eax for a valid
system call number. At one point we loaded the stored value back from
the stack to enforce zero-extension, but that was removed in checkin
d4d67150165df8bf1cc05e532f6efca96f907cab. An actual 32-bit process
will not be able to introduce a non-zero-extended number, but it can
happen via ptrace.
Instead of re-introducing the zero-extension, test what we are
actually going to use, i.e. %rax. This only adds a handful of REX
prefixes to the code.
Reported-by: Ben Hawkes <hawkes@sota.gen.nz>
Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
Cc: <stable@kernel.org>
Cc: Roland McGrath <roland@redhat.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
2010-09-14 23:42:41 +04:00
cmpq $ ( I A 3 2 _ N R _ s y s c a l l s - 1 ) ,% r a x
2008-03-19 04:23:50 +03:00
ja i n t _ r e t _ f r o m _ s y s _ c a l l / * c s t a r _ t r a c e s y s h a s s e t R A X ( % r s p ) * /
2005-04-17 02:20:36 +04:00
jmp c s t a r _ d o _ c a l l
2006-06-26 15:56:55 +04:00
END( i a32 _ c s t a r _ t a r g e t )
2005-04-17 02:20:36 +04:00
ia32_badarg :
2012-09-21 23:43:12 +04:00
ASM_ C L A C
2005-04-17 02:20:36 +04:00
movq $ - E F A U L T ,% r a x
jmp i a32 _ s y s r e t
CFI_ E N D P R O C
/ *
* Emulated I A 3 2 s y s t e m c a l l s v i a i n t 0 x80 .
*
* Arguments :
* % eax S y s t e m c a l l n u m b e r .
* % ebx A r g 1
* % ecx A r g 2
* % edx A r g 3
* % esi A r g 4
* % edi A r g 5
* % ebp A r g 6 [ n o t e : n o t s a v e d i n t h e s t a c k f r a m e , s h o u l d n o t b e t o u c h e d ]
*
* Notes :
* Uses t h e s a m e s t a c k f r a m e a s t h e x86 - 6 4 v e r s i o n .
* All r e g i s t e r s e x c e p t % e a x m u s t b e s a v e d ( b u t p t r a c e m a y v i o l a t e t h a t )
* Arguments a r e z e r o e x t e n d e d . F o r s y s t e m c a l l s t h a t w a n t s i g n e x t e n s i o n a n d
* take l o n g a r g u m e n t s a w r a p p e r i s n e e d e d . M o s t c a l l s c a n j u s t b e c a l l e d
* directly.
* Assumes i t i s o n l y c a l l e d f r o m u s e r s p a c e a n d e n t e r e d w i t h i n t e r r u p t s o f f .
* /
ENTRY( i a32 _ s y s c a l l )
2007-07-21 19:10:20 +04:00
CFI_ S T A R T P R O C 3 2 s i m p l e
2006-09-26 12:52:41 +04:00
CFI_ S I G N A L _ F R A M E
2005-09-12 20:49:24 +04:00
CFI_ D E F _ C F A r s p ,S S + 8 - R I P
/*CFI_REL_OFFSET ss,SS-RIP*/
CFI_ R E L _ O F F S E T r s p ,R S P - R I P
/*CFI_REL_OFFSET rflags,EFLAGS-RIP*/
/*CFI_REL_OFFSET cs,CS-RIP*/
CFI_ R E L _ O F F S E T r i p ,R I P - R I P
2008-07-09 02:06:28 +04:00
PARAVIRT_ A D J U S T _ E X C E P T I O N _ F R A M E
2008-06-25 08:19:29 +04:00
SWAPGS
2006-07-03 11:24:45 +04:00
/ *
* No n e e d t o f o l l o w t h i s i r q s o n / o f f s e c t i o n : t h e s y s c a l l
* disabled i r q s a n d h e r e w e e n a b l e i t s t r a i g h t a f t e r e n t r y :
* /
2008-06-25 08:19:29 +04:00
ENABLE_ I N T E R R U P T S ( C L B R _ N O N E )
2005-04-17 02:20:36 +04:00
movl % e a x ,% e a x
2011-02-28 18:54:40 +03:00
pushq_ c f i % r a x
2005-04-17 02:20:36 +04:00
cld
/ * note t h e r e g i s t e r s a r e n o t z e r o e x t e n d e d t o t h e s f .
this c o u l d b e a p r o b l e m . * /
x86/asm/entry/64: Always allocate a complete "struct pt_regs" on the kernel stack
The 64-bit entry code was using six stack slots less by not
saving/restoring registers which are callee-preserved according
to the C ABI, and was not allocating space for them.
Only when syscalls needed a complete "struct pt_regs" was
the complete area allocated and filled in.
As an additional twist, on interrupt entry a "slightly less
truncated pt_regs" trick is used, to make nested interrupt
stacks easier to unwind.
This proved to be a source of significant obfuscation and subtle
bugs. For example, 'stub_fork' had to pop the return address,
extend the struct, save registers, and push return address back.
Ugly. 'ia32_ptregs_common' pops return address and "returns" via
jmp insn, throwing a wrench into CPU return stack cache.
This patch changes the code to always allocate a complete
"struct pt_regs" on the kernel stack. The saving of registers
is still done lazily.
"Partial pt_regs" trick on interrupt stack is retained.
Macros which manipulate "struct pt_regs" on stack are reworked:
- ALLOC_PT_GPREGS_ON_STACK allocates the structure.
- SAVE_C_REGS saves to it those registers which are clobbered
by C code.
- SAVE_EXTRA_REGS saves to it all other registers.
- Corresponding RESTORE_* and REMOVE_PT_GPREGS_FROM_STACK macros
reverse it.
'ia32_ptregs_common', 'stub_fork' and friends lost their ugly dance
with the return pointer.
LOAD_ARGS32 in ia32entry.S now uses symbolic stack offsets
instead of magic numbers.
'error_entry' and 'save_paranoid' now use SAVE_C_REGS +
SAVE_EXTRA_REGS instead of having it open-coded yet again.
Patch was run-tested: 64-bit executables, 32-bit executables,
strace works.
Timing tests did not show measurable difference in 32-bit
and 64-bit syscalls.
Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Signed-off-by: Andy Lutomirski <luto@amacapital.net>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/1423778052-21038-2-git-send-email-dvlasenk@redhat.com
Link: http://lkml.kernel.org/r/b89763d354aa23e670b9bdf3a40ae320320a7c2e.1424989793.git.luto@amacapital.net
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-02-27 01:40:27 +03:00
ALLOC_ P T _ G P R E G S _ O N _ S T A C K
SAVE_ C _ R E G S _ E X C E P T _ R 8 9 1 0 1 1
2015-02-27 01:40:30 +03:00
orl $ T S _ C O M P A T ,T I _ s t a t u s + T H R E A D _ I N F O ( % r s p ,R I P )
testl $ _ T I F _ W O R K _ S Y S C A L L _ E N T R Y ,T I _ f l a g s + T H R E A D _ I N F O ( % r s p ,R I P )
2005-04-17 02:20:36 +04:00
jnz i a32 _ t r a c e s y s
x86-64, compat: Test %rax for the syscall number, not %eax
On 64 bits, we always, by necessity, jump through the system call
table via %rax. For 32-bit system calls, in theory the system call
number is stored in %eax, and the code was testing %eax for a valid
system call number. At one point we loaded the stored value back from
the stack to enforce zero-extension, but that was removed in checkin
d4d67150165df8bf1cc05e532f6efca96f907cab. An actual 32-bit process
will not be able to introduce a non-zero-extended number, but it can
happen via ptrace.
Instead of re-introducing the zero-extension, test what we are
actually going to use, i.e. %rax. This only adds a handful of REX
prefixes to the code.
Reported-by: Ben Hawkes <hawkes@sota.gen.nz>
Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
Cc: <stable@kernel.org>
Cc: Roland McGrath <roland@redhat.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
2010-09-14 23:42:41 +04:00
cmpq $ ( I A 3 2 _ N R _ s y s c a l l s - 1 ) ,% r a x
2009-02-07 05:15:18 +03:00
ja i a32 _ b a d s y s
ia32_do_call :
2005-04-17 02:20:36 +04:00
IA3 2 _ A R G _ F I X U P
call * i a32 _ s y s _ c a l l _ t a b l e ( ,% r a x ,8 ) # x x x : r i p r e l a t i v e
ia32_sysret :
2015-02-27 01:40:30 +03:00
movq % r a x ,R A X ( % r s p )
2009-09-30 14:22:11 +04:00
ia32_ret_from_sys_call :
2015-02-27 01:40:30 +03:00
CLEAR_ R R E G S
x86/asm/entry/64: Always allocate a complete "struct pt_regs" on the kernel stack
The 64-bit entry code was using six stack slots less by not
saving/restoring registers which are callee-preserved according
to the C ABI, and was not allocating space for them.
Only when syscalls needed a complete "struct pt_regs" was
the complete area allocated and filled in.
As an additional twist, on interrupt entry a "slightly less
truncated pt_regs" trick is used, to make nested interrupt
stacks easier to unwind.
This proved to be a source of significant obfuscation and subtle
bugs. For example, 'stub_fork' had to pop the return address,
extend the struct, save registers, and push return address back.
Ugly. 'ia32_ptregs_common' pops return address and "returns" via
jmp insn, throwing a wrench into CPU return stack cache.
This patch changes the code to always allocate a complete
"struct pt_regs" on the kernel stack. The saving of registers
is still done lazily.
"Partial pt_regs" trick on interrupt stack is retained.
Macros which manipulate "struct pt_regs" on stack are reworked:
- ALLOC_PT_GPREGS_ON_STACK allocates the structure.
- SAVE_C_REGS saves to it those registers which are clobbered
by C code.
- SAVE_EXTRA_REGS saves to it all other registers.
- Corresponding RESTORE_* and REMOVE_PT_GPREGS_FROM_STACK macros
reverse it.
'ia32_ptregs_common', 'stub_fork' and friends lost their ugly dance
with the return pointer.
LOAD_ARGS32 in ia32entry.S now uses symbolic stack offsets
instead of magic numbers.
'error_entry' and 'save_paranoid' now use SAVE_C_REGS +
SAVE_EXTRA_REGS instead of having it open-coded yet again.
Patch was run-tested: 64-bit executables, 32-bit executables,
strace works.
Timing tests did not show measurable difference in 32-bit
and 64-bit syscalls.
Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Signed-off-by: Andy Lutomirski <luto@amacapital.net>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/1423778052-21038-2-git-send-email-dvlasenk@redhat.com
Link: http://lkml.kernel.org/r/b89763d354aa23e670b9bdf3a40ae320320a7c2e.1424989793.git.luto@amacapital.net
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-02-27 01:40:27 +03:00
jmp i n t _ r e t _ f r o m _ s y s _ c a l l
2005-04-17 02:20:36 +04:00
x86/asm/entry/64: Always allocate a complete "struct pt_regs" on the kernel stack
The 64-bit entry code was using six stack slots less by not
saving/restoring registers which are callee-preserved according
to the C ABI, and was not allocating space for them.
Only when syscalls needed a complete "struct pt_regs" was
the complete area allocated and filled in.
As an additional twist, on interrupt entry a "slightly less
truncated pt_regs" trick is used, to make nested interrupt
stacks easier to unwind.
This proved to be a source of significant obfuscation and subtle
bugs. For example, 'stub_fork' had to pop the return address,
extend the struct, save registers, and push return address back.
Ugly. 'ia32_ptregs_common' pops return address and "returns" via
jmp insn, throwing a wrench into CPU return stack cache.
This patch changes the code to always allocate a complete
"struct pt_regs" on the kernel stack. The saving of registers
is still done lazily.
"Partial pt_regs" trick on interrupt stack is retained.
Macros which manipulate "struct pt_regs" on stack are reworked:
- ALLOC_PT_GPREGS_ON_STACK allocates the structure.
- SAVE_C_REGS saves to it those registers which are clobbered
by C code.
- SAVE_EXTRA_REGS saves to it all other registers.
- Corresponding RESTORE_* and REMOVE_PT_GPREGS_FROM_STACK macros
reverse it.
'ia32_ptregs_common', 'stub_fork' and friends lost their ugly dance
with the return pointer.
LOAD_ARGS32 in ia32entry.S now uses symbolic stack offsets
instead of magic numbers.
'error_entry' and 'save_paranoid' now use SAVE_C_REGS +
SAVE_EXTRA_REGS instead of having it open-coded yet again.
Patch was run-tested: 64-bit executables, 32-bit executables,
strace works.
Timing tests did not show measurable difference in 32-bit
and 64-bit syscalls.
Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Signed-off-by: Andy Lutomirski <luto@amacapital.net>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/1423778052-21038-2-git-send-email-dvlasenk@redhat.com
Link: http://lkml.kernel.org/r/b89763d354aa23e670b9bdf3a40ae320320a7c2e.1424989793.git.luto@amacapital.net
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-02-27 01:40:27 +03:00
ia32_tracesys :
SAVE_ E X T R A _ R E G S
2007-07-21 19:10:20 +04:00
CLEAR_ R R E G S
2008-03-17 07:57:41 +03:00
movq $ - E N O S Y S ,R A X ( % r s p ) / * p t r a c e c a n c h a n g e t h i s f o r a b a d s y s c a l l * /
2005-04-17 02:20:36 +04:00
movq % r s p ,% r d i / * & p t _ r e g s - > a r g 1 * /
call s y s c a l l _ t r a c e _ e n t e r
2015-02-27 01:40:30 +03:00
LOAD_ A R G S 3 2 / * r e l o a d a r g s f r o m s t a c k i n c a s e p t r a c e c h a n g e d i t * /
x86/asm/entry/64: Always allocate a complete "struct pt_regs" on the kernel stack
The 64-bit entry code was using six stack slots less by not
saving/restoring registers which are callee-preserved according
to the C ABI, and was not allocating space for them.
Only when syscalls needed a complete "struct pt_regs" was
the complete area allocated and filled in.
As an additional twist, on interrupt entry a "slightly less
truncated pt_regs" trick is used, to make nested interrupt
stacks easier to unwind.
This proved to be a source of significant obfuscation and subtle
bugs. For example, 'stub_fork' had to pop the return address,
extend the struct, save registers, and push return address back.
Ugly. 'ia32_ptregs_common' pops return address and "returns" via
jmp insn, throwing a wrench into CPU return stack cache.
This patch changes the code to always allocate a complete
"struct pt_regs" on the kernel stack. The saving of registers
is still done lazily.
"Partial pt_regs" trick on interrupt stack is retained.
Macros which manipulate "struct pt_regs" on stack are reworked:
- ALLOC_PT_GPREGS_ON_STACK allocates the structure.
- SAVE_C_REGS saves to it those registers which are clobbered
by C code.
- SAVE_EXTRA_REGS saves to it all other registers.
- Corresponding RESTORE_* and REMOVE_PT_GPREGS_FROM_STACK macros
reverse it.
'ia32_ptregs_common', 'stub_fork' and friends lost their ugly dance
with the return pointer.
LOAD_ARGS32 in ia32entry.S now uses symbolic stack offsets
instead of magic numbers.
'error_entry' and 'save_paranoid' now use SAVE_C_REGS +
SAVE_EXTRA_REGS instead of having it open-coded yet again.
Patch was run-tested: 64-bit executables, 32-bit executables,
strace works.
Timing tests did not show measurable difference in 32-bit
and 64-bit syscalls.
Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Signed-off-by: Andy Lutomirski <luto@amacapital.net>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/1423778052-21038-2-git-send-email-dvlasenk@redhat.com
Link: http://lkml.kernel.org/r/b89763d354aa23e670b9bdf3a40ae320320a7c2e.1424989793.git.luto@amacapital.net
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-02-27 01:40:27 +03:00
RESTORE_ E X T R A _ R E G S
x86-64, compat: Test %rax for the syscall number, not %eax
On 64 bits, we always, by necessity, jump through the system call
table via %rax. For 32-bit system calls, in theory the system call
number is stored in %eax, and the code was testing %eax for a valid
system call number. At one point we loaded the stored value back from
the stack to enforce zero-extension, but that was removed in checkin
d4d67150165df8bf1cc05e532f6efca96f907cab. An actual 32-bit process
will not be able to introduce a non-zero-extended number, but it can
happen via ptrace.
Instead of re-introducing the zero-extension, test what we are
actually going to use, i.e. %rax. This only adds a handful of REX
prefixes to the code.
Reported-by: Ben Hawkes <hawkes@sota.gen.nz>
Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
Cc: <stable@kernel.org>
Cc: Roland McGrath <roland@redhat.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
2010-09-14 23:42:41 +04:00
cmpq $ ( I A 3 2 _ N R _ s y s c a l l s - 1 ) ,% r a x
2009-02-07 05:15:18 +03:00
ja i n t _ r e t _ f r o m _ s y s _ c a l l / * i a32 _ t r a c e s y s h a s s e t R A X ( % r s p ) * /
jmp i a32 _ d o _ c a l l
2006-06-26 15:56:55 +04:00
END( i a32 _ s y s c a l l )
2005-04-17 02:20:36 +04:00
ia32_badsys :
2015-02-27 01:40:30 +03:00
movq $ 0 ,O R I G _ R A X ( % r s p )
2009-09-30 14:22:11 +04:00
movq $ - E N O S Y S ,% r a x
jmp i a32 _ s y s r e t
2005-04-17 02:20:36 +04:00
CFI_ E N D P R O C
2013-07-10 22:04:28 +04:00
.macro PTREGSCALL label, f u n c
2011-11-29 15:24:10 +04:00
ALIGN
GLOBAL( \ l a b e l )
2005-04-17 02:20:36 +04:00
leaq \ f u n c ( % r i p ) ,% r a x
jmp i a32 _ p t r e g s _ c o m m o n
.endm
2006-01-12 00:41:59 +03:00
CFI_ S T A R T P R O C 3 2
2005-09-12 20:49:24 +04:00
2012-11-12 23:32:42 +04:00
PTREGSCALL s t u b32 _ r t _ s i g r e t u r n , s y s32 _ r t _ s i g r e t u r n
PTREGSCALL s t u b32 _ s i g r e t u r n , s y s32 _ s i g r e t u r n
PTREGSCALL s t u b32 _ e x e c v e , c o m p a t _ s y s _ e x e c v e
2014-12-13 03:57:33 +03:00
PTREGSCALL s t u b32 _ e x e c v e a t , c o m p a t _ s y s _ e x e c v e a t
2012-11-12 23:32:42 +04:00
PTREGSCALL s t u b32 _ f o r k , s y s _ f o r k
PTREGSCALL s t u b32 _ v f o r k , s y s _ v f o r k
2005-04-17 02:20:36 +04:00
2012-10-23 06:34:11 +04:00
ALIGN
GLOBAL( s t u b32 _ c l o n e )
leaq s y s _ c l o n e ( % r i p ) ,% r a x
mov % r8 , % r c x
jmp i a32 _ p t r e g s _ c o m m o n
2011-11-29 15:24:10 +04:00
ALIGN
ia32_ptregs_common :
2006-01-12 00:41:59 +03:00
CFI_ E N D P R O C
CFI_ S T A R T P R O C 3 2 s i m p l e
2006-09-26 12:52:41 +04:00
CFI_ S I G N A L _ F R A M E
2015-02-27 01:40:30 +03:00
CFI_ D E F _ C F A r s p ,S S + 8
CFI_ R E L _ O F F S E T r a x ,R A X
CFI_ R E L _ O F F S E T r c x ,R C X
CFI_ R E L _ O F F S E T r d x ,R D X
CFI_ R E L _ O F F S E T r s i ,R S I
CFI_ R E L _ O F F S E T r d i ,R D I
CFI_ R E L _ O F F S E T r i p ,R I P
/* CFI_REL_OFFSET cs,CS*/
/* CFI_REL_OFFSET rflags,EFLAGS*/
CFI_ R E L _ O F F S E T r s p ,R S P
/* CFI_REL_OFFSET ss,SS*/
x86/asm/entry/64: Always allocate a complete "struct pt_regs" on the kernel stack
The 64-bit entry code was using six stack slots less by not
saving/restoring registers which are callee-preserved according
to the C ABI, and was not allocating space for them.
Only when syscalls needed a complete "struct pt_regs" was
the complete area allocated and filled in.
As an additional twist, on interrupt entry a "slightly less
truncated pt_regs" trick is used, to make nested interrupt
stacks easier to unwind.
This proved to be a source of significant obfuscation and subtle
bugs. For example, 'stub_fork' had to pop the return address,
extend the struct, save registers, and push return address back.
Ugly. 'ia32_ptregs_common' pops return address and "returns" via
jmp insn, throwing a wrench into CPU return stack cache.
This patch changes the code to always allocate a complete
"struct pt_regs" on the kernel stack. The saving of registers
is still done lazily.
"Partial pt_regs" trick on interrupt stack is retained.
Macros which manipulate "struct pt_regs" on stack are reworked:
- ALLOC_PT_GPREGS_ON_STACK allocates the structure.
- SAVE_C_REGS saves to it those registers which are clobbered
by C code.
- SAVE_EXTRA_REGS saves to it all other registers.
- Corresponding RESTORE_* and REMOVE_PT_GPREGS_FROM_STACK macros
reverse it.
'ia32_ptregs_common', 'stub_fork' and friends lost their ugly dance
with the return pointer.
LOAD_ARGS32 in ia32entry.S now uses symbolic stack offsets
instead of magic numbers.
'error_entry' and 'save_paranoid' now use SAVE_C_REGS +
SAVE_EXTRA_REGS instead of having it open-coded yet again.
Patch was run-tested: 64-bit executables, 32-bit executables,
strace works.
Timing tests did not show measurable difference in 32-bit
and 64-bit syscalls.
Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Signed-off-by: Andy Lutomirski <luto@amacapital.net>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/1423778052-21038-2-git-send-email-dvlasenk@redhat.com
Link: http://lkml.kernel.org/r/b89763d354aa23e670b9bdf3a40ae320320a7c2e.1424989793.git.luto@amacapital.net
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-02-27 01:40:27 +03:00
SAVE_ E X T R A _ R E G S 8
2005-04-17 02:20:36 +04:00
call * % r a x
x86/asm/entry/64: Always allocate a complete "struct pt_regs" on the kernel stack
The 64-bit entry code was using six stack slots less by not
saving/restoring registers which are callee-preserved according
to the C ABI, and was not allocating space for them.
Only when syscalls needed a complete "struct pt_regs" was
the complete area allocated and filled in.
As an additional twist, on interrupt entry a "slightly less
truncated pt_regs" trick is used, to make nested interrupt
stacks easier to unwind.
This proved to be a source of significant obfuscation and subtle
bugs. For example, 'stub_fork' had to pop the return address,
extend the struct, save registers, and push return address back.
Ugly. 'ia32_ptregs_common' pops return address and "returns" via
jmp insn, throwing a wrench into CPU return stack cache.
This patch changes the code to always allocate a complete
"struct pt_regs" on the kernel stack. The saving of registers
is still done lazily.
"Partial pt_regs" trick on interrupt stack is retained.
Macros which manipulate "struct pt_regs" on stack are reworked:
- ALLOC_PT_GPREGS_ON_STACK allocates the structure.
- SAVE_C_REGS saves to it those registers which are clobbered
by C code.
- SAVE_EXTRA_REGS saves to it all other registers.
- Corresponding RESTORE_* and REMOVE_PT_GPREGS_FROM_STACK macros
reverse it.
'ia32_ptregs_common', 'stub_fork' and friends lost their ugly dance
with the return pointer.
LOAD_ARGS32 in ia32entry.S now uses symbolic stack offsets
instead of magic numbers.
'error_entry' and 'save_paranoid' now use SAVE_C_REGS +
SAVE_EXTRA_REGS instead of having it open-coded yet again.
Patch was run-tested: 64-bit executables, 32-bit executables,
strace works.
Timing tests did not show measurable difference in 32-bit
and 64-bit syscalls.
Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Signed-off-by: Andy Lutomirski <luto@amacapital.net>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/1423778052-21038-2-git-send-email-dvlasenk@redhat.com
Link: http://lkml.kernel.org/r/b89763d354aa23e670b9bdf3a40ae320320a7c2e.1424989793.git.luto@amacapital.net
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-02-27 01:40:27 +03:00
RESTORE_ E X T R A _ R E G S 8
ret
2005-04-17 02:20:36 +04:00
CFI_ E N D P R O C
2006-06-26 15:56:55 +04:00
END( i a32 _ p t r e g s _ c o m m o n )
