2012-03-05 15:49:27 +04:00
/ *
* Low- l e v e l e x c e p t i o n h a n d l i n g c o d e
*
* Copyright ( C ) 2 0 1 2 A R M L t d .
* Authors : Catalin M a r i n a s < c a t a l i n . m a r i n a s @arm.com>
* Will D e a c o n < w i l l . d e a c o n @arm.com>
*
* This p r o g r a m i s f r e e s o f t w a r e ; you can redistribute it and/or modify
* it u n d e r t h e t e r m s o f t h e G N U G e n e r a l P u b l i c L i c e n s e v e r s i o n 2 a s
* published b y t h e F r e e S o f t w a r e F o u n d a t i o n .
*
* This p r o g r a m i s d i s t r i b u t e d i n t h e h o p e t h a t i t w i l l b e u s e f u l ,
* but W I T H O U T A N Y W A R R A N T Y ; without even the implied warranty of
* MERCHANTABILITY o r F I T N E S S F O R A P A R T I C U L A R P U R P O S E . S e e t h e
* GNU G e n e r a l P u b l i c L i c e n s e f o r m o r e d e t a i l s .
*
* You s h o u l d h a v e r e c e i v e d a c o p y o f t h e G N U G e n e r a l P u b l i c L i c e n s e
* along w i t h t h i s p r o g r a m . I f n o t , s e e < h t t p : / / w w w . g n u . o r g / l i c e n s e s / > .
* /
# include < l i n u x / i n i t . h >
# include < l i n u x / l i n k a g e . h >
# include < a s m / a s s e m b l e r . h >
# include < a s m / a s m - o f f s e t s . h >
# include < a s m / e r r n o . h >
2013-04-08 20:17:03 +04:00
# include < a s m / e s r . h >
2012-03-05 15:49:27 +04:00
# include < a s m / t h r e a d _ i n f o . h >
# include < a s m / u n i s t d . h >
2012-10-10 18:27:04 +04:00
# include < a s m / u n i s t d32 . h >
2012-03-05 15:49:27 +04:00
/ *
* Bad A b o r t n u m b e r s
* - - - - - - - - - - - - - - - - -
* /
# define B A D _ S Y N C 0
# define B A D _ I R Q 1
# define B A D _ F I Q 2
# define B A D _ E R R O R 3
.macro _ e n t r y , e l , r e g s i z e = 6 4
sub s p , s p , #S _ F R A M E _ S I Z E - S _ L R / / r o o m f o r L R , S P , S P S R , E L R
.if \ regsize = = 3 2
mov w0 , w0 / / z e r o u p p e r 3 2 b i t s o f x0
.endif
push x28 , x29
push x26 , x27
push x24 , x25
push x22 , x23
push x20 , x21
push x18 , x19
push x16 , x17
push x14 , x15
push x12 , x13
push x10 , x11
push x8 , x9
push x6 , x7
push x4 , x5
push x2 , x3
push x0 , x1
.if \ el = = 0
mrs x21 , s p _ e l 0
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-29 22:04:06 +04:00
get_ t h r e a d _ i n f o t s k / / E n s u r e M D S C R _ E L 1 . S S i s c l e a r ,
ldr x19 , [ t s k , #T I _ F L A G S ] / / s i n c e w e c a n u n m a s k d e b u g
disable_ s t e p _ t s k x19 , x20 / / e x c e p t i o n s w h e n s c h e d u l i n g .
2012-03-05 15:49:27 +04:00
.else
add x21 , s p , #S _ F R A M E _ S I Z E
.endif
mrs x22 , e l r _ e l 1
mrs x23 , s p s r _ e l 1
stp l r , x21 , [ s p , #S _ L R ]
stp x22 , x23 , [ s p , #S _ P C ]
/ *
* Set s y s c a l l n o t o - 1 b y d e f a u l t ( o v e r r i d d e n l a t e r i f r e a l s y s c a l l ) .
* /
.if \ el = = 0
mvn x21 , x z r
str x21 , [ s p , #S _ S Y S C A L L N O ]
.endif
/ *
* Registers t h a t m a y b e u s e f u l a f t e r t h i s m a c r o i s i n v o k e d :
*
* x2 1 - a b o r t e d S P
* x2 2 - a b o r t e d P C
* x2 3 - a b o r t e d P S T A T E
* /
.endm
.macro _ e x i t , e l , r e t = 0
ldp x21 , x22 , [ s p , #S _ P C ] / / l o a d E L R , S P S R
.if \ el = = 0
ldr x23 , [ s p , #S _ S P ] / / l o a d r e t u r n s t a c k p o i n t e r
.endif
.if \ ret
ldr x1 , [ s p , #S _ X 1 ] / / p r e s e r v e x0 ( s y s c a l l r e t u r n )
add s p , s p , S _ X 2
.else
pop x0 , x1
.endif
pop x2 , x3 / / l o a d t h e r e s t o f t h e r e g i s t e r s
pop x4 , x5
pop x6 , x7
pop x8 , x9
msr e l r _ e l 1 , x21 / / s e t u p t h e r e t u r n d a t a
msr s p s r _ e l 1 , x22
.if \ el = = 0
msr s p _ e l 0 , x23
.endif
pop x10 , x11
pop x12 , x13
pop x14 , x15
pop x16 , x17
pop x18 , x19
pop x20 , x21
pop x22 , x23
pop x24 , x25
pop x26 , x27
pop x28 , x29
ldr l r , [ s p ] , #S _ F R A M E _ S I Z E - S _ L R / / l o a d L R a n d r e s t o r e S P
eret / / r e t u r n t o k e r n e l
.endm
.macro _ t h r e a d _ i n f o , r d
mov \ r d , s p
2013-07-23 21:52:31 +04:00
and \ r d , \ r d , #~ ( T H R E A D _ S I Z E - 1 ) / / t o p o f s t a c k
2012-03-05 15:49:27 +04:00
.endm
/ *
* These a r e t h e r e g i s t e r s u s e d i n t h e s y s c a l l h a n d l e r , a n d a l l o w u s t o
* have i n t h e o r y u p t o 7 a r g u m e n t s t o a f u n c t i o n - x0 t o x6 .
*
* x7 i s r e s e r v e d f o r t h e s y s t e m c a l l n u m b e r i n 3 2 - b i t m o d e .
* /
sc_ n r . r e q x25 / / n u m b e r o f s y s t e m c a l l s
scno . r e q x26 / / s y s c a l l n u m b e r
stbl . r e q x27 / / s y s c a l l t a b l e p o i n t e r
tsk . r e q x28 / / c u r r e n t t h r e a d _ i n f o
/ *
* Interrupt h a n d l i n g .
* /
.macro irq_handler
ldr x1 , h a n d l e _ a r c h _ i r q
mov x0 , s p
blr x1
.endm
.text
/ *
* Exception v e c t o r s .
* /
.align 11
ENTRY( v e c t o r s )
ventry e l 1 _ s y n c _ i n v a l i d / / S y n c h r o n o u s E L 1 t
ventry e l 1 _ i r q _ i n v a l i d / / I R Q E L 1 t
ventry e l 1 _ f i q _ i n v a l i d / / F I Q E L 1 t
ventry e l 1 _ e r r o r _ i n v a l i d / / E r r o r E L 1 t
ventry e l 1 _ s y n c / / S y n c h r o n o u s E L 1 h
ventry e l 1 _ i r q / / I R Q E L 1 h
ventry e l 1 _ f i q _ i n v a l i d / / F I Q E L 1 h
ventry e l 1 _ e r r o r _ i n v a l i d / / E r r o r E L 1 h
ventry e l 0 _ s y n c / / S y n c h r o n o u s 6 4 - b i t E L 0
ventry e l 0 _ i r q / / I R Q 6 4 - b i t E L 0
ventry e l 0 _ f i q _ i n v a l i d / / F I Q 6 4 - b i t E L 0
ventry e l 0 _ e r r o r _ i n v a l i d / / E r r o r 6 4 - b i t E L 0
# ifdef C O N F I G _ C O M P A T
ventry e l 0 _ s y n c _ c o m p a t / / S y n c h r o n o u s 3 2 - b i t E L 0
ventry e l 0 _ i r q _ c o m p a t / / I R Q 3 2 - b i t E L 0
ventry e l 0 _ f i q _ i n v a l i d _ c o m p a t / / F I Q 3 2 - b i t E L 0
ventry e l 0 _ e r r o r _ i n v a l i d _ c o m p a t / / E r r o r 3 2 - b i t E L 0
# else
ventry e l 0 _ s y n c _ i n v a l i d / / S y n c h r o n o u s 3 2 - b i t E L 0
ventry e l 0 _ i r q _ i n v a l i d / / I R Q 3 2 - b i t E L 0
ventry e l 0 _ f i q _ i n v a l i d / / F I Q 3 2 - b i t E L 0
ventry e l 0 _ e r r o r _ i n v a l i d / / E r r o r 3 2 - b i t E L 0
# endif
END( v e c t o r s )
/ *
* Invalid m o d e h a n d l e r s
* /
.macro _ e n t r y , e l , r e a s o n , r e g s i z e = 6 4
kernel_ e n t r y e l , \ r e g s i z e
mov x0 , s p
mov x1 , #\ r e a s o n
mrs x2 , e s r _ e l 1
b b a d _ m o d e
.endm
el0_sync_invalid :
inv_ e n t r y 0 , B A D _ S Y N C
ENDPROC( e l 0 _ s y n c _ i n v a l i d )
el0_irq_invalid :
inv_ e n t r y 0 , B A D _ I R Q
ENDPROC( e l 0 _ i r q _ i n v a l i d )
el0_fiq_invalid :
inv_ e n t r y 0 , B A D _ F I Q
ENDPROC( e l 0 _ f i q _ i n v a l i d )
el0_error_invalid :
inv_ e n t r y 0 , B A D _ E R R O R
ENDPROC( e l 0 _ e r r o r _ i n v a l i d )
# ifdef C O N F I G _ C O M P A T
el0_fiq_invalid_compat :
inv_ e n t r y 0 , B A D _ F I Q , 3 2
ENDPROC( e l 0 _ f i q _ i n v a l i d _ c o m p a t )
el0_error_invalid_compat :
inv_ e n t r y 0 , B A D _ E R R O R , 3 2
ENDPROC( e l 0 _ e r r o r _ i n v a l i d _ c o m p a t )
# endif
el1_sync_invalid :
inv_ e n t r y 1 , B A D _ S Y N C
ENDPROC( e l 1 _ s y n c _ i n v a l i d )
el1_irq_invalid :
inv_ e n t r y 1 , B A D _ I R Q
ENDPROC( e l 1 _ i r q _ i n v a l i d )
el1_fiq_invalid :
inv_ e n t r y 1 , B A D _ F I Q
ENDPROC( e l 1 _ f i q _ i n v a l i d )
el1_error_invalid :
inv_ e n t r y 1 , B A D _ E R R O R
ENDPROC( e l 1 _ e r r o r _ i n v a l i d )
/ *
* EL1 m o d e h a n d l e r s .
* /
.align 6
el1_sync :
kernel_ e n t r y 1
mrs x1 , e s r _ e l 1 / / r e a d t h e s y n d r o m e r e g i s t e r
2013-04-08 20:17:03 +04:00
lsr x24 , x1 , #E S R _ E L 1 _ E C _ S H I F T / / e x c e p t i o n c l a s s
cmp x24 , #E S R _ E L 1 _ E C _ D A B T _ E L 1 / / d a t a a b o r t i n E L 1
2012-03-05 15:49:27 +04:00
b. e q e l 1 _ d a
2013-04-08 20:17:03 +04:00
cmp x24 , #E S R _ E L 1 _ E C _ S Y S 6 4 / / c o n f i g u r a b l e t r a p
2012-03-05 15:49:27 +04:00
b. e q e l 1 _ u n d e f
2013-04-08 20:17:03 +04:00
cmp x24 , #E S R _ E L 1 _ E C _ S P _ A L I G N / / s t a c k a l i g n m e n t e x c e p t i o n
2012-03-05 15:49:27 +04:00
b. e q e l 1 _ s p _ p c
2013-04-08 20:17:03 +04:00
cmp x24 , #E S R _ E L 1 _ E C _ P C _ A L I G N / / p c a l i g n m e n t e x c e p t i o n
2012-03-05 15:49:27 +04:00
b. e q e l 1 _ s p _ p c
2013-04-08 20:17:03 +04:00
cmp x24 , #E S R _ E L 1 _ E C _ U N K N O W N / / u n k n o w n e x c e p t i o n i n E L 1
2012-03-05 15:49:27 +04:00
b. e q e l 1 _ u n d e f
2013-04-08 20:17:03 +04:00
cmp x24 , #E S R _ E L 1 _ E C _ B R E A K P T _ E L 1 / / d e b u g e x c e p t i o n i n E L 1
2012-03-05 15:49:27 +04:00
b. g e e l 1 _ d b g
b e l 1 _ i n v
el1_da :
/ *
* Data a b o r t h a n d l i n g
* /
mrs x0 , f a r _ e l 1
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-29 22:04:06 +04:00
enable_ d b g
2012-03-05 15:49:27 +04:00
/ / re- e n a b l e i n t e r r u p t s i f t h e y w e r e e n a b l e d i n t h e a b o r t e d c o n t e x t
tbnz x23 , #7 , 1 f / / P S R _ I _ B I T
enable_ i r q
1 :
mov x2 , s p / / s t r u c t p t _ r e g s
bl d o _ m e m _ a b o r t
/ / disable i n t e r r u p t s b e f o r e p u l l i n g p r e s e r v e d d a t a o f f t h e s t a c k
disable_ i r q
kernel_ e x i t 1
el1_sp_pc :
/ *
* Stack o r P C a l i g n m e n t e x c e p t i o n h a n d l i n g
* /
mrs x0 , f a r _ e l 1
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-29 22:04:06 +04:00
enable_ d b g
2012-03-05 15:49:27 +04:00
mov x2 , s p
b d o _ s p _ p c _ a b o r t
el1_undef :
/ *
* Undefined i n s t r u c t i o n
* /
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-29 22:04:06 +04:00
enable_ d b g
2012-03-05 15:49:27 +04:00
mov x0 , s p
b d o _ u n d e f i n s t r
el1_dbg :
/ *
* Debug e x c e p t i o n h a n d l i n g
* /
2013-12-04 09:50:20 +04:00
cmp x24 , #E S R _ E L 1 _ E C _ B R K 6 4 / / i f B R K 6 4
cinc x24 , x24 , e q / / s e t b i t ' 0 '
2012-03-05 15:49:27 +04:00
tbz x24 , #0 , e l 1 _ i n v / / E L 1 o n l y
mrs x0 , f a r _ e l 1
mov x2 , s p / / s t r u c t p t _ r e g s
bl d o _ d e b u g _ e x c e p t i o n
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-29 22:04:06 +04:00
enable_ d b g
2012-03-05 15:49:27 +04:00
kernel_ e x i t 1
el1_inv :
/ / TODO : add s u p p o r t f o r u n d e f i n e d i n s t r u c t i o n s i n k e r n e l m o d e
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-29 22:04:06 +04:00
enable_ d b g
2012-03-05 15:49:27 +04:00
mov x0 , s p
mov x1 , #B A D _ S Y N C
mrs x2 , e s r _ e l 1
b b a d _ m o d e
ENDPROC( e l 1 _ s y n c )
.align 6
el1_irq :
kernel_ e n t r y 1
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-29 22:04:06 +04:00
enable_ d b g
2012-03-05 15:49:27 +04:00
# ifdef C O N F I G _ T R A C E _ I R Q F L A G S
bl t r a c e _ h a r d i r q s _ o f f
# endif
2013-11-12 21:11:53 +04:00
2012-03-05 15:49:27 +04:00
irq_ h a n d l e r
2013-11-12 21:11:53 +04:00
2012-03-05 15:49:27 +04:00
# ifdef C O N F I G _ P R E E M P T
2013-11-12 21:11:53 +04:00
get_ t h r e a d _ i n f o t s k
2014-01-13 12:57:56 +04:00
ldr w24 , [ t s k , #T I _ P R E E M P T ] / / g e t p r e e m p t c o u n t
2013-11-05 00:14:58 +04:00
cbnz w24 , 1 f / / p r e e m p t c o u n t ! = 0
2012-03-05 15:49:27 +04:00
ldr x0 , [ t s k , #T I _ F L A G S ] / / g e t f l a g s
tbz x0 , #T I F _ N E E D _ R E S C H E D , 1 f / / n e e d s r e s c h e d u l i n g ?
bl e l 1 _ p r e e m p t
1 :
# endif
# ifdef C O N F I G _ T R A C E _ I R Q F L A G S
bl t r a c e _ h a r d i r q s _ o n
# endif
kernel_ e x i t 1
ENDPROC( e l 1 _ i r q )
# ifdef C O N F I G _ P R E E M P T
el1_preempt :
mov x24 , l r
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-29 22:04:06 +04:00
1 : bl p r e e m p t _ s c h e d u l e _ i r q / / i r q e n / d i s a b l e i s d o n e i n s i d e
2012-03-05 15:49:27 +04:00
ldr x0 , [ t s k , #T I _ F L A G S ] / / g e t n e w t a s k s T I _ F L A G S
tbnz x0 , #T I F _ N E E D _ R E S C H E D , 1 b / / n e e d s r e s c h e d u l i n g ?
ret x24
# endif
/ *
* EL0 m o d e h a n d l e r s .
* /
.align 6
el0_sync :
kernel_ e n t r y 0
mrs x25 , e s r _ e l 1 / / r e a d t h e s y n d r o m e r e g i s t e r
2013-04-08 20:17:03 +04:00
lsr x24 , x25 , #E S R _ E L 1 _ E C _ S H I F T / / e x c e p t i o n c l a s s
cmp x24 , #E S R _ E L 1 _ E C _ S V C 6 4 / / S V C i n 6 4 - b i t s t a t e
2012-03-05 15:49:27 +04:00
b. e q e l 0 _ s v c
2013-04-08 20:17:03 +04:00
cmp x24 , #E S R _ E L 1 _ E C _ D A B T _ E L 0 / / d a t a a b o r t i n E L 0
2012-03-05 15:49:27 +04:00
b. e q e l 0 _ d a
2013-04-08 20:17:03 +04:00
cmp x24 , #E S R _ E L 1 _ E C _ I A B T _ E L 0 / / i n s t r u c t i o n a b o r t i n E L 0
2012-03-05 15:49:27 +04:00
b. e q e l 0 _ i a
2013-04-08 20:17:03 +04:00
cmp x24 , #E S R _ E L 1 _ E C _ F P _ A S I M D / / F P / A S I M D a c c e s s
2012-03-05 15:49:27 +04:00
b. e q e l 0 _ f p s i m d _ a c c
2013-04-08 20:17:03 +04:00
cmp x24 , #E S R _ E L 1 _ E C _ F P _ E X C 6 4 / / F P / A S I M D e x c e p t i o n
2012-03-05 15:49:27 +04:00
b. e q e l 0 _ f p s i m d _ e x c
2013-04-08 20:17:03 +04:00
cmp x24 , #E S R _ E L 1 _ E C _ S Y S 6 4 / / c o n f i g u r a b l e t r a p
2012-03-05 15:49:27 +04:00
b. e q e l 0 _ u n d e f
2013-04-08 20:17:03 +04:00
cmp x24 , #E S R _ E L 1 _ E C _ S P _ A L I G N / / s t a c k a l i g n m e n t e x c e p t i o n
2012-03-05 15:49:27 +04:00
b. e q e l 0 _ s p _ p c
2013-04-08 20:17:03 +04:00
cmp x24 , #E S R _ E L 1 _ E C _ P C _ A L I G N / / p c a l i g n m e n t e x c e p t i o n
2012-03-05 15:49:27 +04:00
b. e q e l 0 _ s p _ p c
2013-04-08 20:17:03 +04:00
cmp x24 , #E S R _ E L 1 _ E C _ U N K N O W N / / u n k n o w n e x c e p t i o n i n E L 0
2012-03-05 15:49:27 +04:00
b. e q e l 0 _ u n d e f
2013-04-08 20:17:03 +04:00
cmp x24 , #E S R _ E L 1 _ E C _ B R E A K P T _ E L 0 / / d e b u g e x c e p t i o n i n E L 0
2012-03-05 15:49:27 +04:00
b. g e e l 0 _ d b g
b e l 0 _ i n v
# ifdef C O N F I G _ C O M P A T
.align 6
el0_sync_compat :
kernel_ e n t r y 0 , 3 2
mrs x25 , e s r _ e l 1 / / r e a d t h e s y n d r o m e r e g i s t e r
2013-04-08 20:17:03 +04:00
lsr x24 , x25 , #E S R _ E L 1 _ E C _ S H I F T / / e x c e p t i o n c l a s s
cmp x24 , #E S R _ E L 1 _ E C _ S V C 3 2 / / S V C i n 3 2 - b i t s t a t e
2012-03-05 15:49:27 +04:00
b. e q e l 0 _ s v c _ c o m p a t
2013-04-08 20:17:03 +04:00
cmp x24 , #E S R _ E L 1 _ E C _ D A B T _ E L 0 / / d a t a a b o r t i n E L 0
2012-03-05 15:49:27 +04:00
b. e q e l 0 _ d a
2013-04-08 20:17:03 +04:00
cmp x24 , #E S R _ E L 1 _ E C _ I A B T _ E L 0 / / i n s t r u c t i o n a b o r t i n E L 0
2012-03-05 15:49:27 +04:00
b. e q e l 0 _ i a
2013-04-08 20:17:03 +04:00
cmp x24 , #E S R _ E L 1 _ E C _ F P _ A S I M D / / F P / A S I M D a c c e s s
2012-03-05 15:49:27 +04:00
b. e q e l 0 _ f p s i m d _ a c c
2013-04-08 20:17:03 +04:00
cmp x24 , #E S R _ E L 1 _ E C _ F P _ E X C 3 2 / / F P / A S I M D e x c e p t i o n
2012-03-05 15:49:27 +04:00
b. e q e l 0 _ f p s i m d _ e x c
2013-04-08 20:17:03 +04:00
cmp x24 , #E S R _ E L 1 _ E C _ U N K N O W N / / u n k n o w n e x c e p t i o n i n E L 0
2012-03-05 15:49:27 +04:00
b. e q e l 0 _ u n d e f
2013-05-24 15:02:35 +04:00
cmp x24 , #E S R _ E L 1 _ E C _ C P 1 5 _ 3 2 / / C P 1 5 M R C / M C R t r a p
b. e q e l 0 _ u n d e f
cmp x24 , #E S R _ E L 1 _ E C _ C P 1 5 _ 6 4 / / C P 1 5 M R R C / M C R R t r a p
b. e q e l 0 _ u n d e f
cmp x24 , #E S R _ E L 1 _ E C _ C P 1 4 _ M R / / C P 1 4 M R C / M C R t r a p
b. e q e l 0 _ u n d e f
cmp x24 , #E S R _ E L 1 _ E C _ C P 1 4 _ L S / / C P 1 4 L D C / S T C t r a p
b. e q e l 0 _ u n d e f
cmp x24 , #E S R _ E L 1 _ E C _ C P 1 4 _ 6 4 / / C P 1 4 M R R C / M C R R t r a p
b. e q e l 0 _ u n d e f
2013-04-08 20:17:03 +04:00
cmp x24 , #E S R _ E L 1 _ E C _ B R E A K P T _ E L 0 / / d e b u g e x c e p t i o n i n E L 0
2012-03-05 15:49:27 +04:00
b. g e e l 0 _ d b g
b e l 0 _ i n v
el0_svc_compat :
/ *
* AArch3 2 s y s c a l l h a n d l i n g
* /
adr s t b l , c o m p a t _ s y s _ c a l l _ t a b l e / / l o a d c o m p a t s y s c a l l t a b l e p o i n t e r
uxtw s c n o , w7 / / s y s c a l l n u m b e r i n w7 ( r7 )
mov s c _ n r , #_ _ N R _ c o m p a t _ s y s c a l l s
b e l 0 _ s v c _ n a k e d
.align 6
el0_irq_compat :
kernel_ e n t r y 0 , 3 2
b e l 0 _ i r q _ n a k e d
# endif
el0_da :
/ *
* Data a b o r t h a n d l i n g
* /
2014-05-30 23:34:14 +04:00
mrs x26 , f a r _ e l 1
2012-03-05 15:49:27 +04:00
/ / enable i n t e r r u p t s b e f o r e c a l l i n g t h e m a i n h a n d l e r
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-29 22:04:06 +04:00
enable_ d b g _ a n d _ i r q
2014-05-30 23:34:14 +04:00
bic x0 , x26 , #( 0xff < < 5 6 )
2012-03-05 15:49:27 +04:00
mov x1 , x25
mov x2 , s p
2014-05-30 23:34:14 +04:00
adr l r , r e t _ t o _ u s e r
2012-03-05 15:49:27 +04:00
b d o _ m e m _ a b o r t
el0_ia :
/ *
* Instruction a b o r t h a n d l i n g
* /
2014-05-30 23:34:14 +04:00
mrs x26 , f a r _ e l 1
2012-03-05 15:49:27 +04:00
/ / enable i n t e r r u p t s b e f o r e c a l l i n g t h e m a i n h a n d l e r
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-29 22:04:06 +04:00
enable_ d b g _ a n d _ i r q
2014-05-30 23:34:14 +04:00
mov x0 , x26
2012-03-05 15:49:27 +04:00
orr x1 , x25 , #1 < < 2 4 / / u s e r e s e r v e d I S S b i t f o r i n s t r u c t i o n a b o r t s
mov x2 , s p
2014-05-30 23:34:14 +04:00
adr l r , r e t _ t o _ u s e r
2012-03-05 15:49:27 +04:00
b d o _ m e m _ a b o r t
el0_fpsimd_acc :
/ *
* Floating P o i n t o r A d v a n c e d S I M D a c c e s s
* /
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-29 22:04:06 +04:00
enable_ d b g
2012-03-05 15:49:27 +04:00
mov x0 , x25
mov x1 , s p
2014-05-30 23:34:14 +04:00
adr l r , r e t _ t o _ u s e r
2012-03-05 15:49:27 +04:00
b d o _ f p s i m d _ a c c
el0_fpsimd_exc :
/ *
* Floating P o i n t o r A d v a n c e d S I M D e x c e p t i o n
* /
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-29 22:04:06 +04:00
enable_ d b g
2012-03-05 15:49:27 +04:00
mov x0 , x25
mov x1 , s p
2014-05-30 23:34:14 +04:00
adr l r , r e t _ t o _ u s e r
2012-03-05 15:49:27 +04:00
b d o _ f p s i m d _ e x c
el0_sp_pc :
/ *
* Stack o r P C a l i g n m e n t e x c e p t i o n h a n d l i n g
* /
2014-05-30 23:34:14 +04:00
mrs x26 , f a r _ e l 1
2012-03-05 15:49:27 +04:00
/ / enable i n t e r r u p t s b e f o r e c a l l i n g t h e m a i n h a n d l e r
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-29 22:04:06 +04:00
enable_ d b g _ a n d _ i r q
2014-05-30 23:34:14 +04:00
mov x0 , x26
2012-03-05 15:49:27 +04:00
mov x1 , x25
mov x2 , s p
2014-05-30 23:34:14 +04:00
adr l r , r e t _ t o _ u s e r
2012-03-05 15:49:27 +04:00
b d o _ s p _ p c _ a b o r t
el0_undef :
/ *
* Undefined i n s t r u c t i o n
* /
2013-08-22 14:47:37 +04:00
/ / enable i n t e r r u p t s b e f o r e c a l l i n g t h e m a i n h a n d l e r
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-29 22:04:06 +04:00
enable_ d b g _ a n d _ i r q
mov x0 , s p
2014-05-30 23:34:14 +04:00
adr l r , r e t _ t o _ u s e r
2012-03-05 15:49:27 +04:00
b d o _ u n d e f i n s t r
el0_dbg :
/ *
* Debug e x c e p t i o n h a n d l i n g
* /
tbnz x24 , #0 , e l 0 _ i n v / / E L 0 o n l y
mrs x0 , f a r _ e l 1
mov x1 , x25
mov x2 , s p
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-29 22:04:06 +04:00
bl d o _ d e b u g _ e x c e p t i o n
enable_ d b g
b r e t _ t o _ u s e r
2012-03-05 15:49:27 +04:00
el0_inv :
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-29 22:04:06 +04:00
enable_ d b g
2012-03-05 15:49:27 +04:00
mov x0 , s p
mov x1 , #B A D _ S Y N C
mrs x2 , e s r _ e l 1
2014-05-30 23:34:14 +04:00
adr l r , r e t _ t o _ u s e r
2012-03-05 15:49:27 +04:00
b b a d _ m o d e
ENDPROC( e l 0 _ s y n c )
.align 6
el0_irq :
kernel_ e n t r y 0
el0_irq_naked :
enable_ d b g
# ifdef C O N F I G _ T R A C E _ I R Q F L A G S
bl t r a c e _ h a r d i r q s _ o f f
# endif
2013-11-12 21:11:53 +04:00
2012-03-05 15:49:27 +04:00
irq_ h a n d l e r
2013-11-12 21:11:53 +04:00
2012-03-05 15:49:27 +04:00
# ifdef C O N F I G _ T R A C E _ I R Q F L A G S
bl t r a c e _ h a r d i r q s _ o n
# endif
b r e t _ t o _ u s e r
ENDPROC( e l 0 _ i r q )
/ *
* Register s w i t c h f o r A A r c h64 . T h e c a l l e e - s a v e d r e g i s t e r s n e e d t o b e s a v e d
* and r e s t o r e d . O n e n t r y :
* x0 = p r e v i o u s t a s k _ s t r u c t ( m u s t b e p r e s e r v e d a c r o s s t h e s w i t c h )
* x1 = n e x t t a s k _ s t r u c t
* Previous a n d n e x t a r e g u a r a n t e e d n o t t o b e t h e s a m e .
*
* /
ENTRY( c p u _ s w i t c h _ t o )
add x8 , x0 , #T H R E A D _ C P U _ C O N T E X T
mov x9 , s p
stp x19 , x20 , [ x8 ] , #16 / / s t o r e c a l l e e - s a v e d r e g i s t e r s
stp x21 , x22 , [ x8 ] , #16
stp x23 , x24 , [ x8 ] , #16
stp x25 , x26 , [ x8 ] , #16
stp x27 , x28 , [ x8 ] , #16
stp x29 , x9 , [ x8 ] , #16
str l r , [ x8 ]
add x8 , x1 , #T H R E A D _ C P U _ C O N T E X T
ldp x19 , x20 , [ x8 ] , #16 / / r e s t o r e c a l l e e - s a v e d r e g i s t e r s
ldp x21 , x22 , [ x8 ] , #16
ldp x23 , x24 , [ x8 ] , #16
ldp x25 , x26 , [ x8 ] , #16
ldp x27 , x28 , [ x8 ] , #16
ldp x29 , x9 , [ x8 ] , #16
ldr l r , [ x8 ]
mov s p , x9
ret
ENDPROC( c p u _ s w i t c h _ t o )
/ *
* This i s t h e f a s t s y s c a l l r e t u r n p a t h . W e d o a s l i t t l e a s p o s s i b l e h e r e ,
* and t h i s i n c l u d e s s a v i n g x0 b a c k i n t o t h e k e r n e l s t a c k .
* /
ret_fast_syscall :
disable_ i r q / / d i s a b l e i n t e r r u p t s
ldr x1 , [ t s k , #T I _ F L A G S ]
and x2 , x1 , #_ T I F _ W O R K _ M A S K
cbnz x2 , f a s t _ w o r k _ p e n d i n g
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-29 22:04:06 +04:00
enable_ s t e p _ t s k x1 , x2
2012-03-05 15:49:27 +04:00
kernel_ e x i t 0 , r e t = 1
/ *
* Ok, w e n e e d t o d o e x t r a p r o c e s s i n g , e n t e r t h e s l o w p a t h .
* /
fast_work_pending :
str x0 , [ s p , #S _ X 0 ] / / r e t u r n e d x0
work_pending :
tbnz x1 , #T I F _ N E E D _ R E S C H E D , w o r k _ r e s c h e d
2014-05-08 13:20:23 +04:00
/* TIF_SIGPENDING, TIF_NOTIFY_RESUME or TIF_FOREIGN_FPSTATE case */
2012-03-05 15:49:27 +04:00
ldr x2 , [ s p , #S _ P S T A T E ]
mov x0 , s p / / ' r e g s '
tst x2 , #P S R _ M O D E _ M A S K / / u s e r m o d e r e g s ?
b. n e n o _ w o r k _ p e n d i n g / / r e t u r n i n g t o k e r n e l
2012-10-08 21:04:21 +04:00
enable_ i r q / / e n a b l e i n t e r r u p t s f o r d o _ n o t i f y _ r e s u m e ( )
2012-03-05 15:49:27 +04:00
bl d o _ n o t i f y _ r e s u m e
b r e t _ t o _ u s e r
work_resched :
bl s c h e d u l e
/ *
* " slow" s y s c a l l r e t u r n p a t h .
* /
2012-09-10 19:11:46 +04:00
ret_to_user :
2012-03-05 15:49:27 +04:00
disable_ i r q / / d i s a b l e i n t e r r u p t s
ldr x1 , [ t s k , #T I _ F L A G S ]
and x2 , x1 , #_ T I F _ W O R K _ M A S K
cbnz x2 , w o r k _ p e n d i n g
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-29 22:04:06 +04:00
enable_ s t e p _ t s k x1 , x2
2012-03-05 15:49:27 +04:00
no_work_pending :
kernel_ e x i t 0 , r e t = 0
ENDPROC( r e t _ t o _ u s e r )
/ *
* This i s h o w w e r e t u r n f r o m a f o r k .
* /
ENTRY( r e t _ f r o m _ f o r k )
bl s c h e d u l e _ t a i l
2012-10-05 15:31:20 +04:00
cbz x19 , 1 f / / n o t a k e r n e l t h r e a d
mov x0 , x20
blr x19
1 : get_ t h r e a d _ i n f o t s k
2012-03-05 15:49:27 +04:00
b r e t _ t o _ u s e r
ENDPROC( r e t _ f r o m _ f o r k )
/ *
* SVC h a n d l e r .
* /
.align 6
el0_svc :
adrp s t b l , s y s _ c a l l _ t a b l e / / l o a d s y s c a l l t a b l e p o i n t e r
uxtw s c n o , w8 / / s y s c a l l n u m b e r i n w8
mov s c _ n r , #_ _ N R _ s y s c a l l s
el0_svc_naked : / / compat e n t r y p o i n t
stp x0 , s c n o , [ s p , #S _ O R I G _ X 0 ] / / s a v e t h e o r i g i n a l x0 a n d s y s c a l l n u m b e r
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-29 22:04:06 +04:00
enable_ d b g _ a n d _ i r q
2012-03-05 15:49:27 +04:00
2014-04-30 13:51:29 +04:00
ldr x16 , [ t s k , #T I _ F L A G S ] / / c h e c k f o r s y s c a l l h o o k s
tst x16 , #_ T I F _ S Y S C A L L _ W O R K
b. n e _ _ s y s _ t r a c e
2012-03-05 15:49:27 +04:00
adr l r , r e t _ f a s t _ s y s c a l l / / r e t u r n a d d r e s s
cmp s c n o , s c _ n r / / c h e c k u p p e r s y s c a l l l i m i t
b. h s n i _ s y s
ldr x16 , [ s t b l , s c n o , l s l #3 ] / / a d d r e s s i n t h e s y s c a l l t a b l e
br x16 / / c a l l s y s _ * r o u t i n e
ni_sys :
mov x0 , s p
b d o _ n i _ s y s c a l l
ENDPROC( e l 0 _ s v c )
/ *
* This i s t h e r e a l l y s l o w p a t h . W e ' r e g o i n g t o b e d o i n g c o n t e x t
* switches, a n d w a i t i n g f o r o u r p a r e n t t o r e s p o n d .
* /
__sys_trace :
2014-04-30 13:51:30 +04:00
mov x0 , s p
bl s y s c a l l _ t r a c e _ e n t e r
2012-03-05 15:49:27 +04:00
adr l r , _ _ s y s _ t r a c e _ r e t u r n / / r e t u r n a d d r e s s
uxtw s c n o , w0 / / s y s c a l l n u m b e r ( p o s s i b l y n e w )
mov x1 , s p / / p o i n t e r t o r e g s
cmp s c n o , s c _ n r / / c h e c k u p p e r s y s c a l l l i m i t
b. h s n i _ s y s
ldp x0 , x1 , [ s p ] / / r e s t o r e t h e s y s c a l l a r g s
ldp x2 , x3 , [ s p , #S _ X 2 ]
ldp x4 , x5 , [ s p , #S _ X 4 ]
ldp x6 , x7 , [ s p , #S _ X 6 ]
ldr x16 , [ s t b l , s c n o , l s l #3 ] / / a d d r e s s i n t h e s y s c a l l t a b l e
br x16 / / c a l l s y s _ * r o u t i n e
__sys_trace_return :
str x0 , [ s p ] / / s a v e r e t u r n e d x0
2014-04-30 13:51:30 +04:00
mov x0 , s p
bl s y s c a l l _ t r a c e _ e x i t
2012-03-05 15:49:27 +04:00
b r e t _ t o _ u s e r
/ *
* Special s y s t e m c a l l w r a p p e r s .
* /
ENTRY( s y s _ r t _ s i g r e t u r n _ w r a p p e r )
mov x0 , s p
b s y s _ r t _ s i g r e t u r n
ENDPROC( s y s _ r t _ s i g r e t u r n _ w r a p p e r )
ENTRY( h a n d l e _ a r c h _ i r q )
.quad 0
