2012-03-05 11:49:26 +00:00
/*
* Based on arch / arm / include / asm / assembler . h
*
* Copyright ( C ) 1996 - 2000 Russell King
* Copyright ( C ) 2012 ARM Ltd .
*
* This program is free software ; you can redistribute it and / or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation .
*
* This program is distributed in the hope that it will be useful ,
* but WITHOUT ANY WARRANTY ; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE . See the
* GNU General Public License for more details .
*
* You should have received a copy of the GNU General Public License
* along with this program . If not , see < http : //www.gnu.org/licenses/>.
*/
# ifndef __ASSEMBLY__
# error "Only include this from assembly code"
# endif
2015-02-20 13:53:13 +00:00
# ifndef __ASM_ASSEMBLER_H
# define __ASM_ASSEMBLER_H
2012-03-05 11:49:26 +00:00
# include <asm/ptrace.h>
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-29 19:04:06 +01:00
# include <asm/thread_info.h>
2012-03-05 11:49:26 +00:00
/*
* Enable and disable interrupts .
*/
. macro disable_irq
msr daifset , # 2
. endm
. macro enable_irq
msr daifclr , # 2
. endm
/*
* Enable and disable debug exceptions .
*/
. macro disable_dbg
msr daifset , # 8
. endm
. macro enable_dbg
msr daifclr , # 8
. endm
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-29 19:04:06 +01:00
. macro disable_step_tsk , flgs , tmp
tbz \ flgs , # TIF_SINGLESTEP , 9990f
2012-03-05 11:49:26 +00:00
mrs \ tmp , mdscr_el1
bic \ tmp , \ tmp , # 1
msr mdscr_el1 , \ tmp
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-29 19:04:06 +01:00
isb // Synchronise with enable_dbg
9990 :
2012-03-05 11:49:26 +00:00
. endm
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-29 19:04:06 +01:00
. macro enable_step_tsk , flgs , tmp
tbz \ flgs , # TIF_SINGLESTEP , 9990f
disable_dbg
2012-03-05 11:49:26 +00:00
mrs \ tmp , mdscr_el1
orr \ tmp , \ tmp , # 1
msr mdscr_el1 , \ tmp
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-29 19:04:06 +01:00
9990 :
2012-03-05 11:49:26 +00:00
. endm
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-29 19:04:06 +01:00
/*
* Enable both debug exceptions and interrupts . This is likely to be
* faster than two daifclr operations , since writes to this register
* are self - synchronising .
*/
. macro enable_dbg_and_irq
msr daifclr , # ( 8 | 2 )
2012-03-05 11:49:26 +00:00
. endm
/*
* SMP data memory barrier
*/
. macro smp_dmb , opt
dmb \ opt
. endm
2016-01-01 15:02:12 +01:00
/*
* Emit an entry into the exception table
*/
. macro _asm_extable , from , to
. pushsection __ex_table , " a "
. align 3
. long ( \ from - . ) , ( \ to - . )
. popsection
. endm
2012-03-05 11:49:26 +00:00
# define USER(l, x...) \
9999 : x ; \
2016-01-01 15:02:12 +01:00
_asm_extable 9999 b , l
2012-03-05 11:49:26 +00:00
/*
* Register aliases .
*/
lr . req x30 // link register
2012-10-19 17:37:35 +01:00
/*
* Vector entry
*/
. macro ventry label
. align 7
b \ label
. endm
2013-10-11 14:52:15 +01:00
/*
* Select code when configured for BE .
*/
# ifdef CONFIG_CPU_BIG_ENDIAN
# define CPU_BE(code...) code
# else
# define CPU_BE(code...)
# endif
/*
* Select code when configured for LE .
*/
# ifdef CONFIG_CPU_BIG_ENDIAN
# define CPU_LE(code...)
# else
# define CPU_LE(code...) code
# endif
2013-10-11 14:52:13 +01:00
/*
* Define a macro that constructs a 64 - bit value by concatenating two
* 32 - bit registers . Note that on big endian systems the order of the
* registers is swapped .
*/
# ifndef CONFIG_CPU_BIG_ENDIAN
. macro regs_to_64 , rd , lbits , hbits
# else
. macro regs_to_64 , rd , hbits , lbits
# endif
orr \ rd , \ lbits , \ hbits , lsl # 32
. endm
2015-02-20 13:53:13 +00:00
2015-03-04 19:45:38 +01:00
/*
* Pseudo - ops for PC - relative adr / ldr / str < reg > , < symbol > where
* < symbol > is within the range + / - 4 GB of the PC .
*/
/*
* @ dst : destination register ( 64 bit wide )
* @ sym : name of the symbol
* @ tmp : optional scratch register to be used if < dst > = = sp , which
* is not allowed in an adrp instruction
*/
. macro adr_l , dst , sym , tmp =
. ifb \ tmp
adrp \ dst , \ sym
add \ dst , \ dst , : lo12 : \ sym
. else
adrp \ tmp , \ sym
add \ dst , \ tmp , : lo12 : \ sym
. endif
. endm
/*
* @ dst : destination register ( 32 or 64 bit wide )
* @ sym : name of the symbol
* @ tmp : optional 64 - bit scratch register to be used if < dst > is a
* 32 - bit wide register , in which case it cannot be used to hold
* the address
*/
. macro ldr_l , dst , sym , tmp =
. ifb \ tmp
adrp \ dst , \ sym
ldr \ dst , [ \ dst , : lo12 : \ sym ]
. else
adrp \ tmp , \ sym
ldr \ dst , [ \ tmp , : lo12 : \ sym ]
. endif
. endm
/*
* @ src : source register ( 32 or 64 bit wide )
* @ sym : name of the symbol
* @ tmp : mandatory 64 - bit scratch register to calculate the address
* while < src > needs to be preserved .
*/
. macro str_l , src , sym , tmp
adrp \ tmp , \ sym
str \ src , [ \ tmp , : lo12 : \ sym ]
. endm
2015-12-10 10:22:39 +00:00
/*
* @ sym : The name of the per - cpu variable
* @ reg : Result of per_cpu ( sym , smp_processor_id ( ) )
* @ tmp : scratch register
*/
. macro this_cpu_ptr , sym , reg , tmp
adr_l \ reg , \ sym
mrs \ tmp , tpidr_el1
add \ reg , \ reg , \ tmp
. endm
2015-10-08 20:02:03 +01:00
/*
* Annotate a function as position independent , i . e . , safe to be called before
* the kernel virtual mapping is activated .
*/
# define ENDPIPROC(x) \
. globl __pi_ # # x ; \
. type __pi_ # # x , % function ; \
. set __pi_ # # x , x ; \
. size __pi_ # # x , . - x ; \
ENDPROC ( x )
2015-12-26 13:48:02 +01:00
/*
* Emit a 64 - bit absolute little endian symbol reference in a way that
* ensures that it will be resolved at build time , even when building a
* PIE binary . This requires cooperation from the linker script , which
* must emit the lo32 / hi32 halves individually .
*/
. macro le64sym , sym
. long \ sym \ ( ) _lo32
. long \ sym \ ( ) _hi32
. endm
2016-04-18 17:09:44 +02:00
/*
* mov_q - move an immediate constant into a 64 - bit register using
* between 2 and 4 movz / movk instructions ( depending on the
* magnitude and sign of the operand )
*/
. macro mov_q , reg , val
. if ( ( ( \ val ) > > 31 ) = = 0 | | ( ( \ val ) > > 31 ) = = 0x1ffffffff )
movz \ reg , : abs_g1_s : \ val
. else
. if ( ( ( \ val ) > > 47 ) = = 0 | | ( ( \ val ) > > 47 ) = = 0x1ffff )
movz \ reg , : abs_g2_s : \ val
. else
movz \ reg , : abs_g3 : \ val
movk \ reg , : abs_g2_nc : \ val
. endif
movk \ reg , : abs_g1_nc : \ val
. endif
movk \ reg , : abs_g0_nc : \ val
. endm
2015-02-20 13:53:13 +00:00
# endif /* __ASM_ASSEMBLER_H */
