iv/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

powerpc/64: Implement soft interrupt replay in C

Browse Source 
When local_irq_enable() finds a pending soft-masked interrupt, it
"replays" it by setting up registers like the initial interrupt entry,
then calls into the low level handler to set up an interrupt stack
frame and process the interrupt.

This is not necessary, and uses more stack than needed. The high level
interrupt handler can be called directly from C, with just pt_regs set
up on stack. This should be faster and use less stack.

Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://lore.kernel.org/r/20200225173541.1549955-28-npiggin@gmail.com
This commit is contained in:
Nicholas Piggin 2020-02-26 03:35:36 +10:00 committed by Michael Ellerman
parent 993c670a4d
commit 3282a3da25
4 changed files with 130 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
arch/powerpc/include/asm/hw_irq.h
View File

@ -52,7 +52,6 @@
#ifndef __ASSEMBLY__
extern void replay_system_reset(void);
extern void __replay_interrupt(unsigned int vector);
extern void timer_interrupt(struct pt_regs *);
extern void timer_broadcast_interrupt(void);

32
arch/powerpc/kernel/exceptions-64e.S
View File

@ -1002,38 +1002,6 @@ masked_interrupt_book3e_0x280:
masked_interrupt_book3e_0x2c0:
masked_interrupt_book3e PACA_IRQ_DBELL 0
/*
* Called from arch_local_irq_enable when an interrupt needs
* to be resent. r3 contains either 0x500,0x900,0x260 or 0x280
* to indicate the kind of interrupt. MSR:EE is already off.
* We generate a stackframe like if a real interrupt had happened.
*
* Note: While MSR:EE is off, we need to make sure that _MSR
* in the generated frame has EE set to 1 or the exception
* handler will not properly re-enable them.
*/
_GLOBAL(__replay_interrupt)
/* We are going to jump to the exception common code which
* will retrieve various register values from the PACA which
* we don't give a damn about.
*/
mflr r10
mfmsr r11
mfcr r4
mtspr SPRN_SPRG_GEN_SCRATCH,r13;
std r1,PACA_EXGEN+EX_R1(r13);
stw r4,PACA_EXGEN+EX_CR(r13);
ori r11,r11,MSR_EE
subi r1,r1,INT_FRAME_SIZE;
cmpwi cr0,r3,0x500
beq exc_0x500_common
cmpwi cr0,r3,0x900
beq exc_0x900_common
cmpwi cr0,r3,0x280
beq exc_0x280_common
blr
/*
* This is called from 0x300 and 0x400 handlers after the prologs with
* r14 and r15 containing the fault address and error code, with the

47
arch/powerpc/kernel/exceptions-64s.S
View File

@ -3165,50 +3165,3 @@ doorbell_super_common_msgclr:
LOAD_REG_IMMEDIATE(r3, PPC_DBELL_MSGTYPE << (63-36))
PPC_MSGCLRP(3)
b doorbell_super_common_virt
/*
* Called from arch_local_irq_enable when an interrupt needs
* to be resent. r3 contains 0x500, 0x900, 0xa00 or 0xe80 to indicate
* which kind of interrupt. MSR:EE is already off. We generate a
* stackframe like if a real interrupt had happened.
*
* Note: While MSR:EE is off, we need to make sure that _MSR
* in the generated frame has EE set to 1 or the exception
* handler will not properly re-enable them.
*
* Note that we don't specify LR as the NIP (return address) for
* the interrupt because that would unbalance the return branch
* predictor.
*/
_GLOBAL(__replay_interrupt)
/* We are going to jump to the exception common code which
* will retrieve various register values from the PACA which
* we don't give a damn about, so we don't bother storing them.
*/
mfmsr r12
LOAD_REG_ADDR(r11, replay_interrupt_return)
mfcr r9
ori r12,r12,MSR_EE
cmpwi r3,0x900
beq decrementer_common_virt
cmpwi r3,0x500
BEGIN_FTR_SECTION
beq h_virt_irq_common_virt
FTR_SECTION_ELSE
beq hardware_interrupt_common_virt
ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_300)
cmpwi r3,0xf00
beq performance_monitor_common_virt
BEGIN_FTR_SECTION
cmpwi r3,0xa00
beq h_doorbell_common_msgclr
cmpwi r3,0xe60
beq hmi_exception_common_virt
FTR_SECTION_ELSE
cmpwi r3,0xa00
beq doorbell_super_common_msgclr
ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE)
replay_interrupt_return:
blr
_ASM_NOKPROBE_SYMBOL(__replay_interrupt)

165
arch/powerpc/kernel/irq.c
View File

@ -70,6 +70,7 @@
#include <asm/paca.h>
#include <asm/firmware.h>
#include <asm/lv1call.h>
#include <asm/dbell.h>
#endif
#define CREATE_TRACE_POINTS
#include <asm/trace.h>
@ -230,10 +231,121 @@ notrace unsigned int __check_irq_replay(void)
return 0;
}
static void replay_soft_interrupts(void)
{
/*
* We use local_paca rather than get_paca() to avoid all
* the debug_smp_processor_id() business in this low level
* function
*/
unsigned char happened = local_paca->irq_happened;
struct pt_regs regs;
ppc_save_regs(&regs);
regs.softe = IRQS_ALL_DISABLED;
again:
if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
WARN_ON_ONCE(mfmsr() & MSR_EE);
if (happened & PACA_IRQ_HARD_DIS) {
/*
* We may have missed a decrementer interrupt if hard disabled.
* Check the decrementer register in case we had a rollover
* while hard disabled.
*/
if (!(happened & PACA_IRQ_DEC)) {
if (decrementer_check_overflow())
happened |= PACA_IRQ_DEC;
}
}
/*
* Force the delivery of pending soft-disabled interrupts on PS3.
* Any HV call will have this side effect.
*/
if (firmware_has_feature(FW_FEATURE_PS3_LV1)) {
u64 tmp, tmp2;
lv1_get_version_info(&tmp, &tmp2);
}
/*
* Check if an hypervisor Maintenance interrupt happened.
* This is a higher priority interrupt than the others, so
* replay it first.
*/
if (IS_ENABLED(CONFIG_PPC_BOOK3S) && (happened & PACA_IRQ_HMI)) {
local_paca->irq_happened &= ~PACA_IRQ_HMI;
regs.trap = 0xe60;
handle_hmi_exception(&regs);
if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS))
hard_irq_disable();
}
if (happened & PACA_IRQ_DEC) {
local_paca->irq_happened &= ~PACA_IRQ_DEC;
regs.trap = 0x900;
timer_interrupt(&regs);
if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS))
hard_irq_disable();
}
if (happened & PACA_IRQ_EE) {
local_paca->irq_happened &= ~PACA_IRQ_EE;
regs.trap = 0x500;
do_IRQ(&regs);
if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS))
hard_irq_disable();
}
/*
* Check if an EPR external interrupt happened this bit is typically
* set if we need to handle another "edge" interrupt from within the
* MPIC "EPR" handler.
*/
if (IS_ENABLED(CONFIG_PPC_BOOK3E) && (happened & PACA_IRQ_EE_EDGE)) {
local_paca->irq_happened &= ~PACA_IRQ_EE_EDGE;
regs.trap = 0x500;
do_IRQ(&regs);
if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS))
hard_irq_disable();
}
if (IS_ENABLED(CONFIG_PPC_DOORBELL) && (happened & PACA_IRQ_DBELL)) {
local_paca->irq_happened &= ~PACA_IRQ_DBELL;
if (IS_ENABLED(CONFIG_PPC_BOOK3E))
regs.trap = 0x280;
else
regs.trap = 0xa00;
doorbell_exception(&regs);
if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS))
hard_irq_disable();
}
/* Book3E does not support soft-masking PMI interrupts */
if (IS_ENABLED(CONFIG_PPC_BOOK3S) && (happened & PACA_IRQ_PMI)) {
local_paca->irq_happened &= ~PACA_IRQ_PMI;
regs.trap = 0xf00;
performance_monitor_exception(&regs);
if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS))
hard_irq_disable();
}
happened = local_paca->irq_happened;
if (happened & ~PACA_IRQ_HARD_DIS) {
/*
* We are responding to the next interrupt, so interrupt-off
* latencies should be reset here.
*/
trace_hardirqs_on();
trace_hardirqs_off();
goto again;
}
}
notrace void arch_local_irq_restore(unsigned long mask)
{
unsigned char irq_happened;
unsigned int replay;
/* Write the new soft-enabled value */
irq_soft_mask_set(mask);
@ -255,24 +367,16 @@ notrace void arch_local_irq_restore(unsigned long mask)
*/
irq_happened = get_irq_happened();
if (!irq_happened) {
#ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG
WARN_ON_ONCE(!(mfmsr() & MSR_EE));
#endif
if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
WARN_ON_ONCE(!(mfmsr() & MSR_EE));
return;
}
/*
* We need to hard disable to get a trusted value from
* __check_irq_replay(). We also need to soft-disable
* again to avoid warnings in there due to the use of
* per-cpu variables.
*/
/* We need to hard disable to replay. */
if (!(irq_happened & PACA_IRQ_HARD_DIS)) {
#ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG
WARN_ON_ONCE(!(mfmsr() & MSR_EE));
#endif
if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
WARN_ON_ONCE(!(mfmsr() & MSR_EE));
__hard_irq_disable();
#ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG
} else {
/*
* We should already be hard disabled here. We had bugs
@ -280,35 +384,26 @@ notrace void arch_local_irq_restore(unsigned long mask)
* warn if we are wrong. Only do that when IRQ tracing
* is enabled as mfmsr() can be costly.
*/
if (WARN_ON_ONCE(mfmsr() & MSR_EE))
__hard_irq_disable();
#endif
if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG)) {
if (WARN_ON_ONCE(mfmsr() & MSR_EE))
__hard_irq_disable();
}
if (irq_happened == PACA_IRQ_HARD_DIS) {
local_paca->irq_happened = 0;
__hard_irq_enable();
return;
}
}
irq_soft_mask_set(IRQS_ALL_DISABLED);
trace_hardirqs_off();
/*
* Check if anything needs to be re-emitted. We haven't
* soft-enabled yet to avoid warnings in decrementer_check_overflow
* accessing per-cpu variables
*/
replay = __check_irq_replay();
replay_soft_interrupts();
local_paca->irq_happened = 0;
/* We can soft-enable now */
trace_hardirqs_on();
irq_soft_mask_set(IRQS_ENABLED);
/*
* And replay if we have to. This will return with interrupts
* hard-enabled.
*/
if (replay) {
__replay_interrupt(replay);
return;
}
/* Finally, let's ensure we are hard enabled */
__hard_irq_enable();
}
EXPORT_SYMBOL(arch_local_irq_restore);