iv/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

x86/softirq: Remove indirection in do_softirq_own_stack()

Browse Source 
Use the new inline stack switching and remove the old ASM indirect call
implementation.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Kees Cook <keescook@chromium.org>
Link: https://lore.kernel.org/r/20210210002512.972714001@linutronix.de
This commit is contained in:
Thomas Gleixner 2021-02-10 00:40:50 +01:00
parent 359f01d181
commit 52d743f3b7
3 changed files with 17 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
arch/x86/entry/entry_64.S
View File

@ -756,45 +756,6 @@ SYM_CODE_START_LOCAL_NOALIGN(.Lbad_gs)
SYM_CODE_END(.Lbad_gs) SYM_CODE_END(.Lbad_gs)
.previous .previous
/*
* rdi: New stack pointer points to the top word of the stack
* rsi: Function pointer
* rdx: Function argument (can be NULL if none)
*/
SYM_FUNC_START(asm_call_on_stack)
/*
* Save the frame pointer unconditionally. This allows the ORC
* unwinder to handle the stack switch.
*/
pushq %rbp
mov %rsp, %rbp
/*
* The unwinder relies on the word at the top of the new stack
* page linking back to the previous RSP.
*/
mov %rsp, (%rdi)
mov %rdi, %rsp
/* Move the argument to the right place */
mov %rdx, %rdi
1:
.pushsection .discard.instr_begin
.long 1b - .
.popsection
CALL_NOSPEC rsi
2:
.pushsection .discard.instr_end
.long 2b - .
.popsection
/* Restore the previous stack pointer from RBP. */
leaveq
ret
SYM_FUNC_END(asm_call_on_stack)
#ifdef CONFIG_XEN_PV #ifdef CONFIG_XEN_PV
/* /*
* A note on the "critical region" in our callback handler. * A note on the "critical region" in our callback handler.

52
arch/x86/include/asm/irq_stack.h
View File

@ -185,20 +185,23 @@
IRQ_CONSTRAINTS, regs, vector); \ IRQ_CONSTRAINTS, regs, vector); \
} }
static __always_inline bool irqstack_active(void) #define ASM_CALL_SOFTIRQ \
{ "call %P[__func] \n"
return __this_cpu_read(hardirq_stack_inuse);
}
void asm_call_on_stack(void *sp, void (*func)(void), void *arg); /*
* Macro to invoke __do_softirq on the irq stack. Contrary to the above
static __always_inline void __run_on_irqstack(void (*func)(void)) * the only check which is necessary is whether the interrupt stack is
{ * in use already.
void *tos = __this_cpu_read(hardirq_stack_ptr); */
#define run_softirq_on_irqstack_cond() \
__this_cpu_write(hardirq_stack_inuse, true); { \
asm_call_on_stack(tos, func, NULL); if (__this_cpu_read(hardirq_stack_inuse)) { \
__this_cpu_write(hardirq_stack_inuse, false); __do_softirq(); \
} else { \
__this_cpu_write(hardirq_stack_inuse, true); \
call_on_irqstack(__do_softirq, ASM_CALL_SOFTIRQ); \
__this_cpu_write(hardirq_stack_inuse, false); \
} \
} }
#else /* CONFIG_X86_64 */ #else /* CONFIG_X86_64 */
@ -219,29 +222,6 @@ static __always_inline void __run_on_irqstack(void (*func)(void))
irq_exit_rcu(); \ irq_exit_rcu(); \
} }
static inline bool irqstack_active(void) { return false; }
static inline void __run_on_irqstack(void (*func)(void)) { }
#endif /* !CONFIG_X86_64 */ #endif /* !CONFIG_X86_64 */
static __always_inline bool irq_needs_irq_stack(struct pt_regs *regs)
{
if (IS_ENABLED(CONFIG_X86_32))
return false;
if (!regs)
return !irqstack_active();
return !user_mode(regs) && !irqstack_active();
}
static __always_inline void run_on_irqstack_cond(void (*func)(void),
struct pt_regs *regs)
{
lockdep_assert_irqs_disabled();
if (irq_needs_irq_stack(regs))
__run_on_irqstack(func);
else
func();
}
#endif #endif

2
arch/x86/kernel/irq_64.c
View File

@ -76,5 +76,5 @@ int irq_init_percpu_irqstack(unsigned int cpu)
void do_softirq_own_stack(void) void do_softirq_own_stack(void)
{ {
run_on_irqstack_cond(__do_softirq, NULL); run_softirq_on_irqstack_cond();
} }