Merge patch series "riscv: Add independent irq/softirq stacks support"
guoren@kernel.org <guoren@kernel.org> says: From: Guo Ren <guoren@linux.alibaba.com> This patch series adds independent irq/softirq stacks to decrease the press of the thread stack. Also, add a thread STACK_SIZE config for users to adjust the proper size during compile time. * b4-shazam-merge: riscv: stack: Add config of thread stack size riscv: stack: Support HAVE_SOFTIRQ_ON_OWN_STACK riscv: stack: Support HAVE_IRQ_EXIT_ON_IRQ_STACK Link: https://lore.kernel.org/r/20230614013018.2168426-1-guoren@kernel.org Signed-off-by: Palmer Dabbelt <palmer@rivosinc.com>
This commit is contained in:
Palmer Dabbelt
commit
b5e13f3ace
@ -591,6 +591,25 @@ config FPU
|
||||
|
||||
If you don't know what to do here, say Y.
|
||||
|
||||
config IRQ_STACKS
|
||||
bool "Independent irq & softirq stacks" if EXPERT
|
||||
default y
|
||||
select HAVE_IRQ_EXIT_ON_IRQ_STACK
|
||||
select HAVE_SOFTIRQ_ON_OWN_STACK
|
||||
help
|
||||
Add independent irq & softirq stacks for percpu to prevent kernel stack
|
||||
overflows. We may save some memory footprint by disabling IRQ_STACKS.
|
||||
|
||||
config THREAD_SIZE_ORDER
|
||||
int "Kernel stack size (in power-of-two numbers of page size)" if VMAP_STACK && EXPERT
|
||||
range 0 4
|
||||
default 1 if 32BIT && !KASAN
|
||||
default 3 if 64BIT && KASAN
|
||||
default 2
|
||||
help
|
||||
Specify the Pages of thread stack size (from 4KB to 64KB), which also
|
||||
affects irq stack size, which is equal to thread stack size.
|
||||
|
||||
endmenu # "Platform type"
|
||||
|
||||
menu "Kernel features"
|
||||
|
||||
30
arch/riscv/include/asm/irq_stack.h
Normal file
30
arch/riscv/include/asm/irq_stack.h
Normal file
@ -0,0 +1,30 @@
|
||||
/* SPDX-License-Identifier: GPL-2.0 */
|
||||
|
||||
#ifndef _ASM_RISCV_IRQ_STACK_H
|
||||
#define _ASM_RISCV_IRQ_STACK_H
|
||||
|
||||
#include <linux/bug.h>
|
||||
#include <linux/gfp.h>
|
||||
#include <linux/kconfig.h>
|
||||
#include <linux/vmalloc.h>
|
||||
#include <linux/pgtable.h>
|
||||
#include <asm/thread_info.h>
|
||||
|
||||
DECLARE_PER_CPU(ulong *, irq_stack_ptr);
|
||||
|
||||
#ifdef CONFIG_VMAP_STACK
|
||||
/*
|
||||
* To ensure that VMAP'd stack overflow detection works correctly, all VMAP'd
|
||||
* stacks need to have the same alignment.
|
||||
*/
|
||||
static inline unsigned long *arch_alloc_vmap_stack(size_t stack_size, int node)
|
||||
{
|
||||
void *p;
|
||||
|
||||
p = __vmalloc_node(stack_size, THREAD_ALIGN, THREADINFO_GFP, node,
|
||||
__builtin_return_address(0));
|
||||
return kasan_reset_tag(p);
|
||||
}
|
||||
#endif /* CONFIG_VMAP_STACK */
|
||||
|
||||
#endif /* _ASM_RISCV_IRQ_STACK_H */
|
||||
@ -11,18 +11,8 @@
|
||||
#include <asm/page.h>
|
||||
#include <linux/const.h>
|
||||
|
||||
#ifdef CONFIG_KASAN
|
||||
#define KASAN_STACK_ORDER 1
|
||||
#else
|
||||
#define KASAN_STACK_ORDER 0
|
||||
#endif
|
||||
|
||||
/* thread information allocation */
|
||||
#ifdef CONFIG_64BIT
|
||||
#define THREAD_SIZE_ORDER (2 + KASAN_STACK_ORDER)
|
||||
#else
|
||||
#define THREAD_SIZE_ORDER (1 + KASAN_STACK_ORDER)
|
||||
#endif
|
||||
#define THREAD_SIZE_ORDER CONFIG_THREAD_SIZE_ORDER
|
||||
#define THREAD_SIZE (PAGE_SIZE << THREAD_SIZE_ORDER)
|
||||
|
||||
/*
|
||||
@ -40,6 +30,8 @@
|
||||
#define OVERFLOW_STACK_SIZE SZ_4K
|
||||
#define SHADOW_OVERFLOW_STACK_SIZE (1024)
|
||||
|
||||
#define IRQ_STACK_SIZE THREAD_SIZE
|
||||
|
||||
#ifndef __ASSEMBLY__
|
||||
|
||||
extern long shadow_stack[SHADOW_OVERFLOW_STACK_SIZE / sizeof(long)];
|
||||
|
||||
@ -11,6 +11,9 @@
|
||||
#include <linux/module.h>
|
||||
#include <linux/seq_file.h>
|
||||
#include <asm/sbi.h>
|
||||
#include <asm/smp.h>
|
||||
#include <asm/softirq_stack.h>
|
||||
#include <asm/stacktrace.h>
|
||||
|
||||
static struct fwnode_handle *(*__get_intc_node)(void);
|
||||
|
||||
@ -28,6 +31,70 @@ struct fwnode_handle *riscv_get_intc_hwnode(void)
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(riscv_get_intc_hwnode);
|
||||
|
||||
#ifdef CONFIG_IRQ_STACKS
|
||||
#include <asm/irq_stack.h>
|
||||
|
||||
DEFINE_PER_CPU(ulong *, irq_stack_ptr);
|
||||
|
||||
#ifdef CONFIG_VMAP_STACK
|
||||
static void init_irq_stacks(void)
|
||||
{
|
||||
int cpu;
|
||||
ulong *p;
|
||||
|
||||
for_each_possible_cpu(cpu) {
|
||||
p = arch_alloc_vmap_stack(IRQ_STACK_SIZE, cpu_to_node(cpu));
|
||||
per_cpu(irq_stack_ptr, cpu) = p;
|
||||
}
|
||||
}
|
||||
#else
|
||||
/* irq stack only needs to be 16 byte aligned - not IRQ_STACK_SIZE aligned. */
|
||||
DEFINE_PER_CPU_ALIGNED(ulong [IRQ_STACK_SIZE/sizeof(ulong)], irq_stack);
|
||||
|
||||
static void init_irq_stacks(void)
|
||||
{
|
||||
int cpu;
|
||||
|
||||
for_each_possible_cpu(cpu)
|
||||
per_cpu(irq_stack_ptr, cpu) = per_cpu(irq_stack, cpu);
|
||||
}
|
||||
#endif /* CONFIG_VMAP_STACK */
|
||||
|
||||
#ifdef CONFIG_HAVE_SOFTIRQ_ON_OWN_STACK
|
||||
void do_softirq_own_stack(void)
|
||||
{
|
||||
#ifdef CONFIG_IRQ_STACKS
|
||||
if (on_thread_stack()) {
|
||||
ulong *sp = per_cpu(irq_stack_ptr, smp_processor_id())
|
||||
+ IRQ_STACK_SIZE/sizeof(ulong);
|
||||
__asm__ __volatile(
|
||||
"addi sp, sp, -"RISCV_SZPTR "\n"
|
||||
REG_S" ra, (sp) \n"
|
||||
"addi sp, sp, -"RISCV_SZPTR "\n"
|
||||
REG_S" s0, (sp) \n"
|
||||
"addi s0, sp, 2*"RISCV_SZPTR "\n"
|
||||
"move sp, %[sp] \n"
|
||||
"call __do_softirq \n"
|
||||
"addi sp, s0, -2*"RISCV_SZPTR"\n"
|
||||
REG_L" s0, (sp) \n"
|
||||
"addi sp, sp, "RISCV_SZPTR "\n"
|
||||
REG_L" ra, (sp) \n"
|
||||
"addi sp, sp, "RISCV_SZPTR "\n"
|
||||
:
|
||||
: [sp] "r" (sp)
|
||||
: "a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7",
|
||||
"t0", "t1", "t2", "t3", "t4", "t5", "t6",
|
||||
"memory");
|
||||
} else
|
||||
#endif
|
||||
__do_softirq();
|
||||
}
|
||||
#endif /* CONFIG_HAVE_SOFTIRQ_ON_OWN_STACK */
|
||||
|
||||
#else
|
||||
static void init_irq_stacks(void) {}
|
||||
#endif /* CONFIG_IRQ_STACKS */
|
||||
|
||||
int arch_show_interrupts(struct seq_file *p, int prec)
|
||||
{
|
||||
show_ipi_stats(p, prec);
|
||||
@ -36,6 +103,7 @@ int arch_show_interrupts(struct seq_file *p, int prec)
|
||||
|
||||
void __init init_IRQ(void)
|
||||
{
|
||||
init_irq_stacks();
|
||||
irqchip_init();
|
||||
if (!handle_arch_irq)
|
||||
panic("No interrupt controller found.");
|
||||
|
||||
@ -27,6 +27,7 @@
|
||||
#include <asm/syscall.h>
|
||||
#include <asm/thread_info.h>
|
||||
#include <asm/vector.h>
|
||||
#include <asm/irq_stack.h>
|
||||
|
||||
int show_unhandled_signals = 1;
|
||||
|
||||
@ -327,16 +328,46 @@ asmlinkage __visible noinstr void do_page_fault(struct pt_regs *regs)
|
||||
}
|
||||
#endif
|
||||
|
||||
asmlinkage __visible noinstr void do_irq(struct pt_regs *regs)
|
||||
static void noinstr handle_riscv_irq(struct pt_regs *regs)
|
||||
{
|
||||
struct pt_regs *old_regs;
|
||||
irqentry_state_t state = irqentry_enter(regs);
|
||||
|
||||
irq_enter_rcu();
|
||||
old_regs = set_irq_regs(regs);
|
||||
handle_arch_irq(regs);
|
||||
set_irq_regs(old_regs);
|
||||
irq_exit_rcu();
|
||||
}
|
||||
|
||||
asmlinkage void noinstr do_irq(struct pt_regs *regs)
|
||||
{
|
||||
irqentry_state_t state = irqentry_enter(regs);
|
||||
#ifdef CONFIG_IRQ_STACKS
|
||||
if (on_thread_stack()) {
|
||||
ulong *sp = per_cpu(irq_stack_ptr, smp_processor_id())
|
||||
+ IRQ_STACK_SIZE/sizeof(ulong);
|
||||
__asm__ __volatile(
|
||||
"addi sp, sp, -"RISCV_SZPTR "\n"
|
||||
REG_S" ra, (sp) \n"
|
||||
"addi sp, sp, -"RISCV_SZPTR "\n"
|
||||
REG_S" s0, (sp) \n"
|
||||
"addi s0, sp, 2*"RISCV_SZPTR "\n"
|
||||
"move sp, %[sp] \n"
|
||||
"move a0, %[regs] \n"
|
||||
"call handle_riscv_irq \n"
|
||||
"addi sp, s0, -2*"RISCV_SZPTR"\n"
|
||||
REG_L" s0, (sp) \n"
|
||||
"addi sp, sp, "RISCV_SZPTR "\n"
|
||||
REG_L" ra, (sp) \n"
|
||||
"addi sp, sp, "RISCV_SZPTR "\n"
|
||||
:
|
||||
: [sp] "r" (sp), [regs] "r" (regs)
|
||||
: "a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7",
|
||||
"t0", "t1", "t2", "t3", "t4", "t5", "t6",
|
||||
"memory");
|
||||
} else
|
||||
#endif
|
||||
handle_riscv_irq(regs);
|
||||
|
||||
irqentry_exit(regs, state);
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user