iv/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Core code updates:

Browse Source 
- Reduce the amount of work to release a task stack in context
     switch. There is no real reason to do cgroup accounting and memory
     freeing in this performance sensitive context. Aside of this the
     invoked functions cannot be called from this preemption disabled
     context on PREEMPT_RT enabled kernels. Solve this by moving the
     accounting into do_exit() and delaying the freeing of the stack unless
     the vmap stack can be cached.
 
   - Provide a mechanism to delay raising signals from atomic context on
     PREEMPT_RT enabled kernels as sighand::lock cannot be acquired.  Store
     the information in the task struct and raise it in the exit path.
 -----BEGIN PGP SIGNATURE-----
 
 iQJHBAABCgAxFiEEQp8+kY+LLUocC4bMphj1TA10mKEFAmI4U6gTHHRnbHhAbGlu
 dXRyb25peC5kZQAKCRCmGPVMDXSYoSpkEACwgaaQUbqVrpw5yb6LbwzUPnjEdFNN
 uUQCv0XZD8LWbfhcQQVSPWGho7S/w2Mkpdhi0DkVb2K0dkB7EvITSNEC4KoS/yez
 8iQBpv6Lm00quHdNLjkQySSZ4NYB8M1GasBI7zSBjROK/+sRqioTPQsM0oDemGmD
 uMvw0dgDJRlB8X4LZv0xuJbYLdSzu2VOlWd5aJG9BUgHkd7PfUWMlHsa29FP0hkP
 A5yziOnr9kMsmCAsgmiyDW/GmefrEealby5M/jgnxTruF/OLnDsP+PYMlws47fPx
 g6xpHkT5H0zQJ/nMJtK2JAlxpnbIl4cLuUnpn7wX316yjBpP2s3Pw04AVdzPPoBa
 ufAoOLFtnrKN6enIqLWaJHGAsBHEULw6d3/7HoAEQOVWChnQSuWOob8z0QDbvM14
 kKtz+LTrO+P5a15fd4g5+9lFBXJUTnF74SYQNwxIm2cV9hxrf15NhAr8yg+RtUvF
 /ilNNAFtXkASLqs9moEi7U+GyBYwemG+gduVZ3Dw8FBxK/vHmDrhlItcZdKom+UJ
 k4VFDVhzd2GYRHMrcaLfkCYew6ou+LD/rjdPhIU9OQHgILIMLY5aLqxDuyPtHqDz
 TEyF5qsL4wYLIUdsWlqyHISqQQ6LfnpIyko5kb2Zt56sYtrcZr8swDy+yimiEOdL
 G4BzQu0nVbCLhw==
 =uGTc
 -----END PGP SIGNATURE-----

Merge tag 'core-core-2022-03-21' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull core process handling RT latency updates from Thomas Gleixner:

 - Reduce the amount of work to release a task stack in context switch.
   There is no real reason to do cgroup accounting and memory freeing in
   this performance sensitive context.

   Aside of this the invoked functions cannot be called from this
   preemption disabled context on PREEMPT_RT enabled kernels. Solve this
   by moving the accounting into do_exit() and delaying the freeing of
   the stack unless the vmap stack can be cached.

 - Provide a mechanism to delay raising signals from atomic context on
   PREEMPT_RT enabled kernels as sighand::lock cannot be acquired. Store
   the information in the task struct and raise it in the exit path.

* tag 'core-core-2022-03-21' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  signal, x86: Delay calling signals in atomic on RT enabled kernels
  fork: Use IS_ENABLED() in account_kernel_stack()
  fork: Only cache the VMAP stack in finish_task_switch()
  fork: Move task stack accounting to do_exit()
  fork: Move memcg_charge_kernel_stack() into CONFIG_VMAP_STACK
  fork: Don't assign the stack pointer in dup_task_struct()
  fork, IA64: Provide alloc_thread_stack_node() for IA64
  fork: Duplicate task_struct before stack allocation
  fork: Redo ifdefs around task stack handling
This commit is contained in:
Linus Torvalds 2022-03-21 12:37:33 -07:00
commit bba90e0964
9 changed files with 244 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
arch/ia64/include/asm/thread_info.h
View File

@ -55,15 +55,15 @@ struct thread_info {
#ifndef ASM_OFFSETS_C
/* how to get the thread information struct from C */
#define current_thread_info() ((struct thread_info *) ((char *) current + IA64_TASK_SIZE))
#define alloc_thread_stack_node(tsk, node) \
#define arch_alloc_thread_stack_node(tsk, node) \
((unsigned long *) ((char *) (tsk) + IA64_TASK_SIZE))
#define task_thread_info(tsk) ((struct thread_info *) ((char *) (tsk) + IA64_TASK_SIZE))
#else
#define current_thread_info() ((struct thread_info *) 0)
#define alloc_thread_stack_node(tsk, node) ((unsigned long *) 0)
#define arch_alloc_thread_stack_node(tsk, node) ((unsigned long *) 0)
#define task_thread_info(tsk) ((struct thread_info *) 0)
#endif
#define free_thread_stack(tsk) /* nothing */
#define arch_free_thread_stack(tsk) /* nothing */
#define task_stack_page(tsk) ((void *)(tsk))
#define __HAVE_THREAD_FUNCTIONS

1
arch/x86/Kconfig
View File

@ -120,6 +120,7 @@ config X86
select ARCH_WANTS_NO_INSTR
select ARCH_WANT_HUGE_PMD_SHARE
select ARCH_WANT_LD_ORPHAN_WARN
select ARCH_WANTS_RT_DELAYED_SIGNALS
select ARCH_WANTS_THP_SWAP if X86_64
select ARCH_HAS_PARANOID_L1D_FLUSH
select BUILDTIME_TABLE_SORT

3
include/linux/sched.h
View File

@ -1090,6 +1090,9 @@ struct task_struct {
/* Restored if set_restore_sigmask() was used: */
sigset_t saved_sigmask;
struct sigpending pending;
#ifdef CONFIG_RT_DELAYED_SIGNALS
struct kernel_siginfo forced_info;
#endif
unsigned long sas_ss_sp;
size_t sas_ss_size;
unsigned int sas_ss_flags;

2
include/linux/sched/task_stack.h
View File

@ -79,6 +79,8 @@ static inline void *try_get_task_stack(struct task_struct *tsk)
static inline void put_task_stack(struct task_struct *tsk) {}
#endif
void exit_task_stack_account(struct task_struct *tsk);
#define task_stack_end_corrupted(task) \
(*(end_of_stack(task)) != STACK_END_MAGIC)

10
kernel/Kconfig.preempt
View File

@ -132,4 +132,14 @@ config SCHED_CORE
which is the likely usage by Linux distributions, there should
be no measurable impact on performance.
config ARCH_WANTS_RT_DELAYED_SIGNALS
bool
help
This option is selected by architectures where raising signals
can happen in atomic contexts on PREEMPT_RT enabled kernels. This
option delays raising the signal until the return to user space
loop where it is also delivered. X86 requires this to deliver
signals from trap handlers which run on IST stacks.
config RT_DELAYED_SIGNALS
def_bool PREEMPT_RT && ARCH_WANTS_RT_DELAYED_SIGNALS

14
kernel/entry/common.c
View File

@ -148,6 +148,18 @@ static void handle_signal_work(struct pt_regs *regs, unsigned long ti_work)
arch_do_signal_or_restart(regs, ti_work & _TIF_SIGPENDING);
}
#ifdef CONFIG_RT_DELAYED_SIGNALS
static inline void raise_delayed_signal(void)
{
if (unlikely(current->forced_info.si_signo)) {
force_sig_info(&current->forced_info);
current->forced_info.si_signo = 0;
}
}
#else
static inline void raise_delayed_signal(void) { }
#endif
static unsigned long exit_to_user_mode_loop(struct pt_regs *regs,
unsigned long ti_work)
{
@ -162,6 +174,8 @@ static unsigned long exit_to_user_mode_loop(struct pt_regs *regs,
if (ti_work & _TIF_NEED_RESCHED)
schedule();
raise_delayed_signal();
if (ti_work & _TIF_UPROBE)
uprobe_notify_resume(regs);

1
kernel/exit.c
View File

@ -845,6 +845,7 @@ void __noreturn do_exit(long code)
put_page(tsk->task_frag.page);
validate_creds_for_do_exit(tsk);
exit_task_stack_account(tsk);
check_stack_usage();
preempt_disable();

271
kernel/fork.c
View File

@ -186,7 +186,7 @@ static inline void free_task_struct(struct task_struct *tsk)
*/
# if THREAD_SIZE >= PAGE_SIZE || defined(CONFIG_VMAP_STACK)
#ifdef CONFIG_VMAP_STACK
# ifdef CONFIG_VMAP_STACK
/*
* vmalloc() is a bit slow, and calling vfree() enough times will force a TLB
* flush. Try to minimize the number of calls by caching stacks.
@ -194,6 +194,41 @@ static inline void free_task_struct(struct task_struct *tsk)
#define NR_CACHED_STACKS 2
static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]);
struct vm_stack {
struct rcu_head rcu;
struct vm_struct *stack_vm_area;
};
static bool try_release_thread_stack_to_cache(struct vm_struct *vm)
{
unsigned int i;
for (i = 0; i < NR_CACHED_STACKS; i++) {
if (this_cpu_cmpxchg(cached_stacks[i], NULL, vm) != NULL)
continue;
return true;
}
return false;
}
static void thread_stack_free_rcu(struct rcu_head *rh)
{
struct vm_stack *vm_stack = container_of(rh, struct vm_stack, rcu);
if (try_release_thread_stack_to_cache(vm_stack->stack_vm_area))
return;
vfree(vm_stack);
}
static void thread_stack_delayed_free(struct task_struct *tsk)
{
struct vm_stack *vm_stack = tsk->stack;
vm_stack->stack_vm_area = tsk->stack_vm_area;
call_rcu(&vm_stack->rcu, thread_stack_free_rcu);
}
static int free_vm_stack_cache(unsigned int cpu)
{
struct vm_struct **cached_vm_stacks = per_cpu_ptr(cached_stacks, cpu);
@ -211,11 +246,35 @@ static int free_vm_stack_cache(unsigned int cpu)
return 0;
}
#endif
static unsigned long *alloc_thread_stack_node(struct task_struct *tsk, int node)
static int memcg_charge_kernel_stack(struct vm_struct *vm)
{
#ifdef CONFIG_VMAP_STACK
int i;
int ret;
BUILD_BUG_ON(IS_ENABLED(CONFIG_VMAP_STACK) && PAGE_SIZE % 1024 != 0);
BUG_ON(vm->nr_pages != THREAD_SIZE / PAGE_SIZE);
for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) {
ret = memcg_kmem_charge_page(vm->pages[i], GFP_KERNEL, 0);
if (ret)
goto err;
}
return 0;
err:
/*
* If memcg_kmem_charge_page() fails, page's memory cgroup pointer is
* NULL, and memcg_kmem_uncharge_page() in free_thread_stack() will
* ignore this page.
*/
for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++)
memcg_kmem_uncharge_page(vm->pages[i], 0);
return ret;
}
static int alloc_thread_stack_node(struct task_struct *tsk, int node)
{
struct vm_struct *vm;
void *stack;
int i;
@ -233,9 +292,14 @@ static unsigned long *alloc_thread_stack_node(struct task_struct *tsk, int node)
/* Clear stale pointers from reused stack. */
memset(s->addr, 0, THREAD_SIZE);
if (memcg_charge_kernel_stack(s)) {
vfree(s->addr);
return -ENOMEM;
}
tsk->stack_vm_area = s;
tsk->stack = s->addr;
return s->addr;
return 0;
}
/*
@ -248,71 +312,95 @@ static unsigned long *alloc_thread_stack_node(struct task_struct *tsk, int node)
THREADINFO_GFP & ~__GFP_ACCOUNT,
PAGE_KERNEL,
0, node, __builtin_return_address(0));
if (!stack)
return -ENOMEM;
vm = find_vm_area(stack);
if (memcg_charge_kernel_stack(vm)) {
vfree(stack);
return -ENOMEM;
}
/*
* We can't call find_vm_area() in interrupt context, and
* free_thread_stack() can be called in interrupt context,
* so cache the vm_struct.
*/
if (stack) {
tsk->stack_vm_area = find_vm_area(stack);
tsk->stack = stack;
}
return stack;
#else
tsk->stack_vm_area = vm;
tsk->stack = stack;
return 0;
}
static void free_thread_stack(struct task_struct *tsk)
{
if (!try_release_thread_stack_to_cache(tsk->stack_vm_area))
thread_stack_delayed_free(tsk);
tsk->stack = NULL;
tsk->stack_vm_area = NULL;
}
# else /* !CONFIG_VMAP_STACK */
static void thread_stack_free_rcu(struct rcu_head *rh)
{
__free_pages(virt_to_page(rh), THREAD_SIZE_ORDER);
}
static void thread_stack_delayed_free(struct task_struct *tsk)
{
struct rcu_head *rh = tsk->stack;
call_rcu(rh, thread_stack_free_rcu);
}
static int alloc_thread_stack_node(struct task_struct *tsk, int node)
{
struct page *page = alloc_pages_node(node, THREADINFO_GFP,
THREAD_SIZE_ORDER);
if (likely(page)) {
tsk->stack = kasan_reset_tag(page_address(page));
return tsk->stack;
return 0;
}
return NULL;
#endif
return -ENOMEM;
}
static inline void free_thread_stack(struct task_struct *tsk)
static void free_thread_stack(struct task_struct *tsk)
{
#ifdef CONFIG_VMAP_STACK
struct vm_struct *vm = task_stack_vm_area(tsk);
if (vm) {
int i;
for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++)
memcg_kmem_uncharge_page(vm->pages[i], 0);
for (i = 0; i < NR_CACHED_STACKS; i++) {
if (this_cpu_cmpxchg(cached_stacks[i],
NULL, tsk->stack_vm_area) != NULL)
continue;
return;
}
vfree_atomic(tsk->stack);
return;
}
#endif
__free_pages(virt_to_page(tsk->stack), THREAD_SIZE_ORDER);
thread_stack_delayed_free(tsk);
tsk->stack = NULL;
}
# else
# endif /* CONFIG_VMAP_STACK */
# else /* !(THREAD_SIZE >= PAGE_SIZE || defined(CONFIG_VMAP_STACK)) */
static struct kmem_cache *thread_stack_cache;
static unsigned long *alloc_thread_stack_node(struct task_struct *tsk,
int node)
static void thread_stack_free_rcu(struct rcu_head *rh)
{
kmem_cache_free(thread_stack_cache, rh);
}
static void thread_stack_delayed_free(struct task_struct *tsk)
{
struct rcu_head *rh = tsk->stack;
call_rcu(rh, thread_stack_free_rcu);
}
static int alloc_thread_stack_node(struct task_struct *tsk, int node)
{
unsigned long *stack;
stack = kmem_cache_alloc_node(thread_stack_cache, THREADINFO_GFP, node);
stack = kasan_reset_tag(stack);
tsk->stack = stack;
return stack;
return stack ? 0 : -ENOMEM;
}
static void free_thread_stack(struct task_struct *tsk)
{
kmem_cache_free(thread_stack_cache, tsk->stack);
thread_stack_delayed_free(tsk);
tsk->stack = NULL;
}
void thread_stack_cache_init(void)
@ -322,8 +410,26 @@ void thread_stack_cache_init(void)
THREAD_SIZE, NULL);
BUG_ON(thread_stack_cache == NULL);
}
# endif
#endif
# endif /* THREAD_SIZE >= PAGE_SIZE || defined(CONFIG_VMAP_STACK) */
#else /* CONFIG_ARCH_THREAD_STACK_ALLOCATOR */
static int alloc_thread_stack_node(struct task_struct *tsk, int node)
{
unsigned long *stack;
stack = arch_alloc_thread_stack_node(tsk, node);
tsk->stack = stack;
return stack ? 0 : -ENOMEM;
}
static void free_thread_stack(struct task_struct *tsk)
{
arch_free_thread_stack(tsk);
tsk->stack = NULL;
}
#endif /* !CONFIG_ARCH_THREAD_STACK_ALLOCATOR */
/* SLAB cache for signal_struct structures (tsk->signal) */
static struct kmem_cache *signal_cachep;
@ -380,50 +486,34 @@ void vm_area_free(struct vm_area_struct *vma)
static void account_kernel_stack(struct task_struct *tsk, int account)
{
void *stack = task_stack_page(tsk);
struct vm_struct *vm = task_stack_vm_area(tsk);
if (vm) {
if (IS_ENABLED(CONFIG_VMAP_STACK)) {
struct vm_struct *vm = task_stack_vm_area(tsk);
int i;
for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++)
mod_lruvec_page_state(vm->pages[i], NR_KERNEL_STACK_KB,
account * (PAGE_SIZE / 1024));
} else {
void *stack = task_stack_page(tsk);
/* All stack pages are in the same node. */
mod_lruvec_kmem_state(stack, NR_KERNEL_STACK_KB,
account * (THREAD_SIZE / 1024));
}
}
static int memcg_charge_kernel_stack(struct task_struct *tsk)
void exit_task_stack_account(struct task_struct *tsk)
{
#ifdef CONFIG_VMAP_STACK
struct vm_struct *vm = task_stack_vm_area(tsk);
int ret;
account_kernel_stack(tsk, -1);
BUILD_BUG_ON(IS_ENABLED(CONFIG_VMAP_STACK) && PAGE_SIZE % 1024 != 0);
if (vm) {
if (IS_ENABLED(CONFIG_VMAP_STACK)) {
struct vm_struct *vm;
int i;
BUG_ON(vm->nr_pages != THREAD_SIZE / PAGE_SIZE);
for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) {
/*
* If memcg_kmem_charge_page() fails, page's
* memory cgroup pointer is NULL, and
* memcg_kmem_uncharge_page() in free_thread_stack()
* will ignore this page.
*/
ret = memcg_kmem_charge_page(vm->pages[i], GFP_KERNEL,
0);
if (ret)
return ret;
}
vm = task_stack_vm_area(tsk);
for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++)
memcg_kmem_uncharge_page(vm->pages[i], 0);
}
#endif
return 0;
}
static void release_task_stack(struct task_struct *tsk)
@ -431,12 +521,7 @@ static void release_task_stack(struct task_struct *tsk)
if (WARN_ON(READ_ONCE(tsk->__state) != TASK_DEAD))
return; /* Better to leak the stack than to free prematurely */
account_kernel_stack(tsk, -1);
free_thread_stack(tsk);
tsk->stack = NULL;
#ifdef CONFIG_VMAP_STACK
tsk->stack_vm_area = NULL;
#endif
}
#ifdef CONFIG_THREAD_INFO_IN_TASK
@ -875,8 +960,6 @@ void set_task_stack_end_magic(struct task_struct *tsk)
static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
{
struct task_struct *tsk;
unsigned long *stack;
struct vm_struct *stack_vm_area __maybe_unused;
int err;
if (node == NUMA_NO_NODE)
@ -885,32 +968,18 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
if (!tsk)
return NULL;
stack = alloc_thread_stack_node(tsk, node);
if (!stack)
err = arch_dup_task_struct(tsk, orig);
if (err)
goto free_tsk;
if (memcg_charge_kernel_stack(tsk))
goto free_stack;
err = alloc_thread_stack_node(tsk, node);
if (err)
goto free_tsk;
stack_vm_area = task_stack_vm_area(tsk);
err = arch_dup_task_struct(tsk, orig);
/*
* arch_dup_task_struct() clobbers the stack-related fields. Make
* sure they're properly initialized before using any stack-related
* functions again.
*/
tsk->stack = stack;
#ifdef CONFIG_VMAP_STACK
tsk->stack_vm_area = stack_vm_area;
#endif
#ifdef CONFIG_THREAD_INFO_IN_TASK
refcount_set(&tsk->stack_refcount, 1);
#endif
if (err)
goto free_stack;
account_kernel_stack(tsk, 1);
err = scs_prepare(tsk, node);
if (err)
@ -954,8 +1023,6 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
tsk->wake_q.next = NULL;
tsk->worker_private = NULL;
account_kernel_stack(tsk, 1);
kcov_task_init(tsk);
kmap_local_fork(tsk);
@ -978,6 +1045,7 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
return tsk;
free_stack:
exit_task_stack_account(tsk);
free_thread_stack(tsk);
free_tsk:
free_task_struct(tsk);
@ -2450,6 +2518,7 @@ bad_fork_cleanup_count:
exit_creds(p);
bad_fork_free:
WRITE_ONCE(p->__state, TASK_DEAD);
exit_task_stack_account(p);
put_task_stack(p);
delayed_free_task(p);
fork_out:

40
kernel/signal.c
View File

@ -1307,6 +1307,43 @@ enum sig_handler {
HANDLER_EXIT, /* Only visible as the process exit code */
};
/*
* On some archictectures, PREEMPT_RT has to delay sending a signal from a
* trap since it cannot enable preemption, and the signal code's
* spin_locks turn into mutexes. Instead, it must set TIF_NOTIFY_RESUME
* which will send the signal on exit of the trap.
*/
#ifdef CONFIG_RT_DELAYED_SIGNALS
static inline bool force_sig_delayed(struct kernel_siginfo *info,
struct task_struct *t)
{
if (!in_atomic())
return false;
if (WARN_ON_ONCE(t->forced_info.si_signo))
return true;
if (is_si_special(info)) {
WARN_ON_ONCE(info != SEND_SIG_PRIV);
t->forced_info.si_signo = info->si_signo;
t->forced_info.si_errno = 0;
t->forced_info.si_code = SI_KERNEL;
t->forced_info.si_pid = 0;
t->forced_info.si_uid = 0;
} else {
t->forced_info = *info;
}
set_tsk_thread_flag(t, TIF_NOTIFY_RESUME);
return true;
}
#else
static inline bool force_sig_delayed(struct kernel_siginfo *info,
struct task_struct *t)
{
return false;
}
#endif
/*
* Force a signal that the process can't ignore: if necessary
* we unblock the signal and change any SIG_IGN to SIG_DFL.
@ -1327,6 +1364,9 @@ force_sig_info_to_task(struct kernel_siginfo *info, struct task_struct *t,
struct k_sigaction *action;
int sig = info->si_signo;
if (force_sig_delayed(info, t))
return 0;
spin_lock_irqsave(&t->sighand->siglock, flags);
action = &t->sighand->action[sig-1];
ignored = action->sa.sa_handler == SIG_IGN;