ceeadb83ae
If we want to use the schedstats facility to trace other sched classes, we should make it independent of fair sched class. The struct sched_statistics is the schedular statistics of a task_struct or a task_group. So we can move it into struct task_struct and struct task_group to achieve the goal. After the patch, schestats are orgnized as follows, struct task_struct { ... struct sched_entity se; struct sched_rt_entity rt; struct sched_dl_entity dl; ... struct sched_statistics stats; ... }; Regarding the task group, schedstats is only supported for fair group sched, and a new struct sched_entity_stats is introduced, suggested by Peter - struct sched_entity_stats { struct sched_entity se; struct sched_statistics stats; } __no_randomize_layout; Then with the se in a task_group, we can easily get the stats. The sched_statistics members may be frequently modified when schedstats is enabled, in order to avoid impacting on random data which may in the same cacheline with them, the struct sched_statistics is defined as cacheline aligned. As this patch changes the core struct of scheduler, so I verified the performance it may impact on the scheduler with 'perf bench sched pipe', suggested by Mel. Below is the result, in which all the values are in usecs/op. Before After kernel.sched_schedstats=0 5.2~5.4 5.2~5.4 kernel.sched_schedstats=1 5.3~5.5 5.3~5.5 [These data is a little difference with the earlier version, that is because my old test machine is destroyed so I have to use a new different test machine.] Almost no impact on the sched performance. No functional change. [lkp@intel.com: reported build failure in earlier version] Signed-off-by: Yafang Shao <laoar.shao@gmail.com> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Acked-by: Mel Gorman <mgorman@suse.de> Link: https://lore.kernel.org/r/20210905143547.4668-3-laoar.shao@gmail.com
146 lines
3.2 KiB
C
146 lines
3.2 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
/*
|
|
* stop-task scheduling class.
|
|
*
|
|
* The stop task is the highest priority task in the system, it preempts
|
|
* everything and will be preempted by nothing.
|
|
*
|
|
* See kernel/stop_machine.c
|
|
*/
|
|
#include "sched.h"
|
|
|
|
#ifdef CONFIG_SMP
|
|
static int
|
|
select_task_rq_stop(struct task_struct *p, int cpu, int flags)
|
|
{
|
|
return task_cpu(p); /* stop tasks as never migrate */
|
|
}
|
|
|
|
static int
|
|
balance_stop(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
|
|
{
|
|
return sched_stop_runnable(rq);
|
|
}
|
|
#endif /* CONFIG_SMP */
|
|
|
|
static void
|
|
check_preempt_curr_stop(struct rq *rq, struct task_struct *p, int flags)
|
|
{
|
|
/* we're never preempted */
|
|
}
|
|
|
|
static void set_next_task_stop(struct rq *rq, struct task_struct *stop, bool first)
|
|
{
|
|
stop->se.exec_start = rq_clock_task(rq);
|
|
}
|
|
|
|
static struct task_struct *pick_task_stop(struct rq *rq)
|
|
{
|
|
if (!sched_stop_runnable(rq))
|
|
return NULL;
|
|
|
|
return rq->stop;
|
|
}
|
|
|
|
static struct task_struct *pick_next_task_stop(struct rq *rq)
|
|
{
|
|
struct task_struct *p = pick_task_stop(rq);
|
|
|
|
if (p)
|
|
set_next_task_stop(rq, p, true);
|
|
|
|
return p;
|
|
}
|
|
|
|
static void
|
|
enqueue_task_stop(struct rq *rq, struct task_struct *p, int flags)
|
|
{
|
|
add_nr_running(rq, 1);
|
|
}
|
|
|
|
static void
|
|
dequeue_task_stop(struct rq *rq, struct task_struct *p, int flags)
|
|
{
|
|
sub_nr_running(rq, 1);
|
|
}
|
|
|
|
static void yield_task_stop(struct rq *rq)
|
|
{
|
|
BUG(); /* the stop task should never yield, its pointless. */
|
|
}
|
|
|
|
static void put_prev_task_stop(struct rq *rq, struct task_struct *prev)
|
|
{
|
|
struct task_struct *curr = rq->curr;
|
|
u64 delta_exec;
|
|
|
|
delta_exec = rq_clock_task(rq) - curr->se.exec_start;
|
|
if (unlikely((s64)delta_exec < 0))
|
|
delta_exec = 0;
|
|
|
|
schedstat_set(curr->stats.exec_max,
|
|
max(curr->stats.exec_max, delta_exec));
|
|
|
|
curr->se.sum_exec_runtime += delta_exec;
|
|
account_group_exec_runtime(curr, delta_exec);
|
|
|
|
curr->se.exec_start = rq_clock_task(rq);
|
|
cgroup_account_cputime(curr, delta_exec);
|
|
}
|
|
|
|
/*
|
|
* scheduler tick hitting a task of our scheduling class.
|
|
*
|
|
* NOTE: This function can be called remotely by the tick offload that
|
|
* goes along full dynticks. Therefore no local assumption can be made
|
|
* and everything must be accessed through the @rq and @curr passed in
|
|
* parameters.
|
|
*/
|
|
static void task_tick_stop(struct rq *rq, struct task_struct *curr, int queued)
|
|
{
|
|
}
|
|
|
|
static void switched_to_stop(struct rq *rq, struct task_struct *p)
|
|
{
|
|
BUG(); /* its impossible to change to this class */
|
|
}
|
|
|
|
static void
|
|
prio_changed_stop(struct rq *rq, struct task_struct *p, int oldprio)
|
|
{
|
|
BUG(); /* how!?, what priority? */
|
|
}
|
|
|
|
static void update_curr_stop(struct rq *rq)
|
|
{
|
|
}
|
|
|
|
/*
|
|
* Simple, special scheduling class for the per-CPU stop tasks:
|
|
*/
|
|
DEFINE_SCHED_CLASS(stop) = {
|
|
|
|
.enqueue_task = enqueue_task_stop,
|
|
.dequeue_task = dequeue_task_stop,
|
|
.yield_task = yield_task_stop,
|
|
|
|
.check_preempt_curr = check_preempt_curr_stop,
|
|
|
|
.pick_next_task = pick_next_task_stop,
|
|
.put_prev_task = put_prev_task_stop,
|
|
.set_next_task = set_next_task_stop,
|
|
|
|
#ifdef CONFIG_SMP
|
|
.balance = balance_stop,
|
|
.pick_task = pick_task_stop,
|
|
.select_task_rq = select_task_rq_stop,
|
|
.set_cpus_allowed = set_cpus_allowed_common,
|
|
#endif
|
|
|
|
.task_tick = task_tick_stop,
|
|
|
|
.prio_changed = prio_changed_stop,
|
|
.switched_to = switched_to_stop,
|
|
.update_curr = update_curr_stop,
|
|
};
|