With sched_class::task_waking being called only when we do set_task_cpu(), we can make sched_class::migrate_task_rq() do the work and eliminate sched_class::task_waking entirely. Cc: Pavan Kondeti Cc: Ben Segall Cc: Matt Fleming Cc: Mike Galbraith Cc: Morten Rasmussen Cc: Paul Turner Cc: Thomas Gleixner Cc: byungchul.park@lge.com Cc: Andrew Hunter Signed-off-by: Peter Zijlstra (Intel) --- kernel/sched/core.c | 9 ------- kernel/sched/fair.c | 58 ++++++++++++++++++++++++--------------------------- kernel/sched/sched.h | 7 ++---- 3 files changed, 32 insertions(+), 42 deletions(-) --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -1717,11 +1717,8 @@ ttwu_do_activate(struct rq *rq, struct t if (p->sched_contributes_to_load) rq->nr_uninterruptible--; - /* - * If we migrated; we must have called sched_class::task_waking(). - */ if (wake_flags & WF_MIGRATED) - en_flags |= ENQUEUE_WAKING; + en_flags |= ENQUEUE_MIGRATED; #endif ttwu_activate(rq, p, en_flags); @@ -2049,10 +2046,6 @@ try_to_wake_up(struct task_struct *p, un cpu = select_task_rq(p, p->wake_cpu, SD_BALANCE_WAKE, wake_flags); if (task_cpu(p) != cpu) { wake_flags |= WF_MIGRATED; - - if (p->sched_class->task_waking) - p->sched_class->task_waking(p); - set_task_cpu(p, cpu); } #endif /* CONFIG_SMP */ --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -3282,7 +3282,7 @@ static inline void check_schedstat_requi * * WAKEUP (remote) * - * ->task_waking_fair() + * ->migrate_task_rq_fair() (p->state == TASK_WAKING) * vruntime -= min_vruntime * * enqueue @@ -3297,7 +3297,7 @@ static inline void check_schedstat_requi static void enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) { - bool renorm = !(flags & ENQUEUE_WAKEUP) || (flags & ENQUEUE_WAKING); + bool renorm = !(flags & ENQUEUE_WAKEUP) || (flags & ENQUEUE_MIGRATED); bool curr = cfs_rq->curr == se; /* @@ -4865,33 +4865,6 @@ static unsigned long cpu_avg_load_per_ta return 0; } -/* - * Called to migrate a waking task; as blocked tasks retain absolute vruntime - * the migration needs to deal with this by subtracting the old and adding the - * new min_vruntime -- the latter is done by enqueue_entity() when placing - * the task on the new runqueue. - */ -static void task_waking_fair(struct task_struct *p) -{ - struct sched_entity *se = &p->se; - struct cfs_rq *cfs_rq = cfs_rq_of(se); - u64 min_vruntime; - -#ifndef CONFIG_64BIT - u64 min_vruntime_copy; - - do { - min_vruntime_copy = cfs_rq->min_vruntime_copy; - smp_rmb(); - min_vruntime = cfs_rq->min_vruntime; - } while (min_vruntime != min_vruntime_copy); -#else - min_vruntime = cfs_rq->min_vruntime; -#endif - - se->vruntime -= min_vruntime; -} - #ifdef CONFIG_FAIR_GROUP_SCHED /* * effective_load() calculates the load change as seen from the root_task_group @@ -5648,6 +5621,32 @@ select_task_rq_fair(struct task_struct * static void migrate_task_rq_fair(struct task_struct *p) { /* + * As blocked tasks retain absolute vruntime the migration needs to + * deal with this by subtracting the old and adding the new + * min_vruntime -- the latter is done by enqueue_entity() when placing + * the task on the new runqueue. + */ + if (p->state == TASK_WAKING) { + struct sched_entity *se = &p->se; + struct cfs_rq *cfs_rq = cfs_rq_of(se); + u64 min_vruntime; + +#ifndef CONFIG_64BIT + u64 min_vruntime_copy; + + do { + min_vruntime_copy = cfs_rq->min_vruntime_copy; + smp_rmb(); + min_vruntime = cfs_rq->min_vruntime; + } while (min_vruntime != min_vruntime_copy); +#else + min_vruntime = cfs_rq->min_vruntime; +#endif + + se->vruntime -= min_vruntime; + } + + /* * We are supposed to update the task to "current" time, then its up to date * and ready to go to new CPU/cfs_rq. But we have difficulty in getting * what current time is, so simply throw away the out-of-date time. This @@ -8916,7 +8915,6 @@ const struct sched_class fair_sched_clas .rq_online = rq_online_fair, .rq_offline = rq_offline_fair, - .task_waking = task_waking_fair, .task_dead = task_dead_fair, .set_cpus_allowed = set_cpus_allowed_common, #endif --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1164,7 +1164,7 @@ extern const u32 sched_prio_to_wmult[40] * * ENQUEUE_HEAD - place at front of runqueue (tail if not specified) * ENQUEUE_REPLENISH - CBS (replenish runtime and postpone deadline) - * ENQUEUE_WAKING - sched_class::task_waking was called + * ENQUEUE_MIGRATED - the task was migrated during wakeup * */ @@ -1179,9 +1179,9 @@ extern const u32 sched_prio_to_wmult[40] #define ENQUEUE_HEAD 0x08 #define ENQUEUE_REPLENISH 0x10 #ifdef CONFIG_SMP -#define ENQUEUE_WAKING 0x20 +#define ENQUEUE_MIGRATED 0x20 #else -#define ENQUEUE_WAKING 0x00 +#define ENQUEUE_MIGRATED 0x00 #endif #define RETRY_TASK ((void *)-1UL) @@ -1213,7 +1213,6 @@ struct sched_class { int (*select_task_rq)(struct task_struct *p, int task_cpu, int sd_flag, int flags); void (*migrate_task_rq)(struct task_struct *p); - void (*task_waking) (struct task_struct *task); void (*task_woken) (struct rq *this_rq, struct task_struct *task); void (*set_cpus_allowed)(struct task_struct *p,