Per cpu (kernel) threads can currently trick the load balancer in thinking there's something to do and result in moving perfectly placed tasks away. By virtue of the new do_set_cpus_allowed() we can easily add nr_pinned accounting. Which we'll use to change the fbq classification such that we make it less likely to pick such tasks for migration. Note that it is still possible we'll migrate these well placed tasks away, further patches could reduce this still. Suggested-by: Rik van Riel Signed-off-by: Peter Zijlstra (Intel) --- kernel/sched/fair.c | 42 ++++++++++++++++++++++++++++++++++-------- kernel/sched/sched.h | 1 + 2 files changed, 35 insertions(+), 8 deletions(-) --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -858,12 +858,20 @@ static unsigned int task_scan_max(struct static void account_numa_enqueue(struct rq *rq, struct task_struct *p) { + if (p->nr_cpus_allowed == 1) { + p->numa_preferred_nid = -1; + rq->nr_pinned_running++; + } rq->nr_numa_running += (p->numa_preferred_nid != -1); rq->nr_preferred_running += (p->numa_preferred_nid == task_node(p)); } static void account_numa_dequeue(struct rq *rq, struct task_struct *p) { + if (p->nr_cpus_allowed == 1) { + rq->nr_pinned_running--; + WARN_ON_ONCE(p->numa_preferred_nid != -1); + } rq->nr_numa_running -= (p->numa_preferred_nid != -1); rq->nr_preferred_running -= (p->numa_preferred_nid == task_node(p)); } @@ -2266,6 +2274,12 @@ void task_tick_numa(struct rq *rq, struc return; /* + * We don't care about NUMA placement if we can't migrate the task. + */ + if (curr->nr_cpus_allowed == 1) + return; + + /* * Using runtime rather than walltime has the dual advantage that * we (mostly) drive the selection from busy threads and that the * task needs to have done some actual work before we bother with @@ -5567,7 +5581,7 @@ static bool yield_to_task_fair(struct rq static unsigned long __read_mostly max_load_balance_interval = HZ/10; -enum fbq_type { regular, remote, all }; +enum fbq_type { regular, remote, movable, all }; #define LBF_ALL_PINNED 0x01 #define LBF_NEED_BREAK 0x02 @@ -6112,6 +6126,7 @@ struct sg_lb_stats { enum group_type group_type; int group_no_capacity; #ifdef CONFIG_NUMA_BALANCING + unsigned int nr_pinned_running; unsigned int nr_numa_running; unsigned int nr_preferred_running; #endif @@ -6449,6 +6464,7 @@ static inline void update_sg_lb_stats(st *overload = true; #ifdef CONFIG_NUMA_BALANCING + sgs->nr_pinned_running += rq->nr_pinned_running; sgs->nr_numa_running += rq->nr_numa_running; sgs->nr_preferred_running += rq->nr_preferred_running; #endif @@ -6522,19 +6538,27 @@ static bool update_sd_pick_busiest(struc #ifdef CONFIG_NUMA_BALANCING static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs) { - if (sgs->sum_nr_running > sgs->nr_numa_running) + unsigned int nr_migratable = sgs->sum_nr_running - sgs->nr_pinned_running; + + if (nr_migratable > sgs->nr_numa_running) return regular; - if (sgs->sum_nr_running > sgs->nr_preferred_running) + if (nr_migratable > sgs->nr_preferred_running) return remote; + if (nr_migratable) + return movable; return all; } static inline enum fbq_type fbq_classify_rq(struct rq *rq) { - if (rq->nr_running > rq->nr_numa_running) + unsigned int nr_migratable = rq->cfs.h_nr_running - rq->nr_pinned_running; + + if (nr_migratable > rq->nr_numa_running) return regular; - if (rq->nr_running > rq->nr_preferred_running) + if (nr_migratable > rq->nr_preferred_running) return remote; + if (nr_migratable) + return movable; return all; } #else @@ -6938,9 +6962,11 @@ static struct rq *find_busiest_queue(str /* * We classify groups/runqueues into three groups: - * - regular: there are !numa tasks - * - remote: there are numa tasks that run on the 'wrong' node - * - all: there is no distinction + * - regular: there are (migratable) !numa tasks + * - remote: there are (migratable) numa tasks that + * run on the 'wrong' node + * - movable: there are (migratable) tasks + * - all: there are tasks * * In order to avoid migrating ideally placed numa tasks, * ignore those when there's better options. --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -571,6 +571,7 @@ struct rq { #ifdef CONFIG_NUMA_BALANCING unsigned int nr_numa_running; unsigned int nr_preferred_running; + unsigned int nr_pinned_running; #endif #define CPU_LOAD_IDX_MAX 5 unsigned long cpu_load[CPU_LOAD_IDX_MAX]; -- To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/