| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-Id: <1354305521-11583-9-git-send-email-mingo@kernel.org>
Date: Fri, 30 Nov 2012 20:58:39 +0100
From: Ingo Molnar <mingo@...nel.org>
To: linux-kernel@...r.kernel.org, linux-mm@...ck.org
Cc: Peter Zijlstra <a.p.zijlstra@...llo.nl>,
Paul Turner <pjt@...gle.com>,
Lee Schermerhorn <Lee.Schermerhorn@...com>,
Christoph Lameter <cl@...ux.com>,
Rik van Riel <riel@...hat.com>, Mel Gorman <mgorman@...e.de>,
Andrew Morton <akpm@...ux-foundation.org>,
Andrea Arcangeli <aarcange@...hat.com>,
Linus Torvalds <torvalds@...ux-foundation.org>,
Thomas Gleixner <tglx@...utronix.de>,
Johannes Weiner <hannes@...xchg.org>,
Hugh Dickins <hughd@...gle.com>
Subject: [PATCH 08/10] sched: Converge NUMA migrations
Consolidate the various convergence models and add a new one: when
a strongly converged NUMA task migrates, prefer to migrate it in
the direction of its preferred node.
Cc: Linus Torvalds <torvalds@...ux-foundation.org>
Cc: Andrew Morton <akpm@...ux-foundation.org>
Cc: Peter Zijlstra <a.p.zijlstra@...llo.nl>
Cc: Andrea Arcangeli <aarcange@...hat.com>
Cc: Rik van Riel <riel@...hat.com>
Cc: Mel Gorman <mgorman@...e.de>
Cc: Hugh Dickins <hughd@...gle.com>
Signed-off-by: Ingo Molnar <mingo@...nel.org>
---
kernel/sched/fair.c | 59 +++++++++++++++++++++++++++++++++++--------------
kernel/sched/features.h | 3 ++-
2 files changed, 44 insertions(+), 18 deletions(-)
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 1f6104a..10cbfa3 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -4750,6 +4750,35 @@ done:
return target;
}
+static bool numa_allow_migration(struct task_struct *p, int prev_cpu, int new_cpu)
+{
+#ifdef CONFIG_NUMA_BALANCING
+ if (sched_feat(NUMA_CONVERGE_MIGRATIONS)) {
+ /* Help in the direction of expected convergence: */
+ if (p->convergence_node >= 0 && (cpu_to_node(new_cpu) != p->convergence_node))
+ return false;
+
+ return true;
+ }
+
+ if (sched_feat(NUMA_BALANCE_ALL)) {
+ if (task_numa_shared(p) >= 0)
+ return false;
+
+ return true;
+ }
+
+ if (sched_feat(NUMA_BALANCE_INTERNODE)) {
+ if (task_numa_shared(p) >= 0) {
+ if (cpu_to_node(prev_cpu) != cpu_to_node(new_cpu))
+ return false;
+ }
+ }
+#endif
+ return true;
+}
+
+
/*
* sched_balance_self: balance the current task (running on cpu) in domains
* that have the 'flag' flag set. In practice, this is SD_BALANCE_FORK and
@@ -4766,7 +4795,8 @@ select_task_rq_fair(struct task_struct *p, int sd_flag, int wake_flags)
{
struct sched_domain *tmp, *affine_sd = NULL, *sd = NULL;
int cpu = smp_processor_id();
- int prev_cpu = task_cpu(p);
+ int prev0_cpu = task_cpu(p);
+ int prev_cpu = prev0_cpu;
int new_cpu = cpu;
int want_affine = 0;
int sync = wake_flags & WF_SYNC;
@@ -4775,10 +4805,6 @@ select_task_rq_fair(struct task_struct *p, int sd_flag, int wake_flags)
return prev_cpu;
#ifdef CONFIG_NUMA_BALANCING
- /* We do NUMA balancing elsewhere: */
- if (sched_feat(NUMA_BALANCE_ALL) && task_numa_shared(p) >= 0)
- return prev_cpu;
-
if (sched_feat(WAKE_ON_IDEAL_CPU) && p->ideal_cpu >= 0)
return p->ideal_cpu;
#endif
@@ -4857,8 +4883,8 @@ select_task_rq_fair(struct task_struct *p, int sd_flag, int wake_flags)
unlock:
rcu_read_unlock();
- if (sched_feat(NUMA_BALANCE_INTERNODE) && task_numa_shared(p) >= 0 && (cpu_to_node(prev_cpu) != cpu_to_node(new_cpu)))
- return prev_cpu;
+ if (!numa_allow_migration(p, prev0_cpu, new_cpu))
+ return prev0_cpu;
return new_cpu;
}
@@ -5401,8 +5427,11 @@ static bool can_migrate_running_task(struct task_struct *p, struct lb_env *env)
static int can_migrate_task(struct task_struct *p, struct lb_env *env)
{
/* We do NUMA balancing elsewhere: */
- if (sched_feat(NUMA_BALANCE_ALL) && task_numa_shared(p) > 0 && env->failed <= env->sd->cache_nice_tries)
- return false;
+
+ if (env->failed <= env->sd->cache_nice_tries) {
+ if (!numa_allow_migration(p, env->src_rq->cpu, env->dst_cpu))
+ return false;
+ }
if (!can_migrate_pinned_task(p, env))
return false;
@@ -5461,10 +5490,7 @@ static int move_one_task(struct lb_env *env)
if (!can_migrate_task(p, env))
continue;
- if (sched_feat(NUMA_BALANCE_ALL) && task_numa_shared(p) >= 0)
- continue;
-
- if (sched_feat(NUMA_BALANCE_INTERNODE) && task_numa_shared(p) >= 0 && (cpu_to_node(env->src_rq->cpu) != cpu_to_node(env->dst_cpu)))
+ if (!numa_allow_migration(p, env->src_rq->cpu, env->dst_cpu))
continue;
move_task(p, env);
@@ -5527,10 +5553,7 @@ static int move_tasks(struct lb_env *env)
if (!can_migrate_task(p, env))
goto next;
- if (sched_feat(NUMA_BALANCE_ALL) && task_numa_shared(p) >= 0)
- continue;
-
- if (sched_feat(NUMA_BALANCE_INTERNODE) && task_numa_shared(p) >= 0 && (cpu_to_node(env->src_rq->cpu) != cpu_to_node(env->dst_cpu)))
+ if (!numa_allow_migration(p, env->src_rq->cpu, env->dst_cpu))
goto next;
move_task(p, env);
@@ -6520,8 +6543,10 @@ static int load_balance(int this_cpu, struct rq *this_rq,
.iteration = 0,
};
+#ifdef CONFIG_NUMA_BALANCING
if (sched_feat(NUMA_BALANCE_ALL))
return 1;
+#endif
cpumask_copy(cpus, cpu_active_mask);
max_lb_iterations = cpumask_weight(env.dst_grpmask);
diff --git a/kernel/sched/features.h b/kernel/sched/features.h
index fd9db0b..9075faf 100644
--- a/kernel/sched/features.h
+++ b/kernel/sched/features.h
@@ -76,9 +76,10 @@ SCHED_FEAT(WAKE_ON_IDEAL_CPU, false)
/* Do the working set probing faults: */
SCHED_FEAT(NUMA, true)
SCHED_FEAT(NUMA_BALANCE_ALL, false)
-SCHED_FEAT(NUMA_BALANCE_INTERNODE, false)
+SCHED_FEAT(NUMA_BALANCE_INTERNODE, false)
SCHED_FEAT(NUMA_LB, false)
SCHED_FEAT(NUMA_GROUP_LB_COMPRESS, true)
SCHED_FEAT(NUMA_GROUP_LB_SPREAD, true)
SCHED_FEAT(MIGRATE_FAULT_STATS, false)
+SCHED_FEAT(NUMA_CONVERGE_MIGRATIONS, true)
#endif
--
1.7.11.7
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@...r.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
Powered by blists - more mailing lists