| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Date: Thu, 16 Dec 2010 11:36:49 -0800
From: Frank Rowand <frank.rowand@...il.com>
To: frank.rowand@...sony.com, frank.rowand@...il.com
CC: Peter Zijlstra <a.p.zijlstra@...llo.nl>,
Chris Mason <chris.mason@...cle.com>,
Ingo Molnar <mingo@...e.hu>,
Thomas Gleixner <tglx@...utronix.de>,
Mike Galbraith <efault@....de>,
Oleg Nesterov <oleg@...hat.com>, Paul Turner <pjt@...gle.com>,
Jens Axboe <axboe@...nel.dk>, linux-kernel@...r.kernel.org
Subject: Re: [RFC][PATCH 0/5] Reduce runqueue lock contention -v2
patch 2 of 2
Signed-off-by: Frank Rowand <frank.rowand@...sony.com>
---
Index: linux-2.6/include/linux/sched.h
===================================================================
--- linux-2.6.orig/include/linux/sched.h
+++ linux-2.6/include/linux/sched.h
@@ -1060,7 +1060,7 @@ struct sched_class {
#ifdef CONFIG_SMP
int (*select_task_rq)(struct rq *rq, struct task_struct *p,
- int sd_flag, int flags);
+ int sd_flag, int flags, int waking_cpu);
void (*pre_schedule) (struct rq *this_rq, struct task_struct *task);
void (*post_schedule) (struct rq *this_rq);
@@ -1196,6 +1196,7 @@ struct task_struct {
#ifdef CONFIG_SMP
struct task_struct *ttwu_queue_wake_entry;
int ttwu_queue_wake_flags;
+ int ttwu_waking_cpu;
#ifdef __ARCH_WANT_UNLOCKED_CTXSW
int oncpu;
#endif
Index: linux-2.6/kernel/sched.c
===================================================================
--- linux-2.6.orig/kernel/sched.c
+++ linux-2.6/kernel/sched.c
@@ -2262,9 +2262,11 @@ static int select_fallback_rq(int cpu, s
* The caller (fork, wakeup) owns TASK_WAKING, ->cpus_allowed is stable.
*/
static inline
-int select_task_rq(struct rq *rq, struct task_struct *p, int sd_flags, int wake_flags)
+int select_task_rq(struct rq *rq, struct task_struct *p, int sd_flags,
+ int wake_flags, int waking_cpu)
{
- int cpu = p->sched_class->select_task_rq(rq, p, sd_flags, wake_flags);
+ int cpu = p->sched_class->select_task_rq(rq, p, sd_flags, wake_flags,
+ waking_cpu);
/*
* In order not to call set_task_cpu() on a blocking task we need
@@ -2335,12 +2337,14 @@ static inline void ttwu_post_activation(
}
#ifdef CONFIG_SMP
-static void ttwu_queue_wake_up(struct task_struct *p, int cpu, int wake_flags)
+static void ttwu_queue_wake_up(struct task_struct *p, int this_cpu, int p_cpu,
+ int wake_flags)
{
struct task_struct *next = NULL;
- struct rq *rq = cpu_rq(cpu);
+ struct rq *rq = cpu_rq(p_cpu);
p->ttwu_queue_wake_flags = wake_flags;
+ p->ttwu_waking_cpu = this_cpu;
for (;;) {
struct task_struct *old = next;
@@ -2352,7 +2356,7 @@ static void ttwu_queue_wake_up(struct ta
}
if (!next)
- smp_send_reschedule(cpu);
+ smp_send_reschedule(p_cpu);
}
#endif
@@ -2377,8 +2381,6 @@ static int try_to_wake_up(struct task_st
/*
* xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
* todo
- * - pass waking cpu with queued wake up, to be used in call to
- * select_task_rq().
* - handle cpu being offlined
* xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
*/
@@ -2387,7 +2389,7 @@ static int try_to_wake_up(struct task_st
unsigned long en_flags = ENQUEUE_WAKEUP;
struct rq *rq;
#ifdef CONFIG_SMP
- int load;
+ int load, waking_cpu;
#endif
this_cpu = get_cpu();
@@ -2405,8 +2407,12 @@ static int try_to_wake_up(struct task_st
load = task_contributes_to_load(p);
if (cmpxchg(&p->state, task_state, TASK_WAKING) == task_state) {
- if (state == TASK_WAKING)
+ if (state == TASK_WAKING) {
load = wake_flags & WF_LOAD;
+ waking_cpu = p->ttwu_waking_cpu;
+ } else {
+ waking_cpu = this_cpu;
+ }
break;
}
}
@@ -2443,7 +2449,7 @@ static int try_to_wake_up(struct task_st
if (cpu != this_cpu) {
if (load)
wake_flags |= WF_LOAD;
- ttwu_queue_wake_up(p, cpu, wake_flags);
+ ttwu_queue_wake_up(p, this_cpu, cpu, wake_flags);
success = 1;
goto out_nolock;
}
@@ -2482,7 +2488,7 @@ static int try_to_wake_up(struct task_st
en_flags |= ENQUEUE_WAKING;
}
- cpu = select_task_rq(rq, p, SD_BALANCE_WAKE, wake_flags);
+ cpu = select_task_rq(rq, p, SD_BALANCE_WAKE, wake_flags, waking_cpu);
if (cpu != orig_cpu)
set_task_cpu(p, cpu);
/*
@@ -2728,7 +2734,7 @@ void wake_up_new_task(struct task_struct
* We set TASK_WAKING so that select_task_rq() can drop rq->lock
* without people poking at ->cpus_allowed.
*/
- cpu = select_task_rq(rq, p, SD_BALANCE_FORK, 0);
+ cpu = select_task_rq(rq, p, SD_BALANCE_FORK, 0, cpu);
set_task_cpu(p, cpu);
p->state = TASK_RUNNING;
@@ -3327,7 +3333,8 @@ void sched_exec(void)
int dest_cpu;
rq = task_rq_lock(p, &flags);
- dest_cpu = p->sched_class->select_task_rq(rq, p, SD_BALANCE_EXEC, 0);
+ dest_cpu = p->sched_class->select_task_rq(rq, p, SD_BALANCE_EXEC, 0,
+ smp_processor_id());
if (dest_cpu == smp_processor_id())
goto unlock;
Index: linux-2.6/kernel/sched_fair.c
===================================================================
--- linux-2.6.orig/kernel/sched_fair.c
+++ linux-2.6/kernel/sched_fair.c
@@ -1606,10 +1606,10 @@ static int select_idle_sibling(struct ta
* preempt must be disabled.
*/
static int
-select_task_rq_fair(struct rq *rq, struct task_struct *p, int sd_flag, int wake_flags)
+select_task_rq_fair(struct rq *rq, struct task_struct *p, int sd_flag,
+ int wake_flags, int cpu)
{
struct sched_domain *tmp, *affine_sd = NULL, *sd = NULL;
- int cpu = smp_processor_id();
int prev_cpu = task_cpu(p);
int new_cpu = cpu;
int want_affine = 0;
Index: linux-2.6/kernel/sched_idletask.c
===================================================================
--- linux-2.6.orig/kernel/sched_idletask.c
+++ linux-2.6/kernel/sched_idletask.c
@@ -7,7 +7,8 @@
#ifdef CONFIG_SMP
static int
-select_task_rq_idle(struct rq *rq, struct task_struct *p, int sd_flag, int flags)
+select_task_rq_idle(struct rq *rq, struct task_struct *p, int sd_flag,
+ int flags, int waking_cpu)
{
return task_cpu(p); /* IDLE tasks as never migrated */
}
Index: linux-2.6/kernel/sched_rt.c
===================================================================
--- linux-2.6.orig/kernel/sched_rt.c
+++ linux-2.6/kernel/sched_rt.c
@@ -973,7 +973,8 @@ static void yield_task_rt(struct rq *rq)
static int find_lowest_rq(struct task_struct *task);
static int
-select_task_rq_rt(struct rq *rq, struct task_struct *p, int sd_flag, int flags)
+select_task_rq_rt(struct rq *rq, struct task_struct *p, int sd_flag, int flags,
+ int waking_cpu)
{
if (sd_flag != SD_BALANCE_WAKE)
return smp_processor_id();
Index: linux-2.6/kernel/sched_stoptask.c
===================================================================
--- linux-2.6.orig/kernel/sched_stoptask.c
+++ linux-2.6/kernel/sched_stoptask.c
@@ -10,7 +10,7 @@
#ifdef CONFIG_SMP
static int
select_task_rq_stop(struct rq *rq, struct task_struct *p,
- int sd_flag, int flags)
+ int sd_flag, int flags, int waking_cpu)
{
return task_cpu(p); /* stop tasks as never migrate */
}
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@...r.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
Powered by blists - more mailing lists