[ Impact: implement INTERACTIVE feature to increase Xorg responsiveness. ] Apply next buddy logic to interactivity-driven wakeups. Don't pass the interactivity flag across forks to defuse interactivity-based fork-bombs. The goal of this patch is to ensure that Xorg keeps a good interactivity level by ensuring that Xorg and its related threads quickly respond to wakeups caused by user inputs. Derived from a patch from Peter Zijlstra. * This patch also makes sure that as soon as an iowait is perceived, the interactivity chain is stopped. * This patch removes the previously available "NEXT_BUDDY" scheduler feature altogether. On my 2.0GHz uniprocessor desktop, enabling the INTERACTIVE feature makes firefox very responsive even if I overcommit my CPU with a make -j5 kernel build. Signed-off-by: Mathieu Desnoyers CC: Peter Zijlstra --- drivers/input/evdev.c | 2 ++ include/linux/sched.h | 31 +++++++++++++++++++++++-------- kernel/sched.c | 12 +++++++++++- kernel/sched_fair.c | 26 +++++++++++++++++++------- kernel/sched_features.h | 11 ++++------- 5 files changed, 59 insertions(+), 23 deletions(-) Index: linux-2.6-lttng.laptop/drivers/input/evdev.c =================================================================== --- linux-2.6-lttng.laptop.orig/drivers/input/evdev.c +++ linux-2.6-lttng.laptop/drivers/input/evdev.c @@ -78,6 +78,7 @@ static void evdev_event(struct input_han event.code = code; event.value = value; + sched_wake_interactive_enable(); rcu_read_lock(); client = rcu_dereference(evdev->grab); @@ -90,6 +91,7 @@ static void evdev_event(struct input_han rcu_read_unlock(); wake_up_interruptible(&evdev->wait); + sched_wake_interactive_disable(); } static int evdev_fasync(int fd, struct file *file, int on) Index: linux-2.6-lttng.laptop/include/linux/sched.h =================================================================== --- linux-2.6-lttng.laptop.orig/include/linux/sched.h +++ linux-2.6-lttng.laptop/include/linux/sched.h @@ -1024,14 +1024,17 @@ struct sched_domain; /* * wake flags */ -#define WF_SYNC 0x01 /* waker goes to sleep after wakup */ -#define WF_FORK 0x02 /* child wakeup after fork */ +#define WF_SYNC (1 << 0) /* waker goes to sleep after wakup */ +#define WF_FORK (1 << 1) /* child wakeup after fork */ +#define WF_INTERACTIVE (1 << 2) /* interactivity-driven wakeup */ + +#define ENQUEUE_WAKEUP (1 << 0) +#define ENQUEUE_WAKING (1 << 1) +#define ENQUEUE_HEAD (1 << 2) +#define ENQUEUE_IO (1 << 3) +#define ENQUEUE_LATENCY (1 << 4) -#define ENQUEUE_WAKEUP 1 -#define ENQUEUE_WAKING 2 -#define ENQUEUE_HEAD 4 - -#define DEQUEUE_SLEEP 1 +#define DEQUEUE_SLEEP (1 << 0) struct sched_class { const struct sched_class *next; @@ -1124,7 +1127,8 @@ struct sched_entity { struct load_weight load; /* for load-balancing */ struct rb_node run_node; struct list_head group_node; - unsigned int on_rq; + unsigned int on_rq:1, + interactive:1; u64 exec_start; u64 sum_exec_runtime; @@ -1237,6 +1241,7 @@ struct task_struct { unsigned sched_in_iowait:1; /* Called io_schedule() */ unsigned sched_reset_on_fork:1; /* Revert to default * priority/policy on fork */ + unsigned sched_wake_interactive:4; /* User-driven wakeup */ pid_t pid; pid_t tgid; @@ -1502,6 +1507,16 @@ struct task_struct { #endif }; +static inline void sched_wake_interactive_enable(void) +{ + current->sched_wake_interactive++; +} + +static inline void sched_wake_interactive_disable(void) +{ + current->sched_wake_interactive--; +} + /* Future-safe accessor for struct task_struct's cpus_allowed. */ #define tsk_cpus_allowed(tsk) (&(tsk)->cpus_allowed) Index: linux-2.6-lttng.laptop/kernel/sched.c =================================================================== --- linux-2.6-lttng.laptop.orig/kernel/sched.c +++ linux-2.6-lttng.laptop/kernel/sched.c @@ -2288,6 +2288,13 @@ static int try_to_wake_up(struct task_st unsigned long en_flags = ENQUEUE_WAKEUP; struct rq *rq; + if (sched_feat(INTERACTIVE) && !(wake_flags & WF_FORK)) { + if (current->sched_wake_interactive || + wake_flags & WF_INTERACTIVE || + current->se.interactive) + en_flags |= ENQUEUE_LATENCY; + } + this_cpu = get_cpu(); smp_wmb(); @@ -3613,8 +3620,11 @@ need_resched_nonpreemptible: if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) { if (unlikely(signal_pending_state(prev->state, prev))) prev->state = TASK_RUNNING; - else + else { + if (sched_feat(INTERACTIVE)) + prev->se.interactive = 0; deactivate_task(rq, prev, DEQUEUE_SLEEP); + } switch_count = &prev->nvcsw; } Index: linux-2.6-lttng.laptop/kernel/sched_fair.c =================================================================== --- linux-2.6-lttng.laptop.orig/kernel/sched_fair.c +++ linux-2.6-lttng.laptop/kernel/sched_fair.c @@ -774,6 +774,9 @@ enqueue_entity(struct cfs_rq *cfs_rq, st account_entity_enqueue(cfs_rq, se); if (flags & ENQUEUE_WAKEUP) { + if (sched_feat(INTERACTIVE) + && flags & ENQUEUE_LATENCY && !(flags & ENQUEUE_IO)) + se->interactive = 1; place_entity(cfs_rq, se, 0); enqueue_sleeper(cfs_rq, se); } @@ -916,14 +919,14 @@ static struct sched_entity *pick_next_en struct sched_entity *se = __pick_next_entity(cfs_rq); struct sched_entity *left = se; - if (cfs_rq->next && wakeup_preempt_entity(cfs_rq->next, left) < 1) - se = cfs_rq->next; + if (cfs_rq->last && wakeup_preempt_entity(cfs_rq->last, left) < 1) + se = cfs_rq->last; /* - * Prefer last buddy, try to return the CPU to a preempted task. + * Prefer the next buddy, only set through the interactivity logic. */ - if (cfs_rq->last && wakeup_preempt_entity(cfs_rq->last, left) < 1) - se = cfs_rq->last; + if (cfs_rq->next && wakeup_preempt_entity(cfs_rq->next, left) < 1) + se = cfs_rq->next; clear_buddies(cfs_rq, se); @@ -1046,6 +1049,9 @@ enqueue_task_fair(struct rq *rq, struct struct cfs_rq *cfs_rq; struct sched_entity *se = &p->se; + if (p->sched_in_iowait) + flags |= ENQUEUE_IO; + for_each_sched_entity(se) { if (se->on_rq) break; @@ -1657,6 +1663,7 @@ static void check_preempt_wakeup(struct * tasks for there to be buddies. */ int buddies = (cfs_rq->nr_running >= 2); + int preempt = 0; if (unlikely(rt_prio(p->prio))) goto preempt; @@ -1667,8 +1674,13 @@ static void check_preempt_wakeup(struct if (unlikely(se == pse)) return; - if (sched_feat(NEXT_BUDDY) && buddies && !(wake_flags & WF_FORK)) + if (sched_feat(INTERACTIVE) + && !(wake_flags & WF_FORK) && pse->interactive) { + clear_buddies(cfs_rq, NULL); set_next_buddy(pse); + preempt = 1; + buddies = 0; + } /* * We can come here with TIF_NEED_RESCHED already set from new task @@ -1694,7 +1706,7 @@ static void check_preempt_wakeup(struct update_curr(cfs_rq); find_matching_se(&se, &pse); BUG_ON(!pse); - if (wakeup_preempt_entity(se, pse) == 1) + if (preempt || wakeup_preempt_entity(se, pse) == 1) goto preempt; return; Index: linux-2.6-lttng.laptop/kernel/sched_features.h =================================================================== --- linux-2.6-lttng.laptop.orig/kernel/sched_features.h +++ linux-2.6-lttng.laptop/kernel/sched_features.h @@ -26,13 +26,6 @@ SCHED_FEAT(WAKEUP_PREEMPT, 1) SCHED_FEAT(AFFINE_WAKEUPS, 1) /* - * Prefer to schedule the task we woke last (assuming it failed - * wakeup-preemption), since its likely going to consume data we - * touched, increases cache locality. - */ -SCHED_FEAT(NEXT_BUDDY, 0) - -/* * Prefer to schedule the task that ran last (when we did * wake-preempt) as that likely will touch the same data, increases * cache locality. @@ -61,6 +54,10 @@ SCHED_FEAT(ASYM_EFF_LOAD, 1) * ensures the spread does not grow beyond control. */ SCHED_FEAT(DYN_MIN_VRUNTIME, 0) +/* + * Input subsystem next buddy affinity. Not transitive across new task wakeups. + */ +SCHED_FEAT(INTERACTIVE, 0) /* * Spin-wait on mutex acquisition when the mutex owner is running on -- To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/