1 files changed, 37 insertions, 228 deletions

diff --git a/kernel/sched/bfs.c b/kernel/sched/bfs.c
index 4168a5527..60b9ab785 100644
--- a/kernel/sched/bfs.c
+++ b/kernel/sched/bfs.c
@@ -137,7 +137,7 @@
 
 void print_scheduler_version(void)
 {
-	printk(KERN_INFO "BFS CPU scheduler v0.490 by Con Kolivas.\n");
+	printk(KERN_INFO "BFS CPU scheduler v0.497 by Con Kolivas.\n");
 }
 
 /*
@@ -204,7 +204,7 @@ struct global_rq {
 	int iso_ticks;
 	bool iso_refractory;
 
-	skiplist_node *node;
+	skiplist_node node;
 	skiplist *sl;
 };
 
@@ -546,7 +546,7 @@ static inline bool deadline_after(u64 deadline, u64 time)
  */
 static inline bool task_queued(struct task_struct *p)
 {
-	return p->node;
+	return !skiplist_node_empty(&p->node);
 }
 
 /*
@@ -557,8 +557,7 @@ static inline bool task_queued(struct task_struct *p)
  */
 static void dequeue_task(struct task_struct *p)
 {
-	skiplist_delnode(grq.node, grq.sl, p->node);
-	p->node = NULL;
+	skiplist_delete(grq.sl, &p->node);
 	sched_info_dequeued(task_rq(p), p);
 }
 
@@ -622,7 +621,7 @@ static void enqueue_task(struct task_struct *p, struct rq *rq)
 	 * so mask out ~microseconds as the random seed for skiplist insertion.
 	 */
 	randseed = (grq.niffies >> 10) & 0xFFFFFFFF;
-	p->node = skiplist_insert(grq.node, grq.sl, sl_id, p, randseed);
+	skiplist_insert(grq.sl, &p->node, sl_id, p, randseed);
 	sched_info_queued(rq, p);
 }
 
@@ -693,7 +692,7 @@ static int best_smt_bias(struct rq *this_rq)
 
 		if (rq_idle(rq))
 			continue;
-		if (!rq->online)
+		if (unlikely(!rq->online))
 			continue;
 		if (!rq->rq_mm)
 			continue;
@@ -818,10 +817,7 @@ static bool suitable_idle_cpus(struct task_struct *p)
 #define CPUIDLE_CACHE_BUSY	(4)
 #define CPUIDLE_DIFF_CPU	(8)
 #define CPUIDLE_THREAD_BUSY	(16)
-#define CPUIDLE_THROTTLED	(32)
-#define CPUIDLE_DIFF_NODE	(64)
-
-static inline bool scaling_rq(struct rq *rq);
+#define CPUIDLE_DIFF_NODE	(32)
 
 /*
  * The best idle CPU is chosen according to the CPUIDLE ranking above where the
@@ -840,9 +836,9 @@ static inline bool scaling_rq(struct rq *rq);
  */
 static int best_mask_cpu(int best_cpu, struct rq *rq, cpumask_t *tmpmask)
 {
-	int best_ranking = CPUIDLE_DIFF_NODE | CPUIDLE_THROTTLED |
-		CPUIDLE_THREAD_BUSY | CPUIDLE_DIFF_CPU | CPUIDLE_CACHE_BUSY |
-		CPUIDLE_DIFF_CORE | CPUIDLE_DIFF_THREAD;
+	int best_ranking = CPUIDLE_DIFF_NODE | CPUIDLE_THREAD_BUSY |
+		CPUIDLE_DIFF_CPU | CPUIDLE_CACHE_BUSY | CPUIDLE_DIFF_CORE |
+		CPUIDLE_DIFF_THREAD;
 	int cpu_tmp;
 
 	if (cpumask_test_cpu(best_cpu, tmpmask))
@@ -875,9 +871,6 @@ static int best_mask_cpu(int best_cpu, struct rq *rq, cpumask_t *tmpmask)
 		if (!(tmp_rq->siblings_idle(tmp_rq)))
 			ranking |= CPUIDLE_THREAD_BUSY;
 #endif
-		if (scaling_rq(tmp_rq))
-			ranking |= CPUIDLE_THROTTLED;
-
 		if (ranking < best_ranking) {
 			best_cpu = cpu_tmp;
 			best_ranking = ranking;
@@ -887,12 +880,6 @@ out:
 	return best_cpu;
 }
 
-static void resched_best_mask(int best_cpu, struct rq *rq, cpumask_t *tmpmask)
-{
-	best_cpu = best_mask_cpu(best_cpu, rq, tmpmask);
-	resched_curr(cpu_rq(best_cpu));
-}
-
 bool cpus_share_cache(int this_cpu, int that_cpu)
 {
 	struct rq *this_rq = cpu_rq(this_cpu);
@@ -920,25 +907,6 @@ static inline void resched_suitable_idle(struct task_struct *p)
 	if (suitable_idle_cpus(p))
 		resched_best_idle(p);
 }
-/*
- * Flags to tell us whether this CPU is running a CPU frequency governor that
- * has slowed its speed or not. No locking required as the very rare wrongly
- * read value would be harmless.
- */
-void cpu_scaling(int cpu)
-{
-	cpu_rq(cpu)->scaling = true;
-}
-
-void cpu_nonscaling(int cpu)
-{
-	cpu_rq(cpu)->scaling = false;
-}
-
-static inline bool scaling_rq(struct rq *rq)
-{
-	return rq->scaling;
-}
 
 static inline int locality_diff(int cpu, struct rq *rq)
 {
@@ -962,30 +930,11 @@ static inline void resched_suitable_idle(struct task_struct *p)
 {
 }
 
-void cpu_scaling(int __unused)
-{
-}
-
-void cpu_nonscaling(int __unused)
-{
-}
-
-/*
- * Although CPUs can scale in UP, there is nowhere else for tasks to go so this
- * always returns 0.
- */
-static inline bool scaling_rq(struct rq *rq)
-{
-	return false;
-}
-
 static inline int locality_diff(int cpu, struct rq *rq)
 {
 	return 0;
 }
 #endif /* CONFIG_SMP */
-EXPORT_SYMBOL_GPL(cpu_scaling);
-EXPORT_SYMBOL_GPL(cpu_nonscaling);
 
 static inline int normal_prio(struct task_struct *p)
 {
@@ -1018,8 +967,9 @@ static int effective_prio(struct task_struct *p)
 }
 
 /*
- * Update the load average for feeding into cpu frequency governors. Use a rolling
- * average with ~ time constant of 32ms
+ * Update the load average for feeding into cpu frequency governors. Use a
+ * rough estimate of a rolling average with ~ time constant of 32ms.
+ * 80/128 ~ 0.63. * 80 / 32768 / 128 == * 5 / 262144
  */
 static void update_load_avg(struct rq *rq)
 {
@@ -1028,10 +978,10 @@ static void update_load_avg(struct rq *rq)
 		unsigned long us_interval = (rq->clock - rq->load_update) >> 10;
 		long load;
 
-		load = rq->load_avg - (rq->load_avg * us_interval * 80 / 32768 / 128);
+		load = rq->load_avg - (rq->load_avg * us_interval * 5 / 262144);
 		if (unlikely(load < 0))
 			load = 0;
-		load += rq->soft_affined * rq_load_avg(rq) * us_interval * 80 / 32768 / 128;
+		load += rq->soft_affined * rq_load_avg(rq) * us_interval * 5 / 262144;
 		rq->load_avg = load;
 	}
 	rq->load_update = rq->clock;
@@ -1067,8 +1017,6 @@ static void activate_task(struct task_struct *p, struct rq *rq)
 	cpufreq_trigger(grq.niffies, rq->load_avg);
 }
 
-static inline void clear_sticky(struct task_struct *p);
-
 /*
  * deactivate_task - If it's running, it's not on the grq and we can just
  * decrement the nr_running. Enter with grq locked.
@@ -1080,7 +1028,6 @@ static inline void deactivate_task(struct task_struct *p, struct rq *rq)
 	rq->soft_affined--;
 	p->on_rq = 0;
 	grq.nr_running--;
-	clear_sticky(p);
 	update_load_avg(rq);
 	cpufreq_trigger(grq.niffies, rq->load_avg);
 }
@@ -1126,83 +1073,7 @@ void set_task_cpu(struct task_struct *p, unsigned int cpu)
 	}
 	task_thread_info(p)->cpu = cpu;
 }
-
-static inline void clear_sticky(struct task_struct *p)
-{
-	p->sticky = false;
-}
-
-static inline bool task_sticky(struct task_struct *p)
-{
-	return p->sticky;
-}
-
-/* Reschedule the best idle CPU that is not this one. */
-static void
-resched_closest_idle(struct rq *rq, int cpu, struct task_struct *p)
-{
-	cpumask_t tmpmask;
-
-	cpumask_and(&tmpmask, &p->cpus_allowed, &grq.cpu_idle_map);
-	cpumask_clear_cpu(cpu, &tmpmask);
-	if (cpumask_empty(&tmpmask))
-		return;
-	resched_best_mask(cpu, rq, &tmpmask);
-}
-
-/*
- * We set the sticky flag on a task that is descheduled involuntarily meaning
- * it is awaiting further CPU time. If the last sticky task is still sticky
- * but unlucky enough to not be the next task scheduled, we unstick it and try
- * to find it an idle CPU. Realtime tasks do not stick to minimise their
- * latency at all times.
- */
-static inline void
-swap_sticky(struct rq *rq, int cpu, struct task_struct *p)
-{
-	if (rq->sticky_task) {
-		if (rq->sticky_task == p) {
-			p->sticky = true;
-			return;
-		}
-		if (task_sticky(rq->sticky_task)) {
-			clear_sticky(rq->sticky_task);
-			resched_closest_idle(rq, cpu, rq->sticky_task);
-		}
-	}
-	if (!rt_task(p)) {
-		p->sticky = true;
-		rq->sticky_task = p;
-	} else {
-		resched_closest_idle(rq, cpu, p);
-		rq->sticky_task = NULL;
-	}
-}
-
-static inline void unstick_task(struct rq *rq, struct task_struct *p)
-{
-	rq->sticky_task = NULL;
-	clear_sticky(p);
-}
-#else
-static inline void clear_sticky(struct task_struct *p)
-{
-}
-
-static inline bool task_sticky(struct task_struct *p)
-{
-	return false;
-}
-
-static inline void
-swap_sticky(struct rq *rq, int cpu, struct task_struct *p)
-{
-}
-
-static inline void unstick_task(struct rq *rq, struct task_struct *p)
-{
-}
-#endif
+#endif /* CONFIG_SMP */
 
 /*
  * Move a task off the global queue and take it to a cpu for it will
@@ -1212,7 +1083,6 @@ static inline void take_task(int cpu, struct task_struct *p)
 {
 	set_task_cpu(p, cpu);
 	dequeue_task(p);
-	clear_sticky(p);
 	dec_qnr();
 }
 
@@ -1458,17 +1328,10 @@ static inline bool needs_other_cpu(struct task_struct *p, int cpu)
 static void try_preempt(struct task_struct *p, struct rq *this_rq)
 {
 	int cpu, pcpu, highest_prio, highest_cpu;
-	struct rq *highest_prio_rq = NULL;
+	struct rq *highest_prio_rq;
 	u64 latest_deadline;
 	cpumask_t tmp;
 
-	/*
-	 * We clear the sticky flag here because for a task to have called
-	 * try_preempt with the sticky flag enabled means some complicated
-	 * re-scheduling has occurred and we should ignore the sticky flag.
-	 */
-	clear_sticky(p);
-
 	if (suitable_idle_cpus(p) && resched_best_idle(p))
 		return;
 
@@ -1476,10 +1339,7 @@ static void try_preempt(struct task_struct *p, struct rq *this_rq)
 	if (p->policy == SCHED_IDLEPRIO)
 		return;
 
-	if (likely(online_cpus(p)))
-		cpumask_and(&tmp, &cpu_online_map, &p->cpus_allowed);
-	else
-		return;
+	cpumask_and(&tmp, &cpu_online_map, &p->cpus_allowed);
 
 	/* See if this task can preempt the task on the current CPU first. */
 	pcpu = cpu_of(this_rq);
@@ -1492,6 +1352,7 @@ static void try_preempt(struct task_struct *p, struct rq *this_rq)
 	}
 
 	highest_prio = latest_deadline = 0;
+	highest_prio_rq = NULL;
 
 	/* Now look for the CPU with the latest deadline */
 	for_each_cpu(cpu, &tmp) {
@@ -1522,13 +1383,10 @@ static void try_preempt(struct task_struct *p, struct rq *this_rq)
 		return;
 	if (can_preempt(p, highest_prio, latest_deadline)) {
 		/*
-			* If we have decided this task should preempt this CPU,
-			* set the task's CPU to match so there is no discrepancy
-			* in earliest_deadline_task which biases away tasks with
-			* a different CPU set. This means waking tasks are
-			* treated differently to rescheduling tasks in
-			* interactive mode.
-			*/
+		 * If we have decided this task should preempt this CPU,
+		 * set the task's CPU to match thereby speeding up matching
+		 * this task in earliest_deadline_task.
+		 */
 		set_task_cpu(p, highest_cpu);
 		resched_curr(highest_prio_rq);
 	}
@@ -1803,6 +1661,7 @@ int sched_fork(unsigned long __maybe_unused clone_flags, struct task_struct *p)
 	p->sched_time =
 	p->stime_pc =
 	p->utime_pc = 0;
+	skiplist_node_init(&p->node);
 
 	/*
 	 * Revert to default priority/policy on fork if requested.
@@ -1825,13 +1684,11 @@ int sched_fork(unsigned long __maybe_unused clone_flags, struct task_struct *p)
 		p->sched_reset_on_fork = 0;
 	}
 
-	p->node = NULL;
 #ifdef CONFIG_SCHED_INFO
 	if (unlikely(sched_info_on()))
 		memset(&p->sched_info, 0, sizeof(p->sched_info));
 #endif
 	p->on_cpu = false;
-	clear_sticky(p);
 	init_task_preempt_count(p);
 	return 0;
 }
@@ -3394,10 +3251,10 @@ static inline struct
 task_struct *earliest_deadline_task(struct rq *rq, int cpu, struct task_struct *idle)
 {
 	struct task_struct *edt = idle;
-	skiplist_node *node = grq.node;
+	skiplist_node *node = &grq.node;
 	u64 earliest_deadline = ~0ULL;
 
-	while ((node = node->next[0]) != grq.node) {
+	while ((node = node->next[0]) != &grq.node) {
 		struct task_struct *p = node->value;
 		int tcpu;
 
@@ -3409,11 +3266,8 @@ task_struct *earliest_deadline_task(struct rq *rq, int cpu, struct task_struct *
 			continue;
 
 		if (!sched_interactive && (tcpu = task_cpu(p)) != cpu) {
-			u64 dl;
+			u64 dl = p->deadline << locality_diff(tcpu, rq);
 
-			if (task_sticky(p) && scaling_rq(rq))
-				continue;
-			dl = p->deadline << locality_diff(tcpu, rq);
 			if (unlikely(!deadline_before(dl, earliest_deadline)))
 				continue;
 			earliest_deadline = dl;
@@ -3531,7 +3385,7 @@ static void check_smt_siblings(struct rq *this_rq)
 		rq = cpu_rq(other_cpu);
 		if (rq_idle(rq))
 			continue;
-		if (!rq->online)
+		if (unlikely(!rq->online))
 			continue;
 		p = rq->curr;
 		if (!smt_should_schedule(p, this_rq)) {
@@ -3552,7 +3406,7 @@ static void wake_smt_siblings(struct rq *this_rq)
 		struct rq *rq;
 
 		rq = cpu_rq(other_cpu);
-		if (!rq->online)
+		if (unlikely(!rq->online))
 			continue;
 		if (rq_idle(rq)) {
 			struct task_struct *p = rq->curr;
@@ -3689,25 +3543,6 @@ static void __sched notrace __schedule(bool preempt)
 		prev->deadline = rq->rq_deadline;
 		check_deadline(prev);
 		prev->last_ran = rq->clock_task;
-
-		/* Task changed affinity off this CPU */
-		if (likely(!needs_other_cpu(prev, cpu))) {
-			if (!deactivate) {
-				if (!queued_notrunning()) {
-					/*
-					 * We now know prev is the only thing that is
-					 * awaiting CPU so we can bypass rechecking for
-					 * the earliest deadline task and just run it
-					 * again.
-					 */
-					set_rq_task(rq, prev);
-					check_siblings(rq);
-					grq_unlock_irq();
-					goto rerun_prev_unlocked;
-				} else
-					swap_sticky(rq, cpu, prev);
-			}
-		}
 		return_task(prev, rq, deactivate);
 	}
 
@@ -3733,12 +3568,6 @@ static void __sched notrace __schedule(bool preempt)
 		 */
 		if (prev != idle && !deactivate)
 			resched_suitable_idle(prev);
-		/*
-		 * Don't stick tasks when a real time task is going to run as
-		 * they may literally get stuck.
-		 */
-		if (rt_task(next))
-			unstick_task(rq, prev);
 		set_rq_task(rq, next);
 		if (next != idle)
 			check_siblings(rq);
@@ -3758,9 +3587,6 @@ static void __sched notrace __schedule(bool preempt)
 		check_siblings(rq);
 		grq_unlock_irq();
 	}
-
-rerun_prev_unlocked:
-	return;
 }
 
 static inline void sched_submit_work(struct task_struct *tsk)
@@ -5681,7 +5507,6 @@ static void bind_zero(int src_cpu)
 			p->zerobound = true;
 			bound++;
 		}
-		clear_sticky(p);
 	} while_each_thread(t, p);
 
 	if (bound) {
@@ -5956,8 +5781,11 @@ static void set_rq_online(struct rq *rq)
 static void set_rq_offline(struct rq *rq)
 {
 	if (rq->online) {
-		cpumask_clear_cpu(cpu_of(rq), rq->rd->online);
+		int cpu = cpu_of(rq);
+
+		cpumask_clear_cpu(cpu, rq->rd->online);
 		rq->online = false;
+		clear_cpuidle_map(cpu);
 	}
 }
 
@@ -7347,8 +7175,6 @@ LIST_HEAD(task_groups);
 
 /* Cacheline aligned slab cache for task_group */
 static struct kmem_cache *task_group_cache __read_mostly;
-/* task_group_lock serializes the addition/removal of task groups */
-static DEFINE_SPINLOCK(task_group_lock);
 #endif /* CONFIG_CGROUP_SCHED */
 
 void __init sched_init(void)
@@ -7371,8 +7197,9 @@ void __init sched_init(void)
 	grq.iso_ticks = 0;
 	grq.iso_refractory = false;
 	grq.noc = 1;
-	grq.node = skiplist_init();
-	grq.sl = new_skiplist(grq.node);
+	skiplist_init(&grq.node);
+	grq.sl = new_skiplist(&grq.node);
+	skiplist_node_init(&init_task.node);
 
 #ifdef CONFIG_SMP
 	init_defrootdomain();
@@ -7396,7 +7223,6 @@ void __init sched_init(void)
 			      rq->iowait_pc = rq->idle_pc = 0;
 		rq->dither = false;
 #ifdef CONFIG_SMP
-		rq->sticky_task = NULL;
 		rq->last_niffy = 0;
 		rq->sd = NULL;
 		rq->rd = NULL;
@@ -7835,17 +7661,6 @@ struct task_group *sched_create_group(struct task_group *parent)
 
 void sched_online_group(struct task_group *tg, struct task_group *parent)
 {
-	unsigned long flags;
-
-	spin_lock_irqsave(&task_group_lock, flags);
-	list_add_rcu(&tg->list, &task_groups);
-
-	WARN_ON(!parent); /* root should already exist */
-
-	tg->parent = parent;
-	INIT_LIST_HEAD(&tg->children);
-	list_add_rcu(&tg->siblings, &parent->children);
-	spin_unlock_irqrestore(&task_group_lock, flags);
 }
 
 /* rcu callback to free various structures associated with a task group */
@@ -7863,12 +7678,6 @@ void sched_destroy_group(struct task_group *tg)
 
 void sched_offline_group(struct task_group *tg)
 {
-	unsigned long flags;
-
-	spin_lock_irqsave(&task_group_lock, flags);
-	list_del_rcu(&tg->list);
-	list_del_rcu(&tg->siblings);
-	spin_unlock_irqrestore(&task_group_lock, flags);
 }
 
 static inline struct task_group *css_tg(struct cgroup_subsys_state *css)
@@ -7916,7 +7725,7 @@ static void cpu_cgroup_fork(struct task_struct *task)
 
 static int cpu_cgroup_can_attach(struct cgroup_taskset *tset)
 {
-		return 0;
+	return 0;
 }
 
 static void cpu_cgroup_attach(struct cgroup_taskset *tset)