diff options
Diffstat (limited to 'kernel/sched')
| -rw-r--r-- | kernel/sched/bfs.c | 58 | ||||
| -rw-r--r-- | kernel/sched/bfs_sched.h | 13 | ||||
| -rw-r--r-- | kernel/sched/core.c | 22 |
3 files changed, 33 insertions, 60 deletions
diff --git a/kernel/sched/bfs.c b/kernel/sched/bfs.c index 60b9ab785..67f93e752 100644 --- a/kernel/sched/bfs.c +++ b/kernel/sched/bfs.c @@ -35,7 +35,7 @@ #include <linux/init.h> #include <asm/uaccess.h> #include <linux/highmem.h> -#include <asm/mmu_context.h> +#include <linux/mmu_context.h> #include <linux/interrupt.h> #include <linux/capability.h> #include <linux/completion.h> @@ -137,7 +137,7 @@ void print_scheduler_version(void) { - printk(KERN_INFO "BFS CPU scheduler v0.497 by Con Kolivas.\n"); + printk(KERN_INFO "BFS CPU scheduler v0.502 by Con Kolivas.\n"); } /* @@ -676,6 +676,11 @@ static inline int queued_notrunning(void) return grq.qnr; } +static unsigned long rq_load_avg(struct rq *rq) +{ + return rq->soft_affined * SCHED_CAPACITY_SCALE; +} + #ifdef CONFIG_SMT_NICE static const cpumask_t *thread_cpumask(int cpu); @@ -748,41 +753,9 @@ static bool smt_should_schedule(struct task_struct *p, struct rq *this_rq) /* Sorry, you lose */ return false; } - -static unsigned long cpu_load_avg(struct rq *rq) -{ - return rq->soft_affined * SCHED_CAPACITY_SCALE; -} - -/* - * This is the proportion of SCHED_CAPACITY_SCALE (1024) used when each thread - * of a CPU with SMT siblings is in use. - */ -#define SCHED_SMT_LOAD (890) - -/* - * Load of a CPU with smt siblings should be considered to be the load from all - * the SMT siblings, thus will be >1 if both threads are in use since they are - * not full cores. - */ -static unsigned long smt_load_avg(struct rq *rq) -{ - unsigned long load = rq->soft_affined * SCHED_SMT_LOAD; - int cpu; - - for_each_cpu(cpu, thread_cpumask(rq->cpu)) - load += cpu_rq(cpu)->soft_affined * SCHED_SMT_LOAD; - return load; -} - -static unsigned long (*rq_load_avg)(struct rq *rq) = &cpu_load_avg; -#else +#else /* CONFIG_SMT_NICE */ #define smt_schedule(p, this_rq) (true) -static inline unsigned long rq_load_avg(struct rq *rq) -{ - return rq->soft_affined * SCHED_CAPACITY_SCALE; -} -#endif +#endif /* CONFIG_SMT_NICE */ #ifdef CONFIG_SMP /* * The cpu_idle_map stores a bitmap of all the CPUs currently idle to @@ -1055,21 +1028,13 @@ void set_task_cpu(struct task_struct *p, unsigned int cpu) */ smp_wmb(); if (p->on_rq) { - struct rq *rq; + struct rq *rq = task_rq(p); - /* - * set_task_cpu can be set on other CPUs so call cpufreq_trigger - * explicitly telling it what CPU is being updated as the value - * of soft_affined has changed. - */ - rq = task_rq(p); rq->soft_affined--; update_load_avg(rq); - other_cpufreq_trigger(tcpu, grq.niffies, rq->load_avg); rq = cpu_rq(cpu); rq->soft_affined++; update_load_avg(rq); - other_cpufreq_trigger(cpu, grq.niffies, rq->load_avg); } task_thread_info(p)->cpu = cpu; } @@ -7084,7 +7049,7 @@ void __init sched_init_smp(void) /* First check if this cpu is in the same node */ for_each_domain(cpu, sd) { - if (sd->level > SD_LV_NODE) + if (sd->level > SD_LV_MC) continue; /* Set locality to local node if not already found lower */ for_each_cpu(other_cpu, sched_domain_span(sd)) { @@ -7124,7 +7089,6 @@ void __init sched_init_smp(void) check_siblings = &check_smt_siblings; wake_siblings = &wake_smt_siblings; smt_schedule = &smt_should_schedule; - rq_load_avg = &smt_load_avg; } #endif grq_unlock_irq(); diff --git a/kernel/sched/bfs_sched.h b/kernel/sched/bfs_sched.h index d4a8be9be..e7fe1d0a5 100644 --- a/kernel/sched/bfs_sched.h +++ b/kernel/sched/bfs_sched.h @@ -207,23 +207,10 @@ static inline void cpufreq_trigger(u64 time, unsigned long util) if (data) data->func(data, time, util, SCHED_CAPACITY_SCALE); } - -static inline void other_cpufreq_trigger(int cpu, u64 time, unsigned long util) -{ - struct update_util_data *data; - - data = rcu_dereference_sched(*per_cpu_ptr(&cpufreq_update_util_data, cpu)); - if (data) - data->func(data, time, util, SCHED_CAPACITY_SCALE); -} #else static inline void cpufreq_trigger(u64 time, unsigned long util) { } - -static inline void other_cpufreq_trigger(int cpu, u64 time, unsigned long util) -{ -} #endif /* CONFIG_CPU_FREQ */ #ifdef arch_scale_freq_capacity diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 97ee9ac7e..38eacc323 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -2015,6 +2015,28 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) success = 1; /* we're going to change ->state */ cpu = task_cpu(p); + /* + * Ensure we load p->on_rq _after_ p->state, otherwise it would + * be possible to, falsely, observe p->on_rq == 0 and get stuck + * in smp_cond_load_acquire() below. + * + * sched_ttwu_pending() try_to_wake_up() + * [S] p->on_rq = 1; [L] P->state + * UNLOCK rq->lock -----. + * \ + * +--- RMB + * schedule() / + * LOCK rq->lock -----' + * UNLOCK rq->lock + * + * [task p] + * [S] p->state = UNINTERRUPTIBLE [L] p->on_rq + * + * Pairs with the UNLOCK+LOCK on rq->lock from the + * last wakeup of our task and the schedule that got our task + * current. + */ + smp_rmb(); if (p->on_rq && ttwu_remote(p, wake_flags)) goto stat; |