diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/capability.c | 46 | ||||
| -rw-r--r-- | kernel/cred.c | 2 | ||||
| -rw-r--r-- | kernel/fork.c | 4 | ||||
| -rw-r--r-- | kernel/sched/bfs.c | 265 | ||||
| -rw-r--r-- | kernel/sched/bfs_sched.h | 2 | ||||
| -rw-r--r-- | kernel/skip_lists.c | 49 | ||||
| -rw-r--r-- | kernel/time/timekeeping.c | 5 | ||||
| -rw-r--r-- | kernel/time/timekeeping_debug.c | 9 |
8 files changed, 111 insertions, 271 deletions
diff --git a/kernel/capability.c b/kernel/capability.c index 45432b54d..00411c82d 100644 --- a/kernel/capability.c +++ b/kernel/capability.c @@ -361,6 +361,24 @@ bool has_capability_noaudit(struct task_struct *t, int cap) return has_ns_capability_noaudit(t, &init_user_ns, cap); } +static bool ns_capable_common(struct user_namespace *ns, int cap, bool audit) +{ + int capable; + + if (unlikely(!cap_valid(cap))) { + pr_crit("capable() called with invalid cap=%u\n", cap); + BUG(); + } + + capable = audit ? security_capable(current_cred(), ns, cap) : + security_capable_noaudit(current_cred(), ns, cap); + if (capable == 0) { + current->flags |= PF_SUPERPRIV; + return true; + } + return false; +} + /** * ns_capable - Determine if the current task has a superior capability in effect * @ns: The usernamespace we want the capability in @@ -374,19 +392,27 @@ bool has_capability_noaudit(struct task_struct *t, int cap) */ bool ns_capable(struct user_namespace *ns, int cap) { - if (unlikely(!cap_valid(cap))) { - pr_crit("capable() called with invalid cap=%u\n", cap); - BUG(); - } - - if (security_capable(current_cred(), ns, cap) == 0) { - current->flags |= PF_SUPERPRIV; - return true; - } - return false; + return ns_capable_common(ns, cap, true); } EXPORT_SYMBOL(ns_capable); +/** + * ns_capable_noaudit - Determine if the current task has a superior capability + * (unaudited) in effect + * @ns: The usernamespace we want the capability in + * @cap: The capability to be tested for + * + * Return true if the current task has the given superior capability currently + * available for use, false if not. + * + * This sets PF_SUPERPRIV on the task if the capability is available on the + * assumption that it's about to be used. + */ +bool ns_capable_noaudit(struct user_namespace *ns, int cap) +{ + return ns_capable_common(ns, cap, false); +} +EXPORT_SYMBOL(ns_capable_noaudit); /** * capable - Determine if the current task has a superior capability in effect diff --git a/kernel/cred.c b/kernel/cred.c index 0c0cd8a62..5f264fb57 100644 --- a/kernel/cred.c +++ b/kernel/cred.c @@ -689,6 +689,8 @@ EXPORT_SYMBOL(set_security_override_from_ctx); */ int set_create_files_as(struct cred *new, struct inode *inode) { + if (!uid_valid(inode->i_uid) || !gid_valid(inode->i_gid)) + return -EINVAL; new->fsuid = inode->i_uid; new->fsgid = inode->i_gid; return security_kernel_create_files_as(new, inode); diff --git a/kernel/fork.c b/kernel/fork.c index 9c067f88a..767d95257 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1406,7 +1406,6 @@ static struct task_struct *copy_process(unsigned long clone_flags, p->real_start_time = ktime_get_boot_ns(); p->io_context = NULL; p->audit_context = NULL; - threadgroup_change_begin(current); cgroup_fork(p); #ifdef CONFIG_NUMA p->mempolicy = mpol_dup(p->mempolicy); @@ -1558,6 +1557,7 @@ static struct task_struct *copy_process(unsigned long clone_flags, INIT_LIST_HEAD(&p->thread_group); p->task_works = NULL; + threadgroup_change_begin(current); /* * Ensure that the cgroup subsystem policies allow the new process to be * forked. It should be noted the the new process's css_set can be changed @@ -1658,6 +1658,7 @@ static struct task_struct *copy_process(unsigned long clone_flags, bad_fork_cancel_cgroup: cgroup_cancel_fork(p); bad_fork_free_pid: + threadgroup_change_end(current); if (pid != &init_struct_pid) free_pid(pid); bad_fork_cleanup_thread: @@ -1690,7 +1691,6 @@ bad_fork_cleanup_policy: mpol_put(p->mempolicy); bad_fork_cleanup_threadgroup_lock: #endif - threadgroup_change_end(current); delayacct_tsk_free(p); bad_fork_cleanup_count: atomic_dec(&p->cred->user->processes); diff --git a/kernel/sched/bfs.c b/kernel/sched/bfs.c index 4168a5527..60b9ab785 100644 --- a/kernel/sched/bfs.c +++ b/kernel/sched/bfs.c @@ -137,7 +137,7 @@ void print_scheduler_version(void) { - printk(KERN_INFO "BFS CPU scheduler v0.490 by Con Kolivas.\n"); + printk(KERN_INFO "BFS CPU scheduler v0.497 by Con Kolivas.\n"); } /* @@ -204,7 +204,7 @@ struct global_rq { int iso_ticks; bool iso_refractory; - skiplist_node *node; + skiplist_node node; skiplist *sl; }; @@ -546,7 +546,7 @@ static inline bool deadline_after(u64 deadline, u64 time) */ static inline bool task_queued(struct task_struct *p) { - return p->node; + return !skiplist_node_empty(&p->node); } /* @@ -557,8 +557,7 @@ static inline bool task_queued(struct task_struct *p) */ static void dequeue_task(struct task_struct *p) { - skiplist_delnode(grq.node, grq.sl, p->node); - p->node = NULL; + skiplist_delete(grq.sl, &p->node); sched_info_dequeued(task_rq(p), p); } @@ -622,7 +621,7 @@ static void enqueue_task(struct task_struct *p, struct rq *rq) * so mask out ~microseconds as the random seed for skiplist insertion. */ randseed = (grq.niffies >> 10) & 0xFFFFFFFF; - p->node = skiplist_insert(grq.node, grq.sl, sl_id, p, randseed); + skiplist_insert(grq.sl, &p->node, sl_id, p, randseed); sched_info_queued(rq, p); } @@ -693,7 +692,7 @@ static int best_smt_bias(struct rq *this_rq) if (rq_idle(rq)) continue; - if (!rq->online) + if (unlikely(!rq->online)) continue; if (!rq->rq_mm) continue; @@ -818,10 +817,7 @@ static bool suitable_idle_cpus(struct task_struct *p) #define CPUIDLE_CACHE_BUSY (4) #define CPUIDLE_DIFF_CPU (8) #define CPUIDLE_THREAD_BUSY (16) -#define CPUIDLE_THROTTLED (32) -#define CPUIDLE_DIFF_NODE (64) - -static inline bool scaling_rq(struct rq *rq); +#define CPUIDLE_DIFF_NODE (32) /* * The best idle CPU is chosen according to the CPUIDLE ranking above where the @@ -840,9 +836,9 @@ static inline bool scaling_rq(struct rq *rq); */ static int best_mask_cpu(int best_cpu, struct rq *rq, cpumask_t *tmpmask) { - int best_ranking = CPUIDLE_DIFF_NODE | CPUIDLE_THROTTLED | - CPUIDLE_THREAD_BUSY | CPUIDLE_DIFF_CPU | CPUIDLE_CACHE_BUSY | - CPUIDLE_DIFF_CORE | CPUIDLE_DIFF_THREAD; + int best_ranking = CPUIDLE_DIFF_NODE | CPUIDLE_THREAD_BUSY | + CPUIDLE_DIFF_CPU | CPUIDLE_CACHE_BUSY | CPUIDLE_DIFF_CORE | + CPUIDLE_DIFF_THREAD; int cpu_tmp; if (cpumask_test_cpu(best_cpu, tmpmask)) @@ -875,9 +871,6 @@ static int best_mask_cpu(int best_cpu, struct rq *rq, cpumask_t *tmpmask) if (!(tmp_rq->siblings_idle(tmp_rq))) ranking |= CPUIDLE_THREAD_BUSY; #endif - if (scaling_rq(tmp_rq)) - ranking |= CPUIDLE_THROTTLED; - if (ranking < best_ranking) { best_cpu = cpu_tmp; best_ranking = ranking; @@ -887,12 +880,6 @@ out: return best_cpu; } -static void resched_best_mask(int best_cpu, struct rq *rq, cpumask_t *tmpmask) -{ - best_cpu = best_mask_cpu(best_cpu, rq, tmpmask); - resched_curr(cpu_rq(best_cpu)); -} - bool cpus_share_cache(int this_cpu, int that_cpu) { struct rq *this_rq = cpu_rq(this_cpu); @@ -920,25 +907,6 @@ static inline void resched_suitable_idle(struct task_struct *p) if (suitable_idle_cpus(p)) resched_best_idle(p); } -/* - * Flags to tell us whether this CPU is running a CPU frequency governor that - * has slowed its speed or not. No locking required as the very rare wrongly - * read value would be harmless. - */ -void cpu_scaling(int cpu) -{ - cpu_rq(cpu)->scaling = true; -} - -void cpu_nonscaling(int cpu) -{ - cpu_rq(cpu)->scaling = false; -} - -static inline bool scaling_rq(struct rq *rq) -{ - return rq->scaling; -} static inline int locality_diff(int cpu, struct rq *rq) { @@ -962,30 +930,11 @@ static inline void resched_suitable_idle(struct task_struct *p) { } -void cpu_scaling(int __unused) -{ -} - -void cpu_nonscaling(int __unused) -{ -} - -/* - * Although CPUs can scale in UP, there is nowhere else for tasks to go so this - * always returns 0. - */ -static inline bool scaling_rq(struct rq *rq) -{ - return false; -} - static inline int locality_diff(int cpu, struct rq *rq) { return 0; } #endif /* CONFIG_SMP */ -EXPORT_SYMBOL_GPL(cpu_scaling); -EXPORT_SYMBOL_GPL(cpu_nonscaling); static inline int normal_prio(struct task_struct *p) { @@ -1018,8 +967,9 @@ static int effective_prio(struct task_struct *p) } /* - * Update the load average for feeding into cpu frequency governors. Use a rolling - * average with ~ time constant of 32ms + * Update the load average for feeding into cpu frequency governors. Use a + * rough estimate of a rolling average with ~ time constant of 32ms. + * 80/128 ~ 0.63. * 80 / 32768 / 128 == * 5 / 262144 */ static void update_load_avg(struct rq *rq) { @@ -1028,10 +978,10 @@ static void update_load_avg(struct rq *rq) unsigned long us_interval = (rq->clock - rq->load_update) >> 10; long load; - load = rq->load_avg - (rq->load_avg * us_interval * 80 / 32768 / 128); + load = rq->load_avg - (rq->load_avg * us_interval * 5 / 262144); if (unlikely(load < 0)) load = 0; - load += rq->soft_affined * rq_load_avg(rq) * us_interval * 80 / 32768 / 128; + load += rq->soft_affined * rq_load_avg(rq) * us_interval * 5 / 262144; rq->load_avg = load; } rq->load_update = rq->clock; @@ -1067,8 +1017,6 @@ static void activate_task(struct task_struct *p, struct rq *rq) cpufreq_trigger(grq.niffies, rq->load_avg); } -static inline void clear_sticky(struct task_struct *p); - /* * deactivate_task - If it's running, it's not on the grq and we can just * decrement the nr_running. Enter with grq locked. @@ -1080,7 +1028,6 @@ static inline void deactivate_task(struct task_struct *p, struct rq *rq) rq->soft_affined--; p->on_rq = 0; grq.nr_running--; - clear_sticky(p); update_load_avg(rq); cpufreq_trigger(grq.niffies, rq->load_avg); } @@ -1126,83 +1073,7 @@ void set_task_cpu(struct task_struct *p, unsigned int cpu) } task_thread_info(p)->cpu = cpu; } - -static inline void clear_sticky(struct task_struct *p) -{ - p->sticky = false; -} - -static inline bool task_sticky(struct task_struct *p) -{ - return p->sticky; -} - -/* Reschedule the best idle CPU that is not this one. */ -static void -resched_closest_idle(struct rq *rq, int cpu, struct task_struct *p) -{ - cpumask_t tmpmask; - - cpumask_and(&tmpmask, &p->cpus_allowed, &grq.cpu_idle_map); - cpumask_clear_cpu(cpu, &tmpmask); - if (cpumask_empty(&tmpmask)) - return; - resched_best_mask(cpu, rq, &tmpmask); -} - -/* - * We set the sticky flag on a task that is descheduled involuntarily meaning - * it is awaiting further CPU time. If the last sticky task is still sticky - * but unlucky enough to not be the next task scheduled, we unstick it and try - * to find it an idle CPU. Realtime tasks do not stick to minimise their - * latency at all times. - */ -static inline void -swap_sticky(struct rq *rq, int cpu, struct task_struct *p) -{ - if (rq->sticky_task) { - if (rq->sticky_task == p) { - p->sticky = true; - return; - } - if (task_sticky(rq->sticky_task)) { - clear_sticky(rq->sticky_task); - resched_closest_idle(rq, cpu, rq->sticky_task); - } - } - if (!rt_task(p)) { - p->sticky = true; - rq->sticky_task = p; - } else { - resched_closest_idle(rq, cpu, p); - rq->sticky_task = NULL; - } -} - -static inline void unstick_task(struct rq *rq, struct task_struct *p) -{ - rq->sticky_task = NULL; - clear_sticky(p); -} -#else -static inline void clear_sticky(struct task_struct *p) -{ -} - -static inline bool task_sticky(struct task_struct *p) -{ - return false; -} - -static inline void -swap_sticky(struct rq *rq, int cpu, struct task_struct *p) -{ -} - -static inline void unstick_task(struct rq *rq, struct task_struct *p) -{ -} -#endif +#endif /* CONFIG_SMP */ /* * Move a task off the global queue and take it to a cpu for it will @@ -1212,7 +1083,6 @@ static inline void take_task(int cpu, struct task_struct *p) { set_task_cpu(p, cpu); dequeue_task(p); - clear_sticky(p); dec_qnr(); } @@ -1458,17 +1328,10 @@ static inline bool needs_other_cpu(struct task_struct *p, int cpu) static void try_preempt(struct task_struct *p, struct rq *this_rq) { int cpu, pcpu, highest_prio, highest_cpu; - struct rq *highest_prio_rq = NULL; + struct rq *highest_prio_rq; u64 latest_deadline; cpumask_t tmp; - /* - * We clear the sticky flag here because for a task to have called - * try_preempt with the sticky flag enabled means some complicated - * re-scheduling has occurred and we should ignore the sticky flag. - */ - clear_sticky(p); - if (suitable_idle_cpus(p) && resched_best_idle(p)) return; @@ -1476,10 +1339,7 @@ static void try_preempt(struct task_struct *p, struct rq *this_rq) if (p->policy == SCHED_IDLEPRIO) return; - if (likely(online_cpus(p))) - cpumask_and(&tmp, &cpu_online_map, &p->cpus_allowed); - else - return; + cpumask_and(&tmp, &cpu_online_map, &p->cpus_allowed); /* See if this task can preempt the task on the current CPU first. */ pcpu = cpu_of(this_rq); @@ -1492,6 +1352,7 @@ static void try_preempt(struct task_struct *p, struct rq *this_rq) } highest_prio = latest_deadline = 0; + highest_prio_rq = NULL; /* Now look for the CPU with the latest deadline */ for_each_cpu(cpu, &tmp) { @@ -1522,13 +1383,10 @@ static void try_preempt(struct task_struct *p, struct rq *this_rq) return; if (can_preempt(p, highest_prio, latest_deadline)) { /* - * If we have decided this task should preempt this CPU, - * set the task's CPU to match so there is no discrepancy - * in earliest_deadline_task which biases away tasks with - * a different CPU set. This means waking tasks are - * treated differently to rescheduling tasks in - * interactive mode. - */ + * If we have decided this task should preempt this CPU, + * set the task's CPU to match thereby speeding up matching + * this task in earliest_deadline_task. + */ set_task_cpu(p, highest_cpu); resched_curr(highest_prio_rq); } @@ -1803,6 +1661,7 @@ int sched_fork(unsigned long __maybe_unused clone_flags, struct task_struct *p) p->sched_time = p->stime_pc = p->utime_pc = 0; + skiplist_node_init(&p->node); /* * Revert to default priority/policy on fork if requested. @@ -1825,13 +1684,11 @@ int sched_fork(unsigned long __maybe_unused clone_flags, struct task_struct *p) p->sched_reset_on_fork = 0; } - p->node = NULL; #ifdef CONFIG_SCHED_INFO if (unlikely(sched_info_on())) memset(&p->sched_info, 0, sizeof(p->sched_info)); #endif p->on_cpu = false; - clear_sticky(p); init_task_preempt_count(p); return 0; } @@ -3394,10 +3251,10 @@ static inline struct task_struct *earliest_deadline_task(struct rq *rq, int cpu, struct task_struct *idle) { struct task_struct *edt = idle; - skiplist_node *node = grq.node; + skiplist_node *node = &grq.node; u64 earliest_deadline = ~0ULL; - while ((node = node->next[0]) != grq.node) { + while ((node = node->next[0]) != &grq.node) { struct task_struct *p = node->value; int tcpu; @@ -3409,11 +3266,8 @@ task_struct *earliest_deadline_task(struct rq *rq, int cpu, struct task_struct * continue; if (!sched_interactive && (tcpu = task_cpu(p)) != cpu) { - u64 dl; + u64 dl = p->deadline << locality_diff(tcpu, rq); - if (task_sticky(p) && scaling_rq(rq)) - continue; - dl = p->deadline << locality_diff(tcpu, rq); if (unlikely(!deadline_before(dl, earliest_deadline))) continue; earliest_deadline = dl; @@ -3531,7 +3385,7 @@ static void check_smt_siblings(struct rq *this_rq) rq = cpu_rq(other_cpu); if (rq_idle(rq)) continue; - if (!rq->online) + if (unlikely(!rq->online)) continue; p = rq->curr; if (!smt_should_schedule(p, this_rq)) { @@ -3552,7 +3406,7 @@ static void wake_smt_siblings(struct rq *this_rq) struct rq *rq; rq = cpu_rq(other_cpu); - if (!rq->online) + if (unlikely(!rq->online)) continue; if (rq_idle(rq)) { struct task_struct *p = rq->curr; @@ -3689,25 +3543,6 @@ static void __sched notrace __schedule(bool preempt) prev->deadline = rq->rq_deadline; check_deadline(prev); prev->last_ran = rq->clock_task; - - /* Task changed affinity off this CPU */ - if (likely(!needs_other_cpu(prev, cpu))) { - if (!deactivate) { - if (!queued_notrunning()) { - /* - * We now know prev is the only thing that is - * awaiting CPU so we can bypass rechecking for - * the earliest deadline task and just run it - * again. - */ - set_rq_task(rq, prev); - check_siblings(rq); - grq_unlock_irq(); - goto rerun_prev_unlocked; - } else - swap_sticky(rq, cpu, prev); - } - } return_task(prev, rq, deactivate); } @@ -3733,12 +3568,6 @@ static void __sched notrace __schedule(bool preempt) */ if (prev != idle && !deactivate) resched_suitable_idle(prev); - /* - * Don't stick tasks when a real time task is going to run as - * they may literally get stuck. - */ - if (rt_task(next)) - unstick_task(rq, prev); set_rq_task(rq, next); if (next != idle) check_siblings(rq); @@ -3758,9 +3587,6 @@ static void __sched notrace __schedule(bool preempt) check_siblings(rq); grq_unlock_irq(); } - -rerun_prev_unlocked: - return; } static inline void sched_submit_work(struct task_struct *tsk) @@ -5681,7 +5507,6 @@ static void bind_zero(int src_cpu) p->zerobound = true; bound++; } - clear_sticky(p); } while_each_thread(t, p); if (bound) { @@ -5956,8 +5781,11 @@ static void set_rq_online(struct rq *rq) static void set_rq_offline(struct rq *rq) { if (rq->online) { - cpumask_clear_cpu(cpu_of(rq), rq->rd->online); + int cpu = cpu_of(rq); + + cpumask_clear_cpu(cpu, rq->rd->online); rq->online = false; + clear_cpuidle_map(cpu); } } @@ -7347,8 +7175,6 @@ LIST_HEAD(task_groups); /* Cacheline aligned slab cache for task_group */ static struct kmem_cache *task_group_cache __read_mostly; -/* task_group_lock serializes the addition/removal of task groups */ -static DEFINE_SPINLOCK(task_group_lock); #endif /* CONFIG_CGROUP_SCHED */ void __init sched_init(void) @@ -7371,8 +7197,9 @@ void __init sched_init(void) grq.iso_ticks = 0; grq.iso_refractory = false; grq.noc = 1; - grq.node = skiplist_init(); - grq.sl = new_skiplist(grq.node); + skiplist_init(&grq.node); + grq.sl = new_skiplist(&grq.node); + skiplist_node_init(&init_task.node); #ifdef CONFIG_SMP init_defrootdomain(); @@ -7396,7 +7223,6 @@ void __init sched_init(void) rq->iowait_pc = rq->idle_pc = 0; rq->dither = false; #ifdef CONFIG_SMP - rq->sticky_task = NULL; rq->last_niffy = 0; rq->sd = NULL; rq->rd = NULL; @@ -7835,17 +7661,6 @@ struct task_group *sched_create_group(struct task_group *parent) void sched_online_group(struct task_group *tg, struct task_group *parent) { - unsigned long flags; - - spin_lock_irqsave(&task_group_lock, flags); - list_add_rcu(&tg->list, &task_groups); - - WARN_ON(!parent); /* root should already exist */ - - tg->parent = parent; - INIT_LIST_HEAD(&tg->children); - list_add_rcu(&tg->siblings, &parent->children); - spin_unlock_irqrestore(&task_group_lock, flags); } /* rcu callback to free various structures associated with a task group */ @@ -7863,12 +7678,6 @@ void sched_destroy_group(struct task_group *tg) void sched_offline_group(struct task_group *tg) { - unsigned long flags; - - spin_lock_irqsave(&task_group_lock, flags); - list_del_rcu(&tg->list); - list_del_rcu(&tg->siblings); - spin_unlock_irqrestore(&task_group_lock, flags); } static inline struct task_group *css_tg(struct cgroup_subsys_state *css) @@ -7916,7 +7725,7 @@ static void cpu_cgroup_fork(struct task_struct *task) static int cpu_cgroup_can_attach(struct cgroup_taskset *tset) { - return 0; + return 0; } static void cpu_cgroup_attach(struct cgroup_taskset *tset) diff --git a/kernel/sched/bfs_sched.h b/kernel/sched/bfs_sched.h index 06b2cdca7..d4a8be9be 100644 --- a/kernel/sched/bfs_sched.h +++ b/kernel/sched/bfs_sched.h @@ -39,8 +39,6 @@ struct rq { #ifdef CONFIG_SMP int cpu; /* cpu of this runqueue */ bool online; - bool scaling; /* This CPU is managed by a scaling CPU freq governor */ - struct task_struct *sticky_task; struct root_domain *rd; struct sched_domain *sd; diff --git a/kernel/skip_lists.c b/kernel/skip_lists.c index 4d82ee18b..40b7ba240 100644 --- a/kernel/skip_lists.c +++ b/kernel/skip_lists.c @@ -53,34 +53,28 @@ aid of prev<->next pointer manipulation and no searching. #define MaxNumberOfLevels 16 #define MaxLevel (MaxNumberOfLevels - 1) -#define newNode kmalloc(sizeof(skiplist_node), GFP_ATOMIC) -skiplist_node *skiplist_init(void) +void skiplist_init(skiplist_node *slnode) { - skiplist_node *slnode = newNode; int i; - BUG_ON(!slnode); slnode->key = 0xFFFFFFFFFFFFFFFF; slnode->level = 0; slnode->value = NULL; for (i = 0; i < MaxNumberOfLevels; i++) slnode->next[i] = slnode->prev[i] = slnode; - return slnode; } skiplist *new_skiplist(skiplist_node *slnode) { - skiplist *l = kmalloc(sizeof(skiplist), GFP_ATOMIC); + skiplist *l = kzalloc(sizeof(skiplist), GFP_ATOMIC); BUG_ON(!l); - l->entries = 0; - l->level = 0; l->header = slnode; return l; } -void free_skiplist(skiplist_node *slnode, skiplist *l) +void free_skiplist(skiplist *l) { skiplist_node *p, *q; @@ -88,12 +82,17 @@ void free_skiplist(skiplist_node *slnode, skiplist *l) do { q = p->next[0]; p->next[0]->prev[0] = q->prev[0]; - kfree(p); + skiplist_node_init(p); p = q; - } while (p != slnode); + } while (p != l->header); kfree(l); } +void skiplist_node_init(skiplist_node *node) +{ + memset(node, 0, sizeof(skiplist_node)); +} + /* * Returns a pseudo-random number based on the randseed value by masking out * 0-15. As many levels are not required when only few values are on the list, @@ -126,13 +125,13 @@ static inline unsigned int randomLevel(int entries, unsigned int randseed) return randseed & mask; } -skiplist_node *skiplist_insert(skiplist_node *slnode, skiplist *l, keyType key, valueType value, unsigned int randseed) +void skiplist_insert(skiplist *l, skiplist_node *node, keyType key, valueType value, unsigned int randseed) { skiplist_node *update[MaxNumberOfLevels]; skiplist_node *p, *q; int k = l->level; - p = slnode; + p = l->header; do { while (q = p->next[k], q->key <= key) p = q; @@ -142,24 +141,22 @@ skiplist_node *skiplist_insert(skiplist_node *slnode, skiplist *l, keyType key, k = randomLevel(++l->entries, randseed); if (k > l->level) { k = ++l->level; - update[k] = slnode; + update[k] = l->header; } - q = newNode; - q->level = k; - q->key = key; - q->value = value; + node->level = k; + node->key = key; + node->value = value; do { p = update[k]; - q->next[k] = p->next[k]; - p->next[k] = q; - q->prev[k] = p; - q->next[k]->prev[k] = q; + node->next[k] = p->next[k]; + p->next[k] = node; + node->prev[k] = p; + node->next[k]->prev[k] = node; } while (--k >= 0); - return q; } -void skiplist_delnode(skiplist_node *slnode, skiplist *l, skiplist_node *node) +void skiplist_delete(skiplist *l, skiplist_node *node) { int k, m = node->level; @@ -167,9 +164,9 @@ void skiplist_delnode(skiplist_node *slnode, skiplist *l, skiplist_node *node) node->prev[k]->next[k] = node->next[k]; node->next[k]->prev[k] = node->prev[k]; } - kfree(node); + skiplist_node_init(node); if (m == l->level) { - while (l->header->next[m] == slnode && l->header->prev[m] == slnode && m > 0) + while (l->header->next[m] == l->header && l->header->prev[m] == l->header && m > 0) m--; l->level = m; } diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 479d25cd3..b6c394563 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -401,7 +401,10 @@ static __always_inline u64 __ktime_get_fast_ns(struct tk_fast *tkf) do { seq = raw_read_seqcount_latch(&tkf->seq); tkr = tkf->base + (seq & 0x01); - now = ktime_to_ns(tkr->base) + timekeeping_get_ns(tkr); + now = ktime_to_ns(tkr->base); + + now += clocksource_delta(tkr->read(tkr->clock), + tkr->cycle_last, tkr->mask); } while (read_seqcount_retry(&tkf->seq, seq)); return now; diff --git a/kernel/time/timekeeping_debug.c b/kernel/time/timekeeping_debug.c index f6bd65236..107310a6f 100644 --- a/kernel/time/timekeeping_debug.c +++ b/kernel/time/timekeeping_debug.c @@ -23,7 +23,9 @@ #include "timekeeping_internal.h" -static unsigned int sleep_time_bin[32] = {0}; +#define NUM_BINS 32 + +static unsigned int sleep_time_bin[NUM_BINS] = {0}; static int tk_debug_show_sleep_time(struct seq_file *s, void *data) { @@ -69,6 +71,9 @@ late_initcall(tk_debug_sleep_time_init); void tk_debug_account_sleep_time(struct timespec64 *t) { - sleep_time_bin[fls(t->tv_sec)]++; + /* Cap bin index so we don't overflow the array */ + int bin = min(fls(t->tv_sec), NUM_BINS-1); + + sleep_time_bin[bin]++; } |