diff options
Diffstat (limited to 'block/bfq-sched.c')
-rw-r--r-- | block/bfq-sched.c | 209 |
1 files changed, 96 insertions, 113 deletions
diff --git a/block/bfq-sched.c b/block/bfq-sched.c index 9328a1f09..d0890c6d4 100644 --- a/block/bfq-sched.c +++ b/block/bfq-sched.c @@ -10,27 +10,24 @@ * Copyright (C) 2010 Paolo Valente <paolo.valente@unimore.it> */ -#ifdef CONFIG_BFQ_GROUP_IOSCHED +#ifdef CONFIG_CGROUP_BFQIO #define for_each_entity(entity) \ - for (; entity ; entity = entity->parent) + for (; entity != NULL; entity = entity->parent) #define for_each_entity_safe(entity, parent) \ for (; entity && ({ parent = entity->parent; 1; }); entity = parent) - static struct bfq_entity *bfq_lookup_next_entity(struct bfq_sched_data *sd, int extract, struct bfq_data *bfqd); -static struct bfq_group *bfqq_group(struct bfq_queue *bfqq); - -static void bfq_update_budget(struct bfq_entity *next_in_service) +static inline void bfq_update_budget(struct bfq_entity *next_in_service) { struct bfq_entity *bfqg_entity; struct bfq_group *bfqg; struct bfq_sched_data *group_sd; - BUG_ON(!next_in_service); + BUG_ON(next_in_service == NULL); group_sd = next_in_service->sched_data; @@ -41,7 +38,7 @@ static void bfq_update_budget(struct bfq_entity *next_in_service) * as it must never become an in-service entity. */ bfqg_entity = bfqg->my_entity; - if (bfqg_entity) + if (bfqg_entity != NULL) bfqg_entity->budget = next_in_service->budget; } @@ -49,7 +46,7 @@ static int bfq_update_next_in_service(struct bfq_sched_data *sd) { struct bfq_entity *next_in_service; - if (sd->in_service_entity) + if (sd->in_service_entity != NULL) /* will update/requeue at the end of service */ return 0; @@ -63,35 +60,35 @@ static int bfq_update_next_in_service(struct bfq_sched_data *sd) next_in_service = bfq_lookup_next_entity(sd, 0, NULL); sd->next_in_service = next_in_service; - if (next_in_service) + if (next_in_service != NULL) bfq_update_budget(next_in_service); return 1; } -static void bfq_check_next_in_service(struct bfq_sched_data *sd, - struct bfq_entity *entity) +static inline void bfq_check_next_in_service(struct bfq_sched_data *sd, + struct bfq_entity *entity) { BUG_ON(sd->next_in_service != entity); } #else #define for_each_entity(entity) \ - for (; entity ; entity = NULL) + for (; entity != NULL; entity = NULL) #define for_each_entity_safe(entity, parent) \ - for (parent = NULL; entity ; entity = parent) + for (parent = NULL; entity != NULL; entity = parent) -static int bfq_update_next_in_service(struct bfq_sched_data *sd) +static inline int bfq_update_next_in_service(struct bfq_sched_data *sd) { return 0; } -static void bfq_check_next_in_service(struct bfq_sched_data *sd, - struct bfq_entity *entity) +static inline void bfq_check_next_in_service(struct bfq_sched_data *sd, + struct bfq_entity *entity) { } -static void bfq_update_budget(struct bfq_entity *next_in_service) +static inline void bfq_update_budget(struct bfq_entity *next_in_service) { } #endif @@ -112,18 +109,18 @@ static void bfq_update_budget(struct bfq_entity *next_in_service) * * Return @a > @b, dealing with wrapping correctly. */ -static int bfq_gt(u64 a, u64 b) +static inline int bfq_gt(u64 a, u64 b) { return (s64)(a - b) > 0; } -static struct bfq_queue *bfq_entity_to_bfqq(struct bfq_entity *entity) +static inline struct bfq_queue *bfq_entity_to_bfqq(struct bfq_entity *entity) { struct bfq_queue *bfqq = NULL; - BUG_ON(!entity); + BUG_ON(entity == NULL); - if (!entity->my_sched_data) + if (entity->my_sched_data == NULL) bfqq = container_of(entity, struct bfq_queue, entity); return bfqq; @@ -135,7 +132,8 @@ static struct bfq_queue *bfq_entity_to_bfqq(struct bfq_entity *entity) * @service: amount of service. * @weight: scale factor (weight of an entity or weight sum). */ -static u64 bfq_delta(unsigned long service, unsigned long weight) +static inline u64 bfq_delta(unsigned long service, + unsigned long weight) { u64 d = (u64)service << WFQ_SERVICE_SHIFT; @@ -148,7 +146,8 @@ static u64 bfq_delta(unsigned long service, unsigned long weight) * @entity: the entity to act upon. * @service: the service to be charged to the entity. */ -static void bfq_calc_finish(struct bfq_entity *entity, unsigned long service) +static inline void bfq_calc_finish(struct bfq_entity *entity, + unsigned long service) { struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); @@ -157,7 +156,7 @@ static void bfq_calc_finish(struct bfq_entity *entity, unsigned long service) entity->finish = entity->start + bfq_delta(service, entity->weight); - if (bfqq) { + if (bfqq != NULL) { bfq_log_bfqq(bfqq->bfqd, bfqq, "calc_finish: serv %lu, w %d", service, entity->weight); @@ -177,11 +176,11 @@ static void bfq_calc_finish(struct bfq_entity *entity, unsigned long service) * conversion mechanism because, e.g., in the tree walking functions, * the check for a %NULL value would be redundant. */ -static struct bfq_entity *bfq_entity_of(struct rb_node *node) +static inline struct bfq_entity *bfq_entity_of(struct rb_node *node) { struct bfq_entity *entity = NULL; - if (node) + if (node != NULL) entity = rb_entry(node, struct bfq_entity, rb_node); return entity; @@ -192,7 +191,8 @@ static struct bfq_entity *bfq_entity_of(struct rb_node *node) * @root: the tree root. * @entity: the entity to remove. */ -static void bfq_extract(struct rb_root *root, struct bfq_entity *entity) +static inline void bfq_extract(struct rb_root *root, + struct bfq_entity *entity) { BUG_ON(entity->tree != root); @@ -225,7 +225,7 @@ static void bfq_idle_extract(struct bfq_service_tree *st, bfq_extract(&st->idle, entity); - if (bfqq) + if (bfqq != NULL) list_del(&bfqq->bfqq_list); } @@ -243,9 +243,9 @@ static void bfq_insert(struct rb_root *root, struct bfq_entity *entity) struct rb_node **node = &root->rb_node; struct rb_node *parent = NULL; - BUG_ON(entity->tree); + BUG_ON(entity->tree != NULL); - while (*node) { + while (*node != NULL) { parent = *node; entry = rb_entry(parent, struct bfq_entity, rb_node); @@ -271,11 +271,12 @@ static void bfq_insert(struct rb_root *root, struct bfq_entity *entity) * that the subtree rooted at @node (which may be its left or its right * child) has a valid min_start value. */ -static void bfq_update_min(struct bfq_entity *entity, struct rb_node *node) +static inline void bfq_update_min(struct bfq_entity *entity, + struct rb_node *node) { struct bfq_entity *child; - if (node) { + if (node != NULL) { child = rb_entry(node, struct bfq_entity, rb_node); if (bfq_gt(entity->min_start, child->min_start)) entity->min_start = child->min_start; @@ -290,7 +291,7 @@ static void bfq_update_min(struct bfq_entity *entity, struct rb_node *node) * this function updates its min_start value. The left and right subtrees * are assumed to hold a correct min_start value. */ -static void bfq_update_active_node(struct rb_node *node) +static inline void bfq_update_active_node(struct rb_node *node) { struct bfq_entity *entity = rb_entry(node, struct bfq_entity, rb_node); @@ -317,12 +318,12 @@ up: bfq_update_active_node(node); parent = rb_parent(node); - if (!parent) + if (parent == NULL) return; - if (node == parent->rb_left && parent->rb_right) + if (node == parent->rb_left && parent->rb_right != NULL) bfq_update_active_node(parent->rb_right); - else if (parent->rb_left) + else if (parent->rb_left != NULL) bfq_update_active_node(parent->rb_left); node = parent; @@ -354,7 +355,7 @@ static void bfq_active_insert(struct bfq_service_tree *st, { struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); struct rb_node *node = &entity->rb_node; -#ifdef CONFIG_BFQ_GROUP_IOSCHED +#ifdef CONFIG_CGROUP_BFQIO struct bfq_sched_data *sd = NULL; struct bfq_group *bfqg = NULL; struct bfq_data *bfqd = NULL; @@ -362,22 +363,22 @@ static void bfq_active_insert(struct bfq_service_tree *st, bfq_insert(&st->active, entity); - if (node->rb_left) + if (node->rb_left != NULL) node = node->rb_left; - else if (node->rb_right) + else if (node->rb_right != NULL) node = node->rb_right; bfq_update_active_tree(node); -#ifdef CONFIG_BFQ_GROUP_IOSCHED +#ifdef CONFIG_CGROUP_BFQIO sd = entity->sched_data; bfqg = container_of(sd, struct bfq_group, sched_data); BUG_ON(!bfqg); bfqd = (struct bfq_data *)bfqg->bfqd; #endif - if (bfqq) + if (bfqq != NULL) list_add(&bfqq->bfqq_list, &bfqq->bfqd->active_list); -#ifdef CONFIG_BFQ_GROUP_IOSCHED +#ifdef CONFIG_CGROUP_BFQIO else { /* bfq_group */ BUG_ON(!bfqd); bfq_weights_tree_add(bfqd, entity, &bfqd->group_weights_tree); @@ -396,32 +397,31 @@ static void bfq_active_insert(struct bfq_service_tree *st, * bfq_ioprio_to_weight - calc a weight from an ioprio. * @ioprio: the ioprio value to convert. */ -static unsigned short bfq_ioprio_to_weight(int ioprio) +static inline unsigned short bfq_ioprio_to_weight(int ioprio) { BUG_ON(ioprio < 0 || ioprio >= IOPRIO_BE_NR); - return IOPRIO_BE_NR * BFQ_WEIGHT_CONVERSION_COEFF - ioprio; + return IOPRIO_BE_NR - ioprio; } /** * bfq_weight_to_ioprio - calc an ioprio from a weight. * @weight: the weight value to convert. * - * To preserve as much as possible the old only-ioprio user interface, + * To preserve as mush as possible the old only-ioprio user interface, * 0 is used as an escape ioprio value for weights (numerically) equal or - * larger than IOPRIO_BE_NR * BFQ_WEIGHT_CONVERSION_COEFF. + * larger than IOPRIO_BE_NR */ -static unsigned short bfq_weight_to_ioprio(int weight) +static inline unsigned short bfq_weight_to_ioprio(int weight) { BUG_ON(weight < BFQ_MIN_WEIGHT || weight > BFQ_MAX_WEIGHT); - return IOPRIO_BE_NR * BFQ_WEIGHT_CONVERSION_COEFF - weight < 0 ? - 0 : IOPRIO_BE_NR * BFQ_WEIGHT_CONVERSION_COEFF - weight; + return IOPRIO_BE_NR - weight < 0 ? 0 : IOPRIO_BE_NR - weight; } -static void bfq_get_entity(struct bfq_entity *entity) +static inline void bfq_get_entity(struct bfq_entity *entity) { struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); - if (bfqq) { + if (bfqq != NULL) { atomic_inc(&bfqq->ref); bfq_log_bfqq(bfqq->bfqd, bfqq, "get_entity: %p %d", bfqq, atomic_read(&bfqq->ref)); @@ -441,15 +441,15 @@ static struct rb_node *bfq_find_deepest(struct rb_node *node) { struct rb_node *deepest; - if (!node->rb_right && !node->rb_left) + if (node->rb_right == NULL && node->rb_left == NULL) deepest = rb_parent(node); - else if (!node->rb_right) + else if (node->rb_right == NULL) deepest = node->rb_left; - else if (!node->rb_left) + else if (node->rb_left == NULL) deepest = node->rb_right; else { deepest = rb_next(node); - if (deepest->rb_right) + if (deepest->rb_right != NULL) deepest = deepest->rb_right; else if (rb_parent(deepest) != node) deepest = rb_parent(deepest); @@ -468,7 +468,7 @@ static void bfq_active_extract(struct bfq_service_tree *st, { struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); struct rb_node *node; -#ifdef CONFIG_BFQ_GROUP_IOSCHED +#ifdef CONFIG_CGROUP_BFQIO struct bfq_sched_data *sd = NULL; struct bfq_group *bfqg = NULL; struct bfq_data *bfqd = NULL; @@ -477,18 +477,18 @@ static void bfq_active_extract(struct bfq_service_tree *st, node = bfq_find_deepest(&entity->rb_node); bfq_extract(&st->active, entity); - if (node) + if (node != NULL) bfq_update_active_tree(node); -#ifdef CONFIG_BFQ_GROUP_IOSCHED +#ifdef CONFIG_CGROUP_BFQIO sd = entity->sched_data; bfqg = container_of(sd, struct bfq_group, sched_data); BUG_ON(!bfqg); bfqd = (struct bfq_data *)bfqg->bfqd; #endif - if (bfqq) + if (bfqq != NULL) list_del(&bfqq->bfqq_list); -#ifdef CONFIG_BFQ_GROUP_IOSCHED +#ifdef CONFIG_CGROUP_BFQIO else { /* bfq_group */ BUG_ON(!bfqd); bfq_weights_tree_remove(bfqd, entity, @@ -519,14 +519,14 @@ static void bfq_idle_insert(struct bfq_service_tree *st, struct bfq_entity *first_idle = st->first_idle; struct bfq_entity *last_idle = st->last_idle; - if (!first_idle || bfq_gt(first_idle->finish, entity->finish)) + if (first_idle == NULL || bfq_gt(first_idle->finish, entity->finish)) st->first_idle = entity; - if (!last_idle || bfq_gt(entity->finish, last_idle->finish)) + if (last_idle == NULL || bfq_gt(entity->finish, last_idle->finish)) st->last_idle = entity; bfq_insert(&st->idle, entity); - if (bfqq) + if (bfqq != NULL) list_add(&bfqq->bfqq_list, &bfqq->bfqd->idle_list); } @@ -549,7 +549,7 @@ static void bfq_forget_entity(struct bfq_service_tree *st, entity->on_st = 0; st->wsum -= entity->weight; - if (bfqq) { + if (bfqq != NULL) { sd = entity->sched_data; bfq_log_bfqq(bfqq->bfqd, bfqq, "forget_entity: %p %d", bfqq, atomic_read(&bfqq->ref)); @@ -581,7 +581,7 @@ static void bfq_forget_idle(struct bfq_service_tree *st) struct bfq_entity *first_idle = st->first_idle; struct bfq_entity *last_idle = st->last_idle; - if (RB_EMPTY_ROOT(&st->active) && last_idle && + if (RB_EMPTY_ROOT(&st->active) && last_idle != NULL && !bfq_gt(last_idle->finish, st->vtime)) { /* * Forget the whole idle tree, increasing the vtime past @@ -590,7 +590,7 @@ static void bfq_forget_idle(struct bfq_service_tree *st) st->vtime = last_idle->finish; } - if (first_idle && !bfq_gt(first_idle->finish, st->vtime)) + if (first_idle != NULL && !bfq_gt(first_idle->finish, st->vtime)) bfq_put_idle_entity(st, first_idle); } @@ -600,19 +600,19 @@ __bfq_entity_update_weight_prio(struct bfq_service_tree *old_st, { struct bfq_service_tree *new_st = old_st; - if (entity->prio_changed) { + if (entity->ioprio_changed) { struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); unsigned short prev_weight, new_weight; struct bfq_data *bfqd = NULL; struct rb_root *root; -#ifdef CONFIG_BFQ_GROUP_IOSCHED +#ifdef CONFIG_CGROUP_BFQIO struct bfq_sched_data *sd; struct bfq_group *bfqg; #endif - if (bfqq) + if (bfqq != NULL) bfqd = bfqq->bfqd; -#ifdef CONFIG_BFQ_GROUP_IOSCHED +#ifdef CONFIG_CGROUP_BFQIO else { sd = entity->my_sched_data; bfqg = container_of(sd, struct bfq_group, sched_data); @@ -634,14 +634,12 @@ __bfq_entity_update_weight_prio(struct bfq_service_tree *old_st, BUG(); } entity->orig_weight = entity->new_weight; - if (bfqq) - bfqq->ioprio = - bfq_weight_to_ioprio(entity->orig_weight); + entity->ioprio = + bfq_weight_to_ioprio(entity->orig_weight); } - if (bfqq) - bfqq->ioprio_class = bfqq->new_ioprio_class; - entity->prio_changed = 0; + entity->ioprio_class = entity->new_ioprio_class; + entity->ioprio_changed = 0; /* * NOTE: here we may be changing the weight too early, @@ -654,7 +652,7 @@ __bfq_entity_update_weight_prio(struct bfq_service_tree *old_st, prev_weight = entity->weight; new_weight = entity->orig_weight * - (bfqq ? bfqq->wr_coeff : 1); + (bfqq != NULL ? bfqq->wr_coeff : 1); /* * If the weight of the entity changes, remove the entity * from its old weight counter (if there is a counter @@ -685,10 +683,6 @@ __bfq_entity_update_weight_prio(struct bfq_service_tree *old_st, return new_st; } -#ifdef CONFIG_BFQ_GROUP_IOSCHED -static void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg); -#endif - /** * bfq_bfqq_served - update the scheduler status after selection for * service. @@ -699,7 +693,7 @@ static void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg); * are synchronized every time a new bfqq is selected for service. By now, * we keep it to better check consistency. */ -static void bfq_bfqq_served(struct bfq_queue *bfqq, int served) +static void bfq_bfqq_served(struct bfq_queue *bfqq, unsigned long served) { struct bfq_entity *entity = &bfqq->entity; struct bfq_service_tree *st; @@ -714,10 +708,7 @@ static void bfq_bfqq_served(struct bfq_queue *bfqq, int served) st->vtime += bfq_delta(served, st->wsum); bfq_forget_idle(st); } -#ifdef CONFIG_BFQ_GROUP_IOSCHED - bfqg_stats_set_start_empty_time(bfqq_group(bfqq)); -#endif - bfq_log_bfqq(bfqq->bfqd, bfqq, "bfqq_served %d secs", served); + bfq_log_bfqq(bfqq->bfqd, bfqq, "bfqq_served %lu secs", served); } /** @@ -730,7 +721,7 @@ static void bfq_bfqq_served(struct bfq_queue *bfqq, int served) * budget. In this way we should obtain a sort of time-domain * fairness among all the seeky/slow queues. */ -static void bfq_bfqq_charge_full_budget(struct bfq_queue *bfqq) +static inline void bfq_bfqq_charge_full_budget(struct bfq_queue *bfqq) { struct bfq_entity *entity = &bfqq->entity; @@ -755,7 +746,7 @@ static void __bfq_activate_entity(struct bfq_entity *entity) struct bfq_service_tree *st = bfq_entity_service_tree(entity); if (entity == sd->in_service_entity) { - BUG_ON(entity->tree); + BUG_ON(entity->tree != NULL); /* * If we are requeueing the current entity we have * to take care of not charging to it service it has @@ -846,7 +837,7 @@ static int __bfq_deactivate_entity(struct bfq_entity *entity, int requeue) if (!entity->on_st) return 0; - BUG_ON(was_in_service && entity->tree); + BUG_ON(was_in_service && entity->tree != NULL); if (was_in_service) { bfq_calc_finish(entity, entity->service); @@ -855,7 +846,7 @@ static int __bfq_deactivate_entity(struct bfq_entity *entity, int requeue) bfq_active_extract(st, entity); else if (entity->tree == &st->idle) bfq_idle_extract(st, entity); - else if (entity->tree) + else if (entity->tree != NULL) BUG(); if (was_in_service || sd->next_in_service == entity) @@ -893,7 +884,7 @@ static void bfq_deactivate_entity(struct bfq_entity *entity, int requeue) */ break; - if (sd->next_in_service) + if (sd->next_in_service != NULL) /* * The parent entity is still backlogged and * the budgets on the path towards the root @@ -962,7 +953,7 @@ static struct bfq_entity *bfq_first_active_entity(struct bfq_service_tree *st) struct bfq_entity *entry, *first = NULL; struct rb_node *node = st->active.rb_node; - while (node) { + while (node != NULL) { entry = rb_entry(node, struct bfq_entity, rb_node); left: if (!bfq_gt(entry->start, st->vtime)) @@ -970,7 +961,7 @@ left: BUG_ON(bfq_gt(entry->min_start, st->vtime)); - if (node->rb_left) { + if (node->rb_left != NULL) { entry = rb_entry(node->rb_left, struct bfq_entity, rb_node); if (!bfq_gt(entry->min_start, st->vtime)) { @@ -978,12 +969,12 @@ left: goto left; } } - if (first) + if (first != NULL) break; node = node->rb_right; } - BUG_ON(!first && !RB_EMPTY_ROOT(&st->active)); + BUG_ON(first == NULL && !RB_EMPTY_ROOT(&st->active)); return first; } @@ -1039,13 +1030,13 @@ static struct bfq_entity *bfq_lookup_next_entity(struct bfq_sched_data *sd, struct bfq_entity *entity; int i = 0; - BUG_ON(sd->in_service_entity); + BUG_ON(sd->in_service_entity != NULL); - if (bfqd && + if (bfqd != NULL && jiffies - bfqd->bfq_class_idle_last_service > BFQ_CL_IDLE_TIMEOUT) { entity = __bfq_lookup_next_entity(st + BFQ_IOPRIO_CLASSES - 1, true); - if (entity) { + if (entity != NULL) { i = BFQ_IOPRIO_CLASSES - 1; bfqd->bfq_class_idle_last_service = jiffies; sd->next_in_service = entity; @@ -1053,7 +1044,7 @@ static struct bfq_entity *bfq_lookup_next_entity(struct bfq_sched_data *sd, } for (; i < BFQ_IOPRIO_CLASSES; i++) { entity = __bfq_lookup_next_entity(st + i, false); - if (entity) { + if (entity != NULL) { if (extract) { bfq_check_next_in_service(sd, entity); bfq_active_extract(st + i, entity); @@ -1076,27 +1067,27 @@ static struct bfq_queue *bfq_get_next_queue(struct bfq_data *bfqd) struct bfq_sched_data *sd; struct bfq_queue *bfqq; - BUG_ON(bfqd->in_service_queue); + BUG_ON(bfqd->in_service_queue != NULL); if (bfqd->busy_queues == 0) return NULL; sd = &bfqd->root_group->sched_data; - for (; sd ; sd = entity->my_sched_data) { + for (; sd != NULL; sd = entity->my_sched_data) { entity = bfq_lookup_next_entity(sd, 1, bfqd); - BUG_ON(!entity); + BUG_ON(entity == NULL); entity->service = 0; } bfqq = bfq_entity_to_bfqq(entity); - BUG_ON(!bfqq); + BUG_ON(bfqq == NULL); return bfqq; } static void __bfq_bfqd_reset_in_service(struct bfq_data *bfqd) { - if (bfqd->in_service_bic) { + if (bfqd->in_service_bic != NULL) { put_io_context(bfqd->in_service_bic->icq.ioc); bfqd->in_service_bic = NULL; } @@ -1123,10 +1114,6 @@ static void bfq_activate_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq) bfq_activate_entity(entity); } -#ifdef CONFIG_BFQ_GROUP_IOSCHED -static void bfqg_stats_update_dequeue(struct bfq_group *bfqg); -#endif - /* * Called when the bfqq no longer has requests pending, remove it from * the service tree. @@ -1160,10 +1147,6 @@ static void bfq_del_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq, if (bfqq->wr_coeff > 1) bfqd->wr_busy_queues--; -#ifdef CONFIG_BFQ_GROUP_IOSCHED - bfqg_stats_update_dequeue(bfqq_group(bfqq)); -#endif - bfq_deactivate_bfqq(bfqd, bfqq, requeue); } |