diff options
author | André Fabian Silva Delgado <emulatorman@parabola.nu> | 2016-10-20 00:10:27 -0300 |
---|---|---|
committer | André Fabian Silva Delgado <emulatorman@parabola.nu> | 2016-10-20 00:10:27 -0300 |
commit | d0b2f91bede3bd5e3d24dd6803e56eee959c1797 (patch) | |
tree | 7fee4ab0509879c373c4f2cbd5b8a5be5b4041ee /block/bfq-iosched.c | |
parent | e914f8eb445e8f74b00303c19c2ffceaedd16a05 (diff) |
Linux-libre 4.8.2-gnupck-4.8.2-gnu
Diffstat (limited to 'block/bfq-iosched.c')
-rw-r--r-- | block/bfq-iosched.c | 305 |
1 files changed, 178 insertions, 127 deletions
diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c index 76a701abf..9190a5554 100644 --- a/block/bfq-iosched.c +++ b/block/bfq-iosched.c @@ -7,7 +7,9 @@ * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it> * Paolo Valente <paolo.valente@unimore.it> * - * Copyright (C) 2016 Paolo Valente <paolo.valente@unimore.it> + * Copyright (C) 2015 Paolo Valente <paolo.valente@unimore.it> + * + * Copyright (C) 2016 Paolo Valente <paolo.valente@linaro.org> * * Licensed under the GPL-2 as detailed in the accompanying COPYING.BFQ * file. @@ -71,8 +73,8 @@ #include "bfq.h" #include "blk.h" -/* Expiration time of sync (0) and async (1) requests, in jiffies. */ -static const int bfq_fifo_expire[2] = { HZ / 4, HZ / 8 }; +/* Expiration time of sync (0) and async (1) requests, in ns. */ +static const u64 bfq_fifo_expire[2] = { NSEC_PER_SEC / 4, NSEC_PER_SEC / 8 }; /* Maximum backwards seek, in KiB. */ static const int bfq_back_max = 16 * 1024; @@ -80,8 +82,8 @@ static const int bfq_back_max = 16 * 1024; /* Penalty of a backwards seek, in number of sectors. */ static const int bfq_back_penalty = 2; -/* Idling period duration, in jiffies. */ -static int bfq_slice_idle = HZ / 125; +/* Idling period duration, in ns. */ +static u64 bfq_slice_idle = NSEC_PER_SEC / 125; /* Minimum number of assigned budgets for which stats are safe to compute. */ static const int bfq_stats_min_budgets = 194; @@ -102,13 +104,13 @@ static const int bfq_timeout = HZ / 8; struct kmem_cache *bfq_pool; /* Below this threshold (in ms), we consider thinktime immediate. */ -#define BFQ_MIN_TT 2 +#define BFQ_MIN_TT (2 * NSEC_PER_MSEC) /* hw_tag detection: parallel requests threshold and min samples needed. */ #define BFQ_HW_QUEUE_THRESHOLD 4 #define BFQ_HW_QUEUE_SAMPLES 32 -#define BFQQ_SEEK_THR (sector_t)(8 * 100) +#define BFQQ_SEEK_THR (sector_t)(8 * 100) #define BFQQ_CLOSE_THR (sector_t)(8 * 1024) #define BFQQ_SEEKY(bfqq) (hweight32(bfqq->seek_history) > 32/8) @@ -191,10 +193,7 @@ static void bfq_schedule_dispatch(struct bfq_data *bfqd); */ static int bfq_bio_sync(struct bio *bio) { - if (bio_data_dir(bio) == READ || (bio->bi_rw & REQ_SYNC)) - return 1; - - return 0; + return bio_data_dir(bio) == READ || (bio->bi_opf & REQ_SYNC); } /* @@ -223,7 +222,7 @@ static struct request *bfq_choose_req(struct bfq_data *bfqd, unsigned long back_max; #define BFQ_RQ1_WRAP 0x01 /* request 1 wraps */ #define BFQ_RQ2_WRAP 0x02 /* request 2 wraps */ - unsigned wrap = 0; /* bit mask: requests behind the disk head? */ + unsigned int wrap = 0; /* bit mask: requests behind the disk head? */ if (!rq1 || rq1 == rq2) return rq2; @@ -278,12 +277,11 @@ static struct request *bfq_choose_req(struct bfq_data *bfqd, return rq1; else if (d2 < d1) return rq2; - else { - if (s1 >= s2) - return rq1; - else - return rq2; - } + + if (s1 >= s2) + return rq1; + else + return rq2; case BFQ_RQ2_WRAP: return rq1; @@ -339,7 +337,7 @@ bfq_rq_pos_tree_lookup(struct bfq_data *bfqd, struct rb_root *root, *rb_link = p; bfq_log(bfqd, "rq_pos_tree_lookup %llu: returning %d", - (long long unsigned)sector, + (unsigned long long) sector, bfqq ? bfqq->pid : 0); return bfqq; @@ -690,7 +688,7 @@ static void bfq_add_to_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq) */ hlist_for_each_entry(bfqq_item, &bfqd->burst_list, burst_list_node) { - bfq_mark_bfqq_in_large_burst(bfqq_item); + bfq_mark_bfqq_in_large_burst(bfqq_item); bfq_log_bfqq(bfqd, bfqq_item, "marked in large burst"); } bfq_mark_bfqq_in_large_burst(bfqq); @@ -891,6 +889,7 @@ end: static int bfq_bfqq_budget_left(struct bfq_queue *bfqq) { struct bfq_entity *entity = &bfqq->entity; + return entity->budget - entity->service; } @@ -1062,7 +1061,7 @@ static bool bfq_bfqq_update_budg_for_activation(struct bfq_data *bfqd, } entity->budget = max_t(unsigned long, bfqq->max_budget, - bfq_serv_to_charge(bfqq->next_rq,bfqq)); + bfq_serv_to_charge(bfqq->next_rq, bfqq)); BUG_ON(entity->budget < 0); bfq_clear_bfqq_non_blocking_wait_rq(bfqq); @@ -1200,9 +1199,9 @@ static void bfq_bfqq_handle_idle_busy_switch(struct bfq_data *bfqd, * bfq_bfqq_update_budg_for_activation for * details on the usage of the next variable. */ - arrived_in_time = time_is_after_jiffies( + arrived_in_time = ktime_get_ns() <= RQ_BIC(rq)->ttime.last_end_request + - bfqd->bfq_slice_idle * 3); + bfqd->bfq_slice_idle * 3; bfq_log_bfqq(bfqd, bfqq, "bfq_add_request non-busy: " @@ -1217,7 +1216,7 @@ static void bfq_bfqq_handle_idle_busy_switch(struct bfq_data *bfqd, BUG_ON(bfqq == bfqd->in_service_queue); bfqg_stats_update_io_add(bfqq_group(RQ_BFQQ(rq)), bfqq, - rq->cmd_flags); + req_op(rq), rq->cmd_flags); /* * bfqq deserves to be weight-raised if: @@ -1458,7 +1457,7 @@ static void bfq_activate_request(struct request_queue *q, struct request *rq) bfqd->rq_in_driver++; bfqd->last_position = blk_rq_pos(rq) + blk_rq_sectors(rq); bfq_log(bfqd, "activate_request: new bfqd->last_position %llu", - (long long unsigned)bfqd->last_position); + (unsigned long long) bfqd->last_position); } static void bfq_deactivate_request(struct request_queue *q, struct request *rq) @@ -1523,7 +1522,8 @@ static void bfq_remove_request(struct request *rq) BUG_ON(bfqq->meta_pending == 0); bfqq->meta_pending--; } - bfqg_stats_update_io_remove(bfqq_group(bfqq), rq->cmd_flags); + bfqg_stats_update_io_remove(bfqq_group(bfqq), req_op(rq), + rq->cmd_flags); } static int bfq_merge(struct request_queue *q, struct request **req, @@ -1533,7 +1533,7 @@ static int bfq_merge(struct request_queue *q, struct request **req, struct request *__rq; __rq = bfq_find_rq_fmerge(bfqd, bio); - if (__rq && elv_rq_merge_ok(__rq, bio)) { + if (__rq && elv_bio_merge_ok(__rq, bio)) { *req = __rq; return ELEVATOR_FRONT_MERGE; } @@ -1578,7 +1578,8 @@ static void bfq_merged_request(struct request_queue *q, struct request *req, static void bfq_bio_merged(struct request_queue *q, struct request *req, struct bio *bio) { - bfqg_stats_update_io_merged(bfqq_group(RQ_BFQQ(req)), bio->bi_rw); + bfqg_stats_update_io_merged(bfqq_group(RQ_BFQQ(req)), bio_op(bio), + bio->bi_opf); } #endif @@ -1598,7 +1599,7 @@ static void bfq_merged_requests(struct request_queue *q, struct request *rq, */ if (bfqq == next_bfqq && !list_empty(&rq->queuelist) && !list_empty(&next->queuelist) && - time_before(next->fifo_time, rq->fifo_time)) { + next->fifo_time < rq->fifo_time) { list_del_init(&rq->queuelist); list_replace_init(&next->queuelist, &rq->queuelist); rq->fifo_time = next->fifo_time; @@ -1608,7 +1609,8 @@ static void bfq_merged_requests(struct request_queue *q, struct request *rq, bfqq->next_rq = rq; bfq_remove_request(next); - bfqg_stats_update_io_merged(bfqq_group(bfqq), next->cmd_flags); + bfqg_stats_update_io_merged(bfqq_group(bfqq), req_op(next), + next->cmd_flags); } /* Must be called with bfqq != NULL */ @@ -1961,7 +1963,7 @@ bfq_merge_bfqqs(struct bfq_data *bfqd, struct bfq_io_cq *bic, struct bfq_queue *bfqq, struct bfq_queue *new_bfqq) { bfq_log_bfqq(bfqd, bfqq, "merging with queue %lu", - (long unsigned)new_bfqq->pid); + (unsigned long) new_bfqq->pid); /* Save weight raising and idle window of the merged queues */ bfq_bfqq_save_state(bfqq); bfq_bfqq_save_state(new_bfqq); @@ -1983,11 +1985,10 @@ bfq_merge_bfqqs(struct bfq_data *bfqd, struct bfq_io_cq *bic, new_bfqq->wr_cur_max_time = bfqq->wr_cur_max_time; new_bfqq->last_wr_start_finish = bfqq->last_wr_start_finish; if (bfq_bfqq_busy(new_bfqq)) - bfqd->wr_busy_queues++; + bfqd->wr_busy_queues++; new_bfqq->entity.prio_changed = 1; bfq_log_bfqq(bfqd, new_bfqq, - "wr starting after merge with %d, " - "rais_max_time %u", + "wr start after merge with %d, rais_max_time %u", bfqq->pid, jiffies_to_msecs(bfqq->wr_cur_max_time)); } @@ -2028,8 +2029,8 @@ bfq_merge_bfqqs(struct bfq_data *bfqd, struct bfq_io_cq *bic, bfq_put_queue(bfqq); } -static int bfq_allow_merge(struct request_queue *q, struct request *rq, - struct bio *bio) +static int bfq_allow_bio_merge(struct request_queue *q, struct request *rq, + struct bio *bio) { struct bfq_data *bfqd = q->elevator->elevator_data; struct bfq_io_cq *bic; @@ -2039,7 +2040,7 @@ static int bfq_allow_merge(struct request_queue *q, struct request *rq, * Disallow merge of a sync bio into an async request. */ if (bfq_bio_sync(bio) && !rq_is_sync(rq)) - return 0; + return false; /* * Lookup the bfqq that this bio will be queued with. Allow @@ -2048,7 +2049,7 @@ static int bfq_allow_merge(struct request_queue *q, struct request *rq, */ bic = bfq_bic_lookup(bfqd, current->io_context); if (!bic) - return 0; + return false; bfqq = bic_to_bfqq(bic, bfq_bio_sync(bio)); /* @@ -2071,6 +2072,12 @@ static int bfq_allow_merge(struct request_queue *q, struct request *rq, return bfqq == RQ_BFQQ(rq); } +static int bfq_allow_rq_merge(struct request_queue *q, struct request *rq, + struct request *next) +{ + return RQ_BFQQ(rq) == RQ_BFQQ(next); +} + /* * Set the maximum time for the in-service queue to consume its * budget. This prevents seeky processes from lowering the throughput. @@ -2081,6 +2088,7 @@ static void bfq_set_budget_timeout(struct bfq_data *bfqd, struct bfq_queue *bfqq) { unsigned int timeout_coeff; + if (bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time) timeout_coeff = 1; else @@ -2198,13 +2206,15 @@ static void bfq_arm_slice_timer(struct bfq_data *bfqd) */ if (BFQQ_SEEKY(bfqq) && bfqq->wr_coeff == 1 && bfq_symmetric_scenario(bfqd)) - sl = min(sl, msecs_to_jiffies(BFQ_MIN_TT)); + sl = min_t(u64, sl, BFQ_MIN_TT); bfqd->last_idling_start = ktime_get(); - mod_timer(&bfqd->idle_slice_timer, jiffies + sl); + hrtimer_start(&bfqd->idle_slice_timer, ns_to_ktime(sl), + HRTIMER_MODE_REL); bfqg_stats_set_start_idle_time(bfqq_group(bfqq)); - bfq_log(bfqd, "arm idle: %u/%u ms", - jiffies_to_msecs(sl), jiffies_to_msecs(bfqd->bfq_slice_idle)); + bfq_log(bfqd, "arm idle: %llu/%llu ms", + div_u64(sl, NSEC_PER_MSEC), + div_u64(bfqd->bfq_slice_idle, NSEC_PER_MSEC)); } /* @@ -2247,7 +2257,7 @@ static struct request *bfq_check_fifo(struct bfq_queue *bfqq) rq = rq_entry_fifo(bfqq->fifo.next); - if (time_is_after_jiffies(rq->fifo_time)) + if (ktime_get_ns() < rq->fifo_time) return NULL; return rq; @@ -2468,7 +2478,7 @@ static unsigned long bfq_calc_max_budget(struct bfq_data *bfqd) { /* * The max_budget calculated when autotuning is equal to the - * amount of sectors transfered in timeout at the + * amount of sectors transferred in timeout at the * estimated peak rate. */ return bfqd->peak_rate * 1000 * jiffies_to_msecs(bfqd->bfq_timeout) >> @@ -2527,9 +2537,10 @@ static bool bfq_update_peak_rate(struct bfq_data *bfqd, struct bfq_queue *bfqq, /* Don't trust short/unrealistic values. */ if (delta_usecs < 1000 || delta_usecs >= LONG_MAX) { if (blk_queue_nonrot(bfqd->queue)) - *delta_ms = BFQ_MIN_TT; /* give same worst-case - guarantees as - idling for seeky + *delta_ms = BFQ_MIN_TT; /* + * give same worst-case + * guarantees as + * idling for seeky */ else /* Charge at least one seek */ *delta_ms = jiffies_to_msecs(bfq_slice_idle); @@ -2537,7 +2548,7 @@ static bool bfq_update_peak_rate(struct bfq_data *bfqd, struct bfq_queue *bfqq, } delta_ms_tmp = delta_usecs; - do_div(delta_ms_tmp, 1000); + do_div(delta_ms_tmp, NSEC_PER_MSEC); *delta_ms = delta_ms_tmp; /* @@ -2602,6 +2613,7 @@ static bool bfq_update_peak_rate(struct bfq_data *bfqd, struct bfq_queue *bfqq, if (bfqd->peak_rate_samples == BFQ_PEAK_RATE_SAMPLES && update) { int dev_type = blk_queue_nonrot(bfqd->queue); + if (bfqd->bfq_user_max_budget == 0) { bfqd->bfq_max_budget = bfq_calc_max_budget(bfqd); @@ -2619,9 +2631,9 @@ static bool bfq_update_peak_rate(struct bfq_data *bfqd, struct bfq_queue *bfqq, bfqd->RT_prod = R_fast[dev_type] * T_fast[dev_type]; } - bfq_log(bfqd, "dev_speed_class = %d (%d sects/sec), " - "thresh %d setcs/sec", - bfqd->device_speed, + bfq_log(bfqd, + "dev_type %d dev_speed_class = %d (%d sects/sec), thresh %d setcs/sec", + dev_type, bfqd->device_speed, bfqd->device_speed == BFQ_BFQD_FAST ? (1000000*R_fast[dev_type])>>BFQ_RATE_SHIFT : (1000000*R_slow[dev_type])>>BFQ_RATE_SHIFT, @@ -2641,8 +2653,7 @@ static bool bfq_update_peak_rate(struct bfq_data *bfqd, struct bfq_queue *bfqq, } bfq_log_bfqq(bfqd, bfqq, - "update_peak_rate: bw %llu sect/s, peak rate %llu, " - "slow %d", + "update_peak_rate: bw %llu sect/s, peak rate %llu, slow %d", (1000000*bw)>>BFQ_RATE_SHIFT, (1000000*bfqd->peak_rate)>>BFQ_RATE_SHIFT, bw < bfqd->peak_rate / 2); @@ -2706,9 +2717,7 @@ static unsigned long bfq_bfqq_softrt_next_start(struct bfq_data *bfqd, struct bfq_queue *bfqq) { bfq_log_bfqq(bfqd, bfqq, - "softrt_next_start: service_blkg %lu " - "soft_rate %u sects/sec" - "interval %u", +"softrt_next_start: service_blkg %lu soft_rate %u sects/sec interval %u", bfqq->service_from_backlogged, bfqd->bfq_wr_max_softrt_rate, jiffies_to_msecs(HZ * bfqq->service_from_backlogged / @@ -2717,7 +2726,7 @@ static unsigned long bfq_bfqq_softrt_next_start(struct bfq_data *bfqd, return max(bfqq->last_idle_bklogged + HZ * bfqq->service_from_backlogged / bfqd->bfq_wr_max_softrt_rate, - jiffies + bfqq->bfqd->bfq_slice_idle + 4); + jiffies + nsecs_to_jiffies(bfqq->bfqd->bfq_slice_idle) + 4); } /* @@ -2816,7 +2825,7 @@ static void bfq_bfqq_expire(struct bfq_data *bfqd, BUG_ON(bfqq->entity.budget < bfqq->entity.service); if (reason == BFQ_BFQQ_TOO_IDLE && - entity->service <= 2 * entity->budget / 10 ) + entity->service <= 2 * entity->budget / 10) bfq_clear_bfqq_IO_bound(bfqq); if (bfqd->low_latency && bfqq->wr_coeff == 1) @@ -3042,7 +3051,7 @@ static bool bfq_bfqq_may_idle(struct bfq_queue *bfqq) * (i) each of these processes must get the same throughput as * the others; * (ii) all these processes have the same I/O pattern - (either sequential or random). + * (either sequential or random). * In fact, in such a scenario, the drive will tend to treat * the requests of each of these processes in about the same * way as the requests of the others, and thus to provide @@ -3155,9 +3164,11 @@ static bool bfq_bfqq_may_idle(struct bfq_queue *bfqq) * 2) idling either boosts the throughput (without issues), or * is necessary to preserve service guarantees. */ - bfq_log_bfqq(bfqd, bfqq, "may_idle: sync %d idling_boosts_thr %d " - "wr_busy %d boosts %d IO-bound %d guar %d", - bfq_bfqq_sync(bfqq), idling_boosts_thr, + bfq_log_bfqq(bfqd, bfqq, "may_idle: sync %d idling_boosts_thr %d", + bfq_bfqq_sync(bfqq), idling_boosts_thr); + + bfq_log_bfqq(bfqd, bfqq, + "may_idle: wr_busy %d boosts %d IO-bound %d guar %d", bfqd->wr_busy_queues, idling_boosts_thr_without_issues, bfq_bfqq_IO_bound(bfqq), @@ -3204,7 +3215,7 @@ static struct bfq_queue *bfq_select_queue(struct bfq_data *bfqd) bfq_log_bfqq(bfqd, bfqq, "select_queue: already in-service queue"); if (bfq_may_expire_for_budg_timeout(bfqq) && - !timer_pending(&bfqd->idle_slice_timer) && + !hrtimer_active(&bfqd->idle_slice_timer) && !bfq_bfqq_must_idle(bfqq)) goto expire; @@ -3224,7 +3235,8 @@ static struct bfq_queue *bfq_select_queue(struct bfq_data *bfqd) * not disable disk idling even when a new request * arrives. */ - if (timer_pending(&bfqd->idle_slice_timer)) { + if (bfq_bfqq_wait_request(bfqq)) { + BUG_ON(!hrtimer_active(&bfqd->idle_slice_timer)); /* * If we get here: 1) at least a new request * has arrived but we have not disabled the @@ -3239,7 +3251,7 @@ static struct bfq_queue *bfq_select_queue(struct bfq_data *bfqd) * So we disable idling. */ bfq_clear_bfqq_wait_request(bfqq); - del_timer(&bfqd->idle_slice_timer); + hrtimer_try_to_cancel(&bfqd->idle_slice_timer); bfqg_stats_update_idle_time(bfqq_group(bfqq)); } goto keep_queue; @@ -3251,7 +3263,7 @@ static struct bfq_queue *bfq_select_queue(struct bfq_data *bfqd) * for a new request, or has requests waiting for a completion and * may idle after their completion, then keep it anyway. */ - if (timer_pending(&bfqd->idle_slice_timer) || + if (hrtimer_active(&bfqd->idle_slice_timer) || (bfqq->dispatched != 0 && bfq_bfqq_may_idle(bfqq))) { bfqq = NULL; goto keep_queue; @@ -3271,6 +3283,7 @@ keep_queue: static void bfq_update_wr_data(struct bfq_data *bfqd, struct bfq_queue *bfqq) { struct bfq_entity *entity = &bfqq->entity; + if (bfqq->wr_coeff > 1) { /* queue is being weight-raised */ bfq_log_bfqq(bfqd, bfqq, "raising period dur %u/%u msec, old coeff %u, w %d(%d)", @@ -3376,7 +3389,7 @@ static int bfq_dispatch_request(struct bfq_data *bfqd, bfq_log_bfqq(bfqd, bfqq, "dispatched %u sec req (%llu), budg left %d", blk_rq_sectors(rq), - (long long unsigned)blk_rq_pos(rq), + (unsigned long long) blk_rq_pos(rq), bfq_bfqq_budget_left(bfqq)); dispatched++; @@ -3461,8 +3474,7 @@ static int bfq_dispatch_requests(struct request_queue *q, int force) BUG_ON(bfqq->entity.budget < bfqq->entity.service); - bfq_clear_bfqq_wait_request(bfqq); - BUG_ON(timer_pending(&bfqd->idle_slice_timer)); + BUG_ON(bfq_bfqq_wait_request(bfqq)); if (!bfq_dispatch_request(bfqd, bfqq)) return 0; @@ -3554,9 +3566,7 @@ static void bfq_exit_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq) static void bfq_init_icq(struct io_cq *icq) { - struct bfq_io_cq *bic = icq_to_bic(icq); - - bic->ttime.last_end_request = bfq_smallest_from_now(); + icq_to_bic(icq)->ttime.last_end_request = ktime_get_ns() - (1ULL<<32); } static void bfq_exit_icq(struct io_cq *icq) @@ -3620,8 +3630,8 @@ static void bfq_set_next_ioprio_data(struct bfq_queue *bfqq, } if (bfqq->new_ioprio >= IOPRIO_BE_NR) { - printk(KERN_CRIT "bfq_set_next_ioprio_data: new_ioprio %d\n", - bfqq->new_ioprio); + pr_crit("bfq_set_next_ioprio_data: new_ioprio %d\n", + bfqq->new_ioprio); BUG(); } @@ -3735,7 +3745,7 @@ static struct bfq_queue *bfq_get_queue(struct bfq_data *bfqd, rcu_read_lock(); - bfqg = bfq_find_set_group(bfqd,bio_blkcg(bio)); + bfqg = bfq_find_set_group(bfqd, bio_blkcg(bio)); if (!bfqg) { bfqq = &bfqd->oom_bfqq; goto out; @@ -3784,13 +3794,15 @@ out: static void bfq_update_io_thinktime(struct bfq_data *bfqd, struct bfq_io_cq *bic) { - unsigned long elapsed = jiffies - bic->ttime.last_end_request; - unsigned long ttime = min(elapsed, 2UL * bfqd->bfq_slice_idle); + struct bfq_ttime *ttime = &bic->ttime; + u64 elapsed = ktime_get_ns() - bic->ttime.last_end_request; + + elapsed = min(elapsed, 2UL * bfqd->bfq_slice_idle); - bic->ttime.ttime_samples = (7*bic->ttime.ttime_samples + 256) / 8; - bic->ttime.ttime_total = (7*bic->ttime.ttime_total + 256*ttime) / 8; - bic->ttime.ttime_mean = (bic->ttime.ttime_total + 128) / - bic->ttime.ttime_samples; + ttime->ttime_samples = (7*bic->ttime.ttime_samples + 256) / 8; + ttime->ttime_total = div_u64(7*ttime->ttime_total + 256*elapsed, 8); + ttime->ttime_mean = div64_ul(ttime->ttime_total + 128, + ttime->ttime_samples); } @@ -3799,6 +3811,7 @@ bfq_update_io_seektime(struct bfq_data *bfqd, struct bfq_queue *bfqq, struct request *rq) { sector_t sdist = 0; + if (bfqq->last_request_pos) { if (bfqq->last_request_pos < blk_rq_pos(rq)) sdist = blk_rq_pos(rq) - bfqq->last_request_pos; @@ -3906,7 +3919,7 @@ static void bfq_rq_enqueued(struct bfq_data *bfqd, struct bfq_queue *bfqq, * timer. */ bfq_clear_bfqq_wait_request(bfqq); - del_timer(&bfqd->idle_slice_timer); + hrtimer_try_to_cancel(&bfqd->idle_slice_timer); bfqg_stats_update_idle_time(bfqq_group(bfqq)); /* @@ -3964,7 +3977,8 @@ static void bfq_insert_request(struct request_queue *q, struct request *rq) bfq_add_request(rq); - rq->fifo_time = jiffies + bfqd->bfq_fifo_expire[rq_is_sync(rq)]; + rq->fifo_time = ktime_get_ns() + + jiffies_to_nsecs(bfqd->bfq_fifo_expire[rq_is_sync(rq)]); list_add_tail(&rq->queuelist, &bfqq->fifo); bfq_rq_enqueued(bfqd, bfqq, rq); @@ -4012,7 +4026,8 @@ static void bfq_completed_request(struct request_queue *q, struct request *rq) bfqq->dispatched--; bfqg_stats_update_completion(bfqq_group(bfqq), rq_start_time_ns(rq), - rq_io_start_time_ns(rq), rq->cmd_flags); + rq_io_start_time_ns(rq), req_op(rq), + rq->cmd_flags); if (!bfqq->dispatched && !bfq_bfqq_busy(bfqq)) { BUG_ON(!RB_EMPTY_ROOT(&bfqq->sort_list)); @@ -4028,7 +4043,7 @@ static void bfq_completed_request(struct request_queue *q, struct request *rq) &bfqd->queue_weights_tree); } - RQ_BIC(rq)->ttime.last_end_request = jiffies; + RQ_BIC(rq)->ttime.last_end_request = ktime_get_ns(); /* * If we are waiting to discover whether the request pattern @@ -4079,7 +4094,7 @@ static int __bfq_may_queue(struct bfq_queue *bfqq) return ELV_MQUEUE_MAY; } -static int bfq_may_queue(struct request_queue *q, int rw) +static int bfq_may_queue(struct request_queue *q, int op, int op_flags) { struct bfq_data *bfqd = q->elevator->elevator_data; struct task_struct *tsk = current; @@ -4096,7 +4111,7 @@ static int bfq_may_queue(struct request_queue *q, int rw) if (!bic) return ELV_MQUEUE_MAY; - bfqq = bic_to_bfqq(bic, rw_is_sync(rw)); + bfqq = bic_to_bfqq(bic, rw_is_sync(op, op_flags)); if (bfqq) return __bfq_may_queue(bfqq); @@ -4279,9 +4294,10 @@ static void bfq_kick_queue(struct work_struct *work) * Handler of the expiration of the timer running if the in-service queue * is idling inside its time slice. */ -static void bfq_idle_slice_timer(unsigned long data) +static enum hrtimer_restart bfq_idle_slice_timer(struct hrtimer *timer) { - struct bfq_data *bfqd = (struct bfq_data *)data; + struct bfq_data *bfqd = container_of(timer, struct bfq_data, + idle_slice_timer); struct bfq_queue *bfqq; unsigned long flags; enum bfqq_expiration reason; @@ -4299,6 +4315,8 @@ static void bfq_idle_slice_timer(unsigned long data) */ if (bfqq) { bfq_log_bfqq(bfqd, bfqq, "slice_timer expired"); + bfq_clear_bfqq_wait_request(bfqq); + if (bfq_bfqq_budget_timeout(bfqq)) /* * Also here the queue can be safely expired @@ -4324,11 +4342,12 @@ schedule_dispatch: bfq_schedule_dispatch(bfqd); spin_unlock_irqrestore(bfqd->queue->queue_lock, flags); + return HRTIMER_NORESTART; } static void bfq_shutdown_timer_wq(struct bfq_data *bfqd) { - del_timer_sync(&bfqd->idle_slice_timer); + hrtimer_cancel(&bfqd->idle_slice_timer); cancel_work_sync(&bfqd->unplug_work); } @@ -4385,7 +4404,7 @@ static void bfq_exit_queue(struct elevator_queue *e) bfq_shutdown_timer_wq(bfqd); - BUG_ON(timer_pending(&bfqd->idle_slice_timer)); + BUG_ON(hrtimer_active(&bfqd->idle_slice_timer)); #ifdef CONFIG_BFQ_GROUP_IOSCHED blkcg_deactivate_policy(q, &blkcg_policy_bfq); @@ -4460,9 +4479,9 @@ static int bfq_init_queue(struct request_queue *q, struct elevator_type *e) bfq_init_root_group(bfqd->root_group, bfqd); bfq_init_entity(&bfqd->oom_bfqq.entity, bfqd->root_group); - init_timer(&bfqd->idle_slice_timer); + hrtimer_init(&bfqd->idle_slice_timer, CLOCK_MONOTONIC, + HRTIMER_MODE_REL); bfqd->idle_slice_timer.function = bfq_idle_slice_timer; - bfqd->idle_slice_timer.data = (unsigned long)bfqd; bfqd->queue_weights_tree = RB_ROOT; bfqd->group_weights_tree = RB_ROOT; @@ -4528,8 +4547,7 @@ out_free: static void bfq_slab_kill(void) { - if (bfq_pool) - kmem_cache_destroy(bfq_pool); + kmem_cache_destroy(bfq_pool); } static int __init bfq_slab_setup(void) @@ -4542,7 +4560,7 @@ static int __init bfq_slab_setup(void) static ssize_t bfq_var_show(unsigned int var, char *page) { - return sprintf(page, "%d\n", var); + return sprintf(page, "%u\n", var); } static ssize_t bfq_var_store(unsigned long *var, const char *page, @@ -4560,6 +4578,7 @@ static ssize_t bfq_var_store(unsigned long *var, const char *page, static ssize_t bfq_wr_max_time_show(struct elevator_queue *e, char *page) { struct bfq_data *bfqd = e->elevator_data; + return sprintf(page, "%d\n", bfqd->bfq_wr_max_time > 0 ? jiffies_to_msecs(bfqd->bfq_wr_max_time) : jiffies_to_msecs(bfq_wr_duration(bfqd))); @@ -4578,25 +4597,29 @@ static ssize_t bfq_weights_show(struct elevator_queue *e, char *page) num_char += sprintf(page + num_char, "Active:\n"); list_for_each_entry(bfqq, &bfqd->active_list, bfqq_list) { - num_char += sprintf(page + num_char, - "pid%d: weight %hu, nr_queued %d %d, dur %d/%u\n", - bfqq->pid, - bfqq->entity.weight, - bfqq->queued[0], - bfqq->queued[1], - jiffies_to_msecs(jiffies - bfqq->last_wr_start_finish), - jiffies_to_msecs(bfqq->wr_cur_max_time)); + num_char += sprintf(page + num_char, + "pid%d: weight %hu, nr_queued %d %d, ", + bfqq->pid, + bfqq->entity.weight, + bfqq->queued[0], + bfqq->queued[1]); + num_char += sprintf(page + num_char, + "dur %d/%u\n", + jiffies_to_msecs( + jiffies - + bfqq->last_wr_start_finish), + jiffies_to_msecs(bfqq->wr_cur_max_time)); } num_char += sprintf(page + num_char, "Idle:\n"); list_for_each_entry(bfqq, &bfqd->idle_list, bfqq_list) { - num_char += sprintf(page + num_char, - "pid%d: weight %hu, dur %d/%u\n", - bfqq->pid, - bfqq->entity.weight, - jiffies_to_msecs(jiffies - - bfqq->last_wr_start_finish), - jiffies_to_msecs(bfqq->wr_cur_max_time)); + num_char += sprintf(page + num_char, + "pid%d: weight %hu, dur %d/%u\n", + bfqq->pid, + bfqq->entity.weight, + jiffies_to_msecs(jiffies - + bfqq->last_wr_start_finish), + jiffies_to_msecs(bfqq->wr_cur_max_time)); } spin_unlock_irq(bfqd->queue->queue_lock); @@ -4608,16 +4631,18 @@ static ssize_t bfq_weights_show(struct elevator_queue *e, char *page) static ssize_t __FUNC(struct elevator_queue *e, char *page) \ { \ struct bfq_data *bfqd = e->elevator_data; \ - unsigned int __data = __VAR; \ - if (__CONV) \ + u64 __data = __VAR; \ + if (__CONV == 1) \ __data = jiffies_to_msecs(__data); \ + else if (__CONV == 2) \ + __data = div_u64(__data, NSEC_PER_MSEC); \ return bfq_var_show(__data, (page)); \ } -SHOW_FUNCTION(bfq_fifo_expire_sync_show, bfqd->bfq_fifo_expire[1], 1); -SHOW_FUNCTION(bfq_fifo_expire_async_show, bfqd->bfq_fifo_expire[0], 1); +SHOW_FUNCTION(bfq_fifo_expire_sync_show, bfqd->bfq_fifo_expire[1], 2); +SHOW_FUNCTION(bfq_fifo_expire_async_show, bfqd->bfq_fifo_expire[0], 2); SHOW_FUNCTION(bfq_back_seek_max_show, bfqd->bfq_back_max, 0); SHOW_FUNCTION(bfq_back_seek_penalty_show, bfqd->bfq_back_penalty, 0); -SHOW_FUNCTION(bfq_slice_idle_show, bfqd->bfq_slice_idle, 1); +SHOW_FUNCTION(bfq_slice_idle_show, bfqd->bfq_slice_idle, 2); SHOW_FUNCTION(bfq_max_budget_show, bfqd->bfq_user_max_budget, 0); SHOW_FUNCTION(bfq_timeout_sync_show, bfqd->bfq_timeout, 1); SHOW_FUNCTION(bfq_strict_guarantees_show, bfqd->strict_guarantees, 0); @@ -4630,6 +4655,17 @@ SHOW_FUNCTION(bfq_wr_min_inter_arr_async_show, bfqd->bfq_wr_min_inter_arr_async, SHOW_FUNCTION(bfq_wr_max_softrt_rate_show, bfqd->bfq_wr_max_softrt_rate, 0); #undef SHOW_FUNCTION +#define USEC_SHOW_FUNCTION(__FUNC, __VAR) \ +static ssize_t __FUNC(struct elevator_queue *e, char *page) \ +{ \ + struct bfq_data *bfqd = e->elevator_data; \ + u64 __data = __VAR; \ + __data = div_u64(__data, NSEC_PER_USEC); \ + return bfq_var_show(__data, (page)); \ +} +USEC_SHOW_FUNCTION(bfq_slice_idle_us_show, bfqd->bfq_slice_idle); +#undef USEC_SHOW_FUNCTION + #define STORE_FUNCTION(__FUNC, __PTR, MIN, MAX, __CONV) \ static ssize_t \ __FUNC(struct elevator_queue *e, const char *page, size_t count) \ @@ -4641,20 +4677,22 @@ __FUNC(struct elevator_queue *e, const char *page, size_t count) \ __data = (MIN); \ else if (__data > (MAX)) \ __data = (MAX); \ - if (__CONV) \ + if (__CONV == 1) \ *(__PTR) = msecs_to_jiffies(__data); \ + else if (__CONV == 2) \ + *(__PTR) = (u64)__data * NSEC_PER_MSEC; \ else \ *(__PTR) = __data; \ return ret; \ } STORE_FUNCTION(bfq_fifo_expire_sync_store, &bfqd->bfq_fifo_expire[1], 1, - INT_MAX, 1); + INT_MAX, 2); STORE_FUNCTION(bfq_fifo_expire_async_store, &bfqd->bfq_fifo_expire[0], 1, - INT_MAX, 1); + INT_MAX, 2); STORE_FUNCTION(bfq_back_seek_max_store, &bfqd->bfq_back_max, 0, INT_MAX, 0); STORE_FUNCTION(bfq_back_seek_penalty_store, &bfqd->bfq_back_penalty, 1, INT_MAX, 0); -STORE_FUNCTION(bfq_slice_idle_store, &bfqd->bfq_slice_idle, 0, INT_MAX, 1); +STORE_FUNCTION(bfq_slice_idle_store, &bfqd->bfq_slice_idle, 0, INT_MAX, 2); STORE_FUNCTION(bfq_wr_coeff_store, &bfqd->bfq_wr_coeff, 1, INT_MAX, 0); STORE_FUNCTION(bfq_wr_max_time_store, &bfqd->bfq_wr_max_time, 0, INT_MAX, 1); STORE_FUNCTION(bfq_wr_rt_max_time_store, &bfqd->bfq_wr_rt_max_time, 0, INT_MAX, @@ -4667,6 +4705,23 @@ STORE_FUNCTION(bfq_wr_max_softrt_rate_store, &bfqd->bfq_wr_max_softrt_rate, 0, INT_MAX, 0); #undef STORE_FUNCTION +#define USEC_STORE_FUNCTION(__FUNC, __PTR, MIN, MAX) \ +static ssize_t __FUNC(struct elevator_queue *e, const char *page, size_t count)\ +{ \ + struct bfq_data *bfqd = e->elevator_data; \ + unsigned long __data; \ + int ret = bfq_var_store(&__data, (page), count); \ + if (__data < (MIN)) \ + __data = (MIN); \ + else if (__data > (MAX)) \ + __data = (MAX); \ + *(__PTR) = (u64)__data * NSEC_PER_USEC; \ + return ret; \ +} +USEC_STORE_FUNCTION(bfq_slice_idle_us_store, &bfqd->bfq_slice_idle, 0, + UINT_MAX); +#undef USEC_STORE_FUNCTION + /* do nothing for the moment */ static ssize_t bfq_weights_store(struct elevator_queue *e, const char *page, size_t count) @@ -4768,6 +4823,7 @@ static struct elv_fs_entry bfq_attrs[] = { BFQ_ATTR(back_seek_max), BFQ_ATTR(back_seek_penalty), BFQ_ATTR(slice_idle), + BFQ_ATTR(slice_idle_us), BFQ_ATTR(max_budget), BFQ_ATTR(timeout_sync), BFQ_ATTR(strict_guarantees), @@ -4790,7 +4846,8 @@ static struct elevator_type iosched_bfq = { #ifdef CONFIG_BFQ_GROUP_IOSCHED .elevator_bio_merged_fn = bfq_bio_merged, #endif - .elevator_allow_merge_fn = bfq_allow_merge, + .elevator_allow_bio_merge_fn = bfq_allow_bio_merge, + .elevator_allow_rq_merge_fn = bfq_allow_rq_merge, .elevator_dispatch_fn = bfq_dispatch_requests, .elevator_add_req_fn = bfq_insert_request, .elevator_activate_req_fn = bfq_activate_request, @@ -4836,12 +4893,6 @@ static int __init bfq_init(void) int ret; char msg[50] = "BFQ I/O-scheduler: v8r3"; - /* - * Can be 0 on HZ < 1000 setups. - */ - if (bfq_slice_idle == 0) - bfq_slice_idle = 1; - #ifdef CONFIG_BFQ_GROUP_IOSCHED ret = blkcg_policy_register(&blkcg_policy_bfq); if (ret) |