1 files changed, 58 insertions, 17 deletions

diff --git a/include/linux/wbt.h b/include/linux/wbt.h
index 872da2d73..68ba75e3a 100644
--- a/include/linux/wbt.h
+++ b/include/linux/wbt.h
@@ -6,8 +6,24 @@
 #include <linux/timer.h>
 #include <linux/ktime.h>
 
-#define ISSUE_STAT_MASK		(1ULL << 63)
-#define ISSUE_STAT_TIME_MASK	~ISSUE_STAT_MASK
+enum wbt_flags {
+	WBT_TRACKED		= 1,	/* write, tracked for throttling */
+	WBT_READ		= 2,	/* read */
+	WBT_KSWAPD		= 4,	/* write, from kswapd */
+
+	WBT_NR_BITS		= 3,	/* number of bits */
+};
+
+enum {
+	/*
+	 * Set aside 3 bits for state, rest is a time stamp
+	 */
+	ISSUE_STAT_SHIFT	= 64 - WBT_NR_BITS,
+	ISSUE_STAT_MASK 	= ~((1ULL << ISSUE_STAT_SHIFT) - 1),
+	ISSUE_STAT_TIME_MASK	= ~ISSUE_STAT_MASK,
+
+	WBT_NUM_RWQ		= 2,
+};
 
 struct wb_issue_stat {
 	u64 time;
@@ -24,26 +40,42 @@ static inline u64 wbt_issue_stat_get_time(struct wb_issue_stat *stat)
 	return stat->time & ISSUE_STAT_TIME_MASK;
 }
 
-static inline void wbt_mark_tracked(struct wb_issue_stat *stat)
+static inline void wbt_clear_state(struct wb_issue_stat *stat)
 {
-	stat->time |= ISSUE_STAT_MASK;
+	stat->time &= ISSUE_STAT_TIME_MASK;
 }
 
-static inline void wbt_clear_tracked(struct wb_issue_stat *stat)
+static inline enum wbt_flags wbt_stat_to_mask(struct wb_issue_stat *stat)
 {
-	stat->time &= ~ISSUE_STAT_MASK;
+	return (stat->time & ISSUE_STAT_MASK) >> ISSUE_STAT_SHIFT;
 }
 
-static inline bool wbt_tracked(struct wb_issue_stat *stat)
+static inline void wbt_track(struct wb_issue_stat *stat, enum wbt_flags wb_acct)
 {
-	return (stat->time & ISSUE_STAT_MASK) != 0;
+	stat->time |= ((u64) wb_acct) << ISSUE_STAT_SHIFT;
+}
+
+static inline bool wbt_is_tracked(struct wb_issue_stat *stat)
+{
+	return (stat->time >> ISSUE_STAT_SHIFT) & WBT_TRACKED;
+}
+
+static inline bool wbt_is_read(struct wb_issue_stat *stat)
+{
+	return (stat->time >> ISSUE_STAT_SHIFT) & WBT_READ;
 }
 
 struct wb_stat_ops {
 	void (*get)(void *, struct blk_rq_stat *);
+	bool (*is_current)(struct blk_rq_stat *);
 	void (*clear)(void *);
 };
 
+struct rq_wait {
+	wait_queue_head_t wait;
+	atomic_t inflight;
+};
+
 struct rq_wb {
 	/*
 	 * Settings that govern how we throttle
@@ -51,10 +83,8 @@ struct rq_wb {
 	unsigned int wb_background;		/* background writeback */
 	unsigned int wb_normal;			/* normal writeback */
 	unsigned int wb_max;			/* max throughput writeback */
-	unsigned int scale_step;
-
-	u64 win_nsec;				/* default window size */
-	u64 cur_win_nsec;			/* current window size */
+	int scale_step;
+	bool scaled_max;
 
 	/*
 	 * Number of consecutive periods where we don't have enough
@@ -62,6 +92,9 @@ struct rq_wb {
 	 */
 	unsigned int unknown_cnt;
 
+	u64 win_nsec;				/* default window size */
+	u64 cur_win_nsec;			/* current window size */
+
 	struct timer_list window_timer;
 
 	s64 sync_issue;
@@ -74,19 +107,27 @@ struct rq_wb {
 	unsigned long last_comp;		/* last non-throttled comp */
 	unsigned long min_lat_nsec;
 	struct backing_dev_info *bdi;
-	struct request_queue *q;
-	wait_queue_head_t wait;
-	atomic_t inflight;
+	struct rq_wait rq_wait[WBT_NUM_RWQ];
 
 	struct wb_stat_ops *stat_ops;
 	void *ops_data;
 };
 
+static inline unsigned int wbt_inflight(struct rq_wb *rwb)
+{
+	unsigned int i, ret = 0;
+
+	for (i = 0; i < WBT_NUM_RWQ; i++)
+		ret += atomic_read(&rwb->rq_wait[i].inflight);
+
+	return ret;
+}
+
 struct backing_dev_info;
 
-void __wbt_done(struct rq_wb *);
+void __wbt_done(struct rq_wb *, enum wbt_flags);
 void wbt_done(struct rq_wb *, struct wb_issue_stat *);
-bool wbt_wait(struct rq_wb *, unsigned int, spinlock_t *);
+enum wbt_flags wbt_wait(struct rq_wb *, unsigned int, spinlock_t *);
 struct rq_wb *wbt_init(struct backing_dev_info *, struct wb_stat_ops *, void *);
 void wbt_exit(struct rq_wb *);
 void wbt_update_limits(struct rq_wb *);