Skip to content

Commit a790504

Browse files
Josef Bacikaxboe
Josef Bacik
authored and
axboe
committed
blk-rq-qos: refactor out common elements of blk-wbt
blkcg-qos is going to do essentially what wbt does, only on a cgroup basis. Break out the common code that will be shared between blkcg-qos and wbt into blk-rq-qos.* so they can both utilize the same infrastructure. Signed-off-by: Josef Bacik <jbacik@fb.com> Signed-off-by: Jens Axboe <axboe@kernel.dk>
1 parent 2ecbf45 commit a790504

File tree

10 files changed

+478
-251
lines changed

10 files changed

+478
-251
lines changed

‎block/Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ obj-$(CONFIG_BLOCK) := bio.o elevator.o blk-core.o blk-tag.o blk-sysfs.o \
99
blk-lib.o blk-mq.o blk-mq-tag.o blk-stat.o \
1010
blk-mq-sysfs.o blk-mq-cpumap.o blk-mq-sched.o ioctl.o \
1111
genhd.o partition-generic.o ioprio.o \
12-
badblocks.o partitions/
12+
badblocks.o partitions/ blk-rq-qos.o
1313

1414
obj-$(CONFIG_BOUNCE) += bounce.o
1515
obj-$(CONFIG_BLK_SCSI_REQUEST) += scsi_ioctl.o

‎block/blk-core.c

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1645,7 +1645,7 @@ void blk_requeue_request(struct request_queue *q, struct request *rq)
16451645
blk_delete_timer(rq);
16461646
blk_clear_rq_complete(rq);
16471647
trace_block_rq_requeue(q, rq);
1648-
wbt_requeue(q->rq_wb, rq);
1648+
rq_qos_requeue(q, rq);
16491649

16501650
if (rq->rq_flags & RQF_QUEUED)
16511651
blk_queue_end_tag(q, rq);
@@ -1752,7 +1752,7 @@ void __blk_put_request(struct request_queue *q, struct request *req)
17521752
/* this is a bio leak */
17531753
WARN_ON(req->bio != NULL);
17541754

1755-
wbt_done(q->rq_wb, req);
1755+
rq_qos_done(q, req);
17561756

17571757
/*
17581758
* Request may not have originated from ll_rw_blk. if not,
@@ -2044,7 +2044,7 @@ static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio)
20442044
}
20452045

20462046
get_rq:
2047-
wb_acct = wbt_wait(q->rq_wb, bio, q->queue_lock);
2047+
wb_acct = rq_qos_throttle(q, bio, q->queue_lock);
20482048

20492049
/*
20502050
* Grab a free request. This is might sleep but can not fail.
@@ -2054,7 +2054,7 @@ static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio)
20542054
req = get_request(q, bio->bi_opf, bio, 0, GFP_NOIO);
20552055
if (IS_ERR(req)) {
20562056
blk_queue_exit(q);
2057-
__wbt_done(q->rq_wb, wb_acct);
2057+
rq_qos_cleanup(q, wb_acct);
20582058
if (PTR_ERR(req) == -ENOMEM)
20592059
bio->bi_status = BLK_STS_RESOURCE;
20602060
else
@@ -2983,7 +2983,7 @@ void blk_start_request(struct request *req)
29832983
req->throtl_size = blk_rq_sectors(req);
29842984
#endif
29852985
req->rq_flags |= RQF_STATS;
2986-
wbt_issue(req->q->rq_wb, req);
2986+
rq_qos_issue(req->q, req);
29872987
}
29882988

29892989
BUG_ON(blk_rq_is_complete(req));
@@ -3207,7 +3207,7 @@ void blk_finish_request(struct request *req, blk_status_t error)
32073207
blk_account_io_done(req, now);
32083208

32093209
if (req->end_io) {
3210-
wbt_done(req->q->rq_wb, req);
3210+
rq_qos_done(q, req);
32113211
req->end_io(req, error);
32123212
} else {
32133213
if (blk_bidi_rq(req))

‎block/blk-mq.c

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -504,7 +504,7 @@ void blk_mq_free_request(struct request *rq)
504504
if (unlikely(laptop_mode && !blk_rq_is_passthrough(rq)))
505505
laptop_io_completion(q->backing_dev_info);
506506

507-
wbt_done(q->rq_wb, rq);
507+
rq_qos_done(q, rq);
508508

509509
if (blk_rq_rl(rq))
510510
blk_put_rl(blk_rq_rl(rq));
@@ -527,7 +527,7 @@ inline void __blk_mq_end_request(struct request *rq, blk_status_t error)
527527
blk_account_io_done(rq, now);
528528

529529
if (rq->end_io) {
530-
wbt_done(rq->q->rq_wb, rq);
530+
rq_qos_done(rq->q, rq);
531531
rq->end_io(rq, error);
532532
} else {
533533
if (unlikely(blk_bidi_rq(rq)))
@@ -641,7 +641,7 @@ void blk_mq_start_request(struct request *rq)
641641
rq->throtl_size = blk_rq_sectors(rq);
642642
#endif
643643
rq->rq_flags |= RQF_STATS;
644-
wbt_issue(q->rq_wb, rq);
644+
rq_qos_issue(q, rq);
645645
}
646646

647647
WARN_ON_ONCE(blk_mq_rq_state(rq) != MQ_RQ_IDLE);
@@ -667,7 +667,7 @@ static void __blk_mq_requeue_request(struct request *rq)
667667
blk_mq_put_driver_tag(rq);
668668

669669
trace_block_rq_requeue(q, rq);
670-
wbt_requeue(q->rq_wb, rq);
670+
rq_qos_requeue(q, rq);
671671

672672
if (blk_mq_request_started(rq)) {
673673
WRITE_ONCE(rq->state, MQ_RQ_IDLE);
@@ -1806,13 +1806,13 @@ static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio)
18061806
if (blk_mq_sched_bio_merge(q, bio))
18071807
return BLK_QC_T_NONE;
18081808

1809-
wb_acct = wbt_wait(q->rq_wb, bio, NULL);
1809+
wb_acct = rq_qos_throttle(q, bio, NULL);
18101810

18111811
trace_block_getrq(q, bio, bio->bi_opf);
18121812

18131813
rq = blk_mq_get_request(q, bio, bio->bi_opf, &data);
18141814
if (unlikely(!rq)) {
1815-
__wbt_done(q->rq_wb, wb_acct);
1815+
rq_qos_cleanup(q, wb_acct);
18161816
if (bio->bi_opf & REQ_NOWAIT)
18171817
bio_wouldblock_error(bio);
18181818
return BLK_QC_T_NONE;

‎block/blk-rq-qos.c

Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
#include "blk-rq-qos.h"
2+
3+
#include "blk-wbt.h"
4+
5+
/*
6+
* Increment 'v', if 'v' is below 'below'. Returns true if we succeeded,
7+
* false if 'v' + 1 would be bigger than 'below'.
8+
*/
9+
static bool atomic_inc_below(atomic_t *v, int below)
10+
{
11+
int cur = atomic_read(v);
12+
13+
for (;;) {
14+
int old;
15+
16+
if (cur >= below)
17+
return false;
18+
old = atomic_cmpxchg(v, cur, cur + 1);
19+
if (old == cur)
20+
break;
21+
cur = old;
22+
}
23+
24+
return true;
25+
}
26+
27+
bool rq_wait_inc_below(struct rq_wait *rq_wait, int limit)
28+
{
29+
return atomic_inc_below(&rq_wait->inflight, limit);
30+
}
31+
32+
void rq_qos_cleanup(struct request_queue *q, enum wbt_flags wb_acct)
33+
{
34+
struct rq_qos *rqos;
35+
36+
for (rqos = q->rq_qos; rqos; rqos = rqos->next) {
37+
if (rqos->ops->cleanup)
38+
rqos->ops->cleanup(rqos, wb_acct);
39+
}
40+
}
41+
42+
void rq_qos_done(struct request_queue *q, struct request *rq)
43+
{
44+
struct rq_qos *rqos;
45+
46+
for (rqos = q->rq_qos; rqos; rqos = rqos->next) {
47+
if (rqos->ops->done)
48+
rqos->ops->done(rqos, rq);
49+
}
50+
}
51+
52+
void rq_qos_issue(struct request_queue *q, struct request *rq)
53+
{
54+
struct rq_qos *rqos;
55+
56+
for(rqos = q->rq_qos; rqos; rqos = rqos->next) {
57+
if (rqos->ops->issue)
58+
rqos->ops->issue(rqos, rq);
59+
}
60+
}
61+
62+
void rq_qos_requeue(struct request_queue *q, struct request *rq)
63+
{
64+
struct rq_qos *rqos;
65+
66+
for(rqos = q->rq_qos; rqos; rqos = rqos->next) {
67+
if (rqos->ops->requeue)
68+
rqos->ops->requeue(rqos, rq);
69+
}
70+
}
71+
72+
enum wbt_flags rq_qos_throttle(struct request_queue *q, struct bio *bio,
73+
spinlock_t *lock)
74+
{
75+
struct rq_qos *rqos;
76+
enum wbt_flags flags = 0;
77+
78+
for(rqos = q->rq_qos; rqos; rqos = rqos->next) {
79+
if (rqos->ops->throttle)
80+
flags |= rqos->ops->throttle(rqos, bio, lock);
81+
}
82+
return flags;
83+
}
84+
85+
/*
86+
* Return true, if we can't increase the depth further by scaling
87+
*/
88+
bool rq_depth_calc_max_depth(struct rq_depth *rqd)
89+
{
90+
unsigned int depth;
91+
bool ret = false;
92+
93+
/*
94+
* For QD=1 devices, this is a special case. It's important for those
95+
* to have one request ready when one completes, so force a depth of
96+
* 2 for those devices. On the backend, it'll be a depth of 1 anyway,
97+
* since the device can't have more than that in flight. If we're
98+
* scaling down, then keep a setting of 1/1/1.
99+
*/
100+
if (rqd->queue_depth == 1) {
101+
if (rqd->scale_step > 0)
102+
rqd->max_depth = 1;
103+
else {
104+
rqd->max_depth = 2;
105+
ret = true;
106+
}
107+
} else {
108+
/*
109+
* scale_step == 0 is our default state. If we have suffered
110+
* latency spikes, step will be > 0, and we shrink the
111+
* allowed write depths. If step is < 0, we're only doing
112+
* writes, and we allow a temporarily higher depth to
113+
* increase performance.
114+
*/
115+
depth = min_t(unsigned int, rqd->default_depth,
116+
rqd->queue_depth);
117+
if (rqd->scale_step > 0)
118+
depth = 1 + ((depth - 1) >> min(31, rqd->scale_step));
119+
else if (rqd->scale_step < 0) {
120+
unsigned int maxd = 3 * rqd->queue_depth / 4;
121+
122+
depth = 1 + ((depth - 1) << -rqd->scale_step);
123+
if (depth > maxd) {
124+
depth = maxd;
125+
ret = true;
126+
}
127+
}
128+
129+
rqd->max_depth = depth;
130+
}
131+
132+
return ret;
133+
}
134+
135+
void rq_depth_scale_up(struct rq_depth *rqd)
136+
{
137+
/*
138+
* Hit max in previous round, stop here
139+
*/
140+
if (rqd->scaled_max)
141+
return;
142+
143+
rqd->scale_step--;
144+
145+
rqd->scaled_max = rq_depth_calc_max_depth(rqd);
146+
}
147+
148+
/*
149+
* Scale rwb down. If 'hard_throttle' is set, do it quicker, since we
150+
* had a latency violation.
151+
*/
152+
void rq_depth_scale_down(struct rq_depth *rqd, bool hard_throttle)
153+
{
154+
/*
155+
* Stop scaling down when we've hit the limit. This also prevents
156+
* ->scale_step from going to crazy values, if the device can't
157+
* keep up.
158+
*/
159+
if (rqd->max_depth == 1)
160+
return;
161+
162+
if (rqd->scale_step < 0 && hard_throttle)
163+
rqd->scale_step = 0;
164+
else
165+
rqd->scale_step++;
166+
167+
rqd->scaled_max = false;
168+
rq_depth_calc_max_depth(rqd);
169+
}
170+
171+
void rq_qos_exit(struct request_queue *q)
172+
{
173+
while (q->rq_qos) {
174+
struct rq_qos *rqos = q->rq_qos;
175+
q->rq_qos = rqos->next;
176+
rqos->ops->exit(rqos);
177+
}
178+
}

0 commit comments

Comments
 (0)