Skip to content

Commit c1c8038

Browse files
Josef Bacikaxboe
Josef Bacik
authored and
axboe
committed
block: remove external dependency on wbt_flags
We don't really need to save this stuff in the core block code, we can just pass the bio back into the helpers later on to derive the same flags and update the rq->wbt_flags appropriately. Signed-off-by: Josef Bacik <jbacik@fb.com> Signed-off-by: Jens Axboe <axboe@kernel.dk>
1 parent a790504 commit c1c8038

File tree

6 files changed

+68
-42
lines changed

6 files changed

+68
-42
lines changed

‎block/blk-core.c

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@
4242
#include "blk.h"
4343
#include "blk-mq.h"
4444
#include "blk-mq-sched.h"
45-
#include "blk-wbt.h"
45+
#include "blk-rq-qos.h"
4646

4747
#ifdef CONFIG_DEBUG_FS
4848
struct dentry *blk_debugfs_root;
@@ -1986,7 +1986,6 @@ static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio)
19861986
int where = ELEVATOR_INSERT_SORT;
19871987
struct request *req, *free;
19881988
unsigned int request_count = 0;
1989-
unsigned int wb_acct;
19901989

19911990
/*
19921991
* low level driver can indicate that it wants pages above a
@@ -2044,7 +2043,7 @@ static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio)
20442043
}
20452044

20462045
get_rq:
2047-
wb_acct = rq_qos_throttle(q, bio, q->queue_lock);
2046+
rq_qos_throttle(q, bio, q->queue_lock);
20482047

20492048
/*
20502049
* Grab a free request. This is might sleep but can not fail.
@@ -2054,7 +2053,7 @@ static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio)
20542053
req = get_request(q, bio->bi_opf, bio, 0, GFP_NOIO);
20552054
if (IS_ERR(req)) {
20562055
blk_queue_exit(q);
2057-
rq_qos_cleanup(q, wb_acct);
2056+
rq_qos_cleanup(q, bio);
20582057
if (PTR_ERR(req) == -ENOMEM)
20592058
bio->bi_status = BLK_STS_RESOURCE;
20602059
else
@@ -2063,7 +2062,7 @@ static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio)
20632062
goto out_unlock;
20642063
}
20652064

2066-
wbt_track(req, wb_acct);
2065+
rq_qos_track(q, req, bio);
20672066

20682067
/*
20692068
* After dropping the lock and possibly sleeping here, our request

‎block/blk-mq.c

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,8 @@
3434
#include "blk-mq-debugfs.h"
3535
#include "blk-mq-tag.h"
3636
#include "blk-stat.h"
37-
#include "blk-wbt.h"
3837
#include "blk-mq-sched.h"
38+
#include "blk-rq-qos.h"
3939

4040
static bool blk_mq_poll(struct request_queue *q, blk_qc_t cookie);
4141
static void blk_mq_poll_stats_start(struct request_queue *q);
@@ -1790,7 +1790,6 @@ static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio)
17901790
struct blk_plug *plug;
17911791
struct request *same_queue_rq = NULL;
17921792
blk_qc_t cookie;
1793-
unsigned int wb_acct;
17941793

17951794
blk_queue_bounce(q, &bio);
17961795

@@ -1806,19 +1805,19 @@ static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio)
18061805
if (blk_mq_sched_bio_merge(q, bio))
18071806
return BLK_QC_T_NONE;
18081807

1809-
wb_acct = rq_qos_throttle(q, bio, NULL);
1808+
rq_qos_throttle(q, bio, NULL);
18101809

18111810
trace_block_getrq(q, bio, bio->bi_opf);
18121811

18131812
rq = blk_mq_get_request(q, bio, bio->bi_opf, &data);
18141813
if (unlikely(!rq)) {
1815-
rq_qos_cleanup(q, wb_acct);
1814+
rq_qos_cleanup(q, bio);
18161815
if (bio->bi_opf & REQ_NOWAIT)
18171816
bio_wouldblock_error(bio);
18181817
return BLK_QC_T_NONE;
18191818
}
18201819

1821-
wbt_track(rq, wb_acct);
1820+
rq_qos_track(q, rq, bio);
18221821

18231822
cookie = request_to_qc_t(data.hctx, rq);
18241823

‎block/blk-rq-qos.c

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
#include "blk-rq-qos.h"
22

3-
#include "blk-wbt.h"
4-
53
/*
64
* Increment 'v', if 'v' is below 'below'. Returns true if we succeeded,
75
* false if 'v' + 1 would be bigger than 'below'.
@@ -29,13 +27,13 @@ bool rq_wait_inc_below(struct rq_wait *rq_wait, int limit)
2927
return atomic_inc_below(&rq_wait->inflight, limit);
3028
}
3129

32-
void rq_qos_cleanup(struct request_queue *q, enum wbt_flags wb_acct)
30+
void rq_qos_cleanup(struct request_queue *q, struct bio *bio)
3331
{
3432
struct rq_qos *rqos;
3533

3634
for (rqos = q->rq_qos; rqos; rqos = rqos->next) {
3735
if (rqos->ops->cleanup)
38-
rqos->ops->cleanup(rqos, wb_acct);
36+
rqos->ops->cleanup(rqos, bio);
3937
}
4038
}
4139

@@ -69,17 +67,25 @@ void rq_qos_requeue(struct request_queue *q, struct request *rq)
6967
}
7068
}
7169

72-
enum wbt_flags rq_qos_throttle(struct request_queue *q, struct bio *bio,
73-
spinlock_t *lock)
70+
void rq_qos_throttle(struct request_queue *q, struct bio *bio,
71+
spinlock_t *lock)
7472
{
7573
struct rq_qos *rqos;
76-
enum wbt_flags flags = 0;
7774

7875
for(rqos = q->rq_qos; rqos; rqos = rqos->next) {
7976
if (rqos->ops->throttle)
80-
flags |= rqos->ops->throttle(rqos, bio, lock);
77+
rqos->ops->throttle(rqos, bio, lock);
78+
}
79+
}
80+
81+
void rq_qos_track(struct request_queue *q, struct request *rq, struct bio *bio)
82+
{
83+
struct rq_qos *rqos;
84+
85+
for(rqos = q->rq_qos; rqos; rqos = rqos->next) {
86+
if (rqos->ops->track)
87+
rqos->ops->track(rqos, rq, bio);
8188
}
82-
return flags;
8389
}
8490

8591
/*

‎block/blk-rq-qos.h

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -25,12 +25,12 @@ struct rq_qos {
2525
};
2626

2727
struct rq_qos_ops {
28-
enum wbt_flags (*throttle)(struct rq_qos *, struct bio *,
29-
spinlock_t *);
28+
void (*throttle)(struct rq_qos *, struct bio *, spinlock_t *);
29+
void (*track)(struct rq_qos *, struct request *, struct bio *);
3030
void (*issue)(struct rq_qos *, struct request *);
3131
void (*requeue)(struct rq_qos *, struct request *);
3232
void (*done)(struct rq_qos *, struct request *);
33-
void (*cleanup)(struct rq_qos *, enum wbt_flags);
33+
void (*cleanup)(struct rq_qos *, struct bio *);
3434
void (*exit)(struct rq_qos *);
3535
};
3636

@@ -97,10 +97,11 @@ void rq_depth_scale_up(struct rq_depth *rqd);
9797
void rq_depth_scale_down(struct rq_depth *rqd, bool hard_throttle);
9898
bool rq_depth_calc_max_depth(struct rq_depth *rqd);
9999

100-
void rq_qos_cleanup(struct request_queue *, enum wbt_flags);
100+
void rq_qos_cleanup(struct request_queue *, struct bio *);
101101
void rq_qos_done(struct request_queue *, struct request *);
102102
void rq_qos_issue(struct request_queue *, struct request *);
103103
void rq_qos_requeue(struct request_queue *, struct request *);
104-
enum wbt_flags rq_qos_throttle(struct request_queue *, struct bio *, spinlock_t *);
104+
void rq_qos_throttle(struct request_queue *, struct bio *, spinlock_t *);
105+
void rq_qos_track(struct request_queue *q, struct request *, struct bio *);
105106
void rq_qos_exit(struct request_queue *);
106107
#endif

‎block/blk-wbt.c

Lines changed: 39 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -549,41 +549,66 @@ static inline bool wbt_should_throttle(struct rq_wb *rwb, struct bio *bio)
549549
}
550550
}
551551

552+
static enum wbt_flags bio_to_wbt_flags(struct rq_wb *rwb, struct bio *bio)
553+
{
554+
enum wbt_flags flags = 0;
555+
556+
if (bio_op(bio) == REQ_OP_READ) {
557+
flags = WBT_READ;
558+
} else if (wbt_should_throttle(rwb, bio)) {
559+
if (current_is_kswapd())
560+
flags |= WBT_KSWAPD;
561+
if (bio_op(bio) == REQ_OP_DISCARD)
562+
flags |= WBT_DISCARD;
563+
flags |= WBT_TRACKED;
564+
}
565+
return flags;
566+
}
567+
568+
static void wbt_cleanup(struct rq_qos *rqos, struct bio *bio)
569+
{
570+
struct rq_wb *rwb = RQWB(rqos);
571+
enum wbt_flags flags = bio_to_wbt_flags(rwb, bio);
572+
__wbt_done(rqos, flags);
573+
}
574+
552575
/*
553576
* Returns true if the IO request should be accounted, false if not.
554577
* May sleep, if we have exceeded the writeback limits. Caller can pass
555578
* in an irq held spinlock, if it holds one when calling this function.
556579
* If we do sleep, we'll release and re-grab it.
557580
*/
558-
static enum wbt_flags wbt_wait(struct rq_qos *rqos, struct bio *bio,
559-
spinlock_t *lock)
581+
static void wbt_wait(struct rq_qos *rqos, struct bio *bio, spinlock_t *lock)
560582
{
561583
struct rq_wb *rwb = RQWB(rqos);
562-
enum wbt_flags ret = 0;
584+
enum wbt_flags flags;
563585

564586
if (!rwb_enabled(rwb))
565-
return 0;
587+
return;
566588

567-
if (bio_op(bio) == REQ_OP_READ)
568-
ret = WBT_READ;
589+
flags = bio_to_wbt_flags(rwb, bio);
569590

570591
if (!wbt_should_throttle(rwb, bio)) {
571-
if (ret & WBT_READ)
592+
if (flags & WBT_READ)
572593
wb_timestamp(rwb, &rwb->last_issue);
573-
return ret;
594+
return;
574595
}
575596

576597
if (current_is_kswapd())
577-
ret |= WBT_KSWAPD;
598+
flags |= WBT_KSWAPD;
578599
if (bio_op(bio) == REQ_OP_DISCARD)
579-
ret |= WBT_DISCARD;
600+
flags |= WBT_DISCARD;
580601

581-
__wbt_wait(rwb, ret, bio->bi_opf, lock);
602+
__wbt_wait(rwb, flags, bio->bi_opf, lock);
582603

583604
if (!blk_stat_is_active(rwb->cb))
584605
rwb_arm_timer(rwb);
606+
}
585607

586-
return ret | WBT_TRACKED;
608+
static void wbt_track(struct rq_qos *rqos, struct request *rq, struct bio *bio)
609+
{
610+
struct rq_wb *rwb = RQWB(rqos);
611+
rq->wbt_flags |= bio_to_wbt_flags(rwb, bio);
587612
}
588613

589614
void wbt_issue(struct rq_qos *rqos, struct request *rq)
@@ -707,9 +732,10 @@ EXPORT_SYMBOL_GPL(wbt_disable_default);
707732
static struct rq_qos_ops wbt_rqos_ops = {
708733
.throttle = wbt_wait,
709734
.issue = wbt_issue,
735+
.track = wbt_track,
710736
.requeue = wbt_requeue,
711737
.done = wbt_done,
712-
.cleanup = __wbt_done,
738+
.cleanup = wbt_cleanup,
713739
.exit = wbt_exit,
714740
};
715741

‎block/blk-wbt.h

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -87,11 +87,6 @@ static inline unsigned int wbt_inflight(struct rq_wb *rwb)
8787

8888
#ifdef CONFIG_BLK_WBT
8989

90-
static inline void wbt_track(struct request *rq, enum wbt_flags flags)
91-
{
92-
rq->wbt_flags |= flags;
93-
}
94-
9590
int wbt_init(struct request_queue *);
9691
void wbt_update_limits(struct request_queue *);
9792
void wbt_disable_default(struct request_queue *);

0 commit comments

Comments
 (0)