Skip to content

Commit 544ccc8

Browse files
osandovaxboe
osandov
authored and
axboe
committed
block: get rid of struct blk_issue_stat
struct blk_issue_stat squashes three things into one u64: - The time the driver started working on a request - The original size of the request (for the io.low controller) - Flags for writeback throttling It turns out that on x86_64, we have a 4 byte hole in struct request which we can fill with the non-timestamp fields from blk_issue_stat, simplifying things quite a bit. Signed-off-by: Omar Sandoval <osandov@fb.com> Signed-off-by: Jens Axboe <axboe@kernel.dk>
1 parent 5238dcf commit 544ccc8

File tree

10 files changed

+41
-77
lines changed

10 files changed

+41
-77
lines changed

‎block/blk-core.c

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2991,7 +2991,10 @@ void blk_start_request(struct request *req)
29912991
blk_dequeue_request(req);
29922992

29932993
if (test_bit(QUEUE_FLAG_STATS, &req->q->queue_flags)) {
2994-
blk_stat_set_issue(&req->issue_stat, blk_rq_sectors(req));
2994+
req->io_start_time_ns = ktime_get_ns();
2995+
#ifdef CONFIG_BLK_DEV_THROTTLING_LOW
2996+
req->throtl_size = blk_rq_sectors(req);
2997+
#endif
29952998
req->rq_flags |= RQF_STATS;
29962999
wbt_issue(req->q->rq_wb, req);
29973000
}

‎block/blk-mq.c

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -310,6 +310,7 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data,
310310
rq->rq_disk = NULL;
311311
rq->part = NULL;
312312
rq->start_time = jiffies;
313+
rq->io_start_time_ns = 0;
313314
rq->nr_phys_segments = 0;
314315
#if defined(CONFIG_BLK_DEV_INTEGRITY)
315316
rq->nr_integrity_segments = 0;
@@ -329,7 +330,7 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data,
329330
#ifdef CONFIG_BLK_CGROUP
330331
rq->rl = NULL;
331332
set_start_time_ns(rq);
332-
rq->io_start_time_ns = 0;
333+
rq->cgroup_io_start_time_ns = 0;
333334
#endif
334335

335336
data->ctx->rq_dispatched[op_is_sync(op)]++;
@@ -669,7 +670,10 @@ void blk_mq_start_request(struct request *rq)
669670
trace_block_rq_issue(q, rq);
670671

671672
if (test_bit(QUEUE_FLAG_STATS, &q->queue_flags)) {
672-
blk_stat_set_issue(&rq->issue_stat, blk_rq_sectors(rq));
673+
rq->io_start_time_ns = ktime_get_ns();
674+
#ifdef CONFIG_BLK_DEV_THROTTLING_LOW
675+
rq->throtl_size = blk_rq_sectors(rq);
676+
#endif
673677
rq->rq_flags |= RQF_STATS;
674678
wbt_issue(q->rq_wb, rq);
675679
}

‎block/blk-stat.c

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -55,11 +55,8 @@ void blk_stat_add(struct request *rq)
5555
int bucket;
5656
u64 now, value;
5757

58-
now = __blk_stat_time(ktime_to_ns(ktime_get()));
59-
if (now < blk_stat_time(&rq->issue_stat))
60-
return;
61-
62-
value = now - blk_stat_time(&rq->issue_stat);
58+
now = ktime_get_ns();
59+
value = (now >= rq->io_start_time_ns) ? now - rq->io_start_time_ns : 0;
6360

6461
blk_throtl_stat_add(rq, value);
6562

‎block/blk-stat.h

Lines changed: 0 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -8,21 +8,6 @@
88
#include <linux/rcupdate.h>
99
#include <linux/timer.h>
1010

11-
/*
12-
* from upper:
13-
* 4 bits: reserved for other usage
14-
* 12 bits: size
15-
* 48 bits: time
16-
*/
17-
#define BLK_STAT_RES_BITS 4
18-
#define BLK_STAT_SIZE_BITS 12
19-
#define BLK_STAT_RES_SHIFT (64 - BLK_STAT_RES_BITS)
20-
#define BLK_STAT_SIZE_SHIFT (BLK_STAT_RES_SHIFT - BLK_STAT_SIZE_BITS)
21-
#define BLK_STAT_TIME_MASK ((1ULL << BLK_STAT_SIZE_SHIFT) - 1)
22-
#define BLK_STAT_SIZE_MASK \
23-
(((1ULL << BLK_STAT_SIZE_BITS) - 1) << BLK_STAT_SIZE_SHIFT)
24-
#define BLK_STAT_RES_MASK (~((1ULL << BLK_STAT_RES_SHIFT) - 1))
25-
2611
/**
2712
* struct blk_stat_callback - Block statistics callback.
2813
*
@@ -82,34 +67,6 @@ void blk_free_queue_stats(struct blk_queue_stats *);
8267

8368
void blk_stat_add(struct request *);
8469

85-
static inline u64 __blk_stat_time(u64 time)
86-
{
87-
return time & BLK_STAT_TIME_MASK;
88-
}
89-
90-
static inline u64 blk_stat_time(struct blk_issue_stat *stat)
91-
{
92-
return __blk_stat_time(stat->stat);
93-
}
94-
95-
static inline sector_t blk_capped_size(sector_t size)
96-
{
97-
return size & ((1ULL << BLK_STAT_SIZE_BITS) - 1);
98-
}
99-
100-
static inline sector_t blk_stat_size(struct blk_issue_stat *stat)
101-
{
102-
return (stat->stat & BLK_STAT_SIZE_MASK) >> BLK_STAT_SIZE_SHIFT;
103-
}
104-
105-
static inline void blk_stat_set_issue(struct blk_issue_stat *stat,
106-
sector_t size)
107-
{
108-
stat->stat = (stat->stat & BLK_STAT_RES_MASK) |
109-
(ktime_to_ns(ktime_get()) & BLK_STAT_TIME_MASK) |
110-
(((u64)blk_capped_size(size)) << BLK_STAT_SIZE_SHIFT);
111-
}
112-
11370
/* record time/size info in request but not add a callback */
11471
void blk_stat_enable_accounting(struct request_queue *q);
11572

‎block/blk-throttle.c

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2279,8 +2279,7 @@ void blk_throtl_stat_add(struct request *rq, u64 time_ns)
22792279
struct request_queue *q = rq->q;
22802280
struct throtl_data *td = q->td;
22812281

2282-
throtl_track_latency(td, blk_stat_size(&rq->issue_stat),
2283-
req_op(rq), time_ns >> 10);
2282+
throtl_track_latency(td, rq->throtl_size, req_op(rq), time_ns >> 10);
22842283
}
22852284

22862285
void blk_throtl_bio_endio(struct bio *bio)

‎block/blk-wbt.c

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -31,22 +31,22 @@
3131

3232
static inline void wbt_clear_state(struct request *rq)
3333
{
34-
rq->issue_stat.stat &= ~BLK_STAT_RES_MASK;
34+
rq->wbt_flags = 0;
3535
}
3636

3737
static inline enum wbt_flags wbt_flags(struct request *rq)
3838
{
39-
return (rq->issue_stat.stat & BLK_STAT_RES_MASK) >> BLK_STAT_RES_SHIFT;
39+
return rq->wbt_flags;
4040
}
4141

4242
static inline bool wbt_is_tracked(struct request *rq)
4343
{
44-
return (rq->issue_stat.stat >> BLK_STAT_RES_SHIFT) & WBT_TRACKED;
44+
return rq->wbt_flags & WBT_TRACKED;
4545
}
4646

4747
static inline bool wbt_is_read(struct request *rq)
4848
{
49-
return (rq->issue_stat.stat >> BLK_STAT_RES_SHIFT) & WBT_READ;
49+
return rq->wbt_flags & WBT_READ;
5050
}
5151

5252
enum {
@@ -657,7 +657,7 @@ void wbt_issue(struct rq_wb *rwb, struct request *rq)
657657
*/
658658
if (wbt_is_read(rq) && !rwb->sync_issue) {
659659
rwb->sync_cookie = rq;
660-
rwb->sync_issue = blk_stat_time(&rq->issue_stat);
660+
rwb->sync_issue = rq->io_start_time_ns;
661661
}
662662
}
663663

@@ -746,8 +746,6 @@ int wbt_init(struct request_queue *q)
746746
struct rq_wb *rwb;
747747
int i;
748748

749-
BUILD_BUG_ON(WBT_NR_BITS > BLK_STAT_RES_BITS);
750-
751749
rwb = kzalloc(sizeof(*rwb), GFP_KERNEL);
752750
if (!rwb)
753751
return -ENOMEM;

‎block/blk-wbt.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ struct rq_wb {
6363

6464
struct blk_stat_callback *cb;
6565

66-
s64 sync_issue;
66+
u64 sync_issue;
6767
void *sync_cookie;
6868

6969
unsigned int wc;
@@ -90,7 +90,7 @@ static inline unsigned int wbt_inflight(struct rq_wb *rwb)
9090

9191
static inline void wbt_track(struct request *rq, enum wbt_flags flags)
9292
{
93-
rq->issue_stat.stat |= ((u64)flags) << BLK_STAT_RES_SHIFT;
93+
rq->wbt_flags |= flags;
9494
}
9595

9696
void __wbt_done(struct rq_wb *, enum wbt_flags);

‎block/kyber-iosched.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -485,11 +485,11 @@ static void kyber_completed_request(struct request *rq)
485485
if (blk_stat_is_active(kqd->cb))
486486
return;
487487

488-
now = __blk_stat_time(ktime_to_ns(ktime_get()));
489-
if (now < blk_stat_time(&rq->issue_stat))
488+
now = ktime_get_ns();
489+
if (now < rq->io_start_time_ns)
490490
return;
491491

492-
latency = now - blk_stat_time(&rq->issue_stat);
492+
latency = now - rq->io_start_time_ns;
493493

494494
if (latency > target)
495495
blk_stat_activate_msecs(kqd->cb, 10);

‎include/linux/blk_types.h

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -91,10 +91,6 @@ static inline bool blk_path_error(blk_status_t error)
9191
return true;
9292
}
9393

94-
struct blk_issue_stat {
95-
u64 stat;
96-
};
97-
9894
/*
9995
* From most significant bit:
10096
* 1 bit: reserved for other usage, see below

‎include/linux/blkdev.h

Lines changed: 18 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -206,8 +206,18 @@ struct request {
206206
struct gendisk *rq_disk;
207207
struct hd_struct *part;
208208
unsigned long start_time;
209-
struct blk_issue_stat issue_stat;
210-
/* Number of scatter-gather DMA addr+len pairs after
209+
/* Time that I/O was submitted to the device. */
210+
u64 io_start_time_ns;
211+
212+
#ifdef CONFIG_BLK_WBT
213+
unsigned short wbt_flags;
214+
#endif
215+
#ifdef CONFIG_BLK_DEV_THROTTLING_LOW
216+
unsigned short throtl_size;
217+
#endif
218+
219+
/*
220+
* Number of scatter-gather DMA addr+len pairs after
211221
* physical address coalescing is performed.
212222
*/
213223
unsigned short nr_phys_segments;
@@ -267,8 +277,8 @@ struct request {
267277

268278
#ifdef CONFIG_BLK_CGROUP
269279
struct request_list *rl; /* rl this rq is alloced from */
270-
unsigned long long start_time_ns;
271-
unsigned long long io_start_time_ns; /* when passed to hardware */
280+
unsigned long long cgroup_start_time_ns;
281+
unsigned long long cgroup_io_start_time_ns; /* when passed to hardware */
272282
#endif
273283
};
274284

@@ -1797,25 +1807,25 @@ int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork, unsigned lo
17971807
static inline void set_start_time_ns(struct request *req)
17981808
{
17991809
preempt_disable();
1800-
req->start_time_ns = sched_clock();
1810+
req->cgroup_start_time_ns = sched_clock();
18011811
preempt_enable();
18021812
}
18031813

18041814
static inline void set_io_start_time_ns(struct request *req)
18051815
{
18061816
preempt_disable();
1807-
req->io_start_time_ns = sched_clock();
1817+
req->cgroup_io_start_time_ns = sched_clock();
18081818
preempt_enable();
18091819
}
18101820

18111821
static inline uint64_t rq_start_time_ns(struct request *req)
18121822
{
1813-
return req->start_time_ns;
1823+
return req->cgroup_start_time_ns;
18141824
}
18151825

18161826
static inline uint64_t rq_io_start_time_ns(struct request *req)
18171827
{
1818-
return req->io_start_time_ns;
1828+
return req->cgroup_io_start_time_ns;
18191829
}
18201830
#else
18211831
static inline void set_start_time_ns(struct request *req) {}

0 commit comments

Comments
 (0)