Skip to content

Commit 4941115

Browse files
axboeaxboe
committed
blk-mq-tag: cleanup the normal/reserved tag allocation
This is in preparation for having another tag set available. Cleanup the parameters, and allow passing in of tags for blk_mq_put_tag(). Signed-off-by: Jens Axboe <axboe@fb.com> [hch: even more cleanups] Signed-off-by: Christoph Hellwig <hch@lst.de> Reviewed-by: Omar Sandoval <osandov@fb.com>
1 parent 2c3ad66 commit 4941115

File tree

4 files changed

+44
-61
lines changed

4 files changed

+44
-61
lines changed

‎block/blk-mq-tag.c

Lines changed: 36 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -90,32 +90,46 @@ static inline bool hctx_may_queue(struct blk_mq_hw_ctx *hctx,
9090
return atomic_read(&hctx->nr_active) < depth;
9191
}
9292

93-
static int __bt_get(struct blk_mq_hw_ctx *hctx, struct sbitmap_queue *bt)
93+
static int __blk_mq_get_tag(struct blk_mq_hw_ctx *hctx, struct sbitmap_queue *bt)
9494
{
9595
if (!hctx_may_queue(hctx, bt))
9696
return -1;
9797
return __sbitmap_queue_get(bt);
9898
}
9999

100-
static int bt_get(struct blk_mq_alloc_data *data, struct sbitmap_queue *bt,
101-
struct blk_mq_hw_ctx *hctx, struct blk_mq_tags *tags)
100+
unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data)
102101
{
102+
struct blk_mq_tags *tags = blk_mq_tags_from_data(data);
103+
struct sbitmap_queue *bt;
103104
struct sbq_wait_state *ws;
104105
DEFINE_WAIT(wait);
106+
unsigned int tag_offset;
105107
int tag;
106108

107-
tag = __bt_get(hctx, bt);
109+
if (data->flags & BLK_MQ_REQ_RESERVED) {
110+
if (unlikely(!tags->nr_reserved_tags)) {
111+
WARN_ON_ONCE(1);
112+
return BLK_MQ_TAG_FAIL;
113+
}
114+
bt = &tags->breserved_tags;
115+
tag_offset = 0;
116+
} else {
117+
bt = &tags->bitmap_tags;
118+
tag_offset = tags->nr_reserved_tags;
119+
}
120+
121+
tag = __blk_mq_get_tag(data->hctx, bt);
108122
if (tag != -1)
109-
return tag;
123+
goto found_tag;
110124

111125
if (data->flags & BLK_MQ_REQ_NOWAIT)
112-
return -1;
126+
return BLK_MQ_TAG_FAIL;
113127

114-
ws = bt_wait_ptr(bt, hctx);
128+
ws = bt_wait_ptr(bt, data->hctx);
115129
do {
116130
prepare_to_wait(&ws->wait, &wait, TASK_UNINTERRUPTIBLE);
117131

118-
tag = __bt_get(hctx, bt);
132+
tag = __blk_mq_get_tag(data->hctx, bt);
119133
if (tag != -1)
120134
break;
121135

@@ -125,14 +139,14 @@ static int bt_get(struct blk_mq_alloc_data *data, struct sbitmap_queue *bt,
125139
* some to complete. Note that hctx can be NULL here for
126140
* reserved tag allocation.
127141
*/
128-
if (hctx)
129-
blk_mq_run_hw_queue(hctx, false);
142+
if (data->hctx)
143+
blk_mq_run_hw_queue(data->hctx, false);
130144

131145
/*
132146
* Retry tag allocation after running the hardware queue,
133147
* as running the queue may also have found completions.
134148
*/
135-
tag = __bt_get(hctx, bt);
149+
tag = __blk_mq_get_tag(data->hctx, bt);
136150
if (tag != -1)
137151
break;
138152

@@ -142,61 +156,25 @@ static int bt_get(struct blk_mq_alloc_data *data, struct sbitmap_queue *bt,
142156

143157
data->ctx = blk_mq_get_ctx(data->q);
144158
data->hctx = blk_mq_map_queue(data->q, data->ctx->cpu);
145-
if (data->flags & BLK_MQ_REQ_RESERVED) {
146-
bt = &data->hctx->tags->breserved_tags;
147-
} else {
148-
hctx = data->hctx;
149-
bt = &hctx->tags->bitmap_tags;
150-
}
159+
tags = blk_mq_tags_from_data(data);
160+
if (data->flags & BLK_MQ_REQ_RESERVED)
161+
bt = &tags->breserved_tags;
162+
else
163+
bt = &tags->bitmap_tags;
164+
151165
finish_wait(&ws->wait, &wait);
152-
ws = bt_wait_ptr(bt, hctx);
166+
ws = bt_wait_ptr(bt, data->hctx);
153167
} while (1);
154168

155169
finish_wait(&ws->wait, &wait);
156-
return tag;
157-
}
158-
159-
static unsigned int __blk_mq_get_tag(struct blk_mq_alloc_data *data)
160-
{
161-
int tag;
162-
163-
tag = bt_get(data, &data->hctx->tags->bitmap_tags, data->hctx,
164-
data->hctx->tags);
165-
if (tag >= 0)
166-
return tag + data->hctx->tags->nr_reserved_tags;
167-
168-
return BLK_MQ_TAG_FAIL;
169-
}
170-
171-
static unsigned int __blk_mq_get_reserved_tag(struct blk_mq_alloc_data *data)
172-
{
173-
int tag;
174170

175-
if (unlikely(!data->hctx->tags->nr_reserved_tags)) {
176-
WARN_ON_ONCE(1);
177-
return BLK_MQ_TAG_FAIL;
178-
}
179-
180-
tag = bt_get(data, &data->hctx->tags->breserved_tags, NULL,
181-
data->hctx->tags);
182-
if (tag < 0)
183-
return BLK_MQ_TAG_FAIL;
184-
185-
return tag;
171+
found_tag:
172+
return tag + tag_offset;
186173
}
187174

188-
unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data)
175+
void blk_mq_put_tag(struct blk_mq_hw_ctx *hctx, struct blk_mq_tags *tags,
176+
struct blk_mq_ctx *ctx, unsigned int tag)
189177
{
190-
if (data->flags & BLK_MQ_REQ_RESERVED)
191-
return __blk_mq_get_reserved_tag(data);
192-
return __blk_mq_get_tag(data);
193-
}
194-
195-
void blk_mq_put_tag(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
196-
unsigned int tag)
197-
{
198-
struct blk_mq_tags *tags = hctx->tags;
199-
200178
if (tag >= tags->nr_reserved_tags) {
201179
const int real_tag = tag - tags->nr_reserved_tags;
202180

‎block/blk-mq-tag.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,8 @@ extern struct blk_mq_tags *blk_mq_init_tags(unsigned int nr_tags, unsigned int r
2424
extern void blk_mq_free_tags(struct blk_mq_tags *tags);
2525

2626
extern unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data);
27-
extern void blk_mq_put_tag(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
28-
unsigned int tag);
27+
extern void blk_mq_put_tag(struct blk_mq_hw_ctx *hctx, struct blk_mq_tags *tags,
28+
struct blk_mq_ctx *ctx, unsigned int tag);
2929
extern bool blk_mq_has_free_tags(struct blk_mq_tags *tags);
3030
extern ssize_t blk_mq_tag_sysfs_show(struct blk_mq_tags *tags, char *page);
3131
extern int blk_mq_tag_update_depth(struct blk_mq_tags *tags, unsigned int depth);

‎block/blk-mq.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -335,7 +335,7 @@ void __blk_mq_free_request(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
335335

336336
clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
337337
clear_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags);
338-
blk_mq_put_tag(hctx, ctx, tag);
338+
blk_mq_put_tag(hctx, hctx->tags, ctx, tag);
339339
blk_queue_exit(q);
340340
}
341341

‎block/blk-mq.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,11 @@ static inline void blk_mq_set_alloc_data(struct blk_mq_alloc_data *data,
118118
data->hctx = hctx;
119119
}
120120

121+
static inline struct blk_mq_tags *blk_mq_tags_from_data(struct blk_mq_alloc_data *data)
122+
{
123+
return data->hctx->tags;
124+
}
125+
121126
/*
122127
* Internal helpers for request allocation/init/free
123128
*/

0 commit comments

Comments
 (0)