Skip to content

Commit a4a1cc1

Browse files
axboeaxboe
committed
Merge branch 'for-3.20/core' into for-3.20/drivers
We need the tagging changes for the libata conversion.
2 parents 121c7ad + 24391c0 commit a4a1cc1

File tree

16 files changed

+146
-62
lines changed

16 files changed

+146
-62
lines changed

‎block/blk-lib.c

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -283,23 +283,45 @@ static int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
283283
* @sector: start sector
284284
* @nr_sects: number of sectors to write
285285
* @gfp_mask: memory allocation flags (for bio_alloc)
286+
* @discard: whether to discard the block range
286287
*
287288
* Description:
288-
* Generate and issue number of bios with zerofiled pages.
289+
290+
* Zero-fill a block range. If the discard flag is set and the block
291+
* device guarantees that subsequent READ operations to the block range
292+
* in question will return zeroes, the blocks will be discarded. Should
293+
* the discard request fail, if the discard flag is not set, or if
294+
* discard_zeroes_data is not supported, this function will resort to
295+
* zeroing the blocks manually, thus provisioning (allocating,
296+
* anchoring) them. If the block device supports the WRITE SAME command
297+
* blkdev_issue_zeroout() will use it to optimize the process of
298+
* clearing the block range. Otherwise the zeroing will be performed
299+
* using regular WRITE calls.
289300
*/
290301

291302
int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
292-
sector_t nr_sects, gfp_t gfp_mask)
303+
sector_t nr_sects, gfp_t gfp_mask, bool discard)
293304
{
305+
struct request_queue *q = bdev_get_queue(bdev);
306+
unsigned char bdn[BDEVNAME_SIZE];
307+
308+
if (discard && blk_queue_discard(q) && q->limits.discard_zeroes_data) {
309+
310+
if (!blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask, 0))
311+
return 0;
312+
313+
bdevname(bdev, bdn);
314+
pr_warn("%s: DISCARD failed. Manually zeroing.\n", bdn);
315+
}
316+
294317
if (bdev_write_same(bdev)) {
295-
unsigned char bdn[BDEVNAME_SIZE];
296318

297319
if (!blkdev_issue_write_same(bdev, sector, nr_sects, gfp_mask,
298320
ZERO_PAGE(0)))
299321
return 0;
300322

301323
bdevname(bdev, bdn);
302-
pr_err("%s: WRITE SAME failed. Manually zeroing.\n", bdn);
324+
pr_warn("%s: WRITE SAME failed. Manually zeroing.\n", bdn);
303325
}
304326

305327
return __blkdev_issue_zeroout(bdev, sector, nr_sects, gfp_mask);

‎block/blk-mq-tag.c

Lines changed: 49 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -140,35 +140,39 @@ static inline bool hctx_may_queue(struct blk_mq_hw_ctx *hctx,
140140
return atomic_read(&hctx->nr_active) < depth;
141141
}
142142

143-
static int __bt_get_word(struct blk_align_bitmap *bm, unsigned int last_tag)
143+
static int __bt_get_word(struct blk_align_bitmap *bm, unsigned int last_tag,
144+
bool nowrap)
144145
{
145-
int tag, org_last_tag, end;
146-
bool wrap = last_tag != 0;
146+
int tag, org_last_tag = last_tag;
147147

148-
org_last_tag = last_tag;
149-
end = bm->depth;
150-
do {
151-
restart:
152-
tag = find_next_zero_bit(&bm->word, end, last_tag);
153-
if (unlikely(tag >= end)) {
148+
while (1) {
149+
tag = find_next_zero_bit(&bm->word, bm->depth, last_tag);
150+
if (unlikely(tag >= bm->depth)) {
154151
/*
155-
* We started with an offset, start from 0 to
152+
* We started with an offset, and we didn't reset the
153+
* offset to 0 in a failure case, so start from 0 to
156154
* exhaust the map.
157155
*/
158-
if (wrap) {
159-
wrap = false;
160-
end = org_last_tag;
161-
last_tag = 0;
162-
goto restart;
156+
if (org_last_tag && last_tag && !nowrap) {
157+
last_tag = org_last_tag = 0;
158+
continue;
163159
}
164160
return -1;
165161
}
162+
163+
if (!test_and_set_bit(tag, &bm->word))
164+
break;
165+
166166
last_tag = tag + 1;
167-
} while (test_and_set_bit(tag, &bm->word));
167+
if (last_tag >= bm->depth - 1)
168+
last_tag = 0;
169+
}
168170

169171
return tag;
170172
}
171173

174+
#define BT_ALLOC_RR(tags) (tags->alloc_policy == BLK_TAG_ALLOC_RR)
175+
172176
/*
173177
* Straight forward bitmap tag implementation, where each bit is a tag
174178
* (cleared == free, and set == busy). The small twist is using per-cpu
@@ -181,7 +185,7 @@ static int __bt_get_word(struct blk_align_bitmap *bm, unsigned int last_tag)
181185
* until the map is exhausted.
182186
*/
183187
static int __bt_get(struct blk_mq_hw_ctx *hctx, struct blk_mq_bitmap_tags *bt,
184-
unsigned int *tag_cache)
188+
unsigned int *tag_cache, struct blk_mq_tags *tags)
185189
{
186190
unsigned int last_tag, org_last_tag;
187191
int index, i, tag;
@@ -193,15 +197,24 @@ static int __bt_get(struct blk_mq_hw_ctx *hctx, struct blk_mq_bitmap_tags *bt,
193197
index = TAG_TO_INDEX(bt, last_tag);
194198

195199
for (i = 0; i < bt->map_nr; i++) {
196-
tag = __bt_get_word(&bt->map[index], TAG_TO_BIT(bt, last_tag));
200+
tag = __bt_get_word(&bt->map[index], TAG_TO_BIT(bt, last_tag),
201+
BT_ALLOC_RR(tags));
197202
if (tag != -1) {
198203
tag += (index << bt->bits_per_word);
199204
goto done;
200205
}
201206

202-
last_tag = 0;
203-
if (++index >= bt->map_nr)
207+
/*
208+
* Jump to next index, and reset the last tag to be the
209+
* first tag of that index
210+
*/
211+
index++;
212+
last_tag = (index << bt->bits_per_word);
213+
214+
if (index >= bt->map_nr) {
204215
index = 0;
216+
last_tag = 0;
217+
}
205218
}
206219

207220
*tag_cache = 0;
@@ -212,7 +225,7 @@ static int __bt_get(struct blk_mq_hw_ctx *hctx, struct blk_mq_bitmap_tags *bt,
212225
* up using the specific cached tag.
213226
*/
214227
done:
215-
if (tag == org_last_tag) {
228+
if (tag == org_last_tag || unlikely(BT_ALLOC_RR(tags))) {
216229
last_tag = tag + 1;
217230
if (last_tag >= bt->depth - 1)
218231
last_tag = 0;
@@ -241,13 +254,13 @@ static struct bt_wait_state *bt_wait_ptr(struct blk_mq_bitmap_tags *bt,
241254
static int bt_get(struct blk_mq_alloc_data *data,
242255
struct blk_mq_bitmap_tags *bt,
243256
struct blk_mq_hw_ctx *hctx,
244-
unsigned int *last_tag)
257+
unsigned int *last_tag, struct blk_mq_tags *tags)
245258
{
246259
struct bt_wait_state *bs;
247260
DEFINE_WAIT(wait);
248261
int tag;
249262

250-
tag = __bt_get(hctx, bt, last_tag);
263+
tag = __bt_get(hctx, bt, last_tag, tags);
251264
if (tag != -1)
252265
return tag;
253266

@@ -258,7 +271,7 @@ static int bt_get(struct blk_mq_alloc_data *data,
258271
do {
259272
prepare_to_wait(&bs->wait, &wait, TASK_UNINTERRUPTIBLE);
260273

261-
tag = __bt_get(hctx, bt, last_tag);
274+
tag = __bt_get(hctx, bt, last_tag, tags);
262275
if (tag != -1)
263276
break;
264277

@@ -273,7 +286,7 @@ static int bt_get(struct blk_mq_alloc_data *data,
273286
* Retry tag allocation after running the hardware queue,
274287
* as running the queue may also have found completions.
275288
*/
276-
tag = __bt_get(hctx, bt, last_tag);
289+
tag = __bt_get(hctx, bt, last_tag, tags);
277290
if (tag != -1)
278291
break;
279292

@@ -304,7 +317,7 @@ static unsigned int __blk_mq_get_tag(struct blk_mq_alloc_data *data)
304317
int tag;
305318

306319
tag = bt_get(data, &data->hctx->tags->bitmap_tags, data->hctx,
307-
&data->ctx->last_tag);
320+
&data->ctx->last_tag, data->hctx->tags);
308321
if (tag >= 0)
309322
return tag + data->hctx->tags->nr_reserved_tags;
310323

@@ -320,7 +333,8 @@ static unsigned int __blk_mq_get_reserved_tag(struct blk_mq_alloc_data *data)
320333
return BLK_MQ_TAG_FAIL;
321334
}
322335

323-
tag = bt_get(data, &data->hctx->tags->breserved_tags, NULL, &zero);
336+
tag = bt_get(data, &data->hctx->tags->breserved_tags, NULL, &zero,
337+
data->hctx->tags);
324338
if (tag < 0)
325339
return BLK_MQ_TAG_FAIL;
326340

@@ -392,7 +406,8 @@ void blk_mq_put_tag(struct blk_mq_hw_ctx *hctx, unsigned int tag,
392406

393407
BUG_ON(real_tag >= tags->nr_tags);
394408
bt_clear_tag(&tags->bitmap_tags, real_tag);
395-
*last_tag = real_tag;
409+
if (likely(tags->alloc_policy == BLK_TAG_ALLOC_FIFO))
410+
*last_tag = real_tag;
396411
} else {
397412
BUG_ON(tag >= tags->nr_reserved_tags);
398413
bt_clear_tag(&tags->breserved_tags, tag);
@@ -529,10 +544,12 @@ static void bt_free(struct blk_mq_bitmap_tags *bt)
529544
}
530545

531546
static struct blk_mq_tags *blk_mq_init_bitmap_tags(struct blk_mq_tags *tags,
532-
int node)
547+
int node, int alloc_policy)
533548
{
534549
unsigned int depth = tags->nr_tags - tags->nr_reserved_tags;
535550

551+
tags->alloc_policy = alloc_policy;
552+
536553
if (bt_alloc(&tags->bitmap_tags, depth, node, false))
537554
goto enomem;
538555
if (bt_alloc(&tags->breserved_tags, tags->nr_reserved_tags, node, true))
@@ -546,7 +563,8 @@ static struct blk_mq_tags *blk_mq_init_bitmap_tags(struct blk_mq_tags *tags,
546563
}
547564

548565
struct blk_mq_tags *blk_mq_init_tags(unsigned int total_tags,
549-
unsigned int reserved_tags, int node)
566+
unsigned int reserved_tags,
567+
int node, int alloc_policy)
550568
{
551569
struct blk_mq_tags *tags;
552570

@@ -562,7 +580,7 @@ struct blk_mq_tags *blk_mq_init_tags(unsigned int total_tags,
562580
tags->nr_tags = total_tags;
563581
tags->nr_reserved_tags = reserved_tags;
564582

565-
return blk_mq_init_bitmap_tags(tags, node);
583+
return blk_mq_init_bitmap_tags(tags, node, alloc_policy);
566584
}
567585

568586
void blk_mq_free_tags(struct blk_mq_tags *tags)

‎block/blk-mq-tag.h

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,10 +42,12 @@ struct blk_mq_tags {
4242

4343
struct request **rqs;
4444
struct list_head page_list;
45+
46+
int alloc_policy;
4547
};
4648

4749

48-
extern struct blk_mq_tags *blk_mq_init_tags(unsigned int nr_tags, unsigned int reserved_tags, int node);
50+
extern struct blk_mq_tags *blk_mq_init_tags(unsigned int nr_tags, unsigned int reserved_tags, int node, int alloc_policy);
4951
extern void blk_mq_free_tags(struct blk_mq_tags *tags);
5052

5153
extern unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data);

‎block/blk-mq.c

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1374,7 +1374,8 @@ static struct blk_mq_tags *blk_mq_init_rq_map(struct blk_mq_tag_set *set,
13741374
size_t rq_size, left;
13751375

13761376
tags = blk_mq_init_tags(set->queue_depth, set->reserved_tags,
1377-
set->numa_node);
1377+
set->numa_node,
1378+
BLK_MQ_FLAG_TO_ALLOC_POLICY(set->flags));
13781379
if (!tags)
13791380
return NULL;
13801381

‎block/blk-tag.c

Lines changed: 25 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@ init_tag_map(struct request_queue *q, struct blk_queue_tag *tags, int depth)
119119
}
120120

121121
static struct blk_queue_tag *__blk_queue_init_tags(struct request_queue *q,
122-
int depth)
122+
int depth, int alloc_policy)
123123
{
124124
struct blk_queue_tag *tags;
125125

@@ -131,6 +131,8 @@ static struct blk_queue_tag *__blk_queue_init_tags(struct request_queue *q,
131131
goto fail;
132132

133133
atomic_set(&tags->refcnt, 1);
134+
tags->alloc_policy = alloc_policy;
135+
tags->next_tag = 0;
134136
return tags;
135137
fail:
136138
kfree(tags);
@@ -140,10 +142,11 @@ static struct blk_queue_tag *__blk_queue_init_tags(struct request_queue *q,
140142
/**
141143
* blk_init_tags - initialize the tag info for an external tag map
142144
* @depth: the maximum queue depth supported
145+
* @alloc_policy: tag allocation policy
143146
**/
144-
struct blk_queue_tag *blk_init_tags(int depth)
147+
struct blk_queue_tag *blk_init_tags(int depth, int alloc_policy)
145148
{
146-
return __blk_queue_init_tags(NULL, depth);
149+
return __blk_queue_init_tags(NULL, depth, alloc_policy);
147150
}
148151
EXPORT_SYMBOL(blk_init_tags);
149152

@@ -152,19 +155,20 @@ EXPORT_SYMBOL(blk_init_tags);
152155
* @q: the request queue for the device
153156
* @depth: the maximum queue depth supported
154157
* @tags: the tag to use
158+
* @alloc_policy: tag allocation policy
155159
*
156160
* Queue lock must be held here if the function is called to resize an
157161
* existing map.
158162
**/
159163
int blk_queue_init_tags(struct request_queue *q, int depth,
160-
struct blk_queue_tag *tags)
164+
struct blk_queue_tag *tags, int alloc_policy)
161165
{
162166
int rc;
163167

164168
BUG_ON(tags && q->queue_tags && tags != q->queue_tags);
165169

166170
if (!tags && !q->queue_tags) {
167-
tags = __blk_queue_init_tags(q, depth);
171+
tags = __blk_queue_init_tags(q, depth, alloc_policy);
168172

169173
if (!tags)
170174
return -ENOMEM;
@@ -344,16 +348,29 @@ int blk_queue_start_tag(struct request_queue *q, struct request *rq)
344348
}
345349

346350
do {
347-
tag = find_first_zero_bit(bqt->tag_map, max_depth);
348-
if (tag >= max_depth)
349-
return 1;
351+
if (bqt->alloc_policy == BLK_TAG_ALLOC_FIFO) {
352+
tag = find_first_zero_bit(bqt->tag_map, max_depth);
353+
if (tag >= max_depth)
354+
return 1;
355+
} else {
356+
int start = bqt->next_tag;
357+
int size = min_t(int, bqt->max_depth, max_depth + start);
358+
tag = find_next_zero_bit(bqt->tag_map, size, start);
359+
if (tag >= size && start + size > bqt->max_depth) {
360+
size = start + size - bqt->max_depth;
361+
tag = find_first_zero_bit(bqt->tag_map, size);
362+
}
363+
if (tag >= size)
364+
return 1;
365+
}
350366

351367
} while (test_and_set_bit_lock(tag, bqt->tag_map));
352368
/*
353369
* We need lock ordering semantics given by test_and_set_bit_lock.
354370
* See blk_queue_end_tag for details.
355371
*/
356372

373+
bqt->next_tag = (tag + 1) % bqt->max_depth;
357374
rq->cmd_flags |= REQ_QUEUED;
358375
rq->tag = tag;
359376
bqt->tag_index[tag] = rq;

‎block/cfq-iosched.c

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3656,12 +3656,17 @@ static struct cfq_queue *
36563656
cfq_get_queue(struct cfq_data *cfqd, bool is_sync, struct cfq_io_cq *cic,
36573657
struct bio *bio, gfp_t gfp_mask)
36583658
{
3659-
const int ioprio_class = IOPRIO_PRIO_CLASS(cic->ioprio);
3660-
const int ioprio = IOPRIO_PRIO_DATA(cic->ioprio);
3659+
int ioprio_class = IOPRIO_PRIO_CLASS(cic->ioprio);
3660+
int ioprio = IOPRIO_PRIO_DATA(cic->ioprio);
36613661
struct cfq_queue **async_cfqq = NULL;
36623662
struct cfq_queue *cfqq = NULL;
36633663

36643664
if (!is_sync) {
3665+
if (!ioprio_valid(cic->ioprio)) {
3666+
struct task_struct *tsk = current;
3667+
ioprio = task_nice_ioprio(tsk);
3668+
ioprio_class = task_nice_ioclass(tsk);
3669+
}
36653670
async_cfqq = cfq_async_queue_prio(cfqd, ioprio_class, ioprio);
36663671
cfqq = *async_cfqq;
36673672
}

‎block/ioctl.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -198,7 +198,7 @@ static int blk_ioctl_zeroout(struct block_device *bdev, uint64_t start,
198198
if (start + len > (i_size_read(bdev->bd_inode) >> 9))
199199
return -EINVAL;
200200

201-
return blkdev_issue_zeroout(bdev, start, len, GFP_KERNEL);
201+
return blkdev_issue_zeroout(bdev, start, len, GFP_KERNEL, false);
202202
}
203203

204204
static int put_ushort(unsigned long arg, unsigned short val)

0 commit comments

Comments
 (0)