Skip to content

Commit 35d88d9

Browse files
torvaldstorvalds
committed
Merge branch 'for-4.6/core' of git://git.kernel.dk/linux-block
Pull core block updates from Jens Axboe: "Here are the core block changes for this merge window. Not a lot of exciting stuff going on in this round, most of the changes have been on the driver side of things. That pull request is coming next. This pull request contains: - A set of fixes for chained bio handling from Christoph. - A tag bounds check for blk-mq from Hannes, ensuring that we don't do something stupid if a device reports an invalid tag value. - A set of fixes/updates for the CFQ IO scheduler from Jan Kara. - A set of blk-mq fixes from Keith, adding support for dynamic hardware queues, and fixing init of max_dev_sectors for stacking devices. - A fix for the dynamic hw context from Ming. - Enabling of cgroup writeback support on a block device, from Shaohua" * 'for-4.6/core' of git://git.kernel.dk/linux-block: blk-mq: add bounds check on tag-to-rq conversion block: bio_remaining_done() isn't unlikely block: cleanup bio_endio block: factor out chained bio completion block: don't unecessarily clobber bi_error for chained bios block-dev: enable writeback cgroup support blk-mq: Fix NULL pointer updating nr_requests blk-mq: mark request queue as mq asap block: Initialize max_dev_sectors to 0 blk-mq: dynamic h/w context count cfq-iosched: Allow parent cgroup to preempt its child cfq-iosched: Allow sync noidle workloads to preempt each other cfq-iosched: Reorder checks in cfq_should_preempt() cfq-iosched: Don't group_idle if cfqq has big thinktime
2 parents 040e3ab + 4ee86ba commit 35d88d9

File tree

7 files changed

+185
-110
lines changed

7 files changed

+185
-110
lines changed

‎block/bio.c

Lines changed: 26 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -296,13 +296,19 @@ void bio_reset(struct bio *bio)
296296
}
297297
EXPORT_SYMBOL(bio_reset);
298298

299-
static void bio_chain_endio(struct bio *bio)
299+
static struct bio *__bio_chain_endio(struct bio *bio)
300300
{
301301
struct bio *parent = bio->bi_private;
302302

303-
parent->bi_error = bio->bi_error;
304-
bio_endio(parent);
303+
if (!parent->bi_error)
304+
parent->bi_error = bio->bi_error;
305305
bio_put(bio);
306+
return parent;
307+
}
308+
309+
static void bio_chain_endio(struct bio *bio)
310+
{
311+
bio_endio(__bio_chain_endio(bio));
306312
}
307313

308314
/*
@@ -1742,29 +1748,25 @@ static inline bool bio_remaining_done(struct bio *bio)
17421748
**/
17431749
void bio_endio(struct bio *bio)
17441750
{
1745-
while (bio) {
1746-
if (unlikely(!bio_remaining_done(bio)))
1747-
break;
1751+
again:
1752+
if (!bio_remaining_done(bio))
1753+
return;
17481754

1749-
/*
1750-
* Need to have a real endio function for chained bios,
1751-
* otherwise various corner cases will break (like stacking
1752-
* block devices that save/restore bi_end_io) - however, we want
1753-
* to avoid unbounded recursion and blowing the stack. Tail call
1754-
* optimization would handle this, but compiling with frame
1755-
* pointers also disables gcc's sibling call optimization.
1756-
*/
1757-
if (bio->bi_end_io == bio_chain_endio) {
1758-
struct bio *parent = bio->bi_private;
1759-
parent->bi_error = bio->bi_error;
1760-
bio_put(bio);
1761-
bio = parent;
1762-
} else {
1763-
if (bio->bi_end_io)
1764-
bio->bi_end_io(bio);
1765-
bio = NULL;
1766-
}
1755+
/*
1756+
* Need to have a real endio function for chained bios, otherwise
1757+
* various corner cases will break (like stacking block devices that
1758+
* save/restore bi_end_io) - however, we want to avoid unbounded
1759+
* recursion and blowing the stack. Tail call optimization would
1760+
* handle this, but compiling with frame pointers also disables
1761+
* gcc's sibling call optimization.
1762+
*/
1763+
if (bio->bi_end_io == bio_chain_endio) {
1764+
bio = __bio_chain_endio(bio);
1765+
goto again;
17671766
}
1767+
1768+
if (bio->bi_end_io)
1769+
bio->bi_end_io(bio);
17681770
}
17691771
EXPORT_SYMBOL(bio_endio);
17701772

‎block/blk-mq-sysfs.c

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -408,17 +408,18 @@ void blk_mq_unregister_disk(struct gendisk *disk)
408408
blk_mq_enable_hotplug();
409409
}
410410

411+
void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx)
412+
{
413+
kobject_init(&hctx->kobj, &blk_mq_hw_ktype);
414+
}
415+
411416
static void blk_mq_sysfs_init(struct request_queue *q)
412417
{
413-
struct blk_mq_hw_ctx *hctx;
414418
struct blk_mq_ctx *ctx;
415419
int i;
416420

417421
kobject_init(&q->mq_kobj, &blk_mq_ktype);
418422

419-
queue_for_each_hw_ctx(q, hctx, i)
420-
kobject_init(&hctx->kobj, &blk_mq_hw_ktype);
421-
422423
queue_for_each_ctx(q, ctx, i)
423424
kobject_init(&ctx->kobj, &blk_mq_ctx_ktype);
424425
}

‎block/blk-mq.c

Lines changed: 113 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -544,7 +544,10 @@ EXPORT_SYMBOL(blk_mq_abort_requeue_list);
544544

545545
struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag)
546546
{
547-
return tags->rqs[tag];
547+
if (tag < tags->nr_tags)
548+
return tags->rqs[tag];
549+
550+
return NULL;
548551
}
549552
EXPORT_SYMBOL(blk_mq_tag_to_rq);
550553

@@ -1744,31 +1747,6 @@ static int blk_mq_init_hctx(struct request_queue *q,
17441747
return -1;
17451748
}
17461749

1747-
static int blk_mq_init_hw_queues(struct request_queue *q,
1748-
struct blk_mq_tag_set *set)
1749-
{
1750-
struct blk_mq_hw_ctx *hctx;
1751-
unsigned int i;
1752-
1753-
/*
1754-
* Initialize hardware queues
1755-
*/
1756-
queue_for_each_hw_ctx(q, hctx, i) {
1757-
if (blk_mq_init_hctx(q, set, hctx, i))
1758-
break;
1759-
}
1760-
1761-
if (i == q->nr_hw_queues)
1762-
return 0;
1763-
1764-
/*
1765-
* Init failed
1766-
*/
1767-
blk_mq_exit_hw_queues(q, set, i);
1768-
1769-
return 1;
1770-
}
1771-
17721750
static void blk_mq_init_cpu_queues(struct request_queue *q,
17731751
unsigned int nr_hw_queues)
17741752
{
@@ -1826,6 +1804,7 @@ static void blk_mq_map_swqueue(struct request_queue *q,
18261804
continue;
18271805

18281806
hctx = q->mq_ops->map_queue(q, i);
1807+
18291808
cpumask_set_cpu(i, hctx->cpumask);
18301809
ctx->index_hw = hctx->nr_ctx;
18311810
hctx->ctxs[hctx->nr_ctx++] = ctx;
@@ -1974,56 +1953,93 @@ struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set)
19741953
}
19751954
EXPORT_SYMBOL(blk_mq_init_queue);
19761955

1977-
struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
1978-
struct request_queue *q)
1956+
static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set,
1957+
struct request_queue *q)
19791958
{
1980-
struct blk_mq_hw_ctx **hctxs;
1981-
struct blk_mq_ctx __percpu *ctx;
1982-
unsigned int *map;
1983-
int i;
1984-
1985-
ctx = alloc_percpu(struct blk_mq_ctx);
1986-
if (!ctx)
1987-
return ERR_PTR(-ENOMEM);
1988-
1989-
hctxs = kmalloc_node(set->nr_hw_queues * sizeof(*hctxs), GFP_KERNEL,
1990-
set->numa_node);
1991-
1992-
if (!hctxs)
1993-
goto err_percpu;
1994-
1995-
map = blk_mq_make_queue_map(set);
1996-
if (!map)
1997-
goto err_map;
1959+
int i, j;
1960+
struct blk_mq_hw_ctx **hctxs = q->queue_hw_ctx;
19981961

1962+
blk_mq_sysfs_unregister(q);
19991963
for (i = 0; i < set->nr_hw_queues; i++) {
2000-
int node = blk_mq_hw_queue_to_node(map, i);
1964+
int node;
20011965

1966+
if (hctxs[i])
1967+
continue;
1968+
1969+
node = blk_mq_hw_queue_to_node(q->mq_map, i);
20021970
hctxs[i] = kzalloc_node(sizeof(struct blk_mq_hw_ctx),
20031971
GFP_KERNEL, node);
20041972
if (!hctxs[i])
2005-
goto err_hctxs;
1973+
break;
20061974

20071975
if (!zalloc_cpumask_var_node(&hctxs[i]->cpumask, GFP_KERNEL,
2008-
node))
2009-
goto err_hctxs;
1976+
node)) {
1977+
kfree(hctxs[i]);
1978+
hctxs[i] = NULL;
1979+
break;
1980+
}
20101981

20111982
atomic_set(&hctxs[i]->nr_active, 0);
20121983
hctxs[i]->numa_node = node;
20131984
hctxs[i]->queue_num = i;
1985+
1986+
if (blk_mq_init_hctx(q, set, hctxs[i], i)) {
1987+
free_cpumask_var(hctxs[i]->cpumask);
1988+
kfree(hctxs[i]);
1989+
hctxs[i] = NULL;
1990+
break;
1991+
}
1992+
blk_mq_hctx_kobj_init(hctxs[i]);
20141993
}
1994+
for (j = i; j < q->nr_hw_queues; j++) {
1995+
struct blk_mq_hw_ctx *hctx = hctxs[j];
1996+
1997+
if (hctx) {
1998+
if (hctx->tags) {
1999+
blk_mq_free_rq_map(set, hctx->tags, j);
2000+
set->tags[j] = NULL;
2001+
}
2002+
blk_mq_exit_hctx(q, set, hctx, j);
2003+
free_cpumask_var(hctx->cpumask);
2004+
kobject_put(&hctx->kobj);
2005+
kfree(hctx->ctxs);
2006+
kfree(hctx);
2007+
hctxs[j] = NULL;
2008+
2009+
}
2010+
}
2011+
q->nr_hw_queues = i;
2012+
blk_mq_sysfs_register(q);
2013+
}
2014+
2015+
struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
2016+
struct request_queue *q)
2017+
{
2018+
/* mark the queue as mq asap */
2019+
q->mq_ops = set->ops;
2020+
2021+
q->queue_ctx = alloc_percpu(struct blk_mq_ctx);
2022+
if (!q->queue_ctx)
2023+
return ERR_PTR(-ENOMEM);
2024+
2025+
q->queue_hw_ctx = kzalloc_node(nr_cpu_ids * sizeof(*(q->queue_hw_ctx)),
2026+
GFP_KERNEL, set->numa_node);
2027+
if (!q->queue_hw_ctx)
2028+
goto err_percpu;
2029+
2030+
q->mq_map = blk_mq_make_queue_map(set);
2031+
if (!q->mq_map)
2032+
goto err_map;
2033+
2034+
blk_mq_realloc_hw_ctxs(set, q);
2035+
if (!q->nr_hw_queues)
2036+
goto err_hctxs;
20152037

20162038
INIT_WORK(&q->timeout_work, blk_mq_timeout_work);
20172039
blk_queue_rq_timeout(q, set->timeout ? set->timeout : 30 * HZ);
20182040

20192041
q->nr_queues = nr_cpu_ids;
2020-
q->nr_hw_queues = set->nr_hw_queues;
2021-
q->mq_map = map;
2022-
2023-
q->queue_ctx = ctx;
2024-
q->queue_hw_ctx = hctxs;
20252042

2026-
q->mq_ops = set->ops;
20272043
q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT;
20282044

20292045
if (!(set->flags & BLK_MQ_F_SG_MERGE))
@@ -2050,9 +2066,6 @@ struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
20502066

20512067
blk_mq_init_cpu_queues(q, set->nr_hw_queues);
20522068

2053-
if (blk_mq_init_hw_queues(q, set))
2054-
goto err_hctxs;
2055-
20562069
get_online_cpus();
20572070
mutex_lock(&all_q_mutex);
20582071

@@ -2066,17 +2079,11 @@ struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
20662079
return q;
20672080

20682081
err_hctxs:
2069-
kfree(map);
2070-
for (i = 0; i < set->nr_hw_queues; i++) {
2071-
if (!hctxs[i])
2072-
break;
2073-
free_cpumask_var(hctxs[i]->cpumask);
2074-
kfree(hctxs[i]);
2075-
}
2082+
kfree(q->mq_map);
20762083
err_map:
2077-
kfree(hctxs);
2084+
kfree(q->queue_hw_ctx);
20782085
err_percpu:
2079-
free_percpu(ctx);
2086+
free_percpu(q->queue_ctx);
20802087
return ERR_PTR(-ENOMEM);
20812088
}
20822089
EXPORT_SYMBOL(blk_mq_init_allocated_queue);
@@ -2284,9 +2291,13 @@ int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set)
22842291
set->nr_hw_queues = 1;
22852292
set->queue_depth = min(64U, set->queue_depth);
22862293
}
2294+
/*
2295+
* There is no use for more h/w queues than cpus.
2296+
*/
2297+
if (set->nr_hw_queues > nr_cpu_ids)
2298+
set->nr_hw_queues = nr_cpu_ids;
22872299

2288-
set->tags = kmalloc_node(set->nr_hw_queues *
2289-
sizeof(struct blk_mq_tags *),
2300+
set->tags = kzalloc_node(nr_cpu_ids * sizeof(struct blk_mq_tags *),
22902301
GFP_KERNEL, set->numa_node);
22912302
if (!set->tags)
22922303
return -ENOMEM;
@@ -2309,7 +2320,7 @@ void blk_mq_free_tag_set(struct blk_mq_tag_set *set)
23092320
{
23102321
int i;
23112322

2312-
for (i = 0; i < set->nr_hw_queues; i++) {
2323+
for (i = 0; i < nr_cpu_ids; i++) {
23132324
if (set->tags[i])
23142325
blk_mq_free_rq_map(set, set->tags[i], i);
23152326
}
@@ -2330,6 +2341,8 @@ int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr)
23302341

23312342
ret = 0;
23322343
queue_for_each_hw_ctx(q, hctx, i) {
2344+
if (!hctx->tags)
2345+
continue;
23332346
ret = blk_mq_tag_update_depth(hctx->tags, nr);
23342347
if (ret)
23352348
break;
@@ -2341,6 +2354,35 @@ int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr)
23412354
return ret;
23422355
}
23432356

2357+
void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues)
2358+
{
2359+
struct request_queue *q;
2360+
2361+
if (nr_hw_queues > nr_cpu_ids)
2362+
nr_hw_queues = nr_cpu_ids;
2363+
if (nr_hw_queues < 1 || nr_hw_queues == set->nr_hw_queues)
2364+
return;
2365+
2366+
list_for_each_entry(q, &set->tag_list, tag_set_list)
2367+
blk_mq_freeze_queue(q);
2368+
2369+
set->nr_hw_queues = nr_hw_queues;
2370+
list_for_each_entry(q, &set->tag_list, tag_set_list) {
2371+
blk_mq_realloc_hw_ctxs(set, q);
2372+
2373+
if (q->nr_hw_queues > 1)
2374+
blk_queue_make_request(q, blk_mq_make_request);
2375+
else
2376+
blk_queue_make_request(q, blk_sq_make_request);
2377+
2378+
blk_mq_queue_reinit(q, cpu_online_mask);
2379+
}
2380+
2381+
list_for_each_entry(q, &set->tag_list, tag_set_list)
2382+
blk_mq_unfreeze_queue(q);
2383+
}
2384+
EXPORT_SYMBOL_GPL(blk_mq_update_nr_hw_queues);
2385+
23442386
void blk_mq_disable_hotplug(void)
23452387
{
23462388
mutex_lock(&all_q_mutex);

‎block/blk-mq.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ extern int blk_mq_hw_queue_to_node(unsigned int *map, unsigned int);
5757
*/
5858
extern int blk_mq_sysfs_register(struct request_queue *q);
5959
extern void blk_mq_sysfs_unregister(struct request_queue *q);
60+
extern void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx);
6061

6162
extern void blk_mq_rq_timed_out(struct request *req, bool reserved);
6263

0 commit comments

Comments
 (0)