Skip to content

Commit 0d9c1ab

Browse files
idryomovidryomov
committed
libceph: preallocate message data items
Currently message data items are allocated with ceph_msg_data_create() in setup_request_data() inside send_request(). send_request() has never been allowed to fail, so each allocation is followed by a BUG_ON: data = ceph_msg_data_create(...); BUG_ON(!data); It's been this way since support for multiple message data items was added in commit 6644ed7 ("libceph: make message data be a pointer") in 3.10. There is no reason to delay the allocation of message data items until the last possible moment and we certainly don't need a linked list of them as they are only ever appended to the end and never erased. Make ceph_msg_new2() take max_data_items and adapt the rest of the code. Reported-by: Jerry Lee <leisurelysw24@gmail.com> Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
1 parent 26f887e commit 0d9c1ab

File tree

6 files changed

+157
-115
lines changed

6 files changed

+157
-115
lines changed

‎fs/ceph/mds_client.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2071,7 +2071,7 @@ static struct ceph_msg *create_request_message(struct ceph_mds_client *mdsc,
20712071
if (req->r_old_dentry_drop)
20722072
len += req->r_old_dentry->d_name.len;
20732073

2074-
msg = ceph_msg_new(CEPH_MSG_CLIENT_REQUEST, len, GFP_NOFS, false);
2074+
msg = ceph_msg_new2(CEPH_MSG_CLIENT_REQUEST, len, 1, GFP_NOFS, false);
20752075
if (!msg) {
20762076
msg = ERR_PTR(-ENOMEM);
20772077
goto out_free2;
@@ -3129,7 +3129,7 @@ static void send_mds_reconnect(struct ceph_mds_client *mdsc,
31293129
if (!pagelist)
31303130
goto fail_nopagelist;
31313131

3132-
reply = ceph_msg_new(CEPH_MSG_CLIENT_RECONNECT, 0, GFP_NOFS, false);
3132+
reply = ceph_msg_new2(CEPH_MSG_CLIENT_RECONNECT, 0, 1, GFP_NOFS, false);
31333133
if (!reply)
31343134
goto fail_nomsg;
31353135

‎include/linux/ceph/messenger.h

Lines changed: 5 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -82,22 +82,6 @@ enum ceph_msg_data_type {
8282
CEPH_MSG_DATA_BVECS, /* data source/destination is a bio_vec array */
8383
};
8484

85-
static __inline__ bool ceph_msg_data_type_valid(enum ceph_msg_data_type type)
86-
{
87-
switch (type) {
88-
case CEPH_MSG_DATA_NONE:
89-
case CEPH_MSG_DATA_PAGES:
90-
case CEPH_MSG_DATA_PAGELIST:
91-
#ifdef CONFIG_BLOCK
92-
case CEPH_MSG_DATA_BIO:
93-
#endif /* CONFIG_BLOCK */
94-
case CEPH_MSG_DATA_BVECS:
95-
return true;
96-
default:
97-
return false;
98-
}
99-
}
100-
10185
#ifdef CONFIG_BLOCK
10286

10387
struct ceph_bio_iter {
@@ -181,7 +165,6 @@ struct ceph_bvec_iter {
181165
} while (0)
182166

183167
struct ceph_msg_data {
184-
struct list_head links; /* ceph_msg->data */
185168
enum ceph_msg_data_type type;
186169
union {
187170
#ifdef CONFIG_BLOCK
@@ -202,7 +185,6 @@ struct ceph_msg_data {
202185

203186
struct ceph_msg_data_cursor {
204187
size_t total_resid; /* across all data items */
205-
struct list_head *data_head; /* = &ceph_msg->data */
206188

207189
struct ceph_msg_data *data; /* current data item */
208190
size_t resid; /* bytes not yet consumed */
@@ -240,7 +222,9 @@ struct ceph_msg {
240222
struct ceph_buffer *middle;
241223

242224
size_t data_length;
243-
struct list_head data;
225+
struct ceph_msg_data *data;
226+
int num_data_items;
227+
int max_data_items;
244228
struct ceph_msg_data_cursor cursor;
245229

246230
struct ceph_connection *con;
@@ -381,6 +365,8 @@ void ceph_msg_data_add_bio(struct ceph_msg *msg, struct ceph_bio_iter *bio_pos,
381365
void ceph_msg_data_add_bvecs(struct ceph_msg *msg,
382366
struct ceph_bvec_iter *bvec_pos);
383367

368+
struct ceph_msg *ceph_msg_new2(int type, int front_len, int max_data_items,
369+
gfp_t flags, bool can_fail);
384370
extern struct ceph_msg *ceph_msg_new(int type, int front_len, gfp_t flags,
385371
bool can_fail);
386372

‎include/linux/ceph/msgpool.h

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,15 @@ struct ceph_msgpool {
1313
mempool_t *pool;
1414
int type; /* preallocated message type */
1515
int front_len; /* preallocated payload size */
16+
int max_data_items;
1617
};
1718

18-
extern int ceph_msgpool_init(struct ceph_msgpool *pool, int type,
19-
int front_len, int size, bool blocking,
20-
const char *name);
19+
int ceph_msgpool_init(struct ceph_msgpool *pool, int type,
20+
int front_len, int max_data_items, int size,
21+
const char *name);
2122
extern void ceph_msgpool_destroy(struct ceph_msgpool *pool);
22-
extern struct ceph_msg *ceph_msgpool_get(struct ceph_msgpool *,
23-
int front_len);
23+
struct ceph_msg *ceph_msgpool_get(struct ceph_msgpool *pool, int front_len,
24+
int max_data_items);
2425
extern void ceph_msgpool_put(struct ceph_msgpool *, struct ceph_msg *);
2526

2627
#endif

‎net/ceph/messenger.c

Lines changed: 39 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,6 @@ static bool con_flag_test_and_set(struct ceph_connection *con,
156156
/* Slab caches for frequently-allocated structures */
157157

158158
static struct kmem_cache *ceph_msg_cache;
159-
static struct kmem_cache *ceph_msg_data_cache;
160159

161160
/* static tag bytes (protocol control messages) */
162161
static char tag_msg = CEPH_MSGR_TAG_MSG;
@@ -235,23 +234,11 @@ static int ceph_msgr_slab_init(void)
235234
if (!ceph_msg_cache)
236235
return -ENOMEM;
237236

238-
BUG_ON(ceph_msg_data_cache);
239-
ceph_msg_data_cache = KMEM_CACHE(ceph_msg_data, 0);
240-
if (ceph_msg_data_cache)
241-
return 0;
242-
243-
kmem_cache_destroy(ceph_msg_cache);
244-
ceph_msg_cache = NULL;
245-
246-
return -ENOMEM;
237+
return 0;
247238
}
248239

249240
static void ceph_msgr_slab_exit(void)
250241
{
251-
BUG_ON(!ceph_msg_data_cache);
252-
kmem_cache_destroy(ceph_msg_data_cache);
253-
ceph_msg_data_cache = NULL;
254-
255242
BUG_ON(!ceph_msg_cache);
256243
kmem_cache_destroy(ceph_msg_cache);
257244
ceph_msg_cache = NULL;
@@ -1141,16 +1128,13 @@ static void __ceph_msg_data_cursor_init(struct ceph_msg_data_cursor *cursor)
11411128
static void ceph_msg_data_cursor_init(struct ceph_msg *msg, size_t length)
11421129
{
11431130
struct ceph_msg_data_cursor *cursor = &msg->cursor;
1144-
struct ceph_msg_data *data;
11451131

11461132
BUG_ON(!length);
11471133
BUG_ON(length > msg->data_length);
1148-
BUG_ON(list_empty(&msg->data));
1134+
BUG_ON(!msg->num_data_items);
11491135

1150-
cursor->data_head = &msg->data;
11511136
cursor->total_resid = length;
1152-
data = list_first_entry(&msg->data, struct ceph_msg_data, links);
1153-
cursor->data = data;
1137+
cursor->data = msg->data;
11541138

11551139
__ceph_msg_data_cursor_init(cursor);
11561140
}
@@ -1231,8 +1215,7 @@ static void ceph_msg_data_advance(struct ceph_msg_data_cursor *cursor,
12311215

12321216
if (!cursor->resid && cursor->total_resid) {
12331217
WARN_ON(!cursor->last_piece);
1234-
BUG_ON(list_is_last(&cursor->data->links, cursor->data_head));
1235-
cursor->data = list_next_entry(cursor->data, links);
1218+
cursor->data++;
12361219
__ceph_msg_data_cursor_init(cursor);
12371220
new_piece = true;
12381221
}
@@ -1248,9 +1231,6 @@ static size_t sizeof_footer(struct ceph_connection *con)
12481231

12491232
static void prepare_message_data(struct ceph_msg *msg, u32 data_len)
12501233
{
1251-
BUG_ON(!msg);
1252-
BUG_ON(!data_len);
1253-
12541234
/* Initialize data cursor */
12551235

12561236
ceph_msg_data_cursor_init(msg, (size_t)data_len);
@@ -1590,7 +1570,7 @@ static int write_partial_message_data(struct ceph_connection *con)
15901570

15911571
dout("%s %p msg %p\n", __func__, con, msg);
15921572

1593-
if (list_empty(&msg->data))
1573+
if (!msg->num_data_items)
15941574
return -EINVAL;
15951575

15961576
/*
@@ -2347,8 +2327,7 @@ static int read_partial_msg_data(struct ceph_connection *con)
23472327
u32 crc = 0;
23482328
int ret;
23492329

2350-
BUG_ON(!msg);
2351-
if (list_empty(&msg->data))
2330+
if (!msg->num_data_items)
23522331
return -EIO;
23532332

23542333
if (do_datacrc)
@@ -3256,32 +3235,16 @@ bool ceph_con_keepalive_expired(struct ceph_connection *con,
32563235
return false;
32573236
}
32583237

3259-
static struct ceph_msg_data *ceph_msg_data_create(enum ceph_msg_data_type type)
3238+
static struct ceph_msg_data *ceph_msg_data_add(struct ceph_msg *msg)
32603239
{
3261-
struct ceph_msg_data *data;
3262-
3263-
if (WARN_ON(!ceph_msg_data_type_valid(type)))
3264-
return NULL;
3265-
3266-
data = kmem_cache_zalloc(ceph_msg_data_cache, GFP_NOFS);
3267-
if (!data)
3268-
return NULL;
3269-
3270-
data->type = type;
3271-
INIT_LIST_HEAD(&data->links);
3272-
3273-
return data;
3240+
BUG_ON(msg->num_data_items >= msg->max_data_items);
3241+
return &msg->data[msg->num_data_items++];
32743242
}
32753243

32763244
static void ceph_msg_data_destroy(struct ceph_msg_data *data)
32773245
{
3278-
if (!data)
3279-
return;
3280-
3281-
WARN_ON(!list_empty(&data->links));
32823246
if (data->type == CEPH_MSG_DATA_PAGELIST)
32833247
ceph_pagelist_release(data->pagelist);
3284-
kmem_cache_free(ceph_msg_data_cache, data);
32853248
}
32863249

32873250
void ceph_msg_data_add_pages(struct ceph_msg *msg, struct page **pages,
@@ -3292,13 +3255,12 @@ void ceph_msg_data_add_pages(struct ceph_msg *msg, struct page **pages,
32923255
BUG_ON(!pages);
32933256
BUG_ON(!length);
32943257

3295-
data = ceph_msg_data_create(CEPH_MSG_DATA_PAGES);
3296-
BUG_ON(!data);
3258+
data = ceph_msg_data_add(msg);
3259+
data->type = CEPH_MSG_DATA_PAGES;
32973260
data->pages = pages;
32983261
data->length = length;
32993262
data->alignment = alignment & ~PAGE_MASK;
33003263

3301-
list_add_tail(&data->links, &msg->data);
33023264
msg->data_length += length;
33033265
}
33043266
EXPORT_SYMBOL(ceph_msg_data_add_pages);
@@ -3311,12 +3273,11 @@ void ceph_msg_data_add_pagelist(struct ceph_msg *msg,
33113273
BUG_ON(!pagelist);
33123274
BUG_ON(!pagelist->length);
33133275

3314-
data = ceph_msg_data_create(CEPH_MSG_DATA_PAGELIST);
3315-
BUG_ON(!data);
3276+
data = ceph_msg_data_add(msg);
3277+
data->type = CEPH_MSG_DATA_PAGELIST;
33163278
refcount_inc(&pagelist->refcnt);
33173279
data->pagelist = pagelist;
33183280

3319-
list_add_tail(&data->links, &msg->data);
33203281
msg->data_length += pagelist->length;
33213282
}
33223283
EXPORT_SYMBOL(ceph_msg_data_add_pagelist);
@@ -3327,12 +3288,11 @@ void ceph_msg_data_add_bio(struct ceph_msg *msg, struct ceph_bio_iter *bio_pos,
33273288
{
33283289
struct ceph_msg_data *data;
33293290

3330-
data = ceph_msg_data_create(CEPH_MSG_DATA_BIO);
3331-
BUG_ON(!data);
3291+
data = ceph_msg_data_add(msg);
3292+
data->type = CEPH_MSG_DATA_BIO;
33323293
data->bio_pos = *bio_pos;
33333294
data->bio_length = length;
33343295

3335-
list_add_tail(&data->links, &msg->data);
33363296
msg->data_length += length;
33373297
}
33383298
EXPORT_SYMBOL(ceph_msg_data_add_bio);
@@ -3343,11 +3303,10 @@ void ceph_msg_data_add_bvecs(struct ceph_msg *msg,
33433303
{
33443304
struct ceph_msg_data *data;
33453305

3346-
data = ceph_msg_data_create(CEPH_MSG_DATA_BVECS);
3347-
BUG_ON(!data);
3306+
data = ceph_msg_data_add(msg);
3307+
data->type = CEPH_MSG_DATA_BVECS;
33483308
data->bvec_pos = *bvec_pos;
33493309

3350-
list_add_tail(&data->links, &msg->data);
33513310
msg->data_length += bvec_pos->iter.bi_size;
33523311
}
33533312
EXPORT_SYMBOL(ceph_msg_data_add_bvecs);
@@ -3356,8 +3315,8 @@ EXPORT_SYMBOL(ceph_msg_data_add_bvecs);
33563315
* construct a new message with given type, size
33573316
* the new msg has a ref count of 1.
33583317
*/
3359-
struct ceph_msg *ceph_msg_new(int type, int front_len, gfp_t flags,
3360-
bool can_fail)
3318+
struct ceph_msg *ceph_msg_new2(int type, int front_len, int max_data_items,
3319+
gfp_t flags, bool can_fail)
33613320
{
33623321
struct ceph_msg *m;
33633322

@@ -3371,7 +3330,6 @@ struct ceph_msg *ceph_msg_new(int type, int front_len, gfp_t flags,
33713330

33723331
INIT_LIST_HEAD(&m->list_head);
33733332
kref_init(&m->kref);
3374-
INIT_LIST_HEAD(&m->data);
33753333

33763334
/* front */
33773335
if (front_len) {
@@ -3386,6 +3344,15 @@ struct ceph_msg *ceph_msg_new(int type, int front_len, gfp_t flags,
33863344
}
33873345
m->front_alloc_len = m->front.iov_len = front_len;
33883346

3347+
if (max_data_items) {
3348+
m->data = kmalloc_array(max_data_items, sizeof(*m->data),
3349+
flags);
3350+
if (!m->data)
3351+
goto out2;
3352+
3353+
m->max_data_items = max_data_items;
3354+
}
3355+
33893356
dout("ceph_msg_new %p front %d\n", m, front_len);
33903357
return m;
33913358

@@ -3402,6 +3369,13 @@ struct ceph_msg *ceph_msg_new(int type, int front_len, gfp_t flags,
34023369
}
34033370
return NULL;
34043371
}
3372+
EXPORT_SYMBOL(ceph_msg_new2);
3373+
3374+
struct ceph_msg *ceph_msg_new(int type, int front_len, gfp_t flags,
3375+
bool can_fail)
3376+
{
3377+
return ceph_msg_new2(type, front_len, 0, flags, can_fail);
3378+
}
34053379
EXPORT_SYMBOL(ceph_msg_new);
34063380

34073381
/*
@@ -3497,13 +3471,14 @@ static void ceph_msg_free(struct ceph_msg *m)
34973471
{
34983472
dout("%s %p\n", __func__, m);
34993473
kvfree(m->front.iov_base);
3474+
kfree(m->data);
35003475
kmem_cache_free(ceph_msg_cache, m);
35013476
}
35023477

35033478
static void ceph_msg_release(struct kref *kref)
35043479
{
35053480
struct ceph_msg *m = container_of(kref, struct ceph_msg, kref);
3506-
struct ceph_msg_data *data, *next;
3481+
int i;
35073482

35083483
dout("%s %p\n", __func__, m);
35093484
WARN_ON(!list_empty(&m->list_head));
@@ -3516,11 +3491,8 @@ static void ceph_msg_release(struct kref *kref)
35163491
m->middle = NULL;
35173492
}
35183493

3519-
list_for_each_entry_safe(data, next, &m->data, links) {
3520-
list_del_init(&data->links);
3521-
ceph_msg_data_destroy(data);
3522-
}
3523-
m->data_length = 0;
3494+
for (i = 0; i < m->num_data_items; i++)
3495+
ceph_msg_data_destroy(&m->data[i]);
35243496

35253497
if (m->pool)
35263498
ceph_msgpool_put(m->pool, m);

0 commit comments

Comments
 (0)