Skip to content

Commit 41b4dee

Browse files
jgunthorpedledford
jgunthorpe
authored and
dledford
committed
RDMA/umem: Make ib_umem_odp into a sub structure of ib_umem
These two structures are linked together, use the container_of pattern instead of a double allocation to make the code simpler and easier to follow. Signed-off-by: Jason Gunthorpe <jgg@mellanox.com> Signed-off-by: Leon Romanovsky <leonro@mellanox.com> Signed-off-by: Doug Ledford <dledford@redhat.com>
1 parent b5231b0 commit 41b4dee

File tree

4 files changed

+69
-83
lines changed

4 files changed

+69
-83
lines changed

‎drivers/infiniband/core/umem.c

Lines changed: 22 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -108,34 +108,39 @@ struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr,
108108
if (!can_do_mlock())
109109
return ERR_PTR(-EPERM);
110110

111-
umem = kzalloc(sizeof *umem, GFP_KERNEL);
112-
if (!umem)
113-
return ERR_PTR(-ENOMEM);
111+
if (access & IB_ACCESS_ON_DEMAND) {
112+
umem = kzalloc(sizeof(struct ib_umem_odp), GFP_KERNEL);
113+
if (!umem)
114+
return ERR_PTR(-ENOMEM);
115+
umem->odp_data = to_ib_umem_odp(umem);
116+
} else {
117+
umem = kzalloc(sizeof(*umem), GFP_KERNEL);
118+
if (!umem)
119+
return ERR_PTR(-ENOMEM);
120+
}
114121

115122
umem->context = context;
116123
umem->length = size;
117124
umem->address = addr;
118125
umem->page_shift = PAGE_SHIFT;
119126
umem->writable = ib_access_writable(access);
127+
umem->owning_mm = mm = current->mm;
128+
mmgrab(mm);
120129

121130
if (access & IB_ACCESS_ON_DEMAND) {
122-
ret = ib_umem_odp_get(context, umem, access);
131+
ret = ib_umem_odp_get(to_ib_umem_odp(umem), access);
123132
if (ret)
124133
goto umem_kfree;
125134
return umem;
126135
}
127136

128-
umem->owning_mm = mm = current->mm;
129-
mmgrab(mm);
130-
umem->odp_data = NULL;
131-
132137
/* We assume the memory is from hugetlb until proved otherwise */
133138
umem->hugetlb = 1;
134139

135140
page_list = (struct page **) __get_free_page(GFP_KERNEL);
136141
if (!page_list) {
137142
ret = -ENOMEM;
138-
goto umem_kfree_drop;
143+
goto umem_kfree;
139144
}
140145

141146
/*
@@ -226,20 +231,22 @@ struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr,
226231
if (vma_list)
227232
free_page((unsigned long) vma_list);
228233
free_page((unsigned long) page_list);
229-
umem_kfree_drop:
230-
if (ret)
231-
mmdrop(umem->owning_mm);
232234
umem_kfree:
233-
if (ret)
235+
if (ret) {
236+
mmdrop(umem->owning_mm);
234237
kfree(umem);
238+
}
235239
return ret ? ERR_PTR(ret) : umem;
236240
}
237241
EXPORT_SYMBOL(ib_umem_get);
238242

239243
static void __ib_umem_release_tail(struct ib_umem *umem)
240244
{
241245
mmdrop(umem->owning_mm);
242-
kfree(umem);
246+
if (umem->odp_data)
247+
kfree(to_ib_umem_odp(umem));
248+
else
249+
kfree(umem);
243250
}
244251

245252
static void ib_umem_release_defer(struct work_struct *work)
@@ -263,6 +270,7 @@ void ib_umem_release(struct ib_umem *umem)
263270

264271
if (umem->odp_data) {
265272
ib_umem_odp_release(to_ib_umem_odp(umem));
273+
__ib_umem_release_tail(umem);
266274
return;
267275
}
268276

‎drivers/infiniband/core/umem_odp.c

Lines changed: 30 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ static u64 node_start(struct umem_odp_node *n)
5858
struct ib_umem_odp *umem_odp =
5959
container_of(n, struct ib_umem_odp, interval_tree);
6060

61-
return ib_umem_start(umem_odp->umem);
61+
return ib_umem_start(&umem_odp->umem);
6262
}
6363

6464
/* Note that the representation of the intervals in the interval tree
@@ -71,7 +71,7 @@ static u64 node_last(struct umem_odp_node *n)
7171
struct ib_umem_odp *umem_odp =
7272
container_of(n, struct ib_umem_odp, interval_tree);
7373

74-
return ib_umem_end(umem_odp->umem) - 1;
74+
return ib_umem_end(&umem_odp->umem) - 1;
7575
}
7676

7777
INTERVAL_TREE_DEFINE(struct umem_odp_node, rb, u64, __subtree_last,
@@ -159,7 +159,7 @@ static void ib_ucontext_notifier_end_account(struct ib_ucontext *context)
159159
static int ib_umem_notifier_release_trampoline(struct ib_umem_odp *umem_odp,
160160
u64 start, u64 end, void *cookie)
161161
{
162-
struct ib_umem *umem = umem_odp->umem;
162+
struct ib_umem *umem = &umem_odp->umem;
163163

164164
/*
165165
* Increase the number of notifiers running, to
@@ -198,7 +198,7 @@ static int invalidate_page_trampoline(struct ib_umem_odp *item, u64 start,
198198
u64 end, void *cookie)
199199
{
200200
ib_umem_notifier_start_account(item);
201-
item->umem->context->invalidate_range(item, start, start + PAGE_SIZE);
201+
item->umem.context->invalidate_range(item, start, start + PAGE_SIZE);
202202
ib_umem_notifier_end_account(item);
203203
return 0;
204204
}
@@ -207,7 +207,7 @@ static int invalidate_range_start_trampoline(struct ib_umem_odp *item,
207207
u64 start, u64 end, void *cookie)
208208
{
209209
ib_umem_notifier_start_account(item);
210-
item->umem->context->invalidate_range(item, start, end);
210+
item->umem.context->invalidate_range(item, start, end);
211211
return 0;
212212
}
213213

@@ -277,28 +277,21 @@ static const struct mmu_notifier_ops ib_umem_notifiers = {
277277
struct ib_umem_odp *ib_alloc_odp_umem(struct ib_ucontext *context,
278278
unsigned long addr, size_t size)
279279
{
280-
struct ib_umem *umem;
281280
struct ib_umem_odp *odp_data;
281+
struct ib_umem *umem;
282282
int pages = size >> PAGE_SHIFT;
283283
int ret;
284284

285-
umem = kzalloc(sizeof(*umem), GFP_KERNEL);
286-
if (!umem)
285+
odp_data = kzalloc(sizeof(*odp_data), GFP_KERNEL);
286+
if (!odp_data)
287287
return ERR_PTR(-ENOMEM);
288-
288+
umem = &odp_data->umem;
289289
umem->context = context;
290290
umem->length = size;
291291
umem->address = addr;
292292
umem->page_shift = PAGE_SHIFT;
293293
umem->writable = 1;
294294

295-
odp_data = kzalloc(sizeof(*odp_data), GFP_KERNEL);
296-
if (!odp_data) {
297-
ret = -ENOMEM;
298-
goto out_umem;
299-
}
300-
odp_data->umem = umem;
301-
302295
mutex_init(&odp_data->umem_mutex);
303296
init_completion(&odp_data->notifier_completion);
304297

@@ -334,15 +327,14 @@ struct ib_umem_odp *ib_alloc_odp_umem(struct ib_ucontext *context,
334327
vfree(odp_data->page_list);
335328
out_odp_data:
336329
kfree(odp_data);
337-
out_umem:
338-
kfree(umem);
339330
return ERR_PTR(ret);
340331
}
341332
EXPORT_SYMBOL(ib_alloc_odp_umem);
342333

343-
int ib_umem_odp_get(struct ib_ucontext *context, struct ib_umem *umem,
344-
int access)
334+
int ib_umem_odp_get(struct ib_umem_odp *umem_odp, int access)
345335
{
336+
struct ib_ucontext *context = umem_odp->umem.context;
337+
struct ib_umem *umem = &umem_odp->umem;
346338
int ret_val;
347339
struct pid *our_pid;
348340
struct mm_struct *mm = get_task_mm(current);
@@ -378,30 +370,23 @@ int ib_umem_odp_get(struct ib_ucontext *context, struct ib_umem *umem,
378370
goto out_mm;
379371
}
380372

381-
umem->odp_data = kzalloc(sizeof(*umem->odp_data), GFP_KERNEL);
382-
if (!umem->odp_data) {
383-
ret_val = -ENOMEM;
384-
goto out_mm;
385-
}
386-
umem->odp_data->umem = umem;
387-
388-
mutex_init(&umem->odp_data->umem_mutex);
373+
mutex_init(&umem_odp->umem_mutex);
389374

390-
init_completion(&umem->odp_data->notifier_completion);
375+
init_completion(&umem_odp->notifier_completion);
391376

392377
if (ib_umem_num_pages(umem)) {
393-
umem->odp_data->page_list =
394-
vzalloc(array_size(sizeof(*umem->odp_data->page_list),
378+
umem_odp->page_list =
379+
vzalloc(array_size(sizeof(*umem_odp->page_list),
395380
ib_umem_num_pages(umem)));
396-
if (!umem->odp_data->page_list) {
381+
if (!umem_odp->page_list) {
397382
ret_val = -ENOMEM;
398-
goto out_odp_data;
383+
goto out_mm;
399384
}
400385

401-
umem->odp_data->dma_list =
402-
vzalloc(array_size(sizeof(*umem->odp_data->dma_list),
386+
umem_odp->dma_list =
387+
vzalloc(array_size(sizeof(*umem_odp->dma_list),
403388
ib_umem_num_pages(umem)));
404-
if (!umem->odp_data->dma_list) {
389+
if (!umem_odp->dma_list) {
405390
ret_val = -ENOMEM;
406391
goto out_page_list;
407392
}
@@ -415,13 +400,13 @@ int ib_umem_odp_get(struct ib_ucontext *context, struct ib_umem *umem,
415400
down_write(&context->umem_rwsem);
416401
context->odp_mrs_count++;
417402
if (likely(ib_umem_start(umem) != ib_umem_end(umem)))
418-
rbt_ib_umem_insert(&umem->odp_data->interval_tree,
403+
rbt_ib_umem_insert(&umem_odp->interval_tree,
419404
&context->umem_tree);
420405
if (likely(!atomic_read(&context->notifier_count)) ||
421406
context->odp_mrs_count == 1)
422-
umem->odp_data->mn_counters_active = true;
407+
umem_odp->mn_counters_active = true;
423408
else
424-
list_add(&umem->odp_data->no_private_counters,
409+
list_add(&umem_odp->no_private_counters,
425410
&context->no_private_counters);
426411
downgrade_write(&context->umem_rwsem);
427412

@@ -454,19 +439,17 @@ int ib_umem_odp_get(struct ib_ucontext *context, struct ib_umem *umem,
454439

455440
out_mutex:
456441
up_read(&context->umem_rwsem);
457-
vfree(umem->odp_data->dma_list);
442+
vfree(umem_odp->dma_list);
458443
out_page_list:
459-
vfree(umem->odp_data->page_list);
460-
out_odp_data:
461-
kfree(umem->odp_data);
444+
vfree(umem_odp->page_list);
462445
out_mm:
463446
mmput(mm);
464447
return ret_val;
465448
}
466449

467450
void ib_umem_odp_release(struct ib_umem_odp *umem_odp)
468451
{
469-
struct ib_umem *umem = umem_odp->umem;
452+
struct ib_umem *umem = &umem_odp->umem;
470453
struct ib_ucontext *context = umem->context;
471454

472455
/*
@@ -528,8 +511,6 @@ void ib_umem_odp_release(struct ib_umem_odp *umem_odp)
528511

529512
vfree(umem_odp->dma_list);
530513
vfree(umem_odp->page_list);
531-
kfree(umem_odp);
532-
kfree(umem);
533514
}
534515

535516
/*
@@ -557,7 +538,7 @@ static int ib_umem_odp_map_dma_single_page(
557538
u64 access_mask,
558539
unsigned long current_seq)
559540
{
560-
struct ib_umem *umem = umem_odp->umem;
541+
struct ib_umem *umem = &umem_odp->umem;
561542
struct ib_device *dev = umem->context->device;
562543
dma_addr_t dma_addr;
563544
int stored_page = 0;
@@ -643,7 +624,7 @@ int ib_umem_odp_map_dma_pages(struct ib_umem_odp *umem_odp, u64 user_virt,
643624
u64 bcnt, u64 access_mask,
644625
unsigned long current_seq)
645626
{
646-
struct ib_umem *umem = umem_odp->umem;
627+
struct ib_umem *umem = &umem_odp->umem;
647628
struct task_struct *owning_process = NULL;
648629
struct mm_struct *owning_mm = NULL;
649630
struct page **local_page_list = NULL;
@@ -759,7 +740,7 @@ EXPORT_SYMBOL(ib_umem_odp_map_dma_pages);
759740
void ib_umem_odp_unmap_dma_pages(struct ib_umem_odp *umem_odp, u64 virt,
760741
u64 bound)
761742
{
762-
struct ib_umem *umem = umem_odp->umem;
743+
struct ib_umem *umem = &umem_odp->umem;
763744
int idx;
764745
u64 addr;
765746
struct ib_device *dev = umem->context->device;

0 commit comments

Comments
 (0)