Skip to content

Commit 5b0bf5e

Browse files
Jack MorgensteinRoland Dreier
Jack Morgenstein
authored and
Roland Dreier
committed
mlx4_core: Support ICM tables in coherent memory
Enable having ICM tables in coherent memory, and use coherent memory for the dMPT table. This will allow writing MPT entries for MRs both via the SW2HW_MPT command and also directly by the driver for FMR remapping without needing to flush or worry about cacheline boundaries. Signed-off-by: Jack Morgenstein <jackm@dev.mellanox.co.il> Signed-off-by: Michael S. Tsirkin <mst@dev.mellanox.co.il> Signed-off-by: Roland Dreier <rolandd@cisco.com>
1 parent 04d29b0 commit 5b0bf5e

File tree

4 files changed

+112
-49
lines changed

4 files changed

+112
-49
lines changed

‎drivers/net/mlx4/icm.c

Lines changed: 87 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
#include <linux/init.h>
3535
#include <linux/errno.h>
3636
#include <linux/mm.h>
37+
#include <linux/scatterlist.h>
3738

3839
#include <linux/mlx4/cmd.h>
3940

@@ -50,36 +51,87 @@ enum {
5051
MLX4_TABLE_CHUNK_SIZE = 1 << 18
5152
};
5253

53-
void mlx4_free_icm(struct mlx4_dev *dev, struct mlx4_icm *icm)
54+
static void mlx4_free_icm_pages(struct mlx4_dev *dev, struct mlx4_icm_chunk *chunk)
5455
{
55-
struct mlx4_icm_chunk *chunk, *tmp;
5656
int i;
5757

58-
list_for_each_entry_safe(chunk, tmp, &icm->chunk_list, list) {
59-
if (chunk->nsg > 0)
60-
pci_unmap_sg(dev->pdev, chunk->mem, chunk->npages,
61-
PCI_DMA_BIDIRECTIONAL);
58+
if (chunk->nsg > 0)
59+
pci_unmap_sg(dev->pdev, chunk->mem, chunk->npages,
60+
PCI_DMA_BIDIRECTIONAL);
61+
62+
for (i = 0; i < chunk->npages; ++i)
63+
__free_pages(chunk->mem[i].page,
64+
get_order(chunk->mem[i].length));
65+
}
6266

63-
for (i = 0; i < chunk->npages; ++i)
64-
__free_pages(chunk->mem[i].page,
65-
get_order(chunk->mem[i].length));
67+
static void mlx4_free_icm_coherent(struct mlx4_dev *dev, struct mlx4_icm_chunk *chunk)
68+
{
69+
int i;
70+
71+
for (i = 0; i < chunk->npages; ++i)
72+
dma_free_coherent(&dev->pdev->dev, chunk->mem[i].length,
73+
lowmem_page_address(chunk->mem[i].page),
74+
sg_dma_address(&chunk->mem[i]));
75+
}
76+
77+
void mlx4_free_icm(struct mlx4_dev *dev, struct mlx4_icm *icm, int coherent)
78+
{
79+
struct mlx4_icm_chunk *chunk, *tmp;
80+
81+
if (!icm)
82+
return;
83+
84+
list_for_each_entry_safe(chunk, tmp, &icm->chunk_list, list) {
85+
if (coherent)
86+
mlx4_free_icm_coherent(dev, chunk);
87+
else
88+
mlx4_free_icm_pages(dev, chunk);
6689

6790
kfree(chunk);
6891
}
6992

7093
kfree(icm);
7194
}
7295

96+
static int mlx4_alloc_icm_pages(struct scatterlist *mem, int order, gfp_t gfp_mask)
97+
{
98+
mem->page = alloc_pages(gfp_mask, order);
99+
if (!mem->page)
100+
return -ENOMEM;
101+
102+
mem->length = PAGE_SIZE << order;
103+
mem->offset = 0;
104+
return 0;
105+
}
106+
107+
static int mlx4_alloc_icm_coherent(struct device *dev, struct scatterlist *mem,
108+
int order, gfp_t gfp_mask)
109+
{
110+
void *buf = dma_alloc_coherent(dev, PAGE_SIZE << order,
111+
&sg_dma_address(mem), gfp_mask);
112+
if (!buf)
113+
return -ENOMEM;
114+
115+
sg_set_buf(mem, buf, PAGE_SIZE << order);
116+
BUG_ON(mem->offset);
117+
sg_dma_len(mem) = PAGE_SIZE << order;
118+
return 0;
119+
}
120+
73121
struct mlx4_icm *mlx4_alloc_icm(struct mlx4_dev *dev, int npages,
74-
gfp_t gfp_mask)
122+
gfp_t gfp_mask, int coherent)
75123
{
76124
struct mlx4_icm *icm;
77125
struct mlx4_icm_chunk *chunk = NULL;
78126
int cur_order;
127+
int ret;
128+
129+
/* We use sg_set_buf for coherent allocs, which assumes low memory */
130+
BUG_ON(coherent && (gfp_mask & __GFP_HIGHMEM));
79131

80132
icm = kmalloc(sizeof *icm, gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
81133
if (!icm)
82-
return icm;
134+
return NULL;
83135

84136
icm->refcount = 0;
85137
INIT_LIST_HEAD(&icm->chunk_list);
@@ -101,12 +153,20 @@ struct mlx4_icm *mlx4_alloc_icm(struct mlx4_dev *dev, int npages,
101153
while (1 << cur_order > npages)
102154
--cur_order;
103155

104-
chunk->mem[chunk->npages].page = alloc_pages(gfp_mask, cur_order);
105-
if (chunk->mem[chunk->npages].page) {
106-
chunk->mem[chunk->npages].length = PAGE_SIZE << cur_order;
107-
chunk->mem[chunk->npages].offset = 0;
156+
if (coherent)
157+
ret = mlx4_alloc_icm_coherent(&dev->pdev->dev,
158+
&chunk->mem[chunk->npages],
159+
cur_order, gfp_mask);
160+
else
161+
ret = mlx4_alloc_icm_pages(&chunk->mem[chunk->npages],
162+
cur_order, gfp_mask);
163+
164+
if (!ret) {
165+
++chunk->npages;
108166

109-
if (++chunk->npages == MLX4_ICM_CHUNK_LEN) {
167+
if (coherent)
168+
++chunk->nsg;
169+
else if (chunk->npages == MLX4_ICM_CHUNK_LEN) {
110170
chunk->nsg = pci_map_sg(dev->pdev, chunk->mem,
111171
chunk->npages,
112172
PCI_DMA_BIDIRECTIONAL);
@@ -125,7 +185,7 @@ struct mlx4_icm *mlx4_alloc_icm(struct mlx4_dev *dev, int npages,
125185
}
126186
}
127187

128-
if (chunk) {
188+
if (!coherent && chunk) {
129189
chunk->nsg = pci_map_sg(dev->pdev, chunk->mem,
130190
chunk->npages,
131191
PCI_DMA_BIDIRECTIONAL);
@@ -137,7 +197,7 @@ struct mlx4_icm *mlx4_alloc_icm(struct mlx4_dev *dev, int npages,
137197
return icm;
138198

139199
fail:
140-
mlx4_free_icm(dev, icm);
200+
mlx4_free_icm(dev, icm, coherent);
141201
return NULL;
142202
}
143203

@@ -202,15 +262,15 @@ int mlx4_table_get(struct mlx4_dev *dev, struct mlx4_icm_table *table, int obj)
202262

203263
table->icm[i] = mlx4_alloc_icm(dev, MLX4_TABLE_CHUNK_SIZE >> PAGE_SHIFT,
204264
(table->lowmem ? GFP_KERNEL : GFP_HIGHUSER) |
205-
__GFP_NOWARN);
265+
__GFP_NOWARN, table->coherent);
206266
if (!table->icm[i]) {
207267
ret = -ENOMEM;
208268
goto out;
209269
}
210270

211271
if (mlx4_MAP_ICM(dev, table->icm[i], table->virt +
212272
(u64) i * MLX4_TABLE_CHUNK_SIZE)) {
213-
mlx4_free_icm(dev, table->icm[i]);
273+
mlx4_free_icm(dev, table->icm[i], table->coherent);
214274
table->icm[i] = NULL;
215275
ret = -ENOMEM;
216276
goto out;
@@ -234,7 +294,7 @@ void mlx4_table_put(struct mlx4_dev *dev, struct mlx4_icm_table *table, int obj)
234294
if (--table->icm[i]->refcount == 0) {
235295
mlx4_UNMAP_ICM(dev, table->virt + i * MLX4_TABLE_CHUNK_SIZE,
236296
MLX4_TABLE_CHUNK_SIZE / MLX4_ICM_PAGE_SIZE);
237-
mlx4_free_icm(dev, table->icm[i]);
297+
mlx4_free_icm(dev, table->icm[i], table->coherent);
238298
table->icm[i] = NULL;
239299
}
240300

@@ -309,7 +369,7 @@ void mlx4_table_put_range(struct mlx4_dev *dev, struct mlx4_icm_table *table,
309369

310370
int mlx4_init_icm_table(struct mlx4_dev *dev, struct mlx4_icm_table *table,
311371
u64 virt, int obj_size, int nobj, int reserved,
312-
int use_lowmem)
372+
int use_lowmem, int use_coherent)
313373
{
314374
int obj_per_chunk;
315375
int num_icm;
@@ -327,6 +387,7 @@ int mlx4_init_icm_table(struct mlx4_dev *dev, struct mlx4_icm_table *table,
327387
table->num_obj = nobj;
328388
table->obj_size = obj_size;
329389
table->lowmem = use_lowmem;
390+
table->coherent = use_coherent;
330391
mutex_init(&table->mutex);
331392

332393
for (i = 0; i * MLX4_TABLE_CHUNK_SIZE < reserved * obj_size; ++i) {
@@ -336,11 +397,11 @@ int mlx4_init_icm_table(struct mlx4_dev *dev, struct mlx4_icm_table *table,
336397

337398
table->icm[i] = mlx4_alloc_icm(dev, chunk_size >> PAGE_SHIFT,
338399
(use_lowmem ? GFP_KERNEL : GFP_HIGHUSER) |
339-
__GFP_NOWARN);
400+
__GFP_NOWARN, use_coherent);
340401
if (!table->icm[i])
341402
goto err;
342403
if (mlx4_MAP_ICM(dev, table->icm[i], virt + i * MLX4_TABLE_CHUNK_SIZE)) {
343-
mlx4_free_icm(dev, table->icm[i]);
404+
mlx4_free_icm(dev, table->icm[i], use_coherent);
344405
table->icm[i] = NULL;
345406
goto err;
346407
}
@@ -359,7 +420,7 @@ int mlx4_init_icm_table(struct mlx4_dev *dev, struct mlx4_icm_table *table,
359420
if (table->icm[i]) {
360421
mlx4_UNMAP_ICM(dev, virt + i * MLX4_TABLE_CHUNK_SIZE,
361422
MLX4_TABLE_CHUNK_SIZE / MLX4_ICM_PAGE_SIZE);
362-
mlx4_free_icm(dev, table->icm[i]);
423+
mlx4_free_icm(dev, table->icm[i], use_coherent);
363424
}
364425

365426
return -ENOMEM;
@@ -373,7 +434,7 @@ void mlx4_cleanup_icm_table(struct mlx4_dev *dev, struct mlx4_icm_table *table)
373434
if (table->icm[i]) {
374435
mlx4_UNMAP_ICM(dev, table->virt + i * MLX4_TABLE_CHUNK_SIZE,
375436
MLX4_TABLE_CHUNK_SIZE / MLX4_ICM_PAGE_SIZE);
376-
mlx4_free_icm(dev, table->icm[i]);
437+
mlx4_free_icm(dev, table->icm[i], table->coherent);
377438
}
378439

379440
kfree(table->icm);

‎drivers/net/mlx4/icm.h

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,9 @@ struct mlx4_icm_iter {
6767

6868
struct mlx4_dev;
6969

70-
struct mlx4_icm *mlx4_alloc_icm(struct mlx4_dev *dev, int npages, gfp_t gfp_mask);
71-
void mlx4_free_icm(struct mlx4_dev *dev, struct mlx4_icm *icm);
70+
struct mlx4_icm *mlx4_alloc_icm(struct mlx4_dev *dev, int npages,
71+
gfp_t gfp_mask, int coherent);
72+
void mlx4_free_icm(struct mlx4_dev *dev, struct mlx4_icm *icm, int coherent);
7273

7374
int mlx4_table_get(struct mlx4_dev *dev, struct mlx4_icm_table *table, int obj);
7475
void mlx4_table_put(struct mlx4_dev *dev, struct mlx4_icm_table *table, int obj);
@@ -78,7 +79,7 @@ void mlx4_table_put_range(struct mlx4_dev *dev, struct mlx4_icm_table *table,
7879
int start, int end);
7980
int mlx4_init_icm_table(struct mlx4_dev *dev, struct mlx4_icm_table *table,
8081
u64 virt, int obj_size, int nobj, int reserved,
81-
int use_lowmem);
82+
int use_lowmem, int use_coherent);
8283
void mlx4_cleanup_icm_table(struct mlx4_dev *dev, struct mlx4_icm_table *table);
8384
int mlx4_table_get(struct mlx4_dev *dev, struct mlx4_icm_table *table, int obj);
8485
void mlx4_table_put(struct mlx4_dev *dev, struct mlx4_icm_table *table, int obj);

0 commit comments

Comments
 (0)