Skip to content

Commit 311c7c7

Browse files
Saeed Mahameeddavem330
authored andcommitted
net/mlx5e: Allocate DMA coherent memory on reader NUMA node
By affinity hints and XPS, each mlx5e channel is assigned a CPU core. Channel DMA coherent memory that is written by the NIC and read by SW (e.g CQ buffer) is allocated on the NUMA node of the CPU core assigned for the channel. Channel DMA coherent memory that is written by SW and read by the NIC (e.g SQ/RQ buffer) is allocated on the NUMA node of the NIC. Doorbell record (written by SW and read by the NIC) is an exception since it is accessed by SW more frequently. Signed-off-by: Saeed Mahameed <saeedm@mellanox.com> Signed-off-by: Amir Vadai <amirv@mellanox.com> Signed-off-by: David S. Miller <davem@davemloft.net>
1 parent 2be6967 commit 311c7c7

File tree

6 files changed

+70
-18
lines changed

6 files changed

+70
-18
lines changed

drivers/net/ethernet/mellanox/mlx5/core/alloc.c

Lines changed: 40 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -45,15 +45,34 @@
4545
* register it in a memory region at HCA virtual address 0.
4646
*/
4747

48-
int mlx5_buf_alloc(struct mlx5_core_dev *dev, int size, struct mlx5_buf *buf)
48+
static void *mlx5_dma_zalloc_coherent_node(struct mlx5_core_dev *dev,
49+
size_t size, dma_addr_t *dma_handle,
50+
int node)
51+
{
52+
struct mlx5_priv *priv = &dev->priv;
53+
int original_node;
54+
void *cpu_handle;
55+
56+
mutex_lock(&priv->alloc_mutex);
57+
original_node = dev_to_node(&dev->pdev->dev);
58+
set_dev_node(&dev->pdev->dev, node);
59+
cpu_handle = dma_zalloc_coherent(&dev->pdev->dev, size,
60+
dma_handle, GFP_KERNEL);
61+
set_dev_node(&dev->pdev->dev, original_node);
62+
mutex_unlock(&priv->alloc_mutex);
63+
return cpu_handle;
64+
}
65+
66+
int mlx5_buf_alloc_node(struct mlx5_core_dev *dev, int size,
67+
struct mlx5_buf *buf, int node)
4968
{
5069
dma_addr_t t;
5170

5271
buf->size = size;
5372
buf->npages = 1;
5473
buf->page_shift = (u8)get_order(size) + PAGE_SHIFT;
55-
buf->direct.buf = dma_zalloc_coherent(&dev->pdev->dev,
56-
size, &t, GFP_KERNEL);
74+
buf->direct.buf = mlx5_dma_zalloc_coherent_node(dev, size,
75+
&t, node);
5776
if (!buf->direct.buf)
5877
return -ENOMEM;
5978

@@ -66,6 +85,11 @@ int mlx5_buf_alloc(struct mlx5_core_dev *dev, int size, struct mlx5_buf *buf)
6685

6786
return 0;
6887
}
88+
89+
int mlx5_buf_alloc(struct mlx5_core_dev *dev, int size, struct mlx5_buf *buf)
90+
{
91+
return mlx5_buf_alloc_node(dev, size, buf, dev->priv.numa_node);
92+
}
6993
EXPORT_SYMBOL_GPL(mlx5_buf_alloc);
7094

7195
void mlx5_buf_free(struct mlx5_core_dev *dev, struct mlx5_buf *buf)
@@ -75,7 +99,8 @@ void mlx5_buf_free(struct mlx5_core_dev *dev, struct mlx5_buf *buf)
7599
}
76100
EXPORT_SYMBOL_GPL(mlx5_buf_free);
77101

78-
static struct mlx5_db_pgdir *mlx5_alloc_db_pgdir(struct device *dma_device)
102+
static struct mlx5_db_pgdir *mlx5_alloc_db_pgdir(struct mlx5_core_dev *dev,
103+
int node)
79104
{
80105
struct mlx5_db_pgdir *pgdir;
81106

@@ -84,8 +109,9 @@ static struct mlx5_db_pgdir *mlx5_alloc_db_pgdir(struct device *dma_device)
84109
return NULL;
85110

86111
bitmap_fill(pgdir->bitmap, MLX5_DB_PER_PAGE);
87-
pgdir->db_page = dma_alloc_coherent(dma_device, PAGE_SIZE,
88-
&pgdir->db_dma, GFP_KERNEL);
112+
113+
pgdir->db_page = mlx5_dma_zalloc_coherent_node(dev, PAGE_SIZE,
114+
&pgdir->db_dma, node);
89115
if (!pgdir->db_page) {
90116
kfree(pgdir);
91117
return NULL;
@@ -118,7 +144,7 @@ static int mlx5_alloc_db_from_pgdir(struct mlx5_db_pgdir *pgdir,
118144
return 0;
119145
}
120146

121-
int mlx5_db_alloc(struct mlx5_core_dev *dev, struct mlx5_db *db)
147+
int mlx5_db_alloc_node(struct mlx5_core_dev *dev, struct mlx5_db *db, int node)
122148
{
123149
struct mlx5_db_pgdir *pgdir;
124150
int ret = 0;
@@ -129,7 +155,7 @@ int mlx5_db_alloc(struct mlx5_core_dev *dev, struct mlx5_db *db)
129155
if (!mlx5_alloc_db_from_pgdir(pgdir, db))
130156
goto out;
131157

132-
pgdir = mlx5_alloc_db_pgdir(&(dev->pdev->dev));
158+
pgdir = mlx5_alloc_db_pgdir(dev, node);
133159
if (!pgdir) {
134160
ret = -ENOMEM;
135161
goto out;
@@ -145,6 +171,12 @@ int mlx5_db_alloc(struct mlx5_core_dev *dev, struct mlx5_db *db)
145171

146172
return ret;
147173
}
174+
EXPORT_SYMBOL_GPL(mlx5_db_alloc_node);
175+
176+
int mlx5_db_alloc(struct mlx5_core_dev *dev, struct mlx5_db *db)
177+
{
178+
return mlx5_db_alloc_node(dev, db, dev->priv.numa_node);
179+
}
148180
EXPORT_SYMBOL_GPL(mlx5_db_alloc);
149181

150182
void mlx5_db_free(struct mlx5_core_dev *dev, struct mlx5_db *db)

drivers/net/ethernet/mellanox/mlx5/core/en_main.c

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -272,6 +272,8 @@ static int mlx5e_create_rq(struct mlx5e_channel *c,
272272
int err;
273273
int i;
274274

275+
param->wq.db_numa_node = cpu_to_node(c->cpu);
276+
275277
err = mlx5_wq_ll_create(mdev, &param->wq, rqc_wq, &rq->wq,
276278
&rq->wq_ctrl);
277279
if (err)
@@ -502,6 +504,8 @@ static int mlx5e_create_sq(struct mlx5e_channel *c,
502504
if (err)
503505
return err;
504506

507+
param->wq.db_numa_node = cpu_to_node(c->cpu);
508+
505509
err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, &sq->wq,
506510
&sq->wq_ctrl);
507511
if (err)
@@ -702,7 +706,8 @@ static int mlx5e_create_cq(struct mlx5e_channel *c,
702706
int err;
703707
u32 i;
704708

705-
param->wq.numa = cpu_to_node(c->cpu);
709+
param->wq.buf_numa_node = cpu_to_node(c->cpu);
710+
param->wq.db_numa_node = cpu_to_node(c->cpu);
706711
param->eq_ix = c->ix;
707712

708713
err = mlx5_cqwq_create(mdev, &param->wq, param->cqc, &cq->wq,
@@ -1000,7 +1005,7 @@ static void mlx5e_build_rq_param(struct mlx5e_priv *priv,
10001005
MLX5_SET(wq, wq, log_wq_sz, priv->params.log_rq_size);
10011006
MLX5_SET(wq, wq, pd, priv->pdn);
10021007

1003-
param->wq.numa = dev_to_node(&priv->mdev->pdev->dev);
1008+
param->wq.buf_numa_node = dev_to_node(&priv->mdev->pdev->dev);
10041009
param->wq.linear = 1;
10051010
}
10061011

@@ -1014,7 +1019,7 @@ static void mlx5e_build_sq_param(struct mlx5e_priv *priv,
10141019
MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));
10151020
MLX5_SET(wq, wq, pd, priv->pdn);
10161021

1017-
param->wq.numa = dev_to_node(&priv->mdev->pdev->dev);
1022+
param->wq.buf_numa_node = dev_to_node(&priv->mdev->pdev->dev);
10181023
}
10191024

10201025
static void mlx5e_build_common_cq_param(struct mlx5e_priv *priv,

drivers/net/ethernet/mellanox/mlx5/core/main.c

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -455,7 +455,7 @@ static int mlx5_irq_set_affinity_hint(struct mlx5_core_dev *mdev, int i)
455455
struct mlx5_priv *priv = &mdev->priv;
456456
struct msix_entry *msix = priv->msix_arr;
457457
int irq = msix[i + MLX5_EQ_VEC_COMP_BASE].vector;
458-
int numa_node = dev_to_node(&mdev->pdev->dev);
458+
int numa_node = priv->numa_node;
459459
int err;
460460

461461
if (!zalloc_cpumask_var(&priv->irq_info[i].mask, GFP_KERNEL)) {
@@ -668,6 +668,10 @@ static int mlx5_dev_init(struct mlx5_core_dev *dev, struct pci_dev *pdev)
668668
INIT_LIST_HEAD(&priv->pgdir_list);
669669
spin_lock_init(&priv->mkey_lock);
670670

671+
mutex_init(&priv->alloc_mutex);
672+
673+
priv->numa_node = dev_to_node(&dev->pdev->dev);
674+
671675
priv->dbg_root = debugfs_create_dir(dev_name(&pdev->dev), mlx5_debugfs_root);
672676
if (!priv->dbg_root)
673677
return -ENOMEM;

drivers/net/ethernet/mellanox/mlx5/core/wq.c

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -73,13 +73,14 @@ int mlx5_wq_cyc_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
7373
wq->log_stride = MLX5_GET(wq, wqc, log_wq_stride);
7474
wq->sz_m1 = (1 << MLX5_GET(wq, wqc, log_wq_sz)) - 1;
7575

76-
err = mlx5_db_alloc(mdev, &wq_ctrl->db);
76+
err = mlx5_db_alloc_node(mdev, &wq_ctrl->db, param->db_numa_node);
7777
if (err) {
7878
mlx5_core_warn(mdev, "mlx5_db_alloc() failed, %d\n", err);
7979
return err;
8080
}
8181

82-
err = mlx5_buf_alloc(mdev, mlx5_wq_cyc_get_byte_size(wq), &wq_ctrl->buf);
82+
err = mlx5_buf_alloc_node(mdev, mlx5_wq_cyc_get_byte_size(wq),
83+
&wq_ctrl->buf, param->buf_numa_node);
8384
if (err) {
8485
mlx5_core_warn(mdev, "mlx5_buf_alloc() failed, %d\n", err);
8586
goto err_db_free;
@@ -108,13 +109,14 @@ int mlx5_cqwq_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
108109
wq->log_sz = MLX5_GET(cqc, cqc, log_cq_size);
109110
wq->sz_m1 = (1 << wq->log_sz) - 1;
110111

111-
err = mlx5_db_alloc(mdev, &wq_ctrl->db);
112+
err = mlx5_db_alloc_node(mdev, &wq_ctrl->db, param->db_numa_node);
112113
if (err) {
113114
mlx5_core_warn(mdev, "mlx5_db_alloc() failed, %d\n", err);
114115
return err;
115116
}
116117

117-
err = mlx5_buf_alloc(mdev, mlx5_cqwq_get_byte_size(wq), &wq_ctrl->buf);
118+
err = mlx5_buf_alloc_node(mdev, mlx5_cqwq_get_byte_size(wq),
119+
&wq_ctrl->buf, param->buf_numa_node);
118120
if (err) {
119121
mlx5_core_warn(mdev, "mlx5_buf_alloc() failed, %d\n", err);
120122
goto err_db_free;
@@ -144,7 +146,7 @@ int mlx5_wq_ll_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
144146
wq->log_stride = MLX5_GET(wq, wqc, log_wq_stride);
145147
wq->sz_m1 = (1 << MLX5_GET(wq, wqc, log_wq_sz)) - 1;
146148

147-
err = mlx5_db_alloc(mdev, &wq_ctrl->db);
149+
err = mlx5_db_alloc_node(mdev, &wq_ctrl->db, param->db_numa_node);
148150
if (err) {
149151
mlx5_core_warn(mdev, "mlx5_db_alloc() failed, %d\n", err);
150152
return err;

drivers/net/ethernet/mellanox/mlx5/core/wq.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,8 @@
3737

3838
struct mlx5_wq_param {
3939
int linear;
40-
int numa;
40+
int buf_numa_node;
41+
int db_numa_node;
4142
};
4243

4344
struct mlx5_wq_ctrl {

include/linux/mlx5/driver.h

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -463,6 +463,10 @@ struct mlx5_priv {
463463
/* end: mr staff */
464464

465465
/* start: alloc staff */
466+
/* protect buffer alocation according to numa node */
467+
struct mutex alloc_mutex;
468+
int numa_node;
469+
466470
struct mutex pgdir_mutex;
467471
struct list_head pgdir_list;
468472
/* end: alloc staff */
@@ -672,6 +676,8 @@ void mlx5_health_cleanup(void);
672676
void __init mlx5_health_init(void);
673677
void mlx5_start_health_poll(struct mlx5_core_dev *dev);
674678
void mlx5_stop_health_poll(struct mlx5_core_dev *dev);
679+
int mlx5_buf_alloc_node(struct mlx5_core_dev *dev, int size,
680+
struct mlx5_buf *buf, int node);
675681
int mlx5_buf_alloc(struct mlx5_core_dev *dev, int size, struct mlx5_buf *buf);
676682
void mlx5_buf_free(struct mlx5_core_dev *dev, struct mlx5_buf *buf);
677683
struct mlx5_cmd_mailbox *mlx5_alloc_cmd_mailbox_chain(struct mlx5_core_dev *dev,
@@ -773,6 +779,8 @@ void mlx5_eq_debugfs_cleanup(struct mlx5_core_dev *dev);
773779
int mlx5_cq_debugfs_init(struct mlx5_core_dev *dev);
774780
void mlx5_cq_debugfs_cleanup(struct mlx5_core_dev *dev);
775781
int mlx5_db_alloc(struct mlx5_core_dev *dev, struct mlx5_db *db);
782+
int mlx5_db_alloc_node(struct mlx5_core_dev *dev, struct mlx5_db *db,
783+
int node);
776784
void mlx5_db_free(struct mlx5_core_dev *dev, struct mlx5_db *db);
777785

778786
const char *mlx5_command_str(int command);

0 commit comments

Comments
 (0)