Skip to content

Commit 73898db

Browse files
Haggai Abramovskydavem330
Haggai Abramovsky
authored and
davem330
committed
net/mlx4: Avoid wrong virtual mappings
The dma_alloc_coherent() function returns a virtual address which can be used for coherent access to the underlying memory. On some architectures, like arm64, undefined behavior results if this memory is also accessed via virtual mappings that are not coherent. Because of their undefined nature, operations like virt_to_page() return garbage when passed virtual addresses obtained from dma_alloc_coherent(). Any subsequent mappings via vmap() of the garbage page values are unusable and result in bad things like bus errors (synchronous aborts in ARM64 speak). The mlx4 driver contains code that does the equivalent of: vmap(virt_to_page(dma_alloc_coherent)), this results in an OOPs when the device is opened. Prevent Ethernet driver to run this problematic code by forcing it to allocate contiguous memory. As for the Infiniband driver, at first we are trying to allocate contiguous memory, but in case of failure roll back to work with fragmented memory. Signed-off-by: Haggai Abramovsky <hagaya@mellanox.com> Signed-off-by: Yishai Hadas <yishaih@mellanox.com> Reported-by: David Daney <david.daney@cavium.com> Tested-by: Sinan Kaya <okaya@codeaurora.org> Signed-off-by: David S. Miller <davem@davemloft.net>
1 parent 035cd6b commit 73898db

File tree

9 files changed

+68
-125
lines changed

9 files changed

+68
-125
lines changed

‎drivers/infiniband/hw/mlx4/qp.c

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -419,7 +419,8 @@ static int set_rq_size(struct mlx4_ib_dev *dev, struct ib_qp_cap *cap,
419419
}
420420

421421
static int set_kernel_sq_size(struct mlx4_ib_dev *dev, struct ib_qp_cap *cap,
422-
enum mlx4_ib_qp_type type, struct mlx4_ib_qp *qp)
422+
enum mlx4_ib_qp_type type, struct mlx4_ib_qp *qp,
423+
bool shrink_wqe)
423424
{
424425
int s;
425426

@@ -477,7 +478,7 @@ static int set_kernel_sq_size(struct mlx4_ib_dev *dev, struct ib_qp_cap *cap,
477478
* We set WQE size to at least 64 bytes, this way stamping
478479
* invalidates each WQE.
479480
*/
480-
if (dev->dev->caps.fw_ver >= MLX4_FW_VER_WQE_CTRL_NEC &&
481+
if (shrink_wqe && dev->dev->caps.fw_ver >= MLX4_FW_VER_WQE_CTRL_NEC &&
481482
qp->sq_signal_bits && BITS_PER_LONG == 64 &&
482483
type != MLX4_IB_QPT_SMI && type != MLX4_IB_QPT_GSI &&
483484
!(type & (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_PROXY_SMI |
@@ -642,6 +643,7 @@ static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,
642643
{
643644
int qpn;
644645
int err;
646+
struct ib_qp_cap backup_cap;
645647
struct mlx4_ib_sqp *sqp;
646648
struct mlx4_ib_qp *qp;
647649
enum mlx4_ib_qp_type qp_type = (enum mlx4_ib_qp_type) init_attr->qp_type;
@@ -775,7 +777,9 @@ static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,
775777
goto err;
776778
}
777779

778-
err = set_kernel_sq_size(dev, &init_attr->cap, qp_type, qp);
780+
memcpy(&backup_cap, &init_attr->cap, sizeof(backup_cap));
781+
err = set_kernel_sq_size(dev, &init_attr->cap,
782+
qp_type, qp, true);
779783
if (err)
780784
goto err;
781785

@@ -787,9 +791,20 @@ static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,
787791
*qp->db.db = 0;
788792
}
789793

790-
if (mlx4_buf_alloc(dev->dev, qp->buf_size, PAGE_SIZE * 2, &qp->buf, gfp)) {
791-
err = -ENOMEM;
792-
goto err_db;
794+
if (mlx4_buf_alloc(dev->dev, qp->buf_size, qp->buf_size,
795+
&qp->buf, gfp)) {
796+
memcpy(&init_attr->cap, &backup_cap,
797+
sizeof(backup_cap));
798+
err = set_kernel_sq_size(dev, &init_attr->cap, qp_type,
799+
qp, false);
800+
if (err)
801+
goto err_db;
802+
803+
if (mlx4_buf_alloc(dev->dev, qp->buf_size,
804+
PAGE_SIZE * 2, &qp->buf, gfp)) {
805+
err = -ENOMEM;
806+
goto err_db;
807+
}
793808
}
794809

795810
err = mlx4_mtt_init(dev->dev, qp->buf.npages, qp->buf.page_shift,

‎drivers/net/ethernet/mellanox/mlx4/alloc.c

Lines changed: 40 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -576,41 +576,48 @@ u32 mlx4_zone_free_entries_unique(struct mlx4_zone_allocator *zones, u32 obj, u3
576576

577577
return res;
578578
}
579-
/*
580-
* Handling for queue buffers -- we allocate a bunch of memory and
581-
* register it in a memory region at HCA virtual address 0. If the
582-
* requested size is > max_direct, we split the allocation into
583-
* multiple pages, so we don't require too much contiguous memory.
584-
*/
585579

586-
int mlx4_buf_alloc(struct mlx4_dev *dev, int size, int max_direct,
587-
struct mlx4_buf *buf, gfp_t gfp)
580+
static int mlx4_buf_direct_alloc(struct mlx4_dev *dev, int size,
581+
struct mlx4_buf *buf, gfp_t gfp)
588582
{
589583
dma_addr_t t;
590584

591-
if (size <= max_direct) {
592-
buf->nbufs = 1;
593-
buf->npages = 1;
594-
buf->page_shift = get_order(size) + PAGE_SHIFT;
595-
buf->direct.buf = dma_alloc_coherent(&dev->persist->pdev->dev,
596-
size, &t, gfp);
597-
if (!buf->direct.buf)
598-
return -ENOMEM;
585+
buf->nbufs = 1;
586+
buf->npages = 1;
587+
buf->page_shift = get_order(size) + PAGE_SHIFT;
588+
buf->direct.buf =
589+
dma_zalloc_coherent(&dev->persist->pdev->dev,
590+
size, &t, gfp);
591+
if (!buf->direct.buf)
592+
return -ENOMEM;
599593

600-
buf->direct.map = t;
594+
buf->direct.map = t;
601595

602-
while (t & ((1 << buf->page_shift) - 1)) {
603-
--buf->page_shift;
604-
buf->npages *= 2;
605-
}
596+
while (t & ((1 << buf->page_shift) - 1)) {
597+
--buf->page_shift;
598+
buf->npages *= 2;
599+
}
606600

607-
memset(buf->direct.buf, 0, size);
601+
return 0;
602+
}
603+
604+
/* Handling for queue buffers -- we allocate a bunch of memory and
605+
* register it in a memory region at HCA virtual address 0. If the
606+
* requested size is > max_direct, we split the allocation into
607+
* multiple pages, so we don't require too much contiguous memory.
608+
*/
609+
int mlx4_buf_alloc(struct mlx4_dev *dev, int size, int max_direct,
610+
struct mlx4_buf *buf, gfp_t gfp)
611+
{
612+
if (size <= max_direct) {
613+
return mlx4_buf_direct_alloc(dev, size, buf, gfp);
608614
} else {
615+
dma_addr_t t;
609616
int i;
610617

611-
buf->direct.buf = NULL;
612-
buf->nbufs = (size + PAGE_SIZE - 1) / PAGE_SIZE;
613-
buf->npages = buf->nbufs;
618+
buf->direct.buf = NULL;
619+
buf->nbufs = (size + PAGE_SIZE - 1) / PAGE_SIZE;
620+
buf->npages = buf->nbufs;
614621
buf->page_shift = PAGE_SHIFT;
615622
buf->page_list = kcalloc(buf->nbufs, sizeof(*buf->page_list),
616623
gfp);
@@ -619,28 +626,12 @@ int mlx4_buf_alloc(struct mlx4_dev *dev, int size, int max_direct,
619626

620627
for (i = 0; i < buf->nbufs; ++i) {
621628
buf->page_list[i].buf =
622-
dma_alloc_coherent(&dev->persist->pdev->dev,
623-
PAGE_SIZE,
624-
&t, gfp);
629+
dma_zalloc_coherent(&dev->persist->pdev->dev,
630+
PAGE_SIZE, &t, gfp);
625631
if (!buf->page_list[i].buf)
626632
goto err_free;
627633

628634
buf->page_list[i].map = t;
629-
630-
memset(buf->page_list[i].buf, 0, PAGE_SIZE);
631-
}
632-
633-
if (BITS_PER_LONG == 64) {
634-
struct page **pages;
635-
pages = kmalloc(sizeof *pages * buf->nbufs, gfp);
636-
if (!pages)
637-
goto err_free;
638-
for (i = 0; i < buf->nbufs; ++i)
639-
pages[i] = virt_to_page(buf->page_list[i].buf);
640-
buf->direct.buf = vmap(pages, buf->nbufs, VM_MAP, PAGE_KERNEL);
641-
kfree(pages);
642-
if (!buf->direct.buf)
643-
goto err_free;
644635
}
645636
}
646637

@@ -655,15 +646,11 @@ EXPORT_SYMBOL_GPL(mlx4_buf_alloc);
655646

656647
void mlx4_buf_free(struct mlx4_dev *dev, int size, struct mlx4_buf *buf)
657648
{
658-
int i;
659-
660-
if (buf->nbufs == 1)
649+
if (buf->nbufs == 1) {
661650
dma_free_coherent(&dev->persist->pdev->dev, size,
662-
buf->direct.buf,
663-
buf->direct.map);
664-
else {
665-
if (BITS_PER_LONG == 64)
666-
vunmap(buf->direct.buf);
651+
buf->direct.buf, buf->direct.map);
652+
} else {
653+
int i;
667654

668655
for (i = 0; i < buf->nbufs; ++i)
669656
if (buf->page_list[i].buf)
@@ -789,7 +776,7 @@ void mlx4_db_free(struct mlx4_dev *dev, struct mlx4_db *db)
789776
EXPORT_SYMBOL_GPL(mlx4_db_free);
790777

791778
int mlx4_alloc_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres,
792-
int size, int max_direct)
779+
int size)
793780
{
794781
int err;
795782

@@ -799,7 +786,7 @@ int mlx4_alloc_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres,
799786

800787
*wqres->db.db = 0;
801788

802-
err = mlx4_buf_alloc(dev, size, max_direct, &wqres->buf, GFP_KERNEL);
789+
err = mlx4_buf_direct_alloc(dev, size, &wqres->buf, GFP_KERNEL);
803790
if (err)
804791
goto err_db;
805792

‎drivers/net/ethernet/mellanox/mlx4/en_cq.c

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -73,22 +73,16 @@ int mlx4_en_create_cq(struct mlx4_en_priv *priv,
7373
*/
7474
set_dev_node(&mdev->dev->persist->pdev->dev, node);
7575
err = mlx4_alloc_hwq_res(mdev->dev, &cq->wqres,
76-
cq->buf_size, 2 * PAGE_SIZE);
76+
cq->buf_size);
7777
set_dev_node(&mdev->dev->persist->pdev->dev, mdev->dev->numa_node);
7878
if (err)
7979
goto err_cq;
8080

81-
err = mlx4_en_map_buffer(&cq->wqres.buf);
82-
if (err)
83-
goto err_res;
84-
8581
cq->buf = (struct mlx4_cqe *)cq->wqres.buf.direct.buf;
8682
*pcq = cq;
8783

8884
return 0;
8985

90-
err_res:
91-
mlx4_free_hwq_res(mdev->dev, &cq->wqres, cq->buf_size);
9286
err_cq:
9387
kfree(cq);
9488
*pcq = NULL;
@@ -177,7 +171,6 @@ void mlx4_en_destroy_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq **pcq)
177171
struct mlx4_en_dev *mdev = priv->mdev;
178172
struct mlx4_en_cq *cq = *pcq;
179173

180-
mlx4_en_unmap_buffer(&cq->wqres.buf);
181174
mlx4_free_hwq_res(mdev->dev, &cq->wqres, cq->buf_size);
182175
if (mlx4_is_eq_vector_valid(mdev->dev, priv->port, cq->vector) &&
183176
cq->is_tx == RX)

‎drivers/net/ethernet/mellanox/mlx4/en_netdev.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2928,7 +2928,7 @@ int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port,
29282928

29292929
/* Allocate page for receive rings */
29302930
err = mlx4_alloc_hwq_res(mdev->dev, &priv->res,
2931-
MLX4_EN_PAGE_SIZE, MLX4_EN_PAGE_SIZE);
2931+
MLX4_EN_PAGE_SIZE);
29322932
if (err) {
29332933
en_err(priv, "Failed to allocate page for rx qps\n");
29342934
goto out;

‎drivers/net/ethernet/mellanox/mlx4/en_resources.c

Lines changed: 0 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -107,37 +107,6 @@ int mlx4_en_change_mcast_lb(struct mlx4_en_priv *priv, struct mlx4_qp *qp,
107107
return ret;
108108
}
109109

110-
int mlx4_en_map_buffer(struct mlx4_buf *buf)
111-
{
112-
struct page **pages;
113-
int i;
114-
115-
if (BITS_PER_LONG == 64 || buf->nbufs == 1)
116-
return 0;
117-
118-
pages = kmalloc(sizeof *pages * buf->nbufs, GFP_KERNEL);
119-
if (!pages)
120-
return -ENOMEM;
121-
122-
for (i = 0; i < buf->nbufs; ++i)
123-
pages[i] = virt_to_page(buf->page_list[i].buf);
124-
125-
buf->direct.buf = vmap(pages, buf->nbufs, VM_MAP, PAGE_KERNEL);
126-
kfree(pages);
127-
if (!buf->direct.buf)
128-
return -ENOMEM;
129-
130-
return 0;
131-
}
132-
133-
void mlx4_en_unmap_buffer(struct mlx4_buf *buf)
134-
{
135-
if (BITS_PER_LONG == 64 || buf->nbufs == 1)
136-
return;
137-
138-
vunmap(buf->direct.buf);
139-
}
140-
141110
void mlx4_en_sqp_event(struct mlx4_qp *qp, enum mlx4_event event)
142111
{
143112
return;

‎drivers/net/ethernet/mellanox/mlx4/en_rx.c

Lines changed: 1 addition & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -394,26 +394,18 @@ int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv,
394394

395395
/* Allocate HW buffers on provided NUMA node */
396396
set_dev_node(&mdev->dev->persist->pdev->dev, node);
397-
err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres,
398-
ring->buf_size, 2 * PAGE_SIZE);
397+
err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres, ring->buf_size);
399398
set_dev_node(&mdev->dev->persist->pdev->dev, mdev->dev->numa_node);
400399
if (err)
401400
goto err_info;
402401

403-
err = mlx4_en_map_buffer(&ring->wqres.buf);
404-
if (err) {
405-
en_err(priv, "Failed to map RX buffer\n");
406-
goto err_hwq;
407-
}
408402
ring->buf = ring->wqres.buf.direct.buf;
409403

410404
ring->hwtstamp_rx_filter = priv->hwtstamp_config.rx_filter;
411405

412406
*pring = ring;
413407
return 0;
414408

415-
err_hwq:
416-
mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size);
417409
err_info:
418410
vfree(ring->rx_info);
419411
ring->rx_info = NULL;
@@ -517,7 +509,6 @@ void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv,
517509
struct mlx4_en_dev *mdev = priv->mdev;
518510
struct mlx4_en_rx_ring *ring = *pring;
519511

520-
mlx4_en_unmap_buffer(&ring->wqres.buf);
521512
mlx4_free_hwq_res(mdev->dev, &ring->wqres, size * stride + TXBB_SIZE);
522513
vfree(ring->rx_info);
523514
ring->rx_info = NULL;

‎drivers/net/ethernet/mellanox/mlx4/en_tx.c

Lines changed: 2 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -94,20 +94,13 @@ int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv,
9494

9595
/* Allocate HW buffers on provided NUMA node */
9696
set_dev_node(&mdev->dev->persist->pdev->dev, node);
97-
err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres, ring->buf_size,
98-
2 * PAGE_SIZE);
97+
err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres, ring->buf_size);
9998
set_dev_node(&mdev->dev->persist->pdev->dev, mdev->dev->numa_node);
10099
if (err) {
101100
en_err(priv, "Failed allocating hwq resources\n");
102101
goto err_bounce;
103102
}
104103

105-
err = mlx4_en_map_buffer(&ring->wqres.buf);
106-
if (err) {
107-
en_err(priv, "Failed to map TX buffer\n");
108-
goto err_hwq_res;
109-
}
110-
111104
ring->buf = ring->wqres.buf.direct.buf;
112105

113106
en_dbg(DRV, priv, "Allocated TX ring (addr:%p) - buf:%p size:%d buf_size:%d dma:%llx\n",
@@ -118,7 +111,7 @@ int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv,
118111
MLX4_RESERVE_ETH_BF_QP);
119112
if (err) {
120113
en_err(priv, "failed reserving qp for TX ring\n");
121-
goto err_map;
114+
goto err_hwq_res;
122115
}
123116

124117
err = mlx4_qp_alloc(mdev->dev, ring->qpn, &ring->qp, GFP_KERNEL);
@@ -155,8 +148,6 @@ int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv,
155148

156149
err_reserve:
157150
mlx4_qp_release_range(mdev->dev, ring->qpn, 1);
158-
err_map:
159-
mlx4_en_unmap_buffer(&ring->wqres.buf);
160151
err_hwq_res:
161152
mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size);
162153
err_bounce:
@@ -183,7 +174,6 @@ void mlx4_en_destroy_tx_ring(struct mlx4_en_priv *priv,
183174
mlx4_qp_remove(mdev->dev, &ring->qp);
184175
mlx4_qp_free(mdev->dev, &ring->qp);
185176
mlx4_qp_release_range(priv->mdev->dev, ring->qpn, 1);
186-
mlx4_en_unmap_buffer(&ring->wqres.buf);
187177
mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size);
188178
kfree(ring->bounce_buf);
189179
ring->bounce_buf = NULL;

‎drivers/net/ethernet/mellanox/mlx4/mlx4_en.h

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -672,8 +672,6 @@ void mlx4_en_fill_qp_context(struct mlx4_en_priv *priv, int size, int stride,
672672
int is_tx, int rss, int qpn, int cqn, int user_prio,
673673
struct mlx4_qp_context *context);
674674
void mlx4_en_sqp_event(struct mlx4_qp *qp, enum mlx4_event event);
675-
int mlx4_en_map_buffer(struct mlx4_buf *buf);
676-
void mlx4_en_unmap_buffer(struct mlx4_buf *buf);
677675
int mlx4_en_change_mcast_lb(struct mlx4_en_priv *priv, struct mlx4_qp *qp,
678676
int loopback);
679677
void mlx4_en_calc_rx_buf(struct net_device *dev);

‎include/linux/mlx4/device.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1058,7 +1058,7 @@ int mlx4_buf_alloc(struct mlx4_dev *dev, int size, int max_direct,
10581058
void mlx4_buf_free(struct mlx4_dev *dev, int size, struct mlx4_buf *buf);
10591059
static inline void *mlx4_buf_offset(struct mlx4_buf *buf, int offset)
10601060
{
1061-
if (BITS_PER_LONG == 64 || buf->nbufs == 1)
1061+
if (buf->nbufs == 1)
10621062
return buf->direct.buf + offset;
10631063
else
10641064
return buf->page_list[offset >> PAGE_SHIFT].buf +
@@ -1098,7 +1098,7 @@ int mlx4_db_alloc(struct mlx4_dev *dev, struct mlx4_db *db, int order,
10981098
void mlx4_db_free(struct mlx4_dev *dev, struct mlx4_db *db);
10991099

11001100
int mlx4_alloc_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres,
1101-
int size, int max_direct);
1101+
int size);
11021102
void mlx4_free_hwq_res(struct mlx4_dev *mdev, struct mlx4_hwq_resources *wqres,
11031103
int size);
11041104

0 commit comments

Comments
 (0)