Skip to content

Commit 9f519f6

Browse files
Yevgeny Petrilindavem330
Yevgeny Petrilin
authored and
davem330
committed
mlx4_en: Not using Shared Receive Queues
We use 1:1 mapping between QPs and SRQs on receive side, so additional indirection level not required. Allocated the receive buffers for the RSS QPs. Signed-off-by: Yevgeny Petrilin <yevgenyp@mellanox.co.il> Signed-off-by: David S. Miller <davem@davemloft.net>
1 parent b6b912e commit 9f519f6

File tree

5 files changed

+26
-73
lines changed

5 files changed

+26
-73
lines changed

‎drivers/net/mlx4/en_netdev.c

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -622,8 +622,7 @@ int mlx4_en_start_port(struct net_device *dev)
622622

623623
/* Configure ring */
624624
tx_ring = &priv->tx_ring[i];
625-
err = mlx4_en_activate_tx_ring(priv, tx_ring, cq->mcq.cqn,
626-
priv->rx_ring[0].srq.srqn);
625+
err = mlx4_en_activate_tx_ring(priv, tx_ring, cq->mcq.cqn);
627626
if (err) {
628627
en_err(priv, "Failed allocating Tx ring\n");
629628
mlx4_en_deactivate_cq(priv, cq);

‎drivers/net/mlx4/en_resources.c

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@
3737
#include "mlx4_en.h"
3838

3939
void mlx4_en_fill_qp_context(struct mlx4_en_priv *priv, int size, int stride,
40-
int is_tx, int rss, int qpn, int cqn, int srqn,
40+
int is_tx, int rss, int qpn, int cqn,
4141
struct mlx4_qp_context *context)
4242
{
4343
struct mlx4_en_dev *mdev = priv->mdev;
@@ -46,11 +46,12 @@ void mlx4_en_fill_qp_context(struct mlx4_en_priv *priv, int size, int stride,
4646
context->flags = cpu_to_be32(7 << 16 | rss << 13);
4747
context->pd = cpu_to_be32(mdev->priv_pdn);
4848
context->mtu_msgmax = 0xff;
49-
context->rq_size_stride = 0;
49+
if (!is_tx && !rss)
50+
context->rq_size_stride = ilog2(size) << 3 | (ilog2(stride) - 4);
5051
if (is_tx)
5152
context->sq_size_stride = ilog2(size) << 3 | (ilog2(stride) - 4);
5253
else
53-
context->sq_size_stride = 1;
54+
context->sq_size_stride = ilog2(TXBB_SIZE) - 4;
5455
context->usr_page = cpu_to_be32(mdev->priv_uar.index);
5556
context->local_qpn = cpu_to_be32(qpn);
5657
context->pri_path.ackto = 1 & 0x07;
@@ -59,8 +60,6 @@ void mlx4_en_fill_qp_context(struct mlx4_en_priv *priv, int size, int stride,
5960
context->cqn_send = cpu_to_be32(cqn);
6061
context->cqn_recv = cpu_to_be32(cqn);
6162
context->db_rec_addr = cpu_to_be64(priv->res.db.dma << 2);
62-
if (!rss)
63-
context->srqn = cpu_to_be32(MLX4_EN_USE_SRQ | srqn);
6463
}
6564

6665

‎drivers/net/mlx4/en_rx.c

Lines changed: 17 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -40,16 +40,6 @@
4040

4141
#include "mlx4_en.h"
4242

43-
static void *get_wqe(struct mlx4_en_rx_ring *ring, int n)
44-
{
45-
int offset = n << ring->srq.wqe_shift;
46-
return ring->buf + offset;
47-
}
48-
49-
static void mlx4_en_srq_event(struct mlx4_srq *srq, enum mlx4_event type)
50-
{
51-
return;
52-
}
5343

5444
static int mlx4_en_get_frag_header(struct skb_frag_struct *frags, void **mac_hdr,
5545
void **ip_hdr, void **tcpudp_hdr,
@@ -154,9 +144,6 @@ static void mlx4_en_init_rx_desc(struct mlx4_en_priv *priv,
154144
int possible_frags;
155145
int i;
156146

157-
/* Pre-link descriptor */
158-
rx_desc->next.next_wqe_index = cpu_to_be16((index + 1) & ring->size_mask);
159-
160147
/* Set size and memtype fields */
161148
for (i = 0; i < priv->num_frags; i++) {
162149
skb_frags[i].size = priv->frag_info[i].frag_size;
@@ -294,17 +281,14 @@ int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv,
294281
int err;
295282
int tmp;
296283

297-
/* Sanity check SRQ size before proceeding */
298-
if (size >= mdev->dev->caps.max_srq_wqes)
299-
return -EINVAL;
300284

301285
ring->prod = 0;
302286
ring->cons = 0;
303287
ring->size = size;
304288
ring->size_mask = size - 1;
305289
ring->stride = stride;
306290
ring->log_stride = ffs(ring->stride) - 1;
307-
ring->buf_size = ring->size * ring->stride;
291+
ring->buf_size = ring->size * ring->stride + TXBB_SIZE;
308292

309293
tmp = size * roundup_pow_of_two(MLX4_EN_MAX_RX_FRAGS *
310294
sizeof(struct skb_frag_struct));
@@ -360,15 +344,12 @@ int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv,
360344

361345
int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv)
362346
{
363-
struct mlx4_en_dev *mdev = priv->mdev;
364-
struct mlx4_wqe_srq_next_seg *next;
365347
struct mlx4_en_rx_ring *ring;
366348
int i;
367349
int ring_ind;
368350
int err;
369351
int stride = roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc) +
370352
DS_SIZE * priv->num_frags);
371-
int max_gs = (stride - sizeof(struct mlx4_wqe_srq_next_seg)) / DS_SIZE;
372353

373354
for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {
374355
ring = &priv->rx_ring[ring_ind];
@@ -379,6 +360,9 @@ int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv)
379360
ring->cqn = priv->rx_cq[ring_ind].mcq.cqn;
380361

381362
ring->stride = stride;
363+
if (ring->stride <= TXBB_SIZE)
364+
ring->buf += TXBB_SIZE;
365+
382366
ring->log_stride = ffs(ring->stride) - 1;
383367
ring->buf_size = ring->size * ring->stride;
384368

@@ -405,37 +389,10 @@ int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv)
405389
ring = &priv->rx_ring[ring_ind];
406390

407391
mlx4_en_update_rx_prod_db(ring);
408-
409-
/* Configure SRQ representing the ring */
410-
ring->srq.max = ring->actual_size;
411-
ring->srq.max_gs = max_gs;
412-
ring->srq.wqe_shift = ilog2(ring->stride);
413-
414-
for (i = 0; i < ring->srq.max; ++i) {
415-
next = get_wqe(ring, i);
416-
next->next_wqe_index =
417-
cpu_to_be16((i + 1) & (ring->srq.max - 1));
418-
}
419-
420-
err = mlx4_srq_alloc(mdev->dev, mdev->priv_pdn, &ring->wqres.mtt,
421-
ring->wqres.db.dma, &ring->srq);
422-
if (err){
423-
en_err(priv, "Failed to allocate srq\n");
424-
ring_ind--;
425-
goto err_srq;
426-
}
427-
ring->srq.event = mlx4_en_srq_event;
428392
}
429393

430394
return 0;
431395

432-
err_srq:
433-
while (ring_ind >= 0) {
434-
ring = &priv->rx_ring[ring_ind];
435-
mlx4_srq_free(mdev->dev, &ring->srq);
436-
ring_ind--;
437-
}
438-
439396
err_buffers:
440397
for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++)
441398
mlx4_en_free_rx_buf(priv, &priv->rx_ring[ring_ind]);
@@ -456,18 +413,17 @@ void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv,
456413

457414
kfree(ring->lro.lro_arr);
458415
mlx4_en_unmap_buffer(&ring->wqres.buf);
459-
mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size);
416+
mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size + TXBB_SIZE);
460417
vfree(ring->rx_info);
461418
ring->rx_info = NULL;
462419
}
463420

464421
void mlx4_en_deactivate_rx_ring(struct mlx4_en_priv *priv,
465422
struct mlx4_en_rx_ring *ring)
466423
{
467-
struct mlx4_en_dev *mdev = priv->mdev;
468-
469-
mlx4_srq_free(mdev->dev, &ring->srq);
470424
mlx4_en_free_rx_buf(priv, ring);
425+
if (ring->stride <= TXBB_SIZE)
426+
ring->buf -= TXBB_SIZE;
471427
mlx4_en_destroy_allocator(priv, ring);
472428
}
473429

@@ -835,8 +791,8 @@ void mlx4_en_calc_rx_buf(struct net_device *dev)
835791

836792
/* RSS related functions */
837793

838-
static int mlx4_en_config_rss_qp(struct mlx4_en_priv *priv,
839-
int qpn, int srqn, int cqn,
794+
static int mlx4_en_config_rss_qp(struct mlx4_en_priv *priv, int qpn,
795+
struct mlx4_en_rx_ring *ring,
840796
enum mlx4_qp_state *state,
841797
struct mlx4_qp *qp)
842798
{
@@ -858,13 +814,16 @@ static int mlx4_en_config_rss_qp(struct mlx4_en_priv *priv,
858814
qp->event = mlx4_en_sqp_event;
859815

860816
memset(context, 0, sizeof *context);
861-
mlx4_en_fill_qp_context(priv, 0, 0, 0, 0, qpn, cqn, srqn, context);
817+
mlx4_en_fill_qp_context(priv, ring->size, ring->stride, 0, 0,
818+
qpn, ring->cqn, context);
819+
context->db_rec_addr = cpu_to_be64(ring->wqres.db.dma);
862820

863-
err = mlx4_qp_to_ready(mdev->dev, &priv->res.mtt, context, qp, state);
821+
err = mlx4_qp_to_ready(mdev->dev, &ring->wqres.mtt, context, qp, state);
864822
if (err) {
865823
mlx4_qp_remove(mdev->dev, qp);
866824
mlx4_qp_free(mdev->dev, qp);
867825
}
826+
mlx4_en_update_rx_prod_db(ring);
868827
out:
869828
kfree(context);
870829
return err;
@@ -880,7 +839,7 @@ int mlx4_en_config_rss_steer(struct mlx4_en_priv *priv)
880839
void *ptr;
881840
int rss_xor = mdev->profile.rss_xor;
882841
u8 rss_mask = mdev->profile.rss_mask;
883-
int i, srqn, qpn, cqn;
842+
int i, qpn;
884843
int err = 0;
885844
int good_qps = 0;
886845

@@ -894,10 +853,8 @@ int mlx4_en_config_rss_steer(struct mlx4_en_priv *priv)
894853
}
895854

896855
for (i = 0; i < priv->rx_ring_num; i++) {
897-
cqn = priv->rx_ring[i].cqn;
898-
srqn = priv->rx_ring[i].srq.srqn;
899856
qpn = rss_map->base_qpn + i;
900-
err = mlx4_en_config_rss_qp(priv, qpn, srqn, cqn,
857+
err = mlx4_en_config_rss_qp(priv, qpn, &priv->rx_ring[i],
901858
&rss_map->state[i],
902859
&rss_map->qps[i]);
903860
if (err)
@@ -920,7 +877,7 @@ int mlx4_en_config_rss_steer(struct mlx4_en_priv *priv)
920877
}
921878
rss_map->indir_qp.event = mlx4_en_sqp_event;
922879
mlx4_en_fill_qp_context(priv, 0, 0, 0, 1, priv->base_qpn,
923-
priv->rx_ring[0].cqn, 0, &context);
880+
priv->rx_ring[0].cqn, &context);
924881

925882
ptr = ((void *) &context) + 0x3c;
926883
rss_context = (struct mlx4_en_rss_context *) ptr;

‎drivers/net/mlx4/en_tx.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,7 @@ void mlx4_en_destroy_tx_ring(struct mlx4_en_priv *priv,
150150

151151
int mlx4_en_activate_tx_ring(struct mlx4_en_priv *priv,
152152
struct mlx4_en_tx_ring *ring,
153-
int cq, int srqn)
153+
int cq)
154154
{
155155
struct mlx4_en_dev *mdev = priv->mdev;
156156
int err;
@@ -168,7 +168,7 @@ int mlx4_en_activate_tx_ring(struct mlx4_en_priv *priv,
168168
ring->doorbell_qpn = swab32(ring->qp.qpn << 8);
169169

170170
mlx4_en_fill_qp_context(priv, ring->size, ring->stride, 1, 0, ring->qpn,
171-
ring->cqn, srqn, &ring->context);
171+
ring->cqn, &ring->context);
172172

173173
err = mlx4_qp_to_ready(mdev->dev, &ring->wqres.mtt, &ring->context,
174174
&ring->qp, &ring->qp_state);

‎drivers/net/mlx4/mlx4_en.h

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -274,13 +274,11 @@ struct mlx4_en_tx_ring {
274274
};
275275

276276
struct mlx4_en_rx_desc {
277-
struct mlx4_wqe_srq_next_seg next;
278277
/* actual number of entries depends on rx ring stride */
279278
struct mlx4_wqe_data_seg data[0];
280279
};
281280

282281
struct mlx4_en_rx_ring {
283-
struct mlx4_srq srq;
284282
struct mlx4_hwq_resources wqres;
285283
struct mlx4_en_rx_alloc page_alloc[MLX4_EN_MAX_RX_FRAGS];
286284
struct net_lro_mgr lro;
@@ -527,7 +525,7 @@ int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv, struct mlx4_en_tx_ring *ri
527525
void mlx4_en_destroy_tx_ring(struct mlx4_en_priv *priv, struct mlx4_en_tx_ring *ring);
528526
int mlx4_en_activate_tx_ring(struct mlx4_en_priv *priv,
529527
struct mlx4_en_tx_ring *ring,
530-
int cq, int srqn);
528+
int cq);
531529
void mlx4_en_deactivate_tx_ring(struct mlx4_en_priv *priv,
532530
struct mlx4_en_tx_ring *ring);
533531

@@ -544,7 +542,7 @@ int mlx4_en_process_rx_cq(struct net_device *dev,
544542
int budget);
545543
int mlx4_en_poll_rx_cq(struct napi_struct *napi, int budget);
546544
void mlx4_en_fill_qp_context(struct mlx4_en_priv *priv, int size, int stride,
547-
int is_tx, int rss, int qpn, int cqn, int srqn,
545+
int is_tx, int rss, int qpn, int cqn,
548546
struct mlx4_qp_context *context);
549547
void mlx4_en_sqp_event(struct mlx4_qp *qp, enum mlx4_event event);
550548
int mlx4_en_map_buffer(struct mlx4_buf *buf);

0 commit comments

Comments
 (0)