Skip to content

Commit 116aa03

Browse files
vsdhanaljgunthorpe
vsdhanal
authored and
jgunthorpe
committed
IB/{hfi1, qib, rdmavt}: Move send completion logic to rdmavt
Moving send completion code into rdmavt in order to have shared logic between qib and hfi1 drivers. Reviewed-by: Mike Marciniszyn <mike.marciniszyn@intel.com> Reviewed-by: Brian Welty <brian.welty@intel.com> Signed-off-by: Venkata Sandeep Dhanalakota <venkata.s.dhanalakota@intel.com> Signed-off-by: Harish Chegondi <harish.chegondi@intel.com> Signed-off-by: Dennis Dalessandro <dennis.dalessandro@intel.com> Signed-off-by: Jason Gunthorpe <jgg@mellanox.com>
1 parent 019f118 commit 116aa03

File tree

17 files changed

+124
-114
lines changed

17 files changed

+124
-114
lines changed

‎drivers/infiniband/hw/hfi1/rc.c

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -309,7 +309,7 @@ int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
309309
}
310310
clear_ahg(qp);
311311
wqe = rvt_get_swqe_ptr(qp, qp->s_last);
312-
hfi1_send_complete(qp, wqe, qp->s_last != qp->s_acked ?
312+
rvt_send_complete(qp, wqe, qp->s_last != qp->s_acked ?
313313
IB_WC_SUCCESS : IB_WC_WR_FLUSH_ERR);
314314
/* will get called again */
315315
goto done_free_tx;
@@ -378,9 +378,9 @@ int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
378378
wqe->wr.ex.invalidate_rkey);
379379
local_ops = 1;
380380
}
381-
hfi1_send_complete(qp, wqe,
382-
err ? IB_WC_LOC_PROT_ERR
383-
: IB_WC_SUCCESS);
381+
rvt_send_complete(qp, wqe,
382+
err ? IB_WC_LOC_PROT_ERR
383+
: IB_WC_SUCCESS);
384384
if (local_ops)
385385
atomic_dec(&qp->local_ops_pending);
386386
goto done_free_tx;
@@ -1043,7 +1043,7 @@ void hfi1_restart_rc(struct rvt_qp *qp, u32 psn, int wait)
10431043
hfi1_migrate_qp(qp);
10441044
qp->s_retry = qp->s_retry_cnt;
10451045
} else if (qp->s_last == qp->s_acked) {
1046-
hfi1_send_complete(qp, wqe, IB_WC_RETRY_EXC_ERR);
1046+
rvt_send_complete(qp, wqe, IB_WC_RETRY_EXC_ERR);
10471047
rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR);
10481048
return;
10491049
} else { /* need to handle delayed completion */
@@ -1468,7 +1468,7 @@ static int do_rc_ack(struct rvt_qp *qp, u32 aeth, u32 psn, int opcode,
14681468
ibp->rvp.n_other_naks++;
14691469
class_b:
14701470
if (qp->s_last == qp->s_acked) {
1471-
hfi1_send_complete(qp, wqe, status);
1471+
rvt_send_complete(qp, wqe, status);
14721472
rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR);
14731473
}
14741474
break;
@@ -1706,7 +1706,7 @@ static void rc_rcv_resp(struct hfi1_packet *packet)
17061706
status = IB_WC_LOC_LEN_ERR;
17071707
ack_err:
17081708
if (qp->s_last == qp->s_acked) {
1709-
hfi1_send_complete(qp, wqe, status);
1709+
rvt_send_complete(qp, wqe, status);
17101710
rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR);
17111711
}
17121712
ack_done:

‎drivers/infiniband/hw/hfi1/ruc.c

Lines changed: 2 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -411,7 +411,7 @@ static void ruc_loopback(struct rvt_qp *sqp)
411411
ibp->rvp.n_loop_pkts++;
412412
flush_send:
413413
sqp->s_rnr_retry = sqp->s_rnr_retry_cnt;
414-
hfi1_send_complete(sqp, wqe, send_status);
414+
rvt_send_complete(sqp, wqe, send_status);
415415
if (local_ops) {
416416
atomic_dec(&sqp->local_ops_pending);
417417
local_ops = 0;
@@ -459,7 +459,7 @@ static void ruc_loopback(struct rvt_qp *sqp)
459459

460460
serr:
461461
spin_lock_irqsave(&sqp->s_lock, flags);
462-
hfi1_send_complete(sqp, wqe, send_status);
462+
rvt_send_complete(sqp, wqe, send_status);
463463
if (sqp->ibqp.qp_type == IB_QPT_RC) {
464464
int lastwqe = rvt_error_qp(sqp, IB_WC_WR_FLUSH_ERR);
465465

@@ -922,44 +922,3 @@ void hfi1_do_send(struct rvt_qp *qp, bool in_thread)
922922
iowait_starve_clear(ps.pkts_sent, &priv->s_iowait);
923923
spin_unlock_irqrestore(&qp->s_lock, ps.flags);
924924
}
925-
926-
/*
927-
* This should be called with s_lock held.
928-
*/
929-
void hfi1_send_complete(struct rvt_qp *qp, struct rvt_swqe *wqe,
930-
enum ib_wc_status status)
931-
{
932-
u32 old_last, last;
933-
934-
if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_OR_FLUSH_SEND))
935-
return;
936-
937-
last = qp->s_last;
938-
old_last = last;
939-
trace_hfi1_qp_send_completion(qp, wqe, last);
940-
if (++last >= qp->s_size)
941-
last = 0;
942-
trace_hfi1_qp_send_completion(qp, wqe, last);
943-
qp->s_last = last;
944-
/* See post_send() */
945-
barrier();
946-
rvt_put_swqe(wqe);
947-
if (qp->ibqp.qp_type == IB_QPT_UD ||
948-
qp->ibqp.qp_type == IB_QPT_SMI ||
949-
qp->ibqp.qp_type == IB_QPT_GSI)
950-
atomic_dec(&ibah_to_rvtah(wqe->ud_wr.ah)->refcount);
951-
952-
rvt_qp_swqe_complete(qp,
953-
wqe,
954-
ib_hfi1_wc_opcode[wqe->wr.opcode],
955-
status);
956-
957-
if (qp->s_acked == old_last)
958-
qp->s_acked = last;
959-
if (qp->s_cur == old_last)
960-
qp->s_cur = last;
961-
if (qp->s_tail == old_last)
962-
qp->s_tail = last;
963-
if (qp->state == IB_QPS_SQD && last == qp->s_cur)
964-
qp->s_draining = 0;
965-
}

‎drivers/infiniband/hw/hfi1/uc.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ int hfi1_make_uc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
8888
}
8989
clear_ahg(qp);
9090
wqe = rvt_get_swqe_ptr(qp, qp->s_last);
91-
hfi1_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR);
91+
rvt_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR);
9292
goto done_free_tx;
9393
}
9494

@@ -140,7 +140,7 @@ int hfi1_make_uc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
140140
qp, wqe->wr.ex.invalidate_rkey);
141141
local_ops = 1;
142142
}
143-
hfi1_send_complete(qp, wqe, err ? IB_WC_LOC_PROT_ERR
143+
rvt_send_complete(qp, wqe, err ? IB_WC_LOC_PROT_ERR
144144
: IB_WC_SUCCESS);
145145
if (local_ops)
146146
atomic_dec(&qp->local_ops_pending);

‎drivers/infiniband/hw/hfi1/ud.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -518,7 +518,7 @@ int hfi1_make_ud_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
518518
goto bail;
519519
}
520520
wqe = rvt_get_swqe_ptr(qp, qp->s_last);
521-
hfi1_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR);
521+
rvt_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR);
522522
goto done_free_tx;
523523
}
524524

@@ -560,7 +560,7 @@ int hfi1_make_ud_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
560560
ud_loopback(qp, wqe);
561561
spin_lock_irqsave(&qp->s_lock, tflags);
562562
ps->flags = tflags;
563-
hfi1_send_complete(qp, wqe, IB_WC_SUCCESS);
563+
rvt_send_complete(qp, wqe, IB_WC_SUCCESS);
564564
goto done_free_tx;
565565
}
566566
}

‎drivers/infiniband/hw/hfi1/verbs.c

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -492,7 +492,7 @@ static void verbs_sdma_complete(
492492

493493
spin_lock(&qp->s_lock);
494494
if (tx->wqe) {
495-
hfi1_send_complete(qp, tx->wqe, IB_WC_SUCCESS);
495+
rvt_send_complete(qp, tx->wqe, IB_WC_SUCCESS);
496496
} else if (qp->ibqp.qp_type == IB_QPT_RC) {
497497
struct hfi1_opa_header *hdr;
498498

@@ -938,7 +938,7 @@ int hfi1_verbs_send_pio(struct rvt_qp *qp, struct hfi1_pkt_state *ps,
938938
pio_bail:
939939
if (qp->s_wqe) {
940940
spin_lock_irqsave(&qp->s_lock, flags);
941-
hfi1_send_complete(qp, qp->s_wqe, wc_status);
941+
rvt_send_complete(qp, qp->s_wqe, wc_status);
942942
spin_unlock_irqrestore(&qp->s_lock, flags);
943943
} else if (qp->ibqp.qp_type == IB_QPT_RC) {
944944
spin_lock_irqsave(&qp->s_lock, flags);
@@ -1145,7 +1145,7 @@ int hfi1_verbs_send(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
11451145
hfi1_cdbg(PIO, "%s() Failed. Completing with err",
11461146
__func__);
11471147
spin_lock_irqsave(&qp->s_lock, flags);
1148-
hfi1_send_complete(qp, qp->s_wqe, IB_WC_GENERAL_ERR);
1148+
rvt_send_complete(qp, qp->s_wqe, IB_WC_GENERAL_ERR);
11491149
spin_unlock_irqrestore(&qp->s_lock, flags);
11501150
}
11511151
return -EINVAL;
@@ -1735,6 +1735,9 @@ int hfi1_register_ib_device(struct hfi1_devdata *dd)
17351735
/* post send table */
17361736
dd->verbs_dev.rdi.post_parms = hfi1_post_parms;
17371737

1738+
/* opcode translation table */
1739+
dd->verbs_dev.rdi.wc_opcode = ib_hfi1_wc_opcode;
1740+
17381741
ppd = dd->pport;
17391742
for (i = 0; i < dd->num_pports; i++, ppd++)
17401743
rvt_init_port(&dd->verbs_dev.rdi,

‎drivers/infiniband/hw/hfi1/verbs.h

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -363,9 +363,6 @@ void hfi1_do_send_from_rvt(struct rvt_qp *qp);
363363

364364
void hfi1_do_send(struct rvt_qp *qp, bool in_thread);
365365

366-
void hfi1_send_complete(struct rvt_qp *qp, struct rvt_swqe *wqe,
367-
enum ib_wc_status status);
368-
369366
void hfi1_send_rc_ack(struct hfi1_packet *packet, bool is_fecn);
370367

371368
int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps);

‎drivers/infiniband/hw/qib/qib_rc.c

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -254,7 +254,7 @@ int qib_make_rc_req(struct rvt_qp *qp, unsigned long *flags)
254254
goto bail;
255255
}
256256
wqe = rvt_get_swqe_ptr(qp, qp->s_last);
257-
qib_send_complete(qp, wqe, qp->s_last != qp->s_acked ?
257+
rvt_send_complete(qp, wqe, qp->s_last != qp->s_acked ?
258258
IB_WC_SUCCESS : IB_WC_WR_FLUSH_ERR);
259259
/* will get called again */
260260
goto done;
@@ -838,7 +838,7 @@ void qib_restart_rc(struct rvt_qp *qp, u32 psn, int wait)
838838
qib_migrate_qp(qp);
839839
qp->s_retry = qp->s_retry_cnt;
840840
} else if (qp->s_last == qp->s_acked) {
841-
qib_send_complete(qp, wqe, IB_WC_RETRY_EXC_ERR);
841+
rvt_send_complete(qp, wqe, IB_WC_RETRY_EXC_ERR);
842842
rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR);
843843
return;
844844
} else /* XXX need to handle delayed completion */
@@ -1221,7 +1221,7 @@ static int do_rc_ack(struct rvt_qp *qp, u32 aeth, u32 psn, int opcode,
12211221
ibp->rvp.n_other_naks++;
12221222
class_b:
12231223
if (qp->s_last == qp->s_acked) {
1224-
qib_send_complete(qp, wqe, status);
1224+
rvt_send_complete(qp, wqe, status);
12251225
rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR);
12261226
}
12271227
break;
@@ -1492,7 +1492,7 @@ static void qib_rc_rcv_resp(struct qib_ibport *ibp,
14921492
status = IB_WC_LOC_LEN_ERR;
14931493
ack_err:
14941494
if (qp->s_last == qp->s_acked) {
1495-
qib_send_complete(qp, wqe, status);
1495+
rvt_send_complete(qp, wqe, status);
14961496
rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR);
14971497
}
14981498
ack_done:

‎drivers/infiniband/hw/qib/qib_ruc.c

Lines changed: 2 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -403,7 +403,7 @@ static void qib_ruc_loopback(struct rvt_qp *sqp)
403403
ibp->rvp.n_loop_pkts++;
404404
flush_send:
405405
sqp->s_rnr_retry = sqp->s_rnr_retry_cnt;
406-
qib_send_complete(sqp, wqe, send_status);
406+
rvt_send_complete(sqp, wqe, send_status);
407407
goto again;
408408

409409
rnr_nak:
@@ -447,7 +447,7 @@ static void qib_ruc_loopback(struct rvt_qp *sqp)
447447

448448
serr:
449449
spin_lock_irqsave(&sqp->s_lock, flags);
450-
qib_send_complete(sqp, wqe, send_status);
450+
rvt_send_complete(sqp, wqe, send_status);
451451
if (sqp->ibqp.qp_type == IB_QPT_RC) {
452452
int lastwqe = rvt_error_qp(sqp, IB_WC_WR_FLUSH_ERR);
453453

@@ -613,42 +613,3 @@ void qib_do_send(struct rvt_qp *qp)
613613

614614
spin_unlock_irqrestore(&qp->s_lock, flags);
615615
}
616-
617-
/*
618-
* This should be called with s_lock held.
619-
*/
620-
void qib_send_complete(struct rvt_qp *qp, struct rvt_swqe *wqe,
621-
enum ib_wc_status status)
622-
{
623-
u32 old_last, last;
624-
625-
if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_OR_FLUSH_SEND))
626-
return;
627-
628-
last = qp->s_last;
629-
old_last = last;
630-
if (++last >= qp->s_size)
631-
last = 0;
632-
qp->s_last = last;
633-
/* See post_send() */
634-
barrier();
635-
rvt_put_swqe(wqe);
636-
if (qp->ibqp.qp_type == IB_QPT_UD ||
637-
qp->ibqp.qp_type == IB_QPT_SMI ||
638-
qp->ibqp.qp_type == IB_QPT_GSI)
639-
atomic_dec(&ibah_to_rvtah(wqe->ud_wr.ah)->refcount);
640-
641-
rvt_qp_swqe_complete(qp,
642-
wqe,
643-
ib_qib_wc_opcode[wqe->wr.opcode],
644-
status);
645-
646-
if (qp->s_acked == old_last)
647-
qp->s_acked = last;
648-
if (qp->s_cur == old_last)
649-
qp->s_cur = last;
650-
if (qp->s_tail == old_last)
651-
qp->s_tail = last;
652-
if (qp->state == IB_QPS_SQD && last == qp->s_cur)
653-
qp->s_draining = 0;
654-
}

‎drivers/infiniband/hw/qib/qib_sdma.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -651,7 +651,7 @@ int qib_sdma_verbs_send(struct qib_pportdata *ppd,
651651
if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)
652652
rvt_error_qp(qp, IB_WC_GENERAL_ERR);
653653
} else if (qp->s_wqe)
654-
qib_send_complete(qp, qp->s_wqe, IB_WC_GENERAL_ERR);
654+
rvt_send_complete(qp, qp->s_wqe, IB_WC_GENERAL_ERR);
655655
spin_unlock(&qp->s_lock);
656656
spin_unlock(&qp->r_lock);
657657
/* return zero to process the next send work request */

‎drivers/infiniband/hw/qib/qib_uc.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ int qib_make_uc_req(struct rvt_qp *qp, unsigned long *flags)
6868
goto bail;
6969
}
7070
wqe = rvt_get_swqe_ptr(qp, qp->s_last);
71-
qib_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR);
71+
rvt_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR);
7272
goto done;
7373
}
7474

‎drivers/infiniband/hw/qib/qib_ud.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -260,7 +260,7 @@ int qib_make_ud_req(struct rvt_qp *qp, unsigned long *flags)
260260
goto bail;
261261
}
262262
wqe = rvt_get_swqe_ptr(qp, qp->s_last);
263-
qib_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR);
263+
rvt_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR);
264264
goto done;
265265
}
266266

@@ -304,7 +304,7 @@ int qib_make_ud_req(struct rvt_qp *qp, unsigned long *flags)
304304
qib_ud_loopback(qp, wqe);
305305
spin_lock_irqsave(&qp->s_lock, tflags);
306306
*flags = tflags;
307-
qib_send_complete(qp, wqe, IB_WC_SUCCESS);
307+
rvt_send_complete(qp, wqe, IB_WC_SUCCESS);
308308
goto done;
309309
}
310310
}

‎drivers/infiniband/hw/qib/qib_verbs.c

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -731,7 +731,7 @@ static void sdma_complete(struct qib_sdma_txreq *cookie, int status)
731731

732732
spin_lock(&qp->s_lock);
733733
if (tx->wqe)
734-
qib_send_complete(qp, tx->wqe, IB_WC_SUCCESS);
734+
rvt_send_complete(qp, tx->wqe, IB_WC_SUCCESS);
735735
else if (qp->ibqp.qp_type == IB_QPT_RC) {
736736
struct ib_header *hdr;
737737

@@ -1004,7 +1004,7 @@ static int qib_verbs_send_pio(struct rvt_qp *qp, struct ib_header *ibhdr,
10041004
}
10051005
if (qp->s_wqe) {
10061006
spin_lock_irqsave(&qp->s_lock, flags);
1007-
qib_send_complete(qp, qp->s_wqe, IB_WC_SUCCESS);
1007+
rvt_send_complete(qp, qp->s_wqe, IB_WC_SUCCESS);
10081008
spin_unlock_irqrestore(&qp->s_lock, flags);
10091009
} else if (qp->ibqp.qp_type == IB_QPT_RC) {
10101010
spin_lock_irqsave(&qp->s_lock, flags);
@@ -1491,6 +1491,9 @@ static void qib_fill_device_attr(struct qib_devdata *dd)
14911491
rdi->dparms.props.max_mcast_grp;
14921492
/* post send table */
14931493
dd->verbs_dev.rdi.post_parms = qib_post_parms;
1494+
1495+
/* opcode translation table */
1496+
dd->verbs_dev.rdi.wc_opcode = ib_qib_wc_opcode;
14941497
}
14951498

14961499
/**

‎drivers/infiniband/hw/qib/qib_verbs.h

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -331,9 +331,6 @@ void _qib_do_send(struct work_struct *work);
331331

332332
void qib_do_send(struct rvt_qp *qp);
333333

334-
void qib_send_complete(struct rvt_qp *qp, struct rvt_swqe *wqe,
335-
enum ib_wc_status status);
336-
337334
void qib_send_rc_ack(struct rvt_qp *qp);
338335

339336
int qib_make_rc_req(struct rvt_qp *qp, unsigned long *flags);

0 commit comments

Comments
 (0)