Skip to content

Commit 019f118

Browse files
brianweltyjgunthorpe
brianwelty
authored and
jgunthorpe
committed
IB/{hfi1, qib, rdmavt}: Move copy SGE logic into rdmavt
This patch moves hfi1_copy_sge() into rdmavt for sharing with qib. This patch also moves all the wss_*() functions into rdmavt as several wss_*() functions are called from hfi1_copy_sge() When SGE copy mode is adaptive, cacheless copy may be done in some cases for performance reasons. In those cases, X86 cacheless copy function is called since the drivers that use rdmavt and may set SGE copy mode to adaptive are X86 only. For this reason, this patch adds "depends on X86_64" to rdmavt/Kconfig. Reviewed-by: Ashutosh Dixit <ashutosh.dixit@intel.com> Reviewed-by: Michael J. Ruhl <michael.j.ruhl@intel.com> Reviewed-by: Mike Marciniszyn <mike.marciniszyn@intel.com> Reviewed-by: Dennis Dalessandro <dennis.dalessandro@intel.com> Signed-off-by: Brian Welty <brian.welty@intel.com> Signed-off-by: Harish Chegondi <harish.chegondi@intel.com> Signed-off-by: Dennis Dalessandro <dennis.dalessandro@intel.com> Signed-off-by: Jason Gunthorpe <jgg@mellanox.com>
1 parent b56511c commit 019f118

File tree

19 files changed

+344
-314
lines changed

19 files changed

+344
-314
lines changed

‎drivers/infiniband/hw/hfi1/init.c

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1504,9 +1504,6 @@ static int __init hfi1_mod_init(void)
15041504
idr_init(&hfi1_unit_table);
15051505

15061506
hfi1_dbg_init();
1507-
ret = hfi1_wss_init();
1508-
if (ret < 0)
1509-
goto bail_wss;
15101507
ret = pci_register_driver(&hfi1_pci_driver);
15111508
if (ret < 0) {
15121509
pr_err("Unable to register driver: error %d\n", -ret);
@@ -1515,8 +1512,6 @@ static int __init hfi1_mod_init(void)
15151512
goto bail; /* all OK */
15161513

15171514
bail_dev:
1518-
hfi1_wss_exit();
1519-
bail_wss:
15201515
hfi1_dbg_exit();
15211516
idr_destroy(&hfi1_unit_table);
15221517
dev_cleanup();
@@ -1533,7 +1528,6 @@ static void __exit hfi1_mod_cleanup(void)
15331528
{
15341529
pci_unregister_driver(&hfi1_pci_driver);
15351530
node_affinity_destroy_all();
1536-
hfi1_wss_exit();
15371531
hfi1_dbg_exit();
15381532

15391533
idr_destroy(&hfi1_unit_table);

‎drivers/infiniband/hw/hfi1/rc.c

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1644,7 +1644,8 @@ static void rc_rcv_resp(struct hfi1_packet *packet)
16441644
qp->s_rdma_read_len -= pmtu;
16451645
update_last_psn(qp, psn);
16461646
spin_unlock_irqrestore(&qp->s_lock, flags);
1647-
hfi1_copy_sge(&qp->s_rdma_read_sge, data, pmtu, false, false);
1647+
rvt_copy_sge(qp, &qp->s_rdma_read_sge,
1648+
data, pmtu, false, false);
16481649
goto bail;
16491650

16501651
case OP(RDMA_READ_RESPONSE_ONLY):
@@ -1684,7 +1685,8 @@ static void rc_rcv_resp(struct hfi1_packet *packet)
16841685
if (unlikely(tlen != qp->s_rdma_read_len))
16851686
goto ack_len_err;
16861687
aeth = be32_to_cpu(ohdr->u.aeth);
1687-
hfi1_copy_sge(&qp->s_rdma_read_sge, data, tlen, false, false);
1688+
rvt_copy_sge(qp, &qp->s_rdma_read_sge,
1689+
data, tlen, false, false);
16881690
WARN_ON(qp->s_rdma_read_sge.num_sge);
16891691
(void)do_rc_ack(qp, aeth, psn,
16901692
OP(RDMA_READ_RESPONSE_LAST), 0, rcd);
@@ -2144,7 +2146,7 @@ void hfi1_rc_rcv(struct hfi1_packet *packet)
21442146
qp->r_rcv_len += pmtu;
21452147
if (unlikely(qp->r_rcv_len > qp->r_len))
21462148
goto nack_inv;
2147-
hfi1_copy_sge(&qp->r_sge, data, pmtu, true, false);
2149+
rvt_copy_sge(qp, &qp->r_sge, data, pmtu, true, false);
21482150
break;
21492151

21502152
case OP(RDMA_WRITE_LAST_WITH_IMMEDIATE):
@@ -2200,7 +2202,7 @@ void hfi1_rc_rcv(struct hfi1_packet *packet)
22002202
wc.byte_len = tlen + qp->r_rcv_len;
22012203
if (unlikely(wc.byte_len > qp->r_len))
22022204
goto nack_inv;
2203-
hfi1_copy_sge(&qp->r_sge, data, tlen, true, copy_last);
2205+
rvt_copy_sge(qp, &qp->r_sge, data, tlen, true, copy_last);
22042206
rvt_put_ss(&qp->r_sge);
22052207
qp->r_msn++;
22062208
if (!__test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags))

‎drivers/infiniband/hw/hfi1/ruc.c

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -361,7 +361,8 @@ static void ruc_loopback(struct rvt_qp *sqp)
361361
if (len > sge->sge_length)
362362
len = sge->sge_length;
363363
WARN_ON_ONCE(len == 0);
364-
hfi1_copy_sge(&qp->r_sge, sge->vaddr, len, release, copy_last);
364+
rvt_copy_sge(qp, &qp->r_sge, sge->vaddr,
365+
len, release, copy_last);
365366
sge->vaddr += len;
366367
sge->length -= len;
367368
sge->sge_length -= len;

‎drivers/infiniband/hw/hfi1/uc.c

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -426,7 +426,7 @@ void hfi1_uc_rcv(struct hfi1_packet *packet)
426426
qp->r_rcv_len += pmtu;
427427
if (unlikely(qp->r_rcv_len > qp->r_len))
428428
goto rewind;
429-
hfi1_copy_sge(&qp->r_sge, data, pmtu, false, false);
429+
rvt_copy_sge(qp, &qp->r_sge, data, pmtu, false, false);
430430
break;
431431

432432
case OP(SEND_LAST_WITH_IMMEDIATE):
@@ -449,7 +449,7 @@ void hfi1_uc_rcv(struct hfi1_packet *packet)
449449
if (unlikely(wc.byte_len > qp->r_len))
450450
goto rewind;
451451
wc.opcode = IB_WC_RECV;
452-
hfi1_copy_sge(&qp->r_sge, data, tlen, false, false);
452+
rvt_copy_sge(qp, &qp->r_sge, data, tlen, false, false);
453453
rvt_put_ss(&qp->s_rdma_read_sge);
454454
last_imm:
455455
wc.wr_id = qp->r_wr_id;
@@ -523,7 +523,7 @@ void hfi1_uc_rcv(struct hfi1_packet *packet)
523523
qp->r_rcv_len += pmtu;
524524
if (unlikely(qp->r_rcv_len > qp->r_len))
525525
goto drop;
526-
hfi1_copy_sge(&qp->r_sge, data, pmtu, true, false);
526+
rvt_copy_sge(qp, &qp->r_sge, data, pmtu, true, false);
527527
break;
528528

529529
case OP(RDMA_WRITE_LAST_WITH_IMMEDIATE):
@@ -550,7 +550,7 @@ void hfi1_uc_rcv(struct hfi1_packet *packet)
550550
}
551551
wc.byte_len = qp->r_len;
552552
wc.opcode = IB_WC_RECV_RDMA_WITH_IMM;
553-
hfi1_copy_sge(&qp->r_sge, data, tlen, true, false);
553+
rvt_copy_sge(qp, &qp->r_sge, data, tlen, true, false);
554554
rvt_put_ss(&qp->r_sge);
555555
goto last_imm;
556556

@@ -564,7 +564,7 @@ void hfi1_uc_rcv(struct hfi1_packet *packet)
564564
tlen -= (hdrsize + extra_bytes);
565565
if (unlikely(tlen + qp->r_rcv_len != qp->r_len))
566566
goto drop;
567-
hfi1_copy_sge(&qp->r_sge, data, tlen, true, false);
567+
rvt_copy_sge(qp, &qp->r_sge, data, tlen, true, false);
568568
rvt_put_ss(&qp->r_sge);
569569
break;
570570

‎drivers/infiniband/hw/hfi1/ud.c

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -210,8 +210,8 @@ static void ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe)
210210
}
211211

212212
hfi1_make_grh(ibp, &grh, &grd, 0, 0);
213-
hfi1_copy_sge(&qp->r_sge, &grh,
214-
sizeof(grh), true, false);
213+
rvt_copy_sge(qp, &qp->r_sge, &grh,
214+
sizeof(grh), true, false);
215215
wc.wc_flags |= IB_WC_GRH;
216216
} else {
217217
rvt_skip_sge(&qp->r_sge, sizeof(struct ib_grh), true);
@@ -228,7 +228,7 @@ static void ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe)
228228
if (len > sge->sge_length)
229229
len = sge->sge_length;
230230
WARN_ON_ONCE(len == 0);
231-
hfi1_copy_sge(&qp->r_sge, sge->vaddr, len, true, false);
231+
rvt_copy_sge(qp, &qp->r_sge, sge->vaddr, len, true, false);
232232
sge->vaddr += len;
233233
sge->length -= len;
234234
sge->sge_length -= len;
@@ -1019,8 +1019,8 @@ void hfi1_ud_rcv(struct hfi1_packet *packet)
10191019
goto drop;
10201020
}
10211021
if (packet->grh) {
1022-
hfi1_copy_sge(&qp->r_sge, packet->grh,
1023-
sizeof(struct ib_grh), true, false);
1022+
rvt_copy_sge(qp, &qp->r_sge, packet->grh,
1023+
sizeof(struct ib_grh), true, false);
10241024
wc.wc_flags |= IB_WC_GRH;
10251025
} else if (packet->etype == RHF_RCV_TYPE_BYPASS) {
10261026
struct ib_grh grh;
@@ -1030,14 +1030,14 @@ void hfi1_ud_rcv(struct hfi1_packet *packet)
10301030
* out when creating 16B, add back the GRH here.
10311031
*/
10321032
hfi1_make_ext_grh(packet, &grh, slid, dlid);
1033-
hfi1_copy_sge(&qp->r_sge, &grh,
1034-
sizeof(struct ib_grh), true, false);
1033+
rvt_copy_sge(qp, &qp->r_sge, &grh,
1034+
sizeof(struct ib_grh), true, false);
10351035
wc.wc_flags |= IB_WC_GRH;
10361036
} else {
10371037
rvt_skip_sge(&qp->r_sge, sizeof(struct ib_grh), true);
10381038
}
1039-
hfi1_copy_sge(&qp->r_sge, data, wc.byte_len - sizeof(struct ib_grh),
1040-
true, false);
1039+
rvt_copy_sge(qp, &qp->r_sge, data, wc.byte_len - sizeof(struct ib_grh),
1040+
true, false);
10411041
rvt_put_ss(&qp->r_sge);
10421042
if (!test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags))
10431043
return;

0 commit comments

Comments
 (0)