Skip to content

Commit 1570346

Browse files
vsdhanaljgunthorpe
vsdhanal
authored and
jgunthorpe
committed
IB/{hfi1, qib, rdmavt}: Move ruc_loopback to rdmavt
This patch moves ruc_loopback() from hfi1 into rdmavt for code sharing with the qib driver. Reviewed-by: Brian Welty <brian.welty@intel.com> Reviewed-by: Mike Marciniszyn <mike.marciniszyn@intel.com> Reviewed-by: Dennis Dalessandro <dennis.dalessandro@intel.com> Signed-off-by: Venkata Sandeep Dhanalakota <venkata.s.dhanalakota@intel.com> Signed-off-by: Harish Chegondi <harish.chegondi@intel.com> Signed-off-by: Dennis Dalessandro <dennis.dalessandro@intel.com> Signed-off-by: Jason Gunthorpe <jgg@mellanox.com>
1 parent 116aa03 commit 1570346

File tree

4 files changed

+335
-632
lines changed

4 files changed

+335
-632
lines changed

‎drivers/infiniband/hw/hfi1/ruc.c

Lines changed: 2 additions & 330 deletions
Original file line numberDiff line numberDiff line change
@@ -155,334 +155,6 @@ int hfi1_ruc_check_hdr(struct hfi1_ibport *ibp, struct hfi1_packet *packet)
155155
return 0;
156156
}
157157

158-
/**
159-
* ruc_loopback - handle UC and RC loopback requests
160-
* @sqp: the sending QP
161-
*
162-
* This is called from hfi1_do_send() to
163-
* forward a WQE addressed to the same HFI.
164-
* Note that although we are single threaded due to the send engine, we still
165-
* have to protect against post_send(). We don't have to worry about
166-
* receive interrupts since this is a connected protocol and all packets
167-
* will pass through here.
168-
*/
169-
static void ruc_loopback(struct rvt_qp *sqp)
170-
{
171-
struct hfi1_ibport *ibp = to_iport(sqp->ibqp.device, sqp->port_num);
172-
struct rvt_qp *qp;
173-
struct rvt_swqe *wqe;
174-
struct rvt_sge *sge;
175-
unsigned long flags;
176-
struct ib_wc wc;
177-
u64 sdata;
178-
atomic64_t *maddr;
179-
enum ib_wc_status send_status;
180-
bool release;
181-
int ret;
182-
bool copy_last = false;
183-
int local_ops = 0;
184-
185-
rcu_read_lock();
186-
187-
/*
188-
* Note that we check the responder QP state after
189-
* checking the requester's state.
190-
*/
191-
qp = rvt_lookup_qpn(ib_to_rvt(sqp->ibqp.device), &ibp->rvp,
192-
sqp->remote_qpn);
193-
194-
spin_lock_irqsave(&sqp->s_lock, flags);
195-
196-
/* Return if we are already busy processing a work request. */
197-
if ((sqp->s_flags & (RVT_S_BUSY | HFI1_S_ANY_WAIT)) ||
198-
!(ib_rvt_state_ops[sqp->state] & RVT_PROCESS_OR_FLUSH_SEND))
199-
goto unlock;
200-
201-
sqp->s_flags |= RVT_S_BUSY;
202-
203-
again:
204-
if (sqp->s_last == READ_ONCE(sqp->s_head))
205-
goto clr_busy;
206-
wqe = rvt_get_swqe_ptr(sqp, sqp->s_last);
207-
208-
/* Return if it is not OK to start a new work request. */
209-
if (!(ib_rvt_state_ops[sqp->state] & RVT_PROCESS_NEXT_SEND_OK)) {
210-
if (!(ib_rvt_state_ops[sqp->state] & RVT_FLUSH_SEND))
211-
goto clr_busy;
212-
/* We are in the error state, flush the work request. */
213-
send_status = IB_WC_WR_FLUSH_ERR;
214-
goto flush_send;
215-
}
216-
217-
/*
218-
* We can rely on the entry not changing without the s_lock
219-
* being held until we update s_last.
220-
* We increment s_cur to indicate s_last is in progress.
221-
*/
222-
if (sqp->s_last == sqp->s_cur) {
223-
if (++sqp->s_cur >= sqp->s_size)
224-
sqp->s_cur = 0;
225-
}
226-
spin_unlock_irqrestore(&sqp->s_lock, flags);
227-
228-
if (!qp || !(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) ||
229-
qp->ibqp.qp_type != sqp->ibqp.qp_type) {
230-
ibp->rvp.n_pkt_drops++;
231-
/*
232-
* For RC, the requester would timeout and retry so
233-
* shortcut the timeouts and just signal too many retries.
234-
*/
235-
if (sqp->ibqp.qp_type == IB_QPT_RC)
236-
send_status = IB_WC_RETRY_EXC_ERR;
237-
else
238-
send_status = IB_WC_SUCCESS;
239-
goto serr;
240-
}
241-
242-
memset(&wc, 0, sizeof(wc));
243-
send_status = IB_WC_SUCCESS;
244-
245-
release = true;
246-
sqp->s_sge.sge = wqe->sg_list[0];
247-
sqp->s_sge.sg_list = wqe->sg_list + 1;
248-
sqp->s_sge.num_sge = wqe->wr.num_sge;
249-
sqp->s_len = wqe->length;
250-
switch (wqe->wr.opcode) {
251-
case IB_WR_REG_MR:
252-
goto send_comp;
253-
254-
case IB_WR_LOCAL_INV:
255-
if (!(wqe->wr.send_flags & RVT_SEND_COMPLETION_ONLY)) {
256-
if (rvt_invalidate_rkey(sqp,
257-
wqe->wr.ex.invalidate_rkey))
258-
send_status = IB_WC_LOC_PROT_ERR;
259-
local_ops = 1;
260-
}
261-
goto send_comp;
262-
263-
case IB_WR_SEND_WITH_INV:
264-
if (!rvt_invalidate_rkey(qp, wqe->wr.ex.invalidate_rkey)) {
265-
wc.wc_flags = IB_WC_WITH_INVALIDATE;
266-
wc.ex.invalidate_rkey = wqe->wr.ex.invalidate_rkey;
267-
}
268-
goto send;
269-
270-
case IB_WR_SEND_WITH_IMM:
271-
wc.wc_flags = IB_WC_WITH_IMM;
272-
wc.ex.imm_data = wqe->wr.ex.imm_data;
273-
/* FALLTHROUGH */
274-
case IB_WR_SEND:
275-
send:
276-
ret = rvt_get_rwqe(qp, false);
277-
if (ret < 0)
278-
goto op_err;
279-
if (!ret)
280-
goto rnr_nak;
281-
break;
282-
283-
case IB_WR_RDMA_WRITE_WITH_IMM:
284-
if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE)))
285-
goto inv_err;
286-
wc.wc_flags = IB_WC_WITH_IMM;
287-
wc.ex.imm_data = wqe->wr.ex.imm_data;
288-
ret = rvt_get_rwqe(qp, true);
289-
if (ret < 0)
290-
goto op_err;
291-
if (!ret)
292-
goto rnr_nak;
293-
/* skip copy_last set and qp_access_flags recheck */
294-
goto do_write;
295-
case IB_WR_RDMA_WRITE:
296-
copy_last = rvt_is_user_qp(qp);
297-
if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE)))
298-
goto inv_err;
299-
do_write:
300-
if (wqe->length == 0)
301-
break;
302-
if (unlikely(!rvt_rkey_ok(qp, &qp->r_sge.sge, wqe->length,
303-
wqe->rdma_wr.remote_addr,
304-
wqe->rdma_wr.rkey,
305-
IB_ACCESS_REMOTE_WRITE)))
306-
goto acc_err;
307-
qp->r_sge.sg_list = NULL;
308-
qp->r_sge.num_sge = 1;
309-
qp->r_sge.total_len = wqe->length;
310-
break;
311-
312-
case IB_WR_RDMA_READ:
313-
if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_READ)))
314-
goto inv_err;
315-
if (unlikely(!rvt_rkey_ok(qp, &sqp->s_sge.sge, wqe->length,
316-
wqe->rdma_wr.remote_addr,
317-
wqe->rdma_wr.rkey,
318-
IB_ACCESS_REMOTE_READ)))
319-
goto acc_err;
320-
release = false;
321-
sqp->s_sge.sg_list = NULL;
322-
sqp->s_sge.num_sge = 1;
323-
qp->r_sge.sge = wqe->sg_list[0];
324-
qp->r_sge.sg_list = wqe->sg_list + 1;
325-
qp->r_sge.num_sge = wqe->wr.num_sge;
326-
qp->r_sge.total_len = wqe->length;
327-
break;
328-
329-
case IB_WR_ATOMIC_CMP_AND_SWP:
330-
case IB_WR_ATOMIC_FETCH_AND_ADD:
331-
if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC)))
332-
goto inv_err;
333-
if (unlikely(!rvt_rkey_ok(qp, &qp->r_sge.sge, sizeof(u64),
334-
wqe->atomic_wr.remote_addr,
335-
wqe->atomic_wr.rkey,
336-
IB_ACCESS_REMOTE_ATOMIC)))
337-
goto acc_err;
338-
/* Perform atomic OP and save result. */
339-
maddr = (atomic64_t *)qp->r_sge.sge.vaddr;
340-
sdata = wqe->atomic_wr.compare_add;
341-
*(u64 *)sqp->s_sge.sge.vaddr =
342-
(wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) ?
343-
(u64)atomic64_add_return(sdata, maddr) - sdata :
344-
(u64)cmpxchg((u64 *)qp->r_sge.sge.vaddr,
345-
sdata, wqe->atomic_wr.swap);
346-
rvt_put_mr(qp->r_sge.sge.mr);
347-
qp->r_sge.num_sge = 0;
348-
goto send_comp;
349-
350-
default:
351-
send_status = IB_WC_LOC_QP_OP_ERR;
352-
goto serr;
353-
}
354-
355-
sge = &sqp->s_sge.sge;
356-
while (sqp->s_len) {
357-
u32 len = sqp->s_len;
358-
359-
if (len > sge->length)
360-
len = sge->length;
361-
if (len > sge->sge_length)
362-
len = sge->sge_length;
363-
WARN_ON_ONCE(len == 0);
364-
rvt_copy_sge(qp, &qp->r_sge, sge->vaddr,
365-
len, release, copy_last);
366-
sge->vaddr += len;
367-
sge->length -= len;
368-
sge->sge_length -= len;
369-
if (sge->sge_length == 0) {
370-
if (!release)
371-
rvt_put_mr(sge->mr);
372-
if (--sqp->s_sge.num_sge)
373-
*sge = *sqp->s_sge.sg_list++;
374-
} else if (sge->length == 0 && sge->mr->lkey) {
375-
if (++sge->n >= RVT_SEGSZ) {
376-
if (++sge->m >= sge->mr->mapsz)
377-
break;
378-
sge->n = 0;
379-
}
380-
sge->vaddr =
381-
sge->mr->map[sge->m]->segs[sge->n].vaddr;
382-
sge->length =
383-
sge->mr->map[sge->m]->segs[sge->n].length;
384-
}
385-
sqp->s_len -= len;
386-
}
387-
if (release)
388-
rvt_put_ss(&qp->r_sge);
389-
390-
if (!test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags))
391-
goto send_comp;
392-
393-
if (wqe->wr.opcode == IB_WR_RDMA_WRITE_WITH_IMM)
394-
wc.opcode = IB_WC_RECV_RDMA_WITH_IMM;
395-
else
396-
wc.opcode = IB_WC_RECV;
397-
wc.wr_id = qp->r_wr_id;
398-
wc.status = IB_WC_SUCCESS;
399-
wc.byte_len = wqe->length;
400-
wc.qp = &qp->ibqp;
401-
wc.src_qp = qp->remote_qpn;
402-
wc.slid = rdma_ah_get_dlid(&qp->remote_ah_attr) & U16_MAX;
403-
wc.sl = rdma_ah_get_sl(&qp->remote_ah_attr);
404-
wc.port_num = 1;
405-
/* Signal completion event if the solicited bit is set. */
406-
rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc,
407-
wqe->wr.send_flags & IB_SEND_SOLICITED);
408-
409-
send_comp:
410-
spin_lock_irqsave(&sqp->s_lock, flags);
411-
ibp->rvp.n_loop_pkts++;
412-
flush_send:
413-
sqp->s_rnr_retry = sqp->s_rnr_retry_cnt;
414-
rvt_send_complete(sqp, wqe, send_status);
415-
if (local_ops) {
416-
atomic_dec(&sqp->local_ops_pending);
417-
local_ops = 0;
418-
}
419-
goto again;
420-
421-
rnr_nak:
422-
/* Handle RNR NAK */
423-
if (qp->ibqp.qp_type == IB_QPT_UC)
424-
goto send_comp;
425-
ibp->rvp.n_rnr_naks++;
426-
/*
427-
* Note: we don't need the s_lock held since the BUSY flag
428-
* makes this single threaded.
429-
*/
430-
if (sqp->s_rnr_retry == 0) {
431-
send_status = IB_WC_RNR_RETRY_EXC_ERR;
432-
goto serr;
433-
}
434-
if (sqp->s_rnr_retry_cnt < 7)
435-
sqp->s_rnr_retry--;
436-
spin_lock_irqsave(&sqp->s_lock, flags);
437-
if (!(ib_rvt_state_ops[sqp->state] & RVT_PROCESS_RECV_OK))
438-
goto clr_busy;
439-
rvt_add_rnr_timer(sqp, qp->r_min_rnr_timer <<
440-
IB_AETH_CREDIT_SHIFT);
441-
goto clr_busy;
442-
443-
op_err:
444-
send_status = IB_WC_REM_OP_ERR;
445-
wc.status = IB_WC_LOC_QP_OP_ERR;
446-
goto err;
447-
448-
inv_err:
449-
send_status = IB_WC_REM_INV_REQ_ERR;
450-
wc.status = IB_WC_LOC_QP_OP_ERR;
451-
goto err;
452-
453-
acc_err:
454-
send_status = IB_WC_REM_ACCESS_ERR;
455-
wc.status = IB_WC_LOC_PROT_ERR;
456-
err:
457-
/* responder goes to error state */
458-
rvt_rc_error(qp, wc.status);
459-
460-
serr:
461-
spin_lock_irqsave(&sqp->s_lock, flags);
462-
rvt_send_complete(sqp, wqe, send_status);
463-
if (sqp->ibqp.qp_type == IB_QPT_RC) {
464-
int lastwqe = rvt_error_qp(sqp, IB_WC_WR_FLUSH_ERR);
465-
466-
sqp->s_flags &= ~RVT_S_BUSY;
467-
spin_unlock_irqrestore(&sqp->s_lock, flags);
468-
if (lastwqe) {
469-
struct ib_event ev;
470-
471-
ev.device = sqp->ibqp.device;
472-
ev.element.qp = &sqp->ibqp;
473-
ev.event = IB_EVENT_QP_LAST_WQE_REACHED;
474-
sqp->ibqp.event_handler(&ev, sqp->ibqp.qp_context);
475-
}
476-
goto done;
477-
}
478-
clr_busy:
479-
sqp->s_flags &= ~RVT_S_BUSY;
480-
unlock:
481-
spin_unlock_irqrestore(&sqp->s_lock, flags);
482-
done:
483-
rcu_read_unlock();
484-
}
485-
486158
/**
487159
* hfi1_make_grh - construct a GRH header
488160
* @ibp: a pointer to the IB port
@@ -860,7 +532,7 @@ void hfi1_do_send(struct rvt_qp *qp, bool in_thread)
860532
if (!loopback && ((rdma_ah_get_dlid(&qp->remote_ah_attr) &
861533
~((1 << ps.ppd->lmc) - 1)) ==
862534
ps.ppd->lid)) {
863-
ruc_loopback(qp);
535+
rvt_ruc_loopback(qp);
864536
return;
865537
}
866538
make_req = hfi1_make_rc_req;
@@ -870,7 +542,7 @@ void hfi1_do_send(struct rvt_qp *qp, bool in_thread)
870542
if (!loopback && ((rdma_ah_get_dlid(&qp->remote_ah_attr) &
871543
~((1 << ps.ppd->lmc) - 1)) ==
872544
ps.ppd->lid)) {
873-
ruc_loopback(qp);
545+
rvt_ruc_loopback(qp);
874546
return;
875547
}
876548
make_req = hfi1_make_uc_req;

0 commit comments

Comments
 (0)