Skip to content

Commit f4f3003

Browse files
Dean Luickgregkh
Dean Luick
authored and
gregkh
committed
staging/rdma/hfi1: Thread the receive interrupt.
When under heavy load, the receive interrupt handler can run too long with IRQs disabled. Add a mixed-mode threading scheme. Initially process packets in the handler for quick responses (latency). If there are too many packets to process move to a thread to continue (bandwidth). Reviewed-by: Mike Marciniszyn <mike.marciniszyn@intel.com> Signed-off-by: Dean Luick <dean.luick@intel.com> Signed-off-by: Ira Weiny <ira.weiny@intel.com> Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
1 parent b77d713 commit f4f3003

File tree

4 files changed

+154
-46
lines changed

4 files changed

+154
-46
lines changed

‎drivers/staging/rdma/hfi1/chip.c

Lines changed: 95 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4424,7 +4424,7 @@ static void is_rcv_avail_int(struct hfi1_devdata *dd, unsigned int source)
44244424
rcd = dd->rcd[source];
44254425
if (rcd) {
44264426
if (source < dd->first_user_ctxt)
4427-
rcd->do_interrupt(rcd);
4427+
rcd->do_interrupt(rcd, 0);
44284428
else
44294429
handle_user_interrupt(rcd);
44304430
return; /* OK */
@@ -4590,23 +4590,106 @@ static irqreturn_t sdma_interrupt(int irq, void *data)
45904590
}
45914591

45924592
/*
4593-
* NOTE: this routine expects to be on its own MSI-X interrupt. If
4594-
* multiple receive contexts share the same MSI-X interrupt, then this
4595-
* routine must check for who received it.
4593+
* Clear the receive interrupt, forcing the write and making sure
4594+
* we have data from the chip, pushing everything in front of it
4595+
* back to the host.
4596+
*/
4597+
static inline void clear_recv_intr(struct hfi1_ctxtdata *rcd)
4598+
{
4599+
struct hfi1_devdata *dd = rcd->dd;
4600+
u32 addr = CCE_INT_CLEAR + (8 * rcd->ireg);
4601+
4602+
mmiowb(); /* make sure everything before is written */
4603+
write_csr(dd, addr, rcd->imask);
4604+
/* force the above write on the chip and get a value back */
4605+
(void)read_csr(dd, addr);
4606+
}
4607+
4608+
/* force the receive interrupt */
4609+
static inline void force_recv_intr(struct hfi1_ctxtdata *rcd)
4610+
{
4611+
write_csr(rcd->dd, CCE_INT_FORCE + (8 * rcd->ireg), rcd->imask);
4612+
}
4613+
4614+
/* return non-zero if a packet is present */
4615+
static inline int check_packet_present(struct hfi1_ctxtdata *rcd)
4616+
{
4617+
if (!HFI1_CAP_IS_KSET(DMA_RTAIL))
4618+
return (rcd->seq_cnt ==
4619+
rhf_rcv_seq(rhf_to_cpu(get_rhf_addr(rcd))));
4620+
4621+
/* else is RDMA rtail */
4622+
return (rcd->head != get_rcvhdrtail(rcd));
4623+
}
4624+
4625+
/*
4626+
* Receive packet IRQ handler. This routine expects to be on its own IRQ.
4627+
* This routine will try to handle packets immediately (latency), but if
4628+
* it finds too many, it will invoke the thread handler (bandwitdh). The
4629+
* chip receive interupt is *not* cleared down until this or the thread (if
4630+
* invoked) is finished. The intent is to avoid extra interrupts while we
4631+
* are processing packets anyway.
45964632
*/
45974633
static irqreturn_t receive_context_interrupt(int irq, void *data)
45984634
{
45994635
struct hfi1_ctxtdata *rcd = data;
46004636
struct hfi1_devdata *dd = rcd->dd;
4637+
int disposition;
4638+
int present;
46014639

46024640
trace_hfi1_receive_interrupt(dd, rcd->ctxt);
46034641
this_cpu_inc(*dd->int_counter);
46044642

4605-
/* clear the interrupt */
4606-
write_csr(rcd->dd, CCE_INT_CLEAR + (8*rcd->ireg), rcd->imask);
4643+
/* receive interrupt remains blocked while processing packets */
4644+
disposition = rcd->do_interrupt(rcd, 0);
46074645

4608-
/* handle the interrupt */
4609-
rcd->do_interrupt(rcd);
4646+
/*
4647+
* Too many packets were seen while processing packets in this
4648+
* IRQ handler. Invoke the handler thread. The receive interrupt
4649+
* remains blocked.
4650+
*/
4651+
if (disposition == RCV_PKT_LIMIT)
4652+
return IRQ_WAKE_THREAD;
4653+
4654+
/*
4655+
* The packet processor detected no more packets. Clear the receive
4656+
* interrupt and recheck for a packet packet that may have arrived
4657+
* after the previous check and interrupt clear. If a packet arrived,
4658+
* force another interrupt.
4659+
*/
4660+
clear_recv_intr(rcd);
4661+
present = check_packet_present(rcd);
4662+
if (present)
4663+
force_recv_intr(rcd);
4664+
4665+
return IRQ_HANDLED;
4666+
}
4667+
4668+
/*
4669+
* Receive packet thread handler. This expects to be invoked with the
4670+
* receive interrupt still blocked.
4671+
*/
4672+
static irqreturn_t receive_context_thread(int irq, void *data)
4673+
{
4674+
struct hfi1_ctxtdata *rcd = data;
4675+
int present;
4676+
4677+
/* receive interrupt is still blocked from the IRQ handler */
4678+
(void)rcd->do_interrupt(rcd, 1);
4679+
4680+
/*
4681+
* The packet processor will only return if it detected no more
4682+
* packets. Hold IRQs here so we can safely clear the interrupt and
4683+
* recheck for a packet that may have arrived after the previous
4684+
* check and the interrupt clear. If a packet arrived, force another
4685+
* interrupt.
4686+
*/
4687+
local_irq_disable();
4688+
clear_recv_intr(rcd);
4689+
present = check_packet_present(rcd);
4690+
if (present)
4691+
force_recv_intr(rcd);
4692+
local_irq_enable();
46104693

46114694
return IRQ_HANDLED;
46124695
}
@@ -8858,6 +8941,7 @@ static int request_msix_irqs(struct hfi1_devdata *dd)
88588941
struct hfi1_msix_entry *me = &dd->msix_entries[i];
88598942
const char *err_info;
88608943
irq_handler_t handler;
8944+
irq_handler_t thread = NULL;
88618945
void *arg;
88628946
int idx;
88638947
struct hfi1_ctxtdata *rcd = NULL;
@@ -8894,6 +8978,7 @@ static int request_msix_irqs(struct hfi1_devdata *dd)
88948978
rcd->imask = ((u64)1) <<
88958979
((IS_RCVAVAIL_START+idx) % 64);
88968980
handler = receive_context_interrupt;
8981+
thread = receive_context_thread;
88978982
arg = rcd;
88988983
snprintf(me->name, sizeof(me->name),
88998984
DRIVER_NAME"_%d kctxt%d", dd->unit, idx);
@@ -8912,7 +8997,8 @@ static int request_msix_irqs(struct hfi1_devdata *dd)
89128997
/* make sure the name is terminated */
89138998
me->name[sizeof(me->name)-1] = 0;
89148999

8915-
ret = request_irq(me->msix.vector, handler, 0, me->name, arg);
9000+
ret = request_threaded_irq(me->msix.vector, handler, thread, 0,
9001+
me->name, arg);
89169002
if (ret) {
89179003
dd_dev_err(dd,
89189004
"unable to allocate %s interrupt, vector %d, index %d, err %d\n",

‎drivers/staging/rdma/hfi1/driver.c

Lines changed: 41 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -427,8 +427,7 @@ static inline void init_packet(struct hfi1_ctxtdata *rcd,
427427
packet->rcd = rcd;
428428
packet->updegr = 0;
429429
packet->etail = -1;
430-
packet->rhf_addr = (__le32 *) rcd->rcvhdrq + rcd->head +
431-
rcd->dd->rhf_offset;
430+
packet->rhf_addr = get_rhf_addr(rcd);
432431
packet->rhf = rhf_to_cpu(packet->rhf_addr);
433432
packet->rhqoff = rcd->head;
434433
packet->numpkt = 0;
@@ -619,10 +618,7 @@ static void prescan_rxq(struct hfi1_packet *packet)
619618
}
620619
#endif /* CONFIG_PRESCAN_RXQ */
621620

622-
#define RCV_PKT_OK 0x0
623-
#define RCV_PKT_MAX 0x1
624-
625-
static inline int process_rcv_packet(struct hfi1_packet *packet)
621+
static inline int process_rcv_packet(struct hfi1_packet *packet, int thread)
626622
{
627623
int ret = RCV_PKT_OK;
628624

@@ -664,9 +660,13 @@ static inline int process_rcv_packet(struct hfi1_packet *packet)
664660
if (packet->rhqoff >= packet->maxcnt)
665661
packet->rhqoff = 0;
666662

667-
if (packet->numpkt == MAX_PKT_RECV) {
668-
ret = RCV_PKT_MAX;
669-
this_cpu_inc(*packet->rcd->dd->rcv_limit);
663+
if (unlikely((packet->numpkt & (MAX_PKT_RECV - 1)) == 0)) {
664+
if (thread) {
665+
cond_resched();
666+
} else {
667+
ret = RCV_PKT_LIMIT;
668+
this_cpu_inc(*packet->rcd->dd->rcv_limit);
669+
}
670670
}
671671

672672
packet->rhf_addr = (__le32 *) packet->rcd->rcvhdrq + packet->rhqoff +
@@ -743,57 +743,63 @@ static inline void process_rcv_qp_work(struct hfi1_packet *packet)
743743
/*
744744
* Handle receive interrupts when using the no dma rtail option.
745745
*/
746-
void handle_receive_interrupt_nodma_rtail(struct hfi1_ctxtdata *rcd)
746+
int handle_receive_interrupt_nodma_rtail(struct hfi1_ctxtdata *rcd, int thread)
747747
{
748748
u32 seq;
749-
int last = 0;
749+
int last = RCV_PKT_OK;
750750
struct hfi1_packet packet;
751751

752752
init_packet(rcd, &packet);
753753
seq = rhf_rcv_seq(packet.rhf);
754-
if (seq != rcd->seq_cnt)
754+
if (seq != rcd->seq_cnt) {
755+
last = RCV_PKT_DONE;
755756
goto bail;
757+
}
756758

757759
prescan_rxq(&packet);
758760

759-
while (!last) {
760-
last = process_rcv_packet(&packet);
761+
while (last == RCV_PKT_OK) {
762+
last = process_rcv_packet(&packet, thread);
761763
seq = rhf_rcv_seq(packet.rhf);
762764
if (++rcd->seq_cnt > 13)
763765
rcd->seq_cnt = 1;
764766
if (seq != rcd->seq_cnt)
765-
last = 1;
767+
last = RCV_PKT_DONE;
766768
process_rcv_update(last, &packet);
767769
}
768770
process_rcv_qp_work(&packet);
769771
bail:
770772
finish_packet(&packet);
773+
return last;
771774
}
772775

773-
void handle_receive_interrupt_dma_rtail(struct hfi1_ctxtdata *rcd)
776+
int handle_receive_interrupt_dma_rtail(struct hfi1_ctxtdata *rcd, int thread)
774777
{
775778
u32 hdrqtail;
776-
int last = 0;
779+
int last = RCV_PKT_OK;
777780
struct hfi1_packet packet;
778781

779782
init_packet(rcd, &packet);
780783
hdrqtail = get_rcvhdrtail(rcd);
781-
if (packet.rhqoff == hdrqtail)
784+
if (packet.rhqoff == hdrqtail) {
785+
last = RCV_PKT_DONE;
782786
goto bail;
787+
}
783788
smp_rmb(); /* prevent speculative reads of dma'ed hdrq */
784789

785790
prescan_rxq(&packet);
786791

787-
while (!last) {
788-
last = process_rcv_packet(&packet);
792+
while (last == RCV_PKT_OK) {
793+
last = process_rcv_packet(&packet, thread);
794+
hdrqtail = get_rcvhdrtail(rcd);
789795
if (packet.rhqoff == hdrqtail)
790-
last = 1;
796+
last = RCV_PKT_DONE;
791797
process_rcv_update(last, &packet);
792798
}
793799
process_rcv_qp_work(&packet);
794800
bail:
795801
finish_packet(&packet);
796-
802+
return last;
797803
}
798804

799805
static inline void set_all_nodma_rtail(struct hfi1_devdata *dd)
@@ -821,32 +827,35 @@ static inline void set_all_dma_rtail(struct hfi1_devdata *dd)
821827
* Called from interrupt handler for errors or receive interrupt.
822828
* This is the slow path interrupt handler.
823829
*/
824-
void handle_receive_interrupt(struct hfi1_ctxtdata *rcd)
830+
int handle_receive_interrupt(struct hfi1_ctxtdata *rcd, int thread)
825831
{
826-
827832
struct hfi1_devdata *dd = rcd->dd;
828833
u32 hdrqtail;
829-
int last = 0, needset = 1;
834+
int last = RCV_PKT_OK, needset = 1;
830835
struct hfi1_packet packet;
831836

832837
init_packet(rcd, &packet);
833838

834839
if (!HFI1_CAP_IS_KSET(DMA_RTAIL)) {
835840
u32 seq = rhf_rcv_seq(packet.rhf);
836841

837-
if (seq != rcd->seq_cnt)
842+
if (seq != rcd->seq_cnt) {
843+
last = RCV_PKT_DONE;
838844
goto bail;
845+
}
839846
hdrqtail = 0;
840847
} else {
841848
hdrqtail = get_rcvhdrtail(rcd);
842-
if (packet.rhqoff == hdrqtail)
849+
if (packet.rhqoff == hdrqtail) {
850+
last = RCV_PKT_DONE;
843851
goto bail;
852+
}
844853
smp_rmb(); /* prevent speculative reads of dma'ed hdrq */
845854
}
846855

847856
prescan_rxq(&packet);
848857

849-
while (!last) {
858+
while (last == RCV_PKT_OK) {
850859

851860
if (unlikely(dd->do_drop && atomic_xchg(&dd->drop_packet,
852861
DROP_PACKET_OFF) == DROP_PACKET_ON)) {
@@ -860,7 +869,7 @@ void handle_receive_interrupt(struct hfi1_ctxtdata *rcd)
860869
packet.rhf = rhf_to_cpu(packet.rhf_addr);
861870

862871
} else {
863-
last = process_rcv_packet(&packet);
872+
last = process_rcv_packet(&packet, thread);
864873
}
865874

866875
if (!HFI1_CAP_IS_KSET(DMA_RTAIL)) {
@@ -869,7 +878,7 @@ void handle_receive_interrupt(struct hfi1_ctxtdata *rcd)
869878
if (++rcd->seq_cnt > 13)
870879
rcd->seq_cnt = 1;
871880
if (seq != rcd->seq_cnt)
872-
last = 1;
881+
last = RCV_PKT_DONE;
873882
if (needset) {
874883
dd_dev_info(dd,
875884
"Switching to NO_DMA_RTAIL\n");
@@ -878,7 +887,7 @@ void handle_receive_interrupt(struct hfi1_ctxtdata *rcd)
878887
}
879888
} else {
880889
if (packet.rhqoff == hdrqtail)
881-
last = 1;
890+
last = RCV_PKT_DONE;
882891
if (needset) {
883892
dd_dev_info(dd,
884893
"Switching to DMA_RTAIL\n");
@@ -898,6 +907,7 @@ void handle_receive_interrupt(struct hfi1_ctxtdata *rcd)
898907
* if no packets were processed.
899908
*/
900909
finish_packet(&packet);
910+
return last;
901911
}
902912

903913
/*

‎drivers/staging/rdma/hfi1/hfi.h

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -313,7 +313,7 @@ struct hfi1_ctxtdata {
313313
* be valid. Worst case is we process an extra interrupt and up to 64
314314
* packets with the wrong interrupt handler.
315315
*/
316-
void (*do_interrupt)(struct hfi1_ctxtdata *rcd);
316+
int (*do_interrupt)(struct hfi1_ctxtdata *rcd, int threaded);
317317
};
318318

319319
/*
@@ -1130,9 +1130,21 @@ void hfi1_init_pportdata(struct pci_dev *, struct hfi1_pportdata *,
11301130
struct hfi1_devdata *, u8, u8);
11311131
void hfi1_free_ctxtdata(struct hfi1_devdata *, struct hfi1_ctxtdata *);
11321132

1133-
void handle_receive_interrupt(struct hfi1_ctxtdata *);
1134-
void handle_receive_interrupt_nodma_rtail(struct hfi1_ctxtdata *rcd);
1135-
void handle_receive_interrupt_dma_rtail(struct hfi1_ctxtdata *rcd);
1133+
int handle_receive_interrupt(struct hfi1_ctxtdata *, int);
1134+
int handle_receive_interrupt_nodma_rtail(struct hfi1_ctxtdata *, int);
1135+
int handle_receive_interrupt_dma_rtail(struct hfi1_ctxtdata *, int);
1136+
1137+
/* receive packet handler dispositions */
1138+
#define RCV_PKT_OK 0x0 /* keep going */
1139+
#define RCV_PKT_LIMIT 0x1 /* stop, hit limit, start thread */
1140+
#define RCV_PKT_DONE 0x2 /* stop, no more packets detected */
1141+
1142+
/* calculate the current RHF address */
1143+
static inline __le32 *get_rhf_addr(struct hfi1_ctxtdata *rcd)
1144+
{
1145+
return (__le32 *)rcd->rcvhdrq + rcd->head + rcd->dd->rhf_offset;
1146+
}
1147+
11361148
int hfi1_reset_device(int);
11371149

11381150
/* return the driver's idea of the logical OPA port state */

‎drivers/staging/rdma/hfi1/sdma.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2096,9 +2096,9 @@ int sdma_send_txreq(struct sdma_engine *sde,
20962096
tx->sn = sde->tail_sn++;
20972097
trace_hfi1_sdma_in_sn(sde, tx->sn);
20982098
#endif
2099-
spin_lock_irqsave(&sde->flushlist_lock, flags);
2099+
spin_lock(&sde->flushlist_lock);
21002100
list_add_tail(&tx->list, &sde->flushlist);
2101-
spin_unlock_irqrestore(&sde->flushlist_lock, flags);
2101+
spin_unlock(&sde->flushlist_lock);
21022102
if (wait) {
21032103
wait->tx_count++;
21042104
wait->count += tx->num_desc;

0 commit comments

Comments
 (0)