Skip to content

Commit 0534c8c

Browse files
torvaldstorvalds
committed
Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/roland/infiniband
* 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/roland/infiniband: RDMA/nes: Add support for new SFP+ PHY RDMA/nes: Add wide_ppm_offset parm for switch compatibility RDMA/nes: Fix SFP+ PHY initialization RDMA/nes: Fix nes_nic_cm_xmit() error handling RDMA/nes: Fix error handling issues RDMA/nes: Fix incorrect casts on 32-bit architectures IPoIB: Document newish features RDMA/cma: Create cm id even when IB port is down RDMA/cma: Use rate from IPoIB broadcast when joining IPoIB multicast groups IPoIB: Avoid free_netdev() BUG when destroying a child interface mlx4_core: Don't leak mailbox for SET_PORT on Ethernet ports RDMA/cxgb3: Release dependent resources only when endpoint memory is freed. RDMA/cxgb3: Handle EEH events IB/mlx4: Use pgprot_writecombine() for BlueFlame pages
2 parents 54f93b7 + 07306c0 commit 0534c8c

File tree

18 files changed

+442
-306
lines changed

18 files changed

+442
-306
lines changed

‎Documentation/infiniband/ipoib.txt

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,49 @@ Partitions and P_Keys
2424
The P_Key for any interface is given by the "pkey" file, and the
2525
main interface for a subinterface is in "parent."
2626

27+
Datagram vs Connected modes
28+
29+
The IPoIB driver supports two modes of operation: datagram and
30+
connected. The mode is set and read through an interface's
31+
/sys/class/net/<intf name>/mode file.
32+
33+
In datagram mode, the IB UD (Unreliable Datagram) transport is used
34+
and so the interface MTU has is equal to the IB L2 MTU minus the
35+
IPoIB encapsulation header (4 bytes). For example, in a typical IB
36+
fabric with a 2K MTU, the IPoIB MTU will be 2048 - 4 = 2044 bytes.
37+
38+
In connected mode, the IB RC (Reliable Connected) transport is used.
39+
Connected mode is to takes advantage of the connected nature of the
40+
IB transport and allows an MTU up to the maximal IP packet size of
41+
64K, which reduces the number of IP packets needed for handling
42+
large UDP datagrams, TCP segments, etc and increases the performance
43+
for large messages.
44+
45+
In connected mode, the interface's UD QP is still used for multicast
46+
and communication with peers that don't support connected mode. In
47+
this case, RX emulation of ICMP PMTU packets is used to cause the
48+
networking stack to use the smaller UD MTU for these neighbours.
49+
50+
Stateless offloads
51+
52+
If the IB HW supports IPoIB stateless offloads, IPoIB advertises
53+
TCP/IP checksum and/or Large Send (LSO) offloading capability to the
54+
network stack.
55+
56+
Large Receive (LRO) offloading is also implemented and may be turned
57+
on/off using ethtool calls. Currently LRO is supported only for
58+
checksum offload capable devices.
59+
60+
Stateless offloads are supported only in datagram mode.
61+
62+
Interrupt moderation
63+
64+
If the underlying IB device supports CQ event moderation, one can
65+
use ethtool to set interrupt mitigation parameters and thus reduce
66+
the overhead incurred by handling interrupts. The main code path of
67+
IPoIB doesn't use events for TX completion signaling so only RX
68+
moderation is supported.
69+
2770
Debugging Information
2871

2972
By compiling the IPoIB driver with CONFIG_INFINIBAND_IPOIB_DEBUG set
@@ -55,3 +98,5 @@ References
5598
http://ietf.org/rfc/rfc4391.txt
5699
IP over InfiniBand (IPoIB) Architecture (RFC 4392)
57100
http://ietf.org/rfc/rfc4392.txt
101+
IP over InfiniBand: Connected Mode (RFC 4755)
102+
http://ietf.org/rfc/rfc4755.txt

‎drivers/infiniband/core/cma.c

Lines changed: 31 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -297,21 +297,25 @@ static void cma_detach_from_dev(struct rdma_id_private *id_priv)
297297
id_priv->cma_dev = NULL;
298298
}
299299

300-
static int cma_set_qkey(struct ib_device *device, u8 port_num,
301-
enum rdma_port_space ps,
302-
struct rdma_dev_addr *dev_addr, u32 *qkey)
300+
static int cma_set_qkey(struct rdma_id_private *id_priv)
303301
{
304302
struct ib_sa_mcmember_rec rec;
305303
int ret = 0;
306304

307-
switch (ps) {
305+
if (id_priv->qkey)
306+
return 0;
307+
308+
switch (id_priv->id.ps) {
308309
case RDMA_PS_UDP:
309-
*qkey = RDMA_UDP_QKEY;
310+
id_priv->qkey = RDMA_UDP_QKEY;
310311
break;
311312
case RDMA_PS_IPOIB:
312-
ib_addr_get_mgid(dev_addr, &rec.mgid);
313-
ret = ib_sa_get_mcmember_rec(device, port_num, &rec.mgid, &rec);
314-
*qkey = be32_to_cpu(rec.qkey);
313+
ib_addr_get_mgid(&id_priv->id.route.addr.dev_addr, &rec.mgid);
314+
ret = ib_sa_get_mcmember_rec(id_priv->id.device,
315+
id_priv->id.port_num, &rec.mgid,
316+
&rec);
317+
if (!ret)
318+
id_priv->qkey = be32_to_cpu(rec.qkey);
315319
break;
316320
default:
317321
break;
@@ -341,12 +345,7 @@ static int cma_acquire_dev(struct rdma_id_private *id_priv)
341345
ret = ib_find_cached_gid(cma_dev->device, &gid,
342346
&id_priv->id.port_num, NULL);
343347
if (!ret) {
344-
ret = cma_set_qkey(cma_dev->device,
345-
id_priv->id.port_num,
346-
id_priv->id.ps, dev_addr,
347-
&id_priv->qkey);
348-
if (!ret)
349-
cma_attach_to_dev(id_priv, cma_dev);
348+
cma_attach_to_dev(id_priv, cma_dev);
350349
break;
351350
}
352351
}
@@ -578,6 +577,10 @@ static int cma_ib_init_qp_attr(struct rdma_id_private *id_priv,
578577
*qp_attr_mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT;
579578

580579
if (cma_is_ud_ps(id_priv->id.ps)) {
580+
ret = cma_set_qkey(id_priv);
581+
if (ret)
582+
return ret;
583+
581584
qp_attr->qkey = id_priv->qkey;
582585
*qp_attr_mask |= IB_QP_QKEY;
583586
} else {
@@ -2201,6 +2204,12 @@ static int cma_sidr_rep_handler(struct ib_cm_id *cm_id,
22012204
event.status = ib_event->param.sidr_rep_rcvd.status;
22022205
break;
22032206
}
2207+
ret = cma_set_qkey(id_priv);
2208+
if (ret) {
2209+
event.event = RDMA_CM_EVENT_ADDR_ERROR;
2210+
event.status = -EINVAL;
2211+
break;
2212+
}
22042213
if (id_priv->qkey != rep->qkey) {
22052214
event.event = RDMA_CM_EVENT_UNREACHABLE;
22062215
event.status = -EINVAL;
@@ -2480,10 +2489,14 @@ static int cma_send_sidr_rep(struct rdma_id_private *id_priv,
24802489
const void *private_data, int private_data_len)
24812490
{
24822491
struct ib_cm_sidr_rep_param rep;
2492+
int ret;
24832493

24842494
memset(&rep, 0, sizeof rep);
24852495
rep.status = status;
24862496
if (status == IB_SIDR_SUCCESS) {
2497+
ret = cma_set_qkey(id_priv);
2498+
if (ret)
2499+
return ret;
24872500
rep.qp_num = id_priv->qp_num;
24882501
rep.qkey = id_priv->qkey;
24892502
}
@@ -2713,6 +2726,10 @@ static int cma_join_ib_multicast(struct rdma_id_private *id_priv,
27132726
IB_SA_MCMEMBER_REC_FLOW_LABEL |
27142727
IB_SA_MCMEMBER_REC_TRAFFIC_CLASS;
27152728

2729+
if (id_priv->id.ps == RDMA_PS_IPOIB)
2730+
comp_mask |= IB_SA_MCMEMBER_REC_RATE |
2731+
IB_SA_MCMEMBER_REC_RATE_SELECTOR;
2732+
27162733
mc->multicast.ib = ib_sa_join_multicast(&sa_client, id_priv->id.device,
27172734
id_priv->id.port_num, &rec,
27182735
comp_mask, GFP_KERNEL,

‎drivers/infiniband/hw/cxgb3/cxio_hal.c

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,7 @@ static int cxio_hal_clear_qp_ctx(struct cxio_rdev *rdev_p, u32 qpid)
152152
sge_cmd = qpid << 8 | 3;
153153
wqe->sge_cmd = cpu_to_be64(sge_cmd);
154154
skb->priority = CPL_PRIORITY_CONTROL;
155-
return (cxgb3_ofld_send(rdev_p->t3cdev_p, skb));
155+
return iwch_cxgb3_ofld_send(rdev_p->t3cdev_p, skb);
156156
}
157157

158158
int cxio_create_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq)
@@ -571,7 +571,7 @@ static int cxio_hal_init_ctrl_qp(struct cxio_rdev *rdev_p)
571571
(unsigned long long) rdev_p->ctrl_qp.dma_addr,
572572
rdev_p->ctrl_qp.workq, 1 << T3_CTRL_QP_SIZE_LOG2);
573573
skb->priority = CPL_PRIORITY_CONTROL;
574-
return (cxgb3_ofld_send(rdev_p->t3cdev_p, skb));
574+
return iwch_cxgb3_ofld_send(rdev_p->t3cdev_p, skb);
575575
err:
576576
kfree_skb(skb);
577577
return err;
@@ -701,7 +701,7 @@ static int __cxio_tpt_op(struct cxio_rdev *rdev_p, u32 reset_tpt_entry,
701701
u32 stag_idx;
702702
u32 wptr;
703703

704-
if (rdev_p->flags)
704+
if (cxio_fatal_error(rdev_p))
705705
return -EIO;
706706

707707
stag_state = stag_state > 0;
@@ -858,7 +858,7 @@ int cxio_rdma_init(struct cxio_rdev *rdev_p, struct t3_rdma_init_attr *attr)
858858
wqe->qp_dma_size = cpu_to_be32(attr->qp_dma_size);
859859
wqe->irs = cpu_to_be32(attr->irs);
860860
skb->priority = 0; /* 0=>ToeQ; 1=>CtrlQ */
861-
return (cxgb3_ofld_send(rdev_p->t3cdev_p, skb));
861+
return iwch_cxgb3_ofld_send(rdev_p->t3cdev_p, skb);
862862
}
863863

864864
void cxio_register_ev_cb(cxio_hal_ev_callback_func_t ev_cb)
@@ -1041,9 +1041,9 @@ void cxio_rdev_close(struct cxio_rdev *rdev_p)
10411041
cxio_hal_pblpool_destroy(rdev_p);
10421042
cxio_hal_rqtpool_destroy(rdev_p);
10431043
list_del(&rdev_p->entry);
1044-
rdev_p->t3cdev_p->ulp = NULL;
10451044
cxio_hal_destroy_ctrl_qp(rdev_p);
10461045
cxio_hal_destroy_resource(rdev_p->rscp);
1046+
rdev_p->t3cdev_p->ulp = NULL;
10471047
}
10481048
}
10491049

‎drivers/infiniband/hw/cxgb3/cxio_hal.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,11 @@ struct cxio_rdev {
115115
#define CXIO_ERROR_FATAL 1
116116
};
117117

118+
static inline int cxio_fatal_error(struct cxio_rdev *rdev_p)
119+
{
120+
return rdev_p->flags & CXIO_ERROR_FATAL;
121+
}
122+
118123
static inline int cxio_num_stags(struct cxio_rdev *rdev_p)
119124
{
120125
return min((int)T3_MAX_NUM_STAG, (int)((rdev_p->rnic_info.tpt_top - rdev_p->rnic_info.tpt_base) >> 5));
@@ -188,6 +193,7 @@ void cxio_count_scqes(struct t3_cq *cq, struct t3_wq *wq, int *count);
188193
void cxio_flush_hw_cq(struct t3_cq *cq);
189194
int cxio_poll_cq(struct t3_wq *wq, struct t3_cq *cq, struct t3_cqe *cqe,
190195
u8 *cqe_flushed, u64 *cookie, u32 *credit);
196+
int iwch_cxgb3_ofld_send(struct t3cdev *tdev, struct sk_buff *skb);
191197

192198
#define MOD "iw_cxgb3: "
193199
#define PDBG(fmt, args...) pr_debug(MOD fmt, ## args)

‎drivers/infiniband/hw/cxgb3/iwch.c

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -165,12 +165,19 @@ static void close_rnic_dev(struct t3cdev *tdev)
165165
static void iwch_err_handler(struct t3cdev *tdev, u32 status, u32 error)
166166
{
167167
struct cxio_rdev *rdev = tdev->ulp;
168+
struct iwch_dev *rnicp = rdev_to_iwch_dev(rdev);
169+
struct ib_event event;
168170

169-
if (status == OFFLOAD_STATUS_DOWN)
171+
if (status == OFFLOAD_STATUS_DOWN) {
170172
rdev->flags = CXIO_ERROR_FATAL;
171173

172-
return;
174+
event.device = &rnicp->ibdev;
175+
event.event = IB_EVENT_DEVICE_FATAL;
176+
event.element.port_num = 0;
177+
ib_dispatch_event(&event);
178+
}
173179

180+
return;
174181
}
175182

176183
static int __init iwch_init_module(void)

‎drivers/infiniband/hw/cxgb3/iwch.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,11 @@ static inline struct iwch_dev *to_iwch_dev(struct ib_device *ibdev)
117117
return container_of(ibdev, struct iwch_dev, ibdev);
118118
}
119119

120+
static inline struct iwch_dev *rdev_to_iwch_dev(struct cxio_rdev *rdev)
121+
{
122+
return container_of(rdev, struct iwch_dev, rdev);
123+
}
124+
120125
static inline int t3b_device(const struct iwch_dev *rhp)
121126
{
122127
return rhp->rdev.t3cdev_p->type == T3B;

0 commit comments

Comments
 (0)