Skip to content

Commit dbf727d

Browse files
matanb10dledford
matanb10
authored and
dledford
committed
IB/core: Use GID table in AH creation and dmac resolution
Previously, vlan id and source MAC were used from QP attributes. Since the net device is now stored in the GID attributes, they could be used instead of getting this information from the QP attributes. IB_QP_SMAC, IB_QP_ALT_SMAC, IB_QP_VID and IB_QP_ALT_VID were removed because there is no known libibverbs that uses them. This commit also modifies the vendors (mlx4, ocrdma) drivers in order to use the new approach. ocrdma driver changes were done by Somnath Kotur <Somnath.Kotur@Avagotech.Com> Signed-off-by: Matan Barak <matanb@mellanox.com> Signed-off-by: Doug Ledford <dledford@redhat.com>
1 parent 99b27e3 commit dbf727d

File tree

12 files changed

+193
-110
lines changed

12 files changed

+193
-110
lines changed

‎drivers/infiniband/core/addr.c

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -458,7 +458,7 @@ static void resolve_cb(int status, struct sockaddr *src_addr,
458458
}
459459

460460
int rdma_addr_find_dmac_by_grh(const union ib_gid *sgid, const union ib_gid *dgid,
461-
u8 *dmac, u16 *vlan_id)
461+
u8 *dmac, u16 *vlan_id, int if_index)
462462
{
463463
int ret = 0;
464464
struct rdma_dev_addr dev_addr;
@@ -476,6 +476,7 @@ int rdma_addr_find_dmac_by_grh(const union ib_gid *sgid, const union ib_gid *dgi
476476
rdma_gid2ip(&dgid_addr._sockaddr, dgid);
477477

478478
memset(&dev_addr, 0, sizeof(dev_addr));
479+
dev_addr.bound_dev_if = if_index;
479480

480481
ctx.addr = &dev_addr;
481482
init_completion(&ctx.comp);

‎drivers/infiniband/core/core_priv.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,8 @@ void ib_device_unregister_sysfs(struct ib_device *device);
4646
void ib_cache_setup(void);
4747
void ib_cache_cleanup(void);
4848

49-
int ib_resolve_eth_l2_attrs(struct ib_qp *qp,
50-
struct ib_qp_attr *qp_attr, int *qp_attr_mask);
49+
int ib_resolve_eth_dmac(struct ib_qp *qp,
50+
struct ib_qp_attr *qp_attr, int *qp_attr_mask);
5151

5252
typedef void (*roce_netdev_callback)(struct ib_device *device, u8 port,
5353
struct net_device *idev, void *cookie);

‎drivers/infiniband/core/uverbs_cmd.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2345,7 +2345,7 @@ ssize_t ib_uverbs_modify_qp(struct ib_uverbs_file *file,
23452345
attr->alt_ah_attr.port_num = cmd.alt_dest.port_num;
23462346

23472347
if (qp->real_qp == qp) {
2348-
ret = ib_resolve_eth_l2_attrs(qp, attr, &cmd.attr_mask);
2348+
ret = ib_resolve_eth_dmac(qp, attr, &cmd.attr_mask);
23492349
if (ret)
23502350
goto release_qp;
23512351
ret = qp->device->modify_qp(qp, attr,

‎drivers/infiniband/core/verbs.c

Lines changed: 93 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,9 @@
4141
#include <linux/export.h>
4242
#include <linux/string.h>
4343
#include <linux/slab.h>
44+
#include <linux/in.h>
45+
#include <linux/in6.h>
46+
#include <net/addrconf.h>
4447

4548
#include <rdma/ib_verbs.h>
4649
#include <rdma/ib_cache.h>
@@ -308,6 +311,35 @@ struct ib_ah *ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr)
308311
}
309312
EXPORT_SYMBOL(ib_create_ah);
310313

314+
struct find_gid_index_context {
315+
u16 vlan_id;
316+
};
317+
318+
static bool find_gid_index(const union ib_gid *gid,
319+
const struct ib_gid_attr *gid_attr,
320+
void *context)
321+
{
322+
struct find_gid_index_context *ctx =
323+
(struct find_gid_index_context *)context;
324+
325+
if ((!!(ctx->vlan_id != 0xffff) == !is_vlan_dev(gid_attr->ndev)) ||
326+
(is_vlan_dev(gid_attr->ndev) &&
327+
vlan_dev_vlan_id(gid_attr->ndev) != ctx->vlan_id))
328+
return false;
329+
330+
return true;
331+
}
332+
333+
static int get_sgid_index_from_eth(struct ib_device *device, u8 port_num,
334+
u16 vlan_id, const union ib_gid *sgid,
335+
u16 *gid_index)
336+
{
337+
struct find_gid_index_context context = {.vlan_id = vlan_id};
338+
339+
return ib_find_gid_by_filter(device, sgid, port_num, find_gid_index,
340+
&context, gid_index);
341+
}
342+
311343
int ib_init_ah_from_wc(struct ib_device *device, u8 port_num,
312344
const struct ib_wc *wc, const struct ib_grh *grh,
313345
struct ib_ah_attr *ah_attr)
@@ -318,21 +350,30 @@ int ib_init_ah_from_wc(struct ib_device *device, u8 port_num,
318350

319351
memset(ah_attr, 0, sizeof *ah_attr);
320352
if (rdma_cap_eth_ah(device, port_num)) {
353+
u16 vlan_id = wc->wc_flags & IB_WC_WITH_VLAN ?
354+
wc->vlan_id : 0xffff;
355+
321356
if (!(wc->wc_flags & IB_WC_GRH))
322357
return -EPROTOTYPE;
323358

324-
if (wc->wc_flags & IB_WC_WITH_SMAC &&
325-
wc->wc_flags & IB_WC_WITH_VLAN) {
326-
memcpy(ah_attr->dmac, wc->smac, ETH_ALEN);
327-
ah_attr->vlan_id = wc->vlan_id;
328-
} else {
359+
if (!(wc->wc_flags & IB_WC_WITH_SMAC) ||
360+
!(wc->wc_flags & IB_WC_WITH_VLAN)) {
329361
ret = rdma_addr_find_dmac_by_grh(&grh->dgid, &grh->sgid,
330-
ah_attr->dmac, &ah_attr->vlan_id);
362+
ah_attr->dmac,
363+
wc->wc_flags & IB_WC_WITH_VLAN ?
364+
NULL : &vlan_id,
365+
0);
331366
if (ret)
332367
return ret;
333368
}
334-
} else {
335-
ah_attr->vlan_id = 0xffff;
369+
370+
ret = get_sgid_index_from_eth(device, port_num, vlan_id,
371+
&grh->dgid, &gid_index);
372+
if (ret)
373+
return ret;
374+
375+
if (wc->wc_flags & IB_WC_WITH_SMAC)
376+
memcpy(ah_attr->dmac, wc->smac, ETH_ALEN);
336377
}
337378

338379
ah_attr->dlid = wc->slid;
@@ -344,10 +385,13 @@ int ib_init_ah_from_wc(struct ib_device *device, u8 port_num,
344385
ah_attr->ah_flags = IB_AH_GRH;
345386
ah_attr->grh.dgid = grh->sgid;
346387

347-
ret = ib_find_cached_gid(device, &grh->dgid,
348-
NULL, &port_num, &gid_index);
349-
if (ret)
350-
return ret;
388+
if (!rdma_cap_eth_ah(device, port_num)) {
389+
ret = ib_find_cached_gid_by_port(device, &grh->dgid,
390+
port_num, NULL,
391+
&gid_index);
392+
if (ret)
393+
return ret;
394+
}
351395

352396
ah_attr->grh.sgid_index = (u8) gid_index;
353397
flow_class = be32_to_cpu(grh->version_tclass_flow);
@@ -617,9 +661,7 @@ EXPORT_SYMBOL(ib_create_qp);
617661
static const struct {
618662
int valid;
619663
enum ib_qp_attr_mask req_param[IB_QPT_MAX];
620-
enum ib_qp_attr_mask req_param_add_eth[IB_QPT_MAX];
621664
enum ib_qp_attr_mask opt_param[IB_QPT_MAX];
622-
enum ib_qp_attr_mask opt_param_add_eth[IB_QPT_MAX];
623665
} qp_state_table[IB_QPS_ERR + 1][IB_QPS_ERR + 1] = {
624666
[IB_QPS_RESET] = {
625667
[IB_QPS_RESET] = { .valid = 1 },
@@ -700,12 +742,6 @@ static const struct {
700742
IB_QP_MAX_DEST_RD_ATOMIC |
701743
IB_QP_MIN_RNR_TIMER),
702744
},
703-
.req_param_add_eth = {
704-
[IB_QPT_RC] = (IB_QP_SMAC),
705-
[IB_QPT_UC] = (IB_QP_SMAC),
706-
[IB_QPT_XRC_INI] = (IB_QP_SMAC),
707-
[IB_QPT_XRC_TGT] = (IB_QP_SMAC)
708-
},
709745
.opt_param = {
710746
[IB_QPT_UD] = (IB_QP_PKEY_INDEX |
711747
IB_QP_QKEY),
@@ -726,21 +762,7 @@ static const struct {
726762
[IB_QPT_GSI] = (IB_QP_PKEY_INDEX |
727763
IB_QP_QKEY),
728764
},
729-
.opt_param_add_eth = {
730-
[IB_QPT_RC] = (IB_QP_ALT_SMAC |
731-
IB_QP_VID |
732-
IB_QP_ALT_VID),
733-
[IB_QPT_UC] = (IB_QP_ALT_SMAC |
734-
IB_QP_VID |
735-
IB_QP_ALT_VID),
736-
[IB_QPT_XRC_INI] = (IB_QP_ALT_SMAC |
737-
IB_QP_VID |
738-
IB_QP_ALT_VID),
739-
[IB_QPT_XRC_TGT] = (IB_QP_ALT_SMAC |
740-
IB_QP_VID |
741-
IB_QP_ALT_VID)
742-
}
743-
}
765+
},
744766
},
745767
[IB_QPS_RTR] = {
746768
[IB_QPS_RESET] = { .valid = 1 },
@@ -962,13 +984,6 @@ int ib_modify_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state next_state,
962984
req_param = qp_state_table[cur_state][next_state].req_param[type];
963985
opt_param = qp_state_table[cur_state][next_state].opt_param[type];
964986

965-
if (ll == IB_LINK_LAYER_ETHERNET) {
966-
req_param |= qp_state_table[cur_state][next_state].
967-
req_param_add_eth[type];
968-
opt_param |= qp_state_table[cur_state][next_state].
969-
opt_param_add_eth[type];
970-
}
971-
972987
if ((mask & req_param) != req_param)
973988
return 0;
974989

@@ -979,41 +994,52 @@ int ib_modify_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state next_state,
979994
}
980995
EXPORT_SYMBOL(ib_modify_qp_is_ok);
981996

982-
int ib_resolve_eth_l2_attrs(struct ib_qp *qp,
983-
struct ib_qp_attr *qp_attr, int *qp_attr_mask)
997+
int ib_resolve_eth_dmac(struct ib_qp *qp,
998+
struct ib_qp_attr *qp_attr, int *qp_attr_mask)
984999
{
9851000
int ret = 0;
986-
union ib_gid sgid;
9871001

988-
if ((*qp_attr_mask & IB_QP_AV) &&
989-
(rdma_cap_eth_ah(qp->device, qp_attr->ah_attr.port_num))) {
990-
ret = ib_query_gid(qp->device, qp_attr->ah_attr.port_num,
991-
qp_attr->ah_attr.grh.sgid_index, &sgid,
992-
NULL);
993-
if (ret)
994-
goto out;
1002+
if (*qp_attr_mask & IB_QP_AV) {
1003+
if (qp_attr->ah_attr.port_num < rdma_start_port(qp->device) ||
1004+
qp_attr->ah_attr.port_num > rdma_end_port(qp->device))
1005+
return -EINVAL;
1006+
1007+
if (!rdma_cap_eth_ah(qp->device, qp_attr->ah_attr.port_num))
1008+
return 0;
1009+
9951010
if (rdma_link_local_addr((struct in6_addr *)qp_attr->ah_attr.grh.dgid.raw)) {
996-
rdma_get_ll_mac((struct in6_addr *)qp_attr->ah_attr.grh.dgid.raw, qp_attr->ah_attr.dmac);
997-
rdma_get_ll_mac((struct in6_addr *)sgid.raw, qp_attr->smac);
998-
if (!(*qp_attr_mask & IB_QP_VID))
999-
qp_attr->vlan_id = rdma_get_vlan_id(&sgid);
1011+
rdma_get_ll_mac((struct in6_addr *)qp_attr->ah_attr.grh.dgid.raw,
1012+
qp_attr->ah_attr.dmac);
10001013
} else {
1001-
ret = rdma_addr_find_dmac_by_grh(&sgid, &qp_attr->ah_attr.grh.dgid,
1002-
qp_attr->ah_attr.dmac, &qp_attr->vlan_id);
1003-
if (ret)
1004-
goto out;
1005-
ret = rdma_addr_find_smac_by_sgid(&sgid, qp_attr->smac, NULL);
1006-
if (ret)
1014+
union ib_gid sgid;
1015+
struct ib_gid_attr sgid_attr;
1016+
int ifindex;
1017+
1018+
ret = ib_query_gid(qp->device,
1019+
qp_attr->ah_attr.port_num,
1020+
qp_attr->ah_attr.grh.sgid_index,
1021+
&sgid, &sgid_attr);
1022+
1023+
if (ret || !sgid_attr.ndev) {
1024+
if (!ret)
1025+
ret = -ENXIO;
10071026
goto out;
1027+
}
1028+
1029+
ifindex = sgid_attr.ndev->ifindex;
1030+
1031+
ret = rdma_addr_find_dmac_by_grh(&sgid,
1032+
&qp_attr->ah_attr.grh.dgid,
1033+
qp_attr->ah_attr.dmac,
1034+
NULL, ifindex);
1035+
1036+
dev_put(sgid_attr.ndev);
10081037
}
1009-
*qp_attr_mask |= IB_QP_SMAC;
1010-
if (qp_attr->vlan_id < 0xFFFF)
1011-
*qp_attr_mask |= IB_QP_VID;
10121038
}
10131039
out:
10141040
return ret;
10151041
}
1016-
EXPORT_SYMBOL(ib_resolve_eth_l2_attrs);
1042+
EXPORT_SYMBOL(ib_resolve_eth_dmac);
10171043

10181044

10191045
int ib_modify_qp(struct ib_qp *qp,
@@ -1022,7 +1048,7 @@ int ib_modify_qp(struct ib_qp *qp,
10221048
{
10231049
int ret;
10241050

1025-
ret = ib_resolve_eth_l2_attrs(qp, qp_attr, &qp_attr_mask);
1051+
ret = ib_resolve_eth_dmac(qp, qp_attr, &qp_attr_mask);
10261052
if (ret)
10271053
return ret;
10281054

‎drivers/infiniband/hw/mlx4/ah.c

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,10 @@ static struct ib_ah *create_iboe_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr
7676
struct mlx4_dev *dev = ibdev->dev;
7777
int is_mcast = 0;
7878
struct in6_addr in6;
79-
u16 vlan_tag;
79+
u16 vlan_tag = 0xffff;
80+
union ib_gid sgid;
81+
struct ib_gid_attr gid_attr;
82+
int ret;
8083

8184
memcpy(&in6, ah_attr->grh.dgid.raw, sizeof(in6));
8285
if (rdma_is_multicast_addr(&in6)) {
@@ -85,7 +88,17 @@ static struct ib_ah *create_iboe_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr
8588
} else {
8689
memcpy(ah->av.eth.mac, ah_attr->dmac, ETH_ALEN);
8790
}
88-
vlan_tag = ah_attr->vlan_id;
91+
ret = ib_get_cached_gid(pd->device, ah_attr->port_num,
92+
ah_attr->grh.sgid_index, &sgid, &gid_attr);
93+
if (ret)
94+
return ERR_PTR(ret);
95+
memset(ah->av.eth.s_mac, 0, ETH_ALEN);
96+
if (gid_attr.ndev) {
97+
if (is_vlan_dev(gid_attr.ndev))
98+
vlan_tag = vlan_dev_vlan_id(gid_attr.ndev);
99+
memcpy(ah->av.eth.s_mac, gid_attr.ndev->dev_addr, ETH_ALEN);
100+
dev_put(gid_attr.ndev);
101+
}
89102
if (vlan_tag < 0x1000)
90103
vlan_tag |= (ah_attr->sl & 7) << 13;
91104
ah->av.eth.port_pd = cpu_to_be32(to_mpd(pd)->pdn | (ah_attr->port_num << 24));

‎drivers/infiniband/hw/mlx4/mad.c

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1183,7 +1183,7 @@ static int is_proxy_qp0(struct mlx4_ib_dev *dev, int qpn, int slave)
11831183
int mlx4_ib_send_to_wire(struct mlx4_ib_dev *dev, int slave, u8 port,
11841184
enum ib_qp_type dest_qpt, u16 pkey_index,
11851185
u32 remote_qpn, u32 qkey, struct ib_ah_attr *attr,
1186-
u8 *s_mac, struct ib_mad *mad)
1186+
u8 *s_mac, u16 vlan_id, struct ib_mad *mad)
11871187
{
11881188
struct ib_sge list;
11891189
struct ib_send_wr wr, *bad_wr;
@@ -1270,6 +1270,9 @@ int mlx4_ib_send_to_wire(struct mlx4_ib_dev *dev, int slave, u8 port,
12701270
wr.send_flags = IB_SEND_SIGNALED;
12711271
if (s_mac)
12721272
memcpy(to_mah(ah)->av.eth.s_mac, s_mac, 6);
1273+
if (vlan_id < 0x1000)
1274+
vlan_id |= (attr->sl & 7) << 13;
1275+
to_mah(ah)->av.eth.vlan = cpu_to_be16(vlan_id);
12731276

12741277

12751278
ret = ib_post_send(send_qp, &wr, &bad_wr);
@@ -1306,6 +1309,7 @@ static void mlx4_ib_multiplex_mad(struct mlx4_ib_demux_pv_ctx *ctx, struct ib_wc
13061309
u8 *slave_id;
13071310
int slave;
13081311
int port;
1312+
u16 vlan_id;
13091313

13101314
/* Get slave that sent this packet */
13111315
if (wc->src_qp < dev->dev->phys_caps.base_proxy_sqpn ||
@@ -1394,18 +1398,18 @@ static void mlx4_ib_multiplex_mad(struct mlx4_ib_demux_pv_ctx *ctx, struct ib_wc
13941398
fill_in_real_sgid_index(dev, slave, ctx->port, &ah_attr);
13951399

13961400
memcpy(ah_attr.dmac, tunnel->hdr.mac, 6);
1397-
ah_attr.vlan_id = be16_to_cpu(tunnel->hdr.vlan);
1401+
vlan_id = be16_to_cpu(tunnel->hdr.vlan);
13981402
/* if slave have default vlan use it */
13991403
mlx4_get_slave_default_vlan(dev->dev, ctx->port, slave,
1400-
&ah_attr.vlan_id, &ah_attr.sl);
1404+
&vlan_id, &ah_attr.sl);
14011405

14021406
mlx4_ib_send_to_wire(dev, slave, ctx->port,
14031407
is_proxy_qp0(dev, wc->src_qp, slave) ?
14041408
IB_QPT_SMI : IB_QPT_GSI,
14051409
be16_to_cpu(tunnel->hdr.pkey_index),
14061410
be32_to_cpu(tunnel->hdr.remote_qpn),
14071411
be32_to_cpu(tunnel->hdr.qkey),
1408-
&ah_attr, wc->smac, &tunnel->mad);
1412+
&ah_attr, wc->smac, vlan_id, &tunnel->mad);
14091413
}
14101414

14111415
static int mlx4_ib_alloc_pv_bufs(struct mlx4_ib_demux_pv_ctx *ctx,

‎drivers/infiniband/hw/mlx4/mcg.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -222,7 +222,7 @@ static int send_mad_to_wire(struct mlx4_ib_demux_ctx *ctx, struct ib_mad *mad)
222222
spin_unlock_irqrestore(&dev->sm_lock, flags);
223223
return mlx4_ib_send_to_wire(dev, mlx4_master_func_num(dev->dev),
224224
ctx->port, IB_QPT_GSI, 0, 1, IB_QP1_QKEY,
225-
&ah_attr, NULL, mad);
225+
&ah_attr, NULL, 0xffff, mad);
226226
}
227227

228228
static int send_mad_to_slave(int slave, struct mlx4_ib_demux_ctx *ctx,

‎drivers/infiniband/hw/mlx4/mlx4_ib.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -821,7 +821,7 @@ int mlx4_ib_send_to_slave(struct mlx4_ib_dev *dev, int slave, u8 port,
821821
int mlx4_ib_send_to_wire(struct mlx4_ib_dev *dev, int slave, u8 port,
822822
enum ib_qp_type dest_qpt, u16 pkey_index, u32 remote_qpn,
823823
u32 qkey, struct ib_ah_attr *attr, u8 *s_mac,
824-
struct ib_mad *mad);
824+
u16 vlan_id, struct ib_mad *mad);
825825

826826
__be64 mlx4_ib_get_new_demux_tid(struct mlx4_ib_demux_ctx *ctx);
827827

0 commit comments

Comments
 (0)