Skip to content

Commit a4cf044

Browse files
Ursula Braundavem330
Ursula Braun
authored and
davem330
committed
smc: introduce SMC as an IB-client
* create a list of SMC IB-devices Signed-off-by: Ursula Braun <ubraun@linux.vnet.ibm.com> Signed-off-by: David S. Miller <davem@davemloft.net>
1 parent ac71387 commit a4cf044

File tree

5 files changed

+200
-1
lines changed

5 files changed

+200
-1
lines changed

‎net/smc/Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
obj-$(CONFIG_SMC) += smc.o
2-
smc-y := af_smc.o
2+
smc-y := af_smc.o smc_ib.o

‎net/smc/af_smc.c

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
#include <net/sock.h>
2121

2222
#include "smc.h"
23+
#include "smc_ib.h"
2324

2425
static void smc_set_keepalive(struct sock *sk, int val)
2526
{
@@ -597,8 +598,16 @@ static int __init smc_init(void)
597598
goto out_proto;
598599
}
599600

601+
rc = smc_ib_register_client();
602+
if (rc) {
603+
pr_err("%s: ib_register fails with %d\n", __func__, rc);
604+
goto out_sock;
605+
}
606+
600607
return 0;
601608

609+
out_sock:
610+
sock_unregister(PF_SMC);
602611
out_proto:
603612
proto_unregister(&smc_proto);
604613
out:
@@ -607,6 +616,7 @@ static int __init smc_init(void)
607616

608617
static void __exit smc_exit(void)
609618
{
619+
smc_ib_unregister_client();
610620
sock_unregister(PF_SMC);
611621
proto_unregister(&smc_proto);
612622
}

‎net/smc/smc.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616

1717
#define SMCPROTO_SMC 0 /* SMC protocol */
1818

19+
#define SMC_MAX_PORTS 2 /* Max # of ports */
20+
1921
enum smc_state { /* possible states of an SMC socket */
2022
SMC_ACTIVE = 1,
2123
SMC_INIT = 2,
@@ -34,4 +36,8 @@ static inline struct smc_sock *smc_sk(const struct sock *sk)
3436
return (struct smc_sock *)sk;
3537
}
3638

39+
#define SMC_SYSTEMID_LEN 8
40+
41+
extern u8 local_systemid[SMC_SYSTEMID_LEN]; /* unique system identifier */
42+
3743
#endif /* __SMC_H */

‎net/smc/smc_ib.c

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
/*
2+
* Shared Memory Communications over RDMA (SMC-R) and RoCE
3+
*
4+
* IB infrastructure:
5+
* Establish SMC-R as an Infiniband Client to be notified about added and
6+
* removed IB devices of type RDMA.
7+
* Determine device and port characteristics for these IB devices.
8+
*
9+
* Copyright IBM Corp. 2016
10+
*
11+
* Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com>
12+
*/
13+
14+
#include <linux/random.h>
15+
#include <rdma/ib_verbs.h>
16+
17+
#include "smc_ib.h"
18+
#include "smc.h"
19+
20+
struct smc_ib_devices smc_ib_devices = { /* smc-registered ib devices */
21+
.lock = __SPIN_LOCK_UNLOCKED(smc_ib_devices.lock),
22+
.list = LIST_HEAD_INIT(smc_ib_devices.list),
23+
};
24+
25+
#define SMC_LOCAL_SYSTEMID_RESET "%%%%%%%"
26+
27+
u8 local_systemid[SMC_SYSTEMID_LEN] = SMC_LOCAL_SYSTEMID_RESET; /* unique system
28+
* identifier
29+
*/
30+
31+
static int smc_ib_fill_gid_and_mac(struct smc_ib_device *smcibdev, u8 ibport)
32+
{
33+
struct net_device *ndev;
34+
int rc;
35+
36+
rc = ib_query_gid(smcibdev->ibdev, ibport, 0,
37+
&smcibdev->gid[ibport - 1], NULL);
38+
/* the SMC protocol requires specification of the roce MAC address;
39+
* if net_device cannot be determined, it can be derived from gid 0
40+
*/
41+
ndev = smcibdev->ibdev->get_netdev(smcibdev->ibdev, ibport);
42+
if (ndev) {
43+
memcpy(&smcibdev->mac, ndev->dev_addr, ETH_ALEN);
44+
} else if (!rc) {
45+
memcpy(&smcibdev->mac[ibport - 1][0],
46+
&smcibdev->gid[ibport - 1].raw[8], 3);
47+
memcpy(&smcibdev->mac[ibport - 1][3],
48+
&smcibdev->gid[ibport - 1].raw[13], 3);
49+
smcibdev->mac[ibport - 1][0] &= ~0x02;
50+
}
51+
return rc;
52+
}
53+
54+
/* Create an identifier unique for this instance of SMC-R.
55+
* The MAC-address of the first active registered IB device
56+
* plus a random 2-byte number is used to create this identifier.
57+
* This name is delivered to the peer during connection initialization.
58+
*/
59+
static inline void smc_ib_define_local_systemid(struct smc_ib_device *smcibdev,
60+
u8 ibport)
61+
{
62+
memcpy(&local_systemid[2], &smcibdev->mac[ibport - 1],
63+
sizeof(smcibdev->mac[ibport - 1]));
64+
get_random_bytes(&local_systemid[0], 2);
65+
}
66+
67+
bool smc_ib_port_active(struct smc_ib_device *smcibdev, u8 ibport)
68+
{
69+
return smcibdev->pattr[ibport - 1].state == IB_PORT_ACTIVE;
70+
}
71+
72+
int smc_ib_remember_port_attr(struct smc_ib_device *smcibdev, u8 ibport)
73+
{
74+
int rc;
75+
76+
memset(&smcibdev->pattr[ibport - 1], 0,
77+
sizeof(smcibdev->pattr[ibport - 1]));
78+
rc = ib_query_port(smcibdev->ibdev, ibport,
79+
&smcibdev->pattr[ibport - 1]);
80+
if (rc)
81+
goto out;
82+
rc = smc_ib_fill_gid_and_mac(smcibdev, ibport);
83+
if (rc)
84+
goto out;
85+
if (!strncmp(local_systemid, SMC_LOCAL_SYSTEMID_RESET,
86+
sizeof(local_systemid)) &&
87+
smc_ib_port_active(smcibdev, ibport))
88+
/* create unique system identifier */
89+
smc_ib_define_local_systemid(smcibdev, ibport);
90+
out:
91+
return rc;
92+
}
93+
94+
static struct ib_client smc_ib_client;
95+
96+
/* callback function for ib_register_client() */
97+
static void smc_ib_add_dev(struct ib_device *ibdev)
98+
{
99+
struct smc_ib_device *smcibdev;
100+
101+
if (ibdev->node_type != RDMA_NODE_IB_CA)
102+
return;
103+
104+
smcibdev = kzalloc(sizeof(*smcibdev), GFP_KERNEL);
105+
if (!smcibdev)
106+
return;
107+
108+
smcibdev->ibdev = ibdev;
109+
110+
spin_lock(&smc_ib_devices.lock);
111+
list_add_tail(&smcibdev->list, &smc_ib_devices.list);
112+
spin_unlock(&smc_ib_devices.lock);
113+
ib_set_client_data(ibdev, &smc_ib_client, smcibdev);
114+
}
115+
116+
/* callback function for ib_register_client() */
117+
static void smc_ib_remove_dev(struct ib_device *ibdev, void *client_data)
118+
{
119+
struct smc_ib_device *smcibdev;
120+
121+
smcibdev = ib_get_client_data(ibdev, &smc_ib_client);
122+
ib_set_client_data(ibdev, &smc_ib_client, NULL);
123+
spin_lock(&smc_ib_devices.lock);
124+
list_del_init(&smcibdev->list); /* remove from smc_ib_devices */
125+
spin_unlock(&smc_ib_devices.lock);
126+
kfree(smcibdev);
127+
}
128+
129+
static struct ib_client smc_ib_client = {
130+
.name = "smc_ib",
131+
.add = smc_ib_add_dev,
132+
.remove = smc_ib_remove_dev,
133+
};
134+
135+
int __init smc_ib_register_client(void)
136+
{
137+
return ib_register_client(&smc_ib_client);
138+
}
139+
140+
void smc_ib_unregister_client(void)
141+
{
142+
ib_unregister_client(&smc_ib_client);
143+
}

‎net/smc/smc_ib.h

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
/*
2+
* Shared Memory Communications over RDMA (SMC-R) and RoCE
3+
*
4+
* Definitions for IB environment
5+
*
6+
* Copyright IBM Corp. 2016
7+
*
8+
* Author(s): Ursula Braun <Ursula Braun@linux.vnet.ibm.com>
9+
*/
10+
11+
#ifndef _SMC_IB_H
12+
#define _SMC_IB_H
13+
14+
#include <rdma/ib_verbs.h>
15+
16+
#define SMC_MAX_PORTS 2 /* Max # of ports */
17+
#define SMC_GID_SIZE sizeof(union ib_gid)
18+
19+
struct smc_ib_devices { /* list of smc ib devices definition */
20+
struct list_head list;
21+
spinlock_t lock; /* protects list of smc ib devices */
22+
};
23+
24+
extern struct smc_ib_devices smc_ib_devices; /* list of smc ib devices */
25+
26+
struct smc_ib_device { /* ib-device infos for smc */
27+
struct list_head list;
28+
struct ib_device *ibdev;
29+
struct ib_port_attr pattr[SMC_MAX_PORTS]; /* ib dev. port attrs */
30+
char mac[SMC_MAX_PORTS][6]; /* mac address per port*/
31+
union ib_gid gid[SMC_MAX_PORTS]; /* gid per port */
32+
u8 initialized : 1; /* ib dev CQ, evthdl done */
33+
};
34+
35+
int smc_ib_register_client(void) __init;
36+
void smc_ib_unregister_client(void);
37+
bool smc_ib_port_active(struct smc_ib_device *smcibdev, u8 ibport);
38+
int smc_ib_remember_port_attr(struct smc_ib_device *smcibdev, u8 ibport);
39+
40+
#endif

0 commit comments

Comments
 (0)