Skip to content

Commit 993c401

Browse files
Jack Morgensteinrolandd
Jack Morgenstein
authored and
rolandd
committed
mlx4_core: Add IB port-state machine and port mgmt event propagation
For an IB port, a slave should not show port active until that slave has a valid alias-guid (provided by the subnet manager). Therefore the port-up event should be passed to a slave only after both the port is up, and the slave's alias-guid has been set. Also, provide the infrastructure for propagating port-management events (client-reregister, etc) to slaves. Signed-off-by: Jack Morgenstein <jackm@dev.mellanox.co.il> Signed-off-by: Roland Dreier <roland@purestorage.com>
1 parent 3cf69cc commit 993c401

File tree

3 files changed

+250
-16
lines changed

3 files changed

+250
-16
lines changed

‎drivers/net/ethernet/mellanox/mlx4/eq.c

Lines changed: 221 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,196 @@ static void mlx4_slave_event(struct mlx4_dev *dev, int slave,
200200
slave_event(dev, slave, eqe);
201201
}
202202

203+
int mlx4_gen_pkey_eqe(struct mlx4_dev *dev, int slave, u8 port)
204+
{
205+
struct mlx4_eqe eqe;
206+
207+
struct mlx4_priv *priv = mlx4_priv(dev);
208+
struct mlx4_slave_state *s_slave = &priv->mfunc.master.slave_state[slave];
209+
210+
if (!s_slave->active)
211+
return 0;
212+
213+
memset(&eqe, 0, sizeof eqe);
214+
215+
eqe.type = MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT;
216+
eqe.subtype = MLX4_DEV_PMC_SUBTYPE_PKEY_TABLE;
217+
eqe.event.port_mgmt_change.port = port;
218+
219+
return mlx4_GEN_EQE(dev, slave, &eqe);
220+
}
221+
EXPORT_SYMBOL(mlx4_gen_pkey_eqe);
222+
223+
int mlx4_gen_guid_change_eqe(struct mlx4_dev *dev, int slave, u8 port)
224+
{
225+
struct mlx4_eqe eqe;
226+
227+
/*don't send if we don't have the that slave */
228+
if (dev->num_vfs < slave)
229+
return 0;
230+
memset(&eqe, 0, sizeof eqe);
231+
232+
eqe.type = MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT;
233+
eqe.subtype = MLX4_DEV_PMC_SUBTYPE_GUID_INFO;
234+
eqe.event.port_mgmt_change.port = port;
235+
236+
return mlx4_GEN_EQE(dev, slave, &eqe);
237+
}
238+
EXPORT_SYMBOL(mlx4_gen_guid_change_eqe);
239+
240+
int mlx4_gen_port_state_change_eqe(struct mlx4_dev *dev, int slave, u8 port,
241+
u8 port_subtype_change)
242+
{
243+
struct mlx4_eqe eqe;
244+
245+
/*don't send if we don't have the that slave */
246+
if (dev->num_vfs < slave)
247+
return 0;
248+
memset(&eqe, 0, sizeof eqe);
249+
250+
eqe.type = MLX4_EVENT_TYPE_PORT_CHANGE;
251+
eqe.subtype = port_subtype_change;
252+
eqe.event.port_change.port = cpu_to_be32(port << 28);
253+
254+
mlx4_dbg(dev, "%s: sending: %d to slave: %d on port: %d\n", __func__,
255+
port_subtype_change, slave, port);
256+
return mlx4_GEN_EQE(dev, slave, &eqe);
257+
}
258+
EXPORT_SYMBOL(mlx4_gen_port_state_change_eqe);
259+
260+
enum slave_port_state mlx4_get_slave_port_state(struct mlx4_dev *dev, int slave, u8 port)
261+
{
262+
struct mlx4_priv *priv = mlx4_priv(dev);
263+
struct mlx4_slave_state *s_state = priv->mfunc.master.slave_state;
264+
if (slave >= dev->num_slaves || port > MLX4_MAX_PORTS) {
265+
pr_err("%s: Error: asking for slave:%d, port:%d\n",
266+
__func__, slave, port);
267+
return SLAVE_PORT_DOWN;
268+
}
269+
return s_state[slave].port_state[port];
270+
}
271+
EXPORT_SYMBOL(mlx4_get_slave_port_state);
272+
273+
static int mlx4_set_slave_port_state(struct mlx4_dev *dev, int slave, u8 port,
274+
enum slave_port_state state)
275+
{
276+
struct mlx4_priv *priv = mlx4_priv(dev);
277+
struct mlx4_slave_state *s_state = priv->mfunc.master.slave_state;
278+
279+
if (slave >= dev->num_slaves || port > MLX4_MAX_PORTS || port == 0) {
280+
pr_err("%s: Error: asking for slave:%d, port:%d\n",
281+
__func__, slave, port);
282+
return -1;
283+
}
284+
s_state[slave].port_state[port] = state;
285+
286+
return 0;
287+
}
288+
289+
static void set_all_slave_state(struct mlx4_dev *dev, u8 port, int event)
290+
{
291+
int i;
292+
enum slave_port_gen_event gen_event;
293+
294+
for (i = 0; i < dev->num_slaves; i++)
295+
set_and_calc_slave_port_state(dev, i, port, event, &gen_event);
296+
}
297+
/**************************************************************************
298+
The function get as input the new event to that port,
299+
and according to the prev state change the slave's port state.
300+
The events are:
301+
MLX4_PORT_STATE_DEV_EVENT_PORT_DOWN,
302+
MLX4_PORT_STATE_DEV_EVENT_PORT_UP
303+
MLX4_PORT_STATE_IB_EVENT_GID_VALID
304+
MLX4_PORT_STATE_IB_EVENT_GID_INVALID
305+
***************************************************************************/
306+
int set_and_calc_slave_port_state(struct mlx4_dev *dev, int slave,
307+
u8 port, int event,
308+
enum slave_port_gen_event *gen_event)
309+
{
310+
struct mlx4_priv *priv = mlx4_priv(dev);
311+
struct mlx4_slave_state *ctx = NULL;
312+
unsigned long flags;
313+
int ret = -1;
314+
enum slave_port_state cur_state =
315+
mlx4_get_slave_port_state(dev, slave, port);
316+
317+
*gen_event = SLAVE_PORT_GEN_EVENT_NONE;
318+
319+
if (slave >= dev->num_slaves || port > MLX4_MAX_PORTS || port == 0) {
320+
pr_err("%s: Error: asking for slave:%d, port:%d\n",
321+
__func__, slave, port);
322+
return ret;
323+
}
324+
325+
ctx = &priv->mfunc.master.slave_state[slave];
326+
spin_lock_irqsave(&ctx->lock, flags);
327+
328+
mlx4_dbg(dev, "%s: slave: %d, current state: %d new event :%d\n",
329+
__func__, slave, cur_state, event);
330+
331+
switch (cur_state) {
332+
case SLAVE_PORT_DOWN:
333+
if (MLX4_PORT_STATE_DEV_EVENT_PORT_UP == event)
334+
mlx4_set_slave_port_state(dev, slave, port,
335+
SLAVE_PENDING_UP);
336+
break;
337+
case SLAVE_PENDING_UP:
338+
if (MLX4_PORT_STATE_DEV_EVENT_PORT_DOWN == event)
339+
mlx4_set_slave_port_state(dev, slave, port,
340+
SLAVE_PORT_DOWN);
341+
else if (MLX4_PORT_STATE_IB_PORT_STATE_EVENT_GID_VALID == event) {
342+
mlx4_set_slave_port_state(dev, slave, port,
343+
SLAVE_PORT_UP);
344+
*gen_event = SLAVE_PORT_GEN_EVENT_UP;
345+
}
346+
break;
347+
case SLAVE_PORT_UP:
348+
if (MLX4_PORT_STATE_DEV_EVENT_PORT_DOWN == event) {
349+
mlx4_set_slave_port_state(dev, slave, port,
350+
SLAVE_PORT_DOWN);
351+
*gen_event = SLAVE_PORT_GEN_EVENT_DOWN;
352+
} else if (MLX4_PORT_STATE_IB_EVENT_GID_INVALID ==
353+
event) {
354+
mlx4_set_slave_port_state(dev, slave, port,
355+
SLAVE_PENDING_UP);
356+
*gen_event = SLAVE_PORT_GEN_EVENT_DOWN;
357+
}
358+
break;
359+
default:
360+
pr_err("%s: BUG!!! UNKNOWN state: "
361+
"slave:%d, port:%d\n", __func__, slave, port);
362+
goto out;
363+
}
364+
ret = mlx4_get_slave_port_state(dev, slave, port);
365+
mlx4_dbg(dev, "%s: slave: %d, current state: %d new event"
366+
" :%d gen_event: %d\n",
367+
__func__, slave, cur_state, event, *gen_event);
368+
369+
out:
370+
spin_unlock_irqrestore(&ctx->lock, flags);
371+
return ret;
372+
}
373+
374+
EXPORT_SYMBOL(set_and_calc_slave_port_state);
375+
376+
int mlx4_gen_slaves_port_mgt_ev(struct mlx4_dev *dev, u8 port, int attr)
377+
{
378+
struct mlx4_eqe eqe;
379+
380+
memset(&eqe, 0, sizeof eqe);
381+
382+
eqe.type = MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT;
383+
eqe.subtype = MLX4_DEV_PMC_SUBTYPE_PORT_INFO;
384+
eqe.event.port_mgmt_change.port = port;
385+
eqe.event.port_mgmt_change.params.port_info.changed_attr =
386+
cpu_to_be32((u32) attr);
387+
388+
slave_event(dev, ALL_SLAVES, &eqe);
389+
return 0;
390+
}
391+
EXPORT_SYMBOL(mlx4_gen_slaves_port_mgt_ev);
392+
203393
void mlx4_master_handle_slave_flr(struct work_struct *work)
204394
{
205395
struct mlx4_mfunc_master_ctx *master =
@@ -251,6 +441,7 @@ static int mlx4_eq_int(struct mlx4_dev *dev, struct mlx4_eq *eq)
251441
u32 flr_slave;
252442
u8 update_slave_state;
253443
int i;
444+
enum slave_port_gen_event gen_event;
254445

255446
while ((eqe = next_eqe_sw(eq))) {
256447
/*
@@ -347,35 +538,49 @@ static int mlx4_eq_int(struct mlx4_dev *dev, struct mlx4_eq *eq)
347538
case MLX4_EVENT_TYPE_PORT_CHANGE:
348539
port = be32_to_cpu(eqe->event.port_change.port) >> 28;
349540
if (eqe->subtype == MLX4_PORT_CHANGE_SUBTYPE_DOWN) {
350-
mlx4_dispatch_event(dev,
351-
MLX4_DEV_EVENT_PORT_DOWN,
541+
mlx4_dispatch_event(dev, MLX4_DEV_EVENT_PORT_DOWN,
352542
port);
353543
mlx4_priv(dev)->sense.do_sense_port[port] = 1;
354-
if (mlx4_is_master(dev))
355-
/*change the state of all slave's port
356-
* to down:*/
357-
for (i = 0; i < dev->num_slaves; i++) {
358-
mlx4_dbg(dev, "%s: Sending "
359-
"MLX4_PORT_CHANGE_SUBTYPE_DOWN"
544+
if (!mlx4_is_master(dev))
545+
break;
546+
for (i = 0; i < dev->num_slaves; i++) {
547+
if (dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH) {
548+
if (i == mlx4_master_func_num(dev))
549+
continue;
550+
mlx4_dbg(dev, "%s: Sending MLX4_PORT_CHANGE_SUBTYPE_DOWN"
360551
" to slave: %d, port:%d\n",
361552
__func__, i, port);
362-
if (i == dev->caps.function)
363-
continue;
364553
mlx4_slave_event(dev, i, eqe);
554+
} else { /* IB port */
555+
set_and_calc_slave_port_state(dev, i, port,
556+
MLX4_PORT_STATE_DEV_EVENT_PORT_DOWN,
557+
&gen_event);
558+
/*we can be in pending state, then do not send port_down event*/
559+
if (SLAVE_PORT_GEN_EVENT_DOWN == gen_event) {
560+
if (i == mlx4_master_func_num(dev))
561+
continue;
562+
mlx4_slave_event(dev, i, eqe);
563+
}
365564
}
565+
}
366566
} else {
367-
mlx4_dispatch_event(dev,
368-
MLX4_DEV_EVENT_PORT_UP,
369-
port);
567+
mlx4_dispatch_event(dev, MLX4_DEV_EVENT_PORT_UP, port);
568+
370569
mlx4_priv(dev)->sense.do_sense_port[port] = 0;
371570

372-
if (mlx4_is_master(dev)) {
571+
if (!mlx4_is_master(dev))
572+
break;
573+
if (dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH)
373574
for (i = 0; i < dev->num_slaves; i++) {
374-
if (i == dev->caps.function)
575+
if (i == mlx4_master_func_num(dev))
375576
continue;
376577
mlx4_slave_event(dev, i, eqe);
377578
}
378-
}
579+
else /* IB port */
580+
/* port-up event will be sent to a slave when the
581+
* slave's alias-guid is set. This is done in alias_GUID.c
582+
*/
583+
set_all_slave_state(dev, port, MLX4_DEV_EVENT_PORT_UP);
379584
}
380585
break;
381586

‎drivers/net/ethernet/mellanox/mlx4/mlx4.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -452,6 +452,7 @@ struct mlx4_slave_state {
452452
/*initialized via the kzalloc*/
453453
u8 is_slave_going_down;
454454
u32 cookie;
455+
enum slave_port_state port_state[MLX4_MAX_PORTS + 1];
455456
};
456457

457458
struct slave_list {

‎include/linux/mlx4/device.h

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -192,6 +192,25 @@ enum {
192192
MLX4_FATAL_WARNING_SUBTYPE_WARMING = 0,
193193
};
194194

195+
enum slave_port_state {
196+
SLAVE_PORT_DOWN = 0,
197+
SLAVE_PENDING_UP,
198+
SLAVE_PORT_UP,
199+
};
200+
201+
enum slave_port_gen_event {
202+
SLAVE_PORT_GEN_EVENT_DOWN = 0,
203+
SLAVE_PORT_GEN_EVENT_UP,
204+
SLAVE_PORT_GEN_EVENT_NONE,
205+
};
206+
207+
enum slave_port_state_event {
208+
MLX4_PORT_STATE_DEV_EVENT_PORT_DOWN,
209+
MLX4_PORT_STATE_DEV_EVENT_PORT_UP,
210+
MLX4_PORT_STATE_IB_PORT_STATE_EVENT_GID_VALID,
211+
MLX4_PORT_STATE_IB_EVENT_GID_INVALID,
212+
};
213+
195214
enum {
196215
MLX4_PERM_LOCAL_READ = 1 << 10,
197216
MLX4_PERM_LOCAL_WRITE = 1 << 11,
@@ -945,4 +964,13 @@ void mlx4_sync_pkey_table(struct mlx4_dev *dev, int slave, int port,
945964

946965
int mlx4_get_parav_qkey(struct mlx4_dev *dev, u32 qpn, u32 *qkey);
947966

967+
int mlx4_is_slave_active(struct mlx4_dev *dev, int slave);
968+
int mlx4_gen_pkey_eqe(struct mlx4_dev *dev, int slave, u8 port);
969+
int mlx4_gen_guid_change_eqe(struct mlx4_dev *dev, int slave, u8 port);
970+
int mlx4_gen_slaves_port_mgt_ev(struct mlx4_dev *dev, u8 port, int attr);
971+
int mlx4_gen_port_state_change_eqe(struct mlx4_dev *dev, int slave, u8 port, u8 port_subtype_change);
972+
enum slave_port_state mlx4_get_slave_port_state(struct mlx4_dev *dev, int slave, u8 port);
973+
int set_and_calc_slave_port_state(struct mlx4_dev *dev, int slave, u8 port, int event, enum slave_port_gen_event *gen_event);
974+
975+
948976
#endif /* MLX4_DEVICE_H */

0 commit comments

Comments
 (0)