Skip to content

Commit eeb66cd

Browse files
author
Saeed Mahameed
committed
net/mlx5: Separate between E-Switch and MPFS
Multi-Physical Function Switch (MPFs) is required for when multi-PF configuration is enabled to allow passing user configured unicast MAC addresses to the requesting PF. Before this patch eswitch.c used to manage the HW MPFS l2 table, E-Switch always (regardless of sriov) enabled vport(0) (NIC PF) vport's contexts update on unicast mac address list changes, to populate the PF's MPFS L2 table accordingly. In downstream patch we would like to allow compiling the driver without E-Switch functionalities, for that we move MPFS l2 table logic out of eswitch.c into its own file, and provide Kconfig flag (MLX5_MPFS) to allow compiling out MPFS for those who don't want Multi-PF support. NIC PF netdevice will now directly update MPFS l2 table via the new MPFS API. VF netdevice has no access to MPFS L2 table, so E-Switch will remain responsible of updating its MPFS l2 table on behalf of its VFs. Due to this change we also don't require enabling vport(0) (PF vport) unicast mac changes events anymore, for when SRIOV is not enabled. Which means E-Switch is now activated only on SRIOV activation, and not required otherwise. Signed-off-by: Saeed Mahameed <saeedm@mellanox.com> Cc: Jes Sorensen <jsorensen@fb.com> Cc: kernel-team@fb.com
1 parent a9f7705 commit eeb66cd

File tree

9 files changed

+377
-221
lines changed

9 files changed

+377
-221
lines changed

‎drivers/net/ethernet/mellanox/mlx5/core/Kconfig

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,16 @@ config MLX5_CORE_EN
3434
---help---
3535
Ethernet support in Mellanox Technologies ConnectX-4 NIC.
3636

37+
config MLX5_MPFS
38+
bool "Mellanox Technologies MLX5 MPFS support"
39+
depends on MLX5_CORE_EN
40+
default y
41+
---help---
42+
Mellanox Technologies Ethernet Multi-Physical Function Switch (MPFS)
43+
support in ConnectX NIC. MPFs is required for when multi-PF configuration
44+
is enabled to allow passing user configured unicast MAC addresses to the
45+
requesting PF.
46+
3747
config MLX5_CORE_EN_DCB
3848
bool "Data Center Bridging (DCB) Support"
3949
default y

‎drivers/net/ethernet/mellanox/mlx5/core/Makefile

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ mlx5_core-$(CONFIG_MLX5_CORE_EN) += eswitch.o eswitch_offloads.o \
1616
en_rx.o en_rx_am.o en_txrx.o en_clock.o vxlan.o \
1717
en_tc.o en_arfs.o en_rep.o en_fs_ethtool.o en_selftest.o
1818

19+
mlx5_core-$(CONFIG_MLX5_MPFS) += lib/mpfs.o
20+
1921
mlx5_core-$(CONFIG_MLX5_CORE_EN_DCB) += en_dcbnl.o
2022

2123
mlx5_core-$(CONFIG_MLX5_CORE_IPOIB) += ipoib/ipoib.o ipoib/ethtool.o

‎drivers/net/ethernet/mellanox/mlx5/core/en_fs.c

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
#include <linux/tcp.h>
3737
#include <linux/mlx5/fs.h>
3838
#include "en.h"
39+
#include "lib/mpfs.h"
3940

4041
static int mlx5e_add_l2_flow_rule(struct mlx5e_priv *priv,
4142
struct mlx5e_l2_rule *ai, int type);
@@ -65,6 +66,7 @@ struct mlx5e_l2_hash_node {
6566
struct hlist_node hlist;
6667
u8 action;
6768
struct mlx5e_l2_rule ai;
69+
bool mpfs;
6870
};
6971

7072
static inline int mlx5e_hash_l2(u8 *addr)
@@ -362,17 +364,30 @@ static void mlx5e_del_vlan_rules(struct mlx5e_priv *priv)
362364
static void mlx5e_execute_l2_action(struct mlx5e_priv *priv,
363365
struct mlx5e_l2_hash_node *hn)
364366
{
365-
switch (hn->action) {
367+
u8 action = hn->action;
368+
int l2_err = 0;
369+
370+
switch (action) {
366371
case MLX5E_ACTION_ADD:
367372
mlx5e_add_l2_flow_rule(priv, &hn->ai, MLX5E_FULLMATCH);
373+
if (!is_multicast_ether_addr(hn->ai.addr)) {
374+
l2_err = mlx5_mpfs_add_mac(priv->mdev, hn->ai.addr);
375+
hn->mpfs = !l2_err;
376+
}
368377
hn->action = MLX5E_ACTION_NONE;
369378
break;
370379

371380
case MLX5E_ACTION_DEL:
381+
if (!is_multicast_ether_addr(hn->ai.addr) && hn->mpfs)
382+
l2_err = mlx5_mpfs_del_mac(priv->mdev, hn->ai.addr);
372383
mlx5e_del_l2_flow_rule(priv, &hn->ai);
373384
mlx5e_del_l2_from_hash(hn);
374385
break;
375386
}
387+
388+
if (l2_err)
389+
netdev_warn(priv->netdev, "MPFS, failed to %s mac %pM, err(%d)\n",
390+
action == MLX5E_ACTION_ADD ? "add" : "del", hn->ai.addr, l2_err);
376391
}
377392

378393
static void mlx5e_sync_netdev_addr(struct mlx5e_priv *priv)

‎drivers/net/ethernet/mellanox/mlx5/core/eswitch.c

Lines changed: 38 additions & 152 deletions
Original file line numberDiff line numberDiff line change
@@ -46,19 +46,13 @@ enum {
4646
MLX5_ACTION_DEL = 2,
4747
};
4848

49-
/* E-Switch UC L2 table hash node */
50-
struct esw_uc_addr {
51-
struct l2addr_node node;
52-
u32 table_index;
53-
u32 vport;
54-
};
55-
5649
/* Vport UC/MC hash node */
5750
struct vport_addr {
5851
struct l2addr_node node;
5952
u8 action;
6053
u32 vport;
61-
struct mlx5_flow_handle *flow_rule; /* SRIOV only */
54+
struct mlx5_flow_handle *flow_rule;
55+
bool mpfs; /* UC MAC was added to MPFs */
6256
/* A flag indicating that mac was added due to mc promiscuous vport */
6357
bool mc_promisc;
6458
};
@@ -154,81 +148,6 @@ static int modify_esw_vport_cvlan(struct mlx5_core_dev *dev, u32 vport,
154148
return modify_esw_vport_context_cmd(dev, vport, in, sizeof(in));
155149
}
156150

157-
/* HW L2 Table (MPFS) management */
158-
static int set_l2_table_entry_cmd(struct mlx5_core_dev *dev, u32 index,
159-
u8 *mac, u8 vlan_valid, u16 vlan)
160-
{
161-
u32 in[MLX5_ST_SZ_DW(set_l2_table_entry_in)] = {0};
162-
u32 out[MLX5_ST_SZ_DW(set_l2_table_entry_out)] = {0};
163-
u8 *in_mac_addr;
164-
165-
MLX5_SET(set_l2_table_entry_in, in, opcode,
166-
MLX5_CMD_OP_SET_L2_TABLE_ENTRY);
167-
MLX5_SET(set_l2_table_entry_in, in, table_index, index);
168-
MLX5_SET(set_l2_table_entry_in, in, vlan_valid, vlan_valid);
169-
MLX5_SET(set_l2_table_entry_in, in, vlan, vlan);
170-
171-
in_mac_addr = MLX5_ADDR_OF(set_l2_table_entry_in, in, mac_address);
172-
ether_addr_copy(&in_mac_addr[2], mac);
173-
174-
return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
175-
}
176-
177-
static int del_l2_table_entry_cmd(struct mlx5_core_dev *dev, u32 index)
178-
{
179-
u32 in[MLX5_ST_SZ_DW(delete_l2_table_entry_in)] = {0};
180-
u32 out[MLX5_ST_SZ_DW(delete_l2_table_entry_out)] = {0};
181-
182-
MLX5_SET(delete_l2_table_entry_in, in, opcode,
183-
MLX5_CMD_OP_DELETE_L2_TABLE_ENTRY);
184-
MLX5_SET(delete_l2_table_entry_in, in, table_index, index);
185-
return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
186-
}
187-
188-
static int alloc_l2_table_index(struct mlx5_l2_table *l2_table, u32 *ix)
189-
{
190-
int err = 0;
191-
192-
*ix = find_first_zero_bit(l2_table->bitmap, l2_table->size);
193-
if (*ix >= l2_table->size)
194-
err = -ENOSPC;
195-
else
196-
__set_bit(*ix, l2_table->bitmap);
197-
198-
return err;
199-
}
200-
201-
static void free_l2_table_index(struct mlx5_l2_table *l2_table, u32 ix)
202-
{
203-
__clear_bit(ix, l2_table->bitmap);
204-
}
205-
206-
static int set_l2_table_entry(struct mlx5_core_dev *dev, u8 *mac,
207-
u8 vlan_valid, u16 vlan,
208-
u32 *index)
209-
{
210-
struct mlx5_l2_table *l2_table = &dev->priv.eswitch->l2_table;
211-
int err;
212-
213-
err = alloc_l2_table_index(l2_table, index);
214-
if (err)
215-
return err;
216-
217-
err = set_l2_table_entry_cmd(dev, *index, mac, vlan_valid, vlan);
218-
if (err)
219-
free_l2_table_index(l2_table, *index);
220-
221-
return err;
222-
}
223-
224-
static void del_l2_table_entry(struct mlx5_core_dev *dev, u32 index)
225-
{
226-
struct mlx5_l2_table *l2_table = &dev->priv.eswitch->l2_table;
227-
228-
del_l2_table_entry_cmd(dev, index);
229-
free_l2_table_index(l2_table, index);
230-
}
231-
232151
/* E-Switch FDB */
233152
static struct mlx5_flow_handle *
234153
__esw_fdb_set_vport_rule(struct mlx5_eswitch *esw, u32 vport, bool rx_rule,
@@ -455,65 +374,60 @@ typedef int (*vport_addr_action)(struct mlx5_eswitch *esw,
455374

456375
static int esw_add_uc_addr(struct mlx5_eswitch *esw, struct vport_addr *vaddr)
457376
{
458-
struct hlist_head *hash = esw->l2_table.l2_hash;
459-
struct esw_uc_addr *esw_uc;
460377
u8 *mac = vaddr->node.addr;
461378
u32 vport = vaddr->vport;
462379
int err;
463380

464-
esw_uc = l2addr_hash_find(hash, mac, struct esw_uc_addr);
465-
if (esw_uc) {
381+
/* Skip mlx5_mpfs_add_mac for PFs,
382+
* it is already done by the PF netdev in mlx5e_execute_l2_action
383+
*/
384+
if (!vport)
385+
goto fdb_add;
386+
387+
err = mlx5_mpfs_add_mac(esw->dev, mac);
388+
if (err) {
466389
esw_warn(esw->dev,
467-
"Failed to set L2 mac(%pM) for vport(%d), mac is already in use by vport(%d)\n",
468-
mac, vport, esw_uc->vport);
469-
return -EEXIST;
390+
"Failed to add L2 table mac(%pM) for vport(%d), err(%d)\n",
391+
mac, vport, err);
392+
return err;
470393
}
394+
vaddr->mpfs = true;
471395

472-
esw_uc = l2addr_hash_add(hash, mac, struct esw_uc_addr, GFP_KERNEL);
473-
if (!esw_uc)
474-
return -ENOMEM;
475-
esw_uc->vport = vport;
476-
477-
err = set_l2_table_entry(esw->dev, mac, 0, 0, &esw_uc->table_index);
478-
if (err)
479-
goto abort;
480-
396+
fdb_add:
481397
/* SRIOV is enabled: Forward UC MAC to vport */
482398
if (esw->fdb_table.fdb && esw->mode == SRIOV_LEGACY)
483399
vaddr->flow_rule = esw_fdb_set_vport_rule(esw, mac, vport);
484400

485-
esw_debug(esw->dev, "\tADDED UC MAC: vport[%d] %pM index:%d fr(%p)\n",
486-
vport, mac, esw_uc->table_index, vaddr->flow_rule);
487-
return err;
488-
abort:
489-
l2addr_hash_del(esw_uc);
401+
esw_debug(esw->dev, "\tADDED UC MAC: vport[%d] %pM fr(%p)\n",
402+
vport, mac, vaddr->flow_rule);
403+
490404
return err;
491405
}
492406

493407
static int esw_del_uc_addr(struct mlx5_eswitch *esw, struct vport_addr *vaddr)
494408
{
495-
struct hlist_head *hash = esw->l2_table.l2_hash;
496-
struct esw_uc_addr *esw_uc;
497409
u8 *mac = vaddr->node.addr;
498410
u32 vport = vaddr->vport;
411+
int err = 0;
499412

500-
esw_uc = l2addr_hash_find(hash, mac, struct esw_uc_addr);
501-
if (!esw_uc || esw_uc->vport != vport) {
502-
esw_debug(esw->dev,
503-
"MAC(%pM) doesn't belong to vport (%d)\n",
504-
mac, vport);
505-
return -EINVAL;
506-
}
507-
esw_debug(esw->dev, "\tDELETE UC MAC: vport[%d] %pM index:%d fr(%p)\n",
508-
vport, mac, esw_uc->table_index, vaddr->flow_rule);
413+
/* Skip mlx5_mpfs_del_mac for PFs,
414+
* it is already done by the PF netdev in mlx5e_execute_l2_action
415+
*/
416+
if (!vport || !vaddr->mpfs)
417+
goto fdb_del;
509418

510-
del_l2_table_entry(esw->dev, esw_uc->table_index);
419+
err = mlx5_mpfs_del_mac(esw->dev, mac);
420+
if (err)
421+
esw_warn(esw->dev,
422+
"Failed to del L2 table mac(%pM) for vport(%d), err(%d)\n",
423+
mac, vport, err);
424+
vaddr->mpfs = false;
511425

426+
fdb_del:
512427
if (vaddr->flow_rule)
513428
mlx5_del_flow_rules(vaddr->flow_rule);
514429
vaddr->flow_rule = NULL;
515430

516-
l2addr_hash_del(esw_uc);
517431
return 0;
518432
}
519433

@@ -1635,7 +1549,6 @@ int mlx5_eswitch_enable_sriov(struct mlx5_eswitch *esw, int nvfs, int mode)
16351549

16361550
esw_info(esw->dev, "E-Switch enable SRIOV: nvfs(%d) mode (%d)\n", nvfs, mode);
16371551
esw->mode = mode;
1638-
esw_disable_vport(esw, 0);
16391552

16401553
if (mode == SRIOV_LEGACY)
16411554
err = esw_create_legacy_fdb_table(esw, nvfs + 1);
@@ -1648,7 +1561,11 @@ int mlx5_eswitch_enable_sriov(struct mlx5_eswitch *esw, int nvfs, int mode)
16481561
if (err)
16491562
esw_warn(esw->dev, "Failed to create eswitch TSAR");
16501563

1651-
enabled_events = (mode == SRIOV_LEGACY) ? SRIOV_VPORT_EVENTS : UC_ADDR_CHANGE;
1564+
/* Don't enable vport events when in SRIOV_OFFLOADS mode, since:
1565+
* 1. L2 table (MPFS) is programmed by PF/VF representors netdevs set_rx_mode
1566+
* 2. FDB/Eswitch is programmed by user space tools
1567+
*/
1568+
enabled_events = (mode == SRIOV_LEGACY) ? SRIOV_VPORT_EVENTS : 0;
16521569
for (i = 0; i <= nvfs; i++)
16531570
esw_enable_vport(esw, i, enabled_events);
16541571

@@ -1657,7 +1574,6 @@ int mlx5_eswitch_enable_sriov(struct mlx5_eswitch *esw, int nvfs, int mode)
16571574
return 0;
16581575

16591576
abort:
1660-
esw_enable_vport(esw, 0, UC_ADDR_CHANGE);
16611577
esw->mode = SRIOV_NONE;
16621578
return err;
16631579
}
@@ -1691,30 +1607,10 @@ void mlx5_eswitch_disable_sriov(struct mlx5_eswitch *esw)
16911607
esw_offloads_cleanup(esw, nvports);
16921608

16931609
esw->mode = SRIOV_NONE;
1694-
/* VPORT 0 (PF) must be enabled back with non-sriov configuration */
1695-
esw_enable_vport(esw, 0, UC_ADDR_CHANGE);
1696-
}
1697-
1698-
void mlx5_eswitch_attach(struct mlx5_eswitch *esw)
1699-
{
1700-
if (!ESW_ALLOWED(esw))
1701-
return;
1702-
1703-
esw_enable_vport(esw, 0, UC_ADDR_CHANGE);
1704-
/* VF Vports will be enabled when SRIOV is enabled */
1705-
}
1706-
1707-
void mlx5_eswitch_detach(struct mlx5_eswitch *esw)
1708-
{
1709-
if (!ESW_ALLOWED(esw))
1710-
return;
1711-
1712-
esw_disable_vport(esw, 0);
17131610
}
17141611

17151612
int mlx5_eswitch_init(struct mlx5_core_dev *dev)
17161613
{
1717-
int l2_table_size = 1 << MLX5_CAP_GEN(dev, log_max_l2_table);
17181614
int total_vports = MLX5_TOTAL_VPORTS(dev);
17191615
struct mlx5_eswitch *esw;
17201616
int vport_num;
@@ -1724,8 +1620,8 @@ int mlx5_eswitch_init(struct mlx5_core_dev *dev)
17241620
return 0;
17251621

17261622
esw_info(dev,
1727-
"Total vports %d, l2 table size(%d), per vport: max uc(%d) max mc(%d)\n",
1728-
total_vports, l2_table_size,
1623+
"Total vports %d, per vport: max uc(%d) max mc(%d)\n",
1624+
total_vports,
17291625
MLX5_MAX_UC_PER_VPORT(dev),
17301626
MLX5_MAX_MC_PER_VPORT(dev));
17311627

@@ -1735,14 +1631,6 @@ int mlx5_eswitch_init(struct mlx5_core_dev *dev)
17351631

17361632
esw->dev = dev;
17371633

1738-
esw->l2_table.bitmap = kcalloc(BITS_TO_LONGS(l2_table_size),
1739-
sizeof(uintptr_t), GFP_KERNEL);
1740-
if (!esw->l2_table.bitmap) {
1741-
err = -ENOMEM;
1742-
goto abort;
1743-
}
1744-
esw->l2_table.size = l2_table_size;
1745-
17461634
esw->work_queue = create_singlethread_workqueue("mlx5_esw_wq");
17471635
if (!esw->work_queue) {
17481636
err = -ENOMEM;
@@ -1793,7 +1681,6 @@ int mlx5_eswitch_init(struct mlx5_core_dev *dev)
17931681
abort:
17941682
if (esw->work_queue)
17951683
destroy_workqueue(esw->work_queue);
1796-
kfree(esw->l2_table.bitmap);
17971684
kfree(esw->vports);
17981685
kfree(esw->offloads.vport_reps);
17991686
kfree(esw);
@@ -1809,7 +1696,6 @@ void mlx5_eswitch_cleanup(struct mlx5_eswitch *esw)
18091696

18101697
esw->dev->priv.eswitch = NULL;
18111698
destroy_workqueue(esw->work_queue);
1812-
kfree(esw->l2_table.bitmap);
18131699
kfree(esw->offloads.vport_reps);
18141700
kfree(esw->vports);
18151701
kfree(esw);

0 commit comments

Comments
 (0)