Skip to content

Commit ef1b5da

Browse files
bjorn-helgaasbjorn-helgaas
committed
Merge branch 'pci/virtualization' into next
* pci/virtualization: ixgbe: Use pcie_flr() instead of duplicating it IB/hfi1: Use pcie_flr() instead of duplicating it PCI: Call pcie_flr() from reset_chelsio_generic_dev() PCI: Call pcie_flr() from reset_intel_82599_sfp_virtfn() PCI: Export pcie_flr() PCI: Add sysfs sriov_drivers_autoprobe to control VF driver binding PCI: Avoid FLR for Intel 82579 NICs Conflicts: include/linux/pci.h
2 parents 889e4dd + 63af8f7 commit ef1b5da

File tree

13 files changed

+134
-86
lines changed

13 files changed

+134
-86
lines changed

‎Documentation/ABI/testing/sysfs-bus-pci

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -301,3 +301,25 @@ Contact: Emil Velikov <emil.l.velikov@gmail.com>
301301
Description:
302302
This file contains the revision field of the the PCI device.
303303
The value comes from device config space. The file is read only.
304+
305+
What: /sys/bus/pci/devices/.../sriov_drivers_autoprobe
306+
Date: April 2017
307+
Contact: Bodong Wang<bodong@mellanox.com>
308+
Description:
309+
This file is associated with the PF of a device that
310+
supports SR-IOV. It determines whether newly-enabled VFs
311+
are immediately bound to a driver. It initially contains
312+
1, which means the kernel automatically binds VFs to a
313+
compatible driver immediately after they are enabled. If
314+
an application writes 0 to the file before enabling VFs,
315+
the kernel will not bind VFs to a driver.
316+
317+
A typical use case is to write 0 to this file, then enable
318+
VFs, then assign the newly-created VFs to virtual machines.
319+
Note that changing this file does not affect already-
320+
enabled VFs. In this scenario, the user must first disable
321+
the VFs, write 0 to sriov_drivers_autoprobe, then re-enable
322+
the VFs.
323+
324+
This is similar to /sys/bus/pci/drivers_autoprobe, but
325+
affects only the VFs associated with a specific PF.

‎Documentation/PCI/pci-iov-howto.txt

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,18 @@ To disable SR-IOV capability:
6868
echo 0 > \
6969
/sys/bus/pci/devices/<DOMAIN:BUS:DEVICE.FUNCTION>/sriov_numvfs
7070

71+
To enable auto probing VFs by a compatible driver on the host, run
72+
command below before enabling SR-IOV capabilities. This is the
73+
default behavior.
74+
echo 1 > \
75+
/sys/bus/pci/devices/<DOMAIN:BUS:DEVICE.FUNCTION>/sriov_drivers_autoprobe
76+
77+
To disable auto probing VFs by a compatible driver on the host, run
78+
command below before enabling SR-IOV capabilities. Updating this
79+
entry will not affect VFs which are already probed.
80+
echo 0 > \
81+
/sys/bus/pci/devices/<DOMAIN:BUS:DEVICE.FUNCTION>/sriov_drivers_autoprobe
82+
7183
3.2 Usage example
7284

7385
Following piece of code illustrates the usage of the SR-IOV API.

‎drivers/infiniband/hw/hfi1/chip.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13610,14 +13610,14 @@ static void init_chip(struct hfi1_devdata *dd)
1361013610
dd_dev_info(dd, "Resetting CSRs with FLR\n");
1361113611

1361213612
/* do the FLR, the DC reset will remain */
13613-
hfi1_pcie_flr(dd);
13613+
pcie_flr(dd->pcidev);
1361413614

1361513615
/* restore command and BARs */
1361613616
restore_pci_variables(dd);
1361713617

1361813618
if (is_ax(dd)) {
1361913619
dd_dev_info(dd, "Resetting CSRs with FLR\n");
13620-
hfi1_pcie_flr(dd);
13620+
pcie_flr(dd->pcidev);
1362113621
restore_pci_variables(dd);
1362213622
}
1362313623
} else {

‎drivers/infiniband/hw/hfi1/hfi.h

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1764,7 +1764,6 @@ int hfi1_pcie_init(struct pci_dev *, const struct pci_device_id *);
17641764
void hfi1_pcie_cleanup(struct pci_dev *);
17651765
int hfi1_pcie_ddinit(struct hfi1_devdata *, struct pci_dev *);
17661766
void hfi1_pcie_ddcleanup(struct hfi1_devdata *);
1767-
void hfi1_pcie_flr(struct hfi1_devdata *);
17681767
int pcie_speeds(struct hfi1_devdata *);
17691768
void request_msix(struct hfi1_devdata *, u32 *, struct hfi1_msix_entry *);
17701769
void hfi1_enable_intx(struct pci_dev *);

‎drivers/infiniband/hw/hfi1/pcie.c

Lines changed: 0 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -240,36 +240,6 @@ void hfi1_pcie_ddcleanup(struct hfi1_devdata *dd)
240240
iounmap(dd->piobase);
241241
}
242242

243-
/*
244-
* Do a Function Level Reset (FLR) on the device.
245-
* Based on static function drivers/pci/pci.c:pcie_flr().
246-
*/
247-
void hfi1_pcie_flr(struct hfi1_devdata *dd)
248-
{
249-
int i;
250-
u16 status;
251-
252-
/* no need to check for the capability - we know the device has it */
253-
254-
/* wait for Transaction Pending bit to clear, at most a few ms */
255-
for (i = 0; i < 4; i++) {
256-
if (i)
257-
msleep((1 << (i - 1)) * 100);
258-
259-
pcie_capability_read_word(dd->pcidev, PCI_EXP_DEVSTA, &status);
260-
if (!(status & PCI_EXP_DEVSTA_TRPND))
261-
goto clear;
262-
}
263-
264-
dd_dev_err(dd, "Transaction Pending bit is not clearing, proceeding with reset anyway\n");
265-
266-
clear:
267-
pcie_capability_set_word(dd->pcidev, PCI_EXP_DEVCTL,
268-
PCI_EXP_DEVCTL_BCR_FLR);
269-
/* PCIe spec requires the function to be back within 100ms */
270-
msleep(100);
271-
}
272-
273243
static void msix_setup(struct hfi1_devdata *dd, int pos, u32 *msixcnt,
274244
struct hfi1_msix_entry *hfi1_msix_entry)
275245
{

‎drivers/net/ethernet/intel/ixgbe/ixgbe_main.c

Lines changed: 2 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -7112,18 +7112,6 @@ static void ixgbe_watchdog_flush_tx(struct ixgbe_adapter *adapter)
71127112
}
71137113

71147114
#ifdef CONFIG_PCI_IOV
7115-
static inline void ixgbe_issue_vf_flr(struct ixgbe_adapter *adapter,
7116-
struct pci_dev *vfdev)
7117-
{
7118-
if (!pci_wait_for_pending_transaction(vfdev))
7119-
e_dev_warn("Issuing VFLR with pending transactions\n");
7120-
7121-
e_dev_err("Issuing VFLR for VF %s\n", pci_name(vfdev));
7122-
pcie_capability_set_word(vfdev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR);
7123-
7124-
msleep(100);
7125-
}
7126-
71277115
static void ixgbe_check_for_bad_vf(struct ixgbe_adapter *adapter)
71287116
{
71297117
struct ixgbe_hw *hw = &adapter->hw;
@@ -7156,7 +7144,7 @@ static void ixgbe_check_for_bad_vf(struct ixgbe_adapter *adapter)
71567144
pci_read_config_word(vfdev, PCI_STATUS, &status_reg);
71577145
if (status_reg != IXGBE_FAILED_READ_CFG_WORD &&
71587146
status_reg & PCI_STATUS_REC_MASTER_ABORT)
7159-
ixgbe_issue_vf_flr(adapter, vfdev);
7147+
pcie_flr(vfdev);
71607148
}
71617149
}
71627150

@@ -10244,7 +10232,7 @@ static pci_ers_result_t ixgbe_io_error_detected(struct pci_dev *pdev,
1024410232
* VFLR. Just clean up the AER in that case.
1024510233
*/
1024610234
if (vfdev) {
10247-
ixgbe_issue_vf_flr(adapter, vfdev);
10235+
pcie_flr(vfdev);
1024810236
/* Free device reference count */
1024910237
pci_dev_put(vfdev);
1025010238
}

‎drivers/pci/iov.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -450,6 +450,7 @@ static int sriov_init(struct pci_dev *dev, int pos)
450450
iov->total_VFs = total;
451451
iov->pgsz = pgsz;
452452
iov->self = dev;
453+
iov->drivers_autoprobe = true;
453454
pci_read_config_dword(dev, pos + PCI_SRIOV_CAP, &iov->cap);
454455
pci_read_config_byte(dev, pos + PCI_SRIOV_FUNC_LINK, &iov->link);
455456
if (pci_pcie_type(dev) == PCI_EXP_TYPE_RC_END)

‎drivers/pci/pci-driver.c

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -394,6 +394,18 @@ void __weak pcibios_free_irq(struct pci_dev *dev)
394394
{
395395
}
396396

397+
#ifdef CONFIG_PCI_IOV
398+
static inline bool pci_device_can_probe(struct pci_dev *pdev)
399+
{
400+
return (!pdev->is_virtfn || pdev->physfn->sriov->drivers_autoprobe);
401+
}
402+
#else
403+
static inline bool pci_device_can_probe(struct pci_dev *pdev)
404+
{
405+
return true;
406+
}
407+
#endif
408+
397409
static int pci_device_probe(struct device *dev)
398410
{
399411
int error;
@@ -405,10 +417,12 @@ static int pci_device_probe(struct device *dev)
405417
return error;
406418

407419
pci_dev_get(pci_dev);
408-
error = __pci_device_probe(drv, pci_dev);
409-
if (error) {
410-
pcibios_free_irq(pci_dev);
411-
pci_dev_put(pci_dev);
420+
if (pci_device_can_probe(pci_dev)) {
421+
error = __pci_device_probe(drv, pci_dev);
422+
if (error) {
423+
pcibios_free_irq(pci_dev);
424+
pci_dev_put(pci_dev);
425+
}
412426
}
413427

414428
return error;

‎drivers/pci/pci-sysfs.c

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -526,10 +526,37 @@ static ssize_t sriov_numvfs_store(struct device *dev,
526526
return count;
527527
}
528528

529+
static ssize_t sriov_drivers_autoprobe_show(struct device *dev,
530+
struct device_attribute *attr,
531+
char *buf)
532+
{
533+
struct pci_dev *pdev = to_pci_dev(dev);
534+
535+
return sprintf(buf, "%u\n", pdev->sriov->drivers_autoprobe);
536+
}
537+
538+
static ssize_t sriov_drivers_autoprobe_store(struct device *dev,
539+
struct device_attribute *attr,
540+
const char *buf, size_t count)
541+
{
542+
struct pci_dev *pdev = to_pci_dev(dev);
543+
bool drivers_autoprobe;
544+
545+
if (kstrtobool(buf, &drivers_autoprobe) < 0)
546+
return -EINVAL;
547+
548+
pdev->sriov->drivers_autoprobe = drivers_autoprobe;
549+
550+
return count;
551+
}
552+
529553
static struct device_attribute sriov_totalvfs_attr = __ATTR_RO(sriov_totalvfs);
530554
static struct device_attribute sriov_numvfs_attr =
531555
__ATTR(sriov_numvfs, (S_IRUGO|S_IWUSR|S_IWGRP),
532556
sriov_numvfs_show, sriov_numvfs_store);
557+
static struct device_attribute sriov_drivers_autoprobe_attr =
558+
__ATTR(sriov_drivers_autoprobe, (S_IRUGO|S_IWUSR|S_IWGRP),
559+
sriov_drivers_autoprobe_show, sriov_drivers_autoprobe_store);
533560
#endif /* CONFIG_PCI_IOV */
534561

535562
static ssize_t driver_override_store(struct device *dev,
@@ -1539,6 +1566,7 @@ static struct attribute_group pci_dev_hp_attr_group = {
15391566
static struct attribute *sriov_dev_attrs[] = {
15401567
&sriov_totalvfs_attr.attr,
15411568
&sriov_numvfs_attr.attr,
1569+
&sriov_drivers_autoprobe_attr.attr,
15421570
NULL,
15431571
};
15441572

‎drivers/pci/pci.c

Lines changed: 32 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3860,24 +3860,41 @@ static void pci_flr_wait(struct pci_dev *dev)
38603860
(i - 1) * 100);
38613861
}
38623862

3863-
static int pcie_flr(struct pci_dev *dev, int probe)
3863+
/**
3864+
* pcie_has_flr - check if a device supports function level resets
3865+
* @dev: device to check
3866+
*
3867+
* Returns true if the device advertises support for PCIe function level
3868+
* resets.
3869+
*/
3870+
static bool pcie_has_flr(struct pci_dev *dev)
38643871
{
38653872
u32 cap;
38663873

3867-
pcie_capability_read_dword(dev, PCI_EXP_DEVCAP, &cap);
3868-
if (!(cap & PCI_EXP_DEVCAP_FLR))
3869-
return -ENOTTY;
3874+
if (dev->dev_flags & PCI_DEV_FLAGS_NO_FLR_RESET)
3875+
return false;
38703876

3871-
if (probe)
3872-
return 0;
3877+
pcie_capability_read_dword(dev, PCI_EXP_DEVCAP, &cap);
3878+
return cap & PCI_EXP_DEVCAP_FLR;
3879+
}
38733880

3881+
/**
3882+
* pcie_flr - initiate a PCIe function level reset
3883+
* @dev: device to reset
3884+
*
3885+
* Initiate a function level reset on @dev. The caller should ensure the
3886+
* device supports FLR before calling this function, e.g. by using the
3887+
* pcie_has_flr() helper.
3888+
*/
3889+
void pcie_flr(struct pci_dev *dev)
3890+
{
38743891
if (!pci_wait_for_pending_transaction(dev))
38753892
dev_err(&dev->dev, "timed out waiting for pending transaction; performing function level reset anyway\n");
38763893

38773894
pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR);
38783895
pci_flr_wait(dev);
3879-
return 0;
38803896
}
3897+
EXPORT_SYMBOL_GPL(pcie_flr);
38813898

38823899
static int pci_af_flr(struct pci_dev *dev, int probe)
38833900
{
@@ -3888,6 +3905,9 @@ static int pci_af_flr(struct pci_dev *dev, int probe)
38883905
if (!pos)
38893906
return -ENOTTY;
38903907

3908+
if (dev->dev_flags & PCI_DEV_FLAGS_NO_FLR_RESET)
3909+
return -ENOTTY;
3910+
38913911
pci_read_config_byte(dev, pos + PCI_AF_CAP, &cap);
38923912
if (!(cap & PCI_AF_CAP_TP) || !(cap & PCI_AF_CAP_FLR))
38933913
return -ENOTTY;
@@ -4058,9 +4078,12 @@ static int __pci_dev_reset(struct pci_dev *dev, int probe)
40584078
if (rc != -ENOTTY)
40594079
goto done;
40604080

4061-
rc = pcie_flr(dev, probe);
4062-
if (rc != -ENOTTY)
4081+
if (pcie_has_flr(dev)) {
4082+
if (!probe)
4083+
pcie_flr(dev);
4084+
rc = 0;
40634085
goto done;
4086+
}
40644087

40654088
rc = pci_af_flr(dev, probe);
40664089
if (rc != -ENOTTY)

‎drivers/pci/pci.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -272,6 +272,7 @@ struct pci_sriov {
272272
struct pci_dev *self; /* this PF */
273273
struct mutex lock; /* lock for setting sriov_numvfs in sysfs */
274274
resource_size_t barsz[PCI_SRIOV_NUM_BARS]; /* VF BAR size */
275+
bool drivers_autoprobe; /* auto probing of VFs by driver */
275276
};
276277

277278
/* pci_dev priv_flags */

‎drivers/pci/quirks.c

Lines changed: 13 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -3666,19 +3666,11 @@ static int reset_intel_82599_sfp_virtfn(struct pci_dev *dev, int probe)
36663666
*
36673667
* The 82599 supports FLR on VFs, but FLR support is reported only
36683668
* in the PF DEVCAP (sec 9.3.10.4), not in the VF DEVCAP (sec 9.5).
3669-
* Therefore, we can't use pcie_flr(), which checks the VF DEVCAP.
3669+
* Thus we must call pcie_flr() directly without first checking if it is
3670+
* supported.
36703671
*/
3671-
3672-
if (probe)
3673-
return 0;
3674-
3675-
if (!pci_wait_for_pending_transaction(dev))
3676-
dev_err(&dev->dev, "transaction is not cleared; proceeding with reset anyway\n");
3677-
3678-
pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR);
3679-
3680-
msleep(100);
3681-
3672+
if (!probe)
3673+
pcie_flr(dev);
36823674
return 0;
36833675
}
36843676

@@ -3783,20 +3775,7 @@ static int reset_chelsio_generic_dev(struct pci_dev *dev, int probe)
37833775
PCI_MSIX_FLAGS_ENABLE |
37843776
PCI_MSIX_FLAGS_MASKALL);
37853777

3786-
/*
3787-
* Start of pcie_flr() code sequence. This reset code is a copy of
3788-
* the guts of pcie_flr() because that's not an exported function.
3789-
*/
3790-
3791-
if (!pci_wait_for_pending_transaction(dev))
3792-
dev_err(&dev->dev, "transaction is not cleared; proceeding with reset anyway\n");
3793-
3794-
pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR);
3795-
msleep(100);
3796-
3797-
/*
3798-
* End of pcie_flr() code sequence.
3799-
*/
3778+
pcie_flr(dev);
38003779

38013780
/*
38023781
* Restore the configuration information (BAR values, etc.) including
@@ -4677,3 +4656,11 @@ DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x2030, quirk_no_aersid);
46774656
DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x2031, quirk_no_aersid);
46784657
DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x2032, quirk_no_aersid);
46794658
DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x2033, quirk_no_aersid);
4659+
4660+
/* FLR may cause some 82579 devices to hang. */
4661+
static void quirk_intel_no_flr(struct pci_dev *dev)
4662+
{
4663+
dev->dev_flags |= PCI_DEV_FLAGS_NO_FLR_RESET;
4664+
}
4665+
DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x1502, quirk_intel_no_flr);
4666+
DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x1503, quirk_intel_no_flr);

‎include/linux/pci.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,8 @@ enum pci_dev_flags {
181181
PCI_DEV_FLAGS_VPD_REF_F0 = (__force pci_dev_flags_t) (1 << 8),
182182
/* a non-root bridge where translation occurs, stop alias search here */
183183
PCI_DEV_FLAGS_BRIDGE_XLATE_ROOT = (__force pci_dev_flags_t) (1 << 9),
184+
/* Do not use FLR even if device advertises PCI_AF_CAP */
185+
PCI_DEV_FLAGS_NO_FLR_RESET = (__force pci_dev_flags_t) (1 << 10),
184186
};
185187

186188
enum pci_irq_reroute_variant {
@@ -1037,6 +1039,7 @@ int pcie_get_mps(struct pci_dev *dev);
10371039
int pcie_set_mps(struct pci_dev *dev, int mps);
10381040
int pcie_get_minimum_link(struct pci_dev *dev, enum pci_bus_speed *speed,
10391041
enum pcie_link_width *width);
1042+
void pcie_flr(struct pci_dev *dev);
10401043
int __pci_reset_function(struct pci_dev *dev);
10411044
int __pci_reset_function_locked(struct pci_dev *dev);
10421045
int pci_reset_function(struct pci_dev *dev);

0 commit comments

Comments
 (0)