FIX: Move all tests using subprocess to the same worker on windows

timhoffm · timhoffm · commit 40f20c335abd · 2025-04-29T00:05:01.000+02:00
This is a somewhat wild guess based on #29797 (comment) > which makes me think we are somehow crossing state what launching the subprocesses. Using the `xdist_group` with `--dist=loadgroup` should put all tests of that group to the same worker according to https://pytest-xdist.readthedocs.io/en/stable/distribution.html. I've only added `--dist=loadgroup` to the windows pipelines, so tests on other systems are not affected at all. The first test is to see whether this works in the PR. - But to be sure, it think we would need to put this on master and monitor whether the timeouts disappear.
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -149,6 +149,7 @@ stages:
 
               PYTHONFAULTHANDLER=1 pytest -rfEsXR -n 2 \
                   --maxfail=50 --timeout=300 --durations=25 \
+                  --dist=loadgroup \
                   --junitxml=junit/test-results.xml --cov-report=xml --cov=lib
 
               if [[ $VS_VER == 2022 ]]; then
diff --git a/lib/matplotlib/tests/test_backend_inline.py b/lib/matplotlib/tests/test_backend_inline.py
@@ -12,6 +12,7 @@
 pytest.importorskip('matplotlib_inline')
 
 
+@pytest.mark.xdist_group(name="subprocess")
 def test_ipynb():
     nb_path = Path(__file__).parent / 'test_inline_01.ipynb'
 
diff --git a/lib/matplotlib/tests/test_backend_macosx.py b/lib/matplotlib/tests/test_backend_macosx.py
@@ -25,6 +25,7 @@ def _test_cached_renderer():
 
 
 @pytest.mark.backend('macosx', skip_on_importerror=True)
+@pytest.mark.xdist_group(name="subprocess")
 def test_cached_renderer():
     subprocess_run_helper(_test_cached_renderer, timeout=_test_timeout,
                           extra_env={"MPLBACKEND": "macosx"})
@@ -55,13 +56,15 @@ def new_choose_save_file(title, directory, filename):
 
 
 @pytest.mark.backend('macosx', skip_on_importerror=True)
+@pytest.mark.xdist_group(name="subprocess")
 def test_savefig_rcparam(tmp_path):
     subprocess_run_helper(
         _test_savefig_rcparam, timeout=_test_timeout,
         extra_env={"MPLBACKEND": "macosx", "TEST_SAVEFIG_PATH": tmp_path})
 
 
 @pytest.mark.backend('macosx', skip_on_importerror=True)
+@pytest.mark.xdist_group(name="subprocess")
 def test_ipython():
     from matplotlib.testing import ipython_in_subprocess
     ipython_in_subprocess("osx", {(8, 24): "macosx", (7, 0): "MacOSX"})
@@ -81,6 +84,7 @@ def _test_save_figure_return():
 
 
 @pytest.mark.backend('macosx', skip_on_importerror=True)
+@pytest.mark.xdist_group(name="subprocess")
 def test_save_figure_return():
     subprocess_run_helper(_test_save_figure_return, timeout=_test_timeout,
                           extra_env={"MPLBACKEND": "macosx"})
diff --git a/lib/matplotlib/tests/test_backend_nbagg.py b/lib/matplotlib/tests/test_backend_nbagg.py
@@ -13,6 +13,7 @@
 # From https://blog.thedataincubator.com/2016/06/testing-jupyter-notebooks/
 
 
+@pytest.mark.xdist_group(name="subprocess")
 def test_ipynb():
     nb_path = Path(__file__).parent / 'test_nbagg_01.ipynb'
 
diff --git a/lib/matplotlib/tests/test_backend_tk.py b/lib/matplotlib/tests/test_backend_tk.py
@@ -43,6 +43,7 @@ def _isolated_tk_test(success_count, func=None):
         sys.platform == 'darwin' and sys.version_info[:2] < (3, 11),
         reason='Tk version mismatch on Azure macOS CI'
     )
+    @pytest.mark.xdist_group(name="subprocess")
     @functools.wraps(func)
     def test_func():
         # even if the package exists, may not actually be importable this can
diff --git a/lib/matplotlib/tests/test_backend_webagg.py b/lib/matplotlib/tests/test_backend_webagg.py
@@ -7,6 +7,7 @@
 
 
 @pytest.mark.parametrize("backend", ["webagg", "nbagg"])
+@pytest.mark.xdist_group(name="subprocess")
 def test_webagg_fallback(backend):
     pytest.importorskip("tornado")
     if backend == "nbagg":
diff --git a/lib/matplotlib/tests/test_basic.py b/lib/matplotlib/tests/test_basic.py
@@ -3,6 +3,8 @@
 import sys
 import textwrap
 
+import pytest
+
 from matplotlib.testing import subprocess_run_for_testing
 
 
@@ -28,6 +30,7 @@ def test_override_builtins():
     assert overridden <= ok_to_override
 
 
+@pytest.mark.xdist_group(name="subprocess")
 def test_lazy_imports():
     source = textwrap.dedent("""
     import sys
diff --git a/lib/matplotlib/tests/test_determinism.py b/lib/matplotlib/tests/test_determinism.py
@@ -154,6 +154,7 @@ def draw(self, renderer=None):
         pytest.param("mhip", "svg", True, marks=needs_usetex),
     ]
 )
+@pytest.mark.xdist_group(name="subprocess")
 def test_determinism_check(objects, fmt, usetex):
     """
     Output three times the same graphs and checks that the outputs are exactly
@@ -195,6 +196,7 @@ def test_determinism_check(objects, fmt, usetex):
         ("ps", b"%%CreationDate: Sat Jan 01 00:00:00 2000"),
     ]
 )
+@pytest.mark.xdist_group(name="subprocess")
 def test_determinism_source_date_epoch(fmt, string):
     """
     Test SOURCE_DATE_EPOCH support. Output a document with the environment
diff --git a/lib/matplotlib/tests/test_font_manager.py b/lib/matplotlib/tests/test_font_manager.py
@@ -282,12 +282,14 @@ def bad_idea(n):
             raise RuntimeError("thread failed to join")
 
 
+@pytest.mark.xdist_group(name="subprocess")
 def test_fontcache_thread_safe():
     pytest.importorskip('threading')
 
     subprocess_run_helper(_test_threading, timeout=10)
 
 
+@pytest.mark.xdist_group(name="subprocess")
 def test_lockfilefailure(tmp_path):
     # The logic here:
     # 1. get a temp directory from pytest
diff --git a/lib/matplotlib/tests/test_matplotlib.py b/lib/matplotlib/tests/test_matplotlib.py
@@ -22,6 +22,7 @@ def test_parse_to_version_info(version_str, version_tuple):
                     reason="chmod() doesn't work as is on Windows")
 @pytest.mark.skipif(sys.platform != "win32" and os.geteuid() == 0,
                     reason="chmod() doesn't work as root")
+@pytest.mark.xdist_group(name="subprocess")
 def test_tmpconfigdir_warning(tmp_path):
     """Test that a warning is emitted if a temporary configdir must be used."""
     mode = os.stat(tmp_path).st_mode
@@ -36,6 +37,7 @@ def test_tmpconfigdir_warning(tmp_path):
         os.chmod(tmp_path, mode)
 
 
+@pytest.mark.xdist_group(name="subprocess")
 def test_importable_with_no_home(tmp_path):
     subprocess_run_for_testing(
         [sys.executable, "-c",
@@ -65,6 +67,7 @@ def parse(key):
             set(backend_registry.list_builtin(BackendFilter.NON_INTERACTIVE)))
 
 
+@pytest.mark.xdist_group(name="subprocess")
 def test_importable_with__OO():
     """
     When using -OO or export PYTHONOPTIMIZE=2, docstrings are discarded,
diff --git a/lib/matplotlib/tests/test_pickle.py b/lib/matplotlib/tests/test_pickle.py
@@ -136,6 +136,7 @@ def _pickle_load_subprocess():
 
 
 @mpl.style.context("default")
+@pytest.mark.xdist_group(name="subprocess")
 @check_figures_equal()
 def test_pickle_load_from_subprocess(fig_test, fig_ref, tmp_path):
     _generate_complete_test_figure(fig_ref)
@@ -331,6 +332,7 @@ def _test_axeswidget_interactive():
         sys.platform == 'darwin' and sys.version_info[:2] < (3, 11),
         reason='Tk version mismatch on Azure macOS CI'
     )
+@pytest.mark.xdist_group(name="subprocess")
 def test_axeswidget_interactive():
     subprocess_run_helper(
         _test_axeswidget_interactive,
diff --git a/lib/matplotlib/tests/test_preprocess_data.py b/lib/matplotlib/tests/test_preprocess_data.py
@@ -245,6 +245,7 @@ def funcy(ax, x, y, z, t=None):
                          funcy.__doc__)
 
 
+@pytest.mark.xdist_group(name="subprocess")
 def test_data_parameter_replacement():
     """
     Test that the docstring contains the correct *data* parameter stub
diff --git a/lib/matplotlib/tests/test_pyplot.py b/lib/matplotlib/tests/test_pyplot.py
@@ -11,6 +11,7 @@
 from matplotlib import pyplot as plt
 
 
+@pytest.mark.xdist_group(name="subprocess")
 def test_pyplot_up_to_date(tmp_path):
     pytest.importorskip("black")
 
@@ -348,6 +349,7 @@ def test_set_current_axes_on_subfigure():
     assert plt.gca() == ax
 
 
+@pytest.mark.xdist_group(name="subprocess")
 def test_pylab_integration():
     IPython = pytest.importorskip("IPython")
     mpl.testing.subprocess_run_helper(
diff --git a/lib/matplotlib/tests/test_rcparams.py b/lib/matplotlib/tests/test_rcparams.py
@@ -529,6 +529,7 @@ def test_rcparams_reset_after_fail():
 
 
 @pytest.mark.skipif(sys.platform != "linux", reason="Linux only")
+@pytest.mark.xdist_group(name="subprocess")
 def test_backend_fallback_headless_invalid_backend(tmp_path):
     env = {**os.environ,
            "DISPLAY": "", "WAYLAND_DISPLAY": "",
@@ -546,6 +547,7 @@ def test_backend_fallback_headless_invalid_backend(tmp_path):
 
 
 @pytest.mark.skipif(sys.platform != "linux", reason="Linux only")
+@pytest.mark.xdist_group(name="subprocess")
 def test_backend_fallback_headless_auto_backend(tmp_path):
     # specify a headless mpl environment, but request a graphical (tk) backend
     env = {**os.environ,
@@ -570,6 +572,7 @@ def test_backend_fallback_headless_auto_backend(tmp_path):
 @pytest.mark.skipif(
     sys.platform == "linux" and not _c_internal_utils.xdisplay_is_valid(),
     reason="headless")
+@pytest.mark.xdist_group(name="subprocess")
 def test_backend_fallback_headful(tmp_path):
     if parse_version(pytest.__version__) >= parse_version('8.2.0'):
         pytest_kwargs = dict(exc_type=ImportError)
diff --git a/lib/matplotlib/tests/test_sphinxext.py b/lib/matplotlib/tests/test_sphinxext.py
@@ -32,6 +32,7 @@ def build_sphinx_html(source_dir, doctree_dir, html_dir, extra_args=None):
     assert html_dir.is_dir()
 
 
+@pytest.mark.xdist_group(name="subprocess")
 def test_tinypages(tmp_path):
     shutil.copytree(Path(__file__).parent / 'tinypages', tmp_path,
                     dirs_exist_ok=True)
@@ -124,6 +125,7 @@ def plot_directive_file(num):
     assert filecmp.cmp(range_6, plot_file(5))
 
 
+@pytest.mark.xdist_group(name="subprocess")
 def test_plot_html_show_source_link(tmp_path):
     parent = Path(__file__).parent
     shutil.copyfile(parent / 'tinypages/conf.py', tmp_path / 'conf.py')
@@ -147,6 +149,7 @@ def test_plot_html_show_source_link(tmp_path):
 
 
 @pytest.mark.parametrize('plot_html_show_source_link', [0, 1])
+@pytest.mark.xdist_group(name="subprocess")
 def test_show_source_link_true(tmp_path, plot_html_show_source_link):
     # Test that a source link is generated if :show-source-link: is true,
     # whether or not plot_html_show_source_link is true.
@@ -167,6 +170,7 @@ def test_show_source_link_true(tmp_path, plot_html_show_source_link):
 
 
 @pytest.mark.parametrize('plot_html_show_source_link', [0, 1])
+@pytest.mark.xdist_group(name="subprocess")
 def test_show_source_link_false(tmp_path, plot_html_show_source_link):
     # Test that a source link is NOT generated if :show-source-link: is false,
     # whether or not plot_html_show_source_link is true.
@@ -186,6 +190,7 @@ def test_show_source_link_false(tmp_path, plot_html_show_source_link):
     assert len(list(html_dir.glob("**/index-1.py"))) == 0
 
 
+@pytest.mark.xdist_group(name="subprocess")
 def test_srcset_version(tmp_path):
     shutil.copytree(Path(__file__).parent / 'tinypages', tmp_path,
                     dirs_exist_ok=True)
diff --git a/lib/matplotlib/tests/test_texmanager.py b/lib/matplotlib/tests/test_texmanager.py
@@ -64,6 +64,7 @@ def test_unicode_characters():
 
 
 @needs_usetex
+@pytest.mark.xdist_group(name="subprocess")
 def test_openin_any_paranoid():
     completed = subprocess_run_for_testing(
         [sys.executable, "-c",
diff --git a/lib/matplotlib/tests/test_ticker.py b/lib/matplotlib/tests/test_ticker.py
@@ -1766,6 +1766,7 @@ def _impl_locale_comma():
     assert x == '0,5'
 
 
+@pytest.mark.xdist_group(name="subprocess")
 def test_locale_comma():
     # On some systems/pytest versions, `pytest.skip` in an exception handler
     # does not skip, but is treated as an exception, so directly running this

Original file line number	Diff line number	Diff line change
`@@ -43,6 +43,7 @@ def _isolated_tk_test(success_count, func=None):`
`43`	`43`	`sys.platform == 'darwin' and sys.version_info[:2] < (3, 11),`
`44`	`44`	`reason='Tk version mismatch on Azure macOS CI'`
`45`	`45`	`)`
	`46`	`+ @pytest.mark.xdist_group(name="subprocess")`
`46`	`47`	`@functools.wraps(func)`
`47`	`48`	`def test_func():`
`48`	`49`	`# even if the package exists, may not actually be importable this can`
Original file line number	Diff line number	Diff line change
`@@ -154,6 +154,7 @@ def draw(self, renderer=None):`
`154`	`154`	`pytest.param("mhip", "svg", True, marks=needs_usetex),`
`155`	`155`	`]`
`156`	`156`	`)`
	`157`	`+@pytest.mark.xdist_group(name="subprocess")`
`157`	`158`	`def test_determinism_check(objects, fmt, usetex):`
`158`	`159`	`"""`
`159`	`160`	`Output three times the same graphs and checks that the outputs are exactly`
`@@ -195,6 +196,7 @@ def test_determinism_check(objects, fmt, usetex):`
`195`	`196`	`("ps", b"%%CreationDate: Sat Jan 01 00:00:00 2000"),`
`196`	`197`	`]`
`197`	`198`	`)`
	`199`	`+@pytest.mark.xdist_group(name="subprocess")`
`198`	`200`	`def test_determinism_source_date_epoch(fmt, string):`
`199`	`201`	`"""`
`200`	`202`	`Test SOURCE_DATE_EPOCH support. Output a document with the environment`