diff --git a/.github/workflows/cygwin.yml b/.github/workflows/cygwin.yml new file mode 100644 index 000000000000..7b5096ef3e19 --- /dev/null +++ b/.github/workflows/cygwin.yml @@ -0,0 +1,251 @@ +--- +name: Cygwin Tests +concurrency: + group: ${{ github.workflow }}-${{ github.event.number }}-${{ github.event.ref }} + cancel-in-progress: true + +on: + push: + branches: + - main + - v[0-9]+.[0-9]+.[0-9x]+ + tags: + - v* + paths: + - 'src/**' + - '.github/workflows/cygwin.yml' + pull_request: + types: + - opened + - synchronize + - reopened + - labeled + branches-ignore: + - v[0-9]+.[0-9]+.[0-9x]+-doc + paths: + - 'src/**' + - '.github/workflows/cygwin.yml' + schedule: + # 5:47 UTC on Saturdays + - cron: "47 5 * * 6" + workflow_dispatch: + workflow: "*" + +permissions: + contents: read + +env: + NO_AT_BRIDGE: 1 # Necessary for GTK3 interactive test. + OPENBLAS_NUM_THREADS: 1 + PYTHONFAULTHANDLER: 1 + SHELLOPTS: igncr + CYGWIN_NOWINPATH: 1 + CHERE_INVOKING: 1 + TMP: /tmp + TEMP: /tmp + +jobs: + + test-cygwin: + runs-on: windows-latest + name: Python 3.${{ matrix.python-minor-version }} on Cygwin + if: | + github.event_name == 'workflow_dispatch' || + ( + github.repository == 'matplotlib/matplotlib' && + !contains(github.event.head_commit.message, '[ci skip]') && + !contains(github.event.head_commit.message, '[skip ci]') && + !contains(github.event.head_commit.message, '[skip github]') && + ( + github.event_name == 'push' || + github.event_name == 'pull_request' && + ( + ( + github.event.action == 'labeled' && + github.event.label.name == 'Run cygwin' + ) || + contains(github.event.pull_request.labels.*.name, 'Run cygwin') + ) + ) + ) + strategy: + matrix: + python-minor-version: [8, 9] + + steps: + - name: Fix line endings + run: git config --global core.autocrlf input + + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - uses: cygwin/cygwin-install-action@v2 + with: + packages: >- + ccache gcc-g++ gdb git graphviz libcairo-devel libffi-devel + libgeos-devel libQt5Core-devel pkgconf libglib2.0-devel + noto-cjk-fonts + python3${{ matrix.python-minor-version }}-devel + python3${{ matrix.python-minor-version }}-pip + python3${{ matrix.python-minor-version }}-wheel + python3${{ matrix.python-minor-version }}-setuptools + python3${{ matrix.python-minor-version }}-cycler + python3${{ matrix.python-minor-version }}-dateutil + python3${{ matrix.python-minor-version }}-fonttools + python3${{ matrix.python-minor-version }}-imaging + python3${{ matrix.python-minor-version }}-kiwisolver + python3${{ matrix.python-minor-version }}-numpy + python3${{ matrix.python-minor-version }}-packaging + python3${{ matrix.python-minor-version }}-pyparsing + python3${{ matrix.python-minor-version }}-sip + python3${{ matrix.python-minor-version }}-sphinx + python-cairo-devel + python3${{ matrix.python-minor-version }}-cairo + python3${{ matrix.python-minor-version }}-gi + python3${{ matrix.python-minor-version }}-matplotlib + xorg-server-extra libxcb-icccm4 libxcb-image0 + libxcb-keysyms1 libxcb-randr0 libxcb-render-util0 + libxcb-xinerama0 + make autoconf autoconf2.5 automake automake1.10 libtool m4 + libqhull-devel libfreetype-devel + libjpeg-devel libwebp-devel + + - name: Set runner username to root and id to 0 + shell: bash.exe -eo pipefail -o igncr "{0}" + # GitHub Actions runs everything as Administrator. I don't + # know how to test for this, so set the uid for the CI job so + # that the existing unix root detection will work. + run: | + /bin/mkpasswd.exe -c | sed -e "s/$(id -u)/0/" >/etc/passwd + + - name: Mark test repo safe + shell: bash.exe -eo pipefail -o igncr "{0}" + run: | + git.exe config --global --add safe.directory /proc/cygdrive/d/a/matplotlib/matplotlib + git config --global --add safe.directory /cygdrive/d/a/matplotlib/matplotlib + C:/cygwin/bin/git.exe config --global --add safe.directory D:/a/matplotlib/matplotlib + /usr/bin/git config --global --add safe.directory /cygdrive/d/a/matplotlib/matplotlib + + - name: Use dash for /bin/sh + shell: bash.exe -eo pipefail -o igncr "{0}" + run: | + ls -l /bin/sh.exe /bin/bash.exe /bin/dash.exe + /bin/rm -f /bin/sh.exe || exit 1 + cp -sf /bin/dash.exe /bin/sh.exe || exit 1 + ls -l /bin/sh.exe /bin/bash.exe /bin/dash.exe + # FreeType build fails with bash, succeeds with dash + + - name: Cache pip + uses: actions/cache@v3 + with: + path: C:\cygwin\home\runneradmin\.cache\pip + key: Cygwin-py3.${{ matrix.python-minor-version }}-pip-${{ hashFiles('requirements/*/*.txt') }} + restore-keys: | + ${{ matrix.os }}-py3.${{ matrix.python-minor-version }}-pip- + + - name: Cache ccache + uses: actions/cache@v3 + with: + path: C:\cygwin\home\runneradmin\.ccache + key: Cygwin-py3.${{ matrix.python-minor-version }}-ccache-${{ hashFiles('src/*') }} + restore-keys: Cygwin-py3.${{ matrix.python-minor-version }}-ccache- + + - name: Cache Matplotlib + uses: actions/cache@v3 + with: + path: | + C:\cygwin\home\runneradmin\.cache\matplotlib + !C:\cygwin\home\runneradmin\.cache\matplotlib\tex.cache + !C:\cygwin\home\runneradmin\.cache\matplotlib\test_cache + key: 1-Cygwin-py3.${{ matrix.python-minor-version }}-mpl-${{ github.ref }}-${{ github.sha }} + restore-keys: | + 1-Cygwin-py3.${{ matrix.python-minor-version }}-mpl-${{ github.ref }}- + 1-Cygwin-py3.${{ matrix.python-minor-version }}-mpl- + + - name: Ensure correct Python version + shell: bash.exe -eo pipefail -o igncr "{0}" + run: | + /usr/sbin/alternatives --set python /usr/bin/python3.${{ matrix.python-minor-version }} + /usr/sbin/alternatives --set python3 /usr/bin/python3.${{ matrix.python-minor-version }} + + - name: Install Python dependencies + shell: bash.exe -eo pipefail -o igncr "{0}" + run: | + python -m pip install --upgrade pip 'setuptools<60' wheel + python -m pip install kiwisolver 'numpy!=1.21.*' pillow importlib_resources + grep -v -F -e psutil requirements/testing/all.txt >requirements_test.txt + python -m pip install --upgrade 'contourpy>=1.0.1' cycler fonttools \ + packaging pyparsing python-dateutil setuptools-scm \ + -r requirements_test.txt sphinx ipython + python -m pip install --upgrade pycairo 'cairocffi>=0.8' PyGObject && + python -c 'import gi; gi.require_version("Gtk", "3.0"); from gi.repository import Gtk' && + echo 'PyGObject is available' || + echo 'PyGObject is not available' + python -m pip install --upgrade pyqt5 && + python -c 'import PyQt5.QtCore' && + echo 'PyQt5 is available' || + echo 'PyQt5 is not available' + python -mpip install --upgrade pyside2 && + python -c 'import PySide2.QtCore' && + echo 'PySide2 is available' || + echo 'PySide2 is not available' + python -m pip uninstall --yes wxpython || echo 'wxPython already uninstalled' + + - name: Install Matplotlib + shell: bash.exe -eo pipefail -o igncr "{0}" + env: + AUTOCONF: /usr/bin/autoconf-2.69 + MAKEFLAGS: dw + run: | + ccache -s + git describe + cat <> mplsetup.cfg + [rc_options] + backend=Agg + + [libs] + system_freetype = False + system_qhull = True + EOT + cat mplsetup.cfg + # All dependencies must have been pre-installed, so that the minver + # constraints are held. + python -m pip install --no-deps -ve . + + - name: Find DLLs to rebase + shell: bash.exe -eo pipefail -o igncr "{0}" + run: | + find {/usr,/usr/local}/{bin,lib/python3.*/site-packages} /usr/lib/lapack . -name \*.exe -o -name \*.dll -print >files_to_rebase.txt + + - name: Rebase DLL list + shell: ash.exe "{0}" + run: "rebase --database --filelist=files_to_rebase.txt" + # Inplace modification of DLLs to assign non-overlapping load + # addresses so fork() works as expected. Ash is used as it + # does not link against any Cygwin DLLs that might need to be + # rebased. + + - name: Check that Matplotlib imports + shell: bash.exe -eo pipefail -o igncr "{0}" + run: | + /usr/bin/python -c "import matplotlib as mpl; import matplotlib.pyplot as plt" + + - name: Set ffmpeg path + shell: bash.exe -eo pipefail -o igncr "{0}" + run: | + oldmplrc=$(python -c "from matplotlib import matplotlib_fname as mplrc_file; print(mplrc_file())") + echo "${oldmplrc}" + mkdir -p ~/.matplotlib/ + sed -E -e 's~#animation\.ffmpeg_path:.+~animation.ffmpeg_path: /usr/bin/ffmpeg.exe~' "${oldmplrc}" >~/.matplotlib/matplotlibrc + + - name: Run pytest + shell: bash.exe -eo pipefail -o igncr "{0}" + id: cygwin-run-pytest + run: | + xvfb-run python -mpytest -raR -n auto \ + --maxfail=50 --timeout=300 --durations=25 \ + --cov-report=xml --cov=lib --log-level=DEBUG --color=yes + + - name: Upload code coverage + uses: codecov/codecov-action@v3 diff --git a/lib/matplotlib/testing/__init__.py b/lib/matplotlib/testing/__init__.py index 6f54a4866c18..2df6814efa64 100644 --- a/lib/matplotlib/testing/__init__.py +++ b/lib/matplotlib/testing/__init__.py @@ -50,6 +50,69 @@ def setup(): set_reproducibility_for_testing() +def subprocess_run_for_testing( + command: "list[str]", + env: "dict[str, str]" = None, + timeout: float = None, + stdout=None, + stderr=None, + check: bool = False, + text: bool = True, + capture_output: bool = False +) -> "subprocess.Popen": + """ + Create and run a subprocess. + + Thin wrapper around `subprocess.run`, intended for testing. Will + mark fork() failures on Cygwin as expected failures: not a + success, but not indicating a problem with the code either. + + Parameters + ---------- + args : list of str + env : dict[str, str] + timeout : float + stdout, stderr + check : bool + text : bool + Also called ``universal_newlines`` in subprocess. I chose this + name since the main effect is returning bytes (`False`) vs. str + (`True`), though it also tries to normalize newlines across + platforms. + capture_output : bool + Set stdout and stderr to subprocess.PIPE + + Returns + ------- + proc : subprocess.Popen + + See Also + -------- + subprocess.run + + Raises + ------ + pytest.xfail + If platform is Cygwin and subprocess reports a fork() failure. + """ + if capture_output: + stdout = stderr = subprocess.PIPE + try: + proc = subprocess.run( + command, env=env, + timeout=timeout, check=check, + stdout=stdout, stderr=stderr, + text=text + ) + except BlockingIOError: + if sys.platform == "cygwin": + # Might want to make this more specific + import pytest + pytest.xfail("Fork failure") + raise + return proc + + def subprocess_run_helper(func, *args, timeout, extra_env=None): """ Run a function in a sub-process. @@ -66,16 +129,19 @@ def subprocess_run_helper(func, *args, timeout, extra_env=None): """ target = func.__name__ module = func.__module__ - proc = subprocess.run( - [sys.executable, - "-c", - f"from {module} import {target}; {target}()", - *args], + proc = subprocess_run_for_testing( + [ + sys.executable, + "-c", + f"from {module} import {target}; {target}()", + *args + ], env={**os.environ, "SOURCE_DATE_EPOCH": "0", **(extra_env or {})}, timeout=timeout, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - universal_newlines=True) + text=True + ) return proc diff --git a/lib/matplotlib/tests/test_preprocess_data.py b/lib/matplotlib/tests/test_preprocess_data.py index a95a72e7f78d..693385ee5b40 100644 --- a/lib/matplotlib/tests/test_preprocess_data.py +++ b/lib/matplotlib/tests/test_preprocess_data.py @@ -1,5 +1,4 @@ import re -import subprocess import sys import numpy as np @@ -7,6 +6,7 @@ from matplotlib import _preprocess_data from matplotlib.axes import Axes +from matplotlib.testing import subprocess_run_for_testing from matplotlib.testing.decorators import check_figures_equal # Notes on testing the plotting functions itself @@ -259,7 +259,9 @@ def test_data_parameter_replacement(): "import matplotlib.pyplot as plt" ) cmd = [sys.executable, "-c", program] - completed_proc = subprocess.run(cmd, text=True, capture_output=True) + completed_proc = subprocess_run_for_testing( + cmd, text=True, capture_output=True + ) assert 'data parameter docstring error' not in completed_proc.stderr diff --git a/lib/matplotlib/tests/test_pyplot.py b/lib/matplotlib/tests/test_pyplot.py index 95e3174d8ae8..292f81f1292e 100644 --- a/lib/matplotlib/tests/test_pyplot.py +++ b/lib/matplotlib/tests/test_pyplot.py @@ -1,13 +1,13 @@ import difflib import numpy as np -import subprocess import sys from pathlib import Path import pytest import matplotlib as mpl +from matplotlib.testing import subprocess_run_for_testing from matplotlib import pyplot as plt from matplotlib._api import MatplotlibDeprecationWarning @@ -20,8 +20,9 @@ def test_pyplot_up_to_date(tmpdir): plt_file = tmpdir.join('pyplot.py') plt_file.write_text(orig_contents, 'utf-8') - subprocess.run([sys.executable, str(gen_script), str(plt_file)], - check=True) + subprocess_run_for_testing( + [sys.executable, str(gen_script), str(plt_file)], + check=True) new_contents = plt_file.read_text('utf-8') if orig_contents != new_contents: diff --git a/lib/matplotlib/tests/test_sphinxext.py b/lib/matplotlib/tests/test_sphinxext.py index 41575d3a3ce1..669723be1d55 100644 --- a/lib/matplotlib/tests/test_sphinxext.py +++ b/lib/matplotlib/tests/test_sphinxext.py @@ -4,9 +4,9 @@ import os from pathlib import Path import shutil -from subprocess import Popen, PIPE import sys +from matplotlib.testing import subprocess_run_for_testing import pytest @@ -19,9 +19,11 @@ def build_sphinx_html(source_dir, doctree_dir, html_dir, extra_args=None): extra_args = [] if extra_args is None else extra_args cmd = [sys.executable, '-msphinx', '-W', '-b', 'html', '-d', str(doctree_dir), str(source_dir), str(html_dir), *extra_args] - proc = Popen(cmd, stdout=PIPE, stderr=PIPE, universal_newlines=True, - env={**os.environ, "MPLBACKEND": ""}) - out, err = proc.communicate() + proc = subprocess_run_for_testing( + cmd, capture_output=True, text=True, + env={**os.environ, "MPLBACKEND": ""}) + out = proc.stdout + err = proc.stderr assert proc.returncode == 0, \ f"sphinx build failed with stdout:\n{out}\nstderr:\n{err}\n" @@ -44,10 +46,12 @@ def test_tinypages(tmp_path): # On CI, gcov emits warnings (due to agg headers being included with the # same name in multiple extension modules -- but we don't care about their # coverage anyways); hide them using GCOV_ERROR_FILE. - proc = Popen( - cmd, stdout=PIPE, stderr=PIPE, universal_newlines=True, - env={**os.environ, "MPLBACKEND": "", "GCOV_ERROR_FILE": os.devnull}) - out, err = proc.communicate() + proc = subprocess_run_for_testing( + cmd, capture_output=True, text=True, + env={**os.environ, "MPLBACKEND": "", "GCOV_ERROR_FILE": os.devnull} + ) + out = proc.stdout + err = proc.stderr # Build the pages with warnings turned into errors build_sphinx_html(tmp_path, doctree_dir, html_dir) diff --git a/src/mplutils.h b/src/mplutils.h index 39d98ed02e8f..2eb9e2f563f3 100644 --- a/src/mplutils.h +++ b/src/mplutils.h @@ -6,6 +6,7 @@ #define MPLUTILS_H #define PY_SSIZE_T_CLEAN +#include #include #ifdef _POSIX_C_SOURCE @@ -27,7 +28,6 @@ #endif #endif -#include inline double mpl_round(double v) {