From f4002ce3f78736a37c6b5511f13fe1c2838933a0 Mon Sep 17 00:00:00 2001 From: "Thomas J. Fan" Date: Thu, 2 Mar 2023 10:28:49 -0500 Subject: [PATCH 1/5] CI Disable network when SciPy requires it --- sklearn/conftest.py | 28 +++++++ .../feature_extraction/tests/test_image.py | 84 ++++++++----------- 2 files changed, 61 insertions(+), 51 deletions(-) diff --git a/sklearn/conftest.py b/sklearn/conftest.py index 90b30506e8cae..9a818b2c039de 100644 --- a/sklearn/conftest.py +++ b/sklearn/conftest.py @@ -11,6 +11,7 @@ from sklearn.utils import _IS_32BIT from sklearn.utils._openmp_helpers import _openmp_effective_n_threads from sklearn._min_dependencies import PYTEST_MIN_VERSION +from sklearn.utils.fixes import sp_base_version from sklearn.utils.fixes import parse_version from sklearn.datasets import fetch_20newsgroups from sklearn.datasets import fetch_20newsgroups_vectorized @@ -28,6 +29,29 @@ "at least pytest >= {} installed.".format(PYTEST_MIN_VERSION) ) +scipy_datasets_require_network = sp_base_version >= parse_version("1.10") + + +def raccoon_face_or_skip(): + if scipy_datasets_require_network: + run_network_tests = environ.get("SKLEARN_SKIP_NETWORK_TESTS", "1") == "0" + from scipy.datasets import face + + try: + return face(gray=True) + except ImportError as e: + if run_network_tests: + raise ValueError( + "pooch is required when SKLEARN_SKIP_NETWORK_TESTS=0" + ) from e + pytest.skip("test is enabled when SKLEARN_SKIP_NETWORK_TESTS=0") + + else: + from scipy.misc import face + + return face(gray=True) + + dataset_fetchers = { "fetch_20newsgroups_fxt": fetch_20newsgroups, "fetch_20newsgroups_vectorized_fxt": fetch_20newsgroups_vectorized, @@ -38,6 +62,9 @@ "fetch_rcv1_fxt": fetch_rcv1, } +if scipy_datasets_require_network: + dataset_fetchers["raccoon_face_fxt"] = raccoon_face_or_skip + _SKIP32_MARK = pytest.mark.skipif( environ.get("SKLEARN_RUN_FLOAT32_TESTS", "0") != "1", reason="Set SKLEARN_RUN_FLOAT32_TESTS=1 to run float32 dtype tests", @@ -75,6 +102,7 @@ def wrapped(*args, **kwargs): fetch_kddcup99_fxt = _fetch_fixture(fetch_kddcup99) fetch_olivetti_faces_fxt = _fetch_fixture(fetch_olivetti_faces) fetch_rcv1_fxt = _fetch_fixture(fetch_rcv1) +raccoon_face_fxt = pytest.fixture(raccoon_face_or_skip) def pytest_collection_modifyitems(config, items): diff --git a/sklearn/feature_extraction/tests/test_image.py b/sklearn/feature_extraction/tests/test_image.py index 91b777f83f1c9..52489e7da55be 100644 --- a/sklearn/feature_extraction/tests/test_image.py +++ b/sklearn/feature_extraction/tests/test_image.py @@ -7,7 +7,6 @@ from scipy.sparse.csgraph import connected_components import pytest -from sklearn.utils.fixes import sp_version, parse_version from sklearn.feature_extraction.image import ( img_to_graph, grid_to_graph, @@ -18,17 +17,6 @@ ) -@pytest.fixture(scope="module") -def raccoon_face(): - if sp_version.release >= parse_version("1.10").release: - pytest.importorskip("pooch") - from scipy.datasets import face - else: - from scipy.misc import face - - return face(gray=True) - - def test_img_to_graph(): x, y = np.mgrid[:4, :4] - 10 grad_x = img_to_graph(x) @@ -93,8 +81,8 @@ def test_grid_to_graph(): assert A.dtype == np.float64 -def test_connect_regions(raccoon_face): - face = raccoon_face.copy() +def test_connect_regions(raccoon_face_fxt): + face = raccoon_face_fxt # subsample by 4 to reduce run time face = face[::4, ::4] for thr in (50, 150): @@ -103,8 +91,8 @@ def test_connect_regions(raccoon_face): assert ndimage.label(mask)[1] == connected_components(graph)[0] -def test_connect_regions_with_grid(raccoon_face): - face = raccoon_face.copy() +def test_connect_regions_with_grid(raccoon_face_fxt): + face = raccoon_face_fxt # subsample by 4 to reduce run time face = face[::4, ::4] @@ -118,15 +106,9 @@ def test_connect_regions_with_grid(raccoon_face): assert ndimage.label(mask)[1] == connected_components(graph)[0] -def _downsampled_face(): - if sp_version.release >= parse_version("1.10").release: - pytest.importorskip("pooch") - from scipy.datasets import face as raccoon_face - else: - from scipy.misc import face as raccoon_face - - face = raccoon_face(gray=True) - face = face.astype(np.float32) +@pytest.fixture +def downsampled_face(raccoon_face_fxt): + face = raccoon_face_fxt face = face[::2, ::2] + face[1::2, ::2] + face[::2, 1::2] + face[1::2, 1::2] face = face[::2, ::2] + face[1::2, ::2] + face[::2, 1::2] + face[1::2, 1::2] face = face.astype(np.float32) @@ -134,8 +116,9 @@ def _downsampled_face(): return face -def _orange_face(face=None): - face = _downsampled_face() if face is None else face +@pytest.fixture +def orange_face(downsampled_face): + face = downsampled_face face_color = np.zeros(face.shape + (3,)) face_color[:, :, 0] = 256 - face face_color[:, :, 1] = 256 - face / 2 @@ -143,8 +126,7 @@ def _orange_face(face=None): return face_color -def _make_images(face=None): - face = _downsampled_face() if face is None else face +def _make_images(face): # make a collection of faces images = np.zeros((3,) + face.shape) images[0] = face @@ -153,12 +135,12 @@ def _make_images(face=None): return images -downsampled_face = _downsampled_face() -orange_face = _orange_face(downsampled_face) -face_collection = _make_images(downsampled_face) +@pytest.fixture +def downsampled_face_collection(downsampled_face): + return _make_images(downsampled_face) -def test_extract_patches_all(): +def test_extract_patches_all(downsampled_face): face = downsampled_face i_h, i_w = face.shape p_h, p_w = 16, 16 @@ -167,7 +149,7 @@ def test_extract_patches_all(): assert patches.shape == (expected_n_patches, p_h, p_w) -def test_extract_patches_all_color(): +def test_extract_patches_all_color(orange_face): face = orange_face i_h, i_w = face.shape[:2] p_h, p_w = 16, 16 @@ -176,7 +158,7 @@ def test_extract_patches_all_color(): assert patches.shape == (expected_n_patches, p_h, p_w, 3) -def test_extract_patches_all_rect(): +def test_extract_patches_all_rect(downsampled_face): face = downsampled_face face = face[:, 32:97] i_h, i_w = face.shape @@ -187,7 +169,7 @@ def test_extract_patches_all_rect(): assert patches.shape == (expected_n_patches, p_h, p_w) -def test_extract_patches_max_patches(): +def test_extract_patches_max_patches(downsampled_face): face = downsampled_face i_h, i_w = face.shape p_h, p_w = 16, 16 @@ -205,7 +187,7 @@ def test_extract_patches_max_patches(): extract_patches_2d(face, (p_h, p_w), max_patches=-1.0) -def test_extract_patch_same_size_image(): +def test_extract_patch_same_size_image(downsampled_face): face = downsampled_face # Request patches of the same size as image # Should return just the single patch a.k.a. the image @@ -213,7 +195,7 @@ def test_extract_patch_same_size_image(): assert patches.shape[0] == 1 -def test_extract_patches_less_than_max_patches(): +def test_extract_patches_less_than_max_patches(downsampled_face): face = downsampled_face i_h, i_w = face.shape p_h, p_w = 3 * i_h // 4, 3 * i_w // 4 @@ -224,7 +206,7 @@ def test_extract_patches_less_than_max_patches(): assert patches.shape == (expected_n_patches, p_h, p_w) -def test_reconstruct_patches_perfect(): +def test_reconstruct_patches_perfect(downsampled_face): face = downsampled_face p_h, p_w = 16, 16 @@ -233,7 +215,7 @@ def test_reconstruct_patches_perfect(): np.testing.assert_array_almost_equal(face, face_reconstructed) -def test_reconstruct_patches_perfect_color(): +def test_reconstruct_patches_perfect_color(orange_face): face = orange_face p_h, p_w = 16, 16 @@ -242,14 +224,14 @@ def test_reconstruct_patches_perfect_color(): np.testing.assert_array_almost_equal(face, face_reconstructed) -def test_patch_extractor_fit(): - faces = face_collection +def test_patch_extractor_fit(downsampled_face_collection): + faces = downsampled_face_collection extr = PatchExtractor(patch_size=(8, 8), max_patches=100, random_state=0) assert extr == extr.fit(faces) -def test_patch_extractor_max_patches(): - faces = face_collection +def test_patch_extractor_max_patches(downsampled_face_collection): + faces = downsampled_face_collection i_h, i_w = faces.shape[1:3] p_h, p_w = 8, 8 @@ -272,15 +254,15 @@ def test_patch_extractor_max_patches(): assert patches.shape == (expected_n_patches, p_h, p_w) -def test_patch_extractor_max_patches_default(): - faces = face_collection +def test_patch_extractor_max_patches_default(downsampled_face_collection): + faces = downsampled_face_collection extr = PatchExtractor(max_patches=100, random_state=0) patches = extr.transform(faces) assert patches.shape == (len(faces) * 100, 19, 25) -def test_patch_extractor_all_patches(): - faces = face_collection +def test_patch_extractor_all_patches(downsampled_face_collection): + faces = downsampled_face_collection i_h, i_w = faces.shape[1:3] p_h, p_w = 8, 8 expected_n_patches = len(faces) * (i_h - p_h + 1) * (i_w - p_w + 1) @@ -289,7 +271,7 @@ def test_patch_extractor_all_patches(): assert patches.shape == (expected_n_patches, p_h, p_w) -def test_patch_extractor_color(): +def test_patch_extractor_color(orange_face): faces = _make_images(orange_face) i_h, i_w = faces.shape[1:3] p_h, p_w = 8, 8 @@ -347,7 +329,7 @@ def test_extract_patches_strided(): ).all() -def test_extract_patches_square(): +def test_extract_patches_square(downsampled_face): # test same patch size for all dimensions face = downsampled_face i_h, i_w = face.shape @@ -366,7 +348,7 @@ def test_width_patch(): extract_patches_2d(x, (1, 4)) -def test_patch_extractor_wrong_input(): +def test_patch_extractor_wrong_input(orange_face): """Check that an informative error is raised if the patch_size is not valid.""" faces = _make_images(orange_face) err_msg = "patch_size must be a tuple of two integers" From b975888236c37b6b4b85ad8177928294b54c77ed Mon Sep 17 00:00:00 2001 From: "Thomas J. Fan" Date: Thu, 2 Mar 2023 10:36:20 -0500 Subject: [PATCH 2/5] CI [scipy-dev] From 78165f371f47bdda969e9d9fae6a2e7a04ceeedc Mon Sep 17 00:00:00 2001 From: "Thomas J. Fan" Date: Thu, 2 Mar 2023 11:09:22 -0500 Subject: [PATCH 3/5] MNT Easier to backport [scipy-dev] --- sklearn/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sklearn/conftest.py b/sklearn/conftest.py index 9a818b2c039de..6c66bbe2390fc 100644 --- a/sklearn/conftest.py +++ b/sklearn/conftest.py @@ -11,7 +11,7 @@ from sklearn.utils import _IS_32BIT from sklearn.utils._openmp_helpers import _openmp_effective_n_threads from sklearn._min_dependencies import PYTEST_MIN_VERSION -from sklearn.utils.fixes import sp_base_version +from sklearn.utils.fixes import sp_version from sklearn.utils.fixes import parse_version from sklearn.datasets import fetch_20newsgroups from sklearn.datasets import fetch_20newsgroups_vectorized @@ -29,7 +29,7 @@ "at least pytest >= {} installed.".format(PYTEST_MIN_VERSION) ) -scipy_datasets_require_network = sp_base_version >= parse_version("1.10") +scipy_datasets_require_network = sp_version >= parse_version("1.10") def raccoon_face_or_skip(): From b9300252d43b2ac8a136da3edead9cce49e06a13 Mon Sep 17 00:00:00 2001 From: "Thomas J. Fan" Date: Fri, 3 Mar 2023 11:13:43 -0500 Subject: [PATCH 4/5] DOC Adds comment --- sklearn/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sklearn/conftest.py b/sklearn/conftest.py index 6c66bbe2390fc..907fd01b8f98c 100644 --- a/sklearn/conftest.py +++ b/sklearn/conftest.py @@ -33,6 +33,7 @@ def raccoon_face_or_skip(): + # SciPy >= 1.10 requires network to access to get data if scipy_datasets_require_network: run_network_tests = environ.get("SKLEARN_SKIP_NETWORK_TESTS", "1") == "0" from scipy.datasets import face From 3d51a19ac75078f7b4fd480ed5403f16aee7bd26 Mon Sep 17 00:00:00 2001 From: "Thomas J. Fan" Date: Fri, 3 Mar 2023 11:33:26 -0500 Subject: [PATCH 5/5] CLN Clearer code --- sklearn/conftest.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/sklearn/conftest.py b/sklearn/conftest.py index 907fd01b8f98c..582409760c65b 100644 --- a/sklearn/conftest.py +++ b/sklearn/conftest.py @@ -2,6 +2,8 @@ from functools import wraps import platform import sys +from contextlib import suppress +from unittest import SkipTest import pytest import numpy as np @@ -36,21 +38,19 @@ def raccoon_face_or_skip(): # SciPy >= 1.10 requires network to access to get data if scipy_datasets_require_network: run_network_tests = environ.get("SKLEARN_SKIP_NETWORK_TESTS", "1") == "0" - from scipy.datasets import face + if not run_network_tests: + raise SkipTest("test is enabled when SKLEARN_SKIP_NETWORK_TESTS=0") try: - return face(gray=True) - except ImportError as e: - if run_network_tests: - raise ValueError( - "pooch is required when SKLEARN_SKIP_NETWORK_TESTS=0" - ) from e - pytest.skip("test is enabled when SKLEARN_SKIP_NETWORK_TESTS=0") + import pooch # noqa + except ImportError: + raise SkipTest("test requires pooch to be installed") + from scipy.datasets import face else: from scipy.misc import face - return face(gray=True) + return face(gray=True) dataset_fetchers = { @@ -144,7 +144,8 @@ def pytest_collection_modifyitems(config, items): worker_id = environ.get("PYTEST_XDIST_WORKER", "gw0") if worker_id == "gw0" and run_network_tests: for name in datasets_to_download: - dataset_fetchers[name]() + with suppress(SkipTest): + dataset_fetchers[name]() for item in items: # Known failure on with GradientBoostingClassifier on ARM64