Skip to content

CI Disable network when SciPy requires it #25743

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Mar 6, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 31 additions & 1 deletion sklearn/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
from functools import wraps
import platform
import sys
from contextlib import suppress
from unittest import SkipTest

import pytest
import numpy as np
Expand All @@ -11,6 +13,7 @@
from sklearn.utils import _IS_32BIT
from sklearn.utils._openmp_helpers import _openmp_effective_n_threads
from sklearn._min_dependencies import PYTEST_MIN_VERSION
from sklearn.utils.fixes import sp_version
from sklearn.utils.fixes import parse_version
from sklearn.datasets import fetch_20newsgroups
from sklearn.datasets import fetch_20newsgroups_vectorized
Expand All @@ -28,6 +31,28 @@
"at least pytest >= {} installed.".format(PYTEST_MIN_VERSION)
)

scipy_datasets_require_network = sp_version >= parse_version("1.10")


def raccoon_face_or_skip():
# SciPy >= 1.10 requires network to access to get data
if scipy_datasets_require_network:
run_network_tests = environ.get("SKLEARN_SKIP_NETWORK_TESTS", "1") == "0"
if not run_network_tests:
raise SkipTest("test is enabled when SKLEARN_SKIP_NETWORK_TESTS=0")

try:
import pooch # noqa
except ImportError:
raise SkipTest("test requires pooch to be installed")

from scipy.datasets import face
else:
from scipy.misc import face

return face(gray=True)


dataset_fetchers = {
"fetch_20newsgroups_fxt": fetch_20newsgroups,
"fetch_20newsgroups_vectorized_fxt": fetch_20newsgroups_vectorized,
Expand All @@ -38,6 +63,9 @@
"fetch_rcv1_fxt": fetch_rcv1,
}

if scipy_datasets_require_network:
dataset_fetchers["raccoon_face_fxt"] = raccoon_face_or_skip

_SKIP32_MARK = pytest.mark.skipif(
environ.get("SKLEARN_RUN_FLOAT32_TESTS", "0") != "1",
reason="Set SKLEARN_RUN_FLOAT32_TESTS=1 to run float32 dtype tests",
Expand Down Expand Up @@ -75,6 +103,7 @@ def wrapped(*args, **kwargs):
fetch_kddcup99_fxt = _fetch_fixture(fetch_kddcup99)
fetch_olivetti_faces_fxt = _fetch_fixture(fetch_olivetti_faces)
fetch_rcv1_fxt = _fetch_fixture(fetch_rcv1)
raccoon_face_fxt = pytest.fixture(raccoon_face_or_skip)


def pytest_collection_modifyitems(config, items):
Expand Down Expand Up @@ -115,7 +144,8 @@ def pytest_collection_modifyitems(config, items):
worker_id = environ.get("PYTEST_XDIST_WORKER", "gw0")
if worker_id == "gw0" and run_network_tests:
for name in datasets_to_download:
dataset_fetchers[name]()
with suppress(SkipTest):
dataset_fetchers[name]()

for item in items:
# Known failure on with GradientBoostingClassifier on ARM64
Expand Down
84 changes: 33 additions & 51 deletions sklearn/feature_extraction/tests/test_image.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
from scipy.sparse.csgraph import connected_components
import pytest

from sklearn.utils.fixes import sp_version, parse_version
from sklearn.feature_extraction.image import (
img_to_graph,
grid_to_graph,
Expand All @@ -18,17 +17,6 @@
)


@pytest.fixture(scope="module")
def raccoon_face():
if sp_version.release >= parse_version("1.10").release:
pytest.importorskip("pooch")
from scipy.datasets import face
else:
from scipy.misc import face

return face(gray=True)


def test_img_to_graph():
x, y = np.mgrid[:4, :4] - 10
grad_x = img_to_graph(x)
Expand Down Expand Up @@ -93,8 +81,8 @@ def test_grid_to_graph():
assert A.dtype == np.float64


def test_connect_regions(raccoon_face):
face = raccoon_face.copy()
def test_connect_regions(raccoon_face_fxt):
face = raccoon_face_fxt
# subsample by 4 to reduce run time
face = face[::4, ::4]
for thr in (50, 150):
Expand All @@ -103,8 +91,8 @@ def test_connect_regions(raccoon_face):
assert ndimage.label(mask)[1] == connected_components(graph)[0]


def test_connect_regions_with_grid(raccoon_face):
face = raccoon_face.copy()
def test_connect_regions_with_grid(raccoon_face_fxt):
face = raccoon_face_fxt

# subsample by 4 to reduce run time
face = face[::4, ::4]
Expand All @@ -118,33 +106,27 @@ def test_connect_regions_with_grid(raccoon_face):
assert ndimage.label(mask)[1] == connected_components(graph)[0]


def _downsampled_face():
if sp_version.release >= parse_version("1.10").release:
pytest.importorskip("pooch")
from scipy.datasets import face as raccoon_face
else:
from scipy.misc import face as raccoon_face

face = raccoon_face(gray=True)
face = face.astype(np.float32)
@pytest.fixture
def downsampled_face(raccoon_face_fxt):
face = raccoon_face_fxt
face = face[::2, ::2] + face[1::2, ::2] + face[::2, 1::2] + face[1::2, 1::2]
face = face[::2, ::2] + face[1::2, ::2] + face[::2, 1::2] + face[1::2, 1::2]
face = face.astype(np.float32)
face /= 16.0
return face


def _orange_face(face=None):
face = _downsampled_face() if face is None else face
@pytest.fixture
def orange_face(downsampled_face):
face = downsampled_face
face_color = np.zeros(face.shape + (3,))
face_color[:, :, 0] = 256 - face
face_color[:, :, 1] = 256 - face / 2
face_color[:, :, 2] = 256 - face / 4
return face_color


def _make_images(face=None):
face = _downsampled_face() if face is None else face
def _make_images(face):
# make a collection of faces
images = np.zeros((3,) + face.shape)
images[0] = face
Expand All @@ -153,12 +135,12 @@ def _make_images(face=None):
return images


downsampled_face = _downsampled_face()
orange_face = _orange_face(downsampled_face)
face_collection = _make_images(downsampled_face)
@pytest.fixture
def downsampled_face_collection(downsampled_face):
return _make_images(downsampled_face)


def test_extract_patches_all():
def test_extract_patches_all(downsampled_face):
face = downsampled_face
i_h, i_w = face.shape
p_h, p_w = 16, 16
Expand All @@ -167,7 +149,7 @@ def test_extract_patches_all():
assert patches.shape == (expected_n_patches, p_h, p_w)


def test_extract_patches_all_color():
def test_extract_patches_all_color(orange_face):
face = orange_face
i_h, i_w = face.shape[:2]
p_h, p_w = 16, 16
Expand All @@ -176,7 +158,7 @@ def test_extract_patches_all_color():
assert patches.shape == (expected_n_patches, p_h, p_w, 3)


def test_extract_patches_all_rect():
def test_extract_patches_all_rect(downsampled_face):
face = downsampled_face
face = face[:, 32:97]
i_h, i_w = face.shape
Expand All @@ -187,7 +169,7 @@ def test_extract_patches_all_rect():
assert patches.shape == (expected_n_patches, p_h, p_w)


def test_extract_patches_max_patches():
def test_extract_patches_max_patches(downsampled_face):
face = downsampled_face
i_h, i_w = face.shape
p_h, p_w = 16, 16
Expand All @@ -205,15 +187,15 @@ def test_extract_patches_max_patches():
extract_patches_2d(face, (p_h, p_w), max_patches=-1.0)


def test_extract_patch_same_size_image():
def test_extract_patch_same_size_image(downsampled_face):
face = downsampled_face
# Request patches of the same size as image
# Should return just the single patch a.k.a. the image
patches = extract_patches_2d(face, face.shape, max_patches=2)
assert patches.shape[0] == 1


def test_extract_patches_less_than_max_patches():
def test_extract_patches_less_than_max_patches(downsampled_face):
face = downsampled_face
i_h, i_w = face.shape
p_h, p_w = 3 * i_h // 4, 3 * i_w // 4
Expand All @@ -224,7 +206,7 @@ def test_extract_patches_less_than_max_patches():
assert patches.shape == (expected_n_patches, p_h, p_w)


def test_reconstruct_patches_perfect():
def test_reconstruct_patches_perfect(downsampled_face):
face = downsampled_face
p_h, p_w = 16, 16

Expand All @@ -233,7 +215,7 @@ def test_reconstruct_patches_perfect():
np.testing.assert_array_almost_equal(face, face_reconstructed)


def test_reconstruct_patches_perfect_color():
def test_reconstruct_patches_perfect_color(orange_face):
face = orange_face
p_h, p_w = 16, 16

Expand All @@ -242,14 +224,14 @@ def test_reconstruct_patches_perfect_color():
np.testing.assert_array_almost_equal(face, face_reconstructed)


def test_patch_extractor_fit():
faces = face_collection
def test_patch_extractor_fit(downsampled_face_collection):
faces = downsampled_face_collection
extr = PatchExtractor(patch_size=(8, 8), max_patches=100, random_state=0)
assert extr == extr.fit(faces)


def test_patch_extractor_max_patches():
faces = face_collection
def test_patch_extractor_max_patches(downsampled_face_collection):
faces = downsampled_face_collection
i_h, i_w = faces.shape[1:3]
p_h, p_w = 8, 8

Expand All @@ -272,15 +254,15 @@ def test_patch_extractor_max_patches():
assert patches.shape == (expected_n_patches, p_h, p_w)


def test_patch_extractor_max_patches_default():
faces = face_collection
def test_patch_extractor_max_patches_default(downsampled_face_collection):
faces = downsampled_face_collection
extr = PatchExtractor(max_patches=100, random_state=0)
patches = extr.transform(faces)
assert patches.shape == (len(faces) * 100, 19, 25)


def test_patch_extractor_all_patches():
faces = face_collection
def test_patch_extractor_all_patches(downsampled_face_collection):
faces = downsampled_face_collection
i_h, i_w = faces.shape[1:3]
p_h, p_w = 8, 8
expected_n_patches = len(faces) * (i_h - p_h + 1) * (i_w - p_w + 1)
Expand All @@ -289,7 +271,7 @@ def test_patch_extractor_all_patches():
assert patches.shape == (expected_n_patches, p_h, p_w)


def test_patch_extractor_color():
def test_patch_extractor_color(orange_face):
faces = _make_images(orange_face)
i_h, i_w = faces.shape[1:3]
p_h, p_w = 8, 8
Expand Down Expand Up @@ -347,7 +329,7 @@ def test_extract_patches_strided():
).all()


def test_extract_patches_square():
def test_extract_patches_square(downsampled_face):
# test same patch size for all dimensions
face = downsampled_face
i_h, i_w = face.shape
Expand All @@ -366,7 +348,7 @@ def test_width_patch():
extract_patches_2d(x, (1, 4))


def test_patch_extractor_wrong_input():
def test_patch_extractor_wrong_input(orange_face):
"""Check that an informative error is raised if the patch_size is not valid."""
faces = _make_images(orange_face)
err_msg = "patch_size must be a tuple of two integers"
Expand Down