From 75e71e7330d24d38cb24d47414da48c9a13aad86 Mon Sep 17 00:00:00 2001 From: Stephane Couvreur Date: Thu, 9 May 2019 16:00:30 +0100 Subject: [PATCH 1/7] Created stub for check_estimator verbose flag Created 'verbose' parameter in check_estimator function from sklearn.utils.estimator_checks with default value False. --- sklearn/utils/estimator_checks.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/sklearn/utils/estimator_checks.py b/sklearn/utils/estimator_checks.py index 1dd816246be06..7589eca8b474c 100644 --- a/sklearn/utils/estimator_checks.py +++ b/sklearn/utils/estimator_checks.py @@ -270,7 +270,7 @@ def _yield_all_checks(name, estimator): yield check_fit_idempotent -def check_estimator(Estimator): +def check_estimator(Estimator, verbose=False): """Check if estimator adheres to scikit-learn conventions. This estimator will run an extensive test-suite for input validation, @@ -288,7 +288,13 @@ def check_estimator(Estimator): estimator : estimator object or class Estimator to check. Estimator is a class object or instance. + verbose : boolean + Flag to specify in order to see a progress bar. + """ + if verbose == True: + raise NotImplementedError + if isinstance(Estimator, type): # got a class name = Estimator.__name__ From c96c9926effe1722055d5e496135bce8c1d2a6f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Couvreur?= Date: Fri, 10 May 2019 10:59:47 +0100 Subject: [PATCH 2/7] Update sklearn/utils/estimator_checks.py Co-Authored-By: Guillaume Lemaitre --- sklearn/utils/estimator_checks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sklearn/utils/estimator_checks.py b/sklearn/utils/estimator_checks.py index 7589eca8b474c..a381cdb89b26c 100644 --- a/sklearn/utils/estimator_checks.py +++ b/sklearn/utils/estimator_checks.py @@ -292,7 +292,7 @@ def check_estimator(Estimator, verbose=False): Flag to specify in order to see a progress bar. """ - if verbose == True: + if verbose: raise NotImplementedError if isinstance(Estimator, type): From be3e907078a9ee0eac2ab4e2b991dcd9dc8e334f Mon Sep 17 00:00:00 2001 From: Stephane Couvreur Date: Mon, 17 Jun 2019 12:13:03 +0100 Subject: [PATCH 3/7] Added pytest runner Pytest runner added inside the check_estimator function, still need to generate tests for the collector. On branch check_estimator_progress_verbose_mode Your branch is up to date with 'origin/check_estimator_progress_verbose_mode'. Changes to be committed: modified: estimator_checks.py --- sklearn/utils/estimator_checks.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/sklearn/utils/estimator_checks.py b/sklearn/utils/estimator_checks.py index 7589eca8b474c..d62331ef4ceec 100644 --- a/sklearn/utils/estimator_checks.py +++ b/sklearn/utils/estimator_checks.py @@ -3,6 +3,7 @@ import sys import traceback import pickle +import pytest from copy import deepcopy from functools import partial from inspect import signature @@ -270,7 +271,7 @@ def _yield_all_checks(name, estimator): yield check_fit_idempotent -def check_estimator(Estimator, verbose=False): +def check_estimator(Estimator): """Check if estimator adheres to scikit-learn conventions. This estimator will run an extensive test-suite for input validation, @@ -287,18 +288,12 @@ def check_estimator(Estimator, verbose=False): ---------- estimator : estimator object or class Estimator to check. Estimator is a class object or instance. - - verbose : boolean - Flag to specify in order to see a progress bar. - """ - if verbose == True: - raise NotImplementedError - if isinstance(Estimator, type): # got a class name = Estimator.__name__ estimator = Estimator() + # Generate tests for pytest hooks collector check_parameters_default_constructible(name, Estimator) check_no_attributes_set_in_init(name, estimator) else: @@ -313,6 +308,8 @@ def check_estimator(Estimator, verbose=False): # the only SkipTest thrown currently results from not # being able to import pandas. warnings.warn(str(exception), SkipTestWarning) + # Run tests using pytest + pytest.main(['-vvv', 'tests']) def _boston_subset(n_samples=200): From c82677ed1f28d1d83ea5ccf3e0c96fe1fc8c221e Mon Sep 17 00:00:00 2001 From: Stephane Couvreur Date: Mon, 8 Jul 2019 16:58:47 +0100 Subject: [PATCH 4/7] Made _yield_all_checks public Changed _yield_all_checks to a public method and changed references to its private version. --- sklearn/tests/test_common.py | 7 +------ sklearn/utils/estimator_checks.py | 6 ++---- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/sklearn/tests/test_common.py b/sklearn/tests/test_common.py index 660b38c1ae4c2..855c79349f753 100644 --- a/sklearn/tests/test_common.py +++ b/sklearn/tests/test_common.py @@ -99,12 +99,7 @@ def _rename_partial(val): return type(val).__name__ -@pytest.mark.parametrize( - "estimator, check", - _generate_checks_per_estimator(_yield_all_checks, - _tested_estimators()), - ids=_rename_partial -) +@pytest.mark.estimator_checks(_tested_estimators()) def test_estimators(estimator, check): # Common tests for estimator instances with ignore_warnings(category=(DeprecationWarning, ConvergenceWarning, diff --git a/sklearn/utils/estimator_checks.py b/sklearn/utils/estimator_checks.py index 4e05e89a8cc6a..dc4aa4af2a88d 100644 --- a/sklearn/utils/estimator_checks.py +++ b/sklearn/utils/estimator_checks.py @@ -229,7 +229,7 @@ def _yield_outliers_checks(name, estimator): yield check_estimators_unfitted -def _yield_all_checks(name, estimator): +def yield_all_checks(name, estimator): tags = _safe_tags(estimator) if "2darray" not in tags["X_types"]: warnings.warn("Can't test estimator {} which requires input " @@ -301,15 +301,13 @@ def check_estimator(Estimator): estimator = Estimator name = type(estimator).__name__ - for check in _yield_all_checks(name, estimator): + for check in yield_all_checks(name, estimator): try: check(name, estimator) except SkipTest as exception: # the only SkipTest thrown currently results from not # being able to import pandas. warnings.warn(str(exception), SkipTestWarning) - # Run tests using pytest - pytest.main(['-vvv', 'tests']) def _boston_subset(n_samples=200): From 8cc1a08bccf13c916a883fd0a3e32ed74540b760 Mon Sep 17 00:00:00 2001 From: Stephane Couvreur Date: Mon, 8 Jul 2019 17:04:06 +0100 Subject: [PATCH 5/7] Removed pytest import --- sklearn/utils/estimator_checks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sklearn/utils/estimator_checks.py b/sklearn/utils/estimator_checks.py index dc4aa4af2a88d..1ad17e292a002 100644 --- a/sklearn/utils/estimator_checks.py +++ b/sklearn/utils/estimator_checks.py @@ -3,7 +3,6 @@ import sys import traceback import pickle -import pytest from copy import deepcopy from functools import partial from inspect import signature From a9572707181d906f755a19d607e1f5abc42cf73f Mon Sep 17 00:00:00 2001 From: Stephane Couvreur Date: Mon, 8 Jul 2019 17:28:48 +0100 Subject: [PATCH 6/7] Updated import statement --- sklearn/tests/test_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sklearn/tests/test_common.py b/sklearn/tests/test_common.py index 855c79349f753..48fecdd05c703 100644 --- a/sklearn/tests/test_common.py +++ b/sklearn/tests/test_common.py @@ -31,7 +31,7 @@ from sklearn.discriminant_analysis import LinearDiscriminantAnalysis from sklearn.utils import IS_PYPY from sklearn.utils.estimator_checks import ( - _yield_all_checks, + yield_all_checks, _safe_tags, set_checking_parameters, check_parameters_default_constructible, From a9db592c99b93706e561964f128286b22fea2fec Mon Sep 17 00:00:00 2001 From: Stephane Couvreur Date: Fri, 26 Jul 2019 17:49:59 +0100 Subject: [PATCH 7/7] Added Evaluate flag to check_estimators() Added bool flag for check_estimators() and updated pytest decorator for test_estimators(). Changes to be committed: modified: tests/test_common.py modified: utils/estimator_checks.py --- sklearn/tests/test_common.py | 11 ++++++++++- sklearn/utils/estimator_checks.py | 29 +++++++++++++++++------------ 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/sklearn/tests/test_common.py b/sklearn/tests/test_common.py index 74fb56796638d..4cf4ca9e310ef 100644 --- a/sklearn/tests/test_common.py +++ b/sklearn/tests/test_common.py @@ -12,6 +12,7 @@ import re import pkgutil import functools +import itertools import pytest @@ -28,6 +29,7 @@ from sklearn.discriminant_analysis import LinearDiscriminantAnalysis from sklearn.utils import IS_PYPY from sklearn.utils.estimator_checks import ( + check_estimator, yield_all_checks, _safe_tags, set_checking_parameters, @@ -95,8 +97,15 @@ def _rename_partial(val): if hasattr(val, "get_params") and not isinstance(val, type): return type(val).__name__ +ALL_ESTIMATORS = all_estimators() -@pytest.mark.estimator_checks(_tested_estimators()) +@pytest.mark.parametrize( + "check, name, estimator", + itertools.chain.from_iterable( + check_estimator(Estimator, evaluate=False) + for Estimator in ALL_ESTIMATORS + ) +) def test_estimators(estimator, check): # Common tests for estimator instances with ignore_warnings(category=(DeprecationWarning, ConvergenceWarning, diff --git a/sklearn/utils/estimator_checks.py b/sklearn/utils/estimator_checks.py index fe80ec7b22be0..39d5dc830c536 100644 --- a/sklearn/utils/estimator_checks.py +++ b/sklearn/utils/estimator_checks.py @@ -265,7 +265,7 @@ def yield_all_checks(name, estimator): yield check_fit_idempotent -def check_estimator(Estimator): +def check_estimator(Estimator, evaluate=True): """Check if estimator adheres to scikit-learn conventions. This estimator will run an extensive test-suite for input validation, @@ -282,18 +282,23 @@ def check_estimator(Estimator): ---------- estimator : estimator object or class Estimator to check. Estimator is a class object or instance. + + evaluate : bool + Flag to indicate whether or not to evaluate the passed + estimator. """ - if isinstance(Estimator, type): - # got a class - name = Estimator.__name__ - estimator = Estimator() - # Generate tests for pytest hooks collector - check_parameters_default_constructible(name, Estimator) - check_no_attributes_set_in_init(name, estimator) - else: - # got an instance - estimator = Estimator - name = type(estimator).__name__ + if evaluate: + if isinstance(Estimator, type): + # got a class + name = Estimator.__name__ + estimator = Estimator() + # Generate tests for pytest hooks collector + check_parameters_default_constructible(name, Estimator) + check_no_attributes_set_in_init(name, estimator) + else: + # got an instance + estimator = Estimator + name = type(estimator).__name__ for check in yield_all_checks(name, estimator): try: