diff --git a/doc/whats_new/v1.4.rst b/doc/whats_new/v1.4.rst index c3fd8fc9923df..703feeb36db3e 100644 --- a/doc/whats_new/v1.4.rst +++ b/doc/whats_new/v1.4.rst @@ -280,6 +280,20 @@ Changelog deprecated and will be removed in 1.6. :pr:`26830` by :user:`Stefanie Senger `. +:mod:`sklearn.inspection` +......................... + +- |Enhancement| :class:`inspection.DecisionBoundaryDisplay` now accepts a parameter + `class_of_interest` to select the class of interest when plotting the response + provided by `response_method="predict_proba"` or + `response_method="decision_function"`. It allows to plot the decision boundary for + both binary and multiclass classifiers. + :pr:`27291` by :user:`Guillaume Lemaitre `. + +- |API| :class:`inspection.DecisionBoundaryDisplay` raise an `AttributeError` instead + of a `ValueError` when an estimator does not implement the requested response method. + :pr:`27291` by :user:`Guillaume Lemaitre `. + :mod:`sklearn.linear_model` ........................... diff --git a/examples/classification/plot_classification_probability.py b/examples/classification/plot_classification_probability.py index ec5887b63914d..4e8f0763d3b47 100644 --- a/examples/classification/plot_classification_probability.py +++ b/examples/classification/plot_classification_probability.py @@ -22,10 +22,12 @@ import matplotlib.pyplot as plt import numpy as np +from matplotlib import cm from sklearn import datasets from sklearn.gaussian_process import GaussianProcessClassifier from sklearn.gaussian_process.kernels import RBF +from sklearn.inspection import DecisionBoundaryDisplay from sklearn.linear_model import LogisticRegression from sklearn.metrics import accuracy_score from sklearn.svm import SVC @@ -56,40 +58,39 @@ n_classifiers = len(classifiers) -plt.figure(figsize=(3 * 2, n_classifiers * 2)) -plt.subplots_adjust(bottom=0.2, top=0.95) - -xx = np.linspace(3, 9, 100) -yy = np.linspace(1, 5, 100).T -xx, yy = np.meshgrid(xx, yy) -Xfull = np.c_[xx.ravel(), yy.ravel()] - -for index, (name, classifier) in enumerate(classifiers.items()): - classifier.fit(X, y) - - y_pred = classifier.predict(X) +fig, axes = plt.subplots( + nrows=n_classifiers, + ncols=len(iris.target_names), + figsize=(3 * 2, n_classifiers * 2), +) +for classifier_idx, (name, classifier) in enumerate(classifiers.items()): + y_pred = classifier.fit(X, y).predict(X) accuracy = accuracy_score(y, y_pred) - print("Accuracy (train) for %s: %0.1f%% " % (name, accuracy * 100)) - - # View probabilities: - probas = classifier.predict_proba(Xfull) - n_classes = np.unique(y_pred).size - for k in range(n_classes): - plt.subplot(n_classifiers, n_classes, index * n_classes + k + 1) - plt.title("Class %d" % k) - if k == 0: - plt.ylabel(name) - imshow_handle = plt.imshow( - probas[:, k].reshape((100, 100)), extent=(3, 9, 1, 5), origin="lower" + print(f"Accuracy (train) for {name}: {accuracy:0.1%}") + for label in np.unique(y): + # plot the probability estimate provided by the classifier + disp = DecisionBoundaryDisplay.from_estimator( + classifier, + X, + response_method="predict_proba", + class_of_interest=label, + ax=axes[classifier_idx, label], + vmin=0, + vmax=1, + ) + axes[classifier_idx, label].set_title(f"Class {label}") + # plot data predicted to belong to given class + mask_y_pred = y_pred == label + axes[classifier_idx, label].scatter( + X[mask_y_pred, 0], X[mask_y_pred, 1], marker="o", c="w", edgecolor="k" ) - plt.xticks(()) - plt.yticks(()) - idx = y_pred == k - if idx.any(): - plt.scatter(X[idx, 0], X[idx, 1], marker="o", c="w", edgecolor="k") + axes[classifier_idx, label].set(xticks=(), yticks=()) + axes[classifier_idx, 0].set_ylabel(name) -ax = plt.axes([0.15, 0.04, 0.7, 0.05]) +ax = plt.axes([0.15, 0.04, 0.7, 0.02]) plt.title("Probability") -plt.colorbar(imshow_handle, cax=ax, orientation="horizontal") +_ = plt.colorbar( + cm.ScalarMappable(norm=None, cmap="viridis"), cax=ax, orientation="horizontal" +) plt.show() diff --git a/sklearn/inspection/_plot/decision_boundary.py b/sklearn/inspection/_plot/decision_boundary.py index 0aea9903b4307..a42e744261e0b 100644 --- a/sklearn/inspection/_plot/decision_boundary.py +++ b/sklearn/inspection/_plot/decision_boundary.py @@ -1,10 +1,9 @@ -from functools import reduce - import numpy as np from ...base import is_regressor from ...preprocessing import LabelEncoder from ...utils import _safe_indexing, check_matplotlib_support +from ...utils._response import _get_response_values from ...utils.validation import ( _is_arraylike_not_scalar, _num_features, @@ -12,8 +11,8 @@ ) -def _check_boundary_response_method(estimator, response_method): - """Return prediction method from the `response_method` for decision boundary. +def _check_boundary_response_method(estimator, response_method, class_of_interest): + """Validate the response methods to be used with the fitted estimator. Parameters ---------- @@ -26,10 +25,16 @@ def _check_boundary_response_method(estimator, response_method): If set to 'auto', the response method is tried in the following order: :term:`decision_function`, :term:`predict_proba`, :term:`predict`. + class_of_interest : int, float, bool, str or None + The class considered when plotting the decision. If the label is specified, it + is then possible to plot the decision boundary in multiclass settings. + + .. versionadded:: 1.4 + Returns ------- - prediction_method: callable - Prediction method of estimator. + prediction_method : list of str or str + The name or list of names of the response methods to use. """ has_classes = hasattr(estimator, "classes_") if has_classes and _is_arraylike_not_scalar(estimator.classes_[0]): @@ -37,25 +42,21 @@ def _check_boundary_response_method(estimator, response_method): raise ValueError(msg) if has_classes and len(estimator.classes_) > 2: - if response_method not in {"auto", "predict"}: + if response_method not in {"auto", "predict"} and class_of_interest is None: msg = ( - "Multiclass classifiers are only supported when response_method is" - " 'predict' or 'auto'" + "Multiclass classifiers are only supported when `response_method` is " + "'predict' or 'auto'. Else you must provide `class_of_interest` to " + "plot the decision boundary of a specific class." ) raise ValueError(msg) - methods_list = ["predict"] + prediction_method = "predict" if response_method == "auto" else response_method elif response_method == "auto": - methods_list = ["decision_function", "predict_proba", "predict"] + if is_regressor(estimator): + prediction_method = "predict" + else: + prediction_method = ["decision_function", "predict_proba", "predict"] else: - methods_list = [response_method] - - prediction_method = [getattr(estimator, method, None) for method in methods_list] - prediction_method = reduce(lambda x, y: x or y, prediction_method) - if prediction_method is None: - raise ValueError( - f"{estimator.__class__.__name__} has none of the following attributes: " - f"{', '.join(methods_list)}." - ) + prediction_method = response_method return prediction_method @@ -206,6 +207,7 @@ def from_estimator( eps=1.0, plot_method="contourf", response_method="auto", + class_of_interest=None, xlabel=None, ylabel=None, ax=None, @@ -248,6 +250,14 @@ def from_estimator( For multiclass problems, :term:`predict` is selected when `response_method="auto"`. + class_of_interest : int, float, bool or str, default=None + The class considered when plotting the decision. If None, + `estimator.classes_[1]` is considered as the positive class + for binary classifiers. For multiclass classifiers, passing + an explicit value for `class_of_interest` is mandatory. + + .. versionadded:: 1.4 + xlabel : str, default=None The label used for the x-axis. If `None`, an attempt is made to extract a label from `X` if it is a dataframe, otherwise an empty @@ -342,11 +352,30 @@ def from_estimator( else: X_grid = np.c_[xx0.ravel(), xx1.ravel()] - pred_func = _check_boundary_response_method(estimator, response_method) - response = pred_func(X_grid) + prediction_method = _check_boundary_response_method( + estimator, response_method, class_of_interest + ) + try: + response, _, response_method_used = _get_response_values( + estimator, + X_grid, + response_method=prediction_method, + pos_label=class_of_interest, + return_response_method_used=True, + ) + except ValueError as exc: + if "is not a valid label" in str(exc): + # re-raise a more informative error message since `pos_label` is unknown + # to our user when interacting with + # `DecisionBoundaryDisplay.from_estimator` + raise ValueError( + f"class_of_interest={class_of_interest} is not a valid label: It " + f"should be one of {estimator.classes_}" + ) from exc + raise # convert classes predictions into integers - if pred_func.__name__ == "predict" and hasattr(estimator, "classes_"): + if response_method_used == "predict" and hasattr(estimator, "classes_"): encoder = LabelEncoder() encoder.classes_ = estimator.classes_ response = encoder.transform(response) @@ -355,8 +384,11 @@ def from_estimator( if is_regressor(estimator): raise ValueError("Multi-output regressors are not supported") - # TODO: Support pos_label - response = response[:, 1] + # For the multiclass case, `_get_response_values` returns the response + # as-is. Thus, we have a column per class and we need to select the column + # corresponding to the positive class. + col_idx = np.flatnonzero(estimator.classes_ == class_of_interest)[0] + response = response[:, col_idx] if xlabel is None: xlabel = X.columns[0] if hasattr(X, "columns") else "" diff --git a/sklearn/inspection/_plot/tests/test_boundary_decision_display.py b/sklearn/inspection/_plot/tests/test_boundary_decision_display.py index 47c21e4521c35..e93534b3b9e13 100644 --- a/sklearn/inspection/_plot/tests/test_boundary_decision_display.py +++ b/sklearn/inspection/_plot/tests/test_boundary_decision_display.py @@ -2,10 +2,10 @@ import numpy as np import pytest -from numpy.testing import assert_allclose from sklearn.base import BaseEstimator, ClassifierMixin from sklearn.datasets import ( + load_diabetes, load_iris, make_classification, make_multilabel_classification, @@ -13,7 +13,12 @@ from sklearn.inspection import DecisionBoundaryDisplay from sklearn.inspection._plot.decision_boundary import _check_boundary_response_method from sklearn.linear_model import LogisticRegression +from sklearn.preprocessing import scale from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor +from sklearn.utils._testing import ( + assert_allclose, + assert_array_equal, +) # TODO: Remove when https://github.com/numpy/numpy/issues/14397 is resolved pytestmark = pytest.mark.filterwarnings( @@ -31,6 +36,12 @@ ) +def load_iris_2d_scaled(): + X, y = load_iris(return_X_y=True) + X = scale(X)[:, :2] + return X, y + + @pytest.fixture(scope="module") def fitted_clf(): return LogisticRegression().fit(X, y) @@ -46,43 +57,73 @@ def test_input_data_dimension(pyplot): DecisionBoundaryDisplay.from_estimator(estimator=clf, X=X) -def test_check_boundary_response_method_auto(): - """Check _check_boundary_response_method behavior with 'auto'.""" - - class A: - def decision_function(self): - pass - - a_inst = A() - method = _check_boundary_response_method(a_inst, "auto") - assert method == a_inst.decision_function - - class B: - def predict_proba(self): - pass +def test_check_boundary_response_method_error(): + """Check that we raise an error for the cases not supported by + `_check_boundary_response_method`. + """ - b_inst = B() - method = _check_boundary_response_method(b_inst, "auto") - assert method == b_inst.predict_proba + class MultiLabelClassifier: + classes_ = [np.array([0, 1]), np.array([0, 1])] - class C: - def predict_proba(self): - pass + err_msg = "Multi-label and multi-output multi-class classifiers are not supported" + with pytest.raises(ValueError, match=err_msg): + _check_boundary_response_method(MultiLabelClassifier(), "predict", None) - def decision_function(self): - pass + class MulticlassClassifier: + classes_ = [0, 1, 2] - c_inst = C() - method = _check_boundary_response_method(c_inst, "auto") - assert method == c_inst.decision_function + err_msg = "Multiclass classifiers are only supported when `response_method` is" + for response_method in ("predict_proba", "decision_function"): + with pytest.raises(ValueError, match=err_msg): + _check_boundary_response_method( + MulticlassClassifier(), response_method, None + ) - class D: - def predict(self): - pass - d_inst = D() - method = _check_boundary_response_method(d_inst, "auto") - assert method == d_inst.predict +@pytest.mark.parametrize( + "estimator, response_method, class_of_interest, expected_prediction_method", + [ + (DecisionTreeRegressor(), "predict", None, "predict"), + (DecisionTreeRegressor(), "auto", None, "predict"), + (LogisticRegression().fit(*load_iris_2d_scaled()), "predict", None, "predict"), + (LogisticRegression().fit(*load_iris_2d_scaled()), "auto", None, "predict"), + ( + LogisticRegression().fit(*load_iris_2d_scaled()), + "predict_proba", + 0, + "predict_proba", + ), + ( + LogisticRegression().fit(*load_iris_2d_scaled()), + "decision_function", + 0, + "decision_function", + ), + ( + LogisticRegression().fit(X, y), + "auto", + None, + ["decision_function", "predict_proba", "predict"], + ), + (LogisticRegression().fit(X, y), "predict", None, "predict"), + ( + LogisticRegression().fit(X, y), + ["predict_proba", "decision_function"], + None, + ["predict_proba", "decision_function"], + ), + ], +) +def test_check_boundary_response_method( + estimator, response_method, class_of_interest, expected_prediction_method +): + """Check the behaviour of `_check_boundary_response_method` for the supported + cases. + """ + prediction_method = _check_boundary_response_method( + estimator, response_method, class_of_interest + ) + assert prediction_method == expected_prediction_method @pytest.mark.parametrize("response_method", ["predict_proba", "decision_function"]) @@ -93,8 +134,8 @@ def test_multiclass_error(pyplot, response_method): lr = LogisticRegression().fit(X, y) msg = ( - "Multiclass classifiers are only supported when response_method is 'predict' or" - " 'auto'" + "Multiclass classifiers are only supported when `response_method` is 'predict'" + " or 'auto'" ) with pytest.raises(ValueError, match=msg): DecisionBoundaryDisplay.from_estimator(lr, X, response_method=response_method) @@ -162,7 +203,9 @@ def test_display_plot_input_error(pyplot, fitted_clf): "response_method", ["auto", "predict", "predict_proba", "decision_function"] ) @pytest.mark.parametrize("plot_method", ["contourf", "contour"]) -def test_decision_boundary_display(pyplot, fitted_clf, response_method, plot_method): +def test_decision_boundary_display_classifier( + pyplot, fitted_clf, response_method, plot_method +): """Check that decision boundary is correct.""" fig, ax = pyplot.subplots() eps = 2.0 @@ -197,6 +240,45 @@ def test_decision_boundary_display(pyplot, fitted_clf, response_method, plot_met assert disp.figure_ == fig2 +@pytest.mark.parametrize("response_method", ["auto", "predict"]) +@pytest.mark.parametrize("plot_method", ["contourf", "contour"]) +def test_decision_boundary_display_regressor(pyplot, response_method, plot_method): + """Check that we can display the decision boundary for a regressor.""" + X, y = load_diabetes(return_X_y=True) + X = X[:, :2] + tree = DecisionTreeRegressor().fit(X, y) + fig, ax = pyplot.subplots() + eps = 2.0 + disp = DecisionBoundaryDisplay.from_estimator( + tree, + X, + response_method=response_method, + ax=ax, + eps=eps, + plot_method=plot_method, + ) + assert isinstance(disp.surface_, pyplot.matplotlib.contour.QuadContourSet) + assert disp.ax_ == ax + assert disp.figure_ == fig + + x0, x1 = X[:, 0], X[:, 1] + + x0_min, x0_max = x0.min() - eps, x0.max() + eps + x1_min, x1_max = x1.min() - eps, x1.max() + eps + + assert disp.xx0.min() == pytest.approx(x0_min) + assert disp.xx0.max() == pytest.approx(x0_max) + assert disp.xx1.min() == pytest.approx(x1_min) + assert disp.xx1.max() == pytest.approx(x1_max) + + fig2, ax2 = pyplot.subplots() + # change plotting method for second plot + disp.plot(plot_method="pcolormesh", ax=ax2, shading="auto") + assert isinstance(disp.surface_, pyplot.matplotlib.collections.QuadMesh) + assert disp.ax_ == ax2 + assert disp.figure_ == fig2 + + @pytest.mark.parametrize( "response_method, msg", [ @@ -232,7 +314,7 @@ def fit(self, X, y): clf = MyClassifier().fit(X, y) - with pytest.raises(ValueError, match=msg): + with pytest.raises(AttributeError, match=msg): DecisionBoundaryDisplay.from_estimator(clf, X, response_method=response_method) @@ -274,7 +356,21 @@ def test_multioutput_regressor_error(pyplot): y = np.asarray([[0, 1], [4, 1]]) tree = DecisionTreeRegressor().fit(X, y) with pytest.raises(ValueError, match="Multi-output regressors are not supported"): - DecisionBoundaryDisplay.from_estimator(tree, X) + DecisionBoundaryDisplay.from_estimator(tree, X, response_method="predict") + + +@pytest.mark.parametrize( + "response_method", + ["predict_proba", "decision_function", ["predict_proba", "predict"]], +) +def test_regressor_unsupported_response(pyplot, response_method): + """Check that we can display the decision boundary for a regressor.""" + X, y = load_diabetes(return_X_y=True) + X = X[:, :2] + tree = DecisionTreeRegressor().fit(X, y) + err_msg = "should either be a classifier to be used with response_method" + with pytest.raises(ValueError, match=err_msg): + DecisionBoundaryDisplay.from_estimator(tree, X, response_method=response_method) @pytest.mark.filterwarnings( @@ -353,3 +449,111 @@ def test_dataframe_support(pyplot): # no warnings linked to feature names validation should be raised warnings.simplefilter("error", UserWarning) DecisionBoundaryDisplay.from_estimator(estimator, df, response_method="predict") + + +@pytest.mark.parametrize("response_method", ["predict_proba", "decision_function"]) +def test_class_of_interest_binary(pyplot, response_method): + """Check the behaviour of passing `class_of_interest` for plotting the output of + `predict_proba` and `decision_function` in the binary case. + """ + iris = load_iris() + X = iris.data[:100, :2] + y = iris.target[:100] + assert_array_equal(np.unique(y), [0, 1]) + + estimator = LogisticRegression().fit(X, y) + # We will check that `class_of_interest=None` is equivalent to + # `class_of_interest=estimator.classes_[1]` + disp_default = DecisionBoundaryDisplay.from_estimator( + estimator, + X, + response_method=response_method, + class_of_interest=None, + ) + disp_class_1 = DecisionBoundaryDisplay.from_estimator( + estimator, + X, + response_method=response_method, + class_of_interest=estimator.classes_[1], + ) + + assert_allclose(disp_default.response, disp_class_1.response) + + # we can check that `_get_response_values` modifies the response when targeting + # the other class, i.e. 1 - p(y=1|x) for `predict_proba` and -decision_function + # for `decision_function`. + disp_class_0 = DecisionBoundaryDisplay.from_estimator( + estimator, + X, + response_method=response_method, + class_of_interest=estimator.classes_[0], + ) + + if response_method == "predict_proba": + assert_allclose(disp_default.response, 1 - disp_class_0.response) + else: + assert response_method == "decision_function" + assert_allclose(disp_default.response, -disp_class_0.response) + + +@pytest.mark.parametrize("response_method", ["predict_proba", "decision_function"]) +def test_class_of_interest_multiclass(pyplot, response_method): + """Check the behaviour of passing `class_of_interest` for plotting the output of + `predict_proba` and `decision_function` in the multiclass case. + """ + iris = load_iris() + X = iris.data[:, :2] + y = iris.target # the target are numerical labels + class_of_interest_idx = 2 + + estimator = LogisticRegression().fit(X, y) + disp = DecisionBoundaryDisplay.from_estimator( + estimator, + X, + response_method=response_method, + class_of_interest=class_of_interest_idx, + ) + + # we will check that we plot the expected values as response + grid = np.concatenate([disp.xx0.reshape(-1, 1), disp.xx1.reshape(-1, 1)], axis=1) + response = getattr(estimator, response_method)(grid)[:, class_of_interest_idx] + assert_allclose(response.reshape(*disp.response.shape), disp.response) + + # make the same test but this time using target as strings + y = iris.target_names[iris.target] + estimator = LogisticRegression().fit(X, y) + + disp = DecisionBoundaryDisplay.from_estimator( + estimator, + X, + response_method=response_method, + class_of_interest=iris.target_names[class_of_interest_idx], + ) + + grid = np.concatenate([disp.xx0.reshape(-1, 1), disp.xx1.reshape(-1, 1)], axis=1) + response = getattr(estimator, response_method)(grid)[:, class_of_interest_idx] + assert_allclose(response.reshape(*disp.response.shape), disp.response) + + # check that we raise an error for unknown labels + # this test should already be handled in `_get_response_values` but we can have this + # test here as well + err_msg = "class_of_interest=2 is not a valid label: It should be one of" + with pytest.raises(ValueError, match=err_msg): + DecisionBoundaryDisplay.from_estimator( + estimator, + X, + response_method=response_method, + class_of_interest=class_of_interest_idx, + ) + + # TODO: remove this test when we handle multiclass with class_of_interest=None + # by showing the max of the decision function or the max of the predicted + # probabilities. + err_msg = "Multiclass classifiers are only supported" + with pytest.raises(ValueError, match=err_msg): + DecisionBoundaryDisplay.from_estimator( + estimator, + X, + response_method=response_method, + class_of_interest=None, + ) diff --git a/sklearn/utils/_response.py b/sklearn/utils/_response.py index a8504099eb194..2e68414565aee 100644 --- a/sklearn/utils/_response.py +++ b/sklearn/utils/_response.py @@ -114,6 +114,7 @@ def _get_response_values( X, response_method, pos_label=None, + return_response_method_used=False, ): """Compute the response values of a classifier or a regressor. @@ -156,6 +157,12 @@ def _get_response_values( the metrics. By default, `estimators.classes_[1]` is considered as the positive class. + return_response_method_used : bool, default=False + Whether to return the response method used to compute the response + values. + + .. versionadded:: 1.4 + Returns ------- y_pred : ndarray of shape (n_samples,), (n_samples, n_classes) or \ @@ -167,6 +174,12 @@ def _get_response_values( The class considered as the positive class when computing the metrics. Returns `None` if `estimator` is a regressor. + response_method_used : str + The response method used to compute the response values. Only returned + if `return_response_method_used` is `True`. + + .. versionadded:: 1.4 + Raises ------ ValueError @@ -215,8 +228,11 @@ def _get_response_values( "should be 'predict'. Got a regressor with response_method=" f"{response_method} instead." ) - y_pred, pos_label = estimator.predict(X), None + prediction_method = estimator.predict + y_pred, pos_label = prediction_method(X), None + if return_response_method_used: + return y_pred, pos_label, prediction_method.__name__ return y_pred, pos_label diff --git a/sklearn/utils/tests/test_response.py b/sklearn/utils/tests/test_response.py index e715e94cc60b1..a2c0996d28bc3 100644 --- a/sklearn/utils/tests/test_response.py +++ b/sklearn/utils/tests/test_response.py @@ -35,17 +35,21 @@ def test_get_response_values_regressor_error(response_method): _get_response_values(my_estimator, X, response_method=response_method) -def test_get_response_values_regressor(): +@pytest.mark.parametrize("return_response_method_used", [True, False]) +def test_get_response_values_regressor(return_response_method_used): """Check the behaviour of `_get_response_values` with regressor.""" X, y = make_regression(n_samples=10, random_state=0) regressor = LinearRegression().fit(X, y) - y_pred, pos_label = _get_response_values( + results = _get_response_values( regressor, X, response_method="predict", + return_response_method_used=return_response_method_used, ) - assert_array_equal(y_pred, regressor.predict(X)) - assert pos_label is None + assert_array_equal(results[0], regressor.predict(X)) + assert results[1] is None + if return_response_method_used: + assert results[2] == "predict" @pytest.mark.parametrize( @@ -84,7 +88,10 @@ def test_get_response_values_classifier_inconsistent_y_pred_for_binary_proba(): _get_response_values(classifier, X, response_method="predict_proba") -def test_get_response_values_binary_classifier_decision_function(): +@pytest.mark.parametrize("return_response_method_used", [True, False]) +def test_get_response_values_binary_classifier_decision_function( + return_response_method_used, +): """Check the behaviour of `_get_response_values` with `decision_function` and binary classifier.""" X, y = make_classification( @@ -97,27 +104,36 @@ def test_get_response_values_binary_classifier_decision_function(): response_method = "decision_function" # default `pos_label` - y_pred, pos_label = _get_response_values( + results = _get_response_values( classifier, X, response_method=response_method, pos_label=None, + return_response_method_used=return_response_method_used, ) - assert_allclose(y_pred, classifier.decision_function(X)) - assert pos_label == 1 + assert_allclose(results[0], classifier.decision_function(X)) + assert results[1] == 1 + if return_response_method_used: + assert results[2] == "decision_function" # when forcing `pos_label=classifier.classes_[0]` - y_pred, pos_label = _get_response_values( + results = _get_response_values( classifier, X, response_method=response_method, pos_label=classifier.classes_[0], + return_response_method_used=return_response_method_used, ) - assert_allclose(y_pred, classifier.decision_function(X) * -1) - assert pos_label == 0 + assert_allclose(results[0], classifier.decision_function(X) * -1) + assert results[1] == 0 + if return_response_method_used: + assert results[2] == "decision_function" -def test_get_response_values_binary_classifier_predict_proba(): +@pytest.mark.parametrize("return_response_method_used", [True, False]) +def test_get_response_values_binary_classifier_predict_proba( + return_response_method_used, +): """Check that `_get_response_values` with `predict_proba` and binary classifier.""" X, y = make_classification( @@ -130,21 +146,28 @@ def test_get_response_values_binary_classifier_predict_proba(): response_method = "predict_proba" # default `pos_label` - y_pred, pos_label = _get_response_values( + results = _get_response_values( classifier, X, response_method=response_method, pos_label=None, + return_response_method_used=return_response_method_used, ) - assert_allclose(y_pred, classifier.predict_proba(X)[:, 1]) - assert pos_label == 1 + assert_allclose(results[0], classifier.predict_proba(X)[:, 1]) + assert results[1] == 1 + if return_response_method_used: + assert len(results) == 3 + assert results[2] == "predict_proba" + else: + assert len(results) == 2 # when forcing `pos_label=classifier.classes_[0]` - y_pred, pos_label = _get_response_values( + y_pred, pos_label, *_ = _get_response_values( classifier, X, response_method=response_method, pos_label=classifier.classes_[0], + return_response_method_used=return_response_method_used, ) assert_allclose(y_pred, classifier.predict_proba(X)[:, 0]) assert pos_label == 0 @@ -190,13 +213,13 @@ def test_get_response_predict_proba(): y_proba, pos_label = _get_response_values_binary( classifier, X_binary, response_method="predict_proba" ) - np.testing.assert_allclose(y_proba, classifier.predict_proba(X_binary)[:, 1]) + assert_allclose(y_proba, classifier.predict_proba(X_binary)[:, 1]) assert pos_label == 1 y_proba, pos_label = _get_response_values_binary( classifier, X_binary, response_method="predict_proba", pos_label=0 ) - np.testing.assert_allclose(y_proba, classifier.predict_proba(X_binary)[:, 0]) + assert_allclose(y_proba, classifier.predict_proba(X_binary)[:, 0]) assert pos_label == 0 @@ -206,13 +229,13 @@ def test_get_response_decision_function(): y_score, pos_label = _get_response_values_binary( classifier, X_binary, response_method="decision_function" ) - np.testing.assert_allclose(y_score, classifier.decision_function(X_binary)) + assert_allclose(y_score, classifier.decision_function(X_binary)) assert pos_label == 1 y_score, pos_label = _get_response_values_binary( classifier, X_binary, response_method="decision_function", pos_label=0 ) - np.testing.assert_allclose(y_score, classifier.decision_function(X_binary) * -1) + assert_allclose(y_score, classifier.decision_function(X_binary) * -1) assert pos_label == 0 @@ -238,6 +261,33 @@ def test_get_response_values_multiclass(estimator, response_method): assert np.logical_and(predictions >= 0, predictions <= 1).all() +def test_get_response_values_with_response_list(): + """Check the behaviour of passing a list of responses to `_get_response_values`.""" + classifier = LogisticRegression().fit(X_binary, y_binary) + + # it should use `predict_proba` + y_pred, pos_label, response_method = _get_response_values( + classifier, + X_binary, + response_method=["predict_proba", "decision_function"], + return_response_method_used=True, + ) + assert_allclose(y_pred, classifier.predict_proba(X_binary)[:, 1]) + assert pos_label == 1 + assert response_method == "predict_proba" + + # it should use `decision_function` + y_pred, pos_label, response_method = _get_response_values( + classifier, + X_binary, + response_method=["decision_function", "predict_proba"], + return_response_method_used=True, + ) + assert_allclose(y_pred, classifier.decision_function(X_binary)) + assert pos_label == 1 + assert response_method == "decision_function" + + @pytest.mark.parametrize( "response_method", ["predict_proba", "decision_function", "predict"] )