diff --git a/sklearn/feature_selection/rfe.py b/sklearn/feature_selection/rfe.py index 01c99ceb526f4..4b98106fda483 100644 --- a/sklearn/feature_selection/rfe.py +++ b/sklearn/feature_selection/rfe.py @@ -6,6 +6,7 @@ """Recursive feature elimination for feature ranking""" +from functools import wraps import numpy as np from ..utils import check_arrays, safe_sqr from ..base import BaseEstimator @@ -36,6 +37,7 @@ class RFE(BaseEstimator, MetaEstimatorMixin, SelectorMixin): A supervised learning estimator with a `fit` method that updates a `coef_` attribute that holds the fitted parameters. Important features must correspond to high absolute values in the `coef_` array. + The estimator must also implement a `score` method. For instance, this is the case for most supervised learning algorithms such as Support Vector Classifiers and Generalized @@ -169,7 +171,13 @@ def fit(self, X, y): return self - def predict(self, X): + def _delegate_wrapper(self, delegate): + def wrapper(X, *args, **kwargs): + return delegate(self.transform(X), *args, **kwargs) + return wrapper + + @property + def predict(self): """Reduce X to the selected features and then predict using the underlying estimator. @@ -183,9 +191,10 @@ def predict(self, X): y : array of shape [n_samples] The predicted target values. """ - return self.estimator_.predict(self.transform(X)) + return self._delegate_wrapper(self.estimator_.predict) - def score(self, X, y): + @property + def score(self): """Reduce X to the selected features and then return the score of the underlying estimator. @@ -197,16 +206,22 @@ def score(self, X, y): y : array of shape [n_samples] The target values. """ - return self.estimator_.score(self.transform(X), y) + return self._delegate_wrapper(self.estimator_.score) def _get_support_mask(self): return self.support_ - def decision_function(self, X): - return self.estimator_.decision_function(self.transform(X)) + @property + def decision_function(self): + return self._delegate_wrapper(self.estimator_.decision_function) + + @property + def predict_proba(self): + return self._delegate_wrapper(self.estimator_.predict_proba) - def predict_proba(self, X): - return self.estimator_.predict_proba(self.transform(X)) + @property + def predict_log_proba(self): + return self._delegate_wrapper(self.estimator_.predict_log_proba) class RFECV(RFE, MetaEstimatorMixin): diff --git a/sklearn/grid_search.py b/sklearn/grid_search.py index 8d217521f1269..2904030e77c4a 100644 --- a/sklearn/grid_search.py +++ b/sklearn/grid_search.py @@ -325,20 +325,51 @@ def score(self, X, y=None): @property def predict(self): + """Call predict on the best estimator""" return self.best_estimator_.predict @property def predict_proba(self): + """Call predict_proba on the best estimator""" return self.best_estimator_.predict_proba + @property + def predict_log_proba(self): + """Call predict_log_proba on the best estimator""" + return self.best_estimator_.predict_log_proba + @property def decision_function(self): + """Call decision_function on the best estimator""" return self.best_estimator_.decision_function @property def transform(self): + """Call transform on the best estimator""" return self.best_estimator_.transform + @property + def inverse_transform(self): + """Call inverse_transform on the best estimator""" + return self.best_estimator_.inverse_transform + + def _check_estimator(self): + """Check that estimator can be fitted and score can be computed.""" + if (not hasattr(self.estimator, 'fit') or + not (hasattr(self.estimator, 'predict') + or hasattr(self.estimator, 'score'))): + raise TypeError("estimator should a be an estimator implementing" + " 'fit' and 'predict' or 'score' methods," + " %s (type %s) was passed" % + (self.estimator, type(self.estimator))) + if (self.scoring is None and self.loss_func is None and self.score_func + is None): + if not hasattr(self.estimator, 'score'): + raise TypeError( + "If no scoring is specified, the estimator passed " + "should have a 'score' method. The estimator %s " + "does not." % self.estimator) + def _fit(self, X, y, parameter_iterable): """Actual fitting, performing the search over parameters.""" diff --git a/sklearn/pipeline.py b/sklearn/pipeline.py index e91faba0d61f4..e361f4ce2af3e 100644 --- a/sklearn/pipeline.py +++ b/sklearn/pipeline.py @@ -11,6 +11,8 @@ from collections import defaultdict +from functools import partial + import numpy as np from scipy import sparse @@ -78,7 +80,7 @@ class Pipeline(BaseEstimator): # BaseEstimator interface def __init__(self, steps): - self.named_steps = dict(steps) + self.steps = steps names, estimators = zip(*steps) if len(self.named_steps) != len(steps): raise ValueError("Names provided are not unique: %s" % (names,)) @@ -110,6 +112,19 @@ def get_params(self, deep=True): out['%s__%s' % (name, key)] = value return out + @property + def named_steps(self): + return dict(self.steps) + + @property + def _transforms(self): + """Non-final estimators in (name, est) tuples.""" + return self.steps[:-1] + + @property + def _final_estimator(self): + return self.steps[-1][1] + # Estimator interface def _pre_transform(self, X, y=None, **fit_params): @@ -118,7 +133,9 @@ def _pre_transform(self, X, y=None, **fit_params): step, param = pname.split('__', 1) fit_params_steps[step][param] = pval Xt = X - for name, transform in self.steps[:-1]: + for name, transform in self._transforms: + if transform is None: + continue if hasattr(transform, "fit_transform"): Xt = transform.fit_transform(Xt, y, **fit_params_steps[name]) else: @@ -129,79 +146,192 @@ def _pre_transform(self, X, y=None, **fit_params): def fit(self, X, y=None, **fit_params): """Fit all the transforms one after the other and transform the data, then fit the transformed data using the final estimator. + + Parameters + ---------- + X : array-like, shape = [n_samples, n_features] + Training data, where n_samples in the number of samples and + n_features is the number of features. + y : array-like, shape = [n_samples], optional + Target vector relative to X for classification; + None for unsupervised learning. + fit_params : dict of string -> object + Parameters passed to the `fit` method of each step, where + each parameter name is prefixed such that parameter ``p`` for step + ``s`` has key ``s__p``. """ Xt, fit_params = self._pre_transform(X, y, **fit_params) - self.steps[-1][-1].fit(Xt, y, **fit_params) + self._final_estimator.fit(Xt, y, **fit_params) return self - def fit_transform(self, X, y=None, **fit_params): - """Fit all the transforms one after the other and transform the + @property + def fit_transform(self): + """Pipeline.fit_transform(X, y=None, **fit_params) + + Fit all the transforms one after the other and transform the data, then use fit_transform on transformed data using the final - estimator.""" - Xt, fit_params = self._pre_transform(X, y, **fit_params) - if hasattr(self.steps[-1][-1], 'fit_transform'): - return self.steps[-1][-1].fit_transform(Xt, y, **fit_params) - else: - return self.steps[-1][-1].fit(Xt, y, **fit_params).transform(Xt) + estimator. - def predict(self, X): - """Applies transforms to the data, and the predict method of the - final estimator. Valid only if the final estimator implements - predict.""" + Parameters + ---------- + X : array-like, shape = [n_samples, n_features] + Training data, where n_samples in the number of samples and + n_features is the number of features. + y : array-like, shape = [n_samples], optional + Target vector relative to X for classification; + None for unsupervised learning. + fit_params : dict of string -> object + Parameters passed to the `fit` method of each step, where + each parameter name is prefixed such that parameter ``p`` for step + ``s`` has key ``s__p``. + """ + last_step = self._final_estimator + if ( + not hasattr(last_step, 'fit_transform') + and not hasattr(last_step, 'transform')): + raise AttributeError( + 'last step has neither `transform` nor `fit_transform`') + + def fn(X, y=None, **fit_params): + Xt, fit_params = self._pre_transform(X, y, **fit_params) + if hasattr(last_step, 'fit_transform'): + return last_step.fit_transform(Xt, y, **fit_params) + else: + return last_step.fit(Xt, y, **fit_params).transform(Xt) + return fn + + def _run_pipeline(self, est_fn, X, *args, **kwargs): Xt = X - for name, transform in self.steps[:-1]: - Xt = transform.transform(Xt) - return self.steps[-1][-1].predict(Xt) + for name, transform in self._transforms: + if transform is not None: + Xt = transform.transform(Xt) + return est_fn(Xt, *args, **kwargs) - def predict_proba(self, X): - """Applies transforms to the data, and the predict_proba method of the + @property + def predict(self): + """Pipeline.predict(X) + + Applies transforms to the data, and the `predict` method of the final estimator. Valid only if the final estimator implements - predict_proba.""" - Xt = X - for name, transform in self.steps[:-1]: - Xt = transform.transform(Xt) - return self.steps[-1][-1].predict_proba(Xt) + predict. + + Parameters + ---------- + X : array-like, shape = [n_samples, n_features] + Data samples, where n_samples in the number of samples and + n_features is the number of features. + """ + return partial(self._run_pipeline, self._final_estimator.predict) - def decision_function(self, X): - """Applies transforms to the data, and the decision_function method of + @property + def predict_proba(self): + """Pipeline.predict_proba(X) + + Applies transforms to the data, and the `predict_proba` method of the final estimator. Valid only if the final estimator implements - decision_function.""" - Xt = X - for name, transform in self.steps[:-1]: - Xt = transform.transform(Xt) - return self.steps[-1][-1].decision_function(Xt) + predict_proba. - def predict_log_proba(self, X): - Xt = X - for name, transform in self.steps[:-1]: - Xt = transform.transform(Xt) - return self.steps[-1][-1].predict_log_proba(Xt) + Parameters + ---------- + X : array-like, shape = [n_samples, n_features] + Data samples, where n_samples in the number of samples and + n_features is the number of features. + """ + return partial(self._run_pipeline, self._final_estimator.predict_proba) - def transform(self, X): - """Applies transforms to the data, and the transform method of the + @property + def predict_log_proba(self): + """Pipeline.predict_log_proba(X) + + Applies transforms to the data, and the `predict_log_proba` method + of the final estimator. Valid only if the final estimator implements + predict_log_proba. + + Parameters + ---------- + X : array-like, shape = [n_samples, n_features] + Data samples, where n_samples in the number of samples and + n_features is the number of features. + """ + return partial(self._run_pipeline, + self._final_estimator.predict_log_proba) + + @property + def decision_function(self): + """Pipeline.decision_function(X) + + Applies transforms to the data, and the `decision_function` method + of the final estimator. Valid only if the final estimator implements + decision_function. + + Parameters + ---------- + X : array-like, shape = [n_samples, n_features] + Data samples, where n_samples in the number of samples and + n_features is the number of features. + """ + return partial(self._run_pipeline, + self._final_estimator.decision_function) + + @property + def transform(self): + """Pipeline.transform(X) + + Applies transforms to the data, and the `transform` method of the final estimator. Valid only if the final estimator implements - transform.""" - Xt = X - for name, transform in self.steps: - Xt = transform.transform(Xt) - return Xt + transform. - def inverse_transform(self, X): - if X.ndim == 1: - X = X[None, :] - Xt = X - for name, step in self.steps[::-1]: - Xt = step.inverse_transform(Xt) - return Xt + Parameters + ---------- + X : array-like, shape = [n_samples, n_features] + Data samples, where n_samples in the number of samples and + n_features is the number of features. + """ + return partial(self._run_pipeline, self._final_estimator.transform) + + @property + def inverse_transform(self): + """Pipeline.inverse_transform(X) + + Applies inverse transforms to the data from the last step to the + first. + + Parameters + ---------- + X : array-like, shape = [n_samples, n_features] + Data samples, where n_samples in the number of samples and + n_features is the number of features. + """ + inverse_transforms = [step.inverse_transform + for name, step in self.steps[::-1] if step is not None] + + def fn(X): + if X.ndim == 1: + X = X[None, :] + Xt = X + for inv_transform in inverse_transforms: + Xt = inv_transform(Xt) + return Xt + return fn - def score(self, X, y=None): - """Applies transforms to the data, and the score method of the + @property + def score(self): + """Pipeline.score(X, y=None) + + Applies transforms to the data, and the `score` method of the final estimator. Valid only if the final estimator implements - score.""" - Xt = X - for name, transform in self.steps[:-1]: - Xt = transform.transform(Xt) - return self.steps[-1][-1].score(Xt, y) + score. + + Parameters + ---------- + X : array-like, shape = [n_samples, n_features] + Data samples, where n_samples in the number of samples and + n_features is the number of features. + y : array-like, shape = [n_samples], optional + Target vector relative to X; + None for unsupervised learning. + """ + return partial(self._run_pipeline, self._final_estimator.score) @property def _pairwise(self): diff --git a/sklearn/tests/test_metaestimators.py b/sklearn/tests/test_metaestimators.py new file mode 100644 index 0000000000000..0a36125c07b2a --- /dev/null +++ b/sklearn/tests/test_metaestimators.py @@ -0,0 +1,116 @@ +"""Common tests for metaestimators""" + +import functools + +import numpy as np + +from sklearn.base import BaseEstimator +from sklearn.externals.six import iterkeys +from sklearn.datasets import make_classification +from sklearn.utils.testing import assert_true, assert_false +from sklearn.pipeline import Pipeline +from sklearn.grid_search import GridSearchCV, RandomizedSearchCV +from sklearn.feature_selection import RFE, RFECV + + +class DelegatorData(object): + def __init__(self, name, construct, skip_methods=(), + fit_args=make_classification()): + self.name = name + self.construct = construct + self.fit_args = fit_args + self.skip_methods = skip_methods + + +DELEGATING_METAESTIMATORS = [ + DelegatorData('Pipeline', lambda est: Pipeline([('est', est)])), + DelegatorData('GridSearchCV', + lambda est: GridSearchCV( + est, param_grid={'param': [5]}, cv=2), + skip_methods=['score']), + DelegatorData('RandomizedSearchCV', + lambda est: RandomizedSearchCV( + est, param_distributions={'param': [5]}, cv=2), + skip_methods=['score']), + DelegatorData('RFE', RFE, + skip_methods=['transform', 'inverse_transform', 'score']), + DelegatorData('RFECV', RFECV, + skip_methods=['transform', 'inverse_transform', 'score']), +] + + +def test_metaestimator_delegation(): + """Ensures specified metaestimators have methods iff subestimator does""" + def hides(method): + @property + def wrapper(obj): + if obj.hidden_method == method.__name__: + raise AttributeError('%r is hidden' % obj.hidden_method) + return functools.partial(method, obj) + return wrapper + + class SubEstimator(BaseEstimator): + def __init__(self, param=1, hidden_method=None): + self.param = param + self.hidden_method = hidden_method + + def fit(self, X, y=None, *args, **kwargs): + self.coef_ = np.arange(X.shape[1]) + return True + + @hides + def inverse_transform(self, X, *args, **kwargs): + return X + + @hides + def transform(self, X, *args, **kwargs): + return X + + @hides + def predict(self, X, *args, **kwargs): + return np.ones(X.shape[0]) + + @hides + def predict_proba(self, X, *args, **kwargs): + return np.ones(X.shape[0]) + + @hides + def predict_log_proba(self, X, *args, **kwargs): + return np.ones(X.shape[0]) + + @hides + def decision_function(self, X, *args, **kwargs): + return np.ones(X.shape[0]) + + @hides + def score(self, X, *args, **kwargs): + return 1.0 + + methods = [k for k in iterkeys(SubEstimator.__dict__) + if not k.startswith('_') and not k.startswith('fit')] + methods.sort() + + for delegator_data in DELEGATING_METAESTIMATORS: + delegate = SubEstimator() + delegator = delegator_data.construct(delegate) + delegator.fit(*delegator_data.fit_args) + for method in methods: + if method in delegator_data.skip_methods: + continue + assert_true(hasattr(delegate, method)) + assert_true(hasattr(delegator, method), + msg="%s does not have method %r when its delegate does" + % (delegator_data.name, method)) + # smoke test delegation + getattr(delegator, method)(delegator_data.fit_args[0]) + + for method in methods: + if method in delegator_data.skip_methods: + continue + delegate = SubEstimator(hidden_method=method) + delegator = delegator_data.construct(delegate) + delegator.fit(*delegator_data.fit_args) + assert_false(hasattr(delegate, method)) + assert_false(hasattr(delegator, method), + msg="%s has method %r when its delegate does not" + % (delegator_data.name, method))