diff --git a/sklearn/feature_selection/rfe.py b/sklearn/feature_selection/rfe.py
index 01c99ceb526f4..4b98106fda483 100644
--- a/sklearn/feature_selection/rfe.py
+++ b/sklearn/feature_selection/rfe.py
@@ -6,6 +6,7 @@
 
 """Recursive feature elimination for feature ranking"""
 
+from functools import wraps
 import numpy as np
 from ..utils import check_arrays, safe_sqr
 from ..base import BaseEstimator
@@ -36,6 +37,7 @@ class RFE(BaseEstimator, MetaEstimatorMixin, SelectorMixin):
         A supervised learning estimator with a `fit` method that updates a
         `coef_` attribute that holds the fitted parameters. Important features
         must correspond to high absolute values in the `coef_` array.
+        The estimator must also implement a `score` method.
 
         For instance, this is the case for most supervised learning
         algorithms such as Support Vector Classifiers and Generalized
@@ -169,7 +171,13 @@ def fit(self, X, y):
 
         return self
 
-    def predict(self, X):
+    def _delegate_wrapper(self, delegate):
+        def wrapper(X, *args, **kwargs):
+            return delegate(self.transform(X), *args, **kwargs)
+        return wrapper
+
+    @property
+    def predict(self):
         """Reduce X to the selected features and then predict using the
            underlying estimator.
 
@@ -183,9 +191,10 @@ def predict(self, X):
         y : array of shape [n_samples]
             The predicted target values.
         """
-        return self.estimator_.predict(self.transform(X))
+        return self._delegate_wrapper(self.estimator_.predict)
 
-    def score(self, X, y):
+    @property
+    def score(self):
         """Reduce X to the selected features and then return the score of the
            underlying estimator.
 
@@ -197,16 +206,22 @@ def score(self, X, y):
         y : array of shape [n_samples]
             The target values.
         """
-        return self.estimator_.score(self.transform(X), y)
+        return self._delegate_wrapper(self.estimator_.score)
 
     def _get_support_mask(self):
         return self.support_
 
-    def decision_function(self, X):
-        return self.estimator_.decision_function(self.transform(X))
+    @property
+    def decision_function(self):
+        return self._delegate_wrapper(self.estimator_.decision_function)
+
+    @property
+    def predict_proba(self):
+        return self._delegate_wrapper(self.estimator_.predict_proba)
 
-    def predict_proba(self, X):
-        return self.estimator_.predict_proba(self.transform(X))
+    @property
+    def predict_log_proba(self):
+        return self._delegate_wrapper(self.estimator_.predict_log_proba)
 
 
 class RFECV(RFE, MetaEstimatorMixin):
diff --git a/sklearn/grid_search.py b/sklearn/grid_search.py
index 8d217521f1269..2904030e77c4a 100644
--- a/sklearn/grid_search.py
+++ b/sklearn/grid_search.py
@@ -325,20 +325,51 @@ def score(self, X, y=None):
 
     @property
     def predict(self):
+        """Call predict on the best estimator"""
         return self.best_estimator_.predict
 
     @property
     def predict_proba(self):
+        """Call predict_proba on the best estimator"""
         return self.best_estimator_.predict_proba
 
+    @property
+    def predict_log_proba(self):
+        """Call predict_log_proba on the best estimator"""
+        return self.best_estimator_.predict_log_proba
+
     @property
     def decision_function(self):
+        """Call decision_function on the best estimator"""
         return self.best_estimator_.decision_function
 
     @property
     def transform(self):
+        """Call transform on the best estimator"""
         return self.best_estimator_.transform
 
+    @property
+    def inverse_transform(self):
+        """Call inverse_transform on the best estimator"""
+        return self.best_estimator_.inverse_transform
+
+    def _check_estimator(self):
+        """Check that estimator can be fitted and score can be computed."""
+        if (not hasattr(self.estimator, 'fit') or
+                not (hasattr(self.estimator, 'predict')
+                     or hasattr(self.estimator, 'score'))):
+            raise TypeError("estimator should a be an estimator implementing"
+                            " 'fit' and 'predict' or 'score' methods,"
+                            " %s (type %s) was passed" %
+                            (self.estimator, type(self.estimator)))
+        if (self.scoring is None and self.loss_func is None and self.score_func
+                is None):
+            if not hasattr(self.estimator, 'score'):
+                raise TypeError(
+                    "If no scoring is specified, the estimator passed "
+                    "should have a 'score' method. The estimator %s "
+                    "does not." % self.estimator)
+
     def _fit(self, X, y, parameter_iterable):
         """Actual fitting,  performing the search over parameters."""
 
diff --git a/sklearn/pipeline.py b/sklearn/pipeline.py
index e91faba0d61f4..e361f4ce2af3e 100644
--- a/sklearn/pipeline.py
+++ b/sklearn/pipeline.py
@@ -11,6 +11,8 @@
 
 from collections import defaultdict
 
+from functools import partial
+
 import numpy as np
 from scipy import sparse
 
@@ -78,7 +80,7 @@ class Pipeline(BaseEstimator):
     # BaseEstimator interface
 
     def __init__(self, steps):
-        self.named_steps = dict(steps)
+        self.steps = steps
         names, estimators = zip(*steps)
         if len(self.named_steps) != len(steps):
             raise ValueError("Names provided are not unique: %s" % (names,))
@@ -110,6 +112,19 @@ def get_params(self, deep=True):
                     out['%s__%s' % (name, key)] = value
             return out
 
+    @property
+    def named_steps(self):
+        return dict(self.steps)
+
+    @property
+    def _transforms(self):
+        """Non-final estimators in (name, est) tuples."""
+        return self.steps[:-1]
+
+    @property
+    def _final_estimator(self):
+        return self.steps[-1][1]
+
     # Estimator interface
 
     def _pre_transform(self, X, y=None, **fit_params):
@@ -118,7 +133,9 @@ def _pre_transform(self, X, y=None, **fit_params):
             step, param = pname.split('__', 1)
             fit_params_steps[step][param] = pval
         Xt = X
-        for name, transform in self.steps[:-1]:
+        for name, transform in self._transforms:
+            if transform is None:
+                continue
             if hasattr(transform, "fit_transform"):
                 Xt = transform.fit_transform(Xt, y, **fit_params_steps[name])
             else:
@@ -129,79 +146,192 @@ def _pre_transform(self, X, y=None, **fit_params):
     def fit(self, X, y=None, **fit_params):
         """Fit all the transforms one after the other and transform the
         data, then fit the transformed data using the final estimator.
+
+        Parameters
+        ----------
+        X : array-like, shape = [n_samples, n_features]
+            Training data, where n_samples in the number of samples and
+            n_features is the number of features.
+        y : array-like, shape = [n_samples], optional
+            Target vector relative to X for classification;
+            None for unsupervised learning.
+        fit_params : dict of string -> object
+            Parameters passed to the `fit` method of each step, where
+            each parameter name is prefixed such that parameter ``p`` for step
+            ``s`` has key ``s__p``.
         """
         Xt, fit_params = self._pre_transform(X, y, **fit_params)
-        self.steps[-1][-1].fit(Xt, y, **fit_params)
+        self._final_estimator.fit(Xt, y, **fit_params)
         return self
 
-    def fit_transform(self, X, y=None, **fit_params):
-        """Fit all the transforms one after the other and transform the
+    @property
+    def fit_transform(self):
+        """Pipeline.fit_transform(X, y=None, **fit_params)
+
+        Fit all the transforms one after the other and transform the
         data, then use fit_transform on transformed data using the final
-        estimator."""
-        Xt, fit_params = self._pre_transform(X, y, **fit_params)
-        if hasattr(self.steps[-1][-1], 'fit_transform'):
-            return self.steps[-1][-1].fit_transform(Xt, y, **fit_params)
-        else:
-            return self.steps[-1][-1].fit(Xt, y, **fit_params).transform(Xt)
+        estimator.
 
-    def predict(self, X):
-        """Applies transforms to the data, and the predict method of the
-        final estimator. Valid only if the final estimator implements
-        predict."""
+        Parameters
+        ----------
+        X : array-like, shape = [n_samples, n_features]
+            Training data, where n_samples in the number of samples and
+            n_features is the number of features.
+        y : array-like, shape = [n_samples], optional
+            Target vector relative to X for classification;
+            None for unsupervised learning.
+        fit_params : dict of string -> object
+            Parameters passed to the `fit` method of each step, where
+            each parameter name is prefixed such that parameter ``p`` for step
+            ``s`` has key ``s__p``.
+        """
+        last_step = self._final_estimator
+        if (
+                not hasattr(last_step, 'fit_transform')
+                and not hasattr(last_step, 'transform')):
+            raise AttributeError(
+                'last step has neither `transform` nor `fit_transform`')
+
+        def fn(X, y=None, **fit_params):
+            Xt, fit_params = self._pre_transform(X, y, **fit_params)
+            if hasattr(last_step, 'fit_transform'):
+                return last_step.fit_transform(Xt, y, **fit_params)
+            else:
+                return last_step.fit(Xt, y, **fit_params).transform(Xt)
+        return fn
+
+    def _run_pipeline(self, est_fn, X, *args, **kwargs):
         Xt = X
-        for name, transform in self.steps[:-1]:
-            Xt = transform.transform(Xt)
-        return self.steps[-1][-1].predict(Xt)
+        for name, transform in self._transforms:
+            if transform is not None:
+                Xt = transform.transform(Xt)
+        return est_fn(Xt, *args, **kwargs)
 
-    def predict_proba(self, X):
-        """Applies transforms to the data, and the predict_proba method of the
+    @property
+    def predict(self):
+        """Pipeline.predict(X)
+
+        Applies transforms to the data, and the `predict` method of the
         final estimator. Valid only if the final estimator implements
-        predict_proba."""
-        Xt = X
-        for name, transform in self.steps[:-1]:
-            Xt = transform.transform(Xt)
-        return self.steps[-1][-1].predict_proba(Xt)
+        predict.
+
+        Parameters
+        ----------
+        X : array-like, shape = [n_samples, n_features]
+            Data samples, where n_samples in the number of samples and
+            n_features is the number of features.
+        """
+        return partial(self._run_pipeline, self._final_estimator.predict)
 
-    def decision_function(self, X):
-        """Applies transforms to the data, and the decision_function method of
+    @property
+    def predict_proba(self):
+        """Pipeline.predict_proba(X)
+
+        Applies transforms to the data, and the `predict_proba` method of
         the final estimator. Valid only if the final estimator implements
-        decision_function."""
-        Xt = X
-        for name, transform in self.steps[:-1]:
-            Xt = transform.transform(Xt)
-        return self.steps[-1][-1].decision_function(Xt)
+        predict_proba.
 
-    def predict_log_proba(self, X):
-        Xt = X
-        for name, transform in self.steps[:-1]:
-            Xt = transform.transform(Xt)
-        return self.steps[-1][-1].predict_log_proba(Xt)
+        Parameters
+        ----------
+        X : array-like, shape = [n_samples, n_features]
+            Data samples, where n_samples in the number of samples and
+            n_features is the number of features.
+        """
+        return partial(self._run_pipeline, self._final_estimator.predict_proba)
 
-    def transform(self, X):
-        """Applies transforms to the data, and the transform method of the
+    @property
+    def predict_log_proba(self):
+        """Pipeline.predict_log_proba(X)
+
+        Applies transforms to the data, and the `predict_log_proba` method
+        of the final estimator. Valid only if the final estimator implements
+        predict_log_proba.
+
+        Parameters
+        ----------
+        X : array-like, shape = [n_samples, n_features]
+            Data samples, where n_samples in the number of samples and
+            n_features is the number of features.
+        """
+        return partial(self._run_pipeline,
+            self._final_estimator.predict_log_proba)
+
+    @property
+    def decision_function(self):
+        """Pipeline.decision_function(X)
+
+        Applies transforms to the data, and the `decision_function` method
+        of the final estimator. Valid only if the final estimator implements
+        decision_function.
+
+        Parameters
+        ----------
+        X : array-like, shape = [n_samples, n_features]
+            Data samples, where n_samples in the number of samples and
+            n_features is the number of features.
+        """
+        return partial(self._run_pipeline,
+            self._final_estimator.decision_function)
+
+    @property
+    def transform(self):
+        """Pipeline.transform(X)
+
+        Applies transforms to the data, and the `transform` method of the
         final estimator. Valid only if the final estimator implements
-        transform."""
-        Xt = X
-        for name, transform in self.steps:
-            Xt = transform.transform(Xt)
-        return Xt
+        transform.
 
-    def inverse_transform(self, X):
-        if X.ndim == 1:
-            X = X[None, :]
-        Xt = X
-        for name, step in self.steps[::-1]:
-            Xt = step.inverse_transform(Xt)
-        return Xt
+        Parameters
+        ----------
+        X : array-like, shape = [n_samples, n_features]
+            Data samples, where n_samples in the number of samples and
+            n_features is the number of features.
+        """
+        return partial(self._run_pipeline, self._final_estimator.transform)
+
+    @property
+    def inverse_transform(self):
+        """Pipeline.inverse_transform(X)
+
+        Applies inverse transforms to the data from the last step to the
+        first.
+
+        Parameters
+        ----------
+        X : array-like, shape = [n_samples, n_features]
+            Data samples, where n_samples in the number of samples and
+            n_features is the number of features.
+        """
+        inverse_transforms = [step.inverse_transform
+            for name, step in self.steps[::-1] if step is not None]
+
+        def fn(X):
+            if X.ndim == 1:
+                X = X[None, :]
+            Xt = X
+            for inv_transform in inverse_transforms:
+                Xt = inv_transform(Xt)
+            return Xt
+        return fn
 
-    def score(self, X, y=None):
-        """Applies transforms to the data, and the score method of the
+    @property
+    def score(self):
+        """Pipeline.score(X, y=None)
+
+        Applies transforms to the data, and the `score` method of the
         final estimator. Valid only if the final estimator implements
-        score."""
-        Xt = X
-        for name, transform in self.steps[:-1]:
-            Xt = transform.transform(Xt)
-        return self.steps[-1][-1].score(Xt, y)
+        score.
+
+        Parameters
+        ----------
+        X : array-like, shape = [n_samples, n_features]
+            Data samples, where n_samples in the number of samples and
+            n_features is the number of features.
+        y : array-like, shape = [n_samples], optional
+            Target vector relative to X;
+            None for unsupervised learning.
+        """
+        return partial(self._run_pipeline, self._final_estimator.score)
 
     @property
     def _pairwise(self):
diff --git a/sklearn/tests/test_metaestimators.py b/sklearn/tests/test_metaestimators.py
new file mode 100644
index 0000000000000..0a36125c07b2a
--- /dev/null
+++ b/sklearn/tests/test_metaestimators.py
@@ -0,0 +1,116 @@
+"""Common tests for metaestimators"""
+
+import functools
+
+import numpy as np
+
+from sklearn.base import BaseEstimator
+from sklearn.externals.six import iterkeys
+from sklearn.datasets import make_classification
+from sklearn.utils.testing import assert_true, assert_false
+from sklearn.pipeline import Pipeline
+from sklearn.grid_search import GridSearchCV, RandomizedSearchCV
+from sklearn.feature_selection import RFE, RFECV
+
+
+class DelegatorData(object):
+    def __init__(self, name, construct, skip_methods=(),
+                 fit_args=make_classification()):
+        self.name = name
+        self.construct = construct
+        self.fit_args = fit_args
+        self.skip_methods = skip_methods
+
+
+DELEGATING_METAESTIMATORS = [
+    DelegatorData('Pipeline', lambda est: Pipeline([('est', est)])),
+    DelegatorData('GridSearchCV',
+                  lambda est: GridSearchCV(
+                      est, param_grid={'param': [5]}, cv=2),
+                  skip_methods=['score']),
+    DelegatorData('RandomizedSearchCV',
+                  lambda est: RandomizedSearchCV(
+                      est, param_distributions={'param': [5]}, cv=2),
+                  skip_methods=['score']),
+    DelegatorData('RFE', RFE,
+                  skip_methods=['transform', 'inverse_transform', 'score']),
+    DelegatorData('RFECV', RFECV,
+                  skip_methods=['transform', 'inverse_transform', 'score']),
+]
+
+
+def test_metaestimator_delegation():
+    """Ensures specified metaestimators have methods iff subestimator does"""
+    def hides(method):
+        @property
+        def wrapper(obj):
+            if obj.hidden_method == method.__name__:
+                raise AttributeError('%r is hidden' % obj.hidden_method)
+            return functools.partial(method, obj)
+        return wrapper
+
+    class SubEstimator(BaseEstimator):
+        def __init__(self, param=1, hidden_method=None):
+            self.param = param
+            self.hidden_method = hidden_method
+
+        def fit(self, X, y=None, *args, **kwargs):
+            self.coef_ = np.arange(X.shape[1])
+            return True
+
+        @hides
+        def inverse_transform(self, X, *args, **kwargs):
+            return X
+
+        @hides
+        def transform(self, X, *args, **kwargs):
+            return X
+
+        @hides
+        def predict(self, X, *args, **kwargs):
+            return np.ones(X.shape[0])
+
+        @hides
+        def predict_proba(self, X, *args, **kwargs):
+            return np.ones(X.shape[0])
+
+        @hides
+        def predict_log_proba(self, X, *args, **kwargs):
+            return np.ones(X.shape[0])
+
+        @hides
+        def decision_function(self, X, *args, **kwargs):
+            return np.ones(X.shape[0])
+
+        @hides
+        def score(self, X, *args, **kwargs):
+            return 1.0
+
+    methods = [k for k in iterkeys(SubEstimator.__dict__)
+               if not k.startswith('_') and not k.startswith('fit')]
+    methods.sort()
+
+    for delegator_data in DELEGATING_METAESTIMATORS:
+        delegate = SubEstimator()
+        delegator = delegator_data.construct(delegate)
+        delegator.fit(*delegator_data.fit_args)
+        for method in methods:
+            if method in delegator_data.skip_methods:
+                continue
+            assert_true(hasattr(delegate, method))
+            assert_true(hasattr(delegator, method),
+                        msg="%s does not have method %r when its delegate does"
+                            % (delegator_data.name, method))
+            # smoke test delegation
+            getattr(delegator, method)(delegator_data.fit_args[0])
+
+        for method in methods:
+            if method in delegator_data.skip_methods:
+                continue
+            delegate = SubEstimator(hidden_method=method)
+            delegator = delegator_data.construct(delegate)
+            delegator.fit(*delegator_data.fit_args)
+            assert_false(hasattr(delegate, method))
+            assert_false(hasattr(delegator, method),
+                         msg="%s has method %r when its delegate does not"
+                             % (delegator_data.name, method))