diff --git a/sklearn/multiclass.py b/sklearn/multiclass.py
index e3fad7e08e3e0..66b0a47da9599 100644
--- a/sklearn/multiclass.py
+++ b/sklearn/multiclass.py
@@ -50,7 +50,7 @@
 from .utils.multiclass import (_check_partial_fit_first_call,
                                check_classification_targets,
                                _ovr_decision_function)
-from .utils.metaestimators import _safe_split
+from .utils.metaestimators import _safe_split, if_delegate_has_method
 
 from .externals.joblib import Parallel
 from .externals.joblib import delayed
@@ -309,6 +309,7 @@ def predict(self, X):
                                       shape=(n_samples, len(self.estimators_)))
             return self.label_binarizer_.inverse_transform(indicator)
 
+    @if_delegate_has_method(['_first_estimator', 'estimator'])
     def predict_proba(self, X):
         """Probability estimates.
 
@@ -347,6 +348,7 @@ def predict_proba(self, X):
             Y /= np.sum(Y, axis=1)[:, np.newaxis]
         return Y
 
+    @if_delegate_has_method(['_first_estimator', 'estimator'])
     def decision_function(self, X):
         """Returns the distance of each sample from the decision boundary for
         each class. This can only be used with estimators which implement the
@@ -361,9 +363,6 @@ def decision_function(self, X):
         T : array-like, shape = [n_samples, n_classes]
         """
         check_is_fitted(self, 'estimators_')
-        if not hasattr(self.estimators_[0], "decision_function"):
-            raise AttributeError(
-                "Base estimator doesn't have a decision_function attribute.")
         return np.array([est.decision_function(X).ravel()
                          for est in self.estimators_]).T
 
@@ -400,6 +399,10 @@ def _pairwise(self):
         """Indicate if wrapped estimator is using a precomputed Gram matrix"""
         return getattr(self.estimator, "_pairwise", False)
 
+    @property
+    def _first_estimator(self):
+        return self.estimators_[0]
+
 
 def _fit_ovo_binary(estimator, X, y, i, j):
     """Fit a single binary estimator (one-vs-one)."""
diff --git a/sklearn/tests/test_multiclass.py b/sklearn/tests/test_multiclass.py
index 5bdc13f8d5d9a..ca5909ea72a48 100644
--- a/sklearn/tests/test_multiclass.py
+++ b/sklearn/tests/test_multiclass.py
@@ -314,14 +314,25 @@ def test_ovr_multilabel_predict_proba():
         X_test = X[80:]
         clf = OneVsRestClassifier(base_clf).fit(X_train, Y_train)
 
-        # decision function only estimator. Fails in current implementation.
+        # Decision function only estimator.
         decision_only = OneVsRestClassifier(svm.SVR()).fit(X_train, Y_train)
-        assert_raises(AttributeError, decision_only.predict_proba, X_test)
+        assert_false(hasattr(decision_only, 'predict_proba'))
+        assert_true(hasattr(decision_only, 'decision_function'))
 
         # Estimator with predict_proba disabled, depending on parameters.
         decision_only = OneVsRestClassifier(svm.SVC(probability=False))
+        assert_false(hasattr(decision_only, 'predict_proba'))
         decision_only.fit(X_train, Y_train)
-        assert_raises(AttributeError, decision_only.predict_proba, X_test)
+        assert_false(hasattr(decision_only, 'predict_proba'))
+        assert_true(hasattr(decision_only, 'decision_function'))
+
+        # Estimator which can get predict_proba enabled after fitting
+        gs = GridSearchCV(svm.SVC(probability=False),
+                          param_grid={'probability': [True]})
+        proba_after_fit = OneVsRestClassifier(gs)
+        assert_false(hasattr(proba_after_fit, 'predict_proba'))
+        proba_after_fit.fit(X_train, Y_train)
+        assert_true(hasattr(proba_after_fit, 'predict_proba'))
 
         Y_pred = clf.predict(X_test)
         Y_proba = clf.predict_proba(X_test)
@@ -339,9 +350,10 @@ def test_ovr_single_label_predict_proba():
     X_test = X[80:]
     clf = OneVsRestClassifier(base_clf).fit(X_train, Y_train)
 
-    # decision function only estimator. Fails in current implementation.
+    # Decision function only estimator.
     decision_only = OneVsRestClassifier(svm.SVR()).fit(X_train, Y_train)
-    assert_raises(AttributeError, decision_only.predict_proba, X_test)
+    assert_false(hasattr(decision_only, 'predict_proba'))
+    assert_true(hasattr(decision_only, 'decision_function'))
 
     Y_pred = clf.predict(X_test)
     Y_proba = clf.predict_proba(X_test)