diff --git a/sklearn/metrics/scorer.py b/sklearn/metrics/scorer.py
index 2661a379b4e53..f8b678ae62a5d 100644
--- a/sklearn/metrics/scorer.py
+++ b/sklearn/metrics/scorer.py
@@ -183,7 +183,12 @@ def __call__(self, clf, X, y, sample_weight=None):
                 y_pred = clf.predict_proba(X)
 
                 if y_type == "binary":
-                    y_pred = y_pred[:, 1]
+                    if y_pred.shape[1]==2:
+                        y_pred = y_pred[:, 1]
+                    else:
+                        raise ValueError('The output of the prediction is a vector, which is incompatible with a classifierself.'
+                        'Check that the estimator is a classifier and the number of classes in the dataset is 2 or more.')
+
                 elif isinstance(y_pred, list):
                     y_pred = np.vstack([p[:, -1] for p in y_pred]).T