scikit-learn · mblondel · Jan 9, 2014 · Jan 9, 2014 · Jan 10, 2014 · Jan 10, 2014
diff --git a/examples/plot_validation_contours.py b/examples/plot_validation_contours.py
@@ -0,0 +1,50 @@
+import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.datasets import load_digits
+from sklearn.svm import SVC
+from sklearn.learning_curve import validation_curve
+from sklearn.externals.joblib import Memory
+
+memory = Memory(cachedir=".", verbose=0)
+
+@memory.cache
+def grid(X, y, Cs, gammas):
+    param_grid = {"C": Cs, "gamma": gammas}
+
+    tr, te, times = validation_curve(SVC(kernel="rbf"), X, y, param_grid, cv=3)
+
+    shape = (len(Cs), len(gammas))
+    tr = tr.mean(axis=1).reshape(shape)
+    te = te.mean(axis=1).reshape(shape)
+    times = times.mean(axis=1).reshape(shape)
+
+    return tr, te, times
+
+digits = load_digits()
+X, y = digits.data, digits.target
+
+gammas = np.logspace(-6, -1, 5)
+Cs = np.logspace(-3, 3, 5)
+
+tr, te, times = grid(X, y, Cs, gammas)
+
+
+for title, values in (("Training accuracy", tr),
+                      ("Test accuracy", te),
+                      ("Training time", times)):
+
+    plt.figure()
+
+    plt.title(title)
+    plt.xlabel("C")
+    plt.xscale("log")
+
+    plt.ylabel("gamma")
+    plt.yscale("log")
+
+    X1, X2 = np.meshgrid(Cs, gammas)
+    cs = plt.contour(X1, X2, values)
+
+    plt.colorbar(cs)
+
+plt.show()
diff --git a/examples/plot_validation_curve.py b/examples/plot_validation_curve.py
@@ -23,9 +23,9 @@
 X, y = digits.data, digits.target
 
 param_range = np.logspace(-6, -1, 5)
-train_scores, test_scores = validation_curve(
-    SVC(), X, y, param_name="gamma", param_range=param_range,
-    cv=10, scoring="accuracy", n_jobs=1)
+param_grid = {"gamma": param_range}
+train_scores, test_scores, train_times = validation_curve(
+    SVC(), X, y, param_grid, cv=10, scoring="accuracy", n_jobs=1)
 train_scores_mean = np.mean(train_scores, axis=1)
 train_scores_std = np.std(train_scores, axis=1)
 test_scores_mean = np.mean(test_scores, axis=1)

diff --git a/sklearn/cross_validation.py b/sklearn/cross_validation.py
@@ -27,7 +27,8 @@
 from .utils.fixes import unique
 from .externals.joblib import Parallel, delayed, logger
 from .externals.six import with_metaclass
-from .metrics.scorer import check_scoring
+from .metrics.scorer import check_scoring, _evaluate_scorers
+
 
 __all__ = ['Bootstrap',
            'KFold',
@@ -1041,7 +1042,7 @@ def __len__(self):
 def cross_val_score(estimator, X, y=None, scoring=None, cv=None, n_jobs=1,
                     verbose=0, fit_params=None, score_func=None,
                     pre_dispatch='2*n_jobs'):
-    """Evaluate a score by cross-validation
+    """Evaluate test score by cross-validation
 
     Parameters
     ----------
@@ -1055,10 +1056,12 @@ def cross_val_score(estimator, X, y=None, scoring=None, cv=None, n_jobs=1,
         The target variable to try to predict in the case of
         supervised learning.
 
-    scoring : string, callable or None, optional, default: None
+    scoring : string, callable, list of strings/callables or None, optional,
+              default: None
         A string (see model evaluation documentation) or
         a scorer callable object / function with signature
         ``scorer(estimator, X, y)``.
+        Lists can be used for randomized search of multiple metrics.
 
     cv : cross-validation generator, optional, default: None
         A cross-validation generator. If None, a 3-fold cross
@@ -1094,78 +1097,57 @@ def cross_val_score(estimator, X, y=None, scoring=None, cv=None, n_jobs=1,
 
     Returns
     -------
-    scores : array of float, shape=(len(list(cv)),)
+    scores : array of float, shape=(n_folds,) or (n_scoring, n_folds)
         Array of scores of the estimator for each run of the cross validation.
+        The returned array is 2d if `scoring` is a list.
     """
     X, y = check_arrays(X, y, sparse_format='csr', allow_lists=True)
     cv = _check_cv(cv, X, y, classifier=is_classifier(estimator))
-    scorer = check_scoring(estimator, score_func=score_func, scoring=scoring)
-    # We clone the estimator to make sure that all the folds are
-    # independent, and that it is pickle-able.
-    parallel = Parallel(n_jobs=n_jobs, verbose=verbose,
-                        pre_dispatch=pre_dispatch)
-    scores = parallel(delayed(_fit_and_score)(clone(estimator), X, y, scorer,
-                                              train, test, verbose, None,
-                                              fit_params)
-                      for train, test in cv)
-    return np.array(scores)[:, 0]
-
-
-def _fit_and_score(estimator, X, y, scorer, train, test, verbose, parameters,
-                   fit_params, return_train_score=False,
-                   return_parameters=False):
-    """Fit estimator and compute scores for a given dataset split.
-
-    Parameters
-    ----------
-    estimator : estimator object implementing 'fit'
-        The object to use to fit the data.
-
-    X : array-like of shape at least 2D
-        The data to fit.
-
-    y : array-like, optional, default: None
-        The target variable to try to predict in the case of
-        supervised learning.
-
-    scoring : callable
-        A scorer callable object / function with signature
-        ``scorer(estimator, X, y)``.
 
-    train : array-like, shape = (n_train_samples,)
-        Indices of training samples.
+    if isinstance(scoring, (tuple, list)):
+        scorers = [check_scoring(estimator, scoring=s) for s in scoring]
+        ret_1d = False
+    else:
+        scorers = [check_scoring(estimator, score_func=score_func,
+                               scoring=scoring)]
+        ret_1d = True
 
-    test : array-like, shape = (n_test_samples,)
-        Indices of test samples.
+    parallel = Parallel(n_jobs=n_jobs, verbose=verbose,
+                        pre_dispatch=pre_dispatch)
 
-    verbose : integer
-        The verbosity level.
+    # `out` is a list of size n_folds. Each element of the list is a tuple
+    # (test_scores, n_test, train_time)
+    out = parallel(delayed(_fit_and_score)(clone(estimator), X, y, scorers,
+                                           train, test, verbose, None,
+                                           fit_params)
+                      for train, test in cv)
 
-    parameters : dict or None
-        Parameters to be set on the estimator.
+    # Retrieve n_scorers x n_folds 2d-array.
+    test_scores = np.array([o[0] for o in out]).T
 
-    fit_params : dict or None
-        Parameters that will be passed to ``estimator.fit``.
+    if ret_1d:
+        return test_scores[0]
+    else:
+        return test_scores
 
-    return_train_score : boolean, optional, default: False
-        Compute and return score on training set.
 
-    return_parameters : boolean, optional, default: False
-        Return parameters that has been used for the estimator.
+def _fit_and_score(estimator, X, y, scorers, train, test, verbose, parameters,
+                   fit_params, return_train_scores=False):
+    """Fit estimator and compute scores for a given dataset split.
 
     Returns
     -------
-    test_score : float
-        Score on test set.
+    train_score : array of floats, optional
+        Scores on training set.
 
-    train_score : float, optional
-        Score on training set.
+    test_score : array of floats
+        Scores on test set.
 
     n_test_samples : int
         Number of test samples.
 
-    scoring_time : float
-        Time spent for fitting and scoring in seconds.
+    train_time : float
+        Time spent for fitting in seconds.
 
     parameters : dict or None, optional
         The parameters that have been evaluated.
@@ -1188,30 +1170,35 @@ def _fit_and_score(estimator, X, y, scorer, train, test, verbose, parameters,
     if parameters is not None:
         estimator.set_params(**parameters)
 
-    start_time = time.time()
-
     X_train, y_train = _safe_split(estimator, X, y, train)
     X_test, y_test = _safe_split(estimator, X, y, test, train)
+
+    start_time = time.time()
+
     if y_train is None:
         estimator.fit(X_train, **fit_params)
     else:
         estimator.fit(X_train, y_train, **fit_params)
-    test_score = _score(estimator, X_test, y_test, scorer)
-    if return_train_score:
-        train_score = _score(estimator, X_train, y_train, scorer)
 
-    scoring_time = time.time() - start_time
+    train_time = time.time() - start_time
+
+    test_scores = _evaluate_scorers(estimator, X_test, y_test, scorers)
+
+    if return_train_scores:
+        if len(scorers) == 1:
+            train_scores = np.array([scorers[0](estimator, X_train, y_train)])
+        else:
+            train_scores = _evaluate_scorers(estimator, X_train, y_train,
+                                             scorers)
 
     if verbose > 2:
-        msg += ", score=%f" % test_score
+        msg += ", score=%s" % test_scores
     if verbose > 1:
-        end_msg = "%s -%s" % (msg, logger.short_format_time(scoring_time))
+        end_msg = "%s -%s" % (msg, logger.short_format_time(train_time))
         print("[CV] %s %s" % ((64 - len(end_msg)) * '.', end_msg))
 
-    ret = [train_score] if return_train_score else []
-    ret.extend([test_score, _num_samples(X_test), scoring_time])
-    if return_parameters:
-        ret.append(parameters)
+    ret = [train_scores] if return_train_scores else []
+    ret += [test_scores, _num_samples(X_test), train_time, parameters]
     return ret
 
 
@@ -1247,18 +1234,6 @@ def _safe_split(estimator, X, y, indices, train_indices=None):
     return X_subset, y_subset
 
 
-def _score(estimator, X_test, y_test, scorer):
-    """Compute the score of an estimator on a given test set."""
-    if y_test is None:
-        score = scorer(estimator, X_test)
-    else:
-        score = scorer(estimator, X_test, y_test)
-    if not isinstance(score, numbers.Number):
-        raise ValueError("scoring must return a number, got %s (%s) instead."
-                         % (str(score), type(score)))
-    return score
-
-
 def _permutation_test_score(estimator, X, y, cv, scorer):
     """Auxiliary function for permutation_test_score"""
     avg_score = []

diff --git a/sklearn/feature_selection/rfe.py b/sklearn/feature_selection/rfe.py
@@ -13,7 +13,7 @@
 from ..base import clone
 from ..base import is_classifier
 from ..cross_validation import _check_cv as check_cv
-from ..cross_validation import _safe_split, _score
+from ..cross_validation import _safe_split
 from .base import SelectorMixin
 from ..metrics.scorer import check_scoring
 
@@ -342,7 +342,7 @@ def fit(self, X, y):
                 mask = np.where(ranking_ <= k + 1)[0]
                 estimator = clone(self.estimator)
                 estimator.fit(X_train[:, mask], y_train)
-                score = _score(estimator, X_test[:, mask], y_test, scorer)
+                score = scorer(estimator, X_test[:, mask], y_test)
 
                 if self.verbose > 0:
                     print("Finished fold with %d / %d feature ranks, score=%f"