From 70670183a730ff080456c87c2734202e8a1003de Mon Sep 17 00:00:00 2001
From: Raghav RV <rvraghav93@gmail.com>
Date: Thu, 13 Oct 2016 04:18:07 +0200
Subject: [PATCH 01/20] FIX Be robust to non re-entrant/ non deterministic
 cv.split calls

---
 sklearn/model_selection/_search.py     |  3 ++-
 sklearn/model_selection/_split.py      |  2 +-
 sklearn/model_selection/_validation.py | 13 +++++++------
 3 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/sklearn/model_selection/_search.py b/sklearn/model_selection/_search.py
index 82516f1e6ba4a..d2f5542ebd32f 100644
--- a/sklearn/model_selection/_search.py
+++ b/sklearn/model_selection/_search.py
@@ -550,6 +550,7 @@ def _fit(self, X, y, groups, parameter_iterable):
         base_estimator = clone(self.estimator)
         pre_dispatch = self.pre_dispatch
 
+        cv_iter = list(cv.split(X, y, groups))
         out = Parallel(
             n_jobs=self.n_jobs, verbose=self.verbose,
             pre_dispatch=pre_dispatch
@@ -561,7 +562,7 @@ def _fit(self, X, y, groups, parameter_iterable):
                                   return_times=True, return_parameters=True,
                                   error_score=self.error_score)
           for parameters in parameter_iterable
-          for train, test in cv.split(X, y, groups))
+          for train, test in cv_iter)
 
         # if one choose to see train score, "out" will contain train score info
         if self.return_train_score:
diff --git a/sklearn/model_selection/_split.py b/sklearn/model_selection/_split.py
index cf109e621626b..3a39c1d453d51 100644
--- a/sklearn/model_selection/_split.py
+++ b/sklearn/model_selection/_split.py
@@ -1467,7 +1467,7 @@ def get_n_splits(self, X=None, y=None, groups=None):
 class _CVIterableWrapper(BaseCrossValidator):
     """Wrapper class for old style cv objects and iterables."""
     def __init__(self, cv):
-        self.cv = cv
+        self.cv = list(cv)
 
     def get_n_splits(self, X=None, y=None, groups=None):
         """Returns the number of splitting iterations in the cross-validator
diff --git a/sklearn/model_selection/_validation.py b/sklearn/model_selection/_validation.py
index cc77d7c2845b0..b19b52b7555af 100644
--- a/sklearn/model_selection/_validation.py
+++ b/sklearn/model_selection/_validation.py
@@ -1,4 +1,3 @@
-
 """
 The :mod:`sklearn.model_selection._validation` module includes classes and
 functions to validate the model.
@@ -129,6 +128,7 @@ def cross_val_score(estimator, X, y=None, groups=None, scoring=None, cv=None,
     X, y, groups = indexable(X, y, groups)
 
     cv = check_cv(cv, y, classifier=is_classifier(estimator))
+    cv_iter = cv.split(X, y, groups)
     scorer = check_scoring(estimator, scoring=scoring)
     # We clone the estimator to make sure that all the folds are
     # independent, and that it is pickle-able.
@@ -137,7 +137,7 @@ def cross_val_score(estimator, X, y=None, groups=None, scoring=None, cv=None,
     scores = parallel(delayed(_fit_and_score)(clone(estimator), X, y, scorer,
                                               train, test, verbose, None,
                                               fit_params)
-                      for train, test in cv.split(X, y, groups))
+                      for train, test in cv_iter)
     return np.array(scores)[:, 0]
 
 
@@ -384,6 +384,7 @@ def cross_val_predict(estimator, X, y=None, groups=None, cv=None, n_jobs=1,
     X, y, groups = indexable(X, y, groups)
 
     cv = check_cv(cv, y, classifier=is_classifier(estimator))
+    cv_iter = cv.split(X, y, groups)
 
     # Ensure the estimator has implemented the passed decision function
     if not callable(getattr(estimator, method)):
@@ -396,7 +397,7 @@ def cross_val_predict(estimator, X, y=None, groups=None, cv=None, n_jobs=1,
                         pre_dispatch=pre_dispatch)
     prediction_blocks = parallel(delayed(_fit_and_predict)(
         clone(estimator), X, y, train, test, verbose, fit_params, method)
-        for train, test in cv.split(X, y, groups))
+        for train, test in cv_iter)
 
     # Concatenate the predictions
     predictions = [pred_block_i for pred_block_i, _ in prediction_blocks]
@@ -750,9 +751,8 @@ def learning_curve(estimator, X, y, groups=None,
     X, y, groups = indexable(X, y, groups)
 
     cv = check_cv(cv, y, classifier=is_classifier(estimator))
-    cv_iter = cv.split(X, y, groups)
     # Make a list since we will be iterating multiple times over the folds
-    cv_iter = list(cv_iter)
+    cv_iter = list(cv.split(X, y, groups))
     scorer = check_scoring(estimator, scoring=scoring)
 
     n_max_training_samples = len(cv_iter[0][0])
@@ -961,6 +961,7 @@ def validation_curve(estimator, X, y, param_name, param_range, groups=None,
     X, y, groups = indexable(X, y, groups)
 
     cv = check_cv(cv, y, classifier=is_classifier(estimator))
+    cv_iter = cv.split(X, y, groups)
 
     scorer = check_scoring(estimator, scoring=scoring)
 
@@ -969,7 +970,7 @@ def validation_curve(estimator, X, y, param_name, param_range, groups=None,
     out = parallel(delayed(_fit_and_score)(
         estimator, X, y, scorer, train, test, verbose,
         parameters={param_name: v}, fit_params=None, return_train_score=True)
-        for train, test in cv.split(X, y, groups) for v in param_range)
+        for train, test in cv_iter for v in param_range)
 
     out = np.asarray(out)
     n_params = len(param_range)

From eab1fe779ec36aa6366df2e38320ac40881715bb Mon Sep 17 00:00:00 2001
From: Raghav RV <rvraghav93@gmail.com>
Date: Thu, 13 Oct 2016 07:19:03 +0200
Subject: [PATCH 02/20] TST Ensure custom one time iterators are accepted as cv

---
 sklearn/model_selection/tests/test_search.py | 25 ++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/sklearn/model_selection/tests/test_search.py b/sklearn/model_selection/tests/test_search.py
index 36e6965a11974..6008e4c0556ae 100644
--- a/sklearn/model_selection/tests/test_search.py
+++ b/sklearn/model_selection/tests/test_search.py
@@ -1154,3 +1154,28 @@ def test_search_train_scores_set_to_false():
     gs = GridSearchCV(clf, param_grid={'C': [0.1, 0.2]},
                       return_train_score=False)
     gs.fit(X, y)
+
+
+def test_grid_search_custom_cv_iter():
+    # Check if a one time iterable is accepted as a cv parameter.
+    X, y = make_classification(n_samples=100, random_state=0)
+
+    class CustomSplitter():
+        def __init__(self, n_samples=100):
+            self.indices = KFold(n_splits=5).split(np.ones(n_samples))
+        def split(self, X=None, y=None, groups=None):
+            for index in self.indices:
+                yield index
+        def get_n_splits(self, X=None, y=None, groups=None):
+            return 5
+
+    gs = GridSearchCV(LinearSVC(random_state=0),
+                      param_grid={'C': [0.1, 0.2, 0.3]}, cv=CustomSplitter())
+    gs.fit(X, y)
+
+
+    gs2 = GridSearchCV(LinearSVC(random_state=0),
+                       param_grid={'C': [0.1, 0.2, 0.3]}, cv=KFold(n_splits=5))
+    gs2.fit(X, y)
+
+    np.testing.assert_equal(gs.cv_results_, gs2.cv_results_)
\ No newline at end of file

From f35809264d90c75c52f3f6cdc98c0a933cfecb32 Mon Sep 17 00:00:00 2001
From: Raghav RV <rvraghav93@gmail.com>
Date: Thu, 13 Oct 2016 07:22:18 +0200
Subject: [PATCH 03/20] FIX/TST pop the time keys

---
 sklearn/model_selection/tests/test_search.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/sklearn/model_selection/tests/test_search.py b/sklearn/model_selection/tests/test_search.py
index 6008e4c0556ae..189a8ef9ca83a 100644
--- a/sklearn/model_selection/tests/test_search.py
+++ b/sklearn/model_selection/tests/test_search.py
@@ -1178,4 +1178,8 @@ def get_n_splits(self, X=None, y=None, groups=None):
                        param_grid={'C': [0.1, 0.2, 0.3]}, cv=KFold(n_splits=5))
     gs2.fit(X, y)
 
+    for key in ('mean_score_time', 'std_score_time',
+                'mean_fit_time', 'std_fit_time'):
+        gs.cv_results_.pop(key)
+        gs2.cv_results_.pop(key)
     np.testing.assert_equal(gs.cv_results_, gs2.cv_results_)
\ No newline at end of file

From 494c52cff50071082f5ae3bf8328bdce8fed2228 Mon Sep 17 00:00:00 2001
From: Raghav RV <rvraghav93@gmail.com>
Date: Thu, 13 Oct 2016 14:10:18 +0200
Subject: [PATCH 04/20] FLAKE8

---
 sklearn/model_selection/tests/test_search.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/sklearn/model_selection/tests/test_search.py b/sklearn/model_selection/tests/test_search.py
index 189a8ef9ca83a..a40f920416306 100644
--- a/sklearn/model_selection/tests/test_search.py
+++ b/sklearn/model_selection/tests/test_search.py
@@ -1163,9 +1163,11 @@ def test_grid_search_custom_cv_iter():
     class CustomSplitter():
         def __init__(self, n_samples=100):
             self.indices = KFold(n_splits=5).split(np.ones(n_samples))
+
         def split(self, X=None, y=None, groups=None):
             for index in self.indices:
                 yield index
+
         def get_n_splits(self, X=None, y=None, groups=None):
             return 5
 
@@ -1173,7 +1175,6 @@ def get_n_splits(self, X=None, y=None, groups=None):
                       param_grid={'C': [0.1, 0.2, 0.3]}, cv=CustomSplitter())
     gs.fit(X, y)
 
-
     gs2 = GridSearchCV(LinearSVC(random_state=0),
                        param_grid={'C': [0.1, 0.2, 0.3]}, cv=KFold(n_splits=5))
     gs2.fit(X, y)
@@ -1182,4 +1183,4 @@ def get_n_splits(self, X=None, y=None, groups=None):
                 'mean_fit_time', 'std_fit_time'):
         gs.cv_results_.pop(key)
         gs2.cv_results_.pop(key)
-    np.testing.assert_equal(gs.cv_results_, gs2.cv_results_)
\ No newline at end of file
+    np.testing.assert_equal(gs.cv_results_, gs2.cv_results_)

From 87931ce1f1e267c08dfdb3f0ccc3857d3d3a227b Mon Sep 17 00:00:00 2001
From: Raghav RV <rvraghav93@gmail.com>
Date: Fri, 14 Oct 2016 17:14:44 +0200
Subject: [PATCH 05/20] TST consistency of folds across params in GSCV

---
 sklearn/model_selection/tests/test_search.py | 29 ++++++++++++++++----
 1 file changed, 24 insertions(+), 5 deletions(-)

diff --git a/sklearn/model_selection/tests/test_search.py b/sklearn/model_selection/tests/test_search.py
index a40f920416306..2925764065c0e 100644
--- a/sklearn/model_selection/tests/test_search.py
+++ b/sklearn/model_selection/tests/test_search.py
@@ -1179,8 +1179,27 @@ def get_n_splits(self, X=None, y=None, groups=None):
                        param_grid={'C': [0.1, 0.2, 0.3]}, cv=KFold(n_splits=5))
     gs2.fit(X, y)
 
-    for key in ('mean_score_time', 'std_score_time',
-                'mean_fit_time', 'std_fit_time'):
-        gs.cv_results_.pop(key)
-        gs2.cv_results_.pop(key)
-    np.testing.assert_equal(gs.cv_results_, gs2.cv_results_)
+    def _pop_time_keys(cv_results):
+        for key in ('mean_fit_time', 'std_fit_time',
+                    'mean_score_time', 'std_score_time'):
+            cv_results.pop(key)
+        return cv_results
+
+    np.testing.assert_equal(_pop_time_keys(gs.cv_results_),
+                            _pop_time_keys(gs2.cv_results_))
+
+    # Check consistency of folds across the parameters
+    gs = GridSearchCV(LinearSVC(random_state=0),
+                      param_grid={'C': [0.1, 0.1, 0.2, 0.2]},
+                      cv=KFold(n_splits=5, shuffle=True))
+    gs.fit(X, y)
+
+    per_param_test_scores = {}
+    for param_i in range(4):
+        per_param_test_scores[param_i] = list(
+            gs.cv_results_['split%d_test_score' % s][param_i]
+            for s in range(5))
+    assert_array_almost_equal(per_param_test_scores[0],
+                              per_param_test_scores[1])
+    assert_array_almost_equal(per_param_test_scores[2],
+                              per_param_test_scores[3])

From b996857b4e11344d1a19d55a244f51482420500f Mon Sep 17 00:00:00 2001
From: Raghav RV <rvraghav93@gmail.com>
Date: Sun, 16 Oct 2016 14:09:40 +0200
Subject: [PATCH 06/20] Enlist the cv folds

---
 sklearn/model_selection/_validation.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/sklearn/model_selection/_validation.py b/sklearn/model_selection/_validation.py
index b19b52b7555af..15d086c9b50ea 100644
--- a/sklearn/model_selection/_validation.py
+++ b/sklearn/model_selection/_validation.py
@@ -128,7 +128,7 @@ def cross_val_score(estimator, X, y=None, groups=None, scoring=None, cv=None,
     X, y, groups = indexable(X, y, groups)
 
     cv = check_cv(cv, y, classifier=is_classifier(estimator))
-    cv_iter = cv.split(X, y, groups)
+    cv_iter = list(cv.split(X, y, groups))
     scorer = check_scoring(estimator, scoring=scoring)
     # We clone the estimator to make sure that all the folds are
     # independent, and that it is pickle-able.
@@ -384,7 +384,7 @@ def cross_val_predict(estimator, X, y=None, groups=None, cv=None, n_jobs=1,
     X, y, groups = indexable(X, y, groups)
 
     cv = check_cv(cv, y, classifier=is_classifier(estimator))
-    cv_iter = cv.split(X, y, groups)
+    cv_iter = list(cv.split(X, y, groups))
 
     # Ensure the estimator has implemented the passed decision function
     if not callable(getattr(estimator, method)):
@@ -775,9 +775,8 @@ def learning_curve(estimator, X, y, groups=None,
     if exploit_incremental_learning:
         classes = np.unique(y) if is_classifier(estimator) else None
         out = parallel(delayed(_incremental_fit_estimator)(
-            clone(estimator), X, y, classes, train,
-            test, train_sizes_abs, scorer, verbose)
-            for train, test in cv_iter)
+            clone(estimator), X, y, classes, train, test, train_sizes_abs,
+            scorer, verbose) for train, test in cv_iter)
     else:
         train_test_proportions = []
         for train, test in cv_iter:

From 9300390ec06101eab3b4e0bcbd6f8b8a4a673523 Mon Sep 17 00:00:00 2001
From: Raghav RV <rvraghav93@gmail.com>
Date: Sun, 16 Oct 2016 14:10:31 +0200
Subject: [PATCH 07/20] Move CustomSplitter to the top. Add explanatory comment

---
 sklearn/model_selection/tests/test_search.py | 57 ++++++++++++--------
 1 file changed, 36 insertions(+), 21 deletions(-)

diff --git a/sklearn/model_selection/tests/test_search.py b/sklearn/model_selection/tests/test_search.py
index 2925764065c0e..258beb9b2385b 100644
--- a/sklearn/model_selection/tests/test_search.py
+++ b/sklearn/model_selection/tests/test_search.py
@@ -61,6 +61,20 @@
 from sklearn.linear_model import SGDClassifier
 
 
+class CustomSplitter():
+    """A wrapper to make KFold single entry cv iterator"""
+    def __init__(self, n_splits=4, n_samples=99):
+        self.indices = KFold(n_splits=n_splits).split(np.ones(n_samples))
+
+    def split(self, X=None, y=None, groups=None):
+        """Split can be called only once"""
+        for index in self.indices:
+            yield index
+
+    def get_n_splits(self, X=None, y=None, groups=None):
+        return 4
+
+
 # Neither of the following two estimators inherit from BaseEstimator,
 # to test hyperparameter search on user-defined classifiers.
 class MockClassifier(object):
@@ -1156,21 +1170,10 @@ def test_search_train_scores_set_to_false():
     gs.fit(X, y)
 
 
-def test_grid_search_custom_cv_iter():
+def test_grid_search_cv_splits_consistency():
     # Check if a one time iterable is accepted as a cv parameter.
     X, y = make_classification(n_samples=100, random_state=0)
 
-    class CustomSplitter():
-        def __init__(self, n_samples=100):
-            self.indices = KFold(n_splits=5).split(np.ones(n_samples))
-
-        def split(self, X=None, y=None, groups=None):
-            for index in self.indices:
-                yield index
-
-        def get_n_splits(self, X=None, y=None, groups=None):
-            return 5
-
     gs = GridSearchCV(LinearSVC(random_state=0),
                       param_grid={'C': [0.1, 0.2, 0.3]}, cv=CustomSplitter())
     gs.fit(X, y)
@@ -1185,6 +1188,12 @@ def _pop_time_keys(cv_results):
             cv_results.pop(key)
         return cv_results
 
+    # CustomSplitter is a non-re-entrant cv where split can be called only once
+    # if ``cv.split`` is called once per param setting in GridSearchCV.fit
+    # the 2nd and 3rd parameter will not be evaluated as no train/test indices
+    # will be generated for the 2nd and subsequent cv.split calls.
+    # This is a check to make cv.split is not called once per param
+    # setting.
     np.testing.assert_equal(_pop_time_keys(gs.cv_results_),
                             _pop_time_keys(gs2.cv_results_))
 
@@ -1194,12 +1203,18 @@ def _pop_time_keys(cv_results):
                       cv=KFold(n_splits=5, shuffle=True))
     gs.fit(X, y)
 
-    per_param_test_scores = {}
-    for param_i in range(4):
-        per_param_test_scores[param_i] = list(
-            gs.cv_results_['split%d_test_score' % s][param_i]
-            for s in range(5))
-    assert_array_almost_equal(per_param_test_scores[0],
-                              per_param_test_scores[1])
-    assert_array_almost_equal(per_param_test_scores[2],
-                              per_param_test_scores[3])
+    # As the first two param settings (C=0.1) and the next two param
+    # settings (C=0.2) are same, the test and train scores must also be
+    # same as long as the same train/test indices are generated for all
+    # the cv splits, for both param setting
+    for score_type in ('train', 'test'):
+        per_param_scores = {}
+        for param_i in range(4):
+            per_param_scores[param_i] = list(
+                gs.cv_results_['split%d_%s_score' % (s, %score_type)][param_i]
+                for s in range(5))
+
+        assert_array_almost_equal(per_param_scores[0],
+                                  per_param_scores[1])
+        assert_array_almost_equal(per_param_scores[2],
+                                  per_param_scores[3])

From 36ec4f91a1cb69b904a5751f9135487d382f6821 Mon Sep 17 00:00:00 2001
From: Raghav RV <rvraghav93@gmail.com>
Date: Sun, 16 Oct 2016 14:14:44 +0200
Subject: [PATCH 08/20] Add test to check cv splits consistency in
 validation_curve

---
 .../model_selection/tests/test_validation.py  | 28 +++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/sklearn/model_selection/tests/test_validation.py b/sklearn/model_selection/tests/test_validation.py
index eb29be1a2ad4a..0fc88dd0bc196 100644
--- a/sklearn/model_selection/tests/test_validation.py
+++ b/sklearn/model_selection/tests/test_validation.py
@@ -61,6 +61,7 @@
 from sklearn.datasets import make_multilabel_classification
 
 from sklearn.model_selection.tests.test_split import MockClassifier
+from sklearn.model_selection.tests.test_search import CustomSplitter
 
 
 try:
@@ -764,6 +765,33 @@ def test_validation_curve():
     assert_array_almost_equal(test_scores.mean(axis=1), 1 - param_range)
 
 
+def test_validation_curve_cv_splits_consistency():
+    scores1 = validation_curve(LinearSVC(random_state=0), X, y,
+                               'C', [0.1, 0.1, 0.2, 0.2], cv=CustomSplitter())
+    # The CustomSplitter is a non-re-entrant cv splitter. Unless, the
+    # `split` is called for each parameter, the following should produce
+    # identical results for param setting 1 param setting 2 as both have
+    # the same C value.
+    assert_array_almost_equal(*np.vsplit(np.hstack(scores1)[(0, 2, 1, 3), :],
+                                         2))
+
+    scores2 = validation_curve(LinearSVC(random_state=0), X, y,
+                               'C', [0.1, 0.1, 0.2, 0.2],
+                               cv=KFold(n_splits=5, shuffle=True))
+
+    # For scores2, compare the 1st and 2nd parameter's scores
+    # (Since the C value for 1st two param setting is 0.1, they must be
+    # consistent unless the train test folds differ between the param settings)
+    assert_array_almost_equal(*np.vsplit(np.hstack(scores2)[(0, 2, 1, 3), :],
+                                         2))
+
+    scores3 = validation_curve(LinearSVC(random_state=0), X, y,
+                               'C', [0.1, 0.1, 0.2, 0.2], cv=KFold(n_splits=5))
+
+    # CustomSplitter is basically unshuffled KFold(n_splits=5). Sanity check.
+    assert_array_almost_equal(np.array(scores3), np.array(scores1))
+
+
 def test_check_is_permutation():
     rng = np.random.RandomState(0)
     p = np.arange(100)

From b95e5373c7ebddaa7428abff7f4ec0dd91a05fac Mon Sep 17 00:00:00 2001
From: Raghav RV <rvraghav93@gmail.com>
Date: Sun, 16 Oct 2016 18:24:56 +0200
Subject: [PATCH 09/20] FIX/Improve tests for cv consistency

---
 .../model_selection/tests/test_validation.py  | 102 ++++++++++++++++--
 1 file changed, 92 insertions(+), 10 deletions(-)

diff --git a/sklearn/model_selection/tests/test_validation.py b/sklearn/model_selection/tests/test_validation.py
index 0fc88dd0bc196..76acf2bce648a 100644
--- a/sklearn/model_selection/tests/test_validation.py
+++ b/sklearn/model_selection/tests/test_validation.py
@@ -111,6 +111,38 @@ def partial_fit(self, X, y=None, **params):
         self.x = X[0]
 
 
+class MockImprovingEstimatorRoundedScores(BaseEstimator):
+    """Dummy classifier to test the learning curve
+
+    This will give scores with precision of 0.1 so as to produce similar scores
+    for almost similar train fractions
+    """
+    def __init__(self, n_max_train_sizes):
+        self.n_max_train_sizes = n_max_train_sizes
+        self.train_sizes = 0
+        self.X_subset = None
+
+    def fit(self, X_subset, y_subset=None):
+        self.X_subset = X_subset
+        self.train_sizes = X_subset.shape[0]
+        return self
+
+    def predict(self, X):
+        raise NotImplementedError
+
+    def score(self, X=None, Y=None):
+        # training score becomes worse (2 -> 1), test error better (0 -> 1)
+        if self._is_training_data(X):
+            return np.round((2. - float(self.train_sizes) /
+                             self.n_max_train_sizes), 1)
+        else:
+            return np.round((float(self.train_sizes) /
+                             self.n_max_train_sizes), 1)
+
+    def _is_training_data(self, X):
+        return X is self.X_subset
+
+
 class MockEstimatorWithParameter(BaseEstimator):
     """Dummy classifier to test the validation curve"""
     def __init__(self, param=0.5):
@@ -557,14 +589,17 @@ def test_cross_val_score_sparse_fit_params():
 
 
 def test_learning_curve():
-    X, y = make_classification(n_samples=30, n_features=1, n_informative=1,
-                               n_redundant=0, n_classes=2,
+    n_samples = 30
+    n_splits = 3
+    X, y = make_classification(n_samples=n_samples, n_features=1,
+                               n_informative=1, n_redundant=0, n_classes=2,
                                n_clusters_per_class=1, random_state=0)
-    estimator = MockImprovingEstimator(20)
+    estimator = MockImprovingEstimator(n_samples * ((n_splits - 1) / n_splits))
     for shuffle_train in [False, True]:
         with warnings.catch_warnings(record=True) as w:
             train_sizes, train_scores, test_scores = learning_curve(
-                estimator, X, y, cv=3, train_sizes=np.linspace(0.1, 1.0, 10),
+                estimator, X, y, cv=KFold(n_splits=n_splits),
+                train_sizes=np.linspace(0.1, 1.0, 10),
                 shuffle=shuffle_train)
         if len(w) > 0:
             raise RuntimeError("Unexpected warning: %r" % w[0].message)
@@ -576,6 +611,46 @@ def test_learning_curve():
         assert_array_almost_equal(test_scores.mean(axis=1),
                                   np.linspace(0.1, 1.0, 10))
 
+        # Test a custom cv splitter that can iterate only once
+        with warnings.catch_warnings(record=True) as w:
+            train_sizes2, train_scores2, test_scores2 = learning_curve(
+                estimator, X, y,
+                cv=CustomSplitter(n_splits=n_splits, n_samples=30),
+                train_sizes=np.linspace(0.1, 1.0, 10),
+                shuffle=shuffle_train)
+        if len(w) > 0:
+            raise RuntimeError("Unexpected warning: %r" % w[0].message)
+        assert_array_almost_equal(train_scores2, train_scores)
+        assert_array_almost_equal(test_scores2, test_scores)
+
+        # Test consistency of folds in non-deterministic cv splitter
+        n_samples = 300
+        n_splits = 3
+        X, y = np.zeros((n_samples, 3)), np.ones((n_samples, 1))
+
+        estimator = MockImprovingEstimatorRoundedScores(
+            n_samples * ((n_splits - 1) / n_splits))
+
+        with warnings.catch_warnings(record=True) as w:
+            train_sizes3, train_scores3, test_scores3 = learning_curve(
+                estimator, X, y, cv=KFold(n_splits=n_splits, shuffle=True),
+                train_sizes=(0.1, 0.11, 0.9, 0.91),
+                shuffle=shuffle_train)
+        if len(w) > 0:
+            raise RuntimeError("Unexpected warning: %r" % w[0].message)
+
+        # KFold(..., shuffle=True) is non deterministic as random_state
+        # is not set.
+        # However this should not affect the consistency of cv folds across the
+        # different train sizes to ensure that the cross-vadiation scores
+        # remain comparable.
+        # The MockImprovingEstimatorRoundedScores produces similar scores for
+        # train sizes (0.1 and 0.15) and (0.9 and 0.95)
+        assert_array_almost_equal(train_scores3[0, :], train_scores3[1, :])
+        assert_array_almost_equal(train_scores3[2, :], train_scores3[3, :])
+        assert_array_almost_equal(test_scores3[0, :], test_scores3[1, :])
+        assert_array_almost_equal(test_scores3[2, :], test_scores3[3, :])
+
 
 def test_learning_curve_unsupervised():
     X, _ = make_classification(n_samples=30, n_features=1, n_informative=1,
@@ -766,8 +841,14 @@ def test_validation_curve():
 
 
 def test_validation_curve_cv_splits_consistency():
-    scores1 = validation_curve(LinearSVC(random_state=0), X, y,
-                               'C', [0.1, 0.1, 0.2, 0.2], cv=CustomSplitter())
+    n_samples = 100
+    n_splits = 5
+    X, y = make_classification(n_samples=100, random_state=0)
+
+    scores1 = validation_curve(SVC(kernel='linear', random_state=0), X, y,
+                               'C', [0.1, 0.1, 0.2, 0.2],
+                               cv=CustomSplitter(n_splits=n_splits,
+                                                 n_samples=n_samples))
     # The CustomSplitter is a non-re-entrant cv splitter. Unless, the
     # `split` is called for each parameter, the following should produce
     # identical results for param setting 1 param setting 2 as both have
@@ -775,9 +856,9 @@ def test_validation_curve_cv_splits_consistency():
     assert_array_almost_equal(*np.vsplit(np.hstack(scores1)[(0, 2, 1, 3), :],
                                          2))
 
-    scores2 = validation_curve(LinearSVC(random_state=0), X, y,
+    scores2 = validation_curve(SVC(kernel='linear', random_state=0), X, y,
                                'C', [0.1, 0.1, 0.2, 0.2],
-                               cv=KFold(n_splits=5, shuffle=True))
+                               cv=KFold(n_splits=n_splits, shuffle=True))
 
     # For scores2, compare the 1st and 2nd parameter's scores
     # (Since the C value for 1st two param setting is 0.1, they must be
@@ -785,8 +866,9 @@ def test_validation_curve_cv_splits_consistency():
     assert_array_almost_equal(*np.vsplit(np.hstack(scores2)[(0, 2, 1, 3), :],
                                          2))
 
-    scores3 = validation_curve(LinearSVC(random_state=0), X, y,
-                               'C', [0.1, 0.1, 0.2, 0.2], cv=KFold(n_splits=5))
+    scores3 = validation_curve(SVC(kernel='linear', random_state=0), X, y,
+                               'C', [0.1, 0.1, 0.2, 0.2],
+                               cv=KFold(n_splits=n_splits))
 
     # CustomSplitter is basically unshuffled KFold(n_splits=5). Sanity check.
     assert_array_almost_equal(np.array(scores3), np.array(scores1))

From 0ea6a4ca74186b4e12ca631b78a3041ee9765f43 Mon Sep 17 00:00:00 2001
From: Raghav RV <rvraghav93@gmail.com>
Date: Sun, 16 Oct 2016 18:29:15 +0200
Subject: [PATCH 10/20] FIX/TST cv consistency tests in grid search

---
 sklearn/model_selection/tests/test_search.py | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/sklearn/model_selection/tests/test_search.py b/sklearn/model_selection/tests/test_search.py
index 258beb9b2385b..47531bce4458d 100644
--- a/sklearn/model_selection/tests/test_search.py
+++ b/sklearn/model_selection/tests/test_search.py
@@ -64,6 +64,8 @@
 class CustomSplitter():
     """A wrapper to make KFold single entry cv iterator"""
     def __init__(self, n_splits=4, n_samples=99):
+        self.n_splits = n_splits
+        self.n_samples = n_samples
         self.indices = KFold(n_splits=n_splits).split(np.ones(n_samples))
 
     def split(self, X=None, y=None, groups=None):
@@ -72,7 +74,7 @@ def split(self, X=None, y=None, groups=None):
             yield index
 
     def get_n_splits(self, X=None, y=None, groups=None):
-        return 4
+        return self.n_splits
 
 
 # Neither of the following two estimators inherit from BaseEstimator,
@@ -1172,14 +1174,19 @@ def test_search_train_scores_set_to_false():
 
 def test_grid_search_cv_splits_consistency():
     # Check if a one time iterable is accepted as a cv parameter.
-    X, y = make_classification(n_samples=100, random_state=0)
+    n_samples = 100
+    n_splits = 5
+    X, y = make_classification(n_samples=n_samples, random_state=0)
 
     gs = GridSearchCV(LinearSVC(random_state=0),
-                      param_grid={'C': [0.1, 0.2, 0.3]}, cv=CustomSplitter())
+                      param_grid={'C': [0.1, 0.2, 0.3]},
+                      cv=CustomSplitter(n_splits=n_splits,
+                                        n_samples=n_samples))
     gs.fit(X, y)
 
     gs2 = GridSearchCV(LinearSVC(random_state=0),
-                       param_grid={'C': [0.1, 0.2, 0.3]}, cv=KFold(n_splits=5))
+                       param_grid={'C': [0.1, 0.2, 0.3]},
+                       cv=KFold(n_splits=n_splits))
     gs2.fit(X, y)
 
     def _pop_time_keys(cv_results):
@@ -1200,7 +1207,7 @@ def _pop_time_keys(cv_results):
     # Check consistency of folds across the parameters
     gs = GridSearchCV(LinearSVC(random_state=0),
                       param_grid={'C': [0.1, 0.1, 0.2, 0.2]},
-                      cv=KFold(n_splits=5, shuffle=True))
+                      cv=KFold(n_splits=n_splits, shuffle=True))
     gs.fit(X, y)
 
     # As the first two param settings (C=0.1) and the next two param
@@ -1211,7 +1218,7 @@ def _pop_time_keys(cv_results):
         per_param_scores = {}
         for param_i in range(4):
             per_param_scores[param_i] = list(
-                gs.cv_results_['split%d_%s_score' % (s, %score_type)][param_i]
+                gs.cv_results_['split%d_%s_score' % (s, score_type)][param_i]
                 for s in range(5))
 
         assert_array_almost_equal(per_param_scores[0],

From 59b9f6c71af031c3ed3e5589c520fc6fbedd627e Mon Sep 17 00:00:00 2001
From: Raghav RV <rvraghav93@gmail.com>
Date: Sun, 16 Oct 2016 18:42:39 +0200
Subject: [PATCH 11/20] Arggh. This won't test consistency of splits. The
 scores not dependent on splits

---
 .../model_selection/tests/test_validation.py  | 45 +------------------
 1 file changed, 1 insertion(+), 44 deletions(-)

diff --git a/sklearn/model_selection/tests/test_validation.py b/sklearn/model_selection/tests/test_validation.py
index 76acf2bce648a..49c19dcb0a8a3 100644
--- a/sklearn/model_selection/tests/test_validation.py
+++ b/sklearn/model_selection/tests/test_validation.py
@@ -111,38 +111,6 @@ def partial_fit(self, X, y=None, **params):
         self.x = X[0]
 
 
-class MockImprovingEstimatorRoundedScores(BaseEstimator):
-    """Dummy classifier to test the learning curve
-
-    This will give scores with precision of 0.1 so as to produce similar scores
-    for almost similar train fractions
-    """
-    def __init__(self, n_max_train_sizes):
-        self.n_max_train_sizes = n_max_train_sizes
-        self.train_sizes = 0
-        self.X_subset = None
-
-    def fit(self, X_subset, y_subset=None):
-        self.X_subset = X_subset
-        self.train_sizes = X_subset.shape[0]
-        return self
-
-    def predict(self, X):
-        raise NotImplementedError
-
-    def score(self, X=None, Y=None):
-        # training score becomes worse (2 -> 1), test error better (0 -> 1)
-        if self._is_training_data(X):
-            return np.round((2. - float(self.train_sizes) /
-                             self.n_max_train_sizes), 1)
-        else:
-            return np.round((float(self.train_sizes) /
-                             self.n_max_train_sizes), 1)
-
-    def _is_training_data(self, X):
-        return X is self.X_subset
-
-
 class MockEstimatorWithParameter(BaseEstimator):
     """Dummy classifier to test the validation curve"""
     def __init__(self, param=0.5):
@@ -595,6 +563,7 @@ def test_learning_curve():
                                n_informative=1, n_redundant=0, n_classes=2,
                                n_clusters_per_class=1, random_state=0)
     estimator = MockImprovingEstimator(n_samples * ((n_splits - 1) / n_splits))
+<<<<<<< 0ea6a4ca74186b4e12ca631b78a3041ee9765f43
     for shuffle_train in [False, True]:
         with warnings.catch_warnings(record=True) as w:
             train_sizes, train_scores, test_scores = learning_curve(
@@ -639,18 +608,6 @@ def test_learning_curve():
         if len(w) > 0:
             raise RuntimeError("Unexpected warning: %r" % w[0].message)
 
-        # KFold(..., shuffle=True) is non deterministic as random_state
-        # is not set.
-        # However this should not affect the consistency of cv folds across the
-        # different train sizes to ensure that the cross-vadiation scores
-        # remain comparable.
-        # The MockImprovingEstimatorRoundedScores produces similar scores for
-        # train sizes (0.1 and 0.15) and (0.9 and 0.95)
-        assert_array_almost_equal(train_scores3[0, :], train_scores3[1, :])
-        assert_array_almost_equal(train_scores3[2, :], train_scores3[3, :])
-        assert_array_almost_equal(test_scores3[0, :], test_scores3[1, :])
-        assert_array_almost_equal(test_scores3[2, :], test_scores3[3, :])
-
 
 def test_learning_curve_unsupervised():
     X, _ = make_classification(n_samples=30, n_features=1, n_informative=1,

From 63acab0a9f94a2f523975e32a2744bf100a1f2c8 Mon Sep 17 00:00:00 2001
From: Raghav RV <rvraghav93@gmail.com>
Date: Sun, 16 Oct 2016 19:24:04 +0200
Subject: [PATCH 12/20] enlist

---
 sklearn/model_selection/_validation.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sklearn/model_selection/_validation.py b/sklearn/model_selection/_validation.py
index 15d086c9b50ea..c087264d39f34 100644
--- a/sklearn/model_selection/_validation.py
+++ b/sklearn/model_selection/_validation.py
@@ -960,7 +960,7 @@ def validation_curve(estimator, X, y, param_name, param_range, groups=None,
     X, y, groups = indexable(X, y, groups)
 
     cv = check_cv(cv, y, classifier=is_classifier(estimator))
-    cv_iter = cv.split(X, y, groups)
+    cv_iter = list(cv.split(X, y, groups))
 
     scorer = check_scoring(estimator, scoring=scoring)
 

From 1af6f37b14e79ae58bb5b3d3d644d2c610a27ac8 Mon Sep 17 00:00:00 2001
From: Raghav RV <rvraghav93@gmail.com>
Date: Tue, 18 Oct 2016 17:22:20 +0200
Subject: [PATCH 13/20] TST Add test for _CVIterableWrapper to check for
 reusability

---
 sklearn/model_selection/tests/common.py       | 23 +++++
 sklearn/model_selection/tests/test_search.py  | 16 +---
 sklearn/model_selection/tests/test_split.py   | 72 ++--------------
 .../model_selection/tests/test_validation.py  | 86 ++++++++++++++-----
 4 files changed, 98 insertions(+), 99 deletions(-)
 create mode 100644 sklearn/model_selection/tests/common.py

diff --git a/sklearn/model_selection/tests/common.py b/sklearn/model_selection/tests/common.py
new file mode 100644
index 0000000000000..f20b9436f3ffd
--- /dev/null
+++ b/sklearn/model_selection/tests/common.py
@@ -0,0 +1,23 @@
+"""
+Common utilities for testing model selection.
+"""
+
+import numpy as np
+
+from sklearn.model_selection import KFold
+
+
+class CustomSplitter:
+    """A wrapper to make KFold single entry cv iterator"""
+    def __init__(self, n_splits=4, n_samples=99):
+        self.n_splits = n_splits
+        self.n_samples = n_samples
+        self.indices = KFold(n_splits=n_splits).split(np.ones(n_samples))
+
+    def split(self, X=None, y=None, groups=None):
+        """Split can be called only once"""
+        for index in self.indices:
+            yield index
+
+    def get_n_splits(self, X=None, y=None, groups=None):
+        return self.n_splits
diff --git a/sklearn/model_selection/tests/test_search.py b/sklearn/model_selection/tests/test_search.py
index 47531bce4458d..17b452e77d93a 100644
--- a/sklearn/model_selection/tests/test_search.py
+++ b/sklearn/model_selection/tests/test_search.py
@@ -60,21 +60,7 @@
 from sklearn.pipeline import Pipeline
 from sklearn.linear_model import SGDClassifier
 
-
-class CustomSplitter():
-    """A wrapper to make KFold single entry cv iterator"""
-    def __init__(self, n_splits=4, n_samples=99):
-        self.n_splits = n_splits
-        self.n_samples = n_samples
-        self.indices = KFold(n_splits=n_splits).split(np.ones(n_samples))
-
-    def split(self, X=None, y=None, groups=None):
-        """Split can be called only once"""
-        for index in self.indices:
-            yield index
-
-    def get_n_splits(self, X=None, y=None, groups=None):
-        return self.n_splits
+from sklearn.model_selection.tests.common import CustomSplitter
 
 
 # Neither of the following two estimators inherit from BaseEstimator,
diff --git a/sklearn/model_selection/tests/test_split.py b/sklearn/model_selection/tests/test_split.py
index 4dcd8f55038d8..800908808a679 100644
--- a/sklearn/model_selection/tests/test_split.py
+++ b/sklearn/model_selection/tests/test_split.py
@@ -59,73 +59,9 @@
 
 X = np.ones(10)
 y = np.arange(10) // 2
-P_sparse = coo_matrix(np.eye(5))
 digits = load_digits()
 
 
-class MockClassifier(object):
-    """Dummy classifier to test the cross-validation"""
-
-    def __init__(self, a=0, allow_nd=False):
-        self.a = a
-        self.allow_nd = allow_nd
-
-    def fit(self, X, Y=None, sample_weight=None, class_prior=None,
-            sparse_sample_weight=None, sparse_param=None, dummy_int=None,
-            dummy_str=None, dummy_obj=None, callback=None):
-        """The dummy arguments are to test that this fit function can
-        accept non-array arguments through cross-validation, such as:
-            - int
-            - str (this is actually array-like)
-            - object
-            - function
-        """
-        self.dummy_int = dummy_int
-        self.dummy_str = dummy_str
-        self.dummy_obj = dummy_obj
-        if callback is not None:
-            callback(self)
-
-        if self.allow_nd:
-            X = X.reshape(len(X), -1)
-        if X.ndim >= 3 and not self.allow_nd:
-            raise ValueError('X cannot be d')
-        if sample_weight is not None:
-            assert_true(sample_weight.shape[0] == X.shape[0],
-                        'MockClassifier extra fit_param sample_weight.shape[0]'
-                        ' is {0}, should be {1}'.format(sample_weight.shape[0],
-                                                        X.shape[0]))
-        if class_prior is not None:
-            assert_true(class_prior.shape[0] == len(np.unique(y)),
-                        'MockClassifier extra fit_param class_prior.shape[0]'
-                        ' is {0}, should be {1}'.format(class_prior.shape[0],
-                                                        len(np.unique(y))))
-        if sparse_sample_weight is not None:
-            fmt = ('MockClassifier extra fit_param sparse_sample_weight'
-                   '.shape[0] is {0}, should be {1}')
-            assert_true(sparse_sample_weight.shape[0] == X.shape[0],
-                        fmt.format(sparse_sample_weight.shape[0], X.shape[0]))
-        if sparse_param is not None:
-            fmt = ('MockClassifier extra fit_param sparse_param.shape '
-                   'is ({0}, {1}), should be ({2}, {3})')
-            assert_true(sparse_param.shape == P_sparse.shape,
-                        fmt.format(sparse_param.shape[0],
-                                   sparse_param.shape[1],
-                                   P_sparse.shape[0], P_sparse.shape[1]))
-        return self
-
-    def predict(self, T):
-        if self.allow_nd:
-            T = T.reshape(len(T), -1)
-        return T[:, 0]
-
-    def score(self, X=None, Y=None):
-        return 1. / (1 + np.abs(self.a))
-
-    def get_params(self, deep=False):
-        return {'a': self.a, 'allow_nd': self.allow_nd}
-
-
 @ignore_warnings
 def test_cross_validator_with_default_params():
     n_samples = 4
@@ -908,6 +844,14 @@ def test_cv_iterable_wrapper():
     # Check if get_n_splits works correctly
     assert_equal(len(cv), wrapped_old_skf.get_n_splits())
 
+    kf_iter = KFold(n_splits=5).split(X, y)
+    kf_iter_wrapped = check_cv(kf_iter)
+    # Since the wrapped iterable is enlisted and stored,
+    # split can be called any number of times to produce
+    # consistent results
+    assert_array_equal(list(kf_iter_wrapped.split(X, y)),
+                       list(kf_iter_wrapped.split(X, y)))
+
 
 def test_group_kfold():
     rng = np.random.RandomState(0)
diff --git a/sklearn/model_selection/tests/test_validation.py b/sklearn/model_selection/tests/test_validation.py
index 49c19dcb0a8a3..f467a1666ed95 100644
--- a/sklearn/model_selection/tests/test_validation.py
+++ b/sklearn/model_selection/tests/test_validation.py
@@ -60,8 +60,7 @@
 from sklearn.datasets import make_classification
 from sklearn.datasets import make_multilabel_classification
 
-from sklearn.model_selection.tests.test_split import MockClassifier
-from sklearn.model_selection.tests.test_search import CustomSplitter
+from sklearn.model_selection.tests.common import CustomSplitter
 
 
 try:
@@ -132,6 +131,69 @@ def _is_training_data(self, X):
         return X is self.X_subset
 
 
+class MockClassifier(object):
+    """Dummy classifier to test the cross-validation"""
+
+    def __init__(self, a=0, allow_nd=False):
+        self.a = a
+        self.allow_nd = allow_nd
+
+    def fit(self, X, Y=None, sample_weight=None, class_prior=None,
+            sparse_sample_weight=None, sparse_param=None, dummy_int=None,
+            dummy_str=None, dummy_obj=None, callback=None):
+        """The dummy arguments are to test that this fit function can
+        accept non-array arguments through cross-validation, such as:
+            - int
+            - str (this is actually array-like)
+            - object
+            - function
+        """
+        self.dummy_int = dummy_int
+        self.dummy_str = dummy_str
+        self.dummy_obj = dummy_obj
+        if callback is not None:
+            callback(self)
+
+        if self.allow_nd:
+            X = X.reshape(len(X), -1)
+        if X.ndim >= 3 and not self.allow_nd:
+            raise ValueError('X cannot be d')
+        if sample_weight is not None:
+            assert_true(sample_weight.shape[0] == X.shape[0],
+                        'MockClassifier extra fit_param sample_weight.shape[0]'
+                        ' is {0}, should be {1}'.format(sample_weight.shape[0],
+                                                        X.shape[0]))
+        if class_prior is not None:
+            assert_true(class_prior.shape[0] == len(np.unique(y)),
+                        'MockClassifier extra fit_param class_prior.shape[0]'
+                        ' is {0}, should be {1}'.format(class_prior.shape[0],
+                                                        len(np.unique(y))))
+        if sparse_sample_weight is not None:
+            fmt = ('MockClassifier extra fit_param sparse_sample_weight'
+                   '.shape[0] is {0}, should be {1}')
+            assert_true(sparse_sample_weight.shape[0] == X.shape[0],
+                        fmt.format(sparse_sample_weight.shape[0], X.shape[0]))
+        if sparse_param is not None:
+            fmt = ('MockClassifier extra fit_param sparse_param.shape '
+                   'is ({0}, {1}), should be ({2}, {3})')
+            assert_true(sparse_param.shape == P_sparse.shape,
+                        fmt.format(sparse_param.shape[0],
+                                   sparse_param.shape[1],
+                                   P_sparse.shape[0], P_sparse.shape[1]))
+        return self
+
+    def predict(self, T):
+        if self.allow_nd:
+            T = T.reshape(len(T), -1)
+        return T[:, 0]
+
+    def score(self, X=None, Y=None):
+        return 1. / (1 + np.abs(self.a))
+
+    def get_params(self, deep=False):
+        return {'a': self.a, 'allow_nd': self.allow_nd}
+
+
 # XXX: use 2D array, since 1D X is being detected as a single sample in
 # check_consistent_length
 X = np.ones((10, 2))
@@ -140,6 +202,7 @@ def _is_training_data(self, X):
 # The number of samples per class needs to be > n_splits,
 # for StratifiedKFold(n_splits=3)
 y2 = np.array([1, 1, 1, 2, 2, 2, 3, 3, 3, 3])
+P_sparse = coo_matrix(np.eye(5))
 
 
 def test_cross_val_score():
@@ -563,7 +626,6 @@ def test_learning_curve():
                                n_informative=1, n_redundant=0, n_classes=2,
                                n_clusters_per_class=1, random_state=0)
     estimator = MockImprovingEstimator(n_samples * ((n_splits - 1) / n_splits))
-<<<<<<< 0ea6a4ca74186b4e12ca631b78a3041ee9765f43
     for shuffle_train in [False, True]:
         with warnings.catch_warnings(record=True) as w:
             train_sizes, train_scores, test_scores = learning_curve(
@@ -584,7 +646,7 @@ def test_learning_curve():
         with warnings.catch_warnings(record=True) as w:
             train_sizes2, train_scores2, test_scores2 = learning_curve(
                 estimator, X, y,
-                cv=CustomSplitter(n_splits=n_splits, n_samples=30),
+                cv=CustomSplitter(n_splits=n_splits, n_samples=n_samples),
                 train_sizes=np.linspace(0.1, 1.0, 10),
                 shuffle=shuffle_train)
         if len(w) > 0:
@@ -592,22 +654,6 @@ def test_learning_curve():
         assert_array_almost_equal(train_scores2, train_scores)
         assert_array_almost_equal(test_scores2, test_scores)
 
-        # Test consistency of folds in non-deterministic cv splitter
-        n_samples = 300
-        n_splits = 3
-        X, y = np.zeros((n_samples, 3)), np.ones((n_samples, 1))
-
-        estimator = MockImprovingEstimatorRoundedScores(
-            n_samples * ((n_splits - 1) / n_splits))
-
-        with warnings.catch_warnings(record=True) as w:
-            train_sizes3, train_scores3, test_scores3 = learning_curve(
-                estimator, X, y, cv=KFold(n_splits=n_splits, shuffle=True),
-                train_sizes=(0.1, 0.11, 0.9, 0.91),
-                shuffle=shuffle_train)
-        if len(w) > 0:
-            raise RuntimeError("Unexpected warning: %r" % w[0].message)
-
 
 def test_learning_curve_unsupervised():
     X, _ = make_classification(n_samples=30, n_features=1, n_informative=1,

From f485c38f64a079569773b07d2fcac7a84ed0c2e6 Mon Sep 17 00:00:00 2001
From: Raghav RV <rvraghav93@gmail.com>
Date: Thu, 20 Oct 2016 15:03:23 +0200
Subject: [PATCH 14/20] CustomSplitter --> OneTimeSplitter; use iter to ensure
 one time splitting

---
 sklearn/model_selection/tests/common.py          |  4 ++--
 sklearn/model_selection/tests/test_search.py     |  8 ++++----
 sklearn/model_selection/tests/test_validation.py | 12 ++++++------
 3 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/sklearn/model_selection/tests/common.py b/sklearn/model_selection/tests/common.py
index f20b9436f3ffd..13549eef377b7 100644
--- a/sklearn/model_selection/tests/common.py
+++ b/sklearn/model_selection/tests/common.py
@@ -7,12 +7,12 @@
 from sklearn.model_selection import KFold
 
 
-class CustomSplitter:
+class OneTimeSplitter:
     """A wrapper to make KFold single entry cv iterator"""
     def __init__(self, n_splits=4, n_samples=99):
         self.n_splits = n_splits
         self.n_samples = n_samples
-        self.indices = KFold(n_splits=n_splits).split(np.ones(n_samples))
+        self.indices = iter(KFold(n_splits=n_splits).split(np.ones(n_samples)))
 
     def split(self, X=None, y=None, groups=None):
         """Split can be called only once"""
diff --git a/sklearn/model_selection/tests/test_search.py b/sklearn/model_selection/tests/test_search.py
index 17b452e77d93a..924a290fd4d7d 100644
--- a/sklearn/model_selection/tests/test_search.py
+++ b/sklearn/model_selection/tests/test_search.py
@@ -60,7 +60,7 @@
 from sklearn.pipeline import Pipeline
 from sklearn.linear_model import SGDClassifier
 
-from sklearn.model_selection.tests.common import CustomSplitter
+from sklearn.model_selection.tests.common import OneTimeSplitter
 
 
 # Neither of the following two estimators inherit from BaseEstimator,
@@ -1166,8 +1166,8 @@ def test_grid_search_cv_splits_consistency():
 
     gs = GridSearchCV(LinearSVC(random_state=0),
                       param_grid={'C': [0.1, 0.2, 0.3]},
-                      cv=CustomSplitter(n_splits=n_splits,
-                                        n_samples=n_samples))
+                      cv=OneTimeSplitter(n_splits=n_splits,
+                                         n_samples=n_samples))
     gs.fit(X, y)
 
     gs2 = GridSearchCV(LinearSVC(random_state=0),
@@ -1181,7 +1181,7 @@ def _pop_time_keys(cv_results):
             cv_results.pop(key)
         return cv_results
 
-    # CustomSplitter is a non-re-entrant cv where split can be called only once
+    # OneTimeSplitter is a non-re-entrant cv where split can be called only once
     # if ``cv.split`` is called once per param setting in GridSearchCV.fit
     # the 2nd and 3rd parameter will not be evaluated as no train/test indices
     # will be generated for the 2nd and subsequent cv.split calls.
diff --git a/sklearn/model_selection/tests/test_validation.py b/sklearn/model_selection/tests/test_validation.py
index f467a1666ed95..cf7c90aec11fb 100644
--- a/sklearn/model_selection/tests/test_validation.py
+++ b/sklearn/model_selection/tests/test_validation.py
@@ -60,7 +60,7 @@
 from sklearn.datasets import make_classification
 from sklearn.datasets import make_multilabel_classification
 
-from sklearn.model_selection.tests.common import CustomSplitter
+from sklearn.model_selection.tests.common import OneTimeSplitter
 
 
 try:
@@ -646,7 +646,7 @@ def test_learning_curve():
         with warnings.catch_warnings(record=True) as w:
             train_sizes2, train_scores2, test_scores2 = learning_curve(
                 estimator, X, y,
-                cv=CustomSplitter(n_splits=n_splits, n_samples=n_samples),
+                cv=OneTimeSplitter(n_splits=n_splits, n_samples=n_samples),
                 train_sizes=np.linspace(0.1, 1.0, 10),
                 shuffle=shuffle_train)
         if len(w) > 0:
@@ -850,9 +850,9 @@ def test_validation_curve_cv_splits_consistency():
 
     scores1 = validation_curve(SVC(kernel='linear', random_state=0), X, y,
                                'C', [0.1, 0.1, 0.2, 0.2],
-                               cv=CustomSplitter(n_splits=n_splits,
-                                                 n_samples=n_samples))
-    # The CustomSplitter is a non-re-entrant cv splitter. Unless, the
+                               cv=OneTimeSplitter(n_splits=n_splits,
+                                                  n_samples=n_samples))
+    # The OneTimeSplitter is a non-re-entrant cv splitter. Unless, the
     # `split` is called for each parameter, the following should produce
     # identical results for param setting 1 param setting 2 as both have
     # the same C value.
@@ -873,7 +873,7 @@ def test_validation_curve_cv_splits_consistency():
                                'C', [0.1, 0.1, 0.2, 0.2],
                                cv=KFold(n_splits=n_splits))
 
-    # CustomSplitter is basically unshuffled KFold(n_splits=5). Sanity check.
+    # OneTimeSplitter is basically unshuffled KFold(n_splits=5). Sanity check.
     assert_array_almost_equal(np.array(scores3), np.array(scores1))
 
 

From 252827152b3ee0b0de7aa7159af24eaab9f2bdd6 Mon Sep 17 00:00:00 2001
From: Raghav RV <rvraghav93@gmail.com>
Date: Thu, 20 Oct 2016 16:32:51 +0200
Subject: [PATCH 15/20] Flake8

---
 sklearn/model_selection/tests/test_search.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sklearn/model_selection/tests/test_search.py b/sklearn/model_selection/tests/test_search.py
index 924a290fd4d7d..61529c472fd0a 100644
--- a/sklearn/model_selection/tests/test_search.py
+++ b/sklearn/model_selection/tests/test_search.py
@@ -1181,8 +1181,8 @@ def _pop_time_keys(cv_results):
             cv_results.pop(key)
         return cv_results
 
-    # OneTimeSplitter is a non-re-entrant cv where split can be called only once
-    # if ``cv.split`` is called once per param setting in GridSearchCV.fit
+    # OneTimeSplitter is a non-re-entrant cv where split can be called only
+    # once if ``cv.split`` is called once per param setting in GridSearchCV.fit
     # the 2nd and 3rd parameter will not be evaluated as no train/test indices
     # will be generated for the 2nd and subsequent cv.split calls.
     # This is a check to make cv.split is not called once per param

From 1054b3a3d445776f65139d8e45a78b7c458a66ea Mon Sep 17 00:00:00 2001
From: Raghav RV <rvraghav93@gmail.com>
Date: Mon, 24 Oct 2016 14:31:57 +0200
Subject: [PATCH 16/20] TYPO

---
 sklearn/model_selection/tests/test_search.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sklearn/model_selection/tests/test_search.py b/sklearn/model_selection/tests/test_search.py
index 61529c472fd0a..1ce28755075a4 100644
--- a/sklearn/model_selection/tests/test_search.py
+++ b/sklearn/model_selection/tests/test_search.py
@@ -1185,7 +1185,7 @@ def _pop_time_keys(cv_results):
     # once if ``cv.split`` is called once per param setting in GridSearchCV.fit
     # the 2nd and 3rd parameter will not be evaluated as no train/test indices
     # will be generated for the 2nd and subsequent cv.split calls.
-    # This is a check to make cv.split is not called once per param
+    # This is a check to make sure cv.split is not called once per param
     # setting.
     np.testing.assert_equal(_pop_time_keys(gs.cv_results_),
                             _pop_time_keys(gs2.cv_results_))

From 59c5bf6393a83dbb5ee156c25a751b7eb78f9c32 Mon Sep 17 00:00:00 2001
From: Raghav RV <rvraghav93@gmail.com>
Date: Wed, 26 Oct 2016 15:40:50 +0200
Subject: [PATCH 17/20] COSMIT

---
 sklearn/model_selection/tests/test_validation.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sklearn/model_selection/tests/test_validation.py b/sklearn/model_selection/tests/test_validation.py
index cf7c90aec11fb..e2ccdd39ee9ea 100644
--- a/sklearn/model_selection/tests/test_validation.py
+++ b/sklearn/model_selection/tests/test_validation.py
@@ -854,7 +854,7 @@ def test_validation_curve_cv_splits_consistency():
                                                   n_samples=n_samples))
     # The OneTimeSplitter is a non-re-entrant cv splitter. Unless, the
     # `split` is called for each parameter, the following should produce
-    # identical results for param setting 1 param setting 2 as both have
+    # identical results for param setting 1 and param setting 2 as both have
     # the same C value.
     assert_array_almost_equal(*np.vsplit(np.hstack(scores1)[(0, 2, 1, 3), :],
                                          2))

From 2670f0d3aa32934d90557b80c8efedcba5e7e742 Mon Sep 17 00:00:00 2001
From: Raghav RV <rvraghav93@gmail.com>
Date: Wed, 26 Oct 2016 18:01:25 +0200
Subject: [PATCH 18/20] TST Check if shuffle=True produces different splits

---
 sklearn/model_selection/tests/test_split.py | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/sklearn/model_selection/tests/test_split.py b/sklearn/model_selection/tests/test_split.py
index 800908808a679..e2114b2d4302d 100644
--- a/sklearn/model_selection/tests/test_split.py
+++ b/sklearn/model_selection/tests/test_split.py
@@ -848,9 +848,23 @@ def test_cv_iterable_wrapper():
     kf_iter_wrapped = check_cv(kf_iter)
     # Since the wrapped iterable is enlisted and stored,
     # split can be called any number of times to produce
-    # consistent results
+    # consistent results.
     assert_array_equal(list(kf_iter_wrapped.split(X, y)),
                        list(kf_iter_wrapped.split(X, y)))
+    # If the splits are randomized, successive calls to split yields different
+    # results
+    kf_randomized_iter = KFold(n_splits=5, shuffle=True).split(X, y)
+    kf_randomized_iter_wrapped = check_cv(kf_iter)
+    assert_array_equal(list(kf_randomized_iter_wrapped.split(X, y)),
+                       list(kf_randomized_iter_wrapped.split(X, y)))
+    try:
+        assert_array_equal(list(kf_iter_wrapped.split(X, y)),
+                           list(kf_randomized_iter_wrapped.split(X, y)))
+    except AssertionError:
+        pass
+    else:
+        raise AssertionError("After randomization, wrapped kfold (with "
+                             "shuffle) did not produce differing results.")
 
 
 def test_group_kfold():

From 5e735e1420873408da8575edecd8d610e1b53a5c Mon Sep 17 00:00:00 2001
From: Raghav RV <rvraghav93@gmail.com>
Date: Thu, 27 Oct 2016 17:22:25 +0200
Subject: [PATCH 19/20] TYPO

---
 sklearn/model_selection/tests/test_split.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sklearn/model_selection/tests/test_split.py b/sklearn/model_selection/tests/test_split.py
index e2114b2d4302d..7cffe9a0093c0 100644
--- a/sklearn/model_selection/tests/test_split.py
+++ b/sklearn/model_selection/tests/test_split.py
@@ -854,7 +854,7 @@ def test_cv_iterable_wrapper():
     # If the splits are randomized, successive calls to split yields different
     # results
     kf_randomized_iter = KFold(n_splits=5, shuffle=True).split(X, y)
-    kf_randomized_iter_wrapped = check_cv(kf_iter)
+    kf_randomized_iter_wrapped = check_cv(kf_randomized_iter)
     assert_array_equal(list(kf_randomized_iter_wrapped.split(X, y)),
                        list(kf_randomized_iter_wrapped.split(X, y)))
     try:

From 756bd531855456e0ea65ab995e5a5838401da242 Mon Sep 17 00:00:00 2001
From: Raghav RV <rvraghav93@gmail.com>
Date: Thu, 27 Oct 2016 17:24:37 +0200
Subject: [PATCH 20/20] Use np.any+np.array to assert inequality of splits

---
 sklearn/model_selection/tests/test_split.py | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/sklearn/model_selection/tests/test_split.py b/sklearn/model_selection/tests/test_split.py
index 7cffe9a0093c0..f7291f3a82e0f 100644
--- a/sklearn/model_selection/tests/test_split.py
+++ b/sklearn/model_selection/tests/test_split.py
@@ -857,14 +857,8 @@ def test_cv_iterable_wrapper():
     kf_randomized_iter_wrapped = check_cv(kf_randomized_iter)
     assert_array_equal(list(kf_randomized_iter_wrapped.split(X, y)),
                        list(kf_randomized_iter_wrapped.split(X, y)))
-    try:
-        assert_array_equal(list(kf_iter_wrapped.split(X, y)),
-                           list(kf_randomized_iter_wrapped.split(X, y)))
-    except AssertionError:
-        pass
-    else:
-        raise AssertionError("After randomization, wrapped kfold (with "
-                             "shuffle) did not produce differing results.")
+    assert_true(np.any(np.array(list(kf_iter_wrapped.split(X, y))) !=
+                       np.array(list(kf_randomized_iter_wrapped.split(X, y)))))
 
 
 def test_group_kfold():