scikit-learn · fabianp · Jul 29, 2015 · Jan 18, 2016 · Feb 10, 2016 · Feb 11, 2016
diff --git a/doc/whats_new.rst b/doc/whats_new.rst
@@ -35,6 +35,9 @@ New features
    - Added new supervised learning algorithm: :ref:`Multi-layer Perceptron <multilayer_perceptron>`
      (`#3204 <https://github.com/scikit-learn/scikit-learn/pull/3204>`_) by `Issam H. Laradji`_
 
+   - Gaussian Process-based hyper-parameter optimization through :class:`gp_search.GPSearchCV`, by Sébastien Dubois, Djalel Benbouzid and Fabian Pedregosa.
+
+
 Enhancements
 ............
 

diff --git a/examples/model_selection/plot_gp_search.py b/examples/model_selection/plot_gp_search.py
@@ -0,0 +1,83 @@
+"""
+==================================================================
+Plotting Performance of different hyperparameter selection methods
+==================================================================
+
+In this plot you can see the validation scores of a Ridge regression model
+along steps of different hyperparameter selection methods.
+"""
+print(__doc__)
+
+import matplotlib.pyplot as plt
+
+import numpy as np
+from scipy import stats
+
+from sklearn.model_selection import SequentialSearchCV
+from sklearn.model_selection import GridSearchCV
+from sklearn.model_selection import RandomizedSearchCV
+from sklearn.linear_model import Ridge, LogisticRegression
+from sklearn.datasets import load_iris, make_classification, load_digits
+from sklearn.gaussian_process.kernels import Matern
+
+
+# Make synthetic dataset where not all the features are explanatory
+# iris = load_iris()
+# rng = np.random.RandomState(0)
+# X, y = iris.data, iris.target
+digits = load_digits()
+X, y = digits.data, digits.target
+
+clf = LogisticRegression()
+
+params = {'C': {'bounds': [10**-3, 10**3], 'scale': 'log'}}
+
+# Run SequentialSearch with first 5 iterations random.
+gp_search = SequentialSearchCV(
+  clf, params, n_iter=20, random_state=0, n_init=3)
+gp_search.fit(X, y)
+
+# Retrieve the maximum score at each iteration
+gp_cum_scores = [np.max(
+      [gp_search.grid_scores_[j].mean_validation_score for j in range(i)])
+      for i in range(1, len(gp_search.grid_scores_))]
+
+gp_ext = SequentialSearchCV(
+  clf, params, n_iter=20, random_state=0, n_init=3, search='extensive')
+gp_ext.fit(X, y)
+
+# Retrieve the maximum score at each iteration
+gp_ext_cum_scores = [np.max(
+      [gp_ext.grid_scores_[j].mean_validation_score for j in range(i)])
+      for i in range(1, len(gp_ext.grid_scores_))]
+
+
+# Do the same experiment with randomized search cv
+params = {'C': stats.expon(scale=10**-1)}
+rdm_search = RandomizedSearchCV(
+  clf, params, random_state=0, n_iter=20)
+rdm_search.fit(X, y)
+
+rdm_cum_scores = [np.max(
+      [rdm_search.grid_scores_[j].mean_validation_score for j in range(i)])
+      for i in range(1, len(rdm_search.grid_scores_))]
+
+# Do a standard grid search across pre-defined parameters.
+params = {'C': np.logspace(-3, 3, 20)}
+
+grid_search = GridSearchCV(clf, params)
+grid_search.fit(X, y)
+
+grid_cum_scores = [np.max(
+      [grid_search.grid_scores_[j].mean_validation_score for j in range(i)])
+      for i in range(1, len(grid_search.grid_scores_))]
+
+plt.plot(gp_cum_scores, label='GP local', lw=3)
+plt.plot(rdm_cum_scores, label='Random Search', lw=3)
+plt.plot(grid_cum_scores, label='Grid Search', lw=3)
+plt.plot(gp_ext_cum_scores, label='GP ext', lw=3)
+
+plt.legend(loc='lower right')
+plt.ylabel('Score (higher is better)')
+plt.xlabel('Number of model evaluations')
+plt.show()
diff --git a/sklearn/model_selection/__init__.py b/sklearn/model_selection/__init__.py
@@ -21,6 +21,7 @@
 
 from ._search import GridSearchCV
 from ._search import RandomizedSearchCV
+from ._search import SequentialSearchCV
 from ._search import ParameterGrid
 from ._search import ParameterSampler
 from ._search import fit_grid_point
@@ -38,6 +39,7 @@
            'ParameterSampler',
            'PredefinedSplit',
            'RandomizedSearchCV',
+           'SequentialSearchCV',
            'ShuffleSplit',
            'StratifiedKFold',
            'StratifiedShuffleSplit',