scikit-learn · jnothman · Dec 13, 2017 · Jun 5, 2017 · Jun 5, 2017 · Jun 5, 2017
diff --git a/doc/modules/classes.rst b/doc/modules/classes.rst
@@ -1206,6 +1206,7 @@ Model validation
    preprocessing.QuantileTransformer
    preprocessing.RobustScaler
    preprocessing.StandardScaler
+   preprocessing.TransformedTargetRegressor
 
 .. autosummary::
    :toctree: generated/

diff --git a/doc/modules/preprocessing_targets.rst b/doc/modules/preprocessing_targets.rst
@@ -1,4 +1,3 @@
-
 .. currentmodule:: sklearn.preprocessing
 
 .. _preprocessing_targets:
@@ -7,6 +6,76 @@
 Transforming the prediction target (``y``)
 ==========================================
 
+Transforming target in regression
+---------------------------------
+
+:class:`TransformedTargetRegressor` transforms the targets ``y`` before fitting a
+regression model. The predictions are mapped back to the original space via an
+inverse transform. It takes as an argument the regressor that will be used for
+prediction, and the transformer that will be applied to the target variable::
+
+  >>> import numpy as np
+  >>> from sklearn.datasets import load_boston
+  >>> from sklearn.preprocessing import (TransformedTargetRegressor,
+  ...                                    QuantileTransformer)
+  >>> from sklearn.linear_model import LinearRegression
+  >>> from sklearn.model_selection import train_test_split
+  >>> boston = load_boston()
+  >>> X = boston.data
+  >>> y = boston.target
+  >>> transformer = QuantileTransformer(output_distribution='normal')
+  >>> regressor = LinearRegression()
+  >>> regr = TransformedTargetRegressor(regressor=regressor,
+  ...                                   transformer=transformer)
+  >>> X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
+  >>> regr.fit(X_train, y_train) # doctest: +ELLIPSIS
+  TransformedTargetRegressor(...)
+  >>> print('R2 score: {0:.2f}'.format(regr.score(X_test, y_test)))
+  R2 score: 0.67
+  >>> raw_target_regr = LinearRegression().fit(X_train, y_train)
+  >>> print('R2 score: {0:.2f}'.format(raw_target_regr.score(X_test, y_test)))
+  R2 score: 0.64
+
+For simple transformations, instead of a Transformer object, a pair of
+functions can be passed, defining the transformation and its inverse mapping::
+
+  >>> from __future__ import division
+  >>> def func(x):
+  ...     return np.log(x)
+  >>> def inverse_func(x):
+  ...     return np.exp(x)
+
+Subsequently, the object is created as::
+
+  >>> regr = TransformedTargetRegressor(regressor=regressor,
+  ...                                   func=func,
+  ...                                   inverse_func=inverse_func)
+  >>> regr.fit(X_train, y_train) # doctest: +ELLIPSIS
+  TransformedTargetRegressor(...)
+  >>> print('R2 score: {0:.2f}'.format(regr.score(X_test, y_test)))
+  R2 score: 0.65
+
+By default, the provided functions are checked at each fit to be the inverse of
+each other. However, it is possible to bypass this checking by setting
+``check_inverse`` to ``False``::
+
+  >>> def inverse_func(x):
+  ...     return x
+  >>> regr = TransformedTargetRegressor(regressor=regressor,
+  ...                                   func=func,
+  ...                                   inverse_func=inverse_func,
+  ...                                   check_inverse=False)
+  >>> regr.fit(X_train, y_train) # doctest: +ELLIPSIS
+  TransformedTargetRegressor(...)
+  >>> print('R2 score: {0:.2f}'.format(regr.score(X_test, y_test)))
+  R2 score: -4.50
+
+.. note::
+
+   The transformation can be triggered by setting either ``transformer`` or the
+   pair of functions ``func`` and ``inverse_func``. However, setting both
+   options will raise an error.
+
 Label binarization
 ------------------
 

diff --git a/doc/whats_new/v0.20.rst b/doc/whats_new/v0.20.rst
@@ -77,6 +77,11 @@ Model evaluation
 - Added :class:`multioutput.RegressorChain` for multi-target
   regression. :issue:`9257` by :user:`Kumar Ashutosh <thechargedneutron>`.
 
+- Added the :class:`preprocessing.TransformedTargetRegressor` which transforms
+  the target y before fitting a regression model. The predictions are mapped
+  back to the original space via an inverse transform. :issue:`9041` by
+  `Andreas Müller`_ and :user:`Guillaume Lemaitre <glemaitre>`.
+
 Enhancements
 ............
 

diff --git a/examples/preprocessing/plot_transformed_target.py b/examples/preprocessing/plot_transformed_target.py
@@ -0,0 +1,205 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+======================================================
+Effect of transforming the targets in regression model
+======================================================
+
+In this example, we give an overview of the
+:class:`sklearn.preprocessing.TransformedTargetRegressor`. Two examples
+illustrate the benefit of transforming the targets before learning a linear
+regression model. The first example uses synthetic data while the second
+example is based on the Boston housing data set.
+
+"""
+
+# Author: Guillaume Lemaitre <guillaume.lemaitre@inria.fr>
+# License: BSD 3 clause
+
+from __future__ import print_function, division
+
+import numpy as np
+import matplotlib.pyplot as plt
+
+print(__doc__)
+
+###############################################################################
+# Synthetic example
+###############################################################################
+
+from sklearn.datasets import make_regression
+from sklearn.model_selection import train_test_split
+from sklearn.linear_model import RidgeCV
+from sklearn.preprocessing import TransformedTargetRegressor
+from sklearn.metrics import median_absolute_error, r2_score
+
+###############################################################################
+# A synthetic random regression problem is generated. The targets ``y`` are
+# modified by: (i) translating all targets such that all entries are
+# non-negative and (ii) applying an exponential function to obtain non-linear
+# targets which cannot be fitted using a simple linear model.
+#
+# Therefore, a logarithmic and an exponential function will be used to
+# transform the targets before training a linear regression model and using it
+# for prediction.
+
+
+def log_transform(x):
+    return np.log(x + 1)
+
+
+def exp_transform(x):
+    return np.exp(x) - 1
+
+
+X, y = make_regression(n_samples=10000, noise=100, random_state=0)
+y = np.exp((y + abs(y.min())) / 200)
+y_trans = log_transform(y)
+
+###############################################################################
+# The following illustrate the probability density functions of the target
+# before and after applying the logarithmic functions.
+
+f, (ax0, ax1) = plt.subplots(1, 2)
+
+ax0.hist(y, bins='auto', normed=True)
+ax0.set_xlim([0, 2000])
+ax0.set_ylabel('Probability')
+ax0.set_xlabel('Target')
+ax0.set_title('Target distribution')
+
+ax1.hist(y_trans, bins='auto', normed=True)
+ax1.set_ylabel('Probability')
+ax1.set_xlabel('Target')
+ax1.set_title('Transformed target distribution')
+
+f.suptitle("Synthetic data", y=0.035)
+f.tight_layout(rect=[0.05, 0.05, 0.95, 0.95])
+
+X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
+
+###############################################################################
+# At first, a linear model will be applied on the original targets. Due to the
+# non-linearity, the model trained will not be precise during the
+# prediction. Subsequently, a logarithmic function is used to linearize the
+# targets, allowing better prediction even with a similar linear model as
+# reported by the median absolute error (MAE).
+
+f, (ax0, ax1) = plt.subplots(1, 2, sharey=True)
+
+regr = RidgeCV()
+regr.fit(X_train, y_train)
+y_pred = regr.predict(X_test)
+
+ax0.scatter(y_test, y_pred)
+ax0.plot([0, 2000], [0, 2000], '--k')
+ax0.set_ylabel('Target predicted')
+ax0.set_xlabel('True Target')
+ax0.set_title('Ridge regression \n without target transformation')
+ax0.text(100, 1750, r'$R^2$=%.2f, MAE=%.2f' % (
+    r2_score(y_test, y_pred), median_absolute_error(y_test, y_pred)))
+ax0.set_xlim([0, 2000])
+ax0.set_ylim([0, 2000])
+
+regr_trans = TransformedTargetRegressor(regressor=RidgeCV(),
+                                        func=log_transform,
+                                        inverse_func=exp_transform)
+regr_trans.fit(X_train, y_train)
+y_pred = regr_trans.predict(X_test)
+
+ax1.scatter(y_test, y_pred)
+ax1.plot([0, 2000], [0, 2000], '--k')
+ax1.set_ylabel('Target predicted')
+ax1.set_xlabel('True Target')
+ax1.set_title('Ridge regression \n with target transformation')
+ax1.text(100, 1750, r'$R^2$=%.2f, MAE=%.2f' % (
+    r2_score(y_test, y_pred), median_absolute_error(y_test, y_pred)))
+ax1.set_xlim([0, 2000])
+ax1.set_ylim([0, 2000])
+
+f.suptitle("Synthetic data", y=0.035)
+f.tight_layout(rect=[0.05, 0.05, 0.95, 0.95])
+
+###############################################################################
+# Real-world data set
+###############################################################################
+
+###############################################################################
+# In a similar manner, the boston housing data set is used to show the impact
+# of transforming the targets before learning a model. In this example, the
+# targets to be predicted corresponds to the weighted distances to the five
+# Boston employment centers.
+
+from sklearn.datasets import load_boston
+from sklearn.preprocessing import QuantileTransformer, quantile_transform
+
+dataset = load_boston()
+target = np.array(dataset.feature_names) == "DIS"
+X = dataset.data[:, np.logical_not(target)]
+y = dataset.data[:, target].squeeze()
+y_trans = quantile_transform(dataset.data[:, target],
+                             output_distribution='normal').squeeze()
+
+###############################################################################
+# A :class:`sklearn.preprocessing.QuantileTransformer` is used such that the
+# targets follows a normal distribution before applying a
+# :class:`sklearn.linear_model.RidgeCV` model.
+
+f, (ax0, ax1) = plt.subplots(1, 2)
+
+ax0.hist(y, bins='auto', normed=True)
+ax0.set_ylabel('Probability')
+ax0.set_xlabel('Target')
+ax0.set_title('Target distribution')
+
+ax1.hist(y_trans, bins='auto', normed=True)
+ax1.set_ylabel('Probability')
+ax1.set_xlabel('Target')
+ax1.set_title('Transformed target distribution')
+
+f.suptitle("Boston housing data: distance to employment centers", y=0.035)
+f.tight_layout(rect=[0.05, 0.05, 0.95, 0.95])
+
+X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1)
+
+###############################################################################
+# The effect of the transformer is weaker than on the synthetic data. However,
+# the transform induces a decrease of the MAE.
+
+f, (ax0, ax1) = plt.subplots(1, 2, sharey=True)
+
+regr = RidgeCV()
+regr.fit(X_train, y_train)
+y_pred = regr.predict(X_test)
+
+ax0.scatter(y_test, y_pred)
+ax0.plot([0, 10], [0, 10], '--k')
+ax0.set_ylabel('Target predicted')
+ax0.set_xlabel('True Target')
+ax0.set_title('Ridge regression \n without target transformation')
+ax0.text(1, 9, r'$R^2$=%.2f, MAE=%.2f' % (
+    r2_score(y_test, y_pred), median_absolute_error(y_test, y_pred)))
+ax0.set_xlim([0, 10])
+ax0.set_ylim([0, 10])
+
+regr_trans = TransformedTargetRegressor(
+    regressor=RidgeCV(),
+    transformer=QuantileTransformer(output_distribution='normal'))
+regr_trans.fit(X_train, y_train)
+y_pred = regr_trans.predict(X_test)
+
+ax1.scatter(y_test, y_pred)
+ax1.plot([0, 10], [0, 10], '--k')
+ax1.set_ylabel('Target predicted')
+ax1.set_xlabel('True Target')
+ax1.set_title('Ridge regression \n with target transformation')
+ax1.text(1, 9, r'$R^2$=%.2f, MAE=%.2f' % (
+    r2_score(y_test, y_pred), median_absolute_error(y_test, y_pred)))
+ax1.set_xlim([0, 10])
+ax1.set_ylim([0, 10])
+
+f.suptitle("Boston housing data: distance to employment centers", y=0.035)
+f.tight_layout(rect=[0.05, 0.05, 0.95, 0.95])
+
+plt.show()
diff --git a/sklearn/preprocessing/__init__.py b/sklearn/preprocessing/__init__.py
@@ -25,14 +25,14 @@
 from .data import OneHotEncoder
 from .data import PowerTransformer
 from .data import CategoricalEncoder
-
 from .data import PolynomialFeatures
 
 from .label import label_binarize
 from .label import LabelBinarizer
 from .label import LabelEncoder
 from .label import MultiLabelBinarizer
 
+from ._target import TransformedTargetRegressor
 from .imputation import Imputer
 
 
@@ -53,6 +53,7 @@
     'PowerTransformer',
     'RobustScaler',
     'StandardScaler',
+    'TransformedTargetRegressor',
     'add_dummy_feature',
     'PolynomialFeatures',
     'binarize',