scikit-learn · ogrisel · Apr 26, 2019 · Jan 13, 2019 · Jan 13, 2019 · Jan 13, 2019
diff --git a/.coveragerc b/.coveragerc
@@ -1,7 +1,7 @@
 [run]
 branch = True
 source = sklearn
-include = */sklearn/*
+parallel = True
 omit =
     */sklearn/externals/*
     */benchmarks/*

diff --git a/benchmarks/bench_hist_gradient_boosting.py b/benchmarks/bench_hist_gradient_boosting.py
@@ -0,0 +1,241 @@
+from time import time
+import argparse
+
+import matplotlib.pyplot as plt
+from sklearn.model_selection import train_test_split
+# To use this experimental feature, we need to explicitly ask for it:
+from sklearn.experimental import enable_hist_gradient_boosting  # noqa
+from sklearn.ensemble import HistGradientBoostingRegressor
+from sklearn.ensemble import HistGradientBoostingClassifier
+from sklearn.datasets import make_classification
+from sklearn.datasets import make_regression
+from sklearn.ensemble._hist_gradient_boosting.utils import (
+    get_equivalent_estimator)
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--n-leaf-nodes', type=int, default=31)
+parser.add_argument('--n-trees', type=int, default=10)
+parser.add_argument('--lightgbm', action="store_true", default=False,
+                    help='also plot lightgbm')
+parser.add_argument('--xgboost', action="store_true", default=False,
+                    help='also plot xgboost')
+parser.add_argument('--catboost', action="store_true", default=False,
+                    help='also plot catboost')
+parser.add_argument('--learning-rate', type=float, default=.1)
+parser.add_argument('--problem', type=str, default='classification',
+                    choices=['classification', 'regression'])
+parser.add_argument('--n-classes', type=int, default=2)
+parser.add_argument('--n-samples-max', type=int, default=int(1e6))
+parser.add_argument('--n-features', type=int, default=20)
+parser.add_argument('--max-bins', type=int, default=255)
+args = parser.parse_args()
+
+n_leaf_nodes = args.n_leaf_nodes
+n_trees = args.n_trees
+lr = args.learning_rate
+max_bins = args.max_bins
+
+
+def get_estimator_and_data():
+    if args.problem == 'classification':
+        X, y = make_classification(args.n_samples_max * 2,
+                                   n_features=args.n_features,
+                                   n_classes=args.n_classes,
+                                   n_clusters_per_class=1,
+                                   random_state=0)
+        return X, y, HistGradientBoostingClassifier
+    elif args.problem == 'regression':
+        X, y = make_regression(args.n_samples_max * 2,
+                               n_features=args.n_features, random_state=0)
+        return X, y, HistGradientBoostingRegressor
+
+
+X, y, Estimator = get_estimator_and_data()
+X_train_, X_test_, y_train_, y_test_ = train_test_split(
+    X, y, test_size=0.5, random_state=0)
+
+
+def one_run(n_samples):
+    X_train = X_train_[:n_samples]
+    X_test = X_test_[:n_samples]
+    y_train = y_train_[:n_samples]
+    y_test = y_test_[:n_samples]
+    assert X_train.shape[0] == n_samples
+    assert X_test.shape[0] == n_samples
+    print("Data size: %d samples train, %d samples test."
+          % (n_samples, n_samples))
+    print("Fitting a sklearn model...")
+    tic = time()
+    est = Estimator(learning_rate=lr,
+                    max_iter=n_trees,
+                    max_bins=max_bins,
+                    max_leaf_nodes=n_leaf_nodes,
+                    n_iter_no_change=None,
+                    random_state=0,
+                    verbose=0)
+    est.fit(X_train, y_train)
+    sklearn_fit_duration = time() - tic
+    tic = time()
+    sklearn_score = est.score(X_test, y_test)
+    sklearn_score_duration = time() - tic
+    print("score: {:.4f}".format(sklearn_score))
+    print("fit duration: {:.3f}s,".format(sklearn_fit_duration))
+    print("score duration: {:.3f}s,".format(sklearn_score_duration))
+
+    lightgbm_score = None
+    lightgbm_fit_duration = None
+    lightgbm_score_duration = None
+    if args.lightgbm:
+        print("Fitting a LightGBM model...")
+        # get_lightgbm does not accept loss='auto'
+        if args.problem == 'classification':
+            loss = 'binary_crossentropy' if args.n_classes == 2 else \
+                'categorical_crossentropy'
+            est.set_params(loss=loss)
+        lightgbm_est = get_equivalent_estimator(est, lib='lightgbm')
+
+        tic = time()
+        lightgbm_est.fit(X_train, y_train)
+        lightgbm_fit_duration = time() - tic
+        tic = time()
+        lightgbm_score = lightgbm_est.score(X_test, y_test)
+        lightgbm_score_duration = time() - tic
+        print("score: {:.4f}".format(lightgbm_score))
+        print("fit duration: {:.3f}s,".format(lightgbm_fit_duration))
+        print("score duration: {:.3f}s,".format(lightgbm_score_duration))
+
+    xgb_score = None
+    xgb_fit_duration = None
+    xgb_score_duration = None
+    if args.xgboost:
+        print("Fitting an XGBoost model...")
+        # get_xgb does not accept loss='auto'
+        if args.problem == 'classification':
+            loss = 'binary_crossentropy' if args.n_classes == 2 else \
+                'categorical_crossentropy'
+            est.set_params(loss=loss)
+        xgb_est = get_equivalent_estimator(est, lib='xgboost')
+
+        tic = time()
+        xgb_est.fit(X_train, y_train)
+        xgb_fit_duration = time() - tic
+        tic = time()
+        xgb_score = xgb_est.score(X_test, y_test)
+        xgb_score_duration = time() - tic
+        print("score: {:.4f}".format(xgb_score))
+        print("fit duration: {:.3f}s,".format(xgb_fit_duration))
+        print("score duration: {:.3f}s,".format(xgb_score_duration))
+
+    cat_score = None
+    cat_fit_duration = None
+    cat_score_duration = None
+    if args.catboost:
+        print("Fitting a CatBoost model...")
+        # get_cat does not accept loss='auto'
+        if args.problem == 'classification':
+            loss = 'binary_crossentropy' if args.n_classes == 2 else \
+                'categorical_crossentropy'
+            est.set_params(loss=loss)
+        cat_est = get_equivalent_estimator(est, lib='catboost')
+
+        tic = time()
+        cat_est.fit(X_train, y_train)
+        cat_fit_duration = time() - tic
+        tic = time()
+        cat_score = cat_est.score(X_test, y_test)
+        cat_score_duration = time() - tic
+        print("score: {:.4f}".format(cat_score))
+        print("fit duration: {:.3f}s,".format(cat_fit_duration))
+        print("score duration: {:.3f}s,".format(cat_score_duration))
+
+    return (sklearn_score, sklearn_fit_duration, sklearn_score_duration,
+            lightgbm_score, lightgbm_fit_duration, lightgbm_score_duration,
+            xgb_score, xgb_fit_duration, xgb_score_duration,
+            cat_score, cat_fit_duration, cat_score_duration)
+
+
+n_samples_list = [1000, 10000, 100000, 500000, 1000000, 5000000, 10000000]
+n_samples_list = [n_samples for n_samples in n_samples_list
+                  if n_samples <= args.n_samples_max]
+
+sklearn_scores = []
+sklearn_fit_durations = []
+sklearn_score_durations = []
+lightgbm_scores = []
+lightgbm_fit_durations = []
+lightgbm_score_durations = []
+xgb_scores = []
+xgb_fit_durations = []
+xgb_score_durations = []
+cat_scores = []
+cat_fit_durations = []
+cat_score_durations = []
+
+for n_samples in n_samples_list:
+    (sklearn_score,
+     sklearn_fit_duration,
+     sklearn_score_duration,
+     lightgbm_score,
+     lightgbm_fit_duration,
+     lightgbm_score_duration,
+     xgb_score,
+     xgb_fit_duration,
+     xgb_score_duration,
+     cat_score,
+     cat_fit_duration,
+     cat_score_duration) = one_run(n_samples)
+
+    for scores, score in (
+            (sklearn_scores, sklearn_score),
+            (sklearn_fit_durations, sklearn_fit_duration),
+            (sklearn_score_durations, sklearn_score_duration),
+            (lightgbm_scores, lightgbm_score),
+            (lightgbm_fit_durations, lightgbm_fit_duration),
+            (lightgbm_score_durations, lightgbm_score_duration),
+            (xgb_scores, xgb_score),
+            (xgb_fit_durations, xgb_fit_duration),
+            (xgb_score_durations, xgb_score_duration),
+            (cat_scores, cat_score),
+            (cat_fit_durations, cat_fit_duration),
+            (cat_score_durations, cat_score_duration)):
+        scores.append(score)
+
+fig, axs = plt.subplots(3, sharex=True)
+
+axs[0].plot(n_samples_list, sklearn_scores, label='sklearn')
+axs[1].plot(n_samples_list, sklearn_fit_durations, label='sklearn')
+axs[2].plot(n_samples_list, sklearn_score_durations, label='sklearn')
+
+if args.lightgbm:
+    axs[0].plot(n_samples_list, lightgbm_scores, label='lightgbm')
+    axs[1].plot(n_samples_list, lightgbm_fit_durations, label='lightgbm')
+    axs[2].plot(n_samples_list, lightgbm_score_durations, label='lightgbm')
+
+if args.xgboost:
+    axs[0].plot(n_samples_list, xgb_scores, label='XGBoost')
+    axs[1].plot(n_samples_list, xgb_fit_durations, label='XGBoost')
+    axs[2].plot(n_samples_list, xgb_score_durations, label='XGBoost')
+
+if args.catboost:
+    axs[0].plot(n_samples_list, cat_scores, label='CatBoost')
+    axs[1].plot(n_samples_list, cat_fit_durations, label='CatBoost')
+    axs[2].plot(n_samples_list, cat_score_durations, label='CatBoost')
+
+for ax in axs:
+    ax.set_xscale('log')
+    ax.legend(loc='best')
+    ax.set_xlabel('n_samples')
+
+axs[0].set_title('scores')
+axs[1].set_title('fit duration (s)')
+axs[2].set_title('score duration (s)')
+
+title = args.problem
+if args.problem == 'classification':
+    title += ' n_classes = {}'.format(args.n_classes)
+fig.suptitle(title)
+
+
+plt.tight_layout()
+plt.show()
diff --git a/benchmarks/bench_hist_gradient_boosting_higgsboson.py b/benchmarks/bench_hist_gradient_boosting_higgsboson.py
@@ -0,0 +1,123 @@
+from urllib.request import urlretrieve
+import os
+from gzip import GzipFile
+from time import time
+import argparse
+
+import numpy as np
+import pandas as pd
+from joblib import Memory
+from sklearn.model_selection import train_test_split
+from sklearn.metrics import accuracy_score, roc_auc_score
+# To use this experimental feature, we need to explicitly ask for it:
+from sklearn.experimental import enable_hist_gradient_boosting  # noqa
+from sklearn.ensemble import HistGradientBoostingClassifier
+from sklearn.ensemble._hist_gradient_boosting.utils import (
+    get_equivalent_estimator)
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--n-leaf-nodes', type=int, default=31)
+parser.add_argument('--n-trees', type=int, default=10)
+parser.add_argument('--lightgbm', action="store_true", default=False)
+parser.add_argument('--xgboost', action="store_true", default=False)
+parser.add_argument('--catboost', action="store_true", default=False)
+parser.add_argument('--learning-rate', type=float, default=1.)
+parser.add_argument('--subsample', type=int, default=None)
+parser.add_argument('--max-bins', type=int, default=255)
+args = parser.parse_args()
+
+HERE = os.path.dirname(__file__)
+URL = ("https://archive.ics.uci.edu/ml/machine-learning-databases/00280/"
+       "HIGGS.csv.gz")
+m = Memory(location='/tmp', mmap_mode='r')
+
+n_leaf_nodes = args.n_leaf_nodes
+n_trees = args.n_trees
+subsample = args.subsample
+lr = args.learning_rate
+max_bins = args.max_bins
+
+
+@m.cache
+def load_data():
+    filename = os.path.join(HERE, URL.rsplit('/', 1)[-1])
+    if not os.path.exists(filename):
+        print(f"Downloading {URL} to {filename} (2.6 GB)...")
+        urlretrieve(URL, filename)
+        print("done.")
+
+    print(f"Parsing {filename}...")
+    tic = time()
+    with GzipFile(filename) as f:
+        df = pd.read_csv(f, header=None, dtype=np.float32)
+    toc = time()
+    print(f"Loaded {df.values.nbytes / 1e9:0.3f} GB in {toc - tic:0.3f}s")
+    return df
+
+
+df = load_data()
+target = df.values[:, 0]
+data = np.ascontiguousarray(df.values[:, 1:])
+data_train, data_test, target_train, target_test = train_test_split(
+    data, target, test_size=.2, random_state=0)
+
+if subsample is not None:
+    data_train, target_train = data_train[:subsample], target_train[:subsample]
+
+n_samples, n_features = data_train.shape
+print(f"Training set with {n_samples} records with {n_features} features.")
+
+print("Fitting a sklearn model...")
+tic = time()
+est = HistGradientBoostingClassifier(loss='binary_crossentropy',
+                                     learning_rate=lr,
+                                     max_iter=n_trees,
+                                     max_bins=max_bins,
+                                     max_leaf_nodes=n_leaf_nodes,
+                                     n_iter_no_change=None,
+                                     random_state=0,
+                                     verbose=1)
+est.fit(data_train, target_train)
+toc = time()
+predicted_test = est.predict(data_test)
+predicted_proba_test = est.predict_proba(data_test)
+roc_auc = roc_auc_score(target_test, predicted_proba_test[:, 1])
+acc = accuracy_score(target_test, predicted_test)
+print(f"done in {toc - tic:.3f}s, ROC AUC: {roc_auc:.4f}, ACC: {acc :.4f}")
+
+if args.lightgbm:
+    print("Fitting a LightGBM model...")
+    tic = time()
+    lightgbm_est = get_equivalent_estimator(est, lib='lightgbm')
+    lightgbm_est.fit(data_train, target_train)
+    toc = time()
+    predicted_test = lightgbm_est.predict(data_test)
+    predicted_proba_test = lightgbm_est.predict_proba(data_test)
+    roc_auc = roc_auc_score(target_test, predicted_proba_test[:, 1])
+    acc = accuracy_score(target_test, predicted_test)
+    print(f"done in {toc - tic:.3f}s, ROC AUC: {roc_auc:.4f}, ACC: {acc :.4f}")
+
+if args.xgboost:
+    print("Fitting an XGBoost model...")
+    tic = time()
+    xgboost_est = get_equivalent_estimator(est, lib='xgboost')
+    xgboost_est.fit(data_train, target_train)
+    toc = time()
+    predicted_test = xgboost_est.predict(data_test)
+    predicted_proba_test = xgboost_est.predict_proba(data_test)
+    roc_auc = roc_auc_score(target_test, predicted_proba_test[:, 1])
+    acc = accuracy_score(target_test, predicted_test)
+    print(f"done in {toc - tic:.3f}s, ROC AUC: {roc_auc:.4f}, ACC: {acc :.4f}")
+
+if args.catboost:
+    print("Fitting a Catboost model...")
+    tic = time()
+    catboost_est = get_equivalent_estimator(est, lib='catboost')
+    catboost_est.fit(data_train, target_train)
+    toc = time()
+    predicted_test = catboost_est.predict(data_test)
+    predicted_proba_test = catboost_est.predict_proba(data_test)
+    roc_auc = roc_auc_score(target_test, predicted_proba_test[:, 1])
+    acc = accuracy_score(target_test, predicted_test)
+    print(f"done in {toc - tic:.3f}s, ROC AUC: {roc_auc:.4f}, ACC: {acc :.4f}")
diff --git a/build_tools/azure/test_pytest_soft_dependency.sh b/build_tools/azure/test_pytest_soft_dependency.sh
@@ -8,8 +8,10 @@ conda remove -y py pytest || pip uninstall -y py pytest
 
 if [[ "$COVERAGE" == "true" ]]; then
     # Need to append the coverage to the existing .coverage generated by
-    # running the tests
-    CMD="coverage run --append"
+    # running the tests. Make sure to reuse the same coverage
+    # configuration as the one used by the main pytest run to be
+    # able to combine the results.
+    CMD="coverage run --rcfile=$BUILD_SOURCESDIRECTORY/.coveragerc"
 else
     CMD="python"
 fi