scikit-learn · amascia · Mar 27, 2020 · Mar 27, 2020 · Apr 9, 2020 · Apr 9, 2020
diff --git a/doc/modules/calibration.rst b/doc/modules/calibration.rst
@@ -240,6 +240,28 @@ probabilities, the calibrated probabilities for each class
 are predicted separately. As those probabilities do not necessarily sum to
 one, a postprocessing is performed to normalize them.
 
+The parameters ``class_weight`` and ``sample_weight`` can be used to
+respectively set relative importance to classes and to individual samples.
+
+:class:`CalibratedClassifierCV` can handle such unbalanced dataset with the
+``class_weight`` parameter. ``class_weight`` has to be provided as a
+dictionary of the form ``{class_label : value}``, where value is a strictly
+positive floating point, or as ``class_weight='balanced'`` which will
+automatically adjust weights inversely proportional to class frequencies in
+the input data.
+
+.. note::
+   Setting ``class_weight`` parameter in :class:`CalibratedClassifierCV`,
+   will only affect the training of the chosen regressor (`'isotonic'` or `'sigmoid'`).
+
+For instance, if ``class_weight='balanced'`` is passed to
+:class:`CalibratedClassifierCV`, only the samples used to fit the calibrator
+will be balanced. If one also wants the samples used to fit the estimator
+to be balanced, one would need to set ``class_weight='balanced'`` within
+the `base_estimator` passed to :class:`CalibratedClassifierCV`.
+Alternatively, if `cv="prefit"` is set, the samples are not split and are
+all used to fit the regressor in a balanced manner.
+
 .. topic:: Examples:
 
    * :ref:`sphx_glr_auto_examples_calibration_plot_calibration_curve.py`

diff --git a/doc/whats_new/v1.1.rst b/doc/whats_new/v1.1.rst
@@ -77,6 +77,11 @@ Changelog
   `pos_label` to specify the positive class label.
   :pr:`21032` by :user:`Guillaume Lemaitre <glemaitre>`.
 
+- |Enhancement| :class:`calibration.CalibratedClassifierCV` accepts ``class_weight``
+  which enables specifying weights for classes when training the ``sigmoid`` or
+  ``isotonic`` calibration methods.
+  :pr:`17541` by :user:`Achille Mascia <amascia>`.
+
 - |Enhancement| :class:`CalibrationDisplay` accepts a parameter `pos_label` to
   add this information to the plot.
   :pr:`21038` by :user:`Guillaume Lemaitre <glemaitre>`.

diff --git a/sklearn/calibration.py b/sklearn/calibration.py
@@ -32,6 +32,7 @@
     column_or_1d,
     indexable,
     check_matplotlib_support,
+    compute_class_weight,
 )
 
 from .utils.multiclass import check_classification_targets
@@ -145,6 +146,15 @@ class CalibratedClassifierCV(ClassifierMixin, MetaEstimatorMixin, BaseEstimator)
 
         .. versionadded:: 0.24
 
+    class_weight : dict or 'balanced', default=None
+        Class weights used for the calibration.
+        If a dict, it must be provided in this form: ``{class_label: weight}``.
+
+        Those weights won't be used for the underlying estimator training.
+        See :term:`Glossary <class_weight>` for more details.
+
+        .. versionadded:: 1.1
+
     Attributes
     ----------
     classes_ : ndarray of shape (n_classes,)
@@ -244,12 +254,14 @@ def __init__(
         cv=None,
         n_jobs=None,
         ensemble=True,
+        class_weight=None,
     ):
         self.base_estimator = base_estimator
         self.method = method
         self.cv = cv
         self.n_jobs = n_jobs
         self.ensemble = ensemble
+        self.class_weight = class_weight
 
     def fit(self, X, y, sample_weight=None, **fit_params):
         """Fit the calibrated model.
@@ -276,8 +288,8 @@ def fit(self, X, y, sample_weight=None, **fit_params):
         """
         check_classification_targets(y)
         X, y = indexable(X, y)
-        if sample_weight is not None:
-            sample_weight = _check_sample_weight(sample_weight, X)
+
+        sample_weight = _check_sample_weight(sample_weight, X)
 
         for sample_aligned_params in fit_params.values():
             check_consistent_length(y, sample_aligned_params)
@@ -299,6 +311,13 @@ def fit(self, X, y, sample_weight=None, **fit_params):
             n_classes = len(self.classes_)
             predictions = _compute_predictions(pred_method, method_name, X, n_classes)
 
+            if self.class_weight is not None:
+                self.class_weight_ = compute_class_weight(
+                    self.class_weight, classes=self.classes_, y=y
+                )
+                label_encoder_ = LabelEncoder()
+                sample_weight *= self.class_weight_[label_encoder_.fit_transform(y)]
+
             calibrated_classifier = _fit_calibrator(
                 base_estimator,
                 predictions,
@@ -317,7 +336,7 @@ def fit(self, X, y, sample_weight=None, **fit_params):
             # sample_weight checks
             fit_parameters = signature(base_estimator.fit).parameters
             supports_sw = "sample_weight" in fit_parameters
-            if sample_weight is not None and not supports_sw:
+            if not supports_sw:
                 estimator_name = type(base_estimator).__name__
                 warnings.warn(
                     f"Since {estimator_name} does not appear to accept sample_weight, "
@@ -329,6 +348,17 @@ def fit(self, X, y, sample_weight=None, **fit_params):
                     "incorrect."
                 )
 
+            sample_weight_cal = np.copy(sample_weight)
+            if self.class_weight is not None:
+                # Build sample weights for calibrator.
+                # Those weights will not be used for the training of
+                # the underlying estimator which will then be calibrated.
+                self.class_weight_ = compute_class_weight(
+                    self.class_weight, classes=self.classes_, y=y
+                )
+                label_encoder_ = LabelEncoder()
+                sample_weight_cal *= self.class_weight_[label_encoder_.fit_transform(y)]
+
             # Check that each cross-validation fold can have at least one
             # example per class
             if isinstance(self.cv, int):
@@ -360,18 +390,15 @@ def fit(self, X, y, sample_weight=None, **fit_params):
                         classes=self.classes_,
                         supports_sw=supports_sw,
                         sample_weight=sample_weight,
+                        sample_weight_cal=sample_weight_cal,
                         **fit_params,
                     )
                     for train, test in cv.split(X, y)
                 )
             else:
                 this_estimator = clone(base_estimator)
                 _, method_name = _get_prediction_method(this_estimator)
-                fit_params = (
-                    {"sample_weight": sample_weight}
-                    if sample_weight is not None and supports_sw
-                    else None
-                )
+                fit_params = {"sample_weight": sample_weight} if supports_sw else None
                 pred_method = partial(
                     cross_val_predict,
                     estimator=this_estimator,
@@ -386,7 +413,7 @@ def fit(self, X, y, sample_weight=None, **fit_params):
                     pred_method, method_name, X, n_classes
                 )
 
-                if sample_weight is not None and supports_sw:
+                if supports_sw:
                     this_estimator.fit(X, y, sample_weight=sample_weight)
                 else:
                     this_estimator.fit(X, y)
@@ -398,7 +425,7 @@ def fit(self, X, y, sample_weight=None, **fit_params):
                     y,
                     self.classes_,
                     self.method,
-                    sample_weight,
+                    sample_weight_cal,
                 )
                 self.calibrated_classifiers_.append(calibrated_classifier)
 
@@ -478,6 +505,7 @@ def _fit_classifier_calibrator_pair(
     method,
     classes,
     sample_weight=None,
+    sample_weight_cal=None,
     **fit_params,
 ):
     """Fit a classifier/calibration pair on a given train/test split.
@@ -513,7 +541,10 @@ def _fit_classifier_calibrator_pair(
         The target classes.
 
     sample_weight : array-like, default=None
-        Sample weights for `X`.
+        Sample weights for `X` that are used to fit the estimator.
+
+    sample_weight_cal : array-like, default=None
+        Sample weights for `X` that are used to fit the calibrator.
 
     **fit_params : dict
         Parameters to pass to the `fit` method of the underlying
@@ -527,7 +558,7 @@ def _fit_classifier_calibrator_pair(
     X_train, y_train = _safe_indexing(X, train), _safe_indexing(y, train)
     X_test, y_test = _safe_indexing(X, test), _safe_indexing(y, test)
 
-    if sample_weight is not None and supports_sw:
+    if supports_sw:
         sw_train = _safe_indexing(sample_weight, train)
         estimator.fit(X_train, y_train, sample_weight=sw_train, **fit_params_train)
     else:
@@ -537,7 +568,7 @@ def _fit_classifier_calibrator_pair(
     pred_method, method_name = _get_prediction_method(estimator)
     predictions = _compute_predictions(pred_method, method_name, X_test, n_classes)
 
-    sw_test = None if sample_weight is None else _safe_indexing(sample_weight, test)
+    sw_test = _safe_indexing(sample_weight_cal, test)
     calibrated_classifier = _fit_calibrator(
         estimator, predictions, y_test, classes, method, sample_weight=sw_test
     )

diff --git a/sklearn/tests/test_calibration.py b/sklearn/tests/test_calibration.py
@@ -192,6 +192,44 @@ def test_sample_weight(data, method, ensemble):
     assert diff > 0.1
 
 
+@pytest.mark.parametrize("method", ["sigmoid", "isotonic"])
+@pytest.mark.parametrize("ensemble", [True, False])
+@pytest.mark.parametrize("prefit", [True, False])
+def test_class_weight(data, method, ensemble, prefit):
+    n_samples = 50
+    n_classes = 2
+    class_weight = np.random.RandomState(seed=42).uniform(size=n_classes)
+    X, y = data
+    X_calib, y_calib = X[:n_samples], y[:n_samples]
+    X_test = X[2 * n_samples :]
+
+    cw = dict(zip(np.arange(n_classes), class_weight))
+    base_estimator = LinearSVC(random_state=42)
+    cv = None
+
+    if prefit:
+        X_train, y_train = X[n_samples : 2 * n_samples], y[n_samples : 2 * n_samples]
+        base_estimator.fit(X_train, y_train)
+        cv = "prefit"
+
+    calibrated_clf = CalibratedClassifierCV(
+        base_estimator, method=method, cv=cv, ensemble=ensemble, class_weight=cw
+    )
+    calibrated_clf.fit(X_calib, y_calib)
+    probs_with_cw = calibrated_clf.predict_proba(X_test)
+
+    # As the weights are used for the calibration, they should make
+    # the calibrated estimator yield different predictions.
+    calibrated_clf = CalibratedClassifierCV(
+        base_estimator, method=method, cv=cv, ensemble=ensemble
+    )
+    calibrated_clf.fit(X_calib, y_calib)
+    probs_without_cw = calibrated_clf.predict_proba(X_test)
+
+    diff = np.linalg.norm(probs_with_cw - probs_without_cw)
+    assert diff > 0.1
+
+
 @pytest.mark.parametrize("method", ["sigmoid", "isotonic"])
 @pytest.mark.parametrize("ensemble", [True, False])
 def test_parallel_execution(data, method, ensemble):
@@ -307,7 +345,8 @@ def predict(self, X):
     assert_allclose(probas, 1.0 / clf.n_classes_)
 
 
-def test_calibration_prefit():
+@pytest.mark.parametrize("method", ["sigmoid", "isotonic"])
+def test_calibration_prefit(method):
     """Test calibration for prefitted classifiers"""
     n_samples = 50
     X, y = make_classification(n_samples=3 * n_samples, n_features=6, random_state=42)
@@ -339,19 +378,18 @@ def test_calibration_prefit():
         (X_calib, X_test),
         (sparse.csr_matrix(X_calib), sparse.csr_matrix(X_test)),
     ]:
-        for method in ["isotonic", "sigmoid"]:
-            cal_clf = CalibratedClassifierCV(clf, method=method, cv="prefit")
-
-            for sw in [sw_calib, None]:
-                cal_clf.fit(this_X_calib, y_calib, sample_weight=sw)
-                y_prob = cal_clf.predict_proba(this_X_test)
-                y_pred = cal_clf.predict(this_X_test)
-                prob_pos_cal_clf = y_prob[:, 1]
-                assert_array_equal(y_pred, np.array([0, 1])[np.argmax(y_prob, axis=1)])
-
-                assert brier_score_loss(y_test, prob_pos_clf) > brier_score_loss(
-                    y_test, prob_pos_cal_clf
-                )
+        cal_clf = CalibratedClassifierCV(clf, method=method, cv="prefit")
+
+        for sw in [sw_calib, None]:
+            cal_clf.fit(this_X_calib, y_calib, sample_weight=sw)
+            y_prob = cal_clf.predict_proba(this_X_test)
+            y_pred = cal_clf.predict(this_X_test)
+            prob_pos_cal_clf = y_prob[:, 1]
+            assert_array_equal(y_pred, np.array([0, 1])[np.argmax(y_prob, axis=1)])
+
+            assert brier_score_loss(y_test, prob_pos_clf) > brier_score_loss(
+                y_test, prob_pos_cal_clf
+            )
 
 
 @pytest.mark.parametrize("method", ["sigmoid", "isotonic"])