From 20f979419aa1f7470f3772f83505b374204cfb97 Mon Sep 17 00:00:00 2001 From: mehak-sachdeva Date: Sat, 4 Mar 2017 14:48:27 -0500 Subject: [PATCH 1/2] referred reliability diagrams and added citations --- doc/modules/calibration.rst | 12 ++++++++---- sklearn/calibration.py | 3 ++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/doc/modules/calibration.rst b/doc/modules/calibration.rst index cd74f414fe3d2..44644dd169640 100644 --- a/doc/modules/calibration.rst +++ b/doc/modules/calibration.rst @@ -57,9 +57,9 @@ with different biases per method: ensemble away from 0. We observe this effect most strongly with random forests because the base-level trees trained with random forests have relatively high variance due to feature subseting." As a result, the - calibration curve shows a characteristic sigmoid shape, indicating that the - classifier could trust its "intuition" more and return probabilties closer - to 0 or 1 typically. + calibration curve also referred to as the reliability diagram (Wilks 1995[5]) shows a + characteristic sigmoid shape, indicating that the classifier could trust its + "intuition" more and return probabilties closer to 0 or 1 typically. .. currentmodule:: sklearn.svm @@ -118,7 +118,7 @@ is evaluated with Brier score :func:`brier_score_loss`, reported in the legend :align: center One can observe here that logistic regression is well calibrated as its curve is -nearly diagonal. Linear SVC's calibration curve has a sigmoid curve, which is +nearly diagonal. Linear SVC's calibration curve or reliability diagram has a sigmoid curve, which is typical for an under-confident classifier. In the case of LinearSVC, this is caused by the margin property of the hinge loss, which lets the model focus on hard samples that are close to the decision boundary (the support vectors). Both @@ -200,3 +200,7 @@ a similar decrease in log-loss. .. [4] Predicting Good Probabilities with Supervised Learning, A. Niculescu-Mizil & R. Caruana, ICML 2005 + + .. [5] On the combination of forecast probabilities for + consecutive precipitation periods. Wea. Forecasting, 5, 640– + 650., Wilks, D. S., 1990a diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 35cdab45a1b57..6880f26134558 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -511,7 +511,8 @@ def predict(self, T): def calibration_curve(y_true, y_prob, normalize=False, n_bins=5): - """Compute true and predicted probabilities for a calibration curve. + """Compute true and predicted probabilities for a calibration curve also + referred to as the reliability diagram. Read more in the :ref:`User Guide `. From ab18ceaf7cd1157780611d776791e6424c09d265 Mon Sep 17 00:00:00 2001 From: mehak-sachdeva Date: Sat, 4 Mar 2017 15:13:00 -0500 Subject: [PATCH 2/2] formatting edits --- doc/modules/calibration.rst | 15 ++++++++------- sklearn/calibration.py | 5 +++-- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/doc/modules/calibration.rst b/doc/modules/calibration.rst index 44644dd169640..0c0af594398a0 100644 --- a/doc/modules/calibration.rst +++ b/doc/modules/calibration.rst @@ -118,13 +118,14 @@ is evaluated with Brier score :func:`brier_score_loss`, reported in the legend :align: center One can observe here that logistic regression is well calibrated as its curve is -nearly diagonal. Linear SVC's calibration curve or reliability diagram has a sigmoid curve, which is -typical for an under-confident classifier. In the case of LinearSVC, this is -caused by the margin property of the hinge loss, which lets the model focus on -hard samples that are close to the decision boundary (the support vectors). Both -kinds of calibration can fix this issue and yield nearly identical results. -The next figure shows the calibration curve of Gaussian naive Bayes on -the same data, with both kinds of calibration and also without calibration. +nearly diagonal. Linear SVC's calibration curve or reliability diagram has a +sigmoid curve, which is typical for an under-confident classifier. In the case of +LinearSVC, this is caused by the margin property of the hinge loss, which lets +the model focus on hard samples that are close to the decision boundary +(the support vectors). Both kinds of calibration can fix this issue and yield +nearly identical results. The next figure shows the calibration curve of +Gaussian naive Bayes on the same data, with both kinds of calibration and also +without calibration. .. figure:: ../auto_examples/calibration/images/sphx_glr_plot_calibration_curve_001.png :target: ../auto_examples/calibration/plot_calibration_curve.html diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 6880f26134558..0d2f76cd12239 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -511,8 +511,9 @@ def predict(self, T): def calibration_curve(y_true, y_prob, normalize=False, n_bins=5): - """Compute true and predicted probabilities for a calibration curve also - referred to as the reliability diagram. + """Compute true and predicted probabilities for a calibration curve. + + Calibration curves may also be referred to as reliability diagrams. Read more in the :ref:`User Guide `.