From 4b5f0b4fd248f75278f2df5d1c53a94a444c35aa Mon Sep 17 00:00:00 2001
From: Ting Lee <Ting_Lee@esri.com>
Date: Sat, 11 Jul 2015 15:38:41 -0700
Subject: [PATCH] add Brier_score_loss to model validation doc #4804

---
 doc/modules/model_evaluation.rst | 40 ++++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)

diff --git a/doc/modules/model_evaluation.rst b/doc/modules/model_evaluation.rst
index d6631d055f666..6143859056d2b 100644
--- a/doc/modules/model_evaluation.rst
+++ b/doc/modules/model_evaluation.rst
@@ -1014,6 +1014,46 @@ set [0,1] has an error: ::
     for an example of zero one loss usage to perform recursive feature
     elimination with cross-validation.
 
+.. _brier_score_loss:
+
+Brier score loss
+----------------
+
+The :func:`brier_score_loss` function returns a score of the mean square difference 
+between the actual outcome and the predicted probability of the possible outcome. 
+The actual outcome has to be 1 or 0 (true or false), while the predicted probability 
+of the actual outcome happens can be value between 0 and 1. The brier score loss is 
+also between 0 to 1 and the lower the score (the mean square difference is smaller), 
+the more accurate the prediction is.
+
+.. math::
+
+   BS = \frac{1}{N} \sum_{t=1}^{N}(f_t - o_t)^2
+
+where : :math:`N` is the total number of predictions, :math:`f_t` is the predicted probablity of the actual outcome :math:`o_t` 
+
+  >>> import numpy as np
+  >>> from sklearn.metrics import zero_one_loss
+  >>> y_true = np.array([0, 1, 1, 0])
+  >>> y_true_categorical = np.array(["spam", "ham", "ham", "spam"])
+  >>> y_prob = np.array([0.1, 0.9, 0.8, 0.3])  
+  >>> brier_score_loss(y_true, y_prob)
+  0.25
+  >>> zero_one_loss(y_true, y_pred, normalize=False)
+  1
+  0.037...
+  >>> brier_score_loss(y_true, 1-y_prob, pos_label=0)  
+  0.037...
+  >>> brier_score_loss(y_true_categorical, y_prob, pos_label="ham")  
+  0.037...
+  >>> brier_score_loss(y_true, np.array(y_prob) > 0.5)
+  0.0
+
+.. topic:: Example:
+
+  * See :ref:`example_calibration_plot_calibration.py`
+    for an example of Brier score loss usage to perform probability calibration of classifiers.
+
 
 .. _multilabel_ranking_metrics: