From 577651753e6b22510d02caeef684f145a5fc6372 Mon Sep 17 00:00:00 2001 From: Florian Wilhelm Date: Fri, 10 Oct 2014 20:25:22 +0200 Subject: [PATCH 1/5] ENH: Added median_absolute_error to metrics --- doc/modules/classes.rst | 1 + doc/modules/model_evaluation.rst | 2 +- sklearn/metrics/__init__.py | 2 ++ sklearn/metrics/metrics.py | 1 + sklearn/metrics/regression.py | 38 +++++++++++++++++++++ sklearn/metrics/scorer.py | 11 +++--- sklearn/metrics/tests/test_score_objects.py | 10 +++--- 7 files changed, 56 insertions(+), 9 deletions(-) diff --git a/doc/modules/classes.rst b/doc/modules/classes.rst index 4fd228c0e5c46..62afce7f48f48 100644 --- a/doc/modules/classes.rst +++ b/doc/modules/classes.rst @@ -770,6 +770,7 @@ details. metrics.explained_variance_score metrics.mean_absolute_error metrics.mean_squared_error + metrics.median_absolute_error metrics.r2_score Multilabel ranking metrics diff --git a/doc/modules/model_evaluation.rst b/doc/modules/model_evaluation.rst index f7bd64b815272..60cb2825943b6 100644 --- a/doc/modules/model_evaluation.rst +++ b/doc/modules/model_evaluation.rst @@ -78,7 +78,7 @@ of acceptable values:: >>> model = svm.SVC() >>> cross_validation.cross_val_score(model, X, y, scoring='wrong_choice') Traceback (most recent call last): - ValueError: 'wrong_choice' is not a valid scoring value. Valid options are ['accuracy', 'adjusted_rand_score', 'average_precision', 'f1', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'precision', 'r2', 'recall', 'roc_auc'] + ValueError: 'wrong_choice' is not a valid scoring value. Valid options are ['accuracy', 'adjusted_rand_score', 'average_precision', 'f1', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'r2', 'recall', 'roc_auc'] .. note:: diff --git a/sklearn/metrics/__init__.py b/sklearn/metrics/__init__.py index ead9a092b90d2..16ebd318d6a37 100644 --- a/sklearn/metrics/__init__.py +++ b/sklearn/metrics/__init__.py @@ -47,6 +47,7 @@ from .regression import explained_variance_score from .regression import mean_absolute_error from .regression import mean_squared_error +from .regression import median_absolute_error from .regression import r2_score from .scorer import make_scorer @@ -81,6 +82,7 @@ 'matthews_corrcoef', 'mean_absolute_error', 'mean_squared_error', + 'median_absolute_error', 'mutual_info_score', 'normalized_mutual_info_score', 'pairwise_distances', diff --git a/sklearn/metrics/metrics.py b/sklearn/metrics/metrics.py index efb86a158d917..1ffaff5039c2d 100644 --- a/sklearn/metrics/metrics.py +++ b/sklearn/metrics/metrics.py @@ -29,6 +29,7 @@ from .regression import explained_variance_score from .regression import mean_absolute_error from .regression import mean_squared_error +from .regression import median_absolute_error from .regression import r2_score # Deprecated in 0.16 diff --git a/sklearn/metrics/regression.py b/sklearn/metrics/regression.py index 648422fa84693..339704db45ac4 100644 --- a/sklearn/metrics/regression.py +++ b/sklearn/metrics/regression.py @@ -27,6 +27,7 @@ __ALL__ = [ "mean_absolute_error", "mean_squared_error", + "median_absolute_error", "r2_score", "explained_variance_score" ] @@ -177,6 +178,43 @@ def mean_squared_error(y_true, y_pred, sample_weight=None): weights=sample_weight) +def median_absolute_error(y_true, y_pred): + """Median absolute error regression loss + + The loss is calculated by taking the median of all absolute + differences between the target and the prediction. This metric is + particularly interesting because it is robust to outliers. + + Parameters + ---------- + y_true : array-like of shape = [n_samples] or [n_samples, n_outputs] + Ground truth (correct) target values. + + y_pred : array-like of shape = [n_samples] or [n_samples, n_outputs] + Estimated target values. + + Returns + ------- + loss : float + A positive floating point value (the best value is 0.0). + + Examples + -------- + >>> from sklearn.metrics import median_absolute_error + >>> y_true = [3, -0.5, 2, 7] + >>> y_pred = [2.5, 0.0, 2, 8] + >>> median_absolute_error(y_true, y_pred) + 0.5 + >>> y_true = [[0.5, 1], [-1, 1], [7, -6]] + >>> y_pred = [[0, 2], [-1, 2], [8, -5]] + >>> median_absolute_error(y_true, y_pred) + 1.0 + + """ + y_type, y_true, y_pred = _check_reg_targets(y_true, y_pred) + return np.median(np.abs(y_pred - y_true)) + + def explained_variance_score(y_true, y_pred, sample_weight=None): """Explained variance regression score function diff --git a/sklearn/metrics/scorer.py b/sklearn/metrics/scorer.py index 83e0918051992..6f99deb815d04 100644 --- a/sklearn/metrics/scorer.py +++ b/sklearn/metrics/scorer.py @@ -22,9 +22,9 @@ import numpy as np -from . import (r2_score, mean_absolute_error, mean_squared_error, - accuracy_score, f1_score, roc_auc_score, - average_precision_score, +from . import (r2_score, median_absolute_error, mean_absolute_error, + mean_squared_error, accuracy_score, f1_score, + roc_auc_score, average_precision_score, precision_score, recall_score, log_loss) from .cluster import adjusted_rand_score from ..utils.multiclass import type_of_target @@ -86,7 +86,7 @@ def __call__(self, estimator, X, y_true, sample_weight=None): else: return self._sign * self._score_func(y_true, y_pred, **self._kwargs) - + class _ProbaScorer(_BaseScorer): def __call__(self, clf, X, y, sample_weight=None): @@ -316,6 +316,8 @@ def make_scorer(score_func, greater_is_better=True, needs_proba=False, greater_is_better=False) mean_absolute_error_scorer = make_scorer(mean_absolute_error, greater_is_better=False) +median_absolute_error_scorer = make_scorer(median_absolute_error, + greater_is_better=False) # Standard Classification Scores accuracy_scorer = make_scorer(accuracy_score) @@ -337,6 +339,7 @@ def make_scorer(score_func, greater_is_better=True, needs_proba=False, adjusted_rand_scorer = make_scorer(adjusted_rand_score) SCORERS = dict(r2=r2_scorer, + median_absolute_error=median_absolute_error_scorer, mean_absolute_error=mean_absolute_error_scorer, mean_squared_error=mean_squared_error_scorer, accuracy=accuracy_scorer, f1=f1_scorer, roc_auc=roc_auc_scorer, diff --git a/sklearn/metrics/tests/test_score_objects.py b/sklearn/metrics/tests/test_score_objects.py index 0845b5e5a5709..aef0a33dc4a59 100644 --- a/sklearn/metrics/tests/test_score_objects.py +++ b/sklearn/metrics/tests/test_score_objects.py @@ -28,7 +28,8 @@ from sklearn.multiclass import OneVsRestClassifier -REGRESSION_SCORERS = ['r2', 'mean_absolute_error', 'mean_squared_error'] +REGRESSION_SCORERS = ['r2', 'mean_absolute_error', 'mean_squared_error', + 'median_absolute_error'] CLF_SCORERS = ['accuracy', 'f1', 'roc_auc', 'average_precision', 'precision', 'recall', 'log_loss', 'adjusted_rand_score' # not really, but works @@ -273,9 +274,10 @@ def test_scorer_sample_weight(): "called with sample weights: {1} vs " "{2}".format(name, weighted, unweighted)) assert_almost_equal(weighted, ignored, - err_msg="scorer {0} behaves differently when " - "ignoring samples and setting sample_weight to 0: " - "{1} vs {2}".format(name, weighted, ignored)) + err_msg="scorer {0} behaves differently when " + "ignoring samples and setting sample_weight to" + " 0: {1} vs {2}".format(name, weighted, + ignored)) except TypeError as e: assert_true("sample_weight" in str(e), From afc372aaae485ebaad93fd4e59ac92d0b71e61b8 Mon Sep 17 00:00:00 2001 From: Florian Wilhelm Date: Fri, 10 Oct 2014 20:45:56 +0200 Subject: [PATCH 2/5] DOC: Added doc in model_evaluation.rst --- doc/modules/model_evaluation.rst | 29 +++++++++++++++++++++++++++++ sklearn/metrics/regression.py | 4 ---- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/doc/modules/model_evaluation.rst b/doc/modules/model_evaluation.rst index 60cb2825943b6..94ec15ab45e46 100644 --- a/doc/modules/model_evaluation.rst +++ b/doc/modules/model_evaluation.rst @@ -66,6 +66,7 @@ Scoring Function **Regression** 'mean_absolute_error' :func:`sklearn.metrics.mean_absolute_error` 'mean_squared_error' :func:`sklearn.metrics.mean_squared_error` +'median_absolute_error' :func:`sklearn.metrics.median_absolute_error` 'r2' :func:`sklearn.metrics.r2_score` ====================== ================================================= @@ -1056,6 +1057,34 @@ function:: for an example of mean squared error usage to evaluate gradient boosting regression. +Median absolute error +--------------------- + +The :func:`median_absolute_error` is particularly interesting because it is +robust to outliers. The loss is calculated by taking the median of all absolute +differences between the target and the prediction. + +If :math:`\hat{y}_i` is the predicted value of the :math:`i`-th sample +and :math:`y_i` is the corresponding true value, then the median absolute error +(MedAE) estimated over :math:`n_{\text{samples}}` is defined as + +.. math:: + + \text{MedAE}(y, \hat{y}) = \text{median}(\mid y_1 - \hat{y}_1 \mid, \ldots, \mid y_n - \hat{y}_n \mid). + +Here a small example of usage of the :func:`median_absolute_error` +function:: + + >>> from sklearn.metrics import median_absolute_error + >>> y_true = [3, -0.5, 2, 7] + >>> y_pred = [2.5, 0.0, 2, 8] + >>> median_absolute_error(y_true, y_pred) + 0.5 + >>> y_true = [[0.5, 1], [-1, 1], [7, -6]] + >>> y_pred = [[0, 2], [-1, 2], [8, -5]] + >>> median_absolute_error(y_true, y_pred) + 1.0 + R² score, the coefficient of determination ------------------------------------------- diff --git a/sklearn/metrics/regression.py b/sklearn/metrics/regression.py index 339704db45ac4..e355e3bccf9ee 100644 --- a/sklearn/metrics/regression.py +++ b/sklearn/metrics/regression.py @@ -181,10 +181,6 @@ def mean_squared_error(y_true, y_pred, sample_weight=None): def median_absolute_error(y_true, y_pred): """Median absolute error regression loss - The loss is calculated by taking the median of all absolute - differences between the target and the prediction. This metric is - particularly interesting because it is robust to outliers. - Parameters ---------- y_true : array-like of shape = [n_samples] or [n_samples, n_outputs] From 7f7fa1b829087e0c5d367a042f03f4bd6f278821 Mon Sep 17 00:00:00 2001 From: Florian Wilhelm Date: Fri, 10 Oct 2014 21:45:46 +0200 Subject: [PATCH 3/5] ENH: unit tests for median_absolute_error --- sklearn/metrics/tests/test_common.py | 5 ++++- sklearn/metrics/tests/test_regression.py | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/sklearn/metrics/tests/test_common.py b/sklearn/metrics/tests/test_common.py index efbaf244aee56..6ef099d386e13 100644 --- a/sklearn/metrics/tests/test_common.py +++ b/sklearn/metrics/tests/test_common.py @@ -37,6 +37,7 @@ from sklearn.metrics import matthews_corrcoef from sklearn.metrics import mean_absolute_error from sklearn.metrics import mean_squared_error +from sklearn.metrics import median_absolute_error from sklearn.metrics import precision_score from sklearn.metrics import r2_score from sklearn.metrics import recall_score @@ -87,6 +88,7 @@ REGRESSION_METRICS = { "mean_absolute_error": mean_absolute_error, "mean_squared_error": mean_squared_error, + "median_absolute_error": median_absolute_error, "explained_variance_score": explained_variance_score, "r2_score": r2_score, } @@ -291,7 +293,8 @@ "f1_score", "weighted_f1_score", "micro_f1_score", "macro_f1_score", - "matthews_corrcoef_score", "mean_absolute_error", "mean_squared_error" + "matthews_corrcoef_score", "mean_absolute_error", "mean_squared_error", + "median_absolute_error" ] diff --git a/sklearn/metrics/tests/test_regression.py b/sklearn/metrics/tests/test_regression.py index 55a26074b0702..4fdcc446efd46 100644 --- a/sklearn/metrics/tests/test_regression.py +++ b/sklearn/metrics/tests/test_regression.py @@ -11,6 +11,7 @@ from sklearn.metrics import explained_variance_score from sklearn.metrics import mean_absolute_error from sklearn.metrics import mean_squared_error +from sklearn.metrics import median_absolute_error from sklearn.metrics import r2_score from sklearn.metrics.regression import _check_reg_targets @@ -22,6 +23,7 @@ def test_regression_metrics(n_samples=50): assert_almost_equal(mean_squared_error(y_true, y_pred), 1.) assert_almost_equal(mean_absolute_error(y_true, y_pred), 1.) + assert_almost_equal(median_absolute_error(y_true, y_pred), 1.) assert_almost_equal(r2_score(y_true, y_pred), 0.995, 2) assert_almost_equal(explained_variance_score(y_true, y_pred), 1.) @@ -45,6 +47,7 @@ def test_multioutput_regression(): def test_regression_metrics_at_limits(): assert_almost_equal(mean_squared_error([0.], [0.]), 0.00, 2) assert_almost_equal(mean_absolute_error([0.], [0.]), 0.00, 2) + assert_almost_equal(median_absolute_error([0.], [0.]), 0.00, 2) assert_almost_equal(explained_variance_score([0.], [0.]), 1.00, 2) assert_almost_equal(r2_score([0., 1], [0., 1]), 1.00, 2) From 5dfbc788ccb4f1e662b24ddfc3621ff399f39232 Mon Sep 17 00:00:00 2001 From: Florian Wilhelm Date: Sun, 12 Oct 2014 13:41:16 +0200 Subject: [PATCH 4/5] ENH: test_common.py unit tests pass --- sklearn/metrics/tests/test_common.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sklearn/metrics/tests/test_common.py b/sklearn/metrics/tests/test_common.py index 6ef099d386e13..0e856230cbb3d 100644 --- a/sklearn/metrics/tests/test_common.py +++ b/sklearn/metrics/tests/test_common.py @@ -280,7 +280,8 @@ # Regression metrics with "multioutput-continuous" format support MULTIOUTPUT_METRICS = [ - "mean_absolute_error", "mean_squared_error", "r2_score", + "mean_absolute_error", "mean_squared_error", "median_absolute_error", + "r2_score", ] # Symmetric with respect to their input arguments y_true and y_pred @@ -324,6 +325,7 @@ "hamming_loss", "hinge_loss", "matthews_corrcoef_score", + "median_absolute_error", ] From c9e711b7006740e41da298aa00cb36fa50ab1ad1 Mon Sep 17 00:00:00 2001 From: Florian Wilhelm Date: Sun, 12 Oct 2014 14:41:15 +0200 Subject: [PATCH 5/5] FIX: Small typo --- doc/modules/model_evaluation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/model_evaluation.rst b/doc/modules/model_evaluation.rst index 94ec15ab45e46..2ac27395a0d75 100644 --- a/doc/modules/model_evaluation.rst +++ b/doc/modules/model_evaluation.rst @@ -1072,7 +1072,7 @@ and :math:`y_i` is the corresponding true value, then the median absolute error \text{MedAE}(y, \hat{y}) = \text{median}(\mid y_1 - \hat{y}_1 \mid, \ldots, \mid y_n - \hat{y}_n \mid). -Here a small example of usage of the :func:`median_absolute_error` +Here is a small example of usage of the :func:`median_absolute_error` function:: >>> from sklearn.metrics import median_absolute_error