From 07d03277c184966ac41f1e31b5aea41542acf8a9 Mon Sep 17 00:00:00 2001 From: Hanmin Qin Date: Tue, 12 Feb 2019 18:34:00 +0800 Subject: [PATCH 1/6] consistent error message --- sklearn/metrics/classification.py | 10 +++----- sklearn/metrics/tests/test_classification.py | 27 +++++++++++++++++++- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/sklearn/metrics/classification.py b/sklearn/metrics/classification.py index 7d1653c426c68..96ed965e252aa 100644 --- a/sklearn/metrics/classification.py +++ b/sklearn/metrics/classification.py @@ -1247,13 +1247,9 @@ def precision_recall_fscore_support(y_true, y_pred, beta=1.0, labels=None, if average == 'binary': if y_type == 'binary': - if pos_label not in present_labels: - if len(present_labels) < 2: - # Only negative labels - return (0., 0., 0., 0) - else: - raise ValueError("pos_label=%r is not a valid label: %r" % - (pos_label, present_labels)) + if pos_label not in present_labels and len(present_labels) >= 2: + raise ValueError("pos_label=%r is not a valid label: %r" % + (pos_label, present_labels)) labels = [pos_label] else: raise ValueError("Target is %s but average='binary'. Please " diff --git a/sklearn/metrics/tests/test_classification.py b/sklearn/metrics/tests/test_classification.py index a1ddc4654c462..09972238e7a4d 100644 --- a/sklearn/metrics/tests/test_classification.py +++ b/sklearn/metrics/tests/test_classification.py @@ -1446,6 +1446,17 @@ def test_prf_warnings(): 'being set to 0.0 due to no true samples.') my_assert(w, msg, f, [-1, -1], [1, 1], average='binary') + clean_warning_registry() + with warnings.catch_warnings(record=True) as record: + warnings.simplefilter('always') + precision_recall_fscore_support([0, 0], [0, 0], average="binary") + msg = ('Precision and F-score are ill-defined and ' + 'being set to 0.0 due to no predicted samples.') + assert_equal(str(record.pop().message), msg) + msg = ('Recall and F-score are ill-defined and ' + 'being set to 0.0 due to no true samples.') + assert_equal(str(record.pop().message), msg) + def test_recall_warnings(): assert_no_warnings(recall_score, @@ -1461,19 +1472,26 @@ def test_recall_warnings(): assert_equal(str(record.pop().message), 'Recall is ill-defined and ' 'being set to 0.0 due to no true samples.') + recall_score([0, 0], [0, 0]) + assert_equal(str(record.pop().message), + 'Recall is ill-defined and ' + 'being set to 0.0 due to no true samples.') def test_precision_warnings(): clean_warning_registry() with warnings.catch_warnings(record=True) as record: warnings.simplefilter('always') - precision_score(np.array([[1, 1], [1, 1]]), np.array([[0, 0], [0, 0]]), average='micro') assert_equal(str(record.pop().message), 'Precision is ill-defined and ' 'being set to 0.0 due to no predicted samples.') + precision_score([0, 0], [0, 0]) + assert_equal(str(record.pop().message), + 'Precision is ill-defined and ' + 'being set to 0.0 due to no predicted samples.') assert_no_warnings(precision_score, np.array([[0, 0], [0, 0]]), @@ -1499,6 +1517,13 @@ def test_fscore_warnings(): assert_equal(str(record.pop().message), 'F-score is ill-defined and ' 'being set to 0.0 due to no true samples.') + score([0, 0], [0, 0]) + assert_equal(str(record.pop().message), + 'F-score is ill-defined and ' + 'being set to 0.0 due to no true samples.') + assert_equal(str(record.pop().message), + 'F-score is ill-defined and ' + 'being set to 0.0 due to no predicted samples.') def test_prf_average_binary_data_non_binary(): From 6ebd7037e6cc793602478757bdfaa08dafde41b3 Mon Sep 17 00:00:00 2001 From: Hanmin Qin Date: Tue, 12 Feb 2019 18:35:56 +0800 Subject: [PATCH 2/6] new test --- sklearn/metrics/tests/test_classification.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sklearn/metrics/tests/test_classification.py b/sklearn/metrics/tests/test_classification.py index 09972238e7a4d..bfa2d04157668 100644 --- a/sklearn/metrics/tests/test_classification.py +++ b/sklearn/metrics/tests/test_classification.py @@ -1450,12 +1450,12 @@ def test_prf_warnings(): with warnings.catch_warnings(record=True) as record: warnings.simplefilter('always') precision_recall_fscore_support([0, 0], [0, 0], average="binary") - msg = ('Precision and F-score are ill-defined and ' - 'being set to 0.0 due to no predicted samples.') - assert_equal(str(record.pop().message), msg) msg = ('Recall and F-score are ill-defined and ' 'being set to 0.0 due to no true samples.') assert_equal(str(record.pop().message), msg) + msg = ('Precision and F-score are ill-defined and ' + 'being set to 0.0 due to no predicted samples.') + assert_equal(str(record.pop().message), msg) def test_recall_warnings(): From cd6ca5d2d90f835fec06d4d64bc09bd89f13e57a Mon Sep 17 00:00:00 2001 From: Hanmin Qin Date: Tue, 12 Feb 2019 18:44:44 +0800 Subject: [PATCH 3/6] ignore warnings --- sklearn/metrics/tests/test_classification.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sklearn/metrics/tests/test_classification.py b/sklearn/metrics/tests/test_classification.py index bfa2d04157668..2c684a71a0fac 100644 --- a/sklearn/metrics/tests/test_classification.py +++ b/sklearn/metrics/tests/test_classification.py @@ -198,6 +198,7 @@ def test_precision_recall_f1_score_binary(): (1 + 2 ** 2) * ps * rs / (2 ** 2 * ps + rs), 2) +@ignore_warnings def test_precision_recall_f_binary_single_class(): # Test precision, recall and F1 score behave with a single positive or # negative class @@ -1065,6 +1066,7 @@ def test_classification_report_no_labels_target_names_unequal_length(): y_true, y_pred, target_names=target_names) +@ignore_warnings def test_multilabel_classification_report(): n_classes = 4 n_samples = 50 From 9bd1382eaab650711882a0f3a7bba9904d98f841 Mon Sep 17 00:00:00 2001 From: Hanmin Qin Date: Tue, 12 Feb 2019 22:32:42 +0800 Subject: [PATCH 4/6] notes --- sklearn/metrics/classification.py | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/sklearn/metrics/classification.py b/sklearn/metrics/classification.py index 96ed965e252aa..f53ae380edb7c 100644 --- a/sklearn/metrics/classification.py +++ b/sklearn/metrics/classification.py @@ -922,6 +922,11 @@ def f1_score(y_true, y_pred, labels=None, pos_label=1, average='binary', >>> f1_score(y_true, y_pred, average=None) array([0.8, 0. , 0. ]) + Notes + ----- + When ``true_positive + false positive == 0`` or + ``true_positive + false negative == 0``, f-score will be set to 0 + and ``UndefinedMetricWarning`` will be raised. """ return fbeta_score(y_true, y_pred, 1, labels=labels, pos_label=pos_label, average=average, @@ -1036,6 +1041,11 @@ def fbeta_score(y_true, y_pred, beta, labels=None, pos_label=1, ... # doctest: +ELLIPSIS array([0.71..., 0. , 0. ]) + Notes + ----- + When ``true_positive + false positive == 0`` or + ``true_positive + false negative == 0``, f-score will be set to 0 + and ``UndefinedMetricWarning`` will be raised. """ _, _, f, _ = precision_recall_fscore_support(y_true, y_pred, beta=beta, @@ -1233,6 +1243,15 @@ def precision_recall_fscore_support(y_true, y_pred, beta=1.0, labels=None, array([0., 0., 1.]), array([0. , 0. , 0.8]), array([2, 2, 2])) + Notes + ----- + When ``true_positive + false positive == 0``, precision will be set to 0 + and ``UndefinedMetricWarning`` will be raised. + When ``true_positive + false negative == 0``, recall will be set to 0 + and ``UndefinedMetricWarning`` will be raised. + When ``true_positive + false positive == 0`` or + ``true_positive + false negative == 0``, f-score will be set to 0 + and ``UndefinedMetricWarning`` will be raised. """ average_options = (None, 'micro', 'macro', 'weighted', 'samples') if average not in average_options and average != 'binary': @@ -1275,7 +1294,6 @@ def precision_recall_fscore_support(y_true, y_pred, beta=1.0, labels=None, true_sum = np.array([true_sum.sum()]) # Finally, we have all our sufficient statistics. Divide! # - beta2 = beta ** 2 with np.errstate(divide='ignore', invalid='ignore'): # Divide, and on zero-division, set scores to 0 and warn: @@ -1293,7 +1311,6 @@ def precision_recall_fscore_support(y_true, y_pred, beta=1.0, labels=None, f_score[tp_sum == 0] = 0.0 # Average the results - if average == 'weighted': weights = true_sum if weights.sum() == 0: @@ -1406,6 +1423,10 @@ def precision_score(y_true, y_pred, labels=None, pos_label=1, >>> precision_score(y_true, y_pred, average=None) # doctest: +ELLIPSIS array([0.66..., 0. , 0. ]) + Notes + ----- + When ``true_positive + false positive == 0``, precision will be set to 0 + and ``UndefinedMetricWarning`` will be raised. """ p, _, _, _ = precision_recall_fscore_support(y_true, y_pred, labels=labels, @@ -1508,6 +1529,10 @@ def recall_score(y_true, y_pred, labels=None, pos_label=1, average='binary', >>> recall_score(y_true, y_pred, average=None) array([1., 0., 0.]) + Notes + ----- + When ``true_positive + false negative == 0``, recall will be set to 0 + and ``UndefinedMetricWarning`` will be raised. """ _, r, _, _ = precision_recall_fscore_support(y_true, y_pred, labels=labels, From c2c0aab6010c82042fb7da4b419dbaaa30875929 Mon Sep 17 00:00:00 2001 From: Hanmin Qin Date: Wed, 13 Feb 2019 11:47:40 +0800 Subject: [PATCH 5/6] joel's comment --- sklearn/metrics/classification.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/sklearn/metrics/classification.py b/sklearn/metrics/classification.py index f53ae380edb7c..53df6a12bc624 100644 --- a/sklearn/metrics/classification.py +++ b/sklearn/metrics/classification.py @@ -924,8 +924,8 @@ def f1_score(y_true, y_pred, labels=None, pos_label=1, average='binary', Notes ----- - When ``true_positive + false positive == 0`` or - ``true_positive + false negative == 0``, f-score will be set to 0 + When ``true positive + false positive == 0`` or + ``true positive + false negative == 0``, f-score will be set to 0 and ``UndefinedMetricWarning`` will be raised. """ return fbeta_score(y_true, y_pred, 1, labels=labels, @@ -1043,8 +1043,8 @@ def fbeta_score(y_true, y_pred, beta, labels=None, pos_label=1, Notes ----- - When ``true_positive + false positive == 0`` or - ``true_positive + false negative == 0``, f-score will be set to 0 + When ``true positive + false positive == 0`` or + ``true positive + false negative == 0``, f-score will be set to 0 and ``UndefinedMetricWarning`` will be raised. """ _, _, f, _ = precision_recall_fscore_support(y_true, y_pred, @@ -1245,13 +1245,10 @@ def precision_recall_fscore_support(y_true, y_pred, beta=1.0, labels=None, Notes ----- - When ``true_positive + false positive == 0``, precision will be set to 0 - and ``UndefinedMetricWarning`` will be raised. - When ``true_positive + false negative == 0``, recall will be set to 0 - and ``UndefinedMetricWarning`` will be raised. - When ``true_positive + false positive == 0`` or - ``true_positive + false negative == 0``, f-score will be set to 0 - and ``UndefinedMetricWarning`` will be raised. + When ``true positive + false positive == 0``, precision is undefined; + When ``true positive + false negative == 0``, recall is undefined. + In such cases, the metric will be set to 0, as will f-score, and + ``UndefinedMetricWarning`` will be raised. """ average_options = (None, 'micro', 'macro', 'weighted', 'samples') if average not in average_options and average != 'binary': @@ -1425,7 +1422,7 @@ def precision_score(y_true, y_pred, labels=None, pos_label=1, Notes ----- - When ``true_positive + false positive == 0``, precision will be set to 0 + When ``true positive + false positive == 0``, precision will be set to 0 and ``UndefinedMetricWarning`` will be raised. """ p, _, _, _ = precision_recall_fscore_support(y_true, y_pred, @@ -1531,7 +1528,7 @@ def recall_score(y_true, y_pred, labels=None, pos_label=1, average='binary', Notes ----- - When ``true_positive + false negative == 0``, recall will be set to 0 + When ``true positive + false negative == 0``, recall will be set to 0 and ``UndefinedMetricWarning`` will be raised. """ _, r, _, _ = precision_recall_fscore_support(y_true, y_pred, From 4b82c4450b759f4efd0dbad2840739ffc416e533 Mon Sep 17 00:00:00 2001 From: Hanmin Qin Date: Tue, 26 Feb 2019 09:16:24 +0800 Subject: [PATCH 6/6] adrin's comment --- sklearn/metrics/classification.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/sklearn/metrics/classification.py b/sklearn/metrics/classification.py index 53df6a12bc624..8a49e9a0d33cb 100644 --- a/sklearn/metrics/classification.py +++ b/sklearn/metrics/classification.py @@ -925,8 +925,8 @@ def f1_score(y_true, y_pred, labels=None, pos_label=1, average='binary', Notes ----- When ``true positive + false positive == 0`` or - ``true positive + false negative == 0``, f-score will be set to 0 - and ``UndefinedMetricWarning`` will be raised. + ``true positive + false negative == 0``, f-score returns 0 and raises + ``UndefinedMetricWarning``. """ return fbeta_score(y_true, y_pred, 1, labels=labels, pos_label=pos_label, average=average, @@ -1044,8 +1044,8 @@ def fbeta_score(y_true, y_pred, beta, labels=None, pos_label=1, Notes ----- When ``true positive + false positive == 0`` or - ``true positive + false negative == 0``, f-score will be set to 0 - and ``UndefinedMetricWarning`` will be raised. + ``true positive + false negative == 0``, f-score returns 0 and raises + ``UndefinedMetricWarning``. """ _, _, f, _ = precision_recall_fscore_support(y_true, y_pred, beta=beta, @@ -1422,8 +1422,8 @@ def precision_score(y_true, y_pred, labels=None, pos_label=1, Notes ----- - When ``true positive + false positive == 0``, precision will be set to 0 - and ``UndefinedMetricWarning`` will be raised. + When ``true positive + false positive == 0``, precision returns 0 and + raises ``UndefinedMetricWarning``. """ p, _, _, _ = precision_recall_fscore_support(y_true, y_pred, labels=labels, @@ -1528,8 +1528,8 @@ def recall_score(y_true, y_pred, labels=None, pos_label=1, average='binary', Notes ----- - When ``true positive + false negative == 0``, recall will be set to 0 - and ``UndefinedMetricWarning`` will be raised. + When ``true positive + false negative == 0``, recall returns 0 and raises + ``UndefinedMetricWarning``. """ _, r, _, _ = precision_recall_fscore_support(y_true, y_pred, labels=labels,