diff --git a/doc/whats_new/v1.0.rst b/doc/whats_new/v1.0.rst index e4bff3c124dc5..5eed8bffcb2b1 100644 --- a/doc/whats_new/v1.0.rst +++ b/doc/whats_new/v1.0.rst @@ -462,6 +462,11 @@ Changelog :mod:`sklearn.naive_bayes` .......................... +- |Fix| A new parameter `force_alpha` was added to :class:`BernoulliNB` and + class:`MultinomialNB`, allowing user to set parameter alpha to a very + small number, greater or equal 0, which was earlier automatically changed + to `_ALPHA_MIN` instead. + :pr:`16747`, :pr:`18805` by :user:`arka204` and :user:`hongshaoyang`. - |Fix| The `fit` and `partial_fit` methods of the discrete naive Bayes classifiers (:class:`naive_bayes.BernoulliNB`, diff --git a/sklearn/naive_bayes.py b/sklearn/naive_bayes.py index 9707151eba0ca..dd30914df9cfd 100644 --- a/sklearn/naive_bayes.py +++ b/sklearn/naive_bayes.py @@ -537,11 +537,18 @@ def _check_alpha(self): "with shape [n_features]" ) if np.min(self.alpha) < _ALPHA_MIN: - warnings.warn( - "alpha too small will result in numeric errors, " - "setting alpha = %.1e" % _ALPHA_MIN - ) - return np.maximum(self.alpha, _ALPHA_MIN) + if self.force_alpha: + warnings.warn( + "alpha too small will result in numeric errors. " + "Proceeding with alpha = %.1e, as " + "force_alpha was set to True." % self.alpha + ) + else: + warnings.warn( + "alpha too small will result in numeric errors, " + "setting alpha = %.1e" % _ALPHA_MIN + ) + return np.maximum(self.alpha, _ALPHA_MIN) return self.alpha def partial_fit(self, X, y, classes=None, sample_weight=None): @@ -735,7 +742,14 @@ class MultinomialNB(_BaseDiscreteNB): ---------- alpha : float, default=1.0 Additive (Laplace/Lidstone) smoothing parameter - (0 for no smoothing). + (set alpha=0 and force_alpha=True, for no smoothing). + + force_alpha : bool, default=False + If False and alpha is too close to 0, it will set alpha to + `_ALPHA_MIN`. If True, warn user about potential numeric errors + and proceed with alpha unchanged. + + .. versionadded:: 1.0 fit_prior : bool, default=True Whether to learn class prior probabilities or not. @@ -820,8 +834,11 @@ class MultinomialNB(_BaseDiscreteNB): https://nlp.stanford.edu/IR-book/html/htmledition/naive-bayes-text-classification-1.html """ - def __init__(self, *, alpha=1.0, fit_prior=True, class_prior=None): + def __init__( + self, *, alpha=1.0, force_alpha=False, fit_prior=True, class_prior=None + ): self.alpha = alpha + self.force_alpha = force_alpha self.fit_prior = fit_prior self.class_prior = class_prior @@ -862,7 +879,15 @@ class ComplementNB(_BaseDiscreteNB): Parameters ---------- alpha : float, default=1.0 - Additive (Laplace/Lidstone) smoothing parameter (0 for no smoothing). + Additive (Laplace/Lidstone) smoothing parameter + (set alpha=0 and force_alpha=True, for no smoothing). + + force_alpha : bool, default=False + If False and alpha is too close to 0, it will set alpha to + `_ALPHA_MIN`. If True, warn user about potential numeric errors + and proceed with alpha unchanged. + + .. versionadded:: 1.0 fit_prior : bool, default=True Only used in edge case with a single class in the training set. @@ -949,8 +974,17 @@ class ComplementNB(_BaseDiscreteNB): https://people.csail.mit.edu/jrennie/papers/icml03-nb.pdf """ - def __init__(self, *, alpha=1.0, fit_prior=True, class_prior=None, norm=False): + def __init__( + self, + *, + alpha=1.0, + force_alpha=False, + fit_prior=True, + class_prior=None, + norm=False, + ): self.alpha = alpha + self.force_alpha = force_alpha self.fit_prior = fit_prior self.class_prior = class_prior self.norm = norm @@ -998,7 +1032,14 @@ class BernoulliNB(_BaseDiscreteNB): ---------- alpha : float, default=1.0 Additive (Laplace/Lidstone) smoothing parameter - (0 for no smoothing). + (set alpha=0 and force_alpha=True, for no smoothing). + + force_alpha : bool, default=False + If False and alpha is too close to 0, it will set alpha to + `_ALPHA_MIN`. If True, warn user about potential numeric errors + and proceed with alpha unchanged. + + .. versionadded:: 1.0 binarize : float or None, default=0.0 Threshold for binarizing (mapping to booleans) of sample features. @@ -1079,8 +1120,17 @@ class BernoulliNB(_BaseDiscreteNB): naive Bayes -- Which naive Bayes? 3rd Conf. on Email and Anti-Spam (CEAS). """ - def __init__(self, *, alpha=1.0, binarize=0.0, fit_prior=True, class_prior=None): + def __init__( + self, + *, + alpha=1.0, + force_alpha=False, + binarize=0.0, + fit_prior=True, + class_prior=None, + ): self.alpha = alpha + self.force_alpha = force_alpha self.binarize = binarize self.fit_prior = fit_prior self.class_prior = class_prior @@ -1144,7 +1194,14 @@ class CategoricalNB(_BaseDiscreteNB): ---------- alpha : float, default=1.0 Additive (Laplace/Lidstone) smoothing parameter - (0 for no smoothing). + (set alpha=0 and force_alpha=True, for no smoothing). + + force_alpha : bool, default=False + If False and alpha is too close to 0, it will set alpha to + `_ALPHA_MIN`. If True, warn user about potential numeric errors + and proceed with alpha unchanged. + + .. versionadded:: 1.0 fit_prior : bool, default=True Whether to learn class prior probabilities or not. @@ -1221,9 +1278,16 @@ class CategoricalNB(_BaseDiscreteNB): """ def __init__( - self, *, alpha=1.0, fit_prior=True, class_prior=None, min_categories=None + self, + *, + alpha=1.0, + force_alpha=False, + fit_prior=True, + class_prior=None, + min_categories=None, ): self.alpha = alpha + self.force_alpha = force_alpha self.fit_prior = fit_prior self.class_prior = class_prior self.min_categories = min_categories diff --git a/sklearn/tests/test_naive_bayes.py b/sklearn/tests/test_naive_bayes.py index fccd79aae6af2..969a34e889994 100644 --- a/sklearn/tests/test_naive_bayes.py +++ b/sklearn/tests/test_naive_bayes.py @@ -897,6 +897,31 @@ def test_alpha(): m_nb.partial_fit(X, y, classes=[0, 1]) +def test_check_alpha(): + # Non-regression test for: + # https://github.com/scikit-learn/scikit-learn/issues/10772 + # Test force_alpha if alpha < _ALPHA_MIN + _ALPHA_MIN = 1e-10 # const + msg1 = ( + "alpha too small will result in numeric errors. " + "Proceeding with alpha = .+, as " + "force_alpha was set to True." + ) + msg2 = ( + "alpha too small will result in numeric errors, " + "setting alpha = %.1e" % _ALPHA_MIN + ) + b = BernoulliNB(alpha=0, force_alpha=True) + with pytest.warns(UserWarning, match=msg1): + assert b._check_alpha() == 0 + b = BernoulliNB(alpha=0, force_alpha=False) + with pytest.warns(UserWarning, match=msg2): + assert b._check_alpha() == _ALPHA_MIN + b = BernoulliNB(alpha=0) + with pytest.warns(UserWarning, match=msg2): + assert b._check_alpha() == _ALPHA_MIN + + def test_alpha_vector(): X = np.array([[1, 0], [1, 1]]) y = np.array([0, 1])