scikit-learn · avm19 · Feb 21, 2022 · Feb 22, 2022 · Feb 22, 2022 · Feb 22, 2022
diff --git a/doc/modules/classes.rst b/doc/modules/classes.rst
@@ -1317,6 +1317,7 @@ Visualization
    naive_bayes.ComplementNB
    naive_bayes.GaussianNB
    naive_bayes.MultinomialNB
+   naive_bayes.ColumnwiseNB
 
 
 .. _neighbors_ref:

diff --git a/doc/modules/naive_bayes.rst b/doc/modules/naive_bayes.rst
@@ -259,6 +259,41 @@ that all categories for each feature :math:`i` are represented with numbers
 :math:`0, ..., n_i - 1` where :math:`n_i` is the number of available categories
 of feature :math:`i`.
 
+.. _columnwise_naive_bayes:
+
+Mix and match naive Bayes models
+--------------------------------
+
+A naive Bayes model that assumes different distribution families for different
+features (or subsets of features) can be constructed using :class:`ColumnwiseNB`.
+It is a meta-estimator, whose operation relies on naive Bayes
+sub-estimators, which can be chosen in any number or combination from
+:class:`GaussianNB`, :class:`MultinomialNB`, :class:`ComplementNB`,
+:class:`BernoulliNB`, :class:`CategoricalNB`, and user-defined models
+(provided they implement necessary methods).
+
+When creating a :class:`ColumnwiseNB` estimator, one specifies sub-estimators
+and their respective column subsets as a list of tuples.
+Each sub-estimator is fitted and evaluated independently of the
+others and "sees" only the features assigned to it. The predictions of sub-estimators are
+combined via
+
+.. math::
+
+   \log P(x,y)=\log P(x_{1},y) + \dots + \log P(x_{M},y) - (M - 1)\log P(y),
+
+where :math:`\log P(x,y)` is the joint log-probability predicted by the meta-estimator,
+:math:`\log P(x_{m},y)` is that by the :math:`m` th sub-estimator,
+:math:`\log P(y)` is the class prior used by the meta-estimator, and
+:math:`M\geq1` is the total number of sub-estimators.
+
+See :ref:`sphx_glr_auto_examples_miscellaneous_plot_combining_naive_bayes.py`
+for an example of a mixed naive Bayes model implementation.
+
+See also :ref:`voting_classifier` for a way of combining general classifiers.
+An introduction to processing datasets with heterogeneous features is available at
+:ref:`column_transformer`.
+
 Out-of-core naive Bayes model fitting
 -------------------------------------
 

diff --git a/doc/whats_new/v1.4.rst b/doc/whats_new/v1.4.rst
@@ -237,6 +237,13 @@ Changelog
 - |Enhancement| :func:`sklearn.model_selection.train_test_split` now supports
   Array API compatible inputs. :pr:`26855` by `Tim Head`_.
 
+:mod:`sklearn.naive_bayes`
+..........................
+
+- |Feature| Add :class:`naive_bayes.ColumnwiseNB`, a meta-estimator that allows
+  existing naive Bayes classifiers to be combined and applied to different columns
+  of `X`. :pr:`22574` by :user:`Andrey Melnik <avm19>`.
+
 :mod:`sklearn.neighbors`
 ........................
 

diff --git a/examples/miscellaneous/plot_combining_naive_bayes.py b/examples/miscellaneous/plot_combining_naive_bayes.py
@@ -0,0 +1,135 @@
+"""
+===================================================
+Combining Naive Bayes Estimators using ColumnwiseNB
+===================================================
+
+.. currentmodule:: sklearn
+
+This example shows how to use :class:`~naive_bayes.ColumnwiseNB`
+meta-estimator to construct a naive Bayes model from base naive Bayes
+estimators. The resulting model is applied to a dataset with a mixture of
+discrete and continuous features.
+
+We consider the titanic dataset, in which:
+
+- numerical (continous) features "age" and "fare" are handled by
+  :class:`~naive_bayes.GaussianNB`;
+- categorical (discrete) features "embarked", "sex", and "pclass" are handled
+  by :class:`~naive_bayes.CategoricalNB`.
+"""
+
+# Author: Andrey V. Melnik <andrey.melnik.maths@gmail.com>
+#         Pedro Morales <part.morales@gmail.com>
+#
+# License: BSD 3 clause
+
+# %%
+import pandas as pd
+
+from sklearn import set_config
+from sklearn.datasets import fetch_openml
+
+set_config(transform_output="pandas")
+
+X, y = fetch_openml(
+    "titanic", version=1, as_frame=True, return_X_y=True, n_retries=10, parser="auto"
+)
+X["pclass"] = X["pclass"].astype("category")
+# Add a category for NaNs to the "embarked" feature:
+X["embarked"] = X["embarked"].cat.add_categories("N/A").fillna("N/A")
+
+# %%
+# Build and use a pipeline around ``ColumnwiseNB``
+# ------------------------------------------------
+
+from sklearn.compose import ColumnTransformer
+from sklearn.impute import SimpleImputer
+from sklearn.metrics import accuracy_score
+from sklearn.model_selection import GridSearchCV, train_test_split
+from sklearn.naive_bayes import CategoricalNB, ColumnwiseNB, GaussianNB
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import OrdinalEncoder
+
+numeric_features = ["age", "fare"]
+numeric_transformer = SimpleImputer(strategy="median")
+
+categorical_features = ["embarked", "sex", "pclass"]
+categories = [X[c].unique().to_list() for c in X[categorical_features]]
+categorical_transformer = OrdinalEncoder(categories=categories)
+
+preprocessor = ColumnTransformer(
+    transformers=[
+        ("num", numeric_transformer, numeric_features),
+        ("cat", categorical_transformer, categorical_features),
+    ],
+    verbose_feature_names_out=False,
+)
+
+classifier = ColumnwiseNB(
+    estimators=[
+        ("gnb", GaussianNB(), numeric_features),
+        ("cnb", CategoricalNB(), categorical_features),
+    ]
+)
+
+pipe = Pipeline(steps=[("preprocessor", preprocessor), ("classifier", classifier)])
+pipe
+# %%
+X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=0)
+
+pipe.fit(X_train, y_train)
+y_pred = pipe.predict(X_test)
+print(f"Test accuracy: {accuracy_score(y_test, y_pred)}")
+
+# %%
+# Compare choices of columns using ``GridSearchCV``
+# --------------------------------------------------
+#
+# The allocation of columns to constituent subestimators can be regarded as a
+# hyperparameter. We can explore the combinations of columns' choices and values
+# of other hyperparameters with the help of :class:`~.model_selection.GridSearchCV`.
+
+param_grid = {
+    "classifier__estimators": [
+        [
+            ("gnb", GaussianNB(), ["age", "fare"]),
+            ("cnb", CategoricalNB(), categorical_features),
+        ],
+        [("gnb", GaussianNB(), []), ("cnb", CategoricalNB(), ["pclass"])],
+        [("gnb", GaussianNB(), ["embarked"]), ("cnb", CategoricalNB(), [])],
+    ],
+    "preprocessor__num__strategy": ["mean", "most_frequent"],
+}
+
+grid_search = GridSearchCV(pipe, param_grid, cv=10)
+grid_search
+
+# %%
+# Calling `fit` triggers the cross-validated search for the best
+# hyperparameters combination:
+
+grid_search.fit(X_train, y_train)
+
+print("Best params:")
+print(grid_search.best_params_)
+
+# %%
+# As it turns out, the best results are achieved by the naive Bayes model when "sex"
+# is the only feature used:
+
+cv_results = pd.DataFrame(grid_search.cv_results_)
+cv_results = cv_results.sort_values("mean_test_score", ascending=False)
+cv_results["Columns dictionary"] = cv_results["param_classifier__estimators"].map(
+    lambda l: {e[0]: e[-1] for e in l}
+)
+cv_results["'gnb' columns"] = cv_results["Columns dictionary"].map(lambda d: d["gnb"])
+cv_results["'cnb' columns"] = cv_results["Columns dictionary"].map(lambda d: d["cnb"])
+cv_results[
+    [
+        "mean_test_score",
+        "std_test_score",
+        "param_preprocessor__num__strategy",
+        "'gnb' columns",
+        "'cnb' columns",
+    ]
+]