diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 2cb70269..00000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,63 +0,0 @@ -# Python CircleCI 2.0 configuration file -# -# Check https://circleci.com/docs/2.0/language-python/ for more details -# -version: 2 -jobs: - python3: - docker: - # specify the version you desire here - - image: circleci/python:3.9 - # use `-browsers` prefix for selenium tests, e.g. `3.6.1-browsers` - environment: - - USERNAME: "Menelau" - - DOC_REPO: "DESlib" - - DOC_URL: "" - - EMAIL: "rafaelmenelau@gmail.com" - - MINICONDA_PATH: ~/miniconda - - CONDA_ENV_NAME: testenv - - PYTHON_VERSION: 3 - - # Specify service dependencies here if necessary - # CircleCI maintains a library of pre-built images - # documented at https://circleci.com/docs/2.0/circleci-images/ - # - image: circleci/postgres:9.4 - - working_directory: ~/repo - - steps: - - checkout - - run: - no_output_timeout: 30m - command: | - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh - chmod +x miniconda.sh && ./miniconda.sh -b -p ~/miniconda - export PATH="~/miniconda/bin:$PATH" - conda update --yes --quiet conda - conda create -n testenv --yes --quiet python=3.9 - source activate testenv - conda install --yes pip numpy - pip install -r requirements-dev.txt - pip install . - cd docs - make html - - store_artifacts: - path: docs/_build/html - destination: docs - - store_artifacts: - path: ~/log.txt - - persist_to_workspace: - root: docs/_build/html - paths: . - - attach_workspace: - at: docs/_build/html - - run: ls -ltrh docs/_build/html - filters: - branches: - ignore: gh-pages - -workflows: - version: 2 - build-doc-and-deploy: - jobs: - - python3 diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index e14944e1..00000000 --- a/.coveragerc +++ /dev/null @@ -1,16 +0,0 @@ -[run] -branch = True -source = deslib -include = */deslib/* -omit = - */setup.py - deslib/tests/* - -[report] -exclude_lines = - if self.debug: - pragma: no cover - raise NotImplementedError - if __name__ == .__main__.: -ignore_errors = True -show_missing = True diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 7b80292d..00000000 --- a/.gitignore +++ /dev/null @@ -1,115 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ -cover/ - -# Translations -*.mo -*.pot - -# Sphinx documentation -docs/_build/ - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -.python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -Pipfile.lock - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow -__pypackages__/ - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -dev_env/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ - - -## vscode - -.vscode/ \ No newline at end of file diff --git a/.pep8speaks.yml b/.pep8speaks.yml deleted file mode 100644 index 06bcb450..00000000 --- a/.pep8speaks.yml +++ /dev/null @@ -1,17 +0,0 @@ -# File : .pep8speaks.yml - -scanner: - diff_only: True # If False, the entire file touched by the Pull Request is scanned for errors. If True, only the diff is scanned. - -no_blank_comment: True # If True, no comment is made on PR without any errors. -descending_issues_order: False # If True, PEP 8 issues in message will be displayed in descending order of line numbers in the file -only_mention_files_with_errors: True # If False, a separate status comment for each file is made. - -message: - opened: - header: "Hello @{name}! Thanks for opening this PR. " - footer: "Do see the [Hitchhiker's guide to code style](https://goo.gl/hqbW4r)" - updated: - header: "Hello @{name}! Thanks for updating this PR. " - footer: "" # Why to comment the link to the style guide everytime? :) - no_errors: "There are currently no PEP 8 issues detected in this Pull Request. Cheers! :beers: " \ No newline at end of file diff --git a/.readthedocs.yml b/.readthedocs.yml deleted file mode 100644 index 6a06f655..00000000 --- a/.readthedocs.yml +++ /dev/null @@ -1,8 +0,0 @@ -# .readthedocs.yml - -build: - image: latest - -python: - version: 3.6 - setup_py_install: true \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index c7576f1c..00000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,98 +0,0 @@ -Contributing to DESlib -======================== - -You can contribute to the project in several ways: - -- Reporting bugs -- Requesting features -- Improving the documentation -- Adding examples to use the library -- Implementing new features and fixing bugs - -Reporting Bugs and requesting features: ---------------------------------------- - -We use Github issues to track all bugs and feature requests; feel free to -open an issue if you have found a bug or wish to see a new feature implemented. -Before opening a new issue, please check if the issue is not being currently addressed: -[Issues](https://github.com/Menelau/DESlib/issues) - -For reporting bugs: - -- Include information of your working environment. This information - can be found by running the following code snippet: - - ```python - import platform; print(platform.platform()) - import sys; print("Python", sys.version) - import numpy; print("NumPy", numpy.__version__) - import scipy; print("SciPy", scipy.__version__) - import sklearn; print("Scikit-Learn", sklearn.__version__) - ``` - -- Include a [reproducible](https://stackoverflow.com/help/mcve) code snippet - or link to a [gist](https://gist.github.com). If an exception is raised, - please provide the traceback. - -Documentation: --------------- - -We are glad to accept any sort of documentation: function docstrings, -reStructuredText documents (like this one), tutorials, etc. -reStructuredText documents live in the source code repository under the -doc/ directory. - -You can edit the documentation using any text editor and then generate -the HTML output by typing ``make html`` from the doc/ directory. -Alternatively, ``make`` can be used to quickly generate the -documentation without the example gallery. The resulting HTML files will -be placed in _build/html/ and are viewable in a web browser. See the -README file in the doc/ directory for more information. - -For building the documentation, you will need to install sphinx and sphinx_rtd_theme. This -can be easily done by installing the requirements for development using the following command: - -pip install -r requirements-dev.txt - -Contributing with code: ------------------------ - -The preferred way to contribute is to fork the main repository to your account: - -1. Fork the [project repository](https://github.com/Menelau/DESlib): - click on the 'Fork' button near the top of the page. This creates - a copy of the code under your account on the GitHub server. - -2. Clone this copy to your local disk: - - $ git clone git@github.com:YourLogin/DESlib.git - $ cd DESlib - -3. Install all requirements for development: - - $ pip install -r requirements-dev.txt - $ pip install --editable . - -4. Create a branch to hold your changes: - - $ git checkout -b branch_name - -Where ``branch_name`` is the new feature or bug to be fixed. Do not work directly on the ``master`` branch. - -5. Work on this copy on your computer using Git to do the version - control. To record your changes in Git, then push them to GitHub with: - - $ git push -u origin branch_name - -It is important to assert your code is well covered by test routines (coverage of at least 90%), well documented and -follows PEP8 guidelines. - -6. Create a 'Pull request' to send your changes for review. - - If your pull request addresses an issue, please use the title to describe - the issue and mention the issue number in the pull request description to - ensure a link is created to the original issue. - - - - diff --git a/LICENSE.txt b/LICENSE.txt deleted file mode 100644 index 8ddbf20d..00000000 --- a/LICENSE.txt +++ /dev/null @@ -1,12 +0,0 @@ -Copyright 2018 Rafael Menelau Oliveira e Cruz - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 67601f42..00000000 --- a/MANIFEST.in +++ /dev/null @@ -1,9 +0,0 @@ -include *.rst -include CONTRIBUTING.md -include LICENSE.txt -include requirements.txt -include requirements-dev.txt - -recursive-include examples *.py -recursive-include docs *.rst conf.py *.css Makefile make.bat - diff --git a/README.rst b/README.rst deleted file mode 100644 index d04043d3..00000000 --- a/README.rst +++ /dev/null @@ -1,242 +0,0 @@ -.. -*- mode: rst -*- - -.. _scikit-learn-contrib: https://github.com/scikit-learn-contrib - -|Docs|_ |CircleCI|_ |BSD|_ |PyPi|_ |PythonVersion|_ |Downloads|_ |Wheel|_ |Black|_ - -.. |Docs| image:: https://readthedocs.org/projects/deslib/badge/?version=latest -.. _Docs: http://deslib.readthedocs.io/en/latest/?badge=latest - -.. |CircleCI| image:: https://circleci.com/gh/scikit-learn-contrib/DESlib.svg?style=shield -.. _CircleCI: https://circleci.com/gh/scikit-learn-contrib/DESlib - -.. |BSD| image:: https://img.shields.io/badge/License-BSD%203--Clause-blue.svg -.. _BSD: https://opensource.org/licenses/BSD-3-Clause - -.. |PyPi| image:: https://badge.fury.io/py/DESlib.svg -.. _PyPi: https://pypi.org/project/DESlib/ - -.. |PythonVersion| image:: https://img.shields.io/pypi/pyversions/deslib.svg -.. _PythonVersion: https://pypi.org/project/DESlib/ - -.. |Downloads| image:: https://img.shields.io/pypi/dm/deslib.svg -.. _Downloads: https://pypistats.org/packages/deslib - -.. |Wheel| image:: https://img.shields.io/pypi/wheel/deslib.svg -.. _Wheel: https://img.shields.io/pypi/wheel/deslib.svg - -.. |Black| image:: https://img.shields.io/badge/code%20style-black-000000.svg -.. _Black: :target: https://github.com/psf/black - - -.. |PythonMinVersion| replace:: 3.7 -.. |NumPyMinVersion| replace:: 1.17.3 -.. |SciPyMinVersion| replace:: 1.5.0 -.. |ScikitLearnMinVersion| replace:: 1.0.2 - -DESlib -======== - -DESlib is an easy-to-use ensemble learning library focused on the implementation of the state-of-the-art techniques for dynamic classifier and ensemble selection. -The library is is based on scikit-learn_, using the same method signatures: **fit**, **predict**, **predict_proba** and **score**. -All dynamic selection techniques were implemented according to the definitions from [1]_. - -Dynamic Selection: -------------------- - -Dynamic Selection (DS) refers to techniques in which the base classifiers are selected -dynamically at test time, according to each new sample to be classified. Only the most competent, or an ensemble of the most competent classifiers is selected to predict -the label of a specific test sample. The rationale for these techniques is that not every classifier in -the pool is an expert in classifying all unknown samples, but rather each base classifier is an expert in -a different local region of the feature space. - -DS is one of the most promising MCS approaches (Multiple Classifier Systems) due to an increasing number of empirical studies -reporting superior performance over static combination methods. Such techniques -have achieved better classification performance especially when dealing with small-sized and imbalanced datasets. - -Installation: -------------- - -The package can be installed using pip: - -Stable version: - -.. code-block:: bash - - pip install deslib - -Latest version (under development): - -.. code-block:: bash - - pip install git+https://github.com/scikit-learn-contrib/DESlib - - -Dependencies: -------------- - -The dependency requirements are: - -- Python (>= |PythonMinVersion|) -- NumPy (>= |NumPyMinVersion|) -- SciPy (>= |SciPyMinVersion|) -- Scikit-learn (>= |ScikitLearnMinVersion|) - -These dependencies are automatically installed using the pip commands above. - -Examples: ---------- - -Here we show an example using the KNORA-E method with random forest as a pool of classifiers: - -.. code-block:: python - - from deslib.des.knora_e import KNORAE - - # Train a pool of 10 classifiers - pool_classifiers = RandomForestClassifier(n_estimators=10) - pool_classifiers.fit(X_train, y_train) - - # Initialize the DES model - knorae = KNORAE(pool_classifiers) - - # Preprocess the Dynamic Selection dataset (DSEL) - knorae.fit(X_dsel, y_dsel) - - # Predict new examples: - knorae.predict(X_test) - -The library accepts any list of classifiers (compatible with scikit-learn) as input, including a list containing different classifier models (heterogeneous ensembles). -More examples on how to use the API can be found in the documentation_ and in the Examples directory. - -Organization: -------------- - -The library is divided into four modules: - -1. deslib.des: Implementation of DES techniques (Dynamic Ensemble Selection). -2. deslib.dcs: Implementation of DCS techniques (Dynamic Classifier Selection). -3. deslib.static: Implementation of baseline ensemble methods. -4. deslib.util: A collection of aggregation functions and diversity measures for ensemble of classifiers. - -* DES techniques currently available are: - 1. META-DES [7]_ [8]_ [15]_ - 2. K-Nearest-Oracle-Eliminate (KNORA-E) [3]_ - 3. K-Nearest-Oracle-Union (KNORA-U) [3]_ - 4. Dynamic Ensemble Selection-Performance(DES-P) [12]_ - 5. K-Nearest-Output Profiles (KNOP) [9]_ - 6. Randomized Reference Classifier (DES-RRC) [10]_ - 7. DES Kullback-Leibler Divergence (DES-KL) [12]_ - 8. DES-Exponential [21]_ - 9. DES-Logarithmic [11]_ - 10. DES-Minimum Difference [21]_ - 11. DES-Clustering [16]_ - 12. DES-KNN [16]_ - 13. DES Multiclass Imbalance (DES-MI) [24]_ - -* DCS techniques currently available are: - 1. Modified Classifier Rank (Rank) [19]_ - 2. Overall Local Accuracy (OLA) [4]_ - 3. Local Class Accuracy (LCA) [4]_ - 4. Modified Local Accuracy (MLA) [23]_ - 5. Multiple Classifier Behaviour (MCB) [5]_ - 6. A Priori Selection (A Priori) [6]_ - 7. A Posteriori Selection (A Posteriori) [6]_ - -* Baseline methods: - 1. Oracle [20]_ - 2. Single Best [2]_ - 3. Static Selection [2]_ - 4. Stacked Classifier [25]_ - -Variations of each DES techniques are also provided by the library (e.g., different versions of the META-DES framework). - -The following techniques are also available for all methods: - * For DES techniques, the combination of the selected classifiers can be done as Dynamic Selection (majority voting), Dynamic Weighting (weighted majority voting) or a Hybrid (selection + weighting). - * For all DS techniques, Dynamic Frienemy Pruning (DFP) [13]_ can be used. - * For all DS techniques, Instance Hardness (IH) can be used to classify easy samples with a KNN and hard samples using the DS technique. More details on IH and Dynamic Selection can be found in [14]_. - -As an optional requirement, the fast KNN implementation from FAISS_ can be used to speed-up the computation of the region of competence on GPU. - -Citation ---------- - -If you use DESLib in a scientific paper, please consider citing the following paper: - -Rafael M. O. Cruz, Luiz G. Hafemann, Robert Sabourin and George D. C. Cavalcanti `DESlib: A Dynamic ensemble selection library in Python. `_ arXiv preprint arXiv:1802.04967 (2018). - -.. code-block:: text - - @article{JMLR:v21:18-144, - author = {Rafael M. O. Cruz and Luiz G. Hafemann and Robert Sabourin and George D. C. Cavalcanti}, - title = {DESlib: A Dynamic ensemble selection library in Python}, - journal = {Journal of Machine Learning Research}, - year = {2020}, - volume = {21}, - number = {8}, - pages = {1-5}, - url = {http://jmlr.org/papers/v21/18-144.html} - } - -References: ------------ - -.. [1] : R. M. O. Cruz, R. Sabourin, and G. D. Cavalcanti, “Dynamic classifier selection: Recent advances and perspectives,” Information Fusion, vol. 41, pp. 195 – 216, 2018. - -.. [2] : A. S. Britto, R. Sabourin, L. E. S. de Oliveira, Dynamic selection of classifiers - A comprehensive review, Pattern Recognition 47 (11) (2014) 3665–3680. - -.. [3] : A. H. R. Ko, R. Sabourin, u. S. Britto, Jr., From dynamic classifier selection to dynamic ensemble selection, Pattern Recognition 41 (2008) 1735–1748. - -.. [4] : K. Woods, W. P. Kegelmeyer, Jr., K. Bowyer, Combination of multiple classifiers using local accuracy estimates, IEEE Transactions on Pattern Analysis Machine Intelligence 19 (1997) 405–410. - -.. [5] : G. Giacinto, F. Roli, Dynamic classifier selection based on multiple classifier behaviour, Pattern Recognition 34 (2001) 1879–1881. - -.. [6] : L. Didaci, G. Giacinto, F. Roli, G. L. Marcialis, A study on the performances of dynamic classifier selection based on local accuracy estimation, Pattern Recognition 38 (11) (2005) 2188–2191. - -.. [7] : R. M. O. Cruz, R. Sabourin, G. D. C. Cavalcanti, T. I. Ren, META-DES: A dynamic ensemble selection framework using meta-learning, Pattern Recognition 48 (5) (2015) 1925–1935. - -.. [8] : Cruz, R.M., Sabourin, R. and Cavalcanti, G.D., 2015, July. META-DES. H: a dynamic ensemble selection technique using meta-learning and a dynamic weighting approach. In Neural Networks (IJCNN), 2015 International Joint Conference on (pp. 1-8) - -.. [9] : P. R. Cavalin, R. Sabourin, C. Y. Suen, Dynamic selection approaches for multiple classifier systems, Neural Computing and Applications 22 (3-4) (2013) 673–688. - -.. [10] : T.Woloszynski, M. Kurzynski, A probabilistic model of classifier competence for dynamic ensemble selection, Pattern Recognition 44 (2011) 2656–2668. - -.. [11] : T.Woloszynski, M. Kurzynski, A measure of competence based on randomized reference classifier for dynamic ensemble selection, in: International Conference on Pattern Recognition (ICPR), 2010, pp. 4194–4197. - -.. [12] : T. Woloszynski, M. Kurzynski, P. Podsiadlo, G. W. Stachowiak, A measure of competence based on random classification for dynamic ensemble selection, Information Fusion 13 (3) (2012) 207–213. - -.. [13] : Oliveira, D.V.R., Cavalcanti, G.D.C. and Sabourin, R., Online Pruning of Base Classifiers for Dynamic Ensemble Selection, Pattern Recognition, vol. 72, December 2017, pp 44-58. - -.. [14] : Cruz RM, Zakane HH, Sabourin R, Cavalcanti GD. Dynamic Ensemble Selection VS K-NN: why and when Dynamic Selection obtains higher classification performance?. - -.. [15] : R. M. O. Cruz, R. Sabourin, G. D. C. Cavalcanti, META-DES.Oracle: Meta-learning and feature selection for dynamic ensemble selection, Information Fusion 38 (2017) 84–103.Nov 30;38:84-103. - -.. [16] : R. G. F. Soares, A. Santana, A. M. P. Canuto, M. C. P. de Souto, Using accuracy and diversity to select classifiers to build ensembles, Proceedings of the International Joint Conference on Neural Networks (2006) 1310–1316. - -.. [17] : L. I. Kuncheva, Combining Pattern Classifiers: Methods and Algorithms, Wiley-Interscience, 2004. - -.. [18] : Shipp, Catherine A., and Ludmila I. Kuncheva. "Relationships between combination methods and measures of diversity in combining classifiers." Information fusion 3.2 (2002): 135-148. - -.. [19] : M. Sabourin, A. Mitiche, D. Thomas, G. Nagy, Classifier combination for handprinted digit recognition, International Conference on Document Analysis and Recognition (1993) 163–166. - -.. [20] : L. I. Kuncheva, A theoretical study on six classifier fusion strategies, IEEE Transactions on Pattern Analysis and Machine Intelligence 24 (2) (2002) 281–286. - -.. [21] : B. Antosik, M. Kurzynski, New measures of classifier competence – heuristics and application to the design of multiple classifier systems., in: Computer recognition systems 4., 2011, pp. 197–206. - -.. [22] : Smith, Michael R., Tony Martinez, and Christophe Giraud-Carrier. "An instance level analysis of data complexity." Machine learning 95.2 (2014), pp 225-256. - -.. [23] : P. C. Smits, Multiple classifier systems for supervised remote sensing image classification based on dynamic classifier selection, IEEE Transactions on Geoscience and Remote Sensing 40 (4) (2002) 801–813. - -.. [24] : García, S., Zhang, Z.L., Altalhi, A., Alshomrani, S. and Herrera, F., "Dynamic ensemble selection for multi-class imbalanced datasets." Information Sciences 445 (2018): 22-37. - -.. [25] : Wolpert, David H. "Stacked generalization." Neural networks 5, no. 2 (1992): 241-259. - -.. _scikit-learn: http://scikit-learn.org/stable/ - -.. _numpy: http://www.numpy.org/ - -.. _scipy: https://www.scipy.org/ - -.. _documentation: https://deslib.readthedocs.io - -.. _FAISS: https://github.com/facebookresearch/faiss diff --git a/benchmarks/bench_ds_performance_faiss.py b/benchmarks/bench_ds_performance_faiss.py deleted file mode 100644 index 4d8c076b..00000000 --- a/benchmarks/bench_ds_performance_faiss.py +++ /dev/null @@ -1,79 +0,0 @@ -import gzip -import os -import shutil -import threading -import time -import urllib.request - -import pandas as pd -from sklearn.model_selection import train_test_split - -from deslib.des.knora_e import KNORAE - - -def sk_KNORAE_knn(XTrain, YTrain, k, XTest, YTest): - start = time.clock() - knorae_sk = KNORAE(k=k, knn_classifier='knn') - knorae_sk.fit(XTrain, YTrain) - score = knorae_sk.score(XTest, YTest) - print("sklearn_knn_knorae run_time: {}".format(time.clock() - start)) - print("sklearn_knn_knorae score: {}".format(score)) - - -def faiss_KNORAE_knn(XTrain, YTrain, k, XTest, YTest): - start = time.clock() - knorae_sk = KNORAE(k=k, knn_classifier='faiss') - knorae_sk.fit(XTrain, YTrain) - score = knorae_sk.score(XTest, YTest) - print("faiss_knn_knorae run_time: {}".format(time.clock() - start)) - print("faiss_knn_knorae score: {}".format(score)) - - -if __name__ == "__main__": - url = "https://archive.ics.uci.edu/ml/machine-learning-databases/" \ - "00280/HIGGS.csv.gz" - if not os.path.exists("../../HIGGS.csv"): - print("Downloading HIGGS dataset from {}".format(url)) - if not os.path.exists("../../HIGGS.gz"): - filedata = urllib.request.urlopen(url) - data2write = filedata.read() - with open('../../HIGGS.gz', 'wb') as f: - f.write(data2write) - print("Finished downloading") - print("Extracting HIGGS.gz") - if not os.path.exists("../../HIGGS.csv"): - with gzip.open('../../HIGGS.gz', 'rb') as f: - with open('../../HIGGS.csv', 'wb') as csv_out: - shutil.copyfileobj(f, csv_out) - print("Extracted csv") - - df = pd.read_csv('../../HIGGS.csv', header=None) - data = df.values - X = data[:, 1:] - Y = data[:, 0] - - X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.33) - num_samples_list = [1000000] - num_of_k_list = [2, 5, 7, 10] - num_of_test_inputs = [100, 1000, 10000] - - for nsamples in num_samples_list: - for n_k in num_of_k_list: - for n_t in num_of_test_inputs: - print("running experiment: num_of_train_samples: {}, " - "num_of_k: {}, num_of_tests: {}".format(nsamples, n_k, - n_t)) - faiss_KNORAE_knn(X_train[:nsamples], Y_train[:nsamples], n_k, - X_test[:n_t], Y_test[:n_t]) - t = threading.Thread(target=sk_KNORAE_knn, args=( - X_train[:nsamples], Y_train[:nsamples], n_k, X_test[:n_t], - Y_test[:n_t])) - - t.start() - t.join(timeout=600) - if t.is_alive(): - print( - "sklearn_knn, num_of_train_samples: {}, num_of_k: {}, " - "num_of_tests: {}, run_time: timeout".format(nsamples, - n_k, - n_t)) diff --git a/benchmarks/bench_knn_backbone.py b/benchmarks/bench_knn_backbone.py deleted file mode 100644 index 768e0fab..00000000 --- a/benchmarks/bench_knn_backbone.py +++ /dev/null @@ -1,56 +0,0 @@ -import time -import matplotlib.pyplot as plt - -from sklearn.datasets import make_classification -from sklearn.model_selection import train_test_split -from sklearn.neighbors import KNeighborsClassifier - -from deslib.util.faiss_knn_wrapper import FaissKNNClassifier - -n_samples = [1000, 10000, 100000, 1000000, 10000000] -rng = 42 - -faiss_brute = FaissKNNClassifier(n_neighbors=7, - algorithm='brute') -faiss_voronoi = FaissKNNClassifier(n_neighbors=7, - algorithm='voronoi') -faiss_hierarchical = FaissKNNClassifier(n_neighbors=7, - algorithm='hierarchical') - -all_knns = [faiss_brute, faiss_voronoi, faiss_hierarchical] -names = ['faiss_brute', 'faiss_voronoi', 'faiss_hierarchical'] - -list_fitting_time = [] -list_search_time = [] - -for n in n_samples: - - print("Number of samples: {}" .format(n)) - X, y = make_classification(n_samples=n, - n_features=20, - random_state=rng) - X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=rng) - temp_fitting_time = [] - temp_search_time = [] - for name, knn in zip(names, all_knns): - start = time.clock() - knn.fit(X_train, y_train) - fitting_time = time.clock() - start - print("{} fitting time: {}" .format(name, fitting_time)) - - start = time.clock() - neighbors, dists = knn.kneighbors(X_test) - search_time = time.clock() - start - print("{} neighborhood search time: {}" .format(name, search_time)) - - temp_fitting_time.append(fitting_time) - temp_search_time.append(search_time) - - list_fitting_time.append(temp_fitting_time) - list_search_time.append(temp_search_time) - -plt.plot(n_samples, list_search_time) -plt.legend(names) -plt.xlabel("Number of samples") -plt.ylabel("K neighbors search time") -plt.savefig('knn_backbone_benchmark.png') diff --git a/benchmarks/bench_speed_faiss.py b/benchmarks/bench_speed_faiss.py deleted file mode 100644 index cd574f52..00000000 --- a/benchmarks/bench_speed_faiss.py +++ /dev/null @@ -1,103 +0,0 @@ -import gzip -import os -import shutil -import time -import urllib.request - -import numpy as np -import pandas as pd -from sklearn.ensemble import BaggingClassifier -from sklearn.model_selection import train_test_split - -from deslib.des.knora_e import KNORAE - - -def run_knorae(pool_classifiers, X_DSEL, y_DSEL, X_test, y_test, knn_type): - knorae = KNORAE(pool_classifiers=pool_classifiers, - knn_classifier=knn_type) - - knorae.fit(X_DSEL, y_DSEL) - - start = time.clock() - score = knorae.score(X_test, y_test) - end = time.clock() - start - - return score, end - - -def fetch_HIGGS(): - url = "https://archive.ics.uci.edu/ml/machine-learning-databases/" \ - "00280/HIGGS.csv.gz" - if not os.path.exists("../../HIGGS.csv"): - - print("Downloading HIGGS dataset from {}".format(url)) - - if not os.path.exists("../../HIGGS.gz"): - filedata = urllib.request.urlopen(url) - data2write = filedata.read() - - with open('../../HIGGS.gz', 'wb') as f: - f.write(data2write) - - print("Finished downloading") - print("Extracting HIGGS.gz") - - if not os.path.exists("../../HIGGS.csv"): - with gzip.open('../../HIGGS.gz', 'rb') as f: - with open('../../HIGGS.csv', 'wb') as csv_out: - shutil.copyfileobj(f, csv_out) - - print("Extracted csv") - print('Reading CSV file') - df = pd.read_csv('../../HIGGS.csv', header=None) - data = df.values - X = data[:, 1:] - y = data[:, 0] - - return X, y - - -if __name__ == "__main__": - rng = np.random.RandomState(123456) - - print('Preparing dataset') - X, y = fetch_HIGGS() - X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, - random_state=rng) - - X_DSEL, X_train, y_DSEL, y_train = train_test_split(X_train, y_train, - test_size=0.50, - random_state=rng) - pool_classifiers = BaggingClassifier(n_estimators=100, - random_state=rng, - n_jobs=-1) - - print('Fitting base classifiers...') - pool_classifiers.fit(X_train, y_train) - - n_samples = 1000000 - num_of_test_inputs = [100, 1000, 10000] - - for n_t in num_of_test_inputs: - print("running experiment: num_of_DSEL_samples: {}, " - "num_of_tests: {}".format(y_DSEL.size, n_t)) - - score_sklearn, time_sklearn = run_knorae(pool_classifiers, - X_DSEL[:n_samples], - y_DSEL[:n_samples], - X_test[:n_t], - y_test[:n_t], - knn_type='knn') - - print("sklearn_knorae score = {}, time = {}".format(score_sklearn, - time_sklearn)) - - score_faiss, time_faiss = run_knorae(pool_classifiers, - X_DSEL[:n_samples], - y_DSEL[:n_samples], - X_test[:n_t], - y_test[:n_t], - knn_type='faiss') - - print("faiss_knorae score = {}, time = {}".format(score_faiss, - time_faiss)) diff --git a/deslib/__init__.py b/deslib/__init__.py deleted file mode 100644 index 8d75c301..00000000 --- a/deslib/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -"""A Python library for Dynamic Ensemble Selection. - -``DESlib`` is a library containing the implementation of the state-of-the art -dynamic classifier and ensemble selection techniques. The library also provides -some static ensemble methods that are used as baseline comparison. - -Subpackages ------------ -des - The implementation of several DES techniques. - -dcs - The implementation of several DCS techniques. - -static - The implementation of baseline ensemble methods. - -util - A collection of aggregation functions and diversity measures for ensemble - of classifiers. -""" - -# list of all modules available in the library -__all__ = ['des', 'dcs', 'static', 'util', 'tests'] - -__version__ = '0.3.7' diff --git a/deslib/base.py b/deslib/base.py deleted file mode 100644 index c3f18414..00000000 --- a/deslib/base.py +++ /dev/null @@ -1,726 +0,0 @@ -# coding=utf-8 - -# Author: Rafael Menelau Oliveira e Cruz -# -# License: BSD 3 clause - - -import functools -import math -import warnings -from abc import abstractmethod, ABCMeta - -import numpy as np -from scipy.stats import mode -from sklearn.base import BaseEstimator, ClassifierMixin -from sklearn.ensemble import BaseEnsemble, BaggingClassifier -from sklearn.model_selection import train_test_split -from sklearn.neighbors import KNeighborsClassifier -from sklearn.preprocessing import LabelEncoder, normalize -from sklearn.utils.validation import (check_X_y, check_is_fitted, check_array, - check_random_state) - -from deslib.util import KNNE -from deslib.util import faiss_knn_wrapper -from deslib.util.dfp import frienemy_pruning_preprocessed -from deslib.util.instance_hardness import hardness_region_competence - - -class BaseDS(BaseEstimator, ClassifierMixin): - """Base class for a dynamic classifier selection (dcs) and - dynamic ensemble selection (des) methods. - - All DCS and DES techniques should inherit from this class. - - Warning: This class should not be used directly. - Use derived classes instead. - """ - __metaclass__ = ABCMeta - - @abstractmethod - def __init__(self, pool_classifiers=None, k=7, DFP=False, with_IH=False, - safe_k=None, IH_rate=0.30, needs_proba=False, - random_state=None, knn_classifier='knn', - knn_metric='minkowski', DSEL_perc=0.5, knne=False, n_jobs=-1, - voting=None): - - self.pool_classifiers = pool_classifiers - self.k = k - self.DFP = DFP - self.with_IH = with_IH - self.safe_k = safe_k - self.IH_rate = IH_rate - self.needs_proba = needs_proba - self.random_state = random_state - self.knn_classifier = knn_classifier - self.knn_metric = knn_metric - self.DSEL_perc = DSEL_perc - self.knne = knne - self.n_jobs = n_jobs - self.voting = voting - - # Check optional dependency - if knn_classifier == 'faiss' and not faiss_knn_wrapper.is_available(): - raise ImportError( - 'Using knn_classifier="faiss" requires that the FAISS library ' - 'be installed.Please check the Installation Guide.') - - def fit(self, X, y): - """Prepare the DS model by setting the KNN algorithm and - pre-processing the information required to apply the DS - methods - - Parameters - ---------- - X : array of shape (n_samples, n_features) - The input data. - - y : array of shape (n_samples) - class labels of each example in X. - - Returns - ------- - self - """ - self.random_state_ = check_random_state(self.random_state) - X, y = self._validate_data( - X, - y, - accept_sparse="csr", - dtype=np.float64, - order="C", - accept_large_sparse=False, - ) - - # Check if the pool of classifiers is None. - # If yes, use a BaggingClassifier for the pool. - if self.pool_classifiers is None: - X_dsel, y_dsel = self._fit_pool_classifiers(X, y) - else: - self._check_base_classifier_fitted() - self.pool_classifiers_ = self.pool_classifiers - X_dsel = X - y_dsel = y - - self.n_classifiers_ = len(self.pool_classifiers_) - # allow base models with feature subspaces. - if hasattr(self.pool_classifiers_, "estimators_features_"): - self.estimator_features_ = \ - np.array(self.pool_classifiers_.estimators_features_) - else: - indices = np.arange(X.shape[1]) - self.estimator_features_ = np.tile(indices, - (self.n_classifiers_, 1)) - - # check if the input parameters are correct. - self._setup_label_encoder(y) - y_dsel = self.enc_.transform(y_dsel) - self._set_dsel(X_dsel, y_dsel) - self._set_region_of_competence_algorithm(X_dsel) - self._validate_parameters() - - self.roc_algorithm_.fit(X_dsel, y_dsel) - self.BKS_DSEL_ = self._predict_base(self.DSEL_data_) - self.DSEL_processed_ = self.BKS_DSEL_ == y_dsel[:, np.newaxis] - - return self - - def get_competence_region(self, query, k=None): - """Compute the region of competence of the query sample - using the data belonging to DSEL. - - Parameters - ---------- - query : array of shape (n_samples, n_features) - The test examples. - - k : int (Default = self.k) - The number of neighbors used to in the region of competence. - - Returns - ------- - dists : array of shape (n_samples, k) - The distances between the query and each sample in the region - of competence. The vector is ordered in an ascending fashion. - - idx : array of shape (n_samples, k) - Indices of the instances belonging to the region of competence of - the given query sample. - """ - if k is None: - k = self.k_ - - dists, idx = self.roc_algorithm_.kneighbors(query, - n_neighbors=k, - return_distance=True) - - return np.atleast_2d(dists), np.atleast_2d(idx) - - @abstractmethod - def estimate_competence(self, competence_region, distances=None, - predictions=None): - """estimate the competence of each base classifier :math:`c_{i}` - the classification of the query sample :math:`\\mathbf{x}`. - Returns an array containing the level of competence estimated - for each base classifier. The size of the vector is equals to - the size of the generated_pool of classifiers. - - Parameters - ---------- - competence_region : array of shape (n_samples, n_neighbors) - Indices of the k nearest neighbors according for each - test sample. - - distances : array of shape (n_samples, n_neighbors) - Distances of the k nearest neighbors according for each - test sample. - - predictions : array of shape (n_samples, n_classifiers) - Predictions of the base classifiers for all test examples - Returns - ------- - competences : array (n_classifiers) containing the competence level - estimated for each base classifier - """ - pass - - @abstractmethod - def select(self, competences): - """Select the most competent classifier for - the classification of the query sample x. - The most competent classifier (dcs) or an ensemble - with the most competent classifiers (des) is returned - - Parameters - ---------- - competences : array of shape (n_samples, n_classifiers) - The estimated competence level of each base classifier - for test example - - Returns - ------- - selected_classifiers : array containing the selected base classifiers - for each test sample - - """ - pass - - @abstractmethod - def classify_with_ds(self, predictions, probabilities=None, - neighbors=None, distances=None, DFP_mask=None): - """Predicts the label of the corresponding query sample. - Returns the predicted label. - - Parameters - ---------- - predictions : array of shape (n_samples, n_classifiers) - Predictions of the base classifiers for all test examples - - probabilities : array of shape (n_samples, n_classifiers, n_classes) - Probabilities estimates of each base classifier for all test - examples (For methods that always require probabilities from the - base classifiers) - - neighbors : array of shape (n_samples, n_neighbors) - Indices of the k nearest neighbors. - distances : array of shape (n_samples, n_neighbors) - Distances from the k nearest neighbors to the query - - DFP_mask : array of shape (n_samples, n_classifiers) - Mask containing 1 for the selected base classifier and 0 otherwise. - - Returns - ------- - predicted_label : array of shape (n_samples) - The predicted label for each query - """ - pass - - @abstractmethod - def predict_proba_with_ds(self, predictions, probabilities, - neighbors=None, distances=None, DFP_mask=None): - """Predicts the posterior probabilities of the corresponding - query sample. Returns the probability estimates of each class. - - Parameters - ---------- - predictions : array of shape (n_samples, n_classifiers) - Predictions of the base classifiers for all test examples - - probabilities : array of shape (n_samples, n_classifiers, n_classes) - The predictions of each base classifier for all samples (For - methods that always require probabilities from the base - classifiers). - - neighbors : array of shape (n_samples, n_neighbors) - Indices of the k nearest neighbors. - distances : array of shape (n_samples, n_neighbors) - Distances from the k nearest neighbors to the query - - DFP_mask : array of shape (n_samples, n_classifiers) - Mask containing 1 for the selected base classifier and 0 otherwise. - - Returns - ------- - predicted_proba: array of shape (n_samples, n_classes) - Posterior probabilities estimates for each test example. - """ - pass - - def predict(self, X): - """Predict the class label for each sample in X. - - Parameters - ---------- - X : array of shape (n_samples, n_features) - The input data. - - Returns - ------- - predicted_labels : array of shape (n_samples) - Predicted class label for each sample in X. - """ - X = self._check_predict(X) - preds = np.empty(X.shape[0], dtype=np.intp) - need_proba = self.needs_proba or self.voting == 'soft' - - base_preds, base_probas = self._preprocess_predictions(X, need_proba) - # predict all agree - ind_disagreement, ind_all_agree = self._split_agreement(base_preds) - if ind_all_agree.size: - preds[ind_all_agree] = base_preds[ind_all_agree, 0] - # predict with IH - if ind_disagreement.size: - distances, ind_ds_classifier, neighbors = self._IH_prediction( - X, ind_disagreement, preds, is_proba=False - ) - # Predict with DS - Check if there are still samples to be labeled. - if ind_ds_classifier.size: - DFP_mask = self._get_DFP_mask(neighbors) - inds, sel_preds, sel_probas = self._prepare_indices_DS( - base_preds, base_probas, ind_disagreement, - ind_ds_classifier) - preds_ds = self.classify_with_ds(sel_preds, sel_probas, - neighbors, distances, - DFP_mask) - preds[inds] = preds_ds - - return self.classes_.take(preds) - - def _check_predict(self, X): - check_is_fitted(self, - ["DSEL_processed_", "DSEL_data_", "DSEL_target_"]) - X = check_array(X) - if self.n_features_ != X.shape[1]: - raise ValueError("Number of features of the model must " - "match the input. Model n_features is {0} and " - "input n_features is {1}." - "".format(self.n_features_, X.shape[1])) - return X - - def predict_proba(self, X): - """Estimates the posterior probabilities for sample in X. - - Parameters - ---------- - X : array of shape (n_samples, n_features) - The input data. - - Returns - ------- - predicted_proba : array of shape (n_samples, n_classes) - Probabilities estimates for each sample in X. - """ - X = self._check_predict(X) - - self._check_predict_proba() - probas = np.zeros((X.shape[0], self.n_classes_)) - base_preds, base_probas = self._preprocess_predictions(X, True) - # predict all agree - ind_disagreement, ind_all_agree = self._split_agreement(base_preds) - if ind_all_agree.size: - probas[ind_all_agree] = base_probas[ind_all_agree].mean(axis=1) - # predict with IH - if ind_disagreement.size: - distances, ind_ds_classifier, neighbors = self._IH_prediction( - X, ind_disagreement, probas, is_proba=True) - # Predict with DS - Check if there are still samples to be labeled. - if ind_ds_classifier.size: - DFP_mask = self._get_DFP_mask(neighbors) - inds, sel_preds, sel_probas = self._prepare_indices_DS( - base_preds, base_probas, ind_disagreement, - ind_ds_classifier) - probas_ds = self.predict_proba_with_ds(sel_preds, - sel_probas, - neighbors, distances, - DFP_mask) - probas[inds] = probas_ds - return probas - - def _preprocess_predictions(self, X, req_proba): - if req_proba: - base_probabilities = self._predict_proba_base(X) - base_predictions = base_probabilities.argmax(axis=2) - else: - base_probabilities = None - base_predictions = self._predict_base(X) - return base_predictions, base_probabilities - - def _split_agreement(self, base_predictions): - all_agree_vector = BaseDS._all_classifier_agree(base_predictions) - ind_all_agree = np.where(all_agree_vector)[0] - ind_disagreement = np.where(~all_agree_vector)[0] - return ind_disagreement, ind_all_agree - - def _IH_prediction(self, X, ind_disagree, predicted_proba, is_proba=False): - X_DS = X[ind_disagree, :] - distances, region_competence = self.get_competence_region(X_DS) - if self.with_IH: - ind_hard, ind_easy = self._split_easy_samples(region_competence) - distances, region_competence = self._predict_easy_samples( - X_DS, distances, ind_disagree, ind_easy, - region_competence, predicted_proba, is_proba) - else: - # IH was not considered. So all samples go to predict with DS - ind_hard = np.arange(ind_disagree.size) - return distances, ind_hard, region_competence - - def _split_easy_samples(self, neighbors): - hardness = hardness_region_competence(neighbors, - self.DSEL_target_, - self.safe_k) - # Get the index associated with the easy and hard samples. - # easy samples are classified by the knn. - easy_samples_mask = hardness < self.IH_rate - ind_knn_classifier = np.where(easy_samples_mask)[0] - ind_ds_classifier = np.where(~easy_samples_mask)[0] - return ind_ds_classifier, ind_knn_classifier - - def _predict_easy_samples(self, X_DS, distances, ind_disagreement, - ind_easy, neighbors, predictions, is_proba): - if ind_easy.size: - # Accessing which samples in the original array. - ind_knn_original_matrix = ind_disagreement[ind_easy] - - if is_proba: - predictions[ind_knn_original_matrix] = \ - self.roc_algorithm_.predict_proba( - X_DS[ind_easy]) - else: - y_neighbors = self.DSEL_target_[neighbors[ind_easy, - :self.safe_k]] - predictions_knn, _ = mode(y_neighbors, axis=1) - predictions[ind_knn_original_matrix] = predictions_knn.reshape( - -1, ) - neighbors = np.delete(neighbors, ind_easy, axis=0) - distances = np.delete(distances, ind_easy, axis=0) - return distances, neighbors - - def _prepare_indices_DS(self, base_predictions, base_probabilities, - ind_disagreement, ind_ds_classifier): - # Get the real indices_ of the samples that will be classified - # using a DS algorithm. - ind_ds_original_matrix = ind_disagreement[ind_ds_classifier] - if base_probabilities is not None: - selected_probas = base_probabilities[ - ind_ds_original_matrix] - else: - selected_probas = None - selected_preds = base_predictions[ind_ds_original_matrix] - return ind_ds_original_matrix, selected_preds, selected_probas - - def _get_DFP_mask(self, neighbors): - if self.DFP: - DFP_mask = frienemy_pruning_preprocessed(neighbors, - self.DSEL_target_, - self.DSEL_processed_) - else: - DFP_mask = np.ones((neighbors.shape[0], self.n_classifiers_)) - return DFP_mask - - def _fit_pool_classifiers(self, X, y): - if len(X) < 2: - raise ValueError('More than one sample is needed ' - 'if the pool of classifiers is not informed.') - # Split the dataset into training (for the base classifier) and - # DSEL (for DS) - X_train, X_dsel, y_train, y_dsel = train_test_split( - X, y, test_size=self.DSEL_perc, - random_state=self.random_state_) - self.pool_classifiers_ = BaggingClassifier( - random_state=self.random_state_, n_jobs=self.n_jobs) - self.pool_classifiers_.fit(X_train, y_train) - return X_dsel, y_dsel - - def _check_label_encoder(self): - # Check if base classifiers are not using LabelEncoder (the case for - # scikit-learn's ensembles): - if isinstance(self.pool_classifiers_, BaseEnsemble): - if np.array_equal(self.pool_classifiers_.classes_, - self.pool_classifiers_[0].classes_): - self.base_already_encoded_ = False - else: - self.base_already_encoded_ = True - else: - self.base_already_encoded_ = False - - def _compute_highest_possible_IH(self): - highest_IH = (self.safe_k - math.ceil( - self.safe_k / self.n_classes_)) / self.safe_k - return highest_IH - - def _validate_ih(self): - highest_IH = self._compute_highest_possible_IH() - if self.IH_rate > highest_IH: - warnings.warn("IH_rate is bigger than the highest possible IH.", - category=RuntimeWarning) - - def _validate_k(self): - # validate safe_k - if self.k is None: - self.k_ = self.n_samples_ - elif self.k > self.n_samples_: - msg = "k is bigger than DSEL size. Using All DSEL examples " \ - "for competence estimation." - warnings.warn(msg, category=RuntimeWarning) - self.k_ = self.n_samples_ - 1 - else: - self.k_ = self.k - - # Validate safe_k - if self.with_IH and self.safe_k is None: - self.safe_k = self.k - - def _setup_label_encoder(self, y): - self._check_label_encoder() - self.enc_ = LabelEncoder() - self.enc_.fit(y) - self.classes_ = self.enc_.classes_ - - def _encode_base_labels(self, y): - if self.base_already_encoded_: - return y - else: - return self.enc_.transform(y) - - def _set_dsel(self, X, y): - """Pre-Process the input X and y data into the dynamic selection - dataset(DSEL) and get information about the structure of the data - (e.g., n_classes, n_samples, classes) - - Parameters - ---------- - X : array of shape (n_samples, n_features) - The Input data. - - y : array of shape (n_samples) - class labels of each sample in X. - """ - self.DSEL_data_ = X - self.DSEL_target_ = y - self.n_classes_ = self.classes_.size - self.n_features_ = X.shape[1] - self.n_samples_ = self.DSEL_target_.size - - def _set_region_of_competence_algorithm(self, X): - - algorithm = "auto" - metric_params = None - - if self.knn_metric == 'minkowski': - metric = 'minkowski' - elif self.knn_metric == 'mahalanobis': - metric = 'mahalanobis' - metric_params = {'VI': np.cov(X)} - algorithm = "auto" - else: - raise ValueError('"knn_metric" should be one of the following ' - '["minkowski", "mahalanobis"]') - - if self.knn_classifier is None or self.knn_classifier in ['knn', - 'sklearn']: - knn_class = functools.partial(KNeighborsClassifier, - n_jobs=self.n_jobs, - algorithm=algorithm, - metric=metric, - metric_params=metric_params) - elif self.knn_classifier == 'faiss': - knn_class = functools.partial( - faiss_knn_wrapper.FaissKNNClassifier, - n_jobs=self.n_jobs, algorithm="brute") - elif callable(self.knn_classifier): - knn_class = self.knn_classifier - else: - raise ValueError('"knn_classifier" should be one of the following ' - '["knn", "faiss", None] or an estimator class.') - - if self.knne: - self.knn_class_ = functools.partial( - KNNE, - knn_classifier=knn_class, - n_jobs=self.n_jobs, - algorithm="auto") - else: - self.knn_class_ = knn_class - - self.roc_algorithm_ = self.knn_class_(n_neighbors=self.k) - - def _preprocess_dsel(self): - """Compute the prediction of each base classifier for - all samples in DSEL. Used to speed-up the test phase, by - not requiring to re-classify training samples during test. - - Returns - ------- - DSEL_processed_ : array of shape (n_samples, n_classifiers). - Each element indicates whether the base classifier - predicted the correct label for the corresponding - sample (True), otherwise (False). - - BKS_DSEL_ : array of shape (n_samples, n_classifiers) - Predicted labels of each base classifier for all samples - in DSEL. - """ - BKS_dsel = self._predict_base(self.DSEL_data_) - processed_dsel = BKS_dsel == self.DSEL_target_[:, np.newaxis] - return processed_dsel, BKS_dsel - - def _predict_base(self, X): - """ Get the predictions of each base classifier in the pool for all - samples in X. - - Parameters - ---------- - X : array of shape (n_samples, n_features) - The test examples. - - Returns - ------- - predictions : array of shape (n_samples, n_classifiers) - The predictions of each base classifier for all samples - in X. - """ - predictions = np.zeros((X.shape[0], self.n_classifiers_), - dtype=np.intp) - - for index, clf in enumerate(self.pool_classifiers_): - labels = clf.predict(X[:, self.estimator_features_[index]]) - predictions[:, index] = self._encode_base_labels(labels) - return predictions - - def _predict_proba_base(self, X): - """ Get the predictions (probabilities) of each base classifier in the - pool for all samples in X. - - Parameters - ---------- - X : array of shape (n_samples, n_features) - The test examples. - - Returns - ------- - probabilities : array of shape (n_samples, n_classifiers, n_classes) - Probabilities estimates of each base classifier for all - test samples. - """ - probas = np.zeros( - (X.shape[0], self.n_classifiers_, self.n_classes_)) - - for index, clf in enumerate(self.pool_classifiers_): - probas[:, index] = clf.predict_proba( - X[:, self.estimator_features_[index]]) - return probas - - @staticmethod - def _all_classifier_agree(predictions): - """Check whether there is a difference in opinion among the classifiers - in the generated_pool. - - Parameters - ---------- - predictions : array of shape (n_samples, n_classifiers) - Predictions of the base classifiers for the test examples - - Returns - ------- - array of shape (classes) - containing True if all classifiers in the generated_pool agrees - on the same label, otherwise False. - """ - return np.all(predictions == predictions[:, 0].reshape(-1, 1), axis=1) - - def _validate_parameters(self): - """Verify if the input parameters are correct (generated_pool and k) - raises an error if k < 1 or generated_pool is not fitted. - """ - if self.k is not None: - if not isinstance(self.k, int): - raise TypeError("parameter k should be an integer") - if self.k <= 1: - raise ValueError("parameter k must be higher than 1." - "input k is {} ".format(self.k)) - - if self.safe_k is not None: - if not isinstance(self.safe_k, int): - raise TypeError("parameter safe_k should be an integer") - if self.safe_k <= 1: - raise ValueError("parameter safe_k must be higher than 1." - "input safe_k is {} ".format(self.safe_k)) - - # safe_k should be equals or lower the neighborhood size k. - if self.safe_k is not None and self.k is not None: - if self.safe_k > self.k: - raise ValueError( - "parameter safe_k must be equal or less than parameter k." - "input safe_k is {} and k is {}".format(self.k, - self.safe_k)) - if not isinstance(self.IH_rate, float): - raise TypeError( - "parameter IH_rate should be a float between [0.0, 0.5]") - if self.IH_rate < 0 or self.IH_rate > 0.5: - raise ValueError("Parameter IH_rate should be between [0.0, 0.5]." - "IH_rate = {}".format(self.IH_rate)) - - self._validate_pool_classifiers() - # validate the value of k - self._validate_k() - # validate the IH - if self.with_IH: - self._validate_ih() - - def _validate_pool_classifiers(self): - """ Check the estimator and the n_estimator attribute, set the - `base_estimator_` attribute. - - Raises - ------- - ValueError - If the pool of classifiers is empty. - """ - if self.n_classifiers_ <= 1: - raise ValueError("n_classifiers must be greater than one, " - "got {}.".format(self.n_classifiers_)) - - def _check_predict_proba(self): - """ Checks if each base classifier in the pool implements the - predict_proba method. - - Raises - ------- - ValueError - If the base classifiers do not implements the predict_proba method. - """ - for clf in self.pool_classifiers_: - if "predict_proba" not in dir(clf): - raise ValueError( - "All base classifiers should output probability estimates") - - def _check_base_classifier_fitted(self): - """ Checks if each base classifier in the pool is fitted. - - Raises - ------- - NotFittedError: If any of the base classifiers is not yet fitted. - """ - for clf in self.pool_classifiers: - check_is_fitted(clf, "classes_") diff --git a/deslib/dcs/__init__.py b/deslib/dcs/__init__.py deleted file mode 100644 index 71a6e19a..00000000 --- a/deslib/dcs/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -""" -The :mod:`deslib.dcs` provides a set of key dynamic classifier selection -algorithms (DCS). -""" - -from .a_posteriori import APosteriori -from .a_priori import APriori -from .base import BaseDCS -from .lca import LCA -from .mcb import MCB -from .mla import MLA -from .ola import OLA -from .rank import Rank - -__all__ = ['BaseDCS', - 'APosteriori', - 'APriori', - 'LCA', - 'OLA', - 'MLA', - 'MCB', - 'Rank'] diff --git a/deslib/dcs/a_posteriori.py b/deslib/dcs/a_posteriori.py deleted file mode 100644 index 4d1b5434..00000000 --- a/deslib/dcs/a_posteriori.py +++ /dev/null @@ -1,250 +0,0 @@ -# coding=utf-8 - -# Author: Rafael Menelau Oliveira e Cruz -# -# License: BSD 3 clause - -import numpy as np - -from deslib.dcs.base import BaseDCS - - -class APosteriori(BaseDCS): - """A Posteriori Dynamic classifier selection. - - The A Posteriori method uses the probability of correct classification of a - given base classifier :math:`c_{i}` for each neighbor :math:`x_{k}` with - respect to a single class. Consider a classifier :math:`c_{i}` that assigns - a test sample to class :math:`w_{l}`. Then, only the samples belonging to - class :math:`w_{l}` are taken into account during the competence level - estimates. Base classifiers with a higher probability of correct - classification have a higher competence level. Moreover, the method also - weights the influence of each neighbor :math:`x_{k}` according to its - Euclidean distance to the query sample. The closest neighbors have a higher - influence on the competence level estimate. In cases where no sample in the - region of competence belongs to the predicted class, :math:`w_{l}`, the - competence level estimate of the base classifier is equal to zero. - - A single classifier is selected only if its competence level is - significantly higher than that of the other base classifiers in the pool - (higher than a pre-defined threshold). Otherwise, all classifiers in the - pool are combined using the majority voting rule. The selection methodology - can be modified by modifying the hyper-parameter selection_method. - - Parameters - ---------- - pool_classifiers : list of classifiers (Default = None) - The generated_pool of classifiers trained for the corresponding - classification problem. Each base classifiers should support the method - "predict" and "predict_proba". If None, then the pool of classifiers is - a bagging classifier. - - k : int (Default = 7) - Number of neighbors used to estimate the competence of the base - classifiers. - - DFP : Boolean (Default = False) - Determines if the dynamic frienemy pruning is applied. - - with_IH : Boolean (Default = False) - Whether the hardness level of the region of competence is used to - decide between using the DS algorithm or the KNN for classification of - a given query sample. - - safe_k : int (default = None) - The size of the indecision region. - - IH_rate : float (default = 0.3) - Hardness threshold. If the hardness level of the competence region is - lower than the IH_rate the KNN classifier is used. Otherwise, the DS - algorithm is used for classification. - - selection_method : String (Default = "best") - Determines which method is used to select the base classifier after - the competences are estimated. - - diff_thresh : float (Default = 0.1) - Threshold to measure the difference between the competence level of the - base classifiers for the random and diff selection schemes. If the - difference is lower than the threshold, their performance are - considered equivalent. - - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. - - knn_classifier : {'knn', 'faiss', None} (Default = 'knn') - The algorithm used to estimate the region of competence: - - - 'knn' will use :class:`KNeighborsClassifier` from sklearn - :class:`KNNE` available on `deslib.utils.knne` - - - 'faiss' will use Facebook's Faiss similarity search through the - class :class:`FaissKNNClassifier` - - - None, will use sklearn :class:`KNeighborsClassifier`. - - knn_metric : {'minkowski', 'cosine', 'mahalanobis'} (Default = 'minkowski') - The metric used by the k-NN classifier to estimate distances. - - - 'minkowski' will use minkowski distance. - - - 'cosine' will use the cosine distance. - - - 'mahalanobis' will use the mahalonibis distance. - - knne : bool (Default=False) - Whether to use K-Nearest Neighbor Equality (KNNE) for the region - of competence estimation. - - DSEL_perc : float (Default = 0.5) - Percentage of the input data used to fit DSEL. - Note: This parameter is only used if the pool of classifier is None or - unfitted. - - n_jobs : int, default=-1 - The number of parallel jobs to run. None means 1 unless in - a joblib.parallel_backend context. -1 means using all processors. - Doesn’t affect fit method. - - References - ---------- - G. Giacinto and F. Roli, Methods for Dynamic Classifier Selection - 10th Int. Conf. on Image Anal. and Proc., Venice, Italy (1999), 659-664. - - Ko, Albert HR, Robert Sabourin, and Alceu Souza Britto Jr. "From dynamic - classifier selection to dynamic ensemble selection." - Pattern Recognition 41.5 (2008): 1718-1731. - - Britto, Alceu S., Robert Sabourin, and Luiz ES Oliveira. "Dynamic selection - of classifiers—a comprehensive review." - Pattern Recognition 47.11 (2014): 3665-3680. - - R. M. O. Cruz, R. Sabourin, and G. D. Cavalcanti, “Dynamic classifier - selection: Recent advances and perspectives,” - Information Fusion, vol. 41, pp. 195 – 216, 2018. - - """ - - def __init__(self, pool_classifiers=None, k=7, DFP=False, with_IH=False, - safe_k=None, IH_rate=0.30, selection_method='diff', - diff_thresh=0.1, random_state=None, knn_classifier='knn', - knn_metric='minkowski', knne=False, DSEL_perc=0.5, n_jobs=-1): - super(APosteriori, self).__init__(pool_classifiers=pool_classifiers, - k=k, DFP=DFP, with_IH=with_IH, - safe_k=safe_k, IH_rate=IH_rate, - selection_method=selection_method, - diff_thresh=diff_thresh, - knn_classifier=knn_classifier, - knn_metric=knn_metric, - random_state=random_state, - knne=knne, - DSEL_perc=DSEL_perc, n_jobs=n_jobs) - - def fit(self, X, y): - """Prepare the DS model by setting the KNN algorithm and - pre-processing the information required to apply the DS - method. - - Parameters - ---------- - X : array of shape (n_samples, n_features) - Data used to fit the model. - - y : array of shape (n_samples) - class labels of each example in X. - - Returns - ------- - self - """ - super(APosteriori, self).fit(X, y) - self._check_predict_proba() - - self.dsel_scores_ = self._predict_proba_base(self.DSEL_data_) - return self - - def estimate_competence(self, competence_region, distances, - predictions=None): - """Estimate the competence of each base classifier :math:`c_{i}` for - the classification of the query sample using the A Posteriori method. - - The competence level is estimated based on the probability of correct - classification of the base classifier :math:`c_{i}`, for each neighbor - :math:`x_{k}` belonging to a specific class :math:`w_{l}`. - In this case, :math:`w_{l}` is the class predicted by the base - classifier :math:`c_{i}`, for the query sample. This method also - weights the influence of each training sample according to its - Euclidean distance to the query instance. The closest samples have a - higher influence in the computation of the competence level. The - competence level estimate is represented by the following equation: - - .. math:: \\delta_{i,j} = \\frac{\\sum_{\\mathbf{x}_{k} \\in - \\omega_{l}}P(\\omega_{l} \\mid \\mathbf{x}_{k}, c_{i} )W_{k}} - {\\sum_{k = 1}^{K}P(\\omega_{l} \\mid \\mathbf{x}_{k}, c_{i} )W_{k}} - - where :math:`\\delta_{i,j}` represents the competence level of - :math:`c_{i}` for the classification of query. - - Parameters - ---------- - competence_region : array of shape (n_samples, n_neighbors) - Indices of the k nearest neighbors. - - distances : array of shape (n_samples, n_neighbors) - Distances from the k nearest neighbors to the query. - - predictions : array of shape (n_samples, n_classifiers) - Predictions of the base classifiers for the test examples. - - Returns - ------- - competences : array of shape (n_samples, n_classifiers) - Competence level estimated for each base classifier and test - example. - """ - # Guarantee that these arrays are view as a 2D array for the case where - # a single test sample is passed down. - predictions = np.atleast_2d(predictions) - distances[distances == 0] = 1e-10 - - # Normalize the distances - dists_normalized = 1.0 / distances - - # Expanding the dimensions of the predictions and target arrays in - # order to compare both. - predictions_3d = np.expand_dims(predictions, axis=1) - target_3d = self.DSEL_target_[competence_region, np.newaxis] - - # Create a mask to remove the neighbors belonging to a different class - # than the predicted by the base classifier - mask = (predictions_3d != target_3d) - - # Broadcast the distance array to the same shape as the pre-processed - # information for future calculations - dists_normalized = np.repeat(np.expand_dims(dists_normalized, axis=2), - self.n_classifiers_, axis=2) - - # Multiply the pre-processed correct predictions by the base - # classifiers to the distance array - scores_target = self.dsel_scores_[competence_region, :, - self.DSEL_target_[competence_region]] - scores_target_norm = scores_target * dists_normalized - - # Create masked arrays to remove samples with different label in the - # calculations - masked_preprocessed = np.ma.MaskedArray(scores_target_norm, mask=mask) - masked_dist = np.ma.MaskedArray(dists_normalized, mask=mask) - - # Consider only the neighbor samples where the predicted label is - # equals to the neighbor label - competences_masked = np.ma.sum(masked_preprocessed, - axis=1) / np.ma.sum(masked_dist, axis=1) - - # Fill 0 to the masked values in the resulting array (when no neighbors - # belongs to the class predicted by the corresponding base classifier) - competences = np.ma.filled(competences_masked, 0) - - return competences diff --git a/deslib/dcs/a_priori.py b/deslib/dcs/a_priori.py deleted file mode 100644 index 28792959..00000000 --- a/deslib/dcs/a_priori.py +++ /dev/null @@ -1,217 +0,0 @@ -# coding=utf-8 - -# Author: Rafael Menelau Oliveira e Cruz -# -# License: BSD 3 clause - -import numpy as np - -from deslib.dcs.base import BaseDCS - - -class APriori(BaseDCS): - """A Priori dynamic classifier selection. - - The A Priori method uses the probability of correct classification of a - given base classifier :math:`c_{i}` for each neighbor :math:`x_{k}` for the - competence level estimation. Base classifiers with a higher probability of - correct classification have a higher competence level. Moreover, the method - also weights the influence of each neighbor :math:`x_{k}` according to its - Euclidean distance to the query sample. The closest neighbors have a higher - influence on the competence level estimate. - - A single classifier is selected only if its competence level is - significantly higher than that of the other base classifiers in the pool - (higher than a pre-defined threshold). Otherwise, all classifiers i the - pool are combined using the majority voting rule. - - Parameters - ---------- - pool_classifiers : list of classifiers (Default = None) - The generated_pool of classifiers trained for the corresponding - classification problem. Each base classifiers should support the method - "predict" and "predict_proba". If None, then the pool of classifiers is - a bagging classifier. - - k : int (Default = 7) - Number of neighbors used to estimate the competence of the base - classifiers. - - DFP : Boolean (Default = False) - Determines if the dynamic frienemy pruning is applied. - - with_IH : Boolean (Default = False) - Whether the hardness level of the region of competence is used to - decide between using the DS algorithm or the KNN for classification of - a given query sample. - - safe_k : int (default = None) - The size of the indecision region. - - IH_rate : float (default = 0.3) - Hardness threshold. If the hardness level of the competence region is - lower than the IH_rate the KNN classifier is used. Otherwise, the DS - - selection_method : String (Default = "best") - Determines which method is used to select the base classifier after - the competences are estimated. - - diff_thresh : float (Default = 0.1) - Threshold to measure the difference between the competence level of the - base classifiers for the random and diff selection schemes. If the - difference is lower than the threshold, their performance are - considered equivalent. - - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. - - knn_classifier : {'knn', 'faiss', None} (Default = 'knn') - The algorithm used to estimate the region of competence: - - - 'knn' will use :class:`KNeighborsClassifier` from sklearn - :class:`KNNE` available on `deslib.utils.knne` - - - 'faiss' will use Facebook's Faiss similarity search through the - class :class:`FaissKNNClassifier` - - - None, will use sklearn :class:`KNeighborsClassifier`. - - knn_metric : {'minkowski', 'cosine', 'mahalanobis'} (Default = 'minkowski') - The metric used by the k-NN classifier to estimate distances. - - - 'minkowski' will use minkowski distance. - - - 'cosine' will use the cosine distance. - - - 'mahalanobis' will use the mahalonibis distance. - - knne : bool (Default=False) - Whether to use K-Nearest Neighbor Equality (KNNE) for the region - of competence estimation. - - DSEL_perc : float (Default = 0.5) - Percentage of the input data used to fit DSEL. - Note: This parameter is only used if the pool of classifier is None or - unfitted. - - n_jobs : int, default=-1 - The number of parallel jobs to run. None means 1 unless in - a joblib.parallel_backend context. -1 means using all processors. - Doesn’t affect fit method. - - References - ---------- - G. Giacinto and F. Roli, Methods for Dynamic Classifier Selection - 10th Int. Conf. on Image Anal. and Proc., Venice, Italy (1999), 659-664. - - Ko, Albert HR, Robert Sabourin, and Alceu Souza Britto Jr. "From dynamic - classifier selection to dynamic ensemble selection." - Pattern Recognition 41.5 (2008): 1718-1731. - - Britto, Alceu S., Robert Sabourin, and Luiz ES Oliveira. "Dynamic selection - of classifiers—a comprehensive review." - Pattern Recognition 47.11 (2014): 3665-3680. - - R. M. O. Cruz, R. Sabourin, and G. D. Cavalcanti, “Dynamic classifier - selection: Recent advances and perspectives,” - Information Fusion, vol. 41, pp. 195 – 216, 2018. - - """ - - def __init__(self, pool_classifiers=None, k=7, DFP=False, with_IH=False, - safe_k=None, IH_rate=0.30, selection_method='diff', - diff_thresh=0.1, random_state=None, knn_classifier='knn', - knn_metric='minkowski', knne=False, DSEL_perc=0.5, n_jobs=-1): - super(APriori, self).__init__(pool_classifiers=pool_classifiers, k=k, - DFP=DFP, with_IH=with_IH, safe_k=safe_k, - IH_rate=IH_rate, - selection_method=selection_method, - diff_thresh=diff_thresh, - random_state=random_state, - knn_classifier=knn_classifier, - knn_metric=knn_metric, - knne=knne, - DSEL_perc=DSEL_perc, - n_jobs=n_jobs) - - def fit(self, X, y): - """Prepare the DS model by setting the KNN algorithm and - pre-processing the information required to apply the DS - method. - - Parameters - ---------- - X : array of shape (n_samples, n_features) - Data used to fit the model. - - y : array of shape (n_samples) - class labels of each example in X. - - Returns - ------- - self - """ - super(APriori, self).fit(X, y) - self._check_predict_proba() - - self.dsel_scores_ = self._predict_proba_base(self.DSEL_data_) - return self - - def estimate_competence(self, competence_region, distances, - predictions=None): - """estimate the competence of each base classifier :math:`c_{i}` for - the classification of the query sample using the A Priori rule: - - The competence level is estimated based on the probability of correct - classification of the base classifier :math:`c_{i}`, considering all - samples in the region of competence. This method also weights the - influence of each training sample according to its Euclidean distance - to the query instance. The closest samples have a higher influence in - the computation of the competence level. The competence level estimate - is represented by the following equation: - - .. math:: \\delta_{i,j} = \\frac{\\sum_{k = 1}^{K}P(\\omega_{l} \\mid - \\mathbf{x}_{k} \\in \\omega_{l}, - c_{i} )W_{k}}{\\sum_{k = 1}^{K}W_{k}} - - where :math:`\\delta_{i,j}` represents the competence level of - :math:`c_{i}` for the classification of query. - - Parameters - ---------- - competence_region : array of shape (n_samples, n_neighbors) - Indices of the k nearest neighbors. - - distances : array of shape (n_samples, n_neighbors) - Distances from the k nearest neighbors to the query. - - predictions : array of shape (n_samples, n_classifiers) - Predictions of the base classifiers for the test examples. - - Returns - ------- - competences : array of shape (n_samples, n_classifiers) - Competence level estimated for each base classifier and test - example. - """ - distances[distances == 0] = 1e-10 - dists_normalized = 1.0 / distances - - # Get the ndarray containing the scores obtained for the correct class - # for each neighbor (and test sample) - scores_target_class = self.dsel_scores_[ - competence_region, :, self.DSEL_target_[competence_region]] - - # Multiply the scores obtained for the correct class to the distances - # of each corresponding neighbor - scores_target_class *= np.expand_dims(dists_normalized, axis=2) - - # Sum the scores obtained for each neighbor and divide by the sum of - # all distances - competences = np.sum(scores_target_class, axis=1) / np.sum( - dists_normalized, axis=1, keepdims=True) - - return competences diff --git a/deslib/dcs/base.py b/deslib/dcs/base.py deleted file mode 100644 index 2aec4d5f..00000000 --- a/deslib/dcs/base.py +++ /dev/null @@ -1,294 +0,0 @@ -from abc import ABCMeta -from copy import copy - -import numpy as np -from sklearn.utils.validation import check_random_state - -from deslib.base import BaseDS -from deslib.util.aggregation import majority_voting_rule - - -class BaseDCS(BaseDS): - """Base class for a Dynamic Classifier Selection (dcs) method. - All dynamic classifier selection classes should inherit from this class. - - Warning: This class should not be used directly, use derived classes - instead. - - """ - __metaclass__ = ABCMeta - - def __init__(self, pool_classifiers=None, k=7, DFP=False, safe_k=None, - with_IH=False, IH_rate=0.30, selection_method='best', - diff_thresh=0.1, random_state=None, knn_classifier='knn', - knn_metric='minkowski', DSEL_perc=0.5, - knne=False, n_jobs=-1): - - super(BaseDCS, self).__init__(pool_classifiers=pool_classifiers, k=k, - DFP=DFP, with_IH=with_IH, safe_k=safe_k, - IH_rate=IH_rate, - random_state=random_state, - knn_classifier=knn_classifier, - knn_metric=knn_metric, - DSEL_perc=DSEL_perc, - knne=knne, n_jobs=n_jobs, - voting='single') - - self.selection_method = selection_method - self.diff_thresh = diff_thresh - - def estimate_competence(self, competence_region, distances=None, - predictions=None): - """Estimate the competence of each base classifier for the - classification of the query sample. - - Parameters - ---------- - competence_region : array of shape (n_samples, n_neighbors) - Indices of the k nearest neighbors. - - distances : array of shape (n_samples, n_neighbors) - Distances from the k nearest neighbors to the query. - - predictions : array of shape (n_samples, n_classifiers) - Predictions of the base classifiers for the test examples. - - Returns - ------- - competences : array of shape (n_samples, n_classifiers) - Competence level estimated for each base classifier and test - example. - """ - pass - - def select(self, competences): - """Select the most competent classifier for the classification of the - query sample given the competence level estimates. Four selection - schemes are available. - - Best : The base classifier with the highest competence level is - selected. In cases where more than one base classifier achieves the - same competence level, the one with the lowest index is selected. This - method is the standard for the LCA, OLA, MLA techniques. - - Diff : Select the base classifier that is significantly better than the - others in the pool (when the difference between its competence level - and the competence level of the other base classifiers is higher than a - predefined threshold). If no base classifier is significantly better, - the base classifier is selected randomly among the member with - equivalent competence level. - - Random : Selects a random base classifier among all base classifiers - that achieved the same competence level. - - ALL : all base classifiers with the max competence level estimates are - selected (note that in this case the - DCS technique becomes a DES technique). - - Parameters - ---------- - competences : array of shape (n_samples, n_classifiers) - Competence level estimated for each base classifier and test - example. - - Returns - ------- - selected_classifiers : array of shape [n_samples] - Indices of the selected base classifier for each sample. If the - selection_method is set to 'all', a boolean matrix is returned, - containing True for the selected base classifiers, otherwise false. - - """ - if competences.ndim < 2: - competences = competences.reshape(1, -1) - - selected_classifiers = [] - best_index = np.argmax(competences, axis=1) - - if self.selection_method == 'best': - # Select the classifier with highest competence level - selected_classifiers = best_index - - elif self.selection_method == 'diff': - rng = check_random_state(copy(self.random_state)) - best_competence = competences[ - np.arange(competences.shape[0]), best_index] - # best_competence = np.max(competences) - diff = best_competence.reshape(-1, 1) - competences - # TODO: Improve this part of the code - selected_classifiers = np.zeros(diff.shape[0], dtype=int) - for row in range(diff.shape[0]): - diff_list = list(diff[row, :]) - indices = [idx for idx, _ in enumerate(diff_list) if - diff_list[idx] < self.diff_thresh] - - if len(indices) == 0: - indices = range(self.n_classifiers_) - - selected_classifiers[row] = rng.choice(indices) - - elif self.selection_method == 'random': - # TODO: Improve this part of the code - rng = check_random_state(copy(self.random_state)) - selected_classifiers = np.zeros(competences.shape[0], dtype=int) - best_competence = competences[ - np.arange(competences.shape[0]), best_index] - for row in range(competences.shape[0]): - competence_list = list(competences[row, :]) - - # Select a random classifier among all with same competence - # level - indices = [idx for idx, _ in enumerate(competence_list) if - competence_list[idx] == best_competence[row]] - - selected_classifiers[row] = rng.choice(indices) - - elif self.selection_method == 'all': - # select all base classifiers with max competence estimates. - max_value = np.max(competences, axis=1) - selected_classifiers = ( - competences == max_value.reshape(competences.shape[0], - -1)) - - return selected_classifiers - - def classify_with_ds(self, predictions, probabilities=None, - neighbors=None, distances=None, DFP_mask=None): - """Predicts the class label of the corresponding query sample. - - If self.selection_method == "all", the majority voting scheme is used - to aggregate the predictions of all classifiers with the max competence - level estimates for each test examples. - - Parameters - ---------- - predictions : array of shape (n_samples, n_classifiers) - Predictions of the base classifiers for all test examples - - probabilities : array of shape (n_samples, n_classifiers, n_classes) - Probabilities estimates of each base classifier for all test - examples (For methods that always require probabilities from the - base classifiers) - - neighbors : array of shape (n_samples, n_neighbors) - Indices of the k nearest neighbors. - distances : array of shape (n_samples, n_neighbors) - Distances from the k nearest neighbors to the query - - DFP_mask : array of shape (n_samples, n_classifiers) - Mask containing 1 for the selected base classifier and 0 otherwise. - - Returns - ------- - predicted_label : array of shape (n_samples) - The predicted label for each query - """ - if predictions.ndim < 2: - predictions = predictions.reshape(1, -1) - competences = self.estimate_competence(neighbors, - distances=distances, - predictions=predictions) - - if self.DFP: - competences = competences * DFP_mask - - if self.selection_method != 'all': - # only one classifier is selected - clf_index = self.select(competences) - predicted_label = predictions[ - np.arange(predictions.shape[0]), clf_index] - else: - # Selected ensemble of classifiers is combined using Majority - # Voting - indices = self.select(competences) - votes = np.ma.MaskedArray(predictions, ~indices) - predicted_label = majority_voting_rule(votes) - - return predicted_label - - def predict_proba_with_ds(self, predictions, probabilities, - neighbors=None, distances=None, DFP_mask=None): - """Predicts the posterior probabilities of the corresponding query - sample. - - If self.selection_method == "all", get the probability estimates of the - selected ensemble. Otherwise, the technique gets the probability - estimates from the selected base classifier - - Parameters - ---------- - predictions : array of shape (n_samples, n_classifiers) - Predictions of the base classifiers for all test examples - - probabilities : array of shape (n_samples, n_classifiers, n_classes) - The predictions of each base classifier for all samples (For - methods that always require probabilities from the base - classifiers). - - neighbors : array of shape (n_samples, n_neighbors) - Indices of the k nearest neighbors. - distances : array of shape (n_samples, n_neighbors) - Distances from the k nearest neighbors to the query - - DFP_mask : array of shape (n_samples, n_classifiers) - Mask containing 1 for the selected base classifier and 0 otherwise. - - Returns - ------- - predicted_proba: array of shape (n_samples, n_classes) - Posterior probabilities estimates for each test example. - """ - competences = self.estimate_competence(neighbors, - distances=distances, - predictions=predictions) - - if self.DFP: - competences = competences * DFP_mask - - if self.selection_method != 'all': - # only one classifier is selected - clf_index = self.select(competences) - predicted_proba = probabilities[ - np.arange(probabilities.shape[0]), clf_index] - else: - # Selected ensemble of classifiers is combined using average - # probability - selected_classifiers = self.select(competences) - - # Broadcast the selected classifiers mask (to cover the last axis - # (nClasses): - selected_classifiers = np.expand_dims(selected_classifiers, axis=2) - selected_classifiers = np.broadcast_to(selected_classifiers, - probabilities.shape) - masked_proba = np.ma.MaskedArray(probabilities, - ~selected_classifiers) - - predicted_proba = np.mean(masked_proba, axis=1) - - return predicted_proba - - def _validate_parameters(self): - - super(BaseDCS, self)._validate_parameters() - - if not isinstance(self.selection_method, str): - raise TypeError( - 'The parameter selection_method should be a string.' - ' selection_method = ', type(self.selection_method)) - - if self.selection_method not in ['best', 'all', 'random', 'diff']: - raise ValueError( - 'Invalid value for parameter "selection_method." ' - 'The possible values are: ' - '"best", "all", "random", "diff"') - - if not isinstance(self.diff_thresh, float): - raise TypeError( - 'The parameter diff_thresh should be a float. diff_thresh = ', - self.diff_thresh) - - if self.diff_thresh >= 0.5 or self.diff_thresh < 0.0 or np.isnan( - self.diff_thresh): - raise ValueError( - 'diff_thresh should be lower than 0.5. diff_thresh = ', - self.diff_thresh) diff --git a/deslib/dcs/lca.py b/deslib/dcs/lca.py deleted file mode 100644 index 5e0d90f3..00000000 --- a/deslib/dcs/lca.py +++ /dev/null @@ -1,196 +0,0 @@ -# coding=utf-8 - -# Author: Rafael Menelau Oliveira e Cruz -# -# License: BSD 3 clause - -import numpy as np - -from deslib.dcs.base import BaseDCS - - -class LCA(BaseDCS): - """Local Class Accuracy (LCA). - - Evaluates the competence level of each individual classifiers and - select the most competent one to predict the label of each test sample. - The competence of each base classifier is calculated based on its local - accuracy with respect to some output class. Consider a classifier - :math:`c_{i}` that assigns a test sample to class :math:`w_{l}`. The - competence level of :math:`c_{i}` is estimated by the percentage of the - local training samples assigned to class :math:`w_{l}` that it predicts - the correct class label. - - The LCA method selects the base classifier presenting the highest - competence level. In a case where more than one base classifier achieves - the same competence level, the one that was evaluated first is selected. - The selection methodology can be modified by changing the hyper-parameter - selection_method. - - - Parameters - ---------- - pool_classifiers : list of classifiers (Default = None) - The generated_pool of classifiers trained for the corresponding - classification problem. Each base classifiers should support the method - "predict". If None, then the pool of classifiers is a bagging - classifier. - - k : int (Default = 7) - Number of neighbors used to estimate the competence of the base - classifiers. - - DFP : Boolean (Default = False) - Determines if the dynamic frienemy pruning is applied. - - with_IH : Boolean (Default = False) - Whether the hardness level of the region of competence is used to - decide between using the DS algorithm or the KNN for classification of - a given query sample. - - safe_k : int (default = None) - The size of the indecision region. - - IH_rate : float (default = 0.3) - Hardness threshold. If the hardness level of the competence region is - lower than the IH_rate the KNN classifier is used. Otherwise, the DS - algorithm is used for classification. - - selection_method : String (Default = "best") - Determines which method is used to select the base classifier after - the competences are estimated. - - diff_thresh : float (Default = 0.1) - Threshold to measure the difference between the competence level of the - base classifiers for the random and diff selection schemes. If the - difference is lower than the threshold, their performance are - considered equivalent. - - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. - - knn_classifier : {'knn', 'faiss', None} (Default = 'knn') - The algorithm used to estimate the region of competence: - - - 'knn' : will use :class:`KNeighborsClassifier` from sklearn - :class:`KNNE`. - - - 'faiss' : will use Facebook's Faiss similarity search through the - class :class:`FaissKNNClassifier` - - - `None` : will use sklearn :class:`KNeighborsClassifier`. - - knn_metric : {'minkowski', 'cosine', 'mahalanobis'} (Default = 'minkowski') - The metric used by the k-NN classifier to estimate distances. - - - 'minkowski' will use minkowski distance. - - - 'cosine' will use the cosine distance. - - - 'mahalanobis' will use the mahalonibis distance. - - DSEL_perc : float (Default = 0.5) - Percentage of the input data used to fit DSEL. - Note: This parameter is only used if the pool of classifier is None or - unfitted. - - n_jobs : int, default=-1 - The number of parallel jobs to run. None means 1 unless in - a joblib.parallel_backend context. -1 means using all processors. - Doesn’t affect fit method. - - References - ---------- - Woods, Kevin, W. Philip Kegelmeyer, and Kevin Bowyer. "Combination of - multiple classifiers using local accuracy estimates." IEEE transactions on - pattern analysis and machine intelligence 19.4 (1997): 405-410. - - Britto, Alceu S., Robert Sabourin, and Luiz ES Oliveira. "Dynamic selection - of classifiers—a comprehensive review." - Pattern Recognition 47.11 (2014): 3665-3680. - - R. M. O. Cruz, R. Sabourin, and G. D. Cavalcanti, “Dynamic classifier - selection: Recent advances and perspectives,” - Information Fusion, vol. 41, pp. 195 – 216, 2018. - - """ - - def __init__(self, pool_classifiers=None, k=7, DFP=False, with_IH=False, - safe_k=None, IH_rate=0.30, selection_method='best', - diff_thresh=0.1, random_state=None, knn_classifier='knn', - knn_metric='minkowski', DSEL_perc=0.5, - knne=False, n_jobs=-1): - super(LCA, self).__init__(pool_classifiers=pool_classifiers, k=k, - DFP=DFP, with_IH=with_IH, safe_k=safe_k, - IH_rate=IH_rate, - selection_method=selection_method, - diff_thresh=diff_thresh, - random_state=random_state, - knn_classifier=knn_classifier, - knn_metric=knn_metric, - DSEL_perc=DSEL_perc, - knne=knne, - n_jobs=n_jobs) - - def estimate_competence(self, competence_region, distances=None, - predictions=None): - """estimate the competence of each base classifier :math:`c_{i}` for - the classification of the query sample using the local class accuracy - method. - - In this algorithm the k-Nearest Neighbors of the test sample are - estimated. Then, the local accuracy of the base classifiers is - estimated by its classification accuracy taking into account only the - samples from the class :math:`w_{l}` in this neighborhood. In this - case, :math:`w_{l}` is the class predicted by the base classifier - :math:`c_{i}`, for the query sample. The competence level estimate is - represented by the following equation: - - .. math:: \\delta_{i,j} = \\frac{\\sum_{\\mathbf{x}_{k} \\in - \\omega_{l}}P(\\omega_{l} \\mid \\mathbf{x}_{k}, - c_{i} )}{\\sum_{k = 1}^{K}P(\\omega_{l} \\mid - \\mathbf{x}_{k}, c_{i} )} - - where :math:`\\delta_{i,j}` represents the competence level of - :math:`c_{i}` for the classification of query. - - Parameters - ---------- - competence_region : array of shape (n_samples, n_neighbors) - Indices of the k nearest neighbors. - - distances : array of shape (n_samples, n_neighbors) - Distances from the k nearest neighbors to the query. - - predictions : array of shape (n_samples, n_classifiers) - Predictions of the base classifiers for the test examples. - - Returns - ------- - competences : array of shape (n_samples, n_classifiers) - Competence level estimated for each base classifier and test - example. - """ - predictions = np.atleast_2d(predictions) - - # Expanding the dimensions of the predictions and target arrays in - # order to compare both. - predictions_3d = np.expand_dims(predictions, axis=1) - target_3d = np.expand_dims(self.DSEL_target_[competence_region], - axis=2) - # Create a mask to remove the neighbors belonging to a different class - # than the predicted by the base classifier - mask = (predictions_3d != target_3d) - masked_preprocessed = np.ma.MaskedArray( - self.DSEL_processed_[competence_region, :], mask=mask) - - competences_masked = np.mean(masked_preprocessed, axis=1) - # Fill 0 to the masked values in the resulting array (when no neighbors - # belongs to the class predicted by - # the corresponding base classifier) - competences = np.ma.filled(competences_masked, 0) - - return competences diff --git a/deslib/dcs/mcb.py b/deslib/dcs/mcb.py deleted file mode 100644 index ff4c5f60..00000000 --- a/deslib/dcs/mcb.py +++ /dev/null @@ -1,235 +0,0 @@ -# coding=utf-8 - -# Author: Rafael Menelau Oliveira e Cruz -# -# License: BSD 3 clause - -import numpy as np - -from deslib.dcs.base import BaseDCS - - -class MCB(BaseDCS): - """Multiple Classifier Behaviour (MCB). - - The MCB method evaluates the competence level of each individual - classifiers taking into account the local accuracy of the base classifier - in the region of competence. The region of competence is defined using the - k-NN and behavioral knowledge space (BKS) method. First the k-nearest - neighbors of the test sample are computed. Then, the set containing the - k-nearest neighbors is filtered based on the similarity of the query sample - and its neighbors using the decision space (BKS representation). - - A single classifier :math:`c_{i}` is selected only if its competence level - is significantly higher than that of the other base classifiers in the pool - (higher than a pre-defined threshold). Otherwise, all classifiers in the - pool are combined using the majority voting rule. The selection methodology - can be modified by changing the hyper-parameter selection_method. - - Parameters - ---------- - pool_classifiers : list of classifiers (Default = None) - The generated_pool of classifiers trained for the corresponding - classification problem. Each base classifiers should support the method - "predict". If None, then the pool of classifiers is a bagging - classifier. - - k : int (Default = 7) - Number of neighbors used to estimate the competence of the base - classifiers. - - DFP : Boolean (Default = False) - Determines if the dynamic frienemy pruning is applied. - - with_IH : Boolean (Default = False) - Whether the hardness level of the region of competence is used to - decide between using the DS algorithm or the KNN for classification of - a given query sample. - - safe_k : int (default = None) - The size of the indecision region. - - IH_rate : float (default = 0.3) - Hardness threshold. If the hardness level of the competence region is - lower than the IH_rate the KNN classifier is used. Otherwise, the DS - algorithm is used for classification. - - selection_method : String (Default = "best") - Determines which method is used to select the base classifier after - the competences are estimated. - - diff_thresh : float (Default = 0.1) - Threshold to measure the difference between the competence level of the - base classifiers for the random and diff selection schemes. If the - difference is lower than the threshold, their performance are - considered equivalent. - - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. - - knn_classifier : {'knn', 'faiss', None} (Default = 'knn') - The algorithm used to estimate the region of competence: - - - 'knn' will use :class:`KNeighborsClassifier` from sklearn - :class:`KNNE` available on `deslib.utils.knne` - - - 'faiss' will use Facebook's Faiss similarity search through the - class :class:`FaissKNNClassifier` - - - None, will use sklearn :class:`KNeighborsClassifier`. - - knn_metric : {'minkowski', 'cosine', 'mahalanobis'} (Default = 'minkowski') - The metric used by the k-NN classifier to estimate distances. - - - 'minkowski' will use minkowski distance. - - - 'cosine' will use the cosine distance. - - - 'mahalanobis' will use the mahalonibis distance. - - knne : bool (Default=False) - Whether to use K-Nearest Neighbor Equality (KNNE) for the region - of competence estimation. - - DSEL_perc : float (Default = 0.5) - Percentage of the input data used to fit DSEL. - Note: This parameter is only used if the pool of classifier is None or - unfitted. - - n_jobs : int, default=-1 - The number of parallel jobs to run. None means 1 unless in - a joblib.parallel_backend context. -1 means using all processors. - Doesn’t affect fit method. - - References - ---------- - Giacinto, Giorgio, and Fabio Roli. "Dynamic classifier selection based on - multiple classifier behaviour." - Pattern Recognition 34.9 (2001): 1879-1881. - - Britto, Alceu S., Robert Sabourin, and Luiz ES Oliveira. "Dynamic selection - of classifiers—a comprehensive review." - Pattern Recognition 47.11 (2014): 3665-3680. - - Huang, Yea S., and Ching Y. Suen. "A method of combining multiple experts - for the recognition of unconstrained handwritten numerals." IEEE - Transactions on Pattern Analysis and Machine Intelligence - 17.1 (1995): 90-94. - - Huang, Yea S., and Ching Y. Suen. "The behavior-knowledge space method for - combination of multiple classifiers." IEEE Computer Society Conference on - Computer Vision and Pattern Recognition, 1993. - - R. M. O. Cruz, R. Sabourin, and G. D. Cavalcanti, “Dynamic classifier - selection: Recent advances and perspectives,” - Information Fusion, vol. 41, pp. 195 – 216, 2018. - """ - - def __init__(self, pool_classifiers=None, k=7, DFP=False, with_IH=False, - safe_k=None, IH_rate=0.30, similarity_threshold=0.7, - selection_method='diff', diff_thresh=0.1, random_state=None, - knn_classifier='knn', knn_metric='minkowski', knne=False, - DSEL_perc=0.5, n_jobs=-1): - - super(MCB, self).__init__(pool_classifiers, k, DFP=DFP, - with_IH=with_IH, safe_k=safe_k, - IH_rate=IH_rate, - selection_method=selection_method, - diff_thresh=diff_thresh, - random_state=random_state, - knn_classifier=knn_classifier, - knn_metric=knn_metric, - knne=knne, - DSEL_perc=DSEL_perc, - n_jobs=n_jobs) - - self.similarity_threshold = similarity_threshold - - def estimate_competence(self, competence_region, distances=None, - predictions=None): - """estimate the competence of each base classifier :math:`c_{i}` for - the classification of the query sample using the Multiple Classifier - Behaviour criterion. - - The region of competence in this method is estimated taking into - account the feature space and the decision space (using the behaviour - knowledge space method [4]). First, the k-Nearest Neighbors of the - query sample are defined in the feature space to compose the region of - competence. Then, the similarity in the BKS space between the query and - the instances in the region of competence are estimated using the - following equations: - - .. math:: S(\\tilde{\\mathbf{x}}_{j},\\tilde{\\mathbf{x}}_{k}) = - \\frac{1}{M} - \\sum\\limits_{i = 1}^{M}T(\\mathbf{x}_{j},\\mathbf{x}_{k}) - - .. math:: T(\\mathbf{x}_{j},\\mathbf{x}_{k}) = - \\left\\{\\begin{matrix} 1 & \\text{if} & - c_{i}(\\mathbf{x}_{j}) = c_{i}(\\mathbf{x}_{k}),\\\\ - 0 & \\text{if} & c_{i}(\\mathbf{x}_{j}) \\neq - c_{i}(\\mathbf{x}_{k}). \\end{matrix}\\right. - - Where :math:`S(\\tilde{\\mathbf{x}}_{j},\\tilde{\\mathbf{x}}_{k})` - denotes the similarity between two samples based on the behaviour - knowledge space method (BKS). Instances with similarity lower than a - predefined threshold are removed from the region of competence. The - competence level of the base classifiers are estimated as their - classification accuracy in the final region of competence. - - Parameters - ---------- - competence_region : array of shape (n_samples, n_neighbors) - Indices of the k nearest neighbors. - - distances : array of shape (n_samples, n_neighbors) - Distances from the k nearest neighbors to the query. - - predictions : array of shape (n_samples, n_classifiers) - Predictions of the base classifiers for the test examples. - - Returns - ------- - competences : array of shape (n_samples, n_classifiers) - Competence level estimated for each base classifier and test - example. - """ - - # Use the pre-compute decisions to transform the query to the BKS space - BKS_query = predictions - - T = (self.BKS_DSEL_[competence_region] == BKS_query.reshape( - BKS_query.shape[0], -1, BKS_query.shape[1])) - S = np.sum(T, axis=2) / self.n_classifiers_ - - # get a mask with the neighbors that will be considered for the - # competence estimation for all samples. - boolean_mask = (S > self.similarity_threshold) - boolean_mask[~np.any(boolean_mask, axis=1), :] = True - # Expanding this mask to the third axis (n_classifiers) since it is - # the same for each classifier. - boolean_mask = np.repeat(np.expand_dims(boolean_mask, axis=2), - self.n_classifiers_, axis=2) - - # Use the masked array mean to take into account the removed neighbors - processed_pred = np.ma.MaskedArray( - self.DSEL_processed_[competence_region, :], mask=~boolean_mask) - - competences = np.ma.mean(processed_pred, axis=1) - return competences - - def _validate_parameters(self): - - super(MCB, self)._validate_parameters() - - if not isinstance(self.similarity_threshold, float): - raise TypeError( - 'The parameter similarity_threshold must be a float.' - ' similarity_threshold = ', type(self.similarity_threshold)) - if self.similarity_threshold > 1 or self.similarity_threshold < 0: - raise ValueError( - 'The parameter similarity_threshold should be between' - ' [0 and 1]. similarity_threshold = ', - self.similarity_threshold) diff --git a/deslib/dcs/mla.py b/deslib/dcs/mla.py deleted file mode 100644 index 28224e54..00000000 --- a/deslib/dcs/mla.py +++ /dev/null @@ -1,215 +0,0 @@ -# coding=utf-8 - -# Author: Rafael Menelau Oliveira e Cruz -# -# License: BSD 3 clause - -import numpy as np - -from deslib.dcs.base import BaseDCS - - -class MLA(BaseDCS): - """Modified Local Accuracy (MLA). - - Similar to the LCA technique. The only difference is that the output of - each base classifier is weighted by the distance between the test sample - and each pattern in the region of competence for the estimation of the - classifiers competences. Only the classifier that achieved the highest - competence level is select to predict the label of the test sample x. - - The MLA method selects the base classifier presenting the highest - competence level. In a case where more than one base classifier achieves - the same competence level, the one that was evaluated first is selected. - The selection methodology can be modified by changing the hyper-parameter - selection_method. - - Parameters - ---------- - pool_classifiers : list of classifiers (Default = None) - The generated_pool of classifiers trained for the corresponding - classification problem. Each base classifiers should support the method - "predict". If None, then the pool of classifiers is a bagging - classifier. - - k : int (Default = 7) - Number of neighbors used to estimate the competence of the base - classifiers. - - DFP : Boolean (Default = False) - Determines if the dynamic frienemy pruning is applied. - - with_IH : Boolean (Default = False) - Whether the hardness level of the region of competence is used to - decide between using the DS algorithm or the KNN for classification of - a given query sample. - - safe_k : int (default = None) - The size of the indecision region. - - IH_rate : float (default = 0.3) - Hardness threshold. If the hardness level of the competence region is - lower than the IH_rate the KNN classifier is used. Otherwise, the DS - algorithm is used for classification. - - selection_method : String (Default = "best") - Determines which method is used to select the base classifier after - the competences are estimated. - - diff_thresh : float (Default = 0.1) - Threshold to measure the difference between the competence level of the - base classifiers for the random and diff selection schemes. If the - difference is lower than the threshold, their performance are - considered equivalent. - - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. - - knn_classifier : {'knn', 'faiss', None} (Default = 'knn') - The algorithm used to estimate the region of competence: - - - 'knn' will use :class:`KNeighborsClassifier` from sklearn - :class:`KNNE` available on `deslib.utils.knne` - - - 'faiss' will use Facebook's Faiss similarity search through the - class :class:`FaissKNNClassifier` - - - None, will use sklearn :class:`KNeighborsClassifier`. - - knn_metric : {'minkowski', 'cosine', 'mahalanobis'} (Default = 'minkowski') - The metric used by the k-NN classifier to estimate distances. - - - 'minkowski' will use minkowski distance. - - - 'cosine' will use the cosine distance. - - - 'mahalanobis' will use the mahalonibis distance. - - knne : bool (Default=False) - Whether to use K-Nearest Neighbor Equality (KNNE) for the region - of competence estimation. - - DSEL_perc : float (Default = 0.5) - Percentage of the input data used to fit DSEL. - Note: This parameter is only used if the pool of classifier is None or - unfitted. - - n_jobs : int, default=-1 - The number of parallel jobs to run. None means 1 unless in - a joblib.parallel_backend context. -1 means using all processors. - Doesn’t affect fit method. - - References - ---------- - Woods, Kevin, W. Philip Kegelmeyer, and Kevin Bowyer. "Combination of - multiple classifiers using local accuracy estimates." IEEE transactions on - pattern analysis and machine intelligence 19.4 (1997): 405-410. - - Britto, Alceu S., Robert Sabourin, and Luiz ES Oliveira. "Dynamic selection - of classifiers—a comprehensive review." - Pattern Recognition 47.11 (2014): 3665-3680. - - R. M. O. Cruz, R. Sabourin, and G. D. Cavalcanti, “Dynamic classifier - selection: Recent advances and perspectives,” - Information Fusion, vol. 41, pp. 195 – 216, 2018. - - """ - - def __init__(self, pool_classifiers=None, k=7, DFP=False, with_IH=False, - safe_k=None, IH_rate=0.30, selection_method='best', - diff_thresh=0.1, random_state=None, knn_classifier='knn', - knn_metric='minkowski', knne=False, DSEL_perc=0.5, n_jobs=-1): - super(MLA, self).__init__(pool_classifiers=pool_classifiers, k=k, - DFP=DFP, with_IH=with_IH, safe_k=safe_k, - IH_rate=IH_rate, - selection_method=selection_method, - diff_thresh=diff_thresh, - random_state=random_state, - knn_classifier=knn_classifier, - knn_metric=knn_metric, - knne=knne, - DSEL_perc=DSEL_perc, - n_jobs=n_jobs) - - def estimate_competence(self, competence_region, distances, - predictions=None): - """estimate the competence of each base classifier :math:`c_{i}` for - the classification of the query sample using the Modified Local - Accuracy (MLA) method. - - The competence level of the base classifiers is estimated by its - classification accuracy taking into account only the samples belonging - to a given class :math:`w_{l}`.In this case, :math:`w_{l}` is the class - predicted by the base classifier :math:`c_{i}`, for the query sample. - This method also weights the influence of each training sample - according to its Euclidean distance to the query instance. The closest - samples have a higher influence in the computation of the competence - level. The competence level estimate is represented by the following - equation: - - .. math:: \\delta_{i,j} = \\sum_{k = 1}^{K}P(\\omega_{l} \\mid - \\mathbf{x}_{k} \\in \\omega_{l}, c_{i} )W_{k} - - where :math:`\\delta_{i,j}` represents the competence level of - :math:`c_{i}` for the classification of query. - - Parameters - ---------- - competence_region : array of shape (n_samples, n_neighbors) - Indices of the k nearest neighbors. - - distances : array of shape (n_samples, n_neighbors) - Distances from the k nearest neighbors to the query. - - predictions : array of shape (n_samples, n_classifiers) - Predictions of the base classifiers for the test examples. - - Returns - ------- - competences : array of shape (n_samples, n_classifiers) - Competence level estimated for each base classifier and test - example. - """ - predictions = np.atleast_2d(predictions) - - # Normalize the distances - distances[distances == 0] = 1e-10 - dists_normalized = 1.0 / distances - - # Expanding the dimensions of the predictions and target arrays in - # order to compare both. - predictions_3d = np.expand_dims(predictions, axis=1) - target_3d = np.expand_dims(self.DSEL_target_[competence_region], - axis=2) - # Create a mask to remove the neighbors belonging to a different class - # than the predicted by the base classifier - mask = (predictions_3d != target_3d) - - # Broadcast the distance array to the same shape as the pre-processed - # information for future calculations - dists_normalized = np.repeat(np.expand_dims(dists_normalized, axis=2), - self.n_classifiers_, axis=2) - - # Multiply the pre-processed correct predictions by the base - # classifiers to the distance array - proc_norm = \ - self.DSEL_processed_[competence_region, :] * dists_normalized - - # Create masked arrays to remove samples with different label in the - # calculations - masked_preprocessed = np.ma.MaskedArray(proc_norm, mask=mask) - masked_dist = np.ma.MaskedArray(dists_normalized, mask=mask) - - # Consider only the neighbor samples where the predicted label is - # equals to the neighbor label - competences_masked = np.ma.sum(masked_preprocessed, - axis=1) / np.ma.sum(masked_dist, axis=1) - - # Fill 0 to the masked values in the resulting array (when no neighbors - # belongs to the class predicted by the corresponding base classifier) - competences = np.ma.filled(competences_masked, 0) - - return competences diff --git a/deslib/dcs/ola.py b/deslib/dcs/ola.py deleted file mode 100644 index 05262f9d..00000000 --- a/deslib/dcs/ola.py +++ /dev/null @@ -1,172 +0,0 @@ -# coding=utf-8 - -# Author: Rafael Menelau Oliveira e Cruz -# -# License: BSD 3 clause - -import numpy as np - -from deslib.dcs.base import BaseDCS - - -class OLA(BaseDCS): - """Overall Classifier Accuracy (OLA). - - The OLA method evaluates the competence level of each individual - classifiers and select the most competent one to predict the label of each - test sample x. The competence of each base classifier is calculated as its - classification accuracy in the neighborhood of x (region of competence). - - The OLA method selects the base classifier presenting the highest - competence level. In a case where more than one base classifier achieves - the same competence level, the one that was evaluated first is selected. - The selection methodology can be modified by changing the hyper-parameter - selection_method. - - - Parameters - ---------- - pool_classifiers : list of classifiers (Default = None) - The generated_pool of classifiers trained for the corresponding - classification problem. Each base classifiers should support the method - "predict". If None, then the pool of classifiers is a bagging - classifier. - - k : int (Default = 7) - Number of neighbors used to estimate the competence of the base - classifiers. - - DFP : Boolean (Default = False) - Determines if the dynamic frienemy pruning is applied. - - with_IH : Boolean (Default = False) - Whether the hardness level of the region of competence is used to - decide between using the DS algorithm or the KNN for classification of - a given query sample. - - safe_k : int (default = None) - The size of the indecision region. - - IH_rate : float (default = 0.3) - Hardness threshold. If the hardness level of the competence region is - lower than the IH_rate the KNN classifier is used. Otherwise, the DS - algorithm is used for classification. - - selection_method : String (Default = "best") - Determines which method is used to select the base classifier after - the competences are estimated. - - diff_thresh : float (Default = 0.1) - Threshold to measure the difference between the competence level of the - base classifiers for the random and diff selection schemes. If the - difference is lower than the threshold, their performance are - considered equivalent. - - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. - - knn_classifier : {'knn', 'faiss', None} (Default = 'knn') - The algorithm used to estimate the region of competence: - - - 'knn' will use :class:`KNeighborsClassifier` from sklearn - :class:`KNNE` available on `deslib.utils.knne` - - - 'faiss' will use Facebook's Faiss similarity search through the - class :class:`FaissKNNClassifier` - - - None, will use sklearn :class:`KNeighborsClassifier`. - - knn_metric : {'minkowski', 'cosine', 'mahalanobis'} (Default = 'minkowski') - The metric used by the k-NN classifier to estimate distances. - - - 'minkowski' will use minkowski distance. - - - 'cosine' will use the cosine distance. - - - 'mahalanobis' will use the mahalonibis distance. - - knne : bool (Default=False) - Whether to use K-Nearest Neighbor Equality (KNNE) for the region - of competence estimation. - - DSEL_perc : float (Default = 0.5) - Percentage of the input data used to fit DSEL. - Note: This parameter is only used if the pool of classifier is None or - unfitted. - - n_jobs : int, default=-1 - The number of parallel jobs to run. None means 1 unless in - a joblib.parallel_backend context. -1 means using all processors. - Doesn’t affect fit method. - - References - ---------- - Woods, Kevin, W. Philip Kegelmeyer, and Kevin Bowyer. "Combination of - multiple classifiers using local accuracy estimates." IEEE transactions on - pattern analysis and machine intelligence 19.4 (1997): 405-410. - - Britto, Alceu S., Robert Sabourin, and Luiz ES Oliveira. "Dynamic selection - of classifiers—a comprehensive review." - Pattern Recognition 47.11 (2014): 3665-3680. - - R. M. O. Cruz, R. Sabourin, and G. D. Cavalcanti, “Dynamic classifier - selection: Recent advances and perspectives,” - Information Fusion, vol. 41, pp. 195 – 216, 2018. - - """ - - def __init__(self, pool_classifiers=None, k=7, DFP=False, with_IH=False, - safe_k=None, IH_rate=0.30, selection_method='best', - diff_thresh=0.1, random_state=None, knn_classifier='knn', - knn_metric='minkowski', knne=False, DSEL_perc=0.5, n_jobs=-1): - super(OLA, self).__init__(pool_classifiers=pool_classifiers, k=k, - DFP=DFP, with_IH=with_IH, safe_k=safe_k, - IH_rate=IH_rate, - selection_method=selection_method, - diff_thresh=diff_thresh, - random_state=random_state, - knn_classifier=knn_classifier, - knn_metric=knn_metric, - knne=knne, - DSEL_perc=DSEL_perc, n_jobs=n_jobs) - - def estimate_competence(self, competence_region, distances=None, - predictions=None): - """estimate the competence level of each base classifier :math:`c_{i}` - for the classification of the query sample. - - The competences for each base classifier :math:`c_{i}` is estimated by - its classification accuracy considering the k-Nearest Neighbors (region - of competence). The competence level estimate is represented by the - following equation: - - .. math:: \\delta_{i,j} = \\frac{1}{K}\\sum_{k = 1}^{K} - P(\\omega_{l} \\mid \\mathbf{x}_{k} \\in \\omega_{l}, c_{i} ) - - where :math:`\\delta_{i,j}` represents the competence level of - :math:`c_{i}` for the classification of query. - - Parameters - ---------- - competence_region : array of shape (n_samples, n_neighbors) - Indices of the k nearest neighbors. - - distances : array of shape (n_samples, n_neighbors) - Distances from the k nearest neighbors to the query. - - predictions : array of shape (n_samples, n_classifiers) - Predictions of the base classifiers for the test examples. - - Returns - ------- - competences : array of shape (n_samples, n_classifiers) - Competence level estimated for each base classifier and test - example. - """ - competences = np.mean(self.DSEL_processed_[competence_region, :], - axis=1) - - return competences diff --git a/deslib/dcs/rank.py b/deslib/dcs/rank.py deleted file mode 100644 index adf3fc1e..00000000 --- a/deslib/dcs/rank.py +++ /dev/null @@ -1,182 +0,0 @@ -# coding=utf-8 - -# Author: Rafael Menelau Oliveira e Cruz -# -# License: BSD 3 clause - -import numpy as np - -from deslib.dcs.base import BaseDCS - - -class Rank(BaseDCS): - """Modified Classifier Rank. - - The modified classifier rank method evaluates the competence level of each - individual classifiers and select the most competent one to predict the - label of each test sample :math:`x`. The competence of each base classifier - is calculated as the number of correctly classified samples, starting from - the closest neighbor of :math:`x`. The classifier with the highest number - of correctly classified samples is considered the most competent. - - The Rank method selects the base classifier presenting the highest - competence level. In a case where more than one base classifier achieves - the same competence level, the one that was evaluated first is selected. - The selection methodology can be modified by changing the hyper-parameter - selection_method. - - Parameters - ---------- - pool_classifiers : list of classifiers (Default = None) - The generated_pool of classifiers trained for the corresponding - classification problem. Each base classifiers should support the method - "predict". If None, then the pool of classifiers is a bagging - classifier. - - k : int (Default = 7) - Number of neighbors used to estimate the competence of the base - classifiers. - - DFP : Boolean (Default = False) - Determines if the dynamic frienemy pruning is applied. - - with_IH : Boolean (Default = False) - Whether the hardness level of the region of competence is used to - decide between using the DS algorithm or the KNN for classification of - a given query sample. - - safe_k : int (default = None) - The size of the indecision region. - - IH_rate : float (default = 0.3) - Hardness threshold. If the hardness level of the competence region is - lower than the IH_rate the KNN classifier is used. Otherwise, the DS - algorithm is used for classification. - - selection_method : String (Default = "best") - Determines which method is used to select the base classifier after - the competences are estimated. - - diff_thresh : float (Default = 0.1) - Threshold to measure the difference between the competence level of the - base classifiers for the random and diff selection schemes. If the - difference is lower than the threshold, their performance are - considered equivalent. - - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. - - knn_classifier : {'knn', 'faiss', None} (Default = 'knn') - The algorithm used to estimate the region of competence: - - - 'knn' will use :class:`KNeighborsClassifier` from sklearn - :class:`KNNE` available on `deslib.utils.knne` - - - 'faiss' will use Facebook's Faiss similarity search through the - class :class:`FaissKNNClassifier` - - - None, will use sklearn :class:`KNeighborsClassifier`. - - knn_metric : {'minkowski', 'cosine', 'mahalanobis'} (Default = 'minkowski') - The metric used by the k-NN classifier to estimate distances. - - - 'minkowski' will use minkowski distance. - - - 'cosine' will use the cosine distance. - - - 'mahalanobis' will use the mahalonibis distance. - - knne : bool (Default=False) - Whether to use K-Nearest Neighbor Equality (KNNE) for the region - of competence estimation. - - DSEL_perc : float (Default = 0.5) - Percentage of the input data used to fit DSEL. - Note: This parameter is only used if the pool of classifier is None or - unfitted. - - n_jobs : int, default=-1 - The number of parallel jobs to run. None means 1 unless in - a joblib.parallel_backend context. -1 means using all processors. - Doesn’t affect fit method. - - References - ---------- - Woods, Kevin, W. Philip Kegelmeyer, and Kevin Bowyer. "Combination of - multiple classifiers using local accuracy estimates." IEEE transactions on - pattern analysis and machine intelligence 19.4 (1997): 405-410. - - M. Sabourin, A. Mitiche, D. Thomas, G. Nagy, Classifier combination for - handprinted digit recognition, International Conference on Document - Analysis and Recognition (1993) 163–166. - - Britto, Alceu S., Robert Sabourin, and Luiz ES Oliveira. "Dynamic selection - of classifiers—a comprehensive review." - Pattern Recognition 47.11 (2014): 3665-3680. - - R. M. O. Cruz, R. Sabourin, and G. D. Cavalcanti, “Dynamic classifier - selection: Recent advances and perspectives,” - Information Fusion, vol. 41, pp. 195 – 216, 2018. - - """ - - def __init__(self, pool_classifiers=None, k=7, DFP=False, with_IH=False, - safe_k=None, IH_rate=0.30, selection_method='best', - diff_thresh=0.1, random_state=None, knn_classifier='knn', - knn_metric='minkowski', knne=False, DSEL_perc=0.5, n_jobs=-1): - super(Rank, self).__init__(pool_classifiers=pool_classifiers, k=k, - DFP=DFP, with_IH=with_IH, safe_k=safe_k, - IH_rate=IH_rate, - selection_method=selection_method, - diff_thresh=diff_thresh, - random_state=random_state, - knn_classifier=knn_classifier, - knn_metric=knn_metric, - knne=knne, - DSEL_perc=DSEL_perc, - n_jobs=n_jobs) - - def estimate_competence(self, competence_region, distances=None, - predictions=None): - """estimate the competence level of each base classifier :math:`c_{i}` - for the classification of the query sample using the modified ranking - scheme. The rank of the base classifier is estimated by the number of - consecutive correctly classified samples in the defined region of - competence. - - Parameters - ---------- - competence_region : array of shape (n_samples, n_neighbors) - Indices of the k nearest neighbors. - - distances : array of shape (n_samples, n_neighbors) - Distances from the k nearest neighbors to the query. - - predictions : array of shape (n_samples, n_classifiers) - Predictions of the base classifiers for the test examples. - - Returns - ------- - competences : array of shape (n_samples, n_classifiers) - Competence level estimated for each base classifier and test - example. - """ - results_neighbors = self.DSEL_processed_[competence_region, :] - - # Get the shape of the vector in order to know the number of samples, - # base classifiers and neighbors considered. - shape = results_neighbors.shape - - # add an row with zero for the case where the base classifier correctly - # classifies the whole neighborhood. - # That way the search will always find a zero after comparing to - # self.K + 1 - addition = np.zeros((shape[0], shape[2])) - results_neighbors = np.insert(results_neighbors, shape[1], addition, - axis=1) - competences = np.argmax(results_neighbors == 0, axis=1) - - return competences.astype(np.float32) diff --git a/deslib/des/__init__.py b/deslib/des/__init__.py deleted file mode 100644 index eb57a7a4..00000000 --- a/deslib/des/__init__.py +++ /dev/null @@ -1,36 +0,0 @@ -""" -The :mod:`deslib.des` provides a set of key dynamic ensemble selection -algorithms (DES). -""" - -from .base import BaseDES -from .des_clustering import DESClustering -from .des_knn import DESKNN -from .des_mi import DESMI -from .des_p import DESP -from .knop import KNOP -from .knora_e import KNORAE -from .knora_u import KNORAU -from .meta_des import METADES -from deslib.des.probabilistic.base import BaseProbabilistic -from deslib.des.probabilistic.minimum_difference import MinimumDifference -from deslib.des.probabilistic.deskl import DESKL -from deslib.des.probabilistic.rrc import RRC -from deslib.des.probabilistic.exponential import Exponential -from deslib.des.probabilistic.logarithmic import Logarithmic - -__all__ = ['BaseDES', - 'METADES', - 'KNORAE', - 'KNORAU', - 'KNOP', - 'DESP', - 'DESKNN', - 'DESClustering', - 'DESMI', - 'BaseProbabilistic', - 'RRC', - 'DESKL', - 'MinimumDifference', - 'Exponential', - 'Logarithmic'] diff --git a/deslib/des/base.py b/deslib/des/base.py deleted file mode 100644 index 9f3f973a..00000000 --- a/deslib/des/base.py +++ /dev/null @@ -1,229 +0,0 @@ -from abc import ABCMeta - -import numpy as np - -from deslib.base import BaseDS -from deslib.util.aggregation import (aggregate_proba_ensemble_weighted, - sum_votes_per_class, - get_weighted_votes) - - -class BaseDES(BaseDS): - """Base class for a Dynamic Ensemble Selection (DES). - - All dynamic ensemble selection techniques should inherit from this class. - - Warning: This class should not be instantiated directly, use - derived classes instead. - - """ - __metaclass__ = ABCMeta - - def __init__(self, pool_classifiers=None, k=7, DFP=False, with_IH=False, - safe_k=None, IH_rate=0.30, mode='selection', voting='hard', - needs_proba=False, random_state=None, - knn_classifier='knn', knn_metric='minkowski', knne=False, - DSEL_perc=0.5, n_jobs=-1): - - super(BaseDES, self).__init__(pool_classifiers=pool_classifiers, - k=k, - DFP=DFP, - with_IH=with_IH, - safe_k=safe_k, - IH_rate=IH_rate, - needs_proba=needs_proba, - random_state=random_state, - knn_classifier=knn_classifier, - knn_metric=knn_metric, - knne=knne, - DSEL_perc=DSEL_perc, n_jobs=n_jobs) - self.mode = mode - self.voting = voting - - def classify_with_ds(self, predictions, probabilities=None, - competence_region=None, distances=None, - DFP_mask=None): - """Predicts the label of the corresponding query sample. - - If self.mode == "selection", the selected ensemble is combined using - the majority voting rule - - If self.mode == "weighting", all base classifiers are used for - classification, however their influence in the final decision are - weighted according to their estimated competence level. The weighted - majority voting scheme is used to combine the decisions of the - base classifiers. - - If self.mode == "hybrid", A hybrid Dynamic selection and weighting - approach is used. First an ensemble with the competent base classifiers - are selected. Then, their decisions are aggregated using the weighted - majority voting rule according to its competence level estimates. - - Parameters - ---------- - predictions : array of shape (n_samples, n_classifiers) - Predictions of the base classifier for all test examples. - - probabilities : array of shape (n_samples, n_classifiers, n_classes) - Probabilities estimates of each base classifier for all test - examples. (For methods that always require probabilities from - the base classifiers). - - competence_region : array of shape (n_samples, n_neighbors) - Indices of the k nearest neighbors according for each test sample. - - distances : array of shape (n_samples, n_neighbors) - Distances from the k nearest neighbors to the query - - DFP_mask : array of shape (n_samples, n_classifiers) - Mask containing 1 for the selected base classifier and 0 otherwise. - - Returns - ------- - predicted_label : array of shape (n_samples) - Predicted class label for each test example. - """ - probas = self.predict_proba_with_ds(predictions, probabilities, - competence_region, distances, - DFP_mask) - return probas.argmax(axis=1) - - def predict_proba_with_ds(self, predictions, probabilities=None, - competence_region=None, distances=None, - DFP_mask=None): - """Predicts the posterior probabilities of the corresponding query. - - If self.mode == "selection", the selected ensemble is used to estimate - the probabilities. The average rule is used - to give probabilities estimates. - - If self.mode == "weighting", all base classifiers are used for - estimating the probabilities, however their influence in the final - decision are weighted according to their estimated competence level. - A weighted average method is used to give the probabilities estimates. - - If self.mode == "Hybrid", A hybrid Dynamic selection and weighting - approach is used. First an ensemble with the competent base classifiers - are selected. Then, their decisions are aggregated using a weighted - average rule to give the probabilities estimates. - - Parameters - ---------- - predictions : array of shape (n_samples, n_classifiers) - Predictions of the base classifier for all test examples. - - probabilities : array of shape (n_samples, n_classifiers, n_classes) - Probabilities estimates of each base classifier for all samples. - - competence_region : array of shape (n_samples, n_neighbors) - Indices of the k nearest neighbors. - distances : array of shape (n_samples, n_neighbors) - Distances from the k nearest neighbors to the query - - DFP_mask : array of shape (n_samples, n_classifiers) - Mask containing 1 for the selected base classifier and 0 otherwise. - - Returns - ------- - predicted_proba : array = [n_samples, n_classes] - The probability estimates for all test examples. - """ - if self.needs_proba: - competences = self.estimate_competence_from_proba( - neighbors=competence_region, - distances=distances, - probabilities=probabilities) - else: - competences = self.estimate_competence( - competence_region=competence_region, - distances=distances, - predictions=predictions) - if self.DFP: - # FIRE-DES pruning. - competences = competences * DFP_mask - - if self.mode == "selection": - predicted_proba = self._dynamic_selection(competences, - predictions, - probabilities) - elif self.mode == "weighting": - predicted_proba = self._dynamic_weighting(competences, predictions, - probabilities) - else: - predicted_proba = self._hybrid(competences, predictions, - probabilities) - - return predicted_proba - - def _dynamic_selection(self, competences, predictions, probabilities): - """ Combine models using dynamic ensemble selection. """ - selected_classifiers = self.select(competences) - if self.voting == 'hard': - votes = np.ma.MaskedArray(predictions, ~selected_classifiers) - votes = sum_votes_per_class(votes, self.n_classes_) - predicted_proba = votes / votes.sum(axis=1)[:, None] - else: - masked_proba = self._mask_proba(probabilities, - selected_classifiers) - predicted_proba = np.mean(masked_proba, axis=1) - return predicted_proba - - def _dynamic_weighting(self, competences, predictions, probabilities): - """ Combine models using dynamic weighting. """ - if self.voting == 'hard': - w_votes, _ = get_weighted_votes(predictions, - competences, - np.arange(self.n_classes_)) - predicted_proba = w_votes / w_votes.sum(axis=1)[:, None] - else: - predicted_proba = aggregate_proba_ensemble_weighted( - probabilities, competences) - return predicted_proba - - def _hybrid(self, competences, predictions, probabilities): - """ Combine models using a hybrid dynamic selection + weighting. """ - selected_classifiers = self.select(competences) - if self.voting == 'hard': - votes = np.ma.MaskedArray(predictions, ~selected_classifiers) - w_votes, _ = get_weighted_votes(votes, competences, - np.arange(self.n_classes_)) - predicted_proba = w_votes / w_votes.sum(axis=1)[:, None] - else: - masked_proba = self._mask_proba(probabilities, - selected_classifiers) - predicted_proba = aggregate_proba_ensemble_weighted( - masked_proba, competences) - return predicted_proba - - @staticmethod - def _mask_proba(probabilities, selected_classifiers): - # Broadcast the selected classifiers mask - # to cover the last axis (n_classes): - selected_classifiers = np.expand_dims(selected_classifiers, - axis=2) - selected_classifiers = np.broadcast_to(selected_classifiers, - probabilities.shape) - masked_proba = np.ma.MaskedArray(probabilities, - ~selected_classifiers) - return masked_proba - - def _validate_parameters(self): - super(BaseDES, self)._validate_parameters() - - if not isinstance(self.mode, str): - raise TypeError( - 'Parameter "mode" should be a string.' - ' Currently "mode" = {}' .format(type(self.mode))) - - if self.mode not in ['selection', 'hybrid', 'weighting']: - raise ValueError( - 'Invalid value for parameter "mode".' - ' "mode" should be one of these options ' - '{selection, hybrid, weighting}') - - if self.voting not in ['soft', 'hard']: - raise ValueError('Invalid value for parameter "voting".' - ' "voting" should be one of these options ' - '{selection, hybrid, weighting}') - if self.voting == 'soft': - self._check_predict_proba() diff --git a/deslib/des/des_clustering.py b/deslib/des/des_clustering.py deleted file mode 100644 index 36880095..00000000 --- a/deslib/des/des_clustering.py +++ /dev/null @@ -1,424 +0,0 @@ -# coding=utf-8 - -# Author: Rafael Menelau Oliveira e Cruz -# -# License: BSD 3 clause -import warnings - -import numpy as np -from sklearn import metrics -from sklearn.base import ClusterMixin -from sklearn.cluster import KMeans - -from deslib.base import BaseDS -from deslib.util.aggregation import sum_votes_per_class -from deslib.util.diversity import Q_statistic, ratio_errors, \ - negative_double_fault, compute_pairwise_diversity - - -class DESClustering(BaseDS): - """Dynamic ensemble selection-Clustering (DES-Clustering). - - This method selects an ensemble of classifiers taking into account the - accuracy and diversity of the base classifiers. The K-means algorithm is - used to define the region of competence. For each cluster, the N most - accurate classifiers are first selected. Then, the J more diverse - classifiers from the N most accurate classifiers are selected to - compose the ensemble. - - Parameters - ---------- - pool_classifiers : list of classifiers (Default = None) - The generated_pool of classifiers trained for the corresponding - classification problem. Each base classifiers should support the method - "predict". If None, then the pool of classifiers is a bagging - classifier. - - clustering : sklearn.cluster (Default = None) - The clustering model used to estimate the region of competence. - If None, a KMeans with K = 5 is used. - - pct_accuracy : float (Default = 0.5) - Percentage of base classifiers selected based on accuracy - - pct_diversity : float (Default = 0.33) - Percentage of base classifiers selected based on diversity - - more_diverse : Boolean (Default = True) - Whether we select the most or the least diverse classifiers - to add to the pre-selected ensemble - - metric_diversity : String (Default = 'df') - Metric used to estimate the diversity of the base classifiers. Can be - either the double fault (df), Q-statistics (Q), or error correlation. - - metric_performance : String (Default = 'accuracy_score') - Metric used to estimate the performance of a base classifier on a - cluster. Can be either any metric from sklearn.metrics. - - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. - - DSEL_perc : float (Default = 0.5) - Percentage of the input data used to fit DSEL. - Note: This parameter is only used if the pool of classifier is None or - unfitted. - - voting : {'hard', 'soft'}, default='hard' - If 'hard', uses predicted class labels for majority rule voting. - Else if 'soft', predicts the class label based on the argmax of - the sums of the predicted probabilities, which is recommended for - an ensemble of well-calibrated classifiers. - - n_jobs : int, default=-1 - The number of parallel jobs to run. None means 1 unless in - a joblib.parallel_backend context. -1 means using all processors. - Doesn’t affect fit method. - - References - ---------- - Soares, R. G., Santana, A., Canuto, A. M., & de Souto, M. C. P. - "Using accuracy and more_diverse to select classifiers to build ensembles." - International Joint Conference on Neural Networks (IJCNN)., 2006. - - Britto, Alceu S., Robert Sabourin, and Luiz ES Oliveira. "Dynamic selection - of classifiers—a comprehensive review." - Pattern Recognition 47.11 (2014): 3665-3680. - - R. M. O. Cruz, R. Sabourin, and G. D. Cavalcanti, “Dynamic classifier - selection: Recent advances and perspectives,” - Information Fusion, vol. 41, pp. 195 – 216, 2018. - """ - - def __init__(self, pool_classifiers=None, clustering=None, - pct_accuracy=0.5, voting='hard', - pct_diversity=0.33, more_diverse=True, metric_diversity='DF', - metric_performance='accuracy_score', n_clusters=5, - random_state=None, DSEL_perc=0.5, n_jobs=-1): - - super(DESClustering, self).__init__(pool_classifiers=pool_classifiers, - random_state=random_state, - DSEL_perc=DSEL_perc, - n_jobs=n_jobs, - ) - - self.metric_diversity = metric_diversity - self.metric_performance = metric_performance - self.voting = voting - self.clustering = clustering - self.pct_accuracy = pct_accuracy - self.pct_diversity = pct_diversity - self.more_diverse = more_diverse - self.n_clusters = n_clusters - - def fit(self, X, y): - """ Train the DS model by setting the Clustering algorithm and - pre-processing the information required to apply the DS - methods. - - First the data is divided into K clusters. Then, for each cluster, - the N most accurate classifiers are first selected. Then, the J more - diverse classifiers from the N most accurate classifiers are selected - to compose the ensemble of the corresponding cluster. An ensemble of - classifiers is assigned to each of the K clusters. - - Parameters - ---------- - X : array of shape (n_samples, n_features) - Data used to fit the model. - - y : array of shape (n_samples) - class labels of each example in X. - - Returns - ------- - self - """ - super(DESClustering, self).fit(X, y) - self.DSEL_data_ = self.DSEL_data_.astype(np.double) - self.N_ = int(self.n_classifiers_ * self.pct_accuracy) - self.J_ = int(np.ceil(self.n_classifiers_ * self.pct_diversity)) - - self._check_parameters() - - self.metric_classifier_ = getattr(metrics, self.metric_performance) - - if self.clustering is None: - if self.n_samples_ >= self.n_clusters: - self.clustering_ = KMeans(n_clusters=self.n_clusters, - random_state=self.random_state) - else: - warnings.warn("n_clusters is bigger than DSEL size. " - "Using All DSEL examples as cluster centroids.", - category=RuntimeWarning) - self.clustering_ = KMeans(n_clusters=self.n_samples_, - random_state=self.random_state) - - self.clustering_.fit(self.DSEL_data_) - else: - self.clustering_ = self.clustering.fit(self.DSEL_data_) - - # set the diversity metric used - self._set_diversity_func() - - # Since the clusters are fixed, we can pre-compute the accuracy and - # diversity of each cluster as well as the # selected classifiers - # (indices) for each one. These pre-computed information will be kept - # on those three variables: - self.performance_cluster_ = np.zeros( - (self.clustering_.n_clusters, self.n_classifiers_)) - self.diversity_cluster_ = np.zeros( - (self.clustering_.n_clusters, self.n_classifiers_)) - self.indices_ = np.zeros((self.clustering_.n_clusters, self.J_), - dtype=int) - - self._preprocess_clusters() - return self - - def get_competence_region(self, query, k=None): - distances = self.clustering_.transform(query.astype(np.double)) - region = self.clustering_.predict(query.astype(np.double)) - return distances, region - - def _preprocess_clusters(self): - """Preprocess the competence as well as the average diversity of each - base classifier for each specific cluster. - - This process makes the test routines faster, since the ensemble of - classifiers of each cluster is already predefined. - - The class attributes Accuracy_cluster_ and diversity_cluster_ stores - the accuracy and diversity information respectively of each base - classifier for each cluster. The attribute indices_ stores the - pre-selected base classifiers for each cluster. - """ - labels = self.clustering_.predict(self.DSEL_data_) - - for cluster_index in range(self.clustering_.n_clusters): - - # Get the indices_ of the samples in the corresponding cluster. - sample_indices = np.where(labels == cluster_index)[0] - - # Compute performance metric of each classifier in this cluster - score_classifier = self.get_scores_(sample_indices) - - self.performance_cluster_[cluster_index, :] = score_classifier - - # Get the N_ most accurate classifiers in the cluster - performance_indices = np.argsort(score_classifier)[::-1][0:self.N_] - - # Get the target labels for the samples in the corresponding - # cluster for the diversity calculation. - - targets = self.DSEL_target_[sample_indices] - self.diversity_cluster_[cluster_index, :] = \ - compute_pairwise_diversity(targets, - self.BKS_DSEL_[sample_indices, :], - self.diversity_func_) - - diversity_of_selected = self.diversity_cluster_[ - cluster_index, performance_indices] - - if self.more_diverse: - diversity_indices = np.argsort(diversity_of_selected)[::-1][ - 0:self.J_] - else: - diversity_indices = np.argsort(diversity_of_selected)[ - 0:self.J_] - - self.indices_[cluster_index, :] = performance_indices[ - diversity_indices] - - def estimate_competence(self, competence_region, distances=None, - predictions=None): - """Get the competence estimates of each base classifier :math:`c_{i}` - for the classification of the query sample. - - In this case, the competences were already pre-calculated for each - cluster. So this method computes the nearest cluster and get the - pre-calculated competences of the base classifiers for the - corresponding cluster. - - Parameters - ---------- - predictions : array of shape (n_samples, n_classifiers) - Predictions of the base classifiers for all test examples. - - Returns - ------- - competences : array = [n_samples, n_classifiers] - The competence level estimated for each base classifier. - """ - competences = self.performance_cluster_[competence_region][:] - return competences - - def select(self, competences): - """Select an ensemble with the most accurate and most diverse - classifier for the classification of the query. - - The ensemble for each cluster was already pre-calculated in the fit - method. So, this method calculates the closest cluster, and returns - the ensemble associated to this cluster. - - Parameters - ---------- - competences : array of shape (n_samples) - Array containing closest cluster index. - - Returns - ------- - selected_classifiers : array of shape = [n_samples, self.k] - Indices of the selected base classifier for each test example. - - """ - selected_classifiers = self.indices_[competences, :] - return selected_classifiers - - def classify_with_ds(self, predictions, probabilities=None, - competence_region=None, distances=None, - DFP_mask=None): - """Predicts the label of the corresponding query sample. - - Parameters - ---------- - predictions : array of shape (n_samples, n_classifiers) - Predictions of the base classifiers for all test examples. - - probabilities : array of shape (n_samples, n_classifiers, n_classes) - Probabilities estimates of each base classifier for all test - examples. - - competence_region : array of shape (n_samples) - Indices of the nearest clusters to each sample. - - distances : array of shape (n_samples) - Distances of the nearest clusters to each sample. - - DFP_mask : array of shape (n_samples, n_classifiers) - Mask containing 1 for the selected base classifier and 0 otherwise. - - Returns - ------- - predicted_label : array of shape (n_samples) - Predicted class label for each test example. - """ - proba = self.predict_proba_with_ds(predictions, probabilities, - competence_region, distances, - DFP_mask) - predicted_label = proba.argmax(axis=1) - return predicted_label - - def predict_proba_with_ds(self, predictions, probabilities, - competence_region=None, distances=None, - DFP_mask=None): - """Predicts the label of the corresponding query sample. - - Parameters - ---------- - predictions : array of shape (n_samples, n_classifiers) - Predictions of the base classifiers for all test examples. - - probabilities : array of shape (n_samples, n_classifiers, n_classes) - Probabilities estimates of each base classifier for all test - examples. - - competence_region : array of shape (n_samples) - Indices of the nearest clusters to each sample. - - distances : array of shape (n_samples) - Distances of the nearest clusters to each sample. - - DFP_mask : array of shape (n_samples, n_classifiers) - Mask containing 1 for the selected base classifier and 0 otherwise. - - Returns - ------- - predicted_proba : array of shape (n_samples, n_classes) - Posterior probabilities estimates for each test example. - """ - selected_classifiers = self.select(competence_region) - - if self.voting == 'hard': - votes = predictions[np.arange(predictions.shape[0])[:, None], - selected_classifiers] - votes = sum_votes_per_class(votes, self.n_classes_) - predicted_proba = votes / votes.sum(axis=1)[:, None] - - else: - ensemble_proba = probabilities[ - np.arange(probabilities.shape[0])[:, None], - selected_classifiers, :] - predicted_proba = np.mean(ensemble_proba, axis=1) - - return predicted_proba - - def _check_parameters(self): - """Check if the parameters passed as argument are correct. - - Raises - ------ - ValueError - If the hyper-parameters are incorrect. - """ - if self.metric_diversity not in ['DF', 'Q', 'ratio']: - raise ValueError( - 'Diversity metric must be one of the following values:' - ' "DF", "Q" or "Ratio"') - - try: - getattr(metrics, self.metric_performance) - except AttributeError: - raise ValueError( - "Parameter metric_performance must be a sklearn metrics") - - if self.N_ <= 0 or self.J_ <= 0: - raise ValueError("The values of N_ and J_ should be higher than 0" - "N_ = {}, J_= {} ".format(self.N_, self.J_)) - if self.N_ < self.J_: - raise ValueError( - "The value of N_ should be greater or equals than J_" - "N_ = {}, J_= {} ".format(self.N_, self.J_)) - - if self.clustering is not None: - if not isinstance(self.clustering, ClusterMixin): - raise ValueError( - "Parameter clustering must be a sklearn" - " cluster estimator.") - - if self.voting not in ['soft', 'hard']: - raise ValueError('Invalid value for parameter "mode".' - ' "mode" should be one of these options ' - '{selection, hybrid, weighting}') - - if self.voting == 'soft': - self._check_predict_proba() - - def get_scores_(self, sample_indices): - - def precision_function(label_predicted): - targets = self.DSEL_target_[sample_indices] - return self.metric_classifier_(targets, label_predicted) - - label_predicted = self.BKS_DSEL_[sample_indices, :] - score_classifier = np.apply_along_axis( - precision_function, 0, label_predicted) - - return score_classifier - - def _set_diversity_func(self): - """Set the diversity function to be used according to the - hyper-parameter metric_diversity - - The diversity_func_ can be either the Double Fault, Q-Statistics - or Ratio of errors. - - """ - if self.metric_diversity == 'DF': - self.diversity_func_ = negative_double_fault - elif self.metric_diversity == 'Q': - self.diversity_func_ = Q_statistic - else: - self.diversity_func_ = ratio_errors diff --git a/deslib/des/des_knn.py b/deslib/des/des_knn.py deleted file mode 100644 index 15406a41..00000000 --- a/deslib/des/des_knn.py +++ /dev/null @@ -1,424 +0,0 @@ -# coding=utf-8 - -# Author: Rafael Menelau Oliveira e Cruz -# -# License: BSD 3 clause - -import numpy as np - -from deslib.base import BaseDS -from deslib.util.aggregation import sum_votes_per_class -from deslib.util.diversity import negative_double_fault, Q_statistic, \ - ratio_errors, compute_pairwise_diversity - - -class DESKNN(BaseDS): - """Dynamic ensemble Selection KNN (DES-KNN). - - This method selects an ensemble of classifiers taking into account the - accuracy and diversity of the base classifiers. The k-NN algorithm is used - to define the region of competence. The N most accurate classifiers in the - region of competence are first selected. Then, the J more diverse - classifiers from the N most accurate classifiers are selected to compose - the ensemble. - - Parameters - ---------- - pool_classifiers : list of classifiers (Default = None) - The generated_pool of classifiers trained for the corresponding - classification problem. Each base classifiers should support the method - "predict". If None, then the pool of classifiers is a bagging - classifier. - - k : int (Default = 7) - Number of neighbors used to estimate the competence of the base - classifiers. - - DFP : Boolean (Default = False) - Determines if the dynamic frienemy pruning is applied. - - with_IH : Boolean (Default = False) - Whether the hardness level of the region of competence is used to - decide between using the DS algorithm or the KNN for classification of - a given query sample. - - safe_k : int (default = None) - The size of the indecision region. - - IH_rate : float (default = 0.3) - Hardness threshold. If the hardness level of the competence region is - lower than the IH_rate the KNN classifier is used. Otherwise, the DS - algorithm is used for classification. - - pct_accuracy : float (Default = 0.5) - Percentage of base classifiers selected based on accuracy - - pct_diversity : float (Default = 0.3) - Percentage of base classifiers selected based n diversity - - more_diverse : Boolean (Default = True) - Whether we select the most or the least diverse classifiers to add - to the pre-selected ensemble - - metric : String (Default = 'df') - Metric used to estimate the diversity of the base classifiers. Can be - either the double fault (df), Q-statistics (Q), or error correlation. - - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. - - knn_classifier : {'knn', 'faiss', None} (Default = 'knn') - The algorithm used to estimate the region of competence: - - - 'knn' will use :class:`KNeighborsClassifier` from sklearn - :class:`KNNE` available on `deslib.utils.knne` - - - 'faiss' will use Facebook's Faiss similarity search through the - class :class:`FaissKNNClassifier` - - - None, will use sklearn :class:`KNeighborsClassifier`. - - knn_metric : {'minkowski', 'cosine', 'mahalanobis'} (Default = 'minkowski') - The metric used by the k-NN classifier to estimate distances. - - - 'minkowski' will use minkowski distance. - - - 'cosine' will use the cosine distance. - - - 'mahalanobis' will use the mahalonibis distance. - - knne : bool (Default=False) - Whether to use K-Nearest Neighbor Equality (KNNE) for the region - of competence estimation. - - DSEL_perc : float (Default = 0.5) - Percentage of the input data used to fit DSEL. - Note: This parameter is only used if the pool of classifier is None or - unfitted. - - voting : {'hard', 'soft'}, default='hard' - If 'hard', uses predicted class labels for majority rule voting. - Else if 'soft', predicts the class label based on the argmax of - the sums of the predicted probabilities, which is recommended for - an ensemble of well-calibrated classifiers. - - n_jobs : int, default=-1 - The number of parallel jobs to run. None means 1 unless in - a joblib.parallel_backend context. -1 means using all processors. - Doesn’t affect fit method. - - References - ---------- - Soares, R. G., Santana, A., Canuto, A. M., & de Souto, M. C. P. - "Using accuracy and more_diverse to select classifiers to build ensembles." - International Joint Conference on Neural Networks (IJCNN)., 2006. - - Britto, Alceu S., Robert Sabourin, and Luiz ES Oliveira. "Dynamic selection - of classifiers—a comprehensive review." - Pattern Recognition 47.11 (2014): 3665-3680. - - R. M. O. Cruz, R. Sabourin, and G. D. Cavalcanti, “Dynamic classifier - selection: Recent advances and perspectives,” - Information Fusion, vol. 41, pp. 195 – 216, 2018. - """ - - def __init__(self, pool_classifiers=None, k=7, DFP=False, with_IH=False, - safe_k=None, IH_rate=0.30, pct_accuracy=0.5, - pct_diversity=0.3, more_diverse=True, metric='DF', - random_state=None, knn_classifier='knn', - knn_metric='minkowski', knne=False, DSEL_perc=0.5, n_jobs=-1, - voting='hard'): - - super(DESKNN, self).__init__(pool_classifiers=pool_classifiers, - k=k, - DFP=DFP, - with_IH=with_IH, - safe_k=safe_k, - IH_rate=IH_rate, - random_state=random_state, - knn_classifier=knn_classifier, - knn_metric=knn_metric, - knne=knne, - DSEL_perc=DSEL_perc, - n_jobs=n_jobs) - - self.metric = metric - self.pct_accuracy = pct_accuracy - self.pct_diversity = pct_diversity - self.more_diverse = more_diverse - self.voting = voting - - def fit(self, X, y): - """ Prepare the DS model by setting the KNN algorithm and - pre-processing the information required to apply the DS - method. - - Parameters - ---------- - X : array of shape (n_samples, n_features) - Data used to fit the model. - - y : array of shape (n_samples) - class labels of each example in X. - - Returns - ------- - self - """ - super(DESKNN, self).fit(X, y) - self.N_ = int(self.n_classifiers_ * self.pct_accuracy) - self.J_ = int(np.ceil(self.n_classifiers_ * self.pct_diversity)) - self._check_parameters() - self._set_diversity_func() - return self - - def estimate_competence(self, competence_region, distances=None, - predictions=None): - """estimate the competence level of each base classifier :math:`c_{i}` - for the classification of the query sample. - - The competence is estimated using the accuracy and diversity criteria. - First the classification accuracy of the base classifiers in the - region of competence is estimated. Then the diversity of the - base classifiers is estimated. - - The method returns two arrays: One containing the accuracy and the - other the diversity of each base classifier. - - Parameters - ---------- - competence_region : array of shape (n_samples, n_neighbors) - Indices of the k nearest neighbors according for each test sample. - - distances : array of shape (n_samples, n_neighbors) - Distances from the k nearest neighbors to the query - - - predictions : array of shape (n_samples, n_classifiers) - Predictions of the base classifiers for all test examples. - - Notes - ------ - This technique uses both the accuracy and diversity information to - perform dynamic selection. For this reason the function returns a - dictionary containing these two values instead of a single ndarray - containing the competence level estimates for each base classifier. - - Returns - ------- - accuracy : array of shape = [n_samples, n_classifiers} - Local Accuracy estimates (competences) of the base - classifiers for all query samples. - - diversity : array of shape = [n_samples, n_classifiers} - Average pairwise diversity of each base classifiers for - all test examples. - - """ - accuracy = np.mean(self.DSEL_processed_[competence_region, :], axis=1) - - predicted_matrix = self.BKS_DSEL_[competence_region, :] - targets = self.DSEL_target_[competence_region] - - # TODO: optimize this part with numpy instead of for loops - diversity = np.zeros((competence_region.shape[0], self.n_classifiers_)) - for sample_idx in range(competence_region.shape[0]): - this_diversity = compute_pairwise_diversity(targets[sample_idx, :], - predicted_matrix[ - sample_idx, :, :], - self.diversity_func_) - - diversity[sample_idx, :] = this_diversity - - return accuracy, diversity - - def select(self, accuracy, diversity): - """Select an ensemble containing the N most accurate ant the J most - diverse classifiers for the classification of the query sample. - - Parameters - ---------- - accuracy : array of shape (n_samples, n_classifiers) - Local Accuracy estimates (competence) of each base classifiers. - - diversity : array of shape (n_samples, n_classifiers) - Average pairwise diversity of each base classifiers. - - Returns - ------- - selected_classifiers : array of shape = [n_samples, self.J] - Array containing the indices of the J selected base classifier - for each test example. - """ - # Check if the accuracy and diversity arrays have - # the correct dimensionality. - if accuracy.ndim < 2: - accuracy = accuracy.reshape(1, -1) - - if diversity.ndim < 2: - diversity = diversity.reshape(1, -1) - - # sort the array to remove the most accurate classifiers - competent_indices = np.argsort(accuracy, axis=1)[:, ::-1][:, 0:self.N_] - diversity_of_selected = diversity[ - np.arange(diversity.shape[0])[:, None], competent_indices] - # diversity_of_selected = diversity.take(competent_indices) - - # sort the remaining classifiers to select the most diverse ones - if self.more_diverse: - diversity_indices = np.argsort(diversity_of_selected, axis=1) - diversity_indices = diversity_indices[:, ::-1][:, 0:self.J_] - else: - diversity_indices = np.argsort(diversity_of_selected, axis=1) - diversity_indices = diversity_indices[:, 0:self.J_] - - # Getting the index of all selected base classifiers. - selected_classifiers = competent_indices[ - np.arange(competent_indices.shape[0])[:, None], diversity_indices] - - return selected_classifiers - - def classify_with_ds(self, predictions, probabilities=None, - neighbors=None, distances=None, DFP_mask=None): - """Predicts the label of the corresponding query sample. - - Parameters - ---------- - predictions : array of shape (n_samples, n_classifiers) - Predictions of the base classifiers for all test examples - - probabilities : array of shape (n_samples, n_classifiers, n_classes) - Probabilities estimates of each base classifier for all test - examples. - - neighbors : array of shape (n_samples, n_neighbors) - Indices of the k nearest neighbors according for each test sample. - - distances : array of shape (n_samples, n_neighbors) - Distances from the k nearest neighbors to the query - - DFP_mask : array of shape (n_samples, n_classifiers) - Mask containing 1 for the selected base classifier and 0 otherwise. - - Notes - ------ - Different than other DES techniques, this method is based on a two - stage selection, where first the most accurate classifier are selected, - then the diversity information is used to get the most diverse ensemble - for the probability estimation. Hence, the weighting mode is not - defined. Also, the selected ensemble size is fixed (self.J), so there - is no need to use masked arrays in this class. - - Returns - ------- - predicted_label : array of shape (n_samples) - Predicted class label for each test example. - """ - proba = self.predict_proba_with_ds(predictions, probabilities, - neighbors, distances, DFP_mask) - predicted_label = proba.argmax(axis=1) - return predicted_label - - def predict_proba_with_ds(self, predictions, probabilities, - neighbors=None, distances=None, DFP_mask=None): - """Predicts the posterior probabilities. - - Parameters - ---------- - predictions : array of shape (n_samples, n_classifiers) - Predictions of the base classifiers for all test examples. - - probabilities : array of shape (n_samples, n_classifiers, n_classes) - Probabilities estimates of each base classifier for all test - examples. - - neighbors : array of shape (n_samples, n_neighbors) - Indices of the k nearest neighbors. - - distances : array of shape (n_samples, n_neighbors) - Distances from the k nearest neighbors to the query. - - DFP_mask : array of shape (n_samples, n_classifiers) - Mask containing 1 for the selected base classifier and 0 otherwise. - - Notes - ------ - Different than other DES techniques, this method is based on a two - stage selection, where first the most accurate classifier are selected, - then the diversity information is used to get the most diverse ensemble - for the probability estimation. Hence, the weighting mode is not - available. - - Returns - ------- - predicted_proba : array = [n_samples, n_classes] - Probability estimates for all test examples. - """ - accuracy, diversity = self.estimate_competence(neighbors, - distances=distances, - predictions=predictions) - if self.DFP: - accuracy = accuracy * DFP_mask - - # This method always performs selection. There is no weighted version. - selected_classifiers = self.select(accuracy, diversity) - - if self.voting == 'hard': - votes = predictions[np.arange(predictions.shape[0])[:, None], - selected_classifiers] - votes = sum_votes_per_class(votes, self.n_classes_) - predicted_proba = votes / votes.sum(axis=1)[:, None] - else: - ensemble_proba = probabilities[ - np.arange(probabilities.shape[0])[:, None], - selected_classifiers, :] - - predicted_proba = np.mean(ensemble_proba, axis=1) - - return predicted_proba - - def _check_parameters(self): - """Check if the parameters passed as argument are correct. - - Raises - ------ - ValueError - If the hyper-parameters are incorrect. - """ - if self.metric not in ['DF', 'Q', 'ratio']: - raise ValueError( - 'Diversity metric must be one of the following values:' - ' "DF", "Q" or "Ratio"') - - if self.N_ <= 0 or self.J_ <= 0: - raise ValueError("The values of N_ and J_ should be higher than 0" - "N_ = {}, J_= {} ".format(self.N_, self.J_)) - if self.N_ < self.J_: - raise ValueError( - "The value of N_ should be greater or equals than J_" - "N_ = {}, J_= {} ".format(self.N_, self.J_)) - - if self.voting not in ['soft', 'hard']: - raise ValueError('Invalid value for parameter "mode".' - ' "mode" should be one of these options ' - '{selection, hybrid, weighting}') - if self.voting == 'soft': - self._check_predict_proba() - - def _set_diversity_func(self): - """Set the diversity function to be used according to the - hyper-parameter metric - - The diversity_func_ can be either the Double Fault, Q-Statistics - or Ratio of errors. - ---------- - """ - if self.metric == 'DF': - self.diversity_func_ = negative_double_fault - elif self.metric == 'Q': - self.diversity_func_ = Q_statistic - else: - self.diversity_func_ = ratio_errors diff --git a/deslib/des/des_mi.py b/deslib/des/des_mi.py deleted file mode 100644 index f51b73ab..00000000 --- a/deslib/des/des_mi.py +++ /dev/null @@ -1,336 +0,0 @@ -# coding=utf-8 - -# Author: Qiushi Wang -# -# License: BSD 3 clause - -import numpy as np -from sklearn.preprocessing import normalize - -from deslib.base import BaseDS -from deslib.util.aggregation import sum_votes_per_class - - -class DESMI(BaseDS): - """Dynamic ensemble Selection for multi-class imbalanced datasets (DES-MI). - - Parameters - ---------- - pool_classifiers : list of classifiers (Default = None) - The generated_pool of classifiers trained for the corresponding - classification problem. Each base classifiers should support the method - "predict". If None, then the pool of classifiers is a bagging - classifier. - - k : int (Default = 7) - Number of neighbors used to estimate the competence of the base - classifiers. - - DFP : Boolean (Default = False) - Determines if the dynamic frienemy pruning is applied. - - with_IH : Boolean (Default = False) - Whether the hardness level of the region of competence is used to - decide between using the DS algorithm or the KNN for classification of - a given query sample. - - safe_k : int (default = None) - The size of the indecision region. - - IH_rate : float (default = 0.3) - Hardness threshold. If the hardness level of the competence region is - lower than the IH_rate the KNN classifier is used. Otherwise, the DS - algorithm is used for classification. - - alpha : float (Default = 0.9) - Scaling coefficient to regulate the weight value - - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. - - knn_classifier : {'knn', 'faiss', None} (Default = 'knn') - The algorithm used to estimate the region of competence: - - - 'knn' will use :class:`KNeighborsClassifier` from sklearn - :class:`KNNE` available on `deslib.utils.knne` - - - 'faiss' will use Facebook's Faiss similarity search through the - class :class:`FaissKNNClassifier` - - - None, will use sklearn :class:`KNeighborsClassifier`. - - knn_metric : {'minkowski', 'cosine', 'mahalanobis'} (Default = 'minkowski') - The metric used by the k-NN classifier to estimate distances. - - - 'minkowski' will use minkowski distance. - - - 'cosine' will use the cosine distance. - - - 'mahalanobis' will use the mahalonibis distance. - - knne : bool (Default=False) - Whether to use K-Nearest Neighbor Equality (KNNE) for the region - of competence estimation. - - DSEL_perc : float (Default = 0.5) - Percentage of the input data used to fit DSEL. - Note: This parameter is only used if the pool of classifier is None or - unfitted. - - voting : {'hard', 'soft'}, default='hard' - If 'hard', uses predicted class labels for majority rule voting. - Else if 'soft', predicts the class label based on the argmax of - the sums of the predicted probabilities, which is recommended for - an ensemble of well-calibrated classifiers. - - n_jobs : int, default=-1 - The number of parallel jobs to run. None means 1 unless in - a joblib.parallel_backend context. -1 means using all processors. - Doesn’t affect fit method. - - References - ---------- - García, S.; Zhang, Z.-L.; Altalhi, A.; Alshomrani, S. & Herrera, F. - "Dynamic ensemble selection for multi-class - imbalanced datasets." Information Sciences, 2018, 445-446, 22 - 37 - - Britto, Alceu S., Robert Sabourin, and Luiz ES Oliveira. "Dynamic selection - of classifiers—a comprehensive review." - Pattern Recognition 47.11 (2014): 3665-3680 - - R. M. O. Cruz, R. Sabourin, and G. D. Cavalcanti, “Dynamic classifier - selection: Recent advances and perspectives,” - Information Fusion, vol. 41, pp. 195 – 216, 2018. - """ - - def __init__(self, pool_classifiers=None, k=7, pct_accuracy=0.4, alpha=0.9, - DFP=False, with_IH=False, safe_k=None, - IH_rate=0.30, random_state=None, knn_classifier='knn', - knn_metric='minkowski', knne=False, DSEL_perc=0.5, n_jobs=-1, - voting='hard'): - - super(DESMI, self).__init__(pool_classifiers=pool_classifiers, - k=k, - DFP=DFP, - with_IH=with_IH, - safe_k=safe_k, - IH_rate=IH_rate, - random_state=random_state, - knn_classifier=knn_classifier, - knn_metric=knn_metric, - knne=knne, - DSEL_perc=DSEL_perc, - n_jobs=n_jobs) - - self.alpha = alpha - self.pct_accuracy = pct_accuracy - self.voting = voting - - def estimate_competence(self, competence_region, distances=None, - predictions=None): - """estimate the competence level of each base classifier :math:`c_{i}` - for the classification of the query sample. Returns a ndarray - containing the competence level of each base classifier. - - The competence is estimated using the accuracy criteria. - The accuracy is estimated by the weighted results of classifiers who - correctly classify the members in the competence region. The weight - of member :math:`x_i` is related to the number of samples of the same - class of :math:`x_i` in the training dataset. - For detail, please see the first reference, Algorithm 2. - - Parameters - ---------- - competence_region : array of shape (n_samples, n_neighbors) - Indices of the k nearest neighbors according for each test sample. - - distances : array of shape (n_samples, n_neighbors) - Distances from the k nearest neighbors to the query. - - predictions : array of shape (n_samples, n_classifiers) - Predictions of the base classifiers for all test examples. - - Returns - ------- - accuracy : array of shape = [n_samples, n_classifiers} - Local Accuracy estimates (competences) of the base classifiers - for all query samples. - - """ - # calculate the weight - class_frequency = np.bincount(self.DSEL_target_) - targets = self.DSEL_target_[competence_region] - num = class_frequency[targets] - weight = 1. / (1 + np.exp(self.alpha * num)) - weight = normalize(weight, norm='l1') - correct_num = self.DSEL_processed_[competence_region, :] - correct = np.zeros((competence_region.shape[0], self.k_, - self.n_classifiers_)) - for i in range(self.n_classifiers_): - correct[:, :, i] = correct_num[:, :, i] * weight - - # Apply the weights to each sample for each base classifier - competence = correct_num * weight[:, :, np.newaxis] - # calculate the classifiers mean competence for all - # samples/base classifier - competence = np.sum(competence, axis=1) - - return competence - - def select(self, competences): - """Select an ensemble containing the N most accurate classifiers for - the classification of the query sample. - - Parameters - ---------- - competences : array of shape (n_samples, n_classifiers) - Competence estimates of each base classifiers for all query - samples. - - Returns - ------- - selected_classifiers : array of shape = [n_samples, self.N] - Matrix containing the indices of the N selected base classifier - for each test example. - """ - # Check if the accuracy and diversity arrays have - # the correct dimensionality. - if competences.ndim < 2: - competences = competences.reshape(1, -1) - - # sort the array to remove the most accurate classifiers - selected_classifiers = np.argsort(competences, axis=1) - selected_classifiers = selected_classifiers[:, ::-1][:, 0:self.N_] - - return selected_classifiers - - def classify_with_ds(self, predictions, probabilities=None, - neighbors=None, distances=None, DFP_mask=None): - """Predicts the label of the corresponding query sample. - - Parameters - ---------- - predictions : array of shape (n_samples, n_classifiers) - Predictions of the base classifiers for all test examples. - - probabilities : array of shape (n_samples, n_classifiers, n_classes) - Probabilities estimates of each base classifier for all test - examples. - - neighbors : array of shape (n_samples, n_neighbors) - Indices of the k nearest neighbors according for each test sample. - - distances : array of shape (n_samples, n_neighbors) - Distances from the k nearest neighbors to the query - - DFP_mask : array of shape (n_samples, n_classifiers) - Mask containing 1 for the selected base classifier and 0 otherwise. - - Notes - ------ - Different than other DES techniques, this method only select N - candidates from the pool of classifiers. - - Returns - ------- - predicted_label : array of shape (n_samples) - Predicted class label for each test example. - """ - proba = self.predict_proba_with_ds(predictions, probabilities, - neighbors, distances, DFP_mask) - predicted_label = proba.argmax(axis=1) - return predicted_label - - def predict_proba_with_ds(self, predictions, probabilities, - competence_region=None, distances=None, - DFP_mask=None): - """Predicts the posterior probabilities of the corresponding - query sample. - - Parameters - ---------- - predictions : array of shape (n_samples, n_classifiers) - Predictions of the base classifiers for all test examples. - - probabilities : array of shape (n_samples, n_classifiers, n_classes) - Probabilities estimates of each base classifier for all test - examples. - - competence_region : array of shape (n_samples, n_neighbors) - Indices of the k nearest neighbors according for each test sample. - - distances : array of shape (n_samples, n_neighbors) - Distances from the k nearest neighbors to each test - sample. - - DFP_mask : array of shape (n_samples, n_classifiers) - Mask containing 1 for the selected base classifier and 0 otherwise. - - Returns - ------- - predicted_proba : array = [n_samples, n_classes] - Probability estimates for all test examples. - """ - accuracy = self.estimate_competence( - competence_region=competence_region, - distances=distances) - - if self.DFP: - accuracy = accuracy * DFP_mask - - selected_classifiers = self.select(accuracy) - if self.voting == 'hard': - votes = predictions[np.arange(predictions.shape[0])[:, None], - selected_classifiers] - votes = sum_votes_per_class(votes, self.n_classes_) - predicted_proba = votes / votes.sum(axis=1)[:, None] - - else: - ensemble_proba = probabilities[ - np.arange(probabilities.shape[0])[:, None], - selected_classifiers, :] - - predicted_proba = np.mean(ensemble_proba, axis=1) - - return predicted_proba - - def _validate_parameters(self): - """Check if the parameters passed as argument are correct. - - Raises - ------ - ValueError - If the hyper-parameters are incorrect. - """ - super(DESMI, self)._validate_parameters() - - self.N_ = int(self.n_classifiers_ * self.pct_accuracy) - - if self.N_ <= 0: - raise ValueError("The value of N_ should be higher than 0" - "N_ = {}".format(self.N_)) - - # The value of Scaling coefficient (alpha) should be positive - # to add more weight to the minority class - if self.alpha <= 0: - raise ValueError("The value of alpha should be higher than 0" - "alpha = {}".format(self.alpha)) - - if not isinstance(self.alpha, float): - raise TypeError("parameter alpha should be a float!") - - if self.pct_accuracy <= 0. or self.pct_accuracy > 1: - raise ValueError( - "The value of pct_accuracy should be higher than 0 and lower" - " or equal to 1, " - "pct_accuracy = {}".format(self.pct_accuracy)) - - if self.voting not in ['soft', 'hard']: - raise ValueError('Invalid value for parameter "voting".' - ' "voting" should be one of these options ' - '{selection, hybrid, weighting}') - if self.voting == 'soft': - self._check_predict_proba() diff --git a/deslib/des/des_p.py b/deslib/des/des_p.py deleted file mode 100644 index d7352bc3..00000000 --- a/deslib/des/des_p.py +++ /dev/null @@ -1,195 +0,0 @@ -# coding=utf-8 - -# Author: Rafael Menelau Oliveira e Cruz -# -# License: BSD 3 clause - -import numpy as np - -from deslib.des.base import BaseDES - - -class DESP(BaseDES): - """Dynamic ensemble selection-Performance(DES-P). - - This method selects all base classifiers that achieve a classification - performance, in the region of competence, that is higher than the random - classifier (RC). The performance of the random classifier is defined by - RC = 1/L, where L is the number of classes in the problem. - If no base classifier is selected, the whole pool is used for - classification. - - Parameters - ---------- - pool_classifiers : list of classifiers (Default = None) - The generated_pool of classifiers trained for the corresponding - classification problem. Each base classifiers should support the method - "predict". If None, then the pool of classifiers is a bagging - classifier. - - k : int (Default = 7) - Number of neighbors used to estimate the competence of the base - classifiers. - - DFP : Boolean (Default = False) - Determines if the dynamic frienemy pruning is applied. - - with_IH : Boolean (Default = False) - Whether the hardness level of the region of competence is used to - decide between using the DS algorithm or the KNN for classification of - a given query sample. - - safe_k : int (default = None) - The size of the indecision region. - - IH_rate : float (default = 0.3) - Hardness threshold. If the hardness level of the competence region is - lower than the IH_rate the KNN classifier is used. Otherwise, the DS - algorithm is used for classification. - - - mode : String (Default = "selection") - Whether the technique will perform dynamic selection, - dynamic weighting or an hybrid approach for classification. - - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. - - knn_classifier : {'knn', 'faiss', None} (Default = 'knn') - The algorithm used to estimate the region of competence: - - - 'knn' will use :class:`KNeighborsClassifier` from sklearn - :class:`KNNE` available on `deslib.utils.knne` - - - 'faiss' will use Facebook's Faiss similarity search through the - class :class:`FaissKNNClassifier` - - - None, will use sklearn :class:`KNeighborsClassifier`. - - knn_metric : {'minkowski', 'cosine', 'mahalanobis'} (Default = 'minkowski') - The metric used by the k-NN classifier to estimate distances. - - - 'minkowski' will use minkowski distance. - - - 'cosine' will use the cosine distance. - - - 'mahalanobis' will use the mahalonibis distance. - - knne : bool (Default=False) - Whether to use K-Nearest Neighbor Equality (KNNE) for the region - of competence estimation. - - DSEL_perc : float (Default = 0.5) - Percentage of the input data used to fit DSEL. - Note: This parameter is only used if the pool of classifier is None or - unfitted. - - voting : {'hard', 'soft'}, default='hard' - If 'hard', uses predicted class labels for majority rule voting. - Else if 'soft', predicts the class label based on the argmax of - the sums of the predicted probabilities, which is recommended for - an ensemble of well-calibrated classifiers. - - n_jobs : int, default=-1 - The number of parallel jobs to run. None means 1 unless in - a joblib.parallel_backend context. -1 means using all processors. - Doesn’t affect fit method.. - - References - ---------- - Woloszynski, Tomasz, et al. "A measure of competence based on random - classification for dynamic ensemble selection." - Information Fusion 13.3 (2012): 207-213. - - Woloszynski, Tomasz, and Marek Kurzynski. "A probabilistic model of - classifier competence for dynamic ensemble selection." - Pattern Recognition 44.10 (2011): 2656-2668. - - R. M. O. Cruz, R. Sabourin, and G. D. Cavalcanti, “Dynamic classifier - selection: Recent advances and perspectives,” - Information Fusion, vol. 41, pp. 195 – 216, 2018. - """ - - def __init__(self, pool_classifiers=None, k=7, DFP=False, with_IH=False, - safe_k=None, IH_rate=0.30, mode='selection', - random_state=None, knn_classifier='knn', - knn_metric='minkowski', knne=False, DSEL_perc=0.5, n_jobs=-1, - voting='hard'): - - super(DESP, self).__init__(pool_classifiers=pool_classifiers, - k=k, - DFP=DFP, - with_IH=with_IH, - safe_k=safe_k, - IH_rate=IH_rate, - mode=mode, - random_state=random_state, - knn_classifier=knn_classifier, - knn_metric=knn_metric, - knne=knne, - DSEL_perc=DSEL_perc, - n_jobs=n_jobs, - voting=voting) - - def estimate_competence(self, competence_region, distances=None, - predictions=None): - """estimate the competence of each base classifier :math:`c_{i}` for - the classification of the query sample base on its local performance. - - .. math:: \\delta_{i,j} = \\hat{P}(c_{i} \\mid \\theta_{j} ) - - \\frac{1}{L} - - Parameters - ---------- - competence_region : array of shape (n_samples, n_neighbors) - Indices of the k nearest neighbors according for each test sample. - - distances : array of shape (n_samples, n_neighbors) - Distances from the k nearest neighbors to the query. - - predictions : array of shape (n_samples, n_classifiers) - Predictions of the base classifiers for all test examples. - - Returns - ------- - competences : array of shape (n_samples, n_classifiers) - Competence level estimated for each base classifier and test - example. - """ - competences = np.mean(self.DSEL_processed_[competence_region, :], - axis=1) - - return competences - - def select(self, competences): - """Selects all base classifiers that obtained a local classification - accuracy higher than the Random Classifier. The performance of the - random classifier is denoted 1/L, where L is the number of classes - in the problem. - - Parameters - ---------- - competences : array of shape (n_samples, n_classifiers) - Competence level estimated for each base classifier and test - example. - - Returns - ------- - selected_classifiers : array of shape (n_samples, n_classifiers) - Boolean matrix containing True if the base classifier is selected, - False otherwise. - - """ - if competences.ndim < 2: - competences = competences.reshape(1, -1) - - RC = 1.0 / self.n_classes_ - selected_classifiers = (competences > RC) - - # For the rows that are all False (i.e., no base classifier - # was selected, select all classifiers (set all True) - selected_classifiers[~np.any(selected_classifiers, axis=1), :] = True - return selected_classifiers diff --git a/deslib/des/knop.py b/deslib/des/knop.py deleted file mode 100644 index 753acdcf..00000000 --- a/deslib/des/knop.py +++ /dev/null @@ -1,288 +0,0 @@ -# coding=utf-8 - -# Author: Rafael Menelau Oliveira e Cruz -# -# License: BSD 3 clause - -import numpy as np - -from deslib.des.base import BaseDES - - -class KNOP(BaseDES): - """k-Nearest Output Profiles (KNOP). - - This method selects all classifiers that correctly classified at least - one sample belonging to the region of competence of the query sample. - In this case, the region of competence is estimated using the decisions - of the base classifier (output profiles). Thus, the similarity between - the query and the validation samples are measured in the decision space - rather than the feature space. Each selected classifier has a number of - votes equals to the number of samples in the region of competence that - it predicts the correct label. The votes obtained by all - base classifiers are aggregated to obtain the final ensemble decision. - - Parameters - ---------- - pool_classifiers : list of classifiers (Default = None) - The generated_pool of classifiers trained for the corresponding - classification problem. Each base classifiers should support the method - "predict". If None, then the pool of classifiers is a bagging - classifier. - - k : int (Default = 7) - Number of neighbors used to estimate the competence of the base - classifiers. - - DFP : Boolean (Default = False) - Determines if the dynamic frienemy pruning is applied. - - with_IH : Boolean (Default = False) - Whether the hardness level of the region of competence is used to - decide between using the DS algorithm or the KNN for classification of - a given query sample. - - safe_k : int (default = None) - The size of the indecision region. - - IH_rate : float (default = 0.3) - Hardness threshold. If the hardness level of the competence region is - lower than the IH_rate the KNN classifier is used. Otherwise, the DS - algorithm is used for classification. - - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. - - knn_classifier : {'knn', 'faiss', None} (Default = 'knn') - The algorithm used to estimate the region of competence: - - - 'knn' will use :class:`KNeighborsClassifier` from sklearn - :class:`KNNE` available on `deslib.utils.knne` - - - 'faiss' will use Facebook's Faiss similarity search through the - class :class:`FaissKNNClassifier` - - - None, will use sklearn :class:`KNeighborsClassifier`. - - knne : bool (Default=False) - Whether to use K-Nearest Neighbor Equality (KNNE) for the region - of competence estimation. - - DSEL_perc : float (Default = 0.5) - Percentage of the input data used to fit DSEL. - Note: This parameter is only used if the pool of classifier is None or - unfitted. - - voting : {'hard', 'soft'}, default='hard' - If 'hard', uses predicted class labels for majority rule voting. - Else if 'soft', predicts the class label based on the argmax of - the sums of the predicted probabilities, which is recommended for - an ensemble of well-calibrated classifiers. - - n_jobs : int, default=-1 - The number of parallel jobs to run. None means 1 unless in - a joblib.parallel_backend context. -1 means using all processors. - Doesn’t affect fit method. - - References - ---------- - Cavalin, Paulo R., Robert Sabourin, and Ching Y. Suen. - "LoGID: An adaptive framework combining local and global - incremental learning for dynamic selection of ensembles of HMMs." - Pattern Recognition 45.9 (2012): 3544-3556. - - Cavalin, Paulo R., Robert Sabourin, and Ching Y. Suen. - "Dynamic selection approaches for multiple classifier - systems." Neural Computing and Applications 22.3-4 (2013): 673-688. - - Ko, Albert HR, Robert Sabourin, and Alceu Souza Britto Jr. - "From dynamic classifier selection to dynamic ensemble - selection." Pattern Recognition 41.5 (2008): 1718-1731. - - Britto, Alceu S., Robert Sabourin, and Luiz ES Oliveira. "Dynamic selection - of classifiers—a comprehensive review." - Pattern Recognition 47.11 (2014): 3665-3680 - - R. M. O. Cruz, R. Sabourin, and G. D. Cavalcanti, “Dynamic classifier - selection: Recent advances and perspectives,” - Information Fusion, vol. 41, pp. 195 – 216, 2018. - """ - def __init__(self, pool_classifiers=None, k=7, DFP=False, with_IH=False, - safe_k=None, IH_rate=0.30, random_state=None, - knn_classifier='knn', knne=False, - DSEL_perc=0.5, n_jobs=-1, voting='hard'): - - super(KNOP, self).__init__(pool_classifiers, k, - DFP=DFP, - with_IH=with_IH, - safe_k=safe_k, - IH_rate=IH_rate, - mode='weighting', - needs_proba=True, - random_state=random_state, - knn_classifier=knn_classifier, - knne=knne, - DSEL_perc=DSEL_perc, - n_jobs=n_jobs, - voting=voting) - - def fit(self, X, y): - """Train the DS model by setting the KNN algorithm and - pre-process the information required to apply the DS - methods. In this case, the scores of the base classifiers for - the dynamic selection dataset (DSEL) are pre-calculated to - transform each sample in DSEL into an output profile. - - Parameters - ---------- - X : array of shape (n_samples, n_features) - Data used to fit the model. - - y : array of shape (n_samples) - class labels of each example in X. - - Returns - ------- - self - """ - super(KNOP, self).fit(X, y) - if self.n_classes_ == 1: - raise ValueError( - "Error. KNOP does not accept one class datasets!") - self._check_predict_proba() - self.dsel_scores_ = self._predict_proba_base(self.DSEL_data_) - # Reshape DSEL_scores as a 2-D array for nearest neighbor calculations - dsel_output_profiles = self.dsel_scores_.reshape(self.n_samples_, - self.n_classifiers_ * - self.n_classes_) - - self._fit_OP(dsel_output_profiles, self.DSEL_target_, self.k_) - - return self - - def _fit_OP(self, X_op, y_op, k): - """ Fit the set of output profiles. - - Parameters - ---------- - X_op : array of shape (n_samples, n_features) - Output profiles of the training data. n_features is equals - to (n_classifiers x n_classes). - - y_op : array of shape (n_samples) - Class labels of each sample in X_op. - - k : int - Number of output profiles used in the region of competence - estimation. - - """ - self.op_knn_ = self.knn_class_(k) - - if self.n_classes_ == 2: - # Get only the scores for one class since they are complementary - X_temp = X_op[:, ::2] - self.op_knn_.fit(X_temp, y_op) - else: - self.op_knn_.fit(X_op, y_op) - - def _get_similar_out_profiles(self, probabilities): - """Get the most similar output profiles of the query sample. - - Parameters - ---------- - probabilities : array of shape (n_samples, n_classifiers, n_classes) - predictions of each base classifier for all samples. - - Returns - ------- - dists : list of shape = [n_samples, k] - The distances between the query and each sample in the region - of competence. The vector is ordered in an ascending fashion. - - idx : list of shape = [n_samples, k] - Indices of the instances belonging to the region of competence of - the given query sample. - """ - - if self.n_classes_ == 2: - # Get only the scores for one class since they are complementary - query_op = probabilities[:, :, 0] - else: - query_op = probabilities.reshape((probabilities.shape[0], - self.n_classifiers_ * - self.n_classes_)) - - dists, idx = self.op_knn_.kneighbors(query_op, n_neighbors=self.k_, - return_distance=True) - return dists, np.atleast_2d(idx) - - def estimate_competence_from_proba(self, probabilities, - neighbors=None, distances=None): - - """The competence of the base classifiers is simply estimated as - the number of samples in the region of competence that it correctly - classified. This method received an array with - the pre-calculated probability estimates for each query. - - This information is later used to determine the number of votes - obtained for each base classifier. - - Parameters - ---------- - neighbors : array of shape (n_samples, n_neighbors) - Indices of the k nearest neighbors. - - distances : array of shape (n_samples, n_neighbors) - Distances from the k nearest neighbors to the query. - - probabilities : array of shape (n_samples, n_classifiers, n_classes) - Probabilities estimates obtained by each each base classifier - for each query sample. - - Returns - ------- - competences : array of shape (n_samples, n_classifiers) - Competence level estimated for each base classifier and test - example. - """ - _, idx_neighbors = self._get_similar_out_profiles(probabilities) - competences = np.sum(self.DSEL_processed_[idx_neighbors, :], axis=1, - dtype=float) - - return competences - - def select(self, competences): - """Select the base classifiers for the classification of the query - sample. - - Each base classifier can be selected more than once. The number of - times a base classifier is selected (votes) is equals to the number - of samples it correctly classified in the region of competence. - - Parameters - ---------- - competences : array of shape (n_samples, n_classifiers) - Competence level estimated for each base classifier and test - example. - - Returns - ------- - selected_classifiers : array of shape (n_samples, n_classifiers) - Boolean matrix containing True if the base classifier is selected, - False otherwise. - """ - if competences.ndim < 2: - competences = competences.reshape(1, -1) - - # Select classifier if it correctly classified at least one sample - selected_classifiers = (competences > 0) - - # For the rows that are all False (i.e., no base classifier - # was selected, select all classifiers (set all True) - selected_classifiers[~np.any(selected_classifiers, axis=1), :] = True - - return selected_classifiers diff --git a/deslib/des/knora_e.py b/deslib/des/knora_e.py deleted file mode 100644 index 7217070a..00000000 --- a/deslib/des/knora_e.py +++ /dev/null @@ -1,228 +0,0 @@ -# coding=utf-8 - -# Author: Rafael Menelau Oliveira e Cruz -# -# License: BSD 3 clause - -import numpy as np - -from deslib.des.base import BaseDES - - -class KNORAE(BaseDES): - """k-Nearest Oracles Eliminate (KNORA-E). - - This method searches for a local Oracle, which is a base classifier - that correctly classify all samples belonging to the region of competence - of the test sample. All classifiers with a perfect performance in the - region of competence are selected (local Oracles). In the case that no - classifier achieves a perfect accuracy, the size of the competence region - is reduced (by removing the farthest neighbor) and the performance of the - classifiers are re-evaluated. The outputs of the selected ensemble of - classifiers is combined using the majority voting scheme. If no base - classifier is selected, the whole pool is used for classification. - - Parameters - ---------- - pool_classifiers : list of classifiers (Default = None) - The generated_pool of classifiers trained for the corresponding - classification problem. Each base classifiers should support the method - "predict". If None, then the pool of classifiers is a bagging - classifier. - - k : int (Default = 7) - Number of neighbors used to estimate the competence of the base - classifiers. - - DFP : Boolean (Default = False) - Determines if the dynamic frienemy pruning is applied. - - with_IH : Boolean (Default = False) - Whether the hardness level of the region of competence is used to - decide between using the DS algorithm or the KNN for classification of - a given query sample. - - safe_k : int (default = None) - The size of the indecision region. - - IH_rate : float (default = 0.3) - Hardness threshold. If the hardness level of the competence region is - lower than the IH_rate the KNN classifier is used. Otherwise, the DS - algorithm is used for classification. - - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. - - knn_classifier : {'knn', 'faiss', None} (Default = 'knn') - The algorithm used to estimate the region of competence: - - - 'knn' will use :class:`KNeighborsClassifier` from sklearn - :class:`KNNE` available on `deslib.utils.knne` - - - 'faiss' will use Facebook's Faiss similarity search through the - class :class:`FaissKNNClassifier` - - - None, will use sklearn :class:`KNeighborsClassifier`. - - knn_metric : {'minkowski', 'cosine', 'mahalanobis'} (Default = 'minkowski') - The metric used by the k-NN classifier to estimate distances. - - - 'minkowski' will use minkowski distance. - - - 'cosine' will use the cosine distance. - - - 'mahalanobis' will use the mahalonibis distance. - - knne : bool (Default=False) - Whether to use K-Nearest Neighbor Equality (KNNE) for the region - of competence estimation. - - DSEL_perc : float (Default = 0.5) - Percentage of the input data used to fit DSEL. - Note: This parameter is only used if the pool of classifier is None or - unfitted. - - voting : {'hard', 'soft'}, default='hard' - If 'hard', uses predicted class labels for majority rule voting. - Else if 'soft', predicts the class label based on the argmax of - the sums of the predicted probabilities, which is recommended for - an ensemble of well-calibrated classifiers. - - n_jobs : int, default=-1 - The number of parallel jobs to run. None means 1 unless in - a joblib.parallel_backend context. -1 means using all processors. - Doesn’t affect fit method. - - References - ---------- - Ko, Albert HR, Robert Sabourin, and Alceu Souza Britto Jr. - "From dynamic classifier selection to dynamic ensemble - selection." Pattern Recognition 41.5 (2008): 1718-1731. - - Britto, Alceu S., Robert Sabourin, and Luiz ES Oliveira. "Dynamic selection - of classifiers—a comprehensive review." - Pattern Recognition 47.11 (2014): 3665-3680 - - R. M. O. Cruz, R. Sabourin, and G. D. Cavalcanti, “Dynamic classifier - selection: Recent advances and perspectives,” - Information Fusion, vol. 41, pp. 195 – 216, 2018. - - """ - - def __init__(self, pool_classifiers=None, k=7, DFP=False, with_IH=False, - safe_k=None, IH_rate=0.30, random_state=None, - knn_classifier='knn', knn_metric='minkowski', knne=False, - DSEL_perc=0.5, n_jobs=-1, voting='hard'): - - super(KNORAE, self).__init__(pool_classifiers=pool_classifiers, - k=k, - DFP=DFP, - with_IH=with_IH, - safe_k=safe_k, - IH_rate=IH_rate, - random_state=random_state, - knn_classifier=knn_classifier, - knn_metric=knn_metric, - knne=knne, - DSEL_perc=DSEL_perc, - n_jobs=n_jobs, - voting=voting, - ) - - def estimate_competence(self, competence_region, distances=None, - predictions=None): - """ Estimate the competence of the base classifiers. In the case of - the KNORA-E technique, the classifiers are only considered competent - when they achieve a 100% accuracy in the region of competence. - For each base, we estimate the maximum size of the region of competence - that it is a local oracle. The competence level estimate is then the - maximum size of the region of competence that the corresponding base - classifier is considered a local Oracle. - - Parameters - ---------- - competence_region : array of shape (n_samples, n_neighbors) - Indices of the k nearest neighbors. - - distances : array of shape (n_samples, n_neighbors) - Distances from the k nearest neighbors to the query. - - predictions : array of shape (n_samples, n_classifiers) - Predictions of the base classifiers for all test examples. - - Returns - ------- - competences : array of shape (n_samples, n_classifiers) - Competence level estimated for each base classifier and test - example. - """ - results_neighbors = self.DSEL_processed_[competence_region, :] - - # Get the shape of the vector in order to know the number of samples, - # base classifiers and neighbors considered. - shape = results_neighbors.shape - - # add an row with zero for the case where the base classifier correctly - # classifies the whole neighborhood. That way the search will always - # find a zero after comparing to self.K + 1 and will return self.K - # as the Competence level estimate (correctly classified the whole - # neighborhood) - addition = np.zeros((shape[0], shape[2])) - results_neighbors = np.insert(results_neighbors, shape[1], addition, - axis=1) - - # Look for the first occurrence of a zero in the processed predictions - # (first misclassified sample). The np.argmax can be used here, since - # in case of multiple occurrences of the maximum values, the indices_ - # corresponding to the first occurrence are returned. - competences = np.argmax(results_neighbors == 0, axis=1) - - return competences.astype(float) - - def select(self, competences): - """Selects all base classifiers that obtained a local accuracy of 100% - in the region of competence (i.e., local oracle). In the case that no - base classifiers obtain 100% accuracy, the size of the region of - competence is reduced and the search for the local oracle is restarted. - - Notes - ------ - Instead of re-applying the method several times (reducing the size of - the region of competence), we compute the number of consecutive correct - classification of each base classifier starting from the closest - neighbor to the more distant in the estimate_competence function. - The number of consecutive correct classification represents the size - of the region of competence in which the corresponding base classifier - is an Local Oracle. Then, we select all base classifiers with the - maximum value for the number of consecutive correct classification. - This speed up the selection process. - - Parameters - ---------- - competences : array of shape (n_samples, n_classifiers) - Competence level estimated for each base classifier and test - example. - - Returns - ------- - selected_classifiers : array of shape (n_samples, n_classifiers) - Boolean matrix containing True if the base classifier is selected, - False otherwise. - - """ - if competences.ndim < 2: - competences = competences.reshape(1, -1) - - # Checks which was the max value for each sample - # (i.e., the maximum number of consecutive predictions) - max_value = np.max(competences, axis=1) - - # Select all base classifiers with the maximum number of - # consecutive correct predictions for each sample. - selected_classifiers = ( - competences == max_value.reshape(competences.shape[0], -1)) - - return selected_classifiers diff --git a/deslib/des/knora_u.py b/deslib/des/knora_u.py deleted file mode 100644 index 1b6bc7a3..00000000 --- a/deslib/des/knora_u.py +++ /dev/null @@ -1,191 +0,0 @@ -# coding=utf-8 - -# Author: Rafael Menelau Oliveira e Cruz -# -# License: BSD 3 clause - -import numpy as np - -from deslib.des.base import BaseDES - - -class KNORAU(BaseDES): - """k-Nearest Oracles Union (KNORA-U). - - This method selects all classifiers that correctly classified at least - one sample belonging to the region of competence of the query sample. Each - selected classifier has a number of votes equals to the number of samples - in the region of competence that it predicts the correct label. The votes - obtained by all base classifiers are aggregated to obtain the final - ensemble decision. - - Parameters - ---------- - pool_classifiers : list of classifiers (Default = None) - The generated_pool of classifiers trained for the corresponding - classification problem. Each base classifiers should support the method - "predict". If None, then the pool of classifiers is a bagging - classifier. - - k : int (Default = 7) - Number of neighbors used to estimate the competence of the base - classifiers. - - DFP : Boolean (Default = False) - Determines if the dynamic frienemy pruning is applied. - - with_IH : Boolean (Default = False) - Whether the hardness level of the region of competence is used to - decide between using the DS algorithm or the KNN for classification of - a given query sample. - - safe_k : int (default = None) - The size of the indecision region. - - IH_rate : float (default = 0.3) - Hardness threshold. If the hardness level of the competence region is - lower than the IH_rate the KNN classifier is used. Otherwise, the DS - algorithm is used for classification. - - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. - - knn_classifier : {'knn', 'faiss', None} (Default = 'knn') - The algorithm used to estimate the region of competence: - - - 'knn' will use :class:`KNeighborsClassifier` from sklearn - :class:`KNNE` available on `deslib.utils.knne` - - - 'faiss' will use Facebook's Faiss similarity search through the - class :class:`FaissKNNClassifier` - - - None, will use sklearn :class:`KNeighborsClassifier`. - - knn_metric : {'minkowski', 'cosine', 'mahalanobis'} (Default = 'minkowski') - The metric used by the k-NN classifier to estimate distances. - - - 'minkowski' will use minkowski distance. - - - 'cosine' will use the cosine distance. - - - 'mahalanobis' will use the mahalonibis distance. - - knne : bool (Default=False) - Whether to use K-Nearest Neighbor Equality (KNNE) for the region - of competence estimation. - - DSEL_perc : float (Default = 0.5) - Percentage of the input data used to fit DSEL. - Note: This parameter is only used if the pool of classifier is None or - unfitted. - - voting : {'hard', 'soft'}, default='hard' - If 'hard', uses predicted class labels for majority rule voting. - Else if 'soft', predicts the class label based on the argmax of - the sums of the predicted probabilities, which is recommended for - an ensemble of well-calibrated classifiers. - - n_jobs : int, default=-1 - The number of parallel jobs to run. None means 1 unless in - a joblib.parallel_backend context. -1 means using all processors. - Doesn’t affect fit method. - - References - ---------- - Ko, Albert HR, Robert Sabourin, and Alceu Souza Britto Jr. - "From dynamic classifier selection to dynamic ensemble - selection." Pattern Recognition 41.5 (2008): 1718-1731. - - Britto, Alceu S., Robert Sabourin, and Luiz ES Oliveira. - "Dynamic selection of classifiers—a comprehensive review." - Pattern Recognition 47.11 (2014): 3665-3680. - - R. M. O. Cruz, R. Sabourin, and G. D. Cavalcanti, “Dynamic classifier - selection: Recent advances and perspectives,” - Information Fusion, vol. 41, pp. 195 – 216, 2018. - """ - - def __init__(self, pool_classifiers=None, k=7, DFP=False, with_IH=False, - safe_k=None, IH_rate=0.30, random_state=None, voting='hard', - knn_classifier='knn', knn_metric='minkowski', knne=False, - DSEL_perc=0.5, n_jobs=-1): - super(KNORAU, self).__init__(pool_classifiers, k, - DFP=DFP, - with_IH=with_IH, - safe_k=safe_k, - IH_rate=IH_rate, - mode='weighting', - random_state=random_state, - knn_classifier=knn_classifier, - knn_metric=knn_metric, - knne=knne, - DSEL_perc=DSEL_perc, - n_jobs=n_jobs, - voting=voting) - - def estimate_competence(self, competence_region, distances=None, - predictions=None): - """The competence of the base classifiers is simply estimated as the - number of samples in the region of competence that it - correctly classified. - - This information is later used to determine the number of votes - obtained for each base classifier. - - Parameters - ---------- - competence_region : array of shape (n_samples, n_neighbors) - Indices of the k nearest neighbors. - - distances : array of shape (n_samples, n_neighbors) - Distances from the k nearest neighbors to the query. - - predictions : array of shape (n_samples, n_classifiers) - Predictions of the base classifiers for all test examples. - - Returns - ------- - competences : array of shape (n_samples, n_classifiers) - Competence level estimated for each base classifier and test - example. - - """ - competences = np.sum(self.DSEL_processed_[competence_region, :], - axis=1, dtype=float) - - return competences - - def select(self, competences): - """Select the base classifiers for the classification of the query - sample. - - Each base classifier can be selected more than once. The number of - times a base classifier is selected (votes) is equals to the number - of samples it correctly classified in the region of competence. - - Parameters - ---------- - competences : array of shape (n_samples, n_classifiers) - Competence level estimated for each base classifier and test - example. - - Returns - ------- - selected_classifiers : array of shape (n_samples, n_classifiers) - Boolean matrix containing True if the base classifier is selected, - False otherwise. - """ - if competences.ndim < 2: - competences = competences.reshape(1, -1) - - # Select classifier if it correctly classified at least one sample - selected_classifiers = (competences > 0) - - # For the rows that are all False (i.e., no base classifier was - # selected, select all classifiers (set all True) - selected_classifiers[~np.any(selected_classifiers, axis=1), :] = True - - return selected_classifiers diff --git a/deslib/des/meta_des.py b/deslib/des/meta_des.py deleted file mode 100644 index 2f16d66b..00000000 --- a/deslib/des/meta_des.py +++ /dev/null @@ -1,568 +0,0 @@ -# coding=utf-8 - -# Author: Rafael Menelau Oliveira e Cruz -# -# License: BSD 3 clause - -import warnings - -import numpy as np -from sklearn.naive_bayes import MultinomialNB - -from deslib.des.base import BaseDES - - -class METADES(BaseDES): - """Meta learning for dynamic ensemble selection (META-DES). - - The META-DES framework is based on the assumption that the dynamic ensemble - selection problem can be considered as a meta-problem. This meta-problem - uses different criteria regarding the behavior of a base classifier - :math:`c_{i}`, in order to decide whether it is competent enough to - classify a given test sample. - - The framework performs a meta-training stage, in which, the meta-features - are extracted from each instance belonging to the training and the dynamic - selection dataset (DSEL). Then, the extracted meta-features are used - to train the meta-classifier :math:`\\lambda`. The meta-classifier is - trained to predict whether or not a base classifier :math:`c_{i}` is - competent enough to classify a given input sample. - - When an unknown sample is presented to the system, the meta-features for - each base classifier :math:`c_{i}` in relation to the input sample are - calculated and presented to the meta-classifier. The meta-classifier - estimates the competence level of the base classifier :math:`c_{i}` for - the classification of the query sample. Base classifiers with competence - level higher than a pre-defined threshold are selected. If no base - classifier is selected, the whole pool is used for classification. - - Parameters - ---------- - pool_classifiers : list of classifiers (Default = None) - The generated_pool of classifiers trained for the corresponding - classification problem. Each base classifiers should support the method - "predict". If None, then the pool of classifiers is a bagging - classifier. - - meta_classifier : sklearn.estimator (Default = None) - Classifier model used for the meta-classifier. If None, - a Multinomial naive Bayes classifier is used. - - k : int (Default = 7) - Number of neighbors used to estimate the competence of the base - classifiers. - - Kp : int (Default = 5) - Number of output profiles used to estimate the competence of the - base classifiers. - - Hc : float (Default = 1.0) - Sample selection threshold. - - selection_threshold : float(Default = 0.5) - Threshold used to select the base classifier. Only the base classifiers - with competence level higher than the selection_threshold are selected - to compose the ensemble. - - mode : String (Default = "selection") - Determines the mode of META-des that is used - (selection, weighting or hybrid). - - DFP : Boolean (Default = False) - Determines if the dynamic frienemy pruning is applied. - - with_IH : Boolean (Default = False) - Whether the hardness level of the region of competence is used to - decide between using the DS algorithm or the KNN for classification of - a given query sample. - - safe_k : int (default = None) - The size of the indecision region. - - IH_rate : float (default = 0.3) - Hardness threshold. If the hardness level of the competence region is - lower than the IH_rate the KNN classifier is used. Otherwise, the DS - algorithm is used for classification. - - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. - - knn_classifier : {'knn', 'faiss', None} (Default = 'knn') - The algorithm used to estimate the region of competence: - - - 'knn' will use :class:`KNeighborsClassifier` from sklearn - :class:`KNNE` available on `deslib.utils.knne` - - - 'faiss' will use Facebook's Faiss similarity search through the - class :class:`FaissKNNClassifier` - - - None, will use sklearn :class:`KNeighborsClassifier`. - - knn_metric : {'minkowski', 'cosine', 'mahalanobis'} (Default = 'minkowski') - The metric used by the k-NN classifier to estimate distances. - - - 'minkowski' will use minkowski distance. - - - 'cosine' will use the cosine distance. - - - 'mahalanobis' will use the mahalonibis distance. - - Note: This parameter only affects the neighborhood search applied in - the feature space. - - knne : bool (Default=False) - Whether to use K-Nearest Neighbor Equality (KNNE) for the region - of competence estimation. - - DSEL_perc : float (Default = 0.5) - Percentage of the input data used to fit DSEL. - Note: This parameter is only used if the pool of classifier is None or - unfitted. - - voting : {'hard', 'soft'}, default='hard' - If 'hard', uses predicted class labels for majority rule voting. - Else if 'soft', predicts the class label based on the argmax of - the sums of the predicted probabilities, which is recommended for - an ensemble of well-calibrated classifiers. - - n_jobs : int, default=-1 - The number of parallel jobs to run. None means 1 unless in - a joblib.parallel_backend context. -1 means using all processors. - Doesn’t affect fit method. - - References - ---------- - Cruz, R.M., Sabourin, R., Cavalcanti, G.D. and Ren, T.I., 2015. META-DES: - A dynamic ensemble selection framework using meta-learning. - Pattern Recognition, 48(5), pp.1925-1935. - - Cruz, R.M., Sabourin, R. and Cavalcanti, G.D., 2015, July. META-des. H: - a dynamic ensemble selection technique using meta-learning and a dynamic - weighting approach. In Neural Networks (IJCNN), 2015 International Joint - Conference on (pp. 1-8). - - R. M. O. Cruz, R. Sabourin, and G. D. Cavalcanti, “Dynamic classifier - selection: Recent advances and perspectives,” - Information Fusion, vol. 41, pp. 195 – 216, 2018. - - """ - - def __init__(self, pool_classifiers=None, meta_classifier=None, k=7, Kp=5, - Hc=1.0, selection_threshold=0.5, mode='selection', - DFP=False, with_IH=False, safe_k=None, IH_rate=0.30, - random_state=None, knn_classifier='knn', knne=False, - knn_metric='minkowski', DSEL_perc=0.5, n_jobs=-1, - voting='hard'): - - super(METADES, self).__init__(pool_classifiers=pool_classifiers, - k=k, - DFP=DFP, - with_IH=with_IH, - safe_k=safe_k, - IH_rate=IH_rate, - mode=mode, - needs_proba=True, - random_state=random_state, - knn_classifier=knn_classifier, - knn_metric=knn_metric, - knne=knne, - DSEL_perc=DSEL_perc, - n_jobs=n_jobs, - voting=voting) - - self.meta_classifier = meta_classifier - self.Kp = Kp - self.Hc = Hc - self.selection_threshold = selection_threshold - - def fit(self, X, y): - """Prepare the DS model by setting the KNN algorithm and - pre-processing the information required to apply the DS - method. - - This method also extracts the meta-features and trains the - meta-classifier :math:`\\lambda` if the meta-classifier was - not yet trained. - - Parameters - ---------- - X : array of shape (n_samples, n_features) - Data used to fit the model. - - y : array of shape (n_samples) - class labels of each example in X. - - Returns - ------- - self - """ - super(METADES, self).fit(X, y) - - if self.n_classes_ == 1: - raise ValueError( - "Error. META-DES does not accept one class datasets.") - - self._check_Kp_samples() - - # Check if the base classifier is able to estimate probabilities - self._check_predict_proba() - - self.dsel_scores_ = self._predict_proba_base(self.DSEL_data_) - - # Reshape DSEL_scores as a 2-D array for nearest neighbor calculations - dsel_output_profiles = self.dsel_scores_.reshape(self.n_samples_, - self.n_classifiers_ * - self.n_classes_) - - self._fit_OP(dsel_output_profiles, self.DSEL_target_) - - if self.meta_classifier is None: - self.meta_classifier_ = MultinomialNB() - else: - self.meta_classifier_ = self.meta_classifier - - # check whether the meta-classifier was already trained since - # it could have been pre-processed before - if not hasattr(self.meta_classifier_, "classes_"): - - # IF it is not fitted, generate the meta-training dataset and - # train the meta-classifier - X_meta, y_meta = self._generate_meta_training_set() - self._fit_meta_classifier(X_meta, y_meta) - - # set the number of meta-features - self.n_meta_features_ = (self.k_ * 2) + self.Kp_ + 2 - - return self - - def _fit_OP(self, X_op, y_op): - """ Fit the set of output profiles. - - Parameters - ---------- - X_op : array of shape (n_samples, n_features) - The output profiles of the Input data. n_features is equals to - (n_classifiers x n_classes) - - y_op : array of shape (n_samples) - class labels of each sample in X_op. - - """ - self.op_knn_ = self.knn_class_(self.Kp_) - # guarantees that minkowski metric is used in this case. It is a - # requirement for dealing with the decision space. - self.op_knn_.metric = 'minkowski' - self.op_knn_.metric_params = None - - if self.n_classes_ == 2: - # Get only the scores for one class since they are complementary - X_temp = X_op[:, ::2] - self.op_knn_.fit(X_temp, y_op) - else: - self.op_knn_.fit(X_op, y_op) - - def _sample_selection_agreement(self): - """Check the number of base classifier that predict the correct label - for the query sample. - - Returns - ------- - pct_agree : array of shape [n_samples] - The percentage of the base classifier that predicted the - correct label for each sample in DSEL. - - """ - pct_agree = np.sum(self.DSEL_processed_, axis=1) / self.n_classifiers_ - - return pct_agree - - def compute_meta_features(self, scores, idx_neighbors, idx_neighbors_op): - """Compute the five sets of meta-features used in the META-DES. Returns - the meta-features vector :math:`v_{i,j}`. - - Parameters - ---------- - scores : array of shape (n_samples, n_classifiers, n_classes) - scores (posterior probabilities) obtained by the base classifiers - for each sample to extract the meta features. - - idx_neighbors : array of shape = [n_samples, self.K] - indices_ of K-nearest neighbors for each example. - - idx_neighbors_op : array of shape = [n_samples, self.Kp] - Indices of the most similar output profiles for each example. - - Returns - ------- - meta_feature_vectors : array of shape [n_query x n_classifiers, - n_meta_features] - The five sets of meta-features estimated for each pair - (base classifier, example). - - """ - - idx_neighbors = np.atleast_2d(idx_neighbors) - idx_neighbors_op = np.atleast_2d(idx_neighbors_op) - - f1_all_classifiers = self.DSEL_processed_[idx_neighbors, :] - f1_all_classifiers = f1_all_classifiers.swapaxes(1, 2) - f1_all_classifiers = f1_all_classifiers.reshape(-1, self.k_) - - f2_all_classifiers =\ - self.dsel_scores_[idx_neighbors, :, - self.DSEL_target_[idx_neighbors]] - - f2_all_classifiers = f2_all_classifiers.swapaxes(1, 2) - - f2_all_classifiers = f2_all_classifiers.reshape(-1, self.k_) - - f3_all_classifiers = np.mean(self.DSEL_processed_[idx_neighbors, :], - axis=1).reshape(-1, 1) - - f4_all_classifiers = self.DSEL_processed_[idx_neighbors_op, :] - f4_all_classifiers = f4_all_classifiers.swapaxes(1, 2) - f4_all_classifiers = f4_all_classifiers.reshape(-1, self.Kp_) - - f5_all_classifiers = np.max(scores, axis=2).reshape(-1, 1) - meta_feature_vectors = np.hstack( - (f1_all_classifiers, f2_all_classifiers, f3_all_classifiers, - f4_all_classifiers, f5_all_classifiers)) - - return meta_feature_vectors - - def _generate_meta_training_set(self): - """Routine to generate the meta-training dataset that is further used - to train the meta-classifier (Lambda). - - In this procedure we use a leave-one-out scheme in which - each sample in DSEL is used as reference to generate the meta-features - and all others are used to estimate the region of competence. - - The first step is to apply the sample selection mechanism in order to - decide whether or not the corresponding sample should be used for - meta-training process. Then, for each base classifier, five sets of - meta-features are calculated and added to the meta-training dataset. - - """ - # first compute the agreement of each sample for - # the sample selection mechanism - agreement = self._sample_selection_agreement() - indices_selected = np.hstack((np.where(self.Hc > agreement)[0], - np.where(agreement > (1 - self.Hc))[0])) - indices_selected = np.unique(indices_selected) - # Get the region of competence using the feature space and - # the decision space. Use K + 1 to later remove itself - # from the set. - _, idx_neighbors = self.get_competence_region( - self.DSEL_data_[indices_selected, :], self.k_ + 1) - _, idx_neighbors_op = self._get_similar_out_profiles( - self.dsel_scores_[indices_selected], self.Kp_ + 1) - # Remove the first neighbor (itself) - idx_neighbors = idx_neighbors[:, 1:] - idx_neighbors_op = idx_neighbors_op[:, 1:] - - # Get the scores for the samples that the meta - # features are being extracted - scores = self.dsel_scores_[indices_selected, :, :] - - # Extract the meta-feature vectors for each base - # classifier. vector and target must both be numpy arrays - meta_feature_vector = self.compute_meta_features(scores, idx_neighbors, - idx_neighbors_op) - meta_feature_target = self.DSEL_processed_[indices_selected, :] - meta_feature_target = meta_feature_target.reshape(-1, ) - meta_feature_target.astype(int) - - return meta_feature_vector, meta_feature_target - - def _fit_meta_classifier(self, X_meta, y_meta): - """Train the meta-classifier :math:`\\lambda`, using - the meta-training dataset. - - Parameters - ---------- - X_meta : array of shape = [n_meta_examples, n_meta_features] - The meta-training examples. - - y_meta : array of shape = [n_meta_examples] - Class labels of each example in X_test. 1 whether the base - classifier made the correct prediction, otherwise 0. - - """ - if isinstance(self.meta_classifier_, MultinomialNB): - # Digitize the data (Same implementation we have on PRTools) - X_meta = np.digitize(X_meta, np.linspace(0.1, 1, 10)) - - self.meta_classifier_.fit(X_meta, y_meta) - - def _get_similar_out_profiles(self, probabilities, kp=None): - """Get the most similar output profiles of the query sample. - - Parameters - ---------- - probabilities : array of shape (n_samples, n_classifiers, n_classes) - Probabilities estimates by each each base classifier for each - sample. - - kp : int - The number of output profiles (most similar) to be selected. - - Returns - ------- - dists : list of shape = [n_samples, k] - The distances between the query and each sample in the region - of competence. The vector is ordered in an ascending fashion. - - idx : list of shape = [n_samples, k] - Indices of the instances belonging to the region of competence - of the given query sample. - """ - if kp is None: - kp = self.Kp_ - - if self.n_classes_ == 2: - # Get only the scores for one class since they are complementary - query_op = probabilities[:, :, 0] - else: - query_op = probabilities.reshape((probabilities.shape[0], - self.n_classifiers_ * - self.n_classes_)) - - dists, idx = self.op_knn_.kneighbors(query_op, n_neighbors=kp, - return_distance=True) - return dists, idx - - def select(self, competences): - """Selects the base classifiers that obtained a competence level higher - than the predefined threshold defined in self.selection_threshold. - - Parameters - ---------- - competences : array of shape (n_samples, n_classifiers) - The competence level estimated for each base classifier and test - example. - - Returns - ------- - selected_classifiers : array of shape (n_samples, n_classifiers) - Boolean matrix containing True if the base classifier is selected, - False otherwise. - - """ - if competences.ndim < 2: - competences = competences.reshape(1, -1) - - selected_classifiers = (competences > self.selection_threshold) - # For the rows that are all False (i.e., no base classifier was - # selected, select all classifiers (all True) - selected_classifiers[~np.any(selected_classifiers, axis=1), :] = True - - return selected_classifiers - - def estimate_competence_from_proba(self, neighbors, probabilities, - distances=None): - """Estimate the competence of each base classifier :math:`c_i` - the classification of the query sample. This method received an array - with the pre-calculated probability estimates for each query. - - First, the meta-features of each base classifier :math:`c_i` for the - classification of the query sample are estimated. These meta-features - are passed down to the meta-classifier :math:`\\lambda` - for the competence level estimation. - - Parameters - ---------- - neighbors : array of shape (n_samples, n_neighbors) - Indices of the k nearest neighbors according for each test sample. - - distances : array of shape (n_samples, n_neighbors) - Distances from the k nearest neighbors to the query. - - probabilities : array of shape (n_samples, n_classifiers, n_classes) - Probabilities estimates obtained by each each base classifier for - each query sample. - - Returns - ------- - competences : array of shape (n_samples, n_classifiers) - The competence level estimated for each base classifier and test - example. - """ - _, idx_neighbors_op = self._get_similar_out_profiles(probabilities) - meta_feature_vectors = self.compute_meta_features(probabilities, - neighbors, - idx_neighbors_op) - - # Digitize the data if a Multinomial NB is used as the meta-classifier - if isinstance(self.meta_classifier_, MultinomialNB): - meta_feature_vectors = np.digitize(meta_feature_vectors, - np.linspace(0.1, 1, 10)) - - # Get the probability for class 1 (Competent) - competences = self.meta_classifier_.predict_proba( - meta_feature_vectors)[:, 1] - - # Reshape the array from 1D [n_samples x n_classifiers] - # to 2D [n_samples, n_classifiers] - competences = competences.reshape(-1, self.n_classifiers_) - - return competences - - def _validate_parameters(self): - """Check if the parameters passed as argument are correct. - - Raises - ------- - ValueError - If any of the hyper-parameters are invalid. - """ - if not isinstance(self.Hc, (float, int)): - raise ValueError( - 'Parameter Hc should be either a number.' - ' Currently Hc = {}'.format(type(self.Hc))) - - if self.Hc < 0.5: - raise ValueError( - 'Parameter Hc should be higher than 0.5.' - ' Currently Hc = {}'.format(self.Hc)) - - if not isinstance(self.selection_threshold, float): - raise ValueError( - 'Parameter Hc should be either a float.' - ' Currently Hc = {}'.format(type(self.Hc))) - - if self.selection_threshold < 0.5: - raise ValueError( - 'Parameter selection_threshold should be higher than 0.5. ' - 'Currently selection_threshold = {}'.format( - self.selection_threshold)) - - if (self.meta_classifier is not None and - not hasattr(self.meta_classifier, "predict_proba")): - - raise ValueError( - "The meta-classifier should output probability estimates") - - if self.Kp is not None: - if not isinstance(self.Kp, int): - raise TypeError("parameter Kp should be an integer.") - if self.Kp <= 0: - raise ValueError("parameter Kp must be equal orhigher than 1." - "input Kp is {} .".format(self.Kp)) - else: - raise ValueError("Parameter Kp is 'None'.") - - super()._validate_parameters() - - def _check_Kp_samples(self): - if self.Kp >= self.n_samples_: - warnings.warn( - "kp is bigger than DSEL size. Using All DSEL" - " examples for competence estimation.", - category=RuntimeWarning) - self.Kp_ = self.n_samples_ - 1 - else: - self.Kp_ = self.Kp diff --git a/deslib/des/probabilistic/__init__.py b/deslib/des/probabilistic/__init__.py deleted file mode 100644 index 2adaffe1..00000000 --- a/deslib/des/probabilistic/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -from .base import BaseProbabilistic -from .deskl import DESKL -from .exponential import Exponential -from .logarithmic import Logarithmic -from .minimum_difference import MinimumDifference -from .rrc import RRC - - -__all__ = ['BaseProbabilistic', - 'DESKL', - 'Exponential', - 'Logarithmic', - 'MinimumDifference', - 'RRC'] diff --git a/deslib/des/probabilistic/base.py b/deslib/des/probabilistic/base.py deleted file mode 100644 index 6cdf86ee..00000000 --- a/deslib/des/probabilistic/base.py +++ /dev/null @@ -1,207 +0,0 @@ -# coding=utf-8 - -# Author: Rafael Menelau Oliveira e Cruz -# -# License: BSD 3 clause - -from abc import abstractmethod, ABCMeta - -import numpy as np - -from deslib.des.base import BaseDES - - -class BaseProbabilistic(BaseDES): - """Base class for a DS method based on the potential function model. - All DS methods based on the Potential function should inherit from this - class. - - Warning: This class should not be used directly. - Use derived classes instead. - - """ - __metaclass__ = ABCMeta - - def __init__(self, pool_classifiers=None, k=None, DFP=False, with_IH=False, - safe_k=None, IH_rate=0.30, mode='selection', voting='hard', - selection_threshold=None, random_state=None, - knn_classifier='knn', knn_metric='minkowski', - DSEL_perc=0.5, n_jobs=-1): - - super(BaseProbabilistic, self).__init__( - pool_classifiers=pool_classifiers, - k=k, - DFP=DFP, - with_IH=with_IH, - safe_k=safe_k, - IH_rate=IH_rate, - mode=mode, - random_state=random_state, - knn_classifier=knn_classifier, - knn_metric=knn_metric, - DSEL_perc=DSEL_perc, - n_jobs=n_jobs, - voting=voting) - - self.selection_threshold = selection_threshold - - def fit(self, X, y): - """Train the DS model by setting the KNN algorithm and - pre-processing the information required to apply the DS - methods. In the case of probabilistic techniques, the source of - competence (C_src) is calculated for each data point in DSEL in order - to speed up the process during the testing phases. - - C_src is estimated with the source_competence() function that is - overridden by each DS method based on this paradigm. - - Parameters - ---------- - X : array of shape (n_samples, n_features) - Data used to fit the model. - - y : array of shape (n_samples) - class labels of each example in X. - - Returns - ------- - self : object - Returns self. - """ - super(BaseProbabilistic, self).fit(X, y) - - if self.n_classes_ == 1: - raise ValueError( - "Error. This class does not accept one class datasets!") - - self._check_predict_proba() - - self.dsel_scores_ = self._predict_proba_base(self.DSEL_data_) - - # Pre process the source of competence for the entire DSEL, - # making the method faster during generalization. - self.C_src_ = self.source_competence() - return self - - def _validate_parameters(self): - """ - Check if the input parameters for potential function based methods are - correct. - """ - if self.knn_classifier == 'knne': - raise ValueError( - "Error, this class does not support the KNN-Equality method" - ) - super(BaseProbabilistic, self)._validate_parameters() - - def estimate_competence(self, competence_region, distances, - predictions=None): - """estimate the competence of each base classifier :math:`c_{i}` - using the source of competence :math:`C_{src}` and the potential - function model. The source of competence :math:`C_{src}` for all - data points in DSEL is already pre-computed in the fit() steps. - - .. math:: \\delta_{i,j} = \\frac{\\sum_{k=1}^{N}C_{src} \\: - exp(-d (\\mathbf{x}_{k}, \\mathbf{x}_{q})^{2})} - {exp( -d (\\mathbf{x}_{k}, \\mathbf{x}_{q})^{2} )} - - Parameters - ---------- - competence_region : array of shape (n_samples, n_neighbors) - Indices of the k nearest neighbors according for each test sample. - - distances : array of shape (n_samples, n_neighbors) - Distances from the k nearest neighbors to the query. - - predictions : array of shape (n_samples, n_classifiers) - Predictions of the base classifiers for all test examples. - - Returns - ------- - competences : array of shape (n_samples, n_classifiers) - Competence level estimated for each base classifier and test - example. - """ - potential_dists = self.potential_func(distances) - potential_dists[potential_dists == 0] = 1e-20 - sum_potential = np.sum(potential_dists, axis=1) - - competences = np.einsum('ijk,ij->ik', - self.C_src_[competence_region, :], - potential_dists) - competences = competences / sum_potential.reshape(-1, 1) - - return competences - - def select(self, competences): - """Selects the base classifiers that obtained a competence level higher - than the predefined threshold. In this case, the threshold indicates - the competence of the random classifier. - - Parameters - ---------- - competences : array of shape (n_samples, n_classifiers) - Competence level estimated for each base classifier and test - example. - - Returns - ------- - selected_classifiers : array of shape (n_samples, n_classifiers) - Boolean matrix containing True if the base classifier is selected, - False otherwise. - - """ - if competences.ndim < 2: - competences = competences.reshape(1, -1) - - # Set the threshold as the performance of the random classifier - if self.selection_threshold is None: - selection_threshold = 1.0 / self.n_classes_ - else: - selection_threshold = self.selection_threshold - - selected_classifiers = (competences > selection_threshold) - # For the rows that are all False (i.e., no base classifier was - # selected, select all classifiers (all True) - selected_classifiers[~np.any(selected_classifiers, axis=1), :] = True - - return selected_classifiers - - @staticmethod - def potential_func(dist): - """Gaussian potential function to decrease the - influence of the source of competence as the distance between - :math:`\\mathbf{x}_{k}` and the query :math:`\\mathbf{x}_{q}` - increases. The function is computed using the following equation: - - .. math:: potential = exp( -dist (\\mathbf{x}_{k}, - \\mathbf{x}_{q})^{2} ) - - where dist represents the Euclidean distance between - :math:`\\mathbf{x}_{k}` and :math:`\\mathbf{x}_{q}` - - Parameters - ---------- - dist : array of shape = [self.n_samples] - distance between the corresponding sample to the query - - Returns - ------- - The result of the potential function for each value in (dist) - """ - return np.exp(- (dist ** 2)) - - @abstractmethod - def source_competence(self): - """ Method used to estimate the source of competence at each data - point. - - Each DS technique based on this paradigm should define its - computation of C_src - - Returns - ---------- - C_src : array of shape (n_samples, n_classifiers) - The competence source for each base classifier at each data point. - """ - pass diff --git a/deslib/des/probabilistic/deskl.py b/deslib/des/probabilistic/deskl.py deleted file mode 100644 index b68175fe..00000000 --- a/deslib/des/probabilistic/deskl.py +++ /dev/null @@ -1,150 +0,0 @@ -import numpy as np - -from deslib.des.probabilistic import BaseProbabilistic -from deslib.util import entropy_func - - -class DESKL(BaseProbabilistic): - """Dynamic Ensemble Selection-Kullback-Leibler divergence (DES-KL). - - This method estimates the competence of the classifier from the information - theory perspective. The competence of the base classifiers is calculated as - the KL divergence between the vector of class supports produced by the base - classifier and the outputs of a random classifier (RC) - RC = 1/L, L being the number of classes in the problem. Classifiers with a - competence higher than the competence of the random classifier is selected. - - Parameters - ---------- - pool_classifiers : list of classifiers (Default = None) - The generated_pool of classifiers trained for the corresponding - classification problem. Each base classifiers should support the method - "predict". If None, then the pool of classifiers is a bagging - classifier. - - k : int (Default = 7) - Number of neighbors used to estimate the competence of the base - classifiers. - - DFP : Boolean (Default = False) - Determines if the dynamic frienemy pruning is applied. - - with_IH : Boolean (Default = False) - Whether the hardness level of the region of competence is used to - decide between using the DS algorithm or the KNN for classification of - a given query sample. - - safe_k : int (default = None) - The size of the indecision region. - - IH_rate : float (default = 0.3) - Hardness threshold. If the hardness level of the competence region is - lower than the IH_rate the KNN classifier is used. Otherwise, the DS - algorithm is used for classification. - - mode : String (Default = "selection") - Whether the technique will perform dynamic selection, - dynamic weighting or an hybrid approach for classification. - - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. - - knn_classifier : {'knn', 'faiss', None} (Default = 'knn') - The algorithm used to estimate the region of competence: - - - 'knn' will use :class:`KNeighborsClassifier` from sklearn - - - 'faiss' will use Facebook's Faiss similarity search through the - class :class:`FaissKNNClassifier` - - - None, will use sklearn :class:`KNeighborsClassifier`. - - knn_metric : {'minkowski', 'cosine', 'mahalanobis'} (Default = 'minkowski') - The metric used by the k-NN classifier to estimate distances. - - - 'minkowski' will use minkowski distance. - - - 'cosine' will use the cosine distance. - - - 'mahalanobis' will use the mahalonibis distance. - - DSEL_perc : float (Default = 0.5) - Percentage of the input data used to fit DSEL. - Note: This parameter is only used if the pool of classifier is None or - unfitted. - - voting : {'hard', 'soft'}, default='hard' - If 'hard', uses predicted class labels for majority rule voting. - Else if 'soft', predicts the class label based on the argmax of - the sums of the predicted probabilities, which is recommended for - an ensemble of well-calibrated classifiers. - - n_jobs : int, default=-1 - The number of parallel jobs to run. None means 1 unless in - a joblib.parallel_backend context. -1 means using all processors. - Doesn’t affect fit method. - - References - ---------- - Woloszynski, Tomasz, et al. "A measure of competence based on random - classification for dynamic ensemble selection." - Information Fusion 13.3 (2012): 207-213. - - Woloszynski, Tomasz, and Marek Kurzynski. "A probabilistic model of - classifier competence for dynamic ensemble selection." - Pattern Recognition 44.10 (2011): 2656-2668. - - R. M. O. Cruz, R. Sabourin, and G. D. Cavalcanti, “Dynamic classifier - selection: Recent advances and perspectives,” - Information Fusion, vol. 41, pp. 195 – 216, 2018. - - """ - - def __init__(self, pool_classifiers=None, k=None, DFP=False, with_IH=False, - safe_k=None, IH_rate=0.30, mode='selection', - random_state=None, knn_classifier='knn', - knn_metric='minkowski', DSEL_perc=0.5, n_jobs=-1, - voting='hard'): - super(DESKL, self).__init__(pool_classifiers=pool_classifiers, - k=k, - DFP=DFP, - with_IH=with_IH, - safe_k=safe_k, - IH_rate=IH_rate, - mode=mode, - random_state=random_state, - knn_classifier=knn_classifier, - knn_metric=knn_metric, - DSEL_perc=DSEL_perc, - n_jobs=n_jobs, - voting=voting) - - self.selection_threshold = 0.0 - - def source_competence(self): - """Calculates the source of competence using the KL divergence method. - - The source of competence C_src at the validation point - :math:`\\mathbf{x}_{k}` is calculated by the KL divergence - between the vector of class supports produced by the base classifier - and the outputs of a random classifier (RC) RC = 1/L, L being the - number of classes in the problem. The value of C_src is negative if - the base classifier misclassified the instance :math:`\\mathbf{x}_{k}`. - - Returns - ---------- - C_src : array of shape (n_samples, n_classifiers) - The competence source for each base classifier at each data point. - """ - - C_src = np.zeros((self.n_samples_, self.n_classifiers_)) - for clf_index in range(self.n_classifiers_): - supports = self.dsel_scores_[:, clf_index, :] - is_correct = self.DSEL_processed_[:, clf_index] - C_src[:, clf_index] = entropy_func(self.n_classes_, supports, - is_correct) - - return C_src diff --git a/deslib/des/probabilistic/exponential.py b/deslib/des/probabilistic/exponential.py deleted file mode 100644 index 595dccc8..00000000 --- a/deslib/des/probabilistic/exponential.py +++ /dev/null @@ -1,147 +0,0 @@ -import numpy as np - -from deslib.des.probabilistic import BaseProbabilistic -from deslib.util import exponential_func - - -class Exponential(BaseProbabilistic): - """The source of competence C_src at the validation point - :math:`\\mathbf{x}_{k}` is a product of two factors: The absolute value of - the competence and the sign. The value of the source competence is - inverse proportional to the normalized entropy of its supports vector. - The sign of competence is simply determined by correct/incorrect - classification of :math:`\\mathbf{x}_{k}` [1]. - - The influence of each sample :math:`\\mathbf{x}_{k}` is defined according - to a Gaussian function model[2]. Samples that are closer to the query have - a higher influence in the competence estimation. - - Parameters - ---------- - pool_classifiers : list of classifiers (Default = None) - The generated_pool of classifiers trained for the corresponding - classification problem. Each base classifiers should support the method - "predict". If None, then the pool of classifiers is a bagging - classifier. - - k : int (Default = 7) - Number of neighbors used to estimate the competence of the base - classifiers. - - DFP : Boolean (Default = False) - Determines if the dynamic frienemy pruning is applied. - - with_IH : Boolean (Default = False) - Whether the hardness level of the region of competence is used to - decide between using the DS algorithm or the KNN for classification of - a given query sample. - - safe_k : int (default = None) - The size of the indecision region. - - IH_rate : float (default = 0.3) - Hardness threshold. If the hardness level of the competence region is - lower than the IH_rate the KNN classifier is used. Otherwise, the DS - algorithm is used for classification. - - - mode : String (Default = "selection") - Whether the technique will perform dynamic selection, - dynamic weighting or an hybrid approach for classification. - - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. - - knn_classifier : {'knn', 'faiss', None} (Default = 'knn') - The algorithm used to estimate the region of competence: - - - 'knn' will use :class:`KNeighborsClassifier` from sklearn - - - 'faiss' will use Facebook's Faiss similarity search through the - class :class:`FaissKNNClassifier` - - - None, will use sklearn :class:`KNeighborsClassifier`. - - knn_metric : {'minkowski', 'cosine', 'mahalanobis'} (Default = 'minkowski') - The metric used by the k-NN classifier to estimate distances. - - - 'minkowski' will use minkowski distance. - - - 'cosine' will use the cosine distance. - - - 'mahalanobis' will use the mahalonibis distance. - - DSEL_perc : float (Default = 0.5) - Percentage of the input data used to fit DSEL. - Note: This parameter is only used if the pool of classifier is None or - unfitted. - - voting : {'hard', 'soft'}, default='hard' - If 'hard', uses predicted class labels for majority rule voting. - Else if 'soft', predicts the class label based on the argmax of - the sums of the predicted probabilities, which is recommended for - an ensemble of well-calibrated classifiers. - - n_jobs : int, default=-1 - The number of parallel jobs to run. None means 1 unless in - a joblib.parallel_backend context. -1 means using all processors. - Doesn’t affect fit method. - - References - ---------- - [1] B. Antosik, M. Kurzynski, New measures of classifier competence - – heuristics and application to the design of multiple classifier systems., - in: Computer recognition systems 4., 2011, pp. 197–206. - - [2] Woloszynski, Tomasz, and Marek Kurzynski. "A probabilistic model of - classifier competence for dynamic ensemble selection." - Pattern Recognition 44.10 (2011): 2656-2668. - - """ - - def __init__(self, pool_classifiers=None, k=None, DFP=False, safe_k=None, - with_IH=False, IH_rate=0.30, mode='selection', - random_state=None, knn_classifier='knn', - knn_metric='minkowski', DSEL_perc=0.5, n_jobs=-1, - voting='hard'): - super(Exponential, self).__init__(pool_classifiers=pool_classifiers, - k=k, - DFP=DFP, - with_IH=with_IH, - safe_k=safe_k, - IH_rate=IH_rate, - mode=mode, - random_state=random_state, - knn_classifier=knn_classifier, - knn_metric=knn_metric, - DSEL_perc=DSEL_perc, - n_jobs=n_jobs, - voting=voting) - - self.selection_threshold = 0 - - def source_competence(self): - """The source of competence C_src at the validation point - :math:`\\mathbf{x}_{k}` is a product of two factors: The absolute - value of the competence and the sign. The value of the source - competence is inverse proportional to the normalized entropy of its - supports vector.The sign of competence is simply determined by - correct/incorrect classification of the instance :math:`\\mathbf{x}_k`. - - Returns - ---------- - C_src : array of shape (n_samples, n_classifiers) - The competence source for each base classifier at each data point. - """ - C_src = np.zeros((self.n_samples_, self.n_classifiers_)) - for clf_index in range(self.n_classifiers_): - supports = self.dsel_scores_[:, clf_index, :] - support_correct = supports[ - np.arange(self.n_samples_), self.DSEL_target_] - - C_src[:, clf_index] = exponential_func(self.n_classes_, - support_correct) - return C_src diff --git a/deslib/des/probabilistic/logarithmic.py b/deslib/des/probabilistic/logarithmic.py deleted file mode 100644 index 0015f8a7..00000000 --- a/deslib/des/probabilistic/logarithmic.py +++ /dev/null @@ -1,134 +0,0 @@ -import numpy as np - -from deslib.des.probabilistic import BaseProbabilistic -from deslib.util import log_func - - -class Logarithmic(BaseProbabilistic): - """ This method estimates the competence of the classifier based on - the logarithmic difference between the supports obtained by the - base classifier. - - Parameters - ---------- - pool_classifiers : list of classifiers (Default = None) - The generated_pool of classifiers trained for the corresponding - classification problem. Each base classifiers should support the method - "predict". If None, then the pool of classifiers is a bagging - classifier. - - k : int (Default = 7) - Number of neighbors used to estimate the competence of the base - classifiers. - - DFP : Boolean (Default = False) - Determines if the dynamic frienemy pruning is applied. - - with_IH : Boolean (Default = False) - Whether the hardness level of the region of competence is used to - decide between using the DS algorithm or the KNN for classification of - a given query sample. - - safe_k : int (default = None) - The size of the indecision region. - - IH_rate : float (default = 0.3) - Hardness threshold. If the hardness level of the competence region is - lower than the IH_rate the KNN classifier is used. Otherwise, the DS - algorithm is used for classification. - - mode : String (Default = "selection") - Whether the technique will perform dynamic selection, - dynamic weighting or an hybrid approach for classification. - - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. - - knn_classifier : {'knn', 'faiss', None} (Default = 'knn') - The algorithm used to estimate the region of competence: - - - 'knn' will use :class:`KNeighborsClassifier` from sklearn - - - 'faiss' will use Facebook's Faiss similarity search through the - class :class:`FaissKNNClassifier` - - - None, will use sklearn :class:`KNeighborsClassifier`. - - knn_metric : {'minkowski', 'cosine', 'mahalanobis'} (Default = 'minkowski') - The metric used by the k-NN classifier to estimate distances. - - - 'minkowski' will use minkowski distance. - - - 'cosine' will use the cosine distance. - - - 'mahalanobis' will use the mahalonibis distance. - - DSEL_perc : float (Default = 0.5) - Percentage of the input data used to fit DSEL. - Note: This parameter is only used if the pool of classifier is None or - unfitted. - - voting : {'hard', 'soft'}, default='hard' - If 'hard', uses predicted class labels for majority rule voting. - Else if 'soft', predicts the class label based on the argmax of - the sums of the predicted probabilities, which is recommended for - an ensemble of well-calibrated classifiers. - - n_jobs : int, default=-1 - The number of parallel jobs to run. None means 1 unless in - a joblib.parallel_backend context. -1 means using all processors. - Doesn’t affect fit method. - - References - ---------- - B. Antosik, M. Kurzynski, New measures of classifier competence - – heuristics and application to the design of - multiple classifier systems., in: Computer recognition systems - 4., 2011, pp. 197–206. - - T.Woloszynski, M. Kurzynski, A measure of competence based on randomized - reference classifier for dynamic ensemble selection, in: International - Conference on Pattern Recognition (ICPR), 2010, pp. 4194–4197. - """ - - def __init__(self, pool_classifiers=None, k=None, DFP=False, with_IH=False, - safe_k=None, IH_rate=0.30, mode='selection', - random_state=None, knn_classifier='knn', - knn_metric='minkowski', DSEL_perc=0.5, n_jobs=-1, - voting='hard'): - super(Logarithmic, self).__init__(pool_classifiers=pool_classifiers, - k=k, - DFP=DFP, - with_IH=with_IH, - safe_k=safe_k, - IH_rate=IH_rate, - mode=mode, - random_state=random_state, - knn_classifier=knn_classifier, - knn_metric=knn_metric, - DSEL_perc=DSEL_perc, - n_jobs=n_jobs, - voting=voting) - - def source_competence(self): - """The source of competence C_src at the validation point - :math:`\\mathbf{x}_{k}` is calculated by - logarithm function in the support obtained by the base classifier. - - Returns - ---------- - C_src : array of shape (n_samples, n_classifiers) - The competence source for each base classifier at each data point. - """ - C_src = np.zeros((self.n_samples_, self.n_classifiers_)) - for clf_index in range(self.n_classifiers_): - supports = self.dsel_scores_[:, clf_index, :] - support_correct = supports[ - np.arange(self.n_samples_), self.DSEL_target_] - - C_src[:, clf_index] = log_func(self.n_classes_, support_correct) - - return C_src diff --git a/deslib/des/probabilistic/minimum_difference.py b/deslib/des/probabilistic/minimum_difference.py deleted file mode 100644 index 3e419d95..00000000 --- a/deslib/des/probabilistic/minimum_difference.py +++ /dev/null @@ -1,149 +0,0 @@ -import numpy as np - -from deslib.des.probabilistic import BaseProbabilistic -from deslib.util import min_difference - - -class MinimumDifference(BaseProbabilistic): - """ - Computes the competence level of the classifiers based on the difference - between the support obtained by each class. The competence level at a data - point :math:`\\mathbf{x}_{k}` is equal to the minimum difference between - the support obtained to the correct class and the support obtained for - different classes. - - The influence of each sample xk is defined according to a Gaussian function - model[2]. Samples that are closer to the query have a higher influence - in the competence estimation. - - Parameters - ---------- - pool_classifiers : list of classifiers (Default = None) - The generated_pool of classifiers trained for the corresponding - classification problem. Each base classifiers should support the method - "predict". If None, then the pool of classifiers is a bagging - classifier. - - k : int (Default = 7) - Number of neighbors used to estimate the competence of the base - classifiers. - - DFP : Boolean (Default = False) - Determines if the dynamic frienemy pruning is applied. - - with_IH : Boolean (Default = False) - Whether the hardness level of the region of competence is used to - decide between using the DS algorithm or the KNN for classification of - a given query sample. - - safe_k : int (default = None) - The size of the indecision region. - - IH_rate : float (default = 0.3) - Hardness threshold. If the hardness level of the competence region is - lower than the IH_rate the KNN classifier is used. Otherwise, the DS - algorithm is used for classification. - - - mode : String (Default = "selection") - Whether the technique will perform dynamic selection, - dynamic weighting or an hybrid approach for classification. - - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. - - knn_classifier : {'knn', 'faiss', None} (Default = 'knn') - The algorithm used to estimate the region of competence: - - - 'knn' will use :class:`KNeighborsClassifier` from sklearn - - - 'faiss' will use Facebook's Faiss similarity search through the - class :class:`FaissKNNClassifier` - - - None, will use sklearn :class:`KNeighborsClassifier`. - - knn_metric : {'minkowski', 'cosine', 'mahalanobis'} (Default = 'minkowski') - The metric used by the k-NN classifier to estimate distances. - - - 'minkowski' will use minkowski distance. - - - 'cosine' will use the cosine distance. - - - 'mahalanobis' will use the mahalonibis distance. - - DSEL_perc : float (Default = 0.5) - Percentage of the input data used to fit DSEL. - Note: This parameter is only used if the pool of classifier is None or - unfitted. - - voting : {'hard', 'soft'}, default='hard' - If 'hard', uses predicted class labels for majority rule voting. - Else if 'soft', predicts the class label based on the argmax of - the sums of the predicted probabilities, which is recommended for - an ensemble of well-calibrated classifiers. - - n_jobs : int, default=-1 - The number of parallel jobs to run. None means 1 unless in - a joblib.parallel_backend context. -1 means using all processors. - Doesn’t affect fit method. - - References - ---------- - [1] B. Antosik, M. Kurzynski, New measures of classifier competence - – heuristics and application to the design of - multiple classifier systems., in: Computer recognition systems - 4., 2011, pp. 197–206. - - [2] Woloszynski, Tomasz, and Marek Kurzynski. "A probabilistic model of - classifier competence for dynamic ensemble selection." - Pattern Recognition 44.10 (2011): 2656-2668. - - """ - def __init__(self, pool_classifiers=None, k=None, DFP=False, with_IH=False, - safe_k=None, IH_rate=0.30, mode='selection', - random_state=None, knn_classifier='knn', - knn_metric='minkowski', DSEL_perc=0.5, n_jobs=-1, - voting='hard'): - - super(MinimumDifference, self).__init__( - pool_classifiers=pool_classifiers, - k=k, - DFP=DFP, - with_IH=with_IH, - safe_k=safe_k, - IH_rate=IH_rate, - mode=mode, - random_state=random_state, - knn_classifier=knn_classifier, - knn_metric=knn_metric, - DSEL_perc=DSEL_perc, - n_jobs=n_jobs, - voting=voting) - - # Threshold is 0 since incompetent classifiers should - # have a negative competence level - self.selection_threshold = 0.0 - - def source_competence(self): - """Calculates the source of competence using the - Minimum Difference method. - - The source of competence C_src at the validation point - :math:`\\mathbf{x}_{k}` calculated by the Minimum Difference between - the supports obtained to the correct class and the support obtained by - the other classes - - Returns - ---------- - C_src : array of shape (n_samples, n_classifiers) - The competence source for each base classifier at each data point. - """ - C_src = np.zeros((self.n_samples_, self.n_classifiers_)) - for clf_index in range(self.n_classifiers_): - supports = self.dsel_scores_[:, clf_index, :] - C_src[:, clf_index] = min_difference(supports, self.DSEL_target_) - - return C_src diff --git a/deslib/des/probabilistic/rrc.py b/deslib/des/probabilistic/rrc.py deleted file mode 100644 index 6371558a..00000000 --- a/deslib/des/probabilistic/rrc.py +++ /dev/null @@ -1,142 +0,0 @@ -import numpy as np - -from deslib.des.probabilistic import BaseProbabilistic -from deslib.util import ccprmod - - -class RRC(BaseProbabilistic): - """DES technique based on the Randomized Reference Classifier method - (DES-RRC). - - Parameters - ---------- - pool_classifiers : list of classifiers (Default = None) - The generated_pool of classifiers trained for the corresponding - classification problem. Each base classifiers should support the method - "predict". If None, then the pool of classifiers is a bagging - classifier. - - k : int (Default = 7) - Number of neighbors used to estimate the competence of the base - classifiers. - - DFP : Boolean (Default = False) - Determines if the dynamic frienemy pruning is applied. - - with_IH : Boolean (Default = False) - Whether the hardness level of the region of competence is used to - decide between using the DS algorithm or the KNN for classification of - a given query sample. - - safe_k : int (default = None) - The size of the indecision region. - - IH_rate : float (default = 0.3) - Hardness threshold. If the hardness level of the competence region is - lower than the IH_rate the KNN classifier is used. Otherwise, the DS - algorithm is used for classification. - - mode : String (Default = "selection") - Whether the technique will perform dynamic selection, - dynamic weighting or an hybrid approach for classification. - - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. - - knn_classifier : {'knn', 'faiss', None} (Default = 'knn') - The algorithm used to estimate the region of competence: - - - 'knn' will use :class:`KNeighborsClassifier` from sklearn - - - 'faiss' will use Facebook's Faiss similarity search through the - class :class:`FaissKNNClassifier` - - - None, will use sklearn :class:`KNeighborsClassifier`. - - knn_metric : {'minkowski', 'cosine', 'mahalanobis'} (Default = 'minkowski') - The metric used by the k-NN classifier to estimate distances. - - - 'minkowski' will use minkowski distance. - - - 'cosine' will use the cosine distance. - - - 'mahalanobis' will use the mahalonibis distance. - - DSEL_perc : float (Default = 0.5) - Percentage of the input data used to fit DSEL. - Note: This parameter is only used if the pool of classifier is None or - unfitted. - - voting : {'hard', 'soft'}, default='hard' - If 'hard', uses predicted class labels for majority rule voting. - Else if 'soft', predicts the class label based on the argmax of - the sums of the predicted probabilities, which is recommended for - an ensemble of well-calibrated classifiers. - - n_jobs : int, default=-1 - The number of parallel jobs to run. None means 1 unless in - a joblib.parallel_backend context. -1 means using all processors. - Doesn’t affect fit method. - - References - ---------- - Woloszynski, Tomasz, and Marek Kurzynski. "A probabilistic model of - classifier competence for dynamic ensemble selection." Pattern Recognition - 44.10 (2011): 2656-2668. - - R. M. O. Cruz, R. Sabourin, and G. D. Cavalcanti, “Dynamic classifier - selection: Recent advances and perspectives,” - Information Fusion, vol. 41, pp. 195 – 216, 2018. - - """ - - def __init__(self, pool_classifiers=None, k=None, DFP=False, with_IH=False, - safe_k=None, IH_rate=0.30, mode='selection', - random_state=None, knn_classifier='knn', - knn_metric='minkowski', DSEL_perc=0.5, n_jobs=-1, - voting='hard'): - - super(RRC, self).__init__(pool_classifiers=pool_classifiers, - k=k, - DFP=DFP, - with_IH=with_IH, - safe_k=safe_k, - IH_rate=IH_rate, - mode=mode, - random_state=random_state, - knn_classifier=knn_classifier, - knn_metric=knn_metric, - DSEL_perc=DSEL_perc, - n_jobs=n_jobs, - voting=voting) - - self.selection_threshold = None - - def source_competence(self): - """ - Calculates the source of competence using the randomized reference - classifier (RRC) method. - - The source of competence C_src at the validation point - :math:`\\mathbf{x}_{k}` calculated using the probabilistic model - based on the supports obtained by the base classifier and - randomized reference classifier (RRC) model. The probabilistic - modeling of the classifier competence is calculated using - the ccprmod function. - - Returns - ---------- - C_src : array of shape (n_samples, n_classifiers) - The competence source for each base classifier at each data point. - """ - c_src = np.zeros((self.n_samples_, self.n_classifiers_)) - - for clf_index in range(self.n_classifiers_): - # Get supports for all samples in DSEL - supports = self.dsel_scores_[:, clf_index, :] - c_src[:, clf_index] = ccprmod(supports, self.DSEL_target_) - - return c_src diff --git a/deslib/static/__init__.py b/deslib/static/__init__.py deleted file mode 100644 index 8b7cc3c6..00000000 --- a/deslib/static/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -""" -The :mod:`deslib.static` provides a set of static ensemble methods which are -often used as a baseline to compare the performance of dynamic selection -algorithms. -""" - -from .oracle import Oracle -from .single_best import SingleBest -from .static_selection import StaticSelection -from .stacked import StackedClassifier - -__all__ = ['Oracle', - 'SingleBest', - 'StaticSelection', - 'StackedClassifier'] diff --git a/deslib/static/base.py b/deslib/static/base.py deleted file mode 100644 index f5fc1487..00000000 --- a/deslib/static/base.py +++ /dev/null @@ -1,146 +0,0 @@ -# coding=utf-8 - -from abc import abstractmethod, ABCMeta - -# Author: Rafael Menelau Oliveira e Cruz -# -# License: BSD 3 clause -import numpy as np -from sklearn.base import BaseEstimator, ClassifierMixin -from sklearn.ensemble import BaseEnsemble, BaggingClassifier -from sklearn.preprocessing import LabelEncoder -from sklearn.utils.validation import check_random_state - - -class BaseStaticEnsemble(BaseEstimator, ClassifierMixin): - """Base class for static ensembles. - - All static ensemble techniques should inherit from this class. - - Warning: This class should not be instantiated directly, use derived - classes instead. - - Parameters - ---------- - pool_classifiers : list of classifiers (Default = None) - The generated_pool of classifiers trained for the corresponding - classification problem. Each base classifiers should support the method - "predict". If None, then the pool of classifiers is a bagging - classifier. - - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. - - n_jobs : int, default=-1 - The number of parallel jobs to run. None means 1 unless in - a joblib.parallel_backend context. -1 means using all processors. - Doesn’t affect fit method. - - References - ---------- - Kuncheva, Ludmila I. Combining pattern classifiers: methods and algorithms. - John Wiley & Sons, 2004. - - R. M. O. Cruz, R. Sabourin, and G. D. Cavalcanti, “Dynamic classifier - selection: Recent advances and perspectives,” - Information Fusion, vol. 41, pp. 195 – 216, 2018. - - """ - __metaclass__ = ABCMeta - - @abstractmethod - def __init__(self, pool_classifiers=None, random_state=None, n_jobs=-1): - self.pool_classifiers = pool_classifiers - self.random_state = random_state - self.n_jobs = n_jobs - - def fit(self, X, y): - """Fit the model according to the given training data. - - Parameters - ---------- - X : array of shape (n_samples, n_features) - Data used to fit the model. - - y : array of shape (n_samples) - class labels of each example in X. - - Returns - ------- - self : object - Returns self. - """ - self.random_state_ = check_random_state(self.random_state) - - # Check if the pool of classifiers is None. If yes, use a - # BaggingClassifier for the pool. - if self.pool_classifiers is None: - self.pool_classifiers_ = BaggingClassifier( - random_state=self.random_state_, n_jobs=self.n_jobs) - self.pool_classifiers_.fit(X, y) - - else: - self.pool_classifiers_ = self.pool_classifiers - - self.n_classifiers_ = len(self.pool_classifiers_) - # allow base models with feature subspaces. - if hasattr(self.pool_classifiers_, "estimators_features_"): - self.estimator_features_ = \ - np.array(self.pool_classifiers_.estimators_features_) - else: - indices = np.arange(X.shape[1]) - self.estimator_features_ = np.tile(indices, - (self.n_classifiers_, 1)) - - self._validate_pool() - # dealing with label encoder - self._check_label_encoder() - self.y_enc_ = self._setup_label_encoder(y) - self.n_classes_ = self.classes_.size - self.n_features_ = X.shape[1] - - return self - - def _check_label_encoder(self): - # Check if base classifiers are not using LabelEncoder (the case for - # scikit-learn's ensembles): - if isinstance(self.pool_classifiers_, BaseEnsemble): - if np.array_equal(self.pool_classifiers_.classes_, - self.pool_classifiers_[0].classes_): - self.base_already_encoded_ = False - else: - self.base_already_encoded_ = True - else: - self.base_already_encoded_ = False - - def _setup_label_encoder(self, y): - """ - Setup the label encoder - """ - self.enc_ = LabelEncoder() - y_ind = self.enc_.fit_transform(y) - self.classes_ = self.enc_.classes_ - - return y_ind - - def _encode_base_labels(self, y): - if self.base_already_encoded_: - return y - else: - return self.enc_.transform(y) - - def _validate_pool(self): - """ Check the estimator and the n_estimator attribute, set the - `base_estimator_` attribute. - - Raises - ------- - ValueError - If the pool of classifiers is empty or just a single model. - """ - if self.n_classifiers_ <= 1: - raise ValueError("n_classifiers must be greater than one, " - "got {}.".format(len(self.pool_classifiers))) diff --git a/deslib/static/oracle.py b/deslib/static/oracle.py deleted file mode 100644 index d1d7662e..00000000 --- a/deslib/static/oracle.py +++ /dev/null @@ -1,168 +0,0 @@ -# coding=utf-8 - -# Author: Rafael Menelau Oliveira e Cruz -# -# License: BSD 3 clause - -import numpy as np -from sklearn.utils.validation import check_X_y, check_array - -from deslib.static.base import BaseStaticEnsemble - - -class Oracle(BaseStaticEnsemble): - """ Abstract method that always selects the base classifier that predicts - the correct label if such classifier exists. This method is often used to - measure the upper-limit performance that can be achieved by a dynamic - classifier selection technique. It is used as a benchmark by several - dynamic selection algorithms. - - Parameters - ---------- - pool_classifiers : list of classifiers (Default = None) - The generated_pool of classifiers trained for the corresponding - classification problem. Each base classifiers should support the method - "predict". If None, then the pool of classifiers is a bagging - classifier. - - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. - - n_jobs : int, default=-1 - The number of parallel jobs to run. None means 1 unless in - a joblib.parallel_backend context. -1 means using all processors. - Doesn’t affect fit method. - - References - ---------- - Kuncheva, Ludmila I. "A theoretical study on six classifier fusion - strategies." IEEE Transactions on Pattern Analysis & Machine Intelligence, - (2002): 281-286. - - R. M. O. Cruz, R. Sabourin, and G. D. Cavalcanti, “Dynamic classifier - selection: Recent advances and perspectives,” - Information Fusion, vol. 41, pp. 195 – 216, 2018. - - """ - def __init__(self, pool_classifiers=None, random_state=None, n_jobs=-1): - super(Oracle, self).__init__(pool_classifiers=pool_classifiers, - random_state=random_state, - n_jobs=n_jobs) - - def fit(self, X, y): - """Fit the model according to the given training data. - - Parameters - ---------- - X : array of shape (n_samples, n_features) - Data used to fit the model. - - y : array of shape (n_samples) - class labels of each example in X. - - Returns - ------- - self : object - Returns self. - """ - X, y = self._validate_data( - X, - y, - accept_sparse="csr", - dtype=np.float64, - order="C", - accept_large_sparse=False, - ) - super(Oracle, self).fit(X, y) - return self - - def predict(self, X, y): - """Prepare the labels using the Oracle model. - - Parameters - ---------- - X : array of shape (n_samples, n_features) - The data to be classified - - y : array of shape (n_samples) - Class labels of each sample in X. - - Returns - ------- - predicted_labels : array of shape (n_samples) - Predicted class for each sample in X. - """ - X = check_array(X) - if self.n_features_ != X.shape[1]: - raise ValueError("Number of features of the model must " - "match the input. Model n_features is {0} and " - "input n_features is {1}." - "".format(self.n_features_, X.shape[1])) - - y = self.enc_.transform(y) - preds = [clf.predict(X[:, self.estimator_features_[idx]]) - for idx, clf in enumerate(self.pool_classifiers_)] - preds = np.asarray(preds).T - hit_miss = np.asarray(preds) == y.reshape(-1, 1) - idx_sel_classifier = hit_miss.argmax(axis=1) - predicted_labels = preds[np.arange(preds.shape[0]), idx_sel_classifier] - - return self.classes_.take(predicted_labels.astype(int)) - - def predict_proba(self, X, y): - """Estimates the posterior probabilities for each class for each sample - in X. - - Note that as the Oracle is the ideal classifier selection, the - classifier that estimate the highest probability for the correct class - is the selected one. - - Parameters - ---------- - X : array of shape (n_samples, n_features) - The data to be classified. - - y : array of shape (n_samples) - Class labels of each sample in X. - - Returns - ------- - probas : array of shape (n_samples, n_classes) - Posterior probabilities estimates for each class. - - """ - X = check_array(X) - y = self.enc_.transform(y) - - probas = [clf.predict_proba(X[:, self.estimator_features_[idx]]) - for idx, clf in enumerate(self.pool_classifiers_)] - probas = np.array(probas).transpose((1, 0, 2)) - best_probas_ids = np.argmax(probas[np.arange(y.size), :, y], axis=1) - return probas[np.arange(y.size), best_probas_ids, :] - - def score(self, X, y, sample_weights=None): - """ Return the mean accuracy on the given test data and labels. - - Parameters - ---------- - X : array of shape (n_samples, n_features) - The data to be classified. - - y : array of shape (n_samples) - Class labels of each sample in X. - - sample_weight : array-like of shape (n_samples,), default=None - Sample weights. - - Returns - ------- - accuracy : float - Classification accuracy of the Oracle model. - """ - from sklearn.metrics import accuracy_score - accuracy = accuracy_score(y, self.predict(X, y), - sample_weight=sample_weights) - return accuracy diff --git a/deslib/static/single_best.py b/deslib/static/single_best.py deleted file mode 100644 index 6edb5c1f..00000000 --- a/deslib/static/single_best.py +++ /dev/null @@ -1,170 +0,0 @@ -# coding=utf-8 - -# Author: Rafael Menelau Oliveira e Cruz -# -# License: BSD 3 clause - -import numpy as np -from sklearn.metrics import check_scoring -from sklearn.utils.validation import check_X_y, check_is_fitted, check_array - -from .base import BaseStaticEnsemble - - -class SingleBest(BaseStaticEnsemble): - """Classification method that selects the classifier in the pool with - highest score to be used for classification. Usually, the performance of - the single best classifier is estimated based on the validation data. - - Parameters - ---------- - pool_classifiers : list of classifiers (Default = None) - The generated_pool of classifiers trained for the corresponding - classification problem. Each base classifiers should support the method - "predict". If None, then the pool of classifiers is a bagging - classifier. - - scoring : string, callable (default = None) - A single string or a callable to evaluate the predictions on the - validation set. - - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. - - n_jobs : int, default=-1 - The number of parallel jobs to run. None means 1 unless in - a joblib.parallel_backend context. -1 means using all processors. - Doesn’t affect fit method. - - References - ---------- - Britto, Alceu S., Robert Sabourin, and Luiz ES Oliveira. "Dynamic selection - of classifiers—a comprehensive review." - Pattern Recognition 47.11 (2014): 3665-3680. - - Kuncheva, Ludmila I. Combining pattern classifiers: methods and algorithms. - John Wiley & Sons, 2004. - - R. M. O. Cruz, R. Sabourin, and G. D. Cavalcanti, “Dynamic classifier - selection: Recent advances and perspectives,” - Information Fusion, vol. 41, pp. 195 – 216, 2018. - """ - - def __init__(self, pool_classifiers=None, scoring=None, - random_state=None, n_jobs=-1): - super(SingleBest, self).__init__(pool_classifiers=pool_classifiers, - random_state=random_state, - n_jobs=n_jobs) - self.scoring = scoring - - def fit(self, X, y): - """Fit the model by selecting the base classifier with the highest - accuracy in the dataset. The single best classifier is kept in - self.best_clf and its index is kept in self.best_clf_index. - - Parameters - ---------- - X : array of shape (n_samples, n_features) - Data used to fit the model. - - y : array of shape (n_samples) - class labels of each example in X. - - """ - X, y = self._validate_data( - X, - y, - accept_sparse="csr", - dtype=np.float64, - order="C", - accept_large_sparse=False, - ) - - super(SingleBest, self).fit(X, y) - - if not self.base_already_encoded_: - y_encoded = y - else: - y_encoded = self.enc_.transform(y) - - performances = self._estimate_performances(X, y_encoded) - self.best_clf_index_ = np.argmax(performances) - self.best_clf_ = self.pool_classifiers_[self.best_clf_index_] - - return self - - def _estimate_performances(self, X, y): - performances = np.zeros(self.n_classifiers_) - for idx, clf in enumerate(self.pool_classifiers_): - scorer = check_scoring(clf, self.scoring) - performances[idx] = scorer(clf, - X[:, self.estimator_features_[idx]], y) - return performances - - def predict(self, X): - """Predict the label of each sample in X and returns the predicted - label. - - Parameters - ---------- - X : array of shape (n_samples, n_features) - The data to be classified - - Returns - ------- - predicted_labels : array of shape (n_samples) - Predicted class for each sample in X. - """ - self._check_is_fitted() - X = check_array(X) - if self.n_features_ != X.shape[1]: - raise ValueError("Number of features of the model must " - "match the input. Model n_features is {0} and " - "input n_features is {1}." - "".format(self.n_features_, X.shape[1])) - - predictions = self.best_clf_.predict( - X[:, self.estimator_features_[self.best_clf_index_]]) - - predictions = self._encode_base_labels(predictions) - return self.classes_.take(predictions.astype(int)) - - def predict_proba(self, X): - """Estimates the posterior probabilities for each class for each sample - in X. The returned probability estimates for all classes are ordered by - the label of classes. - - Parameters - ---------- - X : array of shape (n_samples, n_features) - The data to be classified - - Returns - ------- - predicted_proba : array of shape (n_samples, n_classes) - Posterior probabilities estimates for each class. - - """ - self._check_is_fitted() - if "predict_proba" not in dir(self.best_clf_): - raise ValueError( - "Base classifier must support the predict_proba function.") - X = check_array(X) - if self.n_features_ != X.shape[1]: - raise ValueError("Number of features of the model must " - "match the input. Model n_features is {0} and " - "input n_features is {1}." - "".format(self.n_features_, X.shape[1])) - - predicted_proba = self.best_clf_.predict_proba( - X[:, self.estimator_features_[self.best_clf_index_]]) - return predicted_proba - - def _check_is_fitted(self): - """Verify if the estimator algorithm was fitted. Raises an error if it - is not fitted. - """ - check_is_fitted(self, "best_clf_") diff --git a/deslib/static/stacked.py b/deslib/static/stacked.py deleted file mode 100644 index f24b22ed..00000000 --- a/deslib/static/stacked.py +++ /dev/null @@ -1,212 +0,0 @@ -import numpy as np -from sklearn.linear_model import LogisticRegression -from sklearn.utils.validation import check_X_y, check_array, check_is_fitted - -from deslib.static.base import BaseStaticEnsemble - - -class StackedClassifier(BaseStaticEnsemble): - """ A Stacking classifier. - - Parameters - ---------- - pool_classifiers : list of classifiers (Default = None) - The generated_pool of classifiers trained for the corresponding - classification problem. Each base classifiers should support the method - "predict" and "predict_proba". If None, then the pool of classifiers - is a bagging classifier. - - meta_classifier : object or None, optional (default=None) - Classifier model used to aggregate the output of the base classifiers. - If None, a :class:`LogisticRegression` classifier is used. - - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. - - passthrough : bool (default=False) - When False, only the predictions of estimators will be used as - training data for the meta-classifier. When True, the - meta-classifier is trained on the predictions as well as the - original training data. - - n_jobs : int, default=-1 - The number of parallel jobs to run. None means 1 unless in - a joblib.parallel_backend context. -1 means using all processors. - Doesn’t affect fit method. - - References - ---------- - Wolpert, David H. "Stacked generalization." Neural networks 5, - no. 2 (1992): 241-259. - - Kuncheva, Ludmila I. Combining pattern classifiers: methods and algorithms. - John Wiley & Sons, 2004. - """ - - def __init__(self, pool_classifiers=None, meta_classifier=None, - passthrough=False, random_state=None, n_jobs=-1): - - super(StackedClassifier, self).__init__( - pool_classifiers=pool_classifiers, - random_state=random_state, - n_jobs=n_jobs - ) - self.meta_classifier = meta_classifier - self.passthrough = passthrough - - def fit(self, X, y): - """Fit the model by training a meta-classifier on the outputs of the - base classifiers - - Parameters - ---------- - X : array of shape (n_samples, n_features) - Data used to fit the model. - - y : array of shape (n_samples) - class labels of each example in X. - - """ - X, y = self._validate_data( - X, - y, - accept_sparse="csr", - dtype=np.float64, - order="C", - accept_large_sparse=False, - ) - super(StackedClassifier, self).fit(X, y) - base_preds = self._predict_proba_base(X) - X_meta = self._connect_input(X, base_preds) - - # Prepare the meta-classifier - if self.meta_classifier is None: - self.meta_classifier_ = LogisticRegression( - solver='lbfgs', - multi_class='auto', - max_iter=1000, - random_state=self.random_state_) - - else: - self.meta_classifier_ = self.meta_classifier - - self.meta_classifier_.fit(X_meta, self.y_enc_) - - return self - - def predict(self, X): - """Predict the label of each sample in X and returns the predicted - label. - - Parameters - ---------- - X : array of shape (n_samples, n_features) - The data to be classified - - Returns - ------- - predicted_labels : array of shape (n_samples) - Predicted class for each sample in X. - """ - check_is_fitted(self, "meta_classifier_") - X = check_array(X) - if self.n_features_ != X.shape[1]: - raise ValueError("Number of features of the model must " - "match the input. Model n_features is {0} and " - "input n_features is {1}." - "".format(self.n_features_, X.shape[1])) - - base_preds = self._predict_proba_base(X) - X_meta = self._connect_input(X, base_preds) - preds = self.meta_classifier_.predict(X_meta) - return self.classes_.take(preds) - - def predict_proba(self, X): - """Predict the label of each sample in X and returns the predicted - label. - - Parameters - ---------- - X : array of shape (n_samples, n_features) - The data to be classified - - Returns - ------- - predicted_labels : array of shape (n_samples) - Predicted class for each sample in X. - """ - check_is_fitted(self, "meta_classifier_") - X = check_array(X) - if self.n_features_ != X.shape[1]: - raise ValueError("Number of features of the model must " - "match the input. Model n_features is {0} and " - "input n_features is {1}." - "".format(self.n_features_, X.shape[1])) - - # Check if the meta-classifier can output probabilities - if not hasattr(self.meta_classifier_, "predict_proba"): - raise ValueError("Meta-classifier does not implement the" - " predict_proba method.") - - base_preds = self._predict_proba_base(X) - X_meta = self._connect_input(X, base_preds) - - return self.meta_classifier_.predict_proba(X_meta) - - def _connect_input(self, X, base_preds): - if self.passthrough: - X_meta = np.hstack((base_preds, X)) - else: - X_meta = base_preds - return X_meta - - def _predict_proba_base(self, X): - """ Get the predictions (probabilities) of each base classifier in the - pool for all samples in X. - - Parameters - ---------- - X : array of shape (n_samples, n_features) - The test examples. - - Returns - ------- - probabilities : array of shape = [n_samples, n_classifiers x n_classes] - Probabilities estimates of each base classifier for all - test samples. - """ - # Check if base classifiers implement the predict proba method. - self._check_predict_proba() - - probas = np.zeros( - (X.shape[0], self.n_classifiers_, self.n_classes_)) - - for index, clf in enumerate(self.pool_classifiers_): - probas[:, index] = clf.predict_proba( - X[:, self.estimator_features_[index]]) - - probas = probas.reshape(X.shape[0], - self.n_classifiers_ * self.n_classes_) - - # remove first column as both features are collinear. - if self.n_classes_ == 2: - probas = probas[:, ::2] - - return probas - - def _check_predict_proba(self): - """ Checks if each base classifier in the pool implements the - predict_proba method. - - Raises - ------ - ValueError - If the base classifiers do not implements the predict_proba method. - """ - for clf in self.pool_classifiers_: - if "predict_proba" not in dir(clf): - raise ValueError( - "All base classifiers should output probability estimates") diff --git a/deslib/static/static_selection.py b/deslib/static/static_selection.py deleted file mode 100644 index cc0ef95e..00000000 --- a/deslib/static/static_selection.py +++ /dev/null @@ -1,207 +0,0 @@ -# coding=utf-8 - -# Author: Rafael Menelau Oliveira e Cruz -# -# License: BSD 3 clause - -import numpy as np -from sklearn.metrics import check_scoring -from sklearn.utils.validation import check_is_fitted, check_X_y, check_array - -from deslib.util.aggregation import majority_voting_rule -from deslib.util.aggregation import predict_proba_ensemble -from .base import BaseStaticEnsemble - - -class StaticSelection(BaseStaticEnsemble): - """Ensemble model that selects N classifiers with the best performance in a - dataset - - Parameters - ---------- - pool_classifiers : list of classifiers (Default = None) - The generated_pool of classifiers trained for the corresponding - classification problem. Each base classifiers should support the method - "predict". If None, then the pool of classifiers is a bagging - classifier. - - scoring : string, callable (default = None) - A single string or a callable to evaluate the predictions on the - validation set. - - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. - - pct_classifiers : float (Default = 0.5) - Percentage of base classifier that should be selected by the selection - scheme. - - n_jobs : int, default=-1 - The number of parallel jobs to run. None means 1 unless in - a joblib.parallel_backend context. -1 means using all processors. - Doesn’t affect fit method. - - References - ---------- - Britto, Alceu S., Robert Sabourin, and Luiz ES Oliveira. "Dynamic selection - of classifiers—a comprehensive review." - Pattern Recognition 47.11 (2014): 3665-3680. - - Kuncheva, Ludmila I. Combining pattern classifiers: methods and algorithms. - John Wiley & Sons, 2004. - - R. M. O. Cruz, R. Sabourin, and G. D. Cavalcanti, “Dynamic classifier - selection: Recent advances and perspectives,” - Information Fusion, vol. 41, pp. 195 – 216, 2018. - """ - - def __init__(self, pool_classifiers=None, pct_classifiers=0.5, - scoring=None, random_state=None, n_jobs=-1): - - super(StaticSelection, self).__init__( - pool_classifiers=pool_classifiers, - random_state=random_state, - n_jobs=n_jobs) - self.pct_classifiers = pct_classifiers - self.scoring = scoring - - def fit(self, X, y): - """Fit the static selection model by select an ensemble of classifier - containing the base classifiers with highest accuracy in the given - dataset. - - Parameters - ---------- - X : array of shape (n_samples, n_features) - Data used to fit the model. - - y : array of shape (n_samples) - class labels of each example in X. - - Returns - ------- - self : object - Returns self. - """ - X, y = self._validate_data( - X, - y, - accept_sparse="csr", - dtype=np.float64, - order="C", - accept_large_sparse=False, - ) - self._validate_parameters() - - super(StaticSelection, self).fit(X, y) - - self.n_classifiers_ensemble_ = int( - self.n_classifiers_ * self.pct_classifiers) - - performances = np.zeros(self.n_classifiers_) - - if not self.base_already_encoded_: - y_encoded = y - else: - y_encoded = self.enc_.transform(y) - - for clf_idx, clf in enumerate(self.pool_classifiers_): - scorer = check_scoring(clf, self.scoring) - performances[clf_idx] = scorer(clf, X[:, self.estimator_features_[ - clf_idx]], y_encoded) - - self.clf_indices_ = np.argsort(performances)[::-1][ - 0:self.n_classifiers_ensemble_] - self.ensemble_ = [self.pool_classifiers_[clf_idx] for clf_idx in - self.clf_indices_] - - return self - - def predict(self, X): - """Predict the label of each sample in X and returns the predicted - label. - - Parameters - ---------- - X : array of shape (n_samples, n_features) - The data to be classified - - Returns - ------- - predicted_labels : array of shape (n_samples) - Predicted class for each sample in X. - """ - self._check_is_fitted() - X = check_array(X) - if self.n_features_ != X.shape[1]: - raise ValueError("Number of features of the model must " - "match the input. Model n_features is {0} and " - "input n_features is {1}." - "".format(self.n_features_, X.shape[1])) - - votes = np.zeros((X.shape[0], self.n_classifiers_ensemble_)) - for idx, clf in enumerate(self.ensemble_): - X_space = X[:, self.estimator_features_[self.clf_indices_[idx]]] - votes[:, idx] = self._encode_base_labels(clf.predict(X_space)) - - predicted_labels = majority_voting_rule(votes).astype(int) - - return self.classes_.take(predicted_labels) - - def predict_proba(self, X): - """Estimates the posterior probabilities for sample in X. - - Parameters - ---------- - X : array of shape (n_samples, n_features) - The input data. - - Returns - ------- - predicted_proba : array of shape (n_samples, n_classes) - Probabilities estimates for each sample in X. - """ - self._check_is_fitted() - X = check_array(X) - if self.n_features_ != X.shape[1]: - raise ValueError("Number of features of the model must " - "match the input. Model n_features is {0} and " - "input n_features is {1}." - "".format(self.n_features_, X.shape[1])) - - self._check_predict_proba() - features = self.estimator_features_[self.clf_indices_] - proba = predict_proba_ensemble(self.ensemble_, X, features) - return proba - - def _validate_parameters(self): - - if not isinstance(self.pct_classifiers, float): - raise TypeError('pct_classifiers should be a float.') - if self.pct_classifiers > 1 or self.pct_classifiers < 0: - raise ValueError( - 'The parameter pct_classifiers should be a number ' - 'between 0 and 1.') - - def _check_is_fitted(self): - """Verify if the estimator algorithm was fitted. Raises an error if it - is not fitted. - """ - check_is_fitted(self, "ensemble_") - - def _check_predict_proba(self): - """ Checks if each base classifier in the ensemble (selected models) - implements the predict_proba method. - - Raises - ------- - ValueError - If the base classifiers do not implements the predict_proba method. - """ - for clf in self.ensemble_: - if "predict_proba" not in dir(clf): - raise ValueError( - "All base classifiers should output probability estimates") diff --git a/deslib/tests/__init__.py b/deslib/tests/__init__.py deleted file mode 100644 index 9bd34cf0..00000000 --- a/deslib/tests/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .conftest import * diff --git a/deslib/tests/conftest.py b/deslib/tests/conftest.py deleted file mode 100644 index 56c449e4..00000000 --- a/deslib/tests/conftest.py +++ /dev/null @@ -1,178 +0,0 @@ -import pytest -import numpy as np -from unittest.mock import MagicMock -from sklearn.tree import DecisionTreeClassifier - - -@pytest.fixture -def example_estimate_competence(create_X_y): - X, y = create_X_y - - # Pre-processed results on DSEL. This information is used by the majority - # of DS techniques to estimate the classifier competence. - dsel_processed = np.array( - [[1, 1, 1], [0, 0, 1], [0, 0, 1], [1, 1, 0], [1, 1, 1], [0, 0, 1], - [0, 0, 0], [1, 1, 1], - [1, 1, 0], [0, 0, 1], [0, 0, 1], [0, 1, 0], [0, 1, 0], [1, 1, 1], - [1, 1, 1]]) - - # pre-calculated indices of 7 Nearest neighbors for competence estimation. - neighbors = np.array([[8, 11, 4, 7, 13, 10, 1], - [7, 1, 11, 13, 0, 8, 4], - [5, 3, 4, 8, 10, 11, 7]]) - - # Scores obtained for the two classes. This information is used by the - # techniques based on posterior probabilities - dsel_scores = np.tile(np.array([[1.0, 0.0], [0.5, 0.5], [0.8, 0.2]]), - (15, 1, 1)) - - # Distance information is used by the probabilistic techniques - # (des.probabilistic) as well as the MLA, A Priori and A Posteriori methods - distances = np.array([[0.35355339, 0.35355339, 0.55901699, 0.79056942, - 0.79056942, 0.90138782, 1.03077641], - [0.3, 0.39051248, 0.53851648, 0.86023253, 0.94339811, - 1.04403065, 1.28549601], - [0.70710678, 1.41421356, 1.95256242, 2.12132034, - 2.79508497, 2.82842712, 2.91547595]]) - - return X, y, neighbors, distances, dsel_processed, dsel_scores - - -@pytest.fixture -def create_X_y(): - # ex1: The distribution of samples of a test example. - X = np.array([[-1, 1], [-0.75, 0.5], [-1.5, 1.5], - [1, 1], [0.75, 0.5], [1.5, 1.5], - [1, -1], [-0.5, 0.5], [0.5, 0.5], - [0, -1], [0.75, -0.5], [0.0, 0.0], - [-1, -1], [0, -0.5], [1, -1]]) - # Labels associated with the samples. This information is used - # by techniques based on a posteriori information. - y = np.array([0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0]) - return X, y - - -# ----- Test Example all ones ------ -@pytest.fixture -def example_all_ones(example_estimate_competence): - X, y, neighbors = example_estimate_competence[0:3] - dsel_processed = np.ones((15, 3)) - dsel_scores = np.ones((15, 3, 2)) - distances = np.ones((3, 7)) - - return X, y, neighbors, distances, dsel_processed, dsel_scores - - -# ----- Test Example from Combining pattern classifiers ------ -# This example is used to test the results of the A priori, -# A posteriori and MLA techniques -@pytest.fixture -def example_kuncheva(): - distances = np.linspace(1, 15, num=15) - distances = distances.reshape(1, -1) - - # 10 neighbors used in the example - neighbors = np.linspace(0, 14, num=15, dtype=int) - - # target class of each example. independent means that it should be - # used by the a priori technique - y_independent = np.array([2, 1, 2, 2, 3, 1, 2, 1, 3, 3, 2, 1, 2, 2, 1]) - 1 - - # dependent means that it should be used by the a posteriori technique - y_dependent = np.array([1, 0, 1, 1, 2, 0, 1, 0, 0, 2, 1, 2, 1, 1, 0]) - - # Predictions of the base classifier ci. Used to estimate its competence - # level for the A Posteriori - classifier_pred = np.array( - [2, 3, 2, 2, 1, 1, 2, 2, 3, 3, 1, 2, 2, 2, 1]) - 1 - - # whether or not the base classifier made the correct prediction for each - # sample in dsel - dsel_processed = np.transpose( - np.array([[1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 0, 1, 1, 1]])) - - # In this example we consider that the posteriori is always 1 for the - # predicted class (15 samples, 1 classifier, 3 classes) - dsel_scores = np.array([[[0.0, 1.0, 0.0], - [0.0, 0.0, 1.0], - [0.0, 1.0, 0.0], - [0.0, 1.0, 0.0], - [1.0, 0.0, 0.0], - [1.0, 0.0, 0.0], - [0.0, 1.0, 0.0], - [0.0, 1.0, 0.0], - [0.0, 0.0, 1.0], - [0.0, 0.0, 1.0], - [1.0, 0.0, 0.0], - [0.0, 1.0, 0.0], - [0.0, 1.0, 0.0], - [0.0, 1.0, 0.0], - [1.0, 0.0, 0.0]]]).reshape(15, 1, 3) - - k = 15 - n_classes = 3 - dict_return = {"dsel_processed": dsel_processed, - "dsel_scores": dsel_scores, - "distances": distances, - "neighbors": neighbors, - "classifier_pred": classifier_pred, - "y_dependent": y_dependent, - "y_independent": y_independent, - "n_classes": n_classes, - "k": k} - - return dict_return - - -# ----- Routines to generate a pool of classifiers using MagicMock ------ -def create_base_classifier(return_value, return_prob=None): - classifier = MagicMock() - classifier.predict.return_value = return_value - classifier.predict_proba.return_value = return_prob - return classifier - - -@pytest.fixture -def create_pool_classifiers(): - clf_0 = create_base_classifier(return_value=np.zeros(1), - return_prob=np.array([[0.5, 0.5]])) - clf_1 = create_base_classifier(return_value=np.ones(1), - return_prob=np.array([[1.0, 0.0]])) - clf_2 = create_base_classifier(return_value=np.zeros(1), - return_prob=np.array([[0.33, 0.67]])) - pool_classifiers = [clf_0, clf_1, clf_2] - return pool_classifiers - - -@pytest.fixture -def create_pool_all_agree(): - return [create_base_classifier(return_value=np.zeros(1), - return_prob=np.array([[0.61, 0.39]]))] * 100 - - -@pytest.fixture -def example_static_selection(create_X_y): - X, y = create_X_y - n_samples = y.size - pool1 = [create_base_classifier( - return_value=np.zeros(n_samples), - return_prob=np.tile([0.49, 0.51], (n_samples, 1)))] * 50 - - pool2 = [create_base_classifier( - return_value=np.ones(X.shape[0]), - return_prob=np.tile([0.52, 0.48], (n_samples, 1)))] * 50 - for clf in pool1: - clf.score = MagicMock(return_value=0.5) - for clf in pool2: - clf.score = MagicMock(return_value=1.0) - - pool = pool1 + pool2 - return X, y, pool - - -@pytest.fixture -def create_label_encoder_test(): - y = ['one', 'one', 'one', 'zero', 'zero', 'two'] - X = np.random.rand(6, 3) - pool = [DecisionTreeClassifier().fit(X, y) for _ in range(5)] - return X, y, pool diff --git a/deslib/tests/dcs/__init__.py b/deslib/tests/dcs/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/deslib/tests/dcs/test_a_posteriori.py b/deslib/tests/dcs/test_a_posteriori.py deleted file mode 100644 index 37411fed..00000000 --- a/deslib/tests/dcs/test_a_posteriori.py +++ /dev/null @@ -1,141 +0,0 @@ -from unittest.mock import MagicMock - -import numpy as np -import pytest -from sklearn.linear_model import Perceptron -from sklearn.utils.estimator_checks import check_estimator - -from deslib.dcs.a_posteriori import APosteriori - - -def test_check_estimator(): - check_estimator(APosteriori(selection_method='best')) - - -# Should always be 1.0 since the supports for the correct class is always 1. -@pytest.mark.parametrize('index', [0, 1, 2]) -def test_estimate_competence_all_ones(index, example_all_ones): - _, y, neighbors, distances, dsel_processed, dsel_scores = example_all_ones - - query = np.atleast_2d([1, 1]) - - a_posteriori_test = APosteriori() - a_posteriori_test.n_classifiers_ = 3 - a_posteriori_test.DSEL_processed_ = dsel_processed - a_posteriori_test.dsel_scores_ = dsel_scores - a_posteriori_test.DSEL_target_ = y - - neighbors = neighbors[index, :].reshape(1, -1) - distances = distances[index, :].reshape(1, -1) - - expected = [1.0, 1.0, 1.0] - predictions = np.array([0, 1, 0]) - - competences = a_posteriori_test.estimate_competence(neighbors, - distances, - predictions=np.array( - predictions)) - assert np.isclose(competences, expected).all() - - -# Testing example from kuncheva's book (combining pattern classifiers) -def test_estimate_competence_kuncheva_ex(example_kuncheva): - query = np.atleast_2d([1, 1]) - - a_posteriori_test = APosteriori(k=example_kuncheva['k']) - a_posteriori_test.n_classifiers_ = 1 - - a_posteriori_test.DSEL_processed_ = example_kuncheva['dsel_processed'] - a_posteriori_test.dsel_scores_ = example_kuncheva['dsel_scores'] - a_posteriori_test.DSEL_target_ = example_kuncheva['y_dependent'] - a_posteriori_test.n_classes_ = example_kuncheva['n_classes'] - - neighbors = example_kuncheva['neighbors'].reshape(1, -1) - distances = example_kuncheva['distances'].reshape(1, -1) - - predictions = np.array([[1]]) - - competences = a_posteriori_test.estimate_competence(neighbors, - distances, - predictions=np.array( - predictions)) - assert np.isclose(competences, 0.95, atol=0.01) - - -# Testing example from kuncheva's book (combining pattern classifiers) -def test_estimate_competence_kuncheva_ex_batch(example_kuncheva): - # considering a batch composed of 10 samples - query = np.ones((10, 2)) - classifier = MagicMock() - classifier.predict.return_value = [1] - classifier.predict_proba.return_value = None - - a_posteriori_test = APosteriori(pool_classifiers=classifier, - k=example_kuncheva['k']) - - a_posteriori_test.n_classifiers_ = 1 - a_posteriori_test.DSEL_processed_ = example_kuncheva['dsel_processed'] - a_posteriori_test.DSEL_target_ = example_kuncheva['y_dependent'] - a_posteriori_test.dsel_scores_ = example_kuncheva['dsel_scores'] - a_posteriori_test.n_classes_ = example_kuncheva['n_classes'] - - # repeating the same matrix in a new axis to simulate a batch input. - neighbors = example_kuncheva['neighbors'] - distances = example_kuncheva['distances'] - - predictions = [1] - competences = a_posteriori_test.estimate_competence(neighbors, - distances, - predictions=np.array( - predictions)) - assert np.allclose(competences, 0.95, atol=0.01) - - -# in this test case, the target of the neighbors is always different -# than the predicted. So -# the estimation of competence should always be zero -@pytest.mark.parametrize('index', [0, 1, 2]) -def test_estimate_competence_diff_target(index, example_all_ones): - _, _, neighbors, distances, dsel_processed, _ = example_all_ones - - query = np.atleast_2d([1, 1]) - a_posteriori_test = APosteriori() - a_posteriori_test.n_classifiers_ = 3 - a_posteriori_test.DSEL_processed_ = dsel_processed - a_posteriori_test.dsel_scores_ = np.ones((15, 3, 3)) - a_posteriori_test.DSEL_target_ = np.ones(15, dtype=int) * 2 - a_posteriori_test.n_classes_ = 2 - - neighbors = neighbors[index, :].reshape(1, -1) - distances = distances[index, :].reshape(1, -1) - - expected = [0.0, 0.0, 0.0] - - predictions = np.array([0, 1, 0]) - competences = a_posteriori_test.estimate_competence(neighbors, - distances, - predictions=np.array( - predictions)) - assert np.isclose(competences, expected).all() - - -# Check if the fit method is pre-calculating the classifier scores correctly -def test_fit(create_X_y, create_pool_classifiers): - X, y = create_X_y - a_posteriori_test = APosteriori(create_pool_classifiers) - a_posteriori_test.fit(X, y) - expected = np.array([[0.5, 0.5], [1.0, 0.0], [0.33, 0.67]]) - expected = np.tile(expected, (15, 1, 1)) - assert np.array_equal(a_posteriori_test.dsel_scores_, expected) - - -# Test if the class is raising an error when the base classifiers do not -# implements the predict_proba method. Should raise an exception when the -# base classifier cannot estimate posterior probabilities (predict_proba) -# Using Perceptron classifier as it does not implements predict_proba. -def test_not_predict_proba(create_X_y): - X, y = create_X_y - clf1 = Perceptron() - clf1.fit(X, y) - with pytest.raises(ValueError): - APosteriori([clf1, clf1]).fit(X, y) diff --git a/deslib/tests/dcs/test_a_priori.py b/deslib/tests/dcs/test_a_priori.py deleted file mode 100644 index bea79fe2..00000000 --- a/deslib/tests/dcs/test_a_priori.py +++ /dev/null @@ -1,93 +0,0 @@ -import numpy as np -import pytest -from sklearn.linear_model import Perceptron -from sklearn.utils.estimator_checks import check_estimator - -from deslib.dcs.a_priori import APriori - - -def test_check_estimator(): - check_estimator(APriori(selection_method='best')) - - -# Should always be 1.0 since the supports for the correct class is always 1. -@pytest.mark.parametrize('index, expected', [(0, [1.0, 1.0, 1.0]), - (1, [1.0, 1.0, 1.0]), - (2, [1.0, 1.0, 1.0])]) -def test_estimate_competence_all_ones(index, expected, example_all_ones): - X, y, neighbors, distances, dsel_processed, dsel_scores = example_all_ones - - a_priori_test = APriori() - - a_priori_test.DSEL_processed_ = dsel_processed - a_priori_test.dsel_scores_ = dsel_scores - a_priori_test.DSEL_target_ = y - a_priori_test.n_classes_ = 2 - - neighbors = neighbors[index, :].reshape(1, -1) - distances = distances[index, :].reshape(1, -1) - - competences = a_priori_test.estimate_competence(neighbors, distances) - assert np.isclose(competences, expected).all() - - -# Testing example from kuncheva's book (combining pattern classifiers) -def test_estimate_competence_kuncheva_ex(example_kuncheva): - a_priori_test = APriori(k=example_kuncheva['k']) - test_example = example_kuncheva - a_priori_test.DSEL_processed_ = test_example['dsel_processed'] - a_priori_test.dsel_scores_ = test_example['dsel_scores'] - a_priori_test.DSEL_target_ = test_example['y_independent'] - a_priori_test.n_classes_ = test_example['n_classes'] - - neighbors = test_example['neighbors'].reshape(1, -1) - distances = test_example['distances'].reshape(1, -1) - - competences = a_priori_test.estimate_competence(neighbors, distances) - assert np.isclose(competences, 0.70, atol=0.01) - - -# Test the estimate competence method receiving n samples as input -def test_estimate_competence_batch(example_estimate_competence): - _, y, nn, _, dsel_processed, dsel_scores = example_estimate_competence - expected = np.array([[0.333333, 0.50000, 0.40000], - [0.666666, 0.50000, 0.60000], - [0.000000, 0.50000, 0.20000]]) - - # Using 3 neighbors to facilitate the calculations - a_priori_test = APriori(k=3) - - a_priori_test.DSEL_processed_ = dsel_processed - a_priori_test.dsel_scores_ = dsel_scores - a_priori_test.DSEL_target_ = y - a_priori_test.n_classes_ = 2 - - nn = nn[:, 0:3] - distances = np.ones((3, 3)) - - competences = a_priori_test.estimate_competence(nn, - distances) - assert np.allclose(competences, expected, atol=0.01) - - -def test_fit(create_pool_classifiers, create_X_y): - X, y = create_X_y - - a_priori_test = APriori(create_pool_classifiers) - a_priori_test.fit(X, y) - expected = np.array([[0.5, 0.5], [1.0, 0.0], [0.33, 0.67]]) - expected = np.tile(expected, (15, 1, 1)) - assert np.array_equal(a_priori_test.dsel_scores_, expected) - - -# Test if the class is raising an error when the base classifiers do not -# implements the predict_proba method. Should raise an exception when the -# base classifier cannot estimate posterior probabilities (predict_proba) -# Using Perceptron classifier as it does not implements predict_proba. -def test_not_predict_proba(create_X_y): - X, y = create_X_y - - clf1 = Perceptron() - clf1.fit(X, y) - with pytest.raises(ValueError): - APriori([clf1, clf1]).fit(X, y) diff --git a/deslib/tests/dcs/test_base.py b/deslib/tests/dcs/test_base.py deleted file mode 100644 index e0b8dbc0..00000000 --- a/deslib/tests/dcs/test_base.py +++ /dev/null @@ -1,269 +0,0 @@ -from unittest.mock import MagicMock - -import numpy as np -import pytest - -from deslib.dcs.base import BaseDCS - - -@pytest.mark.parametrize('selection_method,', ['a', 'test']) -def test_valid_selection_method(selection_method, create_pool_classifiers): - X = np.random.rand(10, 2) - y = np.ones(10) - with pytest.raises(ValueError): - dcs = BaseDCS(create_pool_classifiers, - selection_method=selection_method) - dcs.fit(X, y) - - -@pytest.mark.parametrize('selection_method,', [1, [1.0, 2.0], None, np.nan]) -def test_selection_method_type(selection_method, create_pool_classifiers): - X = np.random.rand(10, 2) - y = np.ones(10) - with pytest.raises(TypeError): - dcs = BaseDCS(create_pool_classifiers, - selection_method=selection_method) - dcs.fit(X, y) - - -@pytest.mark.parametrize('diff_thresh,', ['test', None, [0.1, 0.2]]) -def test_valid_diff_threshold_type(diff_thresh, create_pool_classifiers): - X = np.random.rand(10, 2) - y = np.ones(10) - with pytest.raises(TypeError): - dcs = BaseDCS(create_pool_classifiers, selection_method='diff', - diff_thresh=diff_thresh) - dcs.fit(X, y) - - -@pytest.mark.parametrize('diff_thresh,', [1.0, -0.15, 0.5, np.nan]) -def test_valid_diff_threshold_value(diff_thresh, create_pool_classifiers): - X = np.random.rand(10, 2) - y = np.ones(10) - with pytest.raises(ValueError): - dcs = BaseDCS(create_pool_classifiers, selection_method='diff', - diff_thresh=diff_thresh) - dcs.fit(X, y) - - -# ------------- Testing different types of selection routines ----------------- -@pytest.mark.parametrize('competences, expected', - [([1.0, 0.5, 0.5], 0), ([0.8, 0.9, 1.0], 2), - ([0.0, 0.0, 0.15], 2)]) -def test_select_best(competences, expected, create_pool_classifiers): - pool_classifiers = create_pool_classifiers - dcs_test = BaseDCS(pool_classifiers, selection_method='best') - selected_clf = dcs_test.select(np.array(competences)) - assert np.allclose(selected_clf, expected) - - -@pytest.mark.parametrize('competences, expected', - [([1.0, 1.0, 0.5], [True, True, False]), - ([0.8, 0.9, 0.9], [False, True, True]), - ([0.15, 0.15, 0.15], [True, True, True]), - ([0.0, 0.0, 0.0], [True, True, True])]) -def test_select_all(competences, expected, create_pool_classifiers): - pool_classifiers = create_pool_classifiers - dcs_test = BaseDCS(pool_classifiers, selection_method='all') - selected_clf = dcs_test.select(np.array(competences)) - assert np.allclose(selected_clf, expected) - - -@pytest.mark.parametrize('competences, expected', - [([1.0, 0.5, 0.5], 0), ([0.8, 0.9, 1.0], 2), - ([0.0, 0.0, 0.15], 2)]) -def test_select_diff(competences, expected): - rng = np.random.RandomState(123456) - dcs_test = BaseDCS(selection_method='diff', diff_thresh=0.15, - random_state=rng) - - selected_clf = dcs_test.select(np.array(competences)) - assert np.allclose(selected_clf, expected) - - -@pytest.mark.parametrize('competences, expected', - [([0.5, 0.5, 0.5], 1), ([0.8, 0.9, 1.0], 2), - ([0.0, 0.10, 0.0], 1)]) -def test_select_random(competences, expected): - rng = np.random.RandomState(123456) - dcs_test = BaseDCS(selection_method='random', random_state=rng) - - selected_clf = dcs_test.select(np.array(competences)) - assert np.allclose(selected_clf, expected) - - -def test_select_best_batch(): - competences = np.array( - [[1.0, 0.5, 0.5], [0.8, 0.9, 1.0], [0.0, 0.0, 0.15]]) - expected = [0, 2, 2] - dcs_test = BaseDCS(selection_method='best') - selected_clf = dcs_test.select(competences) - assert np.array_equal(selected_clf, expected) - - -def test_select_all_batch(): - competences = np.array( - [[1.0, 1.0, 0.5], [0.8, 0.9, 0.9], [0.15, 0.15, 0.15], - [0.0, 0.0, 0.0]]) - expected = np.array( - [[True, True, False], [False, True, True], [True, True, True], - [True, True, True]]) - dcs_test = BaseDCS(selection_method='all') - selected_clf = dcs_test.select(competences) - assert np.array_equal(selected_clf, expected) - - -def test_select_diff_batch(): - competences = np.array( - [[1.0, 0.5, 0.5], [0.8, 0.9, 1.0], [0.0, 0.0, 0.15]]) - expected = np.array([0, 2, 2]) - rng = np.random.RandomState(123456) - dcs_test = BaseDCS(selection_method='diff', diff_thresh=0.15, - random_state=rng) - - selected_clf = dcs_test.select(competences) - assert np.array_equal(selected_clf, expected) - - -def test_select_random_batch(): - competences = np.array( - [[0.5, 0.5, 0.5], [0.8, 0.9, 1.0], [0.0, 0.10, 0.0]]) - expected = np.array([1, 2, 1]) - rng = np.random.RandomState(123456) - dcs_test = BaseDCS(selection_method='random', random_state=rng) - - selected_clf = dcs_test.select(competences) - assert np.array_equal(selected_clf, expected) - - -# --------------- Testing classify_with_ds and predict_proba ----------------- - - -def test_classify_instance(create_pool_classifiers): - query = np.array([-1, 1]) - n_classifiers = 3 - - pool_classifiers = create_pool_classifiers - dcs_test = BaseDCS(pool_classifiers) - competences = np.random.rand(n_classifiers) - - dcs_test.estimate_competence = MagicMock(return_value=competences) - expected = pool_classifiers[np.argmax(competences)].predict(query)[0] - - predictions = [] - for clf in dcs_test.pool_classifiers: - predictions.append(clf.predict(query)[0]) - predicted_label = dcs_test.classify_with_ds(np.array(predictions)) - assert predicted_label == expected - - -def test_classify_instance_batch(create_pool_classifiers): - n_samples = 3 - n_classifiers = 3 - query = np.ones((n_samples, 2)) - pool_classifiers = create_pool_classifiers - dcs_test = BaseDCS(pool_classifiers) - - competences = np.random.rand(n_samples, n_classifiers) - - dcs_test.estimate_competence = MagicMock(return_value=competences) - expected = [] - for ind in range(n_samples): - expected.append( - pool_classifiers[np.argmax(competences[ind, :])].predict(query)[0]) - - predictions = [] - for clf in dcs_test.pool_classifiers: - predictions.append(clf.predict(query)[0]) - predicted_label = dcs_test.classify_with_ds(np.tile(predictions, (3, 1))) - assert np.array_equal(predicted_label, expected) - - -@pytest.mark.parametrize('competences, expected', [([0.6, 0.2, 0.6], 0), - ([0.5, 0.8, 0.5], 1)]) -def test_classify_instance_all(competences, expected, create_pool_classifiers): - query = np.array([-1, 1]) - pool_classifiers = create_pool_classifiers - dcs_test = BaseDCS(pool_classifiers, selection_method='all') - dcs_test.estimate_competence = MagicMock( - return_value=np.array(competences)) - - predictions = [] - for clf in dcs_test.pool_classifiers: - predictions.append(clf.predict(query)[0]) - predicted_label = dcs_test.classify_with_ds(np.array(predictions)) - assert predicted_label == expected - - -def test_classify_instance_all_batch(create_pool_classifiers): - competences = np.array([[0.6, 0.2, 0.6], [0.5, 0.8, 0.5]]) - expected = [0, 1] - n_samples = 2 - query = np.ones((n_samples, 2)) - pool_classifiers = create_pool_classifiers - dcs_test = BaseDCS(pool_classifiers, selection_method='all') - dcs_test.estimate_competence = MagicMock( - return_value=np.array(competences)) - - predictions = [] - for clf in dcs_test.pool_classifiers: - predictions.append(clf.predict(query)[0]) - predicted_label = dcs_test.classify_with_ds(np.tile(predictions, - (n_samples, 1))) - assert np.array_equal(predicted_label, expected) - - -def test_predict_proba_instance(create_pool_classifiers): - query = np.array([-1, 1]) - n_classifiers = 3 - pool_classifiers = create_pool_classifiers - dcs_test = BaseDCS(pool_classifiers) - dcs_test.n_classes_ = 2 - - competences = np.random.rand(n_classifiers) - - dcs_test.estimate_competence = MagicMock(return_value=competences) - expected = pool_classifiers[np.argmax(competences)].predict_proba(query) - - predictions = [] - probabilities = [] - for clf in dcs_test.pool_classifiers: - predictions.append(clf.predict(query)[0]) - probabilities.append(clf.predict_proba(query)[0]) - - query = np.atleast_2d(query) - predictions = np.atleast_2d(predictions) - probabilities = np.array(probabilities) - probabilities = np.expand_dims(probabilities, axis=0) - - predicted_proba = dcs_test.predict_proba_with_ds(predictions, - probabilities) - assert np.array_equal(predicted_proba, expected) - - -@pytest.mark.parametrize('competences, expected', - [([0.6, 0.2, 0.6], [0.415, 0.585]), - ([0.5, 0.8, 0.5], [1.0, 0.0])]) -def test_predict_proba_instance_all(competences, expected, - create_pool_classifiers): - query = np.array([-1, 1]) - pool_classifiers = create_pool_classifiers - dcs_test = BaseDCS(pool_classifiers, selection_method='all') - dcs_test.n_classes_ = 2 - - dcs_test.estimate_competence = MagicMock( - return_value=np.array(competences)) - - predictions = [] - probabilities = [] - for clf in dcs_test.pool_classifiers: - predictions.append(clf.predict(query)[0]) - probabilities.append(clf.predict_proba(query)[0]) - - predictions = np.atleast_2d(predictions) - probabilities = np.array(probabilities) - probabilities = np.expand_dims(probabilities, axis=0) - - predicted_proba = dcs_test.predict_proba_with_ds(predictions, - probabilities) - assert np.isclose(predicted_proba, expected).all() diff --git a/deslib/tests/dcs/test_lca.py b/deslib/tests/dcs/test_lca.py deleted file mode 100644 index c1cdc076..00000000 --- a/deslib/tests/dcs/test_lca.py +++ /dev/null @@ -1,70 +0,0 @@ -import numpy as np -import pytest -from sklearn.linear_model import Perceptron -from sklearn.utils.estimator_checks import check_estimator - -from deslib.dcs.lca import LCA - - -def test_check_estimator(): - check_estimator(LCA()) - - -def test_estimate_competence_batch(example_estimate_competence): - _, y, neighbors, distances, dsel_processed, _ = example_estimate_competence - - expected = np.array([[0.75000000, 0.66666667, 0.75000000], - [0.80000000, 1.00000000, 0.80000000], - [1.00000000, 0.60000000, 0.50000000]]) - lca_test = LCA() - lca_test.DSEL_processed_ = dsel_processed - lca_test.DSEL_target_ = y - - query = np.ones((3, 2)) - - predictions = np.array([[0, 1, 0]]) - competences = lca_test.estimate_competence(neighbors, - distances=distances, - predictions=np.array( - predictions)) - - assert np.isclose(competences, expected).all() - - -# in this test case, the target of the neighbors is always different than -# the predicted class. So the estimation of competence should always be zero -@pytest.mark.parametrize('index', [0, 1, 2]) -def test_estimate_competence_diff_target(index, - example_estimate_competence, - create_pool_classifiers): - _, y, neighbors, distances, dsel_processed, _ = example_estimate_competence - - lca_test = LCA(create_pool_classifiers) - lca_test.DSEL_processed_ = dsel_processed - lca_test.DSEL_target_ = np.ones(15, dtype=int) * 3 - - neighbors = neighbors[index, :].reshape(1, -1) - distances = distances[index, :].reshape(1, -1) - - query = np.atleast_2d([1, 1]) - expected = [0.0, 0.0, 0.0] - - predictions = np.array([[0, 1, 0]]) - competences = lca_test.estimate_competence(neighbors, - distances=distances, - predictions=np.array( - predictions)) - - assert np.isclose(competences, expected).all() - - -# Test if the class is raising an error when the base classifiers do not -# implements the predict_proba method. In this case the test should not raise -# an error since this class does not require base classifiers that -# can estimate probabilities -def test_predict_proba(create_X_y): - X, y = create_X_y - - clf1 = Perceptron() - clf1.fit(X, y) - LCA([clf1, clf1]).fit(X, y) diff --git a/deslib/tests/dcs/test_mcb.py b/deslib/tests/dcs/test_mcb.py deleted file mode 100644 index 482d4c9b..00000000 --- a/deslib/tests/dcs/test_mcb.py +++ /dev/null @@ -1,102 +0,0 @@ -import numpy as np -import pytest -from sklearn.linear_model import Perceptron -from sklearn.utils.estimator_checks import check_estimator - -from deslib.dcs.mcb import MCB - -# ex1 the similarity will always be 100% -bks_dsel_ex1 = np.hstack( - (np.hstack((np.zeros((15, 1)), np.ones((15, 1)))), np.zeros((15, 1)))) - -# Change a bit to check if the filtering by similarity is working as intended. -bks_dsel_ex2 = np.hstack( - (np.hstack((np.zeros((15, 1)), np.ones((15, 1)))), np.zeros((15, 1)))) -bks_dsel_ex2[1, :] = 2 - -bks_dsel_ex3 = bks_dsel_ex1 + 1 - - -def test_check_estimator(): - check_estimator(MCB()) - - -@pytest.mark.parametrize('similarity_threshold', [2.0, -1.0, -0.5]) -def test_similarity_threshold(similarity_threshold, create_X_y): - X, y = create_X_y - with pytest.raises(ValueError): - mcb = MCB(similarity_threshold=similarity_threshold) - mcb.fit(X, y) - - -@pytest.mark.parametrize('similarity_threshold', [None, 'a']) -def test_similarity_threshold_type(similarity_threshold, create_X_y): - X, y = create_X_y - with pytest.raises(TypeError): - mcb = MCB(similarity_threshold=similarity_threshold) - mcb.fit(X, y) - - -@pytest.mark.parametrize('index, expected', [(0, [0.66666666, - 0.83333333, - 0.66666666]), - (1, [0.83333333, - 1.0, - 0.66666666])]) -def test_estimate_competence2(index, expected, example_estimate_competence): - - _, _, neighbors, distances, dsel_processed, _ = example_estimate_competence - - mcb_test = MCB() - mcb_test.n_classifiers_ = 3 - mcb_test.DSEL_processed_ = dsel_processed - - neighbors = neighbors[index, :].reshape(1, -1) - distances = distances[index, :].reshape(1, -1) - # Only changing the pre-processed BKS to see if the filter works. - mcb_test.BKS_DSEL_ = bks_dsel_ex2 - - predictions = np.array([[0, 1, 0]]) - - competences = mcb_test.estimate_competence(neighbors, - distances=distances, - predictions=np.atleast_2d( - predictions)) - assert np.isclose(competences, expected).all() - - -# This third test uses an totally wrong bks matrix, so that the technique -# is obligated to use the whole it also considers batch processing -# region of competence -def test_estimate_competence_batch(example_estimate_competence): - _, _, neighbors, distances, dsel_processed, _ = example_estimate_competence - - expected = np.array([[0.57142857, 0.71428571, 0.71428571], - [0.71428571, 0.85714286, 0.71428571], - [0.57142857, 0.71428571, 0.57142857]]) - mcb_test = MCB() - mcb_test.n_classifiers_ = 3 - mcb_test.DSEL_processed_ = dsel_processed - - # Only changing the pre-processed BKS to see if the filter works. - mcb_test.BKS_DSEL_ = bks_dsel_ex3 - - predictions = np.array([0, 1, 0]) - - competences = mcb_test.estimate_competence(neighbors, - distances=distances, - predictions=np.tile(predictions, - (3, 1))) - assert np.isclose(competences, expected).all() - - -# Test if the class is raising an error when the base classifiers do not -# implements the predict_proba method. # In this case the test should not -# raise an error since this class does not require base classifiers that -# can estimate probabilities -def test_predict_proba(create_X_y): - X, y = create_X_y - - clf1 = Perceptron() - clf1.fit(X, y) - MCB([clf1, clf1]).fit(X, y) diff --git a/deslib/tests/dcs/test_mla.py b/deslib/tests/dcs/test_mla.py deleted file mode 100644 index 39db3876..00000000 --- a/deslib/tests/dcs/test_mla.py +++ /dev/null @@ -1,125 +0,0 @@ -import numpy as np -import pytest -from sklearn.linear_model import Perceptron -from sklearn.utils.estimator_checks import check_estimator - -from deslib.dcs.mla import MLA - - -def test_check_estimator(): - check_estimator(MLA()) - - -# Should always be 1.0 since the supports for the correct class is always 1. -@pytest.mark.parametrize('index', [0, 1, 2]) -def test_estimate_competence_all_ones(index, example_all_ones): - _, y, neighbors, distances, dsel_processed, dsel_scores = example_all_ones - - mla_test = MLA() - mla_test.n_classifiers_ = 3 - - mla_test.DSEL_processed_ = dsel_processed - mla_test.DSEL_scores = dsel_scores - mla_test.DSEL_target_ = y - mla_test.n_classes_ = 2 - - neighbors = neighbors[index, :].reshape(1, -1) - distances = distances[index, :].reshape(1, -1) - - expected = [1.0, 1.0, 1.0] - - predictions = np.array([[0, 1, 0]]) - - competences = mla_test.estimate_competence(neighbors, - distances=distances, - predictions=predictions) - - assert np.isclose(competences, expected).all() - - -def test_estimate_competence_batch(example_estimate_competence): - - _, y, neighbors, _, dsel_processed, _ = example_estimate_competence - - expected = np.array([[0.750, 0.666, 0.750], - [0.800, 1.000, 0.800], - [1.000, 0.600, 0.500]]) - - mla_test = MLA() - mla_test.n_classifiers_ = 3 - mla_test.DSEL_processed_ = dsel_processed - distances = np.ones((3, 7)) - - mla_test.DSEL_target_ = y - mla_test.n_classes_ = 2 - predictions = np.array([[0, 1, 0]]) - - competences = mla_test.estimate_competence(competence_region=neighbors, - distances=distances, - predictions=predictions) - - assert np.allclose(competences, expected, atol=0.01) - - -# in this test case, the target of the neighbors is always different than the -# predicted. So the estimation of competence should always be zero -@pytest.mark.parametrize('index', [0, 1, 2]) -def test_estimate_competence_diff_target(index, example_estimate_competence): - _, _, neighbors, distances, dsel_processed, _ = example_estimate_competence - - mla_test = MLA() - mla_test.n_classifiers_ = 3 - - mla_test.DSEL_processed_ = dsel_processed - mla_test.DSEL_target_ = np.ones(15, dtype=int) * 3 - - neighbors = neighbors[index, :].reshape(1, -1) - distances = distances[index, :].reshape(1, -1) - - expected = [0.0, 0.0, 0.0] - - predictions = np.array([[0, 1, 0]]) - - competences = mla_test.estimate_competence(neighbors, - distances=distances, - predictions=predictions) - - assert np.isclose(competences, expected).all() - - -# Testing example from kuncheva's book (combining pattern classifiers) -def test_estimate_competence_kuncheva_ex(example_kuncheva): - example_kuncheva = example_kuncheva - - mla_test = MLA(k=example_kuncheva['k']) - mla_test.n_classifiers_ = 2 - - mla_test.DSEL_processed_ = np.repeat(example_kuncheva['dsel_processed'], - 2, - axis=1) - - mla_test.dsel_scores_ = example_kuncheva['dsel_scores'] - mla_test.DSEL_target_ = example_kuncheva['y_dependent'] - mla_test.n_classes_ = example_kuncheva['n_classes'] - - neighbors = example_kuncheva['neighbors'].reshape(1, -1) - distances = example_kuncheva['distances'].reshape(1, -1) - - predictions = np.array([[1, 1]]) - competences = mla_test.estimate_competence(neighbors, - distances=distances, - predictions=predictions) - - assert np.allclose(competences, [0.95, 0.95], atol=0.01) - - -# Test if the class is raising an error when the base classifiers do not -# implements the predict_proba method. In this case the test should not raise -# an error since this class does not require base classifiers that -# can estimate probabilities -def test_predict_proba(create_X_y): - X, y = create_X_y - - clf1 = Perceptron() - clf1.fit(X, y) - MLA([clf1, clf1]).fit(X, y) diff --git a/deslib/tests/dcs/test_ola.py b/deslib/tests/dcs/test_ola.py deleted file mode 100644 index a471fe50..00000000 --- a/deslib/tests/dcs/test_ola.py +++ /dev/null @@ -1,35 +0,0 @@ -import numpy as np -from sklearn.linear_model import Perceptron -from sklearn.utils.estimator_checks import check_estimator - -from deslib.dcs.ola import OLA - - -def test_check_estimator(): - check_estimator(OLA()) - - -def test_estimate_competence_batch(example_estimate_competence): - _, _, neighbors, distances, dsel_processed, _ = example_estimate_competence - expected = np.array([[0.57142857, 0.71428571, 0.71428571], - [0.71428571, 0.85714286, 0.71428571], - [0.57142857, 0.71428571, 0.57142857]]) - - ola_test = OLA() - ola_test.DSEL_processed_ = dsel_processed - - ola_test.DFP_mask = np.ones((3, 3)) - competences = ola_test.estimate_competence(neighbors, - distances=distances) - assert np.allclose(competences, expected) - - -# Test if the class is raising an error when the base classifiers do not -# implements the predict_proba method. In this case the test should not raise -# an error since this class does not require base classifiers that -# can estimate probabilities -def test_predict_proba(create_X_y): - X, y = create_X_y - clf1 = Perceptron() - clf1.fit(X, y) - OLA([clf1, clf1]).fit(X, y) diff --git a/deslib/tests/dcs/test_rank.py b/deslib/tests/dcs/test_rank.py deleted file mode 100644 index b25ededc..00000000 --- a/deslib/tests/dcs/test_rank.py +++ /dev/null @@ -1,34 +0,0 @@ -import numpy as np -from sklearn.linear_model import Perceptron -from sklearn.utils.estimator_checks import check_estimator - -from deslib.dcs.rank import Rank - - -def test_check_estimator(): - check_estimator(Rank()) - - -def test_estimate_competence_batch(example_estimate_competence): - _, _, neighbors, distances, dsel_processed, _ = example_estimate_competence - - expected = np.array([[1, 5, 0], - [1, 1, 2], - [0, 0, 1]]) - rank_test = Rank() - rank_test.DSEL_processed_ = dsel_processed - competences = rank_test.estimate_competence(neighbors, - distances=distances) - assert np.allclose(competences, expected) - - -# Test if the class is raising an error when the base classifiers do not -# implements the predict_proba method. In this case the test should not raise -# an error since this class does not require base classifiers that -# can estimate probabilities -def test_predict_proba(create_X_y): - X, y = create_X_y - - clf1 = Perceptron() - clf1.fit(X, y) - Rank([clf1, clf1]).fit(X, y) diff --git a/deslib/tests/des/__init__.py b/deslib/tests/des/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/deslib/tests/des/test_base.py b/deslib/tests/des/test_base.py deleted file mode 100644 index ac443551..00000000 --- a/deslib/tests/des/test_base.py +++ /dev/null @@ -1,225 +0,0 @@ -from unittest.mock import MagicMock - -import numpy as np -import pytest - -from deslib.des.base import BaseDES - - -@pytest.mark.parametrize('mode,', ['a', 'test']) -def test_valid_selection_mode(mode, create_pool_classifiers): - X = np.random.rand(10, 2) - y = np.ones(10) - with pytest.raises(ValueError): - des = BaseDES(create_pool_classifiers, mode=mode) - des.fit(X, y) - - -@pytest.mark.parametrize('mode,', [1, [1.0, 2.0], None]) -def test_selection_method_type(mode, create_pool_classifiers): - X = np.random.rand(10, 2) - y = np.ones(10) - with pytest.raises(TypeError): - des = BaseDES(create_pool_classifiers, mode=mode) - des.fit(X, y) - - -# ------------------------ Testing classify_with_ds ----------------- -"""Example considering a pool composed of 6 base classifiers. The classifiers -with index 0, 2, 3 and 5 predicts class 0 -while classifiers with indices_ 1 and 4 predicts class 1. -""" - - -# In this first example only dynamic selection is considered. Since the -# selected indices_ are 0, 1 and 5 the expected -# prediction should be 0 (2 votes). -def test_classify_instance_selection(create_pool_classifiers): - n_samples = 3 - query = np.ones((n_samples, 2)) - pool_classifiers = create_pool_classifiers + create_pool_classifiers - des_test = BaseDES(pool_classifiers, mode='selection') - des_test.n_classes_ = 2 - selected_index = np.array( - [[True, True, False, False, False, True] * n_samples]) - des_test.select = MagicMock(return_value=selected_index) - - predictions = [] - for clf in des_test.pool_classifiers: - predictions.append(clf.predict(query)[0]) - - predicted_label = des_test.classify_with_ds(np.tile(predictions, - (n_samples, 1))) - assert np.allclose(predicted_label, 0) and predicted_label.size == 3 - - -def test_classify_instance_weighting(create_pool_classifiers): - n_samples = 3 - query = np.ones((n_samples, 2)) - pool_classifiers = create_pool_classifiers + create_pool_classifiers - des_test = BaseDES(pool_classifiers, mode='weighting') - des_test.classes_ = np.array([0, 1]) - des_test.n_classes_ = 2 - - competences = np.tile([0.55, 1.0, 0.2, 0.60, 0.75, 0.3], (3, 1)) - des_test.estimate_competence = MagicMock(return_value=competences) - - predictions = [] - for clf in des_test.pool_classifiers: - predictions.append(clf.predict(query)[0]) - predicted_label = des_test.classify_with_ds(np.tile(predictions, (3, 1))) - assert np.allclose(predicted_label, 1) and predicted_label.size == 3 - - -# Same example of test_classify_instance_selection, however, since the weights -# are also used in the hybrid scheme, -# the function should return 1 instead of 0. -def test_classify_instance_hybrid(create_pool_classifiers): - query = np.ones((3, 2)) - expected = 1 - pool_classifiers = create_pool_classifiers + create_pool_classifiers - des_test = BaseDES(pool_classifiers, mode='hybrid') - des_test.classes_ = np.array([0, 1]) - des_test.n_classes_ = 2 - - selected_indices = np.tile([True, True, False, False, False, True], (3, 1)) - competences = np.tile([0.55, 1.0, 0.2, 0.60, 0.75, 0.3], (3, 1)) - des_test.estimate_competence = MagicMock(return_value=competences) - des_test.select = MagicMock(return_value=selected_indices) - - predictions = [] - for clf in des_test.pool_classifiers: - predictions.append(clf.predict(query)[0]) - - predicted_label = des_test.classify_with_ds(np.tile(predictions, (3, 1))) - assert np.allclose(predicted_label, expected) - - -# ------------------------ Testing predict_proba ----------------- -# The prediction of probability here should be an average_rule of the -# probabilities estimates of the three selected base classifiers -def test_predict_proba_selection_soft_voting(create_pool_classifiers): - query = np.array([[-1, 1]]) - expected = np.array([0.61, 0.39]) - predictions = np.array([[0, 1, 0, 0, 1, 0]]) - probabilities = np.array([[[0.5, 0.5], [1., 0.], [0.33, 0.67], - [0.5, 0.5], [1., 0.], [0.33, 0.67]]]) - selected_indices = np.array([[True, True, False, False, False, True]]) - pool_classifiers = create_pool_classifiers + create_pool_classifiers - - des_test = BaseDES(pool_classifiers, mode='selection', voting='soft') - des_test.select = MagicMock(return_value=selected_indices) - des_test.n_classes_ = 2 - - predicted_proba = des_test.predict_proba_with_ds(predictions, - probabilities) - assert np.isclose(predicted_proba, expected, atol=0.01).all() - - -# The predicted probabilities must also consider the assigned weights of each -# base classifier -def test_predict_proba_weighting_soft_voting(create_pool_classifiers): - query = np.array([[-1, 1]]) - expected = np.array([0.5769, 0.4231]) - competences = np.array([[0.5, 1.0, 0.2]]) - predictions = np.array([[0, 1, 0]]) - probabilities = np.array([[[0.5, 0.5], [1., 0.], [0.33, 0.67]]]) - pool_classifiers = create_pool_classifiers - - des_test = BaseDES(pool_classifiers, mode='weighting', voting='soft') - des_test.estimate_competence = MagicMock(return_value=competences) - des_test.n_classes_ = 2 - - predicted_proba = des_test.predict_proba_with_ds(predictions, - probabilities) - assert np.isclose(predicted_proba, expected, atol=0.01).all() - - -# The predicted probabilities must also consider the assigned weights of each -# base classifier selected -def test_predict_proba_hybrid_soft_voting(create_pool_classifiers): - query = np.array([[-1, 1]]) - expected = np.array([0.5744, 0.4256]) - competences = np.array([[0.55, 1.0, 0.2, 0.60, 0.75, 0.3]]) - predictions = np.array([[0, 1, 0, 0, 1, 0]]) - probabilities = np.array([[[0.5, 0.5], [1., 0.], [0.33, 0.67], - [0.5, 0.5], [1., 0.], [0.33, 0.67]]]) - selected_indices = np.array([[True, True, False, False, False, True]]) - pool_classifiers = create_pool_classifiers + create_pool_classifiers - - des_test = BaseDES(pool_classifiers, mode='hybrid', voting='soft') - des_test.n_classes_ = 2 - des_test.estimate_competence = MagicMock(return_value=competences) - des_test.select = MagicMock(return_value=selected_indices) - predicted_proba = des_test.predict_proba_with_ds(predictions, - probabilities) - assert np.isclose(predicted_proba, expected, atol=0.01).all() - - -# --------------------------Tests Hard Voting---------------------------------- -def test_predict_proba_selection_hard_voting(create_pool_classifiers): - query = np.array([[-1, 1]]) - expected = np.array([0.66, 0.33]) - predictions = np.array([[0, 1, 0, 0, 1, 0]]) - pool_classifiers = create_pool_classifiers + create_pool_classifiers - selected_indices = np.array([[True, True, False, False, False, True]]) - - des_test = BaseDES(pool_classifiers, mode='selection', voting='hard') - des_test.n_classes_ = 2 - des_test.select = MagicMock(return_value=selected_indices) - - predicted_proba = des_test.predict_proba_with_ds(predictions) - assert np.isclose(predicted_proba, expected, atol=0.01).all() - - -# The predicted probabilities must also consider the assigned weights of each -# base classifier -def test_predict_proba_weighting_hard_voting(create_pool_classifiers): - query = np.array([[-1, 1]]) - expected = np.array([0.4117, 0.5883]) - competences = np.array([[0.5, 1.0, 0.2]]) - predictions = np.array([[0, 1, 0]]) - pool_classifiers = create_pool_classifiers - - des_test = BaseDES(pool_classifiers, mode='weighting', voting='hard') - des_test.estimate_competence = MagicMock(return_value=competences) - des_test.n_classes_ = 2 - - predicted_proba = des_test.predict_proba_with_ds(predictions) - assert np.isclose(predicted_proba, expected, atol=0.01).all() - - -# The predicted probabilities must also consider the assigned weights of each -# base classifier selected -def test_predict_proba_hybrid_hard_voting(create_pool_classifiers): - query = np.array([[-1, 1]]) - expected = np.array([0.4594, 0.5405]) - competences = np.array([[0.55, 1.0, 0.2, 0.60, 0.75, 0.3]]) - predictions = np.array([[0, 1, 0, 0, 1, 0]]) - selected_indices = np.array([[True, True, False, False, False, True]]) - pool_classifiers = create_pool_classifiers + create_pool_classifiers - - des_test = BaseDES(pool_classifiers, mode='hybrid', voting='hard') - des_test.n_classes_ = 2 - des_test.estimate_competence = MagicMock(return_value=competences) - des_test.select = MagicMock(return_value=selected_indices) - - predicted_proba = des_test.predict_proba_with_ds(predictions) - assert np.isclose(predicted_proba, expected, atol=0.01).all() - - -def test_soft_voting_no_proba(create_X_y): - from sklearn.linear_model import Perceptron - X, y = create_X_y - clf = Perceptron() - clf.fit(X, y) - with pytest.raises(ValueError): - BaseDES([clf, clf], voting='soft').fit(X, y) - - -@pytest.mark.parametrize('voting', [None, 'product', 1]) -def test_wrong_voting_value(voting, create_X_y, create_pool_classifiers): - X, y = create_X_y - pool = create_pool_classifiers - with pytest.raises(ValueError): - BaseDES(pool, voting=voting).fit(X, y) diff --git a/deslib/tests/des/test_des_clustering.py b/deslib/tests/des/test_des_clustering.py deleted file mode 100644 index e65b8055..00000000 --- a/deslib/tests/des/test_des_clustering.py +++ /dev/null @@ -1,240 +0,0 @@ -from unittest.mock import MagicMock - -import numpy as np -import pytest -from sklearn.cluster import KMeans -from sklearn.linear_model import Perceptron -from sklearn.utils.estimator_checks import check_estimator - -from deslib.des.des_clustering import DESClustering -from deslib.util.diversity import Q_statistic, ratio_errors, \ - negative_double_fault - - -def test_check_estimator(): - check_estimator(DESClustering()) - - -""" Considering a test scenario in which all samples from class 0 are indexed -in cluster n. 0 and classes_ 1 to cluster n. 1. For this example, the base -classifiers that always predicts 0 should me most accurate on the cluster 0, -while the base classifiers that predicts 1 for the cluster with index == 1. -""" - -""" In this test scenario, each cluster contains samples from classes_ 1 and 2. -""" -return_cluster_index_ex2 = np.array( - [0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 0, 1, 1, 1, 1]) - - -def test_fit_homogeneous_clusters(create_pool_classifiers, - example_estimate_competence): - clustering_test = DESClustering(create_pool_classifiers * 2, - clustering=KMeans(n_clusters=2), - pct_accuracy=0.5, pct_diversity=0.33) - - X, y = example_estimate_competence[0:2] - clustering_test.clustering.predict = MagicMock(return_value=y) - - clustering_test.fit(X, y) - - assert (clustering_test.performance_cluster_[0, 1] == 0.0 and - clustering_test.performance_cluster_[0, [0, 2]].all() == 1.0) - assert (clustering_test.performance_cluster_[1, 1] == 1.0 and - clustering_test.performance_cluster_[1, [0, 2]].all() == 0.0) - for idx in clustering_test.indices_[0, :]: - assert idx in (0, 2, 3, 5) - - -def test_fit_heterogeneous_clusters(example_estimate_competence, - create_pool_classifiers): - clustering_test = DESClustering(create_pool_classifiers, - clustering=KMeans(n_clusters=2), - pct_accuracy=0.5, - pct_diversity=0.33) - X, y = example_estimate_competence[0:2] - - clustering_test.clustering.predict = MagicMock( - return_value=return_cluster_index_ex2) - clustering_test.fit(X, y) - - # Index selected should be of any classifier that predicts the label 0 - assert np.isclose(clustering_test.performance_cluster_[:, 1], - [0.428, 0.375], atol=0.01).all() - assert np.isclose(clustering_test.performance_cluster_[:, 0], - [0.572, 0.625], atol=0.01).all() - assert clustering_test.indices_[0, 0] == 0 or clustering_test.indices_[ - 0, 0] == 2 - assert clustering_test.indices_[1, 0] == 0 or clustering_test.indices_[ - 1, 0] == 2 - - -def test_estimate_competence(create_pool_classifiers, - example_estimate_competence): - clustering_test = DESClustering(create_pool_classifiers * 2, - clustering=KMeans(n_clusters=2), - pct_accuracy=0.5, pct_diversity=0.33) - - X, y = example_estimate_competence[0:2] - clustering_test.fit(X, y) - competences = clustering_test.estimate_competence(0) - - assert np.array_equal(competences, - clustering_test.performance_cluster_[0, :]) - - competences = clustering_test.estimate_competence(1) - assert np.array_equal(competences, - clustering_test.performance_cluster_[1, :]) - - -def test_fit_clusters_less_diverse(example_estimate_competence, - create_pool_classifiers): - clustering_test = DESClustering(create_pool_classifiers * 2, - clustering=KMeans(n_clusters=2), - pct_accuracy=0.5, pct_diversity=0.33, - more_diverse=False) - X, y = example_estimate_competence[0:2] - - clustering_test.clustering.predict = MagicMock(return_value=y) - clustering_test.fit(X, y) - - assert (clustering_test.performance_cluster_[0, 1] == 0.0 and - clustering_test.performance_cluster_[0, [0, 2]].all() == 1.0) - assert (clustering_test.performance_cluster_[1, 1] == 1.0 and - clustering_test.performance_cluster_[1, [0, 2]].all() == 0.0) - for idx in clustering_test.indices_[0, :]: - assert idx in (1, 3, 4, 5) - - -def test_select(): - clustering_test = DESClustering() - clustering_test.clustering_ = KMeans() - roc = [0] - clustering_test.indices_ = np.array([[0, 2], [1, 4]]) - assert np.array_equal(clustering_test.select(roc), [[0, 2]]) - roc = [1] - clustering_test.clustering_.predict = MagicMock(return_value=[1]) - assert np.array_equal(clustering_test.select(roc), [[1, 4]]) - - -# Since the majority of the base classifiers selected predicts class 0, -# the final decision of the ensemble should be 0. -def test_classify_instance(create_pool_classifiers): - query = np.ones((1, 2)) - clustering_test = DESClustering(create_pool_classifiers * 4, - clustering=KMeans(n_clusters=2)) - clustering_test.n_classes_ = 2 - clustering_test.select = MagicMock(return_value=[0, 1, 2, 3, 5, 6, 7, 9]) - predictions = [] - for clf in clustering_test.pool_classifiers: - predictions.append(clf.predict(query)[0]) - - predicted = clustering_test.classify_with_ds(np.atleast_2d(predictions)) - assert predicted == 0 - - -def test_input_diversity_parameter(create_X_y): - X, y = create_X_y - - with pytest.raises(ValueError): - des_clustering = DESClustering(metric_diversity='abc') - des_clustering.fit(X, y) - - -def test_J_N_values(create_X_y): - X, y = create_X_y - - with pytest.raises(ValueError): - des_clustering = DESClustering(pct_accuracy=0.5, pct_diversity=0) - des_clustering.fit(X, y) - - -def test_J_higher_than_N(create_X_y): - X, y = create_X_y - - with pytest.raises(ValueError): - des_clustering = DESClustering(pct_accuracy=0.3, pct_diversity=0.5) - des_clustering.fit(X, y) - - -def test_diversity_metric_Q(create_X_y): - X, y = create_X_y - - test = DESClustering(metric_diversity='Q') - # Mocking this method to avoid preprocessing the cluster information - # that is not required in this test. - test._preprocess_clusters = MagicMock(return_value=1) - test.fit(X, y) - assert test.diversity_func_ == Q_statistic - - -def test_diversity_metric_DF(create_X_y): - X, y = create_X_y - - test = DESClustering(metric_diversity='DF') - # Mocking this method to avoid preprocessing the cluster - # information that is not required in this test. - test._preprocess_clusters = MagicMock(return_value=1) - test.fit(X, y) - assert test.diversity_func_ == negative_double_fault - - -def test_diversity_metric_ratio(create_X_y): - X, y = create_X_y - - test = DESClustering(metric_diversity='ratio') - # Mocking this method to avoid preprocessing the cluster - # information that is not required in this test. - test._preprocess_clusters = MagicMock(return_value=1) - test.fit(X, y) - assert test.diversity_func_ == ratio_errors - - -# Test if the class is raising an error when the base classifiers do not -# implements the predict_proba method. In this case the test should not raise -# an error since this class does not require base classifiers that -# can estimate probabilities -def test_predict_proba(example_estimate_competence): - X, y = example_estimate_competence[0:2] - - clf1 = Perceptron() - clf1.fit(X, y) - DESClustering([clf1, clf1]).fit(X, y) - - -def test_not_clustering_algorithm(create_X_y): - X, y = create_X_y - - des_clustering = DESClustering(clustering=Perceptron()) - with pytest.raises(ValueError): - des_clustering.fit(X, y) - - -def test_invalid_metric_performance(create_X_y): - X, y = create_X_y - with pytest.raises(ValueError): - DESClustering(metric_performance='notametric').fit(X, y) - - -def test_single_classifier_pool(create_X_y): - X, y = create_X_y - pool = [Perceptron().fit(X, y)] - with pytest.raises(ValueError): - DESClustering(pool_classifiers=pool).fit(X, y) - - -def test_soft_voting_no_proba(create_X_y): - from sklearn.linear_model import Perceptron - X, y = create_X_y - clf = Perceptron() - clf.fit(X, y) - with pytest.raises(ValueError): - model = DESClustering([clf, clf], voting='soft').fit(X, y) - - -@pytest.mark.parametrize('voting', [None, 'product', 1]) -def test_wrong_voting_value(voting, create_X_y, create_pool_classifiers): - X, y = create_X_y - pool = create_pool_classifiers - with pytest.raises(ValueError): - DESClustering(pool, voting=voting).fit(X, y) diff --git a/deslib/tests/des/test_des_knn.py b/deslib/tests/des/test_des_knn.py deleted file mode 100644 index 6ec02421..00000000 --- a/deslib/tests/des/test_des_knn.py +++ /dev/null @@ -1,318 +0,0 @@ -from unittest.mock import MagicMock - -import numpy as np -import pytest -from sklearn.linear_model import Perceptron -from sklearn.utils.estimator_checks import check_estimator - -from deslib.des.des_knn import DESKNN - - -def test_check_estimator(): - check_estimator(DESKNN()) - - -def test_estimate_competence(): - """ - Test case: - - Correct labels: 0000111 - classifier 1: 1010000 (2/7 correct) - classifier 2: 1000100 (4/7 correct) - classifier 2: 0010110 (5/7 correct) - - Diversity: compute number of common errors (on both classifiers) and - divide by 7: - clf1 x clf2: 3/7 - clf1 x clf3: 2/7 - clf2 x clf3: 1/7 - - clf1 diversity = (3+2)/7 = -5/7 (negative because we use the negative of - double error) - clf2 diversity = (3+1)/7 = -4/7 - clf3 diversity = (2+1)/7 = -3/7 - - """ - x = np.array([0, 1, 2, 3, 4, 5, 6]).reshape(-1, 1) - y = np.array([0, 0, 0, 0, 1, 1, 1]) - - clf1 = create_base_classifier(np.array([1, 0, 1, 0, 0, 0, 0])) - clf2 = create_base_classifier(np.array([1, 0, 0, 0, 1, 0, 0])) - clf3 = create_base_classifier(np.array([0, 0, 1, 0, 1, 1, 0])) - - pool_classifiers = [clf1, clf2, clf3] - - target = DESKNN(pool_classifiers, k=7, pct_accuracy=1, pct_diversity=1) - target.fit(x, y) - neighbors = np.array([[0, 1, 2, 3, 4, 5, 6]]) - competences, diversity = target.estimate_competence(neighbors) - assert np.allclose(competences, [2. / 7, 4. / 7, 5. / 7]) - assert np.allclose(diversity, [-5. / 7, -4. / 7, -3. / 7]) - - -def test_estimate_competence_batch(): - """ - Test case: - - Correct labels: 0000111 - classifier 1: 1010000 (2/7 correct) - classifier 2: 1000100 (4/7 correct) - classifier 2: 0010110 (5/7 correct) - - Diversity: compute number of common errors (on both classifiers) and - divide by 7: - clf1 x clf2: 3/7 - clf1 x clf3: 2/7 - clf2 x clf3: 1/7 - - clf1 diversity = (3+2)/7 = -5/7 (negative because we use the negative of - double error) - clf2 diversity = (3+1)/7 = -4/7 - clf3 diversity = (2+1)/7 = -3/7 - - """ - - n_samples = 10 - x = np.array([0, 1, 2, 3, 4, 5, 6]).reshape(-1, 1) - y = np.array([0, 0, 0, 0, 1, 1, 1]) - - clf1 = create_base_classifier(np.array([1, 0, 1, 0, 0, 0, 0])) - clf2 = create_base_classifier(np.array([1, 0, 0, 0, 1, 0, 0])) - clf3 = create_base_classifier(np.array([0, 0, 1, 0, 1, 1, 0])) - - pool_classifiers = [clf1, clf2, clf3] - - target = DESKNN(pool_classifiers, k=7, pct_accuracy=1, pct_diversity=1) - target.fit(x, y) - neighbors = np.tile([0, 1, 2, 3, 4, 5, 6], (10, 1)) - - competences, diversity = target.estimate_competence(neighbors) - assert np.allclose(competences, [2. / 7, 4. / 7, 5. / 7]) - assert np.allclose(diversity, [-5. / 7, -4. / 7, -3. / 7]) - - -def test_estimate_competence_Q(): - x = np.array([0, 1, 2, 3, 4, 5, 6]).reshape(-1, 1) - y = np.array([0, 0, 0, 0, 1, 1, 1]) - - clf1 = create_base_classifier(np.array([1, 0, 1, 0, 0, 0, 0])) - clf2 = create_base_classifier(np.array([1, 0, 0, 0, 1, 0, 0])) - clf3 = create_base_classifier(np.array([0, 0, 1, 0, 1, 1, 0])) - - pool_classifiers = [clf1, clf2, clf3] - - target = DESKNN(pool_classifiers, k=7, pct_accuracy=1, pct_diversity=1, - metric='Q') - target.fit(x, y) - neighbors = np.array([[0, 1, 2, 3, 4, 5, 6]]) - - competences, diversity = target.estimate_competence(neighbors) - assert np.allclose(competences, [2. / 7, 4. / 7, 5. / 7]) - assert np.allclose(diversity, [2, 1.2, 1.2]) - - -def test_estimate_competence_Q_batch(): - n_samples = 10 - x = np.array([0, 1, 2, 3, 4, 5, 6]).reshape(-1, 1) - y = np.array([0, 0, 0, 0, 1, 1, 1]) - - clf1 = create_base_classifier(np.array([1, 0, 1, 0, 0, 0, 0])) - clf2 = create_base_classifier(np.array([1, 0, 0, 0, 1, 0, 0])) - clf3 = create_base_classifier(np.array([0, 0, 1, 0, 1, 1, 0])) - - pool_classifiers = [clf1, clf2, clf3] - - target = DESKNN(pool_classifiers, k=7, pct_accuracy=1, pct_diversity=1, - metric='Q') - target.fit(x, y) - neighbors = np.tile([0, 1, 2, 3, 4, 5, 6], (n_samples, 1)) - - competences, diversity = target.estimate_competence(neighbors) - assert np.allclose(competences, [2. / 7, 4. / 7, 5. / 7]) - assert np.allclose(diversity, [2, 1.2, 1.2]) - - -def test_estimate_competence_ratio(): - x = np.array([0, 1, 2, 3, 4, 5, 6]).reshape(-1, 1) - y = np.array([0, 0, 0, 0, 1, 1, 1]) - - clf1 = create_base_classifier(np.array([1, 0, 1, 0, 0, 0, 0])) - clf2 = create_base_classifier(np.array([1, 0, 0, 0, 1, 0, 0])) - clf3 = create_base_classifier(np.array([0, 0, 1, 0, 1, 1, 0])) - - pool_classifiers = [clf1, clf2, clf3] - - target = DESKNN(pool_classifiers, k=7, pct_accuracy=1, pct_diversity=1, - metric='ratio') - target.fit(x, y) - neighbors = np.array([[0, 1, 2, 3, 4, 5, 6]]) - - competences, diversity = target.estimate_competence(neighbors) - assert np.allclose(competences, [2. / 7, 4. / 7, 5. / 7]) - assert np.allclose(diversity, [2.166, 3.666, 4.500], atol=0.01) - - -def test_estimate_competence_ratio_batch(): - n_samples = 10 - x = np.array([0, 1, 2, 3, 4, 5, 6]).reshape(-1, 1) - y = np.array([0, 0, 0, 0, 1, 1, 1]) - - clf1 = create_base_classifier(np.array([1, 0, 1, 0, 0, 0, 0])) - clf2 = create_base_classifier(np.array([1, 0, 0, 0, 1, 0, 0])) - clf3 = create_base_classifier(np.array([0, 0, 1, 0, 1, 1, 0])) - - pool_classifiers = [clf1, clf2, clf3] - - target = DESKNN(pool_classifiers, k=7, pct_accuracy=1, pct_diversity=1, - metric='ratio') - target.fit(x, y) - neighbors = np.tile([0, 1, 2, 3, 4, 5, 6], (n_samples, 1)) - - competences, diversity = target.estimate_competence(neighbors) - assert np.allclose(competences, [2. / 7, 4. / 7, 5. / 7]) - assert np.allclose(diversity, [2.166, 3.666, 4.500], atol=0.01) - - -def test_select(): - """ - Test case: 10 base classifiers; select 5 based on accuracy, - then the 3 most diverse - accuracies (/10): 4 6 1 2 9 8 7 9 3 2 - (should select indices_ 1, 4, 5, 6, 7) - diversity: 0 8 0 0 1 6 7 2 0 0 - (should select indices_ 1, 5, 6 as most diverse) - - """ - pool_classifiers = [create_base_classifier(1) for _ in range(10)] - - accuracies = np.array([4, 6, 1, 2, 9, 8, 7, 9, 3, 2]) / 10. - diversity = np.array([0, 8, 0, 0, 1, 6, 7, 2, 0, 0]) - target = DESKNN(pool_classifiers, k=7, pct_accuracy=5. / 10, - pct_diversity=3. / 10) - target.N_ = 5 - target.J_ = 3 - - selected_classifiers = target.select(accuracies, diversity) - expected = np.array([[1, 5, 6]]) - - assert np.array_equal(np.unique(selected_classifiers), np.unique(expected)) - - -def test_select_batch(): - """ - Test case: 10 base classifiers; select 5 based on accuracy, - then the 3 most diverse. - accuracies (/10): 4 6 1 2 9 8 7 9 3 2 - (should select indices_ 1, 4, 5, 6, 7) - diversity: 0 8 0 0 1 6 7 2 0 0 - (should select indices_ 1, 5, 6 as most diverse) - - """ - n_samples = 10 - pool_classifiers = [create_base_classifier(1) for _ in range(10)] - - accuracies = np.tile([4, 6, 1, 2, 9, 8, 7, 9, 3, 2], (n_samples, 1)) / 10. - diversity = np.tile([0, 8, 0, 0, 1, 6, 7, 2, 0, 0], (n_samples, 1)) - target = DESKNN(pool_classifiers, k=7, pct_accuracy=5. / 10, - pct_diversity=3. / 10) - target.N_ = 5 - target.J_ = 3 - - selected_classifiers = target.select(accuracies, diversity) - expected = np.tile([1, 5, 6], (n_samples, 1)) - - assert np.array_equal(np.unique(selected_classifiers), np.unique(expected)) - - -def test_select_less_diverse(): - """ - Test case: 10 base classifiers; select 5 based on accuracy, - then the 3 less diverse - accuracies (/10): 4 6 1 2 9 8 7 9 3 2 - (should select indices_ 1, 4, 5, 6, 7) - diversity: 0 8 0 0 1 6 7 2 0 0 - (should select indices_ 4, 5, 7 as most diverse) - - """ - pool_classifiers = [create_base_classifier(1) for _ in range(10)] - - accuracies = np.array([[4, 6, 1, 2, 9, 8, 7, 9, 3, 2]]) / 10. - diversity = np.array([[0, 8, 0, 0, 1, 6, 7, 2, 0, 0]]) - target = DESKNN(pool_classifiers, k=7, pct_accuracy=5. / 10, - pct_diversity=3. / 10, more_diverse=False) - target.N_ = 5 - target.J_ = 3 - - selected_classifiers = target.select(accuracies, diversity) - expected = np.array([[4, 5, 7]]) - - assert np.array_equal(np.unique(selected_classifiers), np.unique(expected)) - - -def test_input_diversity_parameter(): - X = np.random.rand(10, 2) - y = np.ones(10) - with pytest.raises(ValueError): - desknn = DESKNN(metric='abc') - desknn.fit(X, y) - - -def test_J_N_values(): - X = np.random.rand(10, 2) - y = np.ones(10) - with pytest.raises(ValueError): - desknn = DESKNN(pct_accuracy=0.5, pct_diversity=0) - desknn.fit(X, y) - - -def test_J_higher_than_N(): - X = np.random.rand(10, 2) - y = np.ones(10) - with pytest.raises(ValueError): - desknn = DESKNN(pct_accuracy=0.3, pct_diversity=0.5) - desknn.fit(X, y) - - -def test_pect_zero(): - X = np.random.rand(10, 2) - y = np.ones(10) - with pytest.raises(ValueError): - desknn = DESKNN(pct_accuracy=0.0, pct_diversity=0.0) - desknn.fit(X, y) - - -def create_base_classifier(value): - classifier = MagicMock() - classifier.predict.return_value = value - classifier.predict_proba.return_value = value - - return classifier - - -# Test if the class is raising an error when the base classifiers do not -# implements the predict_proba method. -# In this case the test should not raise an error since this class does not -# require base classifiers that can estimate probabilities -def test_predict_proba(): - X = np.random.randn(15, 5) - y = np.array([0, 1, 0, 0, 0] * 3) - clf1 = Perceptron() - clf1.fit(X, y) - DESKNN([clf1, clf1, clf1]).fit(X, y) - - -def test_soft_voting_no_proba(create_X_y): - from sklearn.linear_model import Perceptron - X, y = create_X_y - clf = Perceptron() - clf.fit(X, y) - with pytest.raises(ValueError): - DESKNN([clf, clf], voting='soft').fit(X, y) - - -@pytest.mark.parametrize('voting', [None, 'product', 1]) -def test_wrong_voting_value(voting, create_X_y, create_pool_classifiers): - X, y = create_X_y - pool = create_pool_classifiers - with pytest.raises(ValueError): - DESKNN(pool, voting=voting).fit(X, y) diff --git a/deslib/tests/des/test_des_mi.py b/deslib/tests/des/test_des_mi.py deleted file mode 100644 index 6cb55e84..00000000 --- a/deslib/tests/des/test_des_mi.py +++ /dev/null @@ -1,141 +0,0 @@ -from unittest.mock import MagicMock - -import numpy as np -import pytest -from sklearn.linear_model import Perceptron -from sklearn.utils.estimator_checks import check_estimator - -from deslib.des.des_mi import DESMI - - -def test_check_estimator(): - check_estimator(DESMI()) - - -# TODO: create test routine for the estimate_competence method - - -@pytest.mark.parametrize('alpha', [-1.0, -0.5, 0.0]) -def test_check_alpha_value(alpha, create_X_y): - X, y = create_X_y - with pytest.raises(ValueError): - desmi = DESMI(alpha=alpha) - desmi.fit(X, y) - - -@pytest.mark.parametrize('alpha', ['a', None, 'string', 1]) -def test_check_alpha_type(alpha, create_X_y): - X, y = create_X_y - with pytest.raises(TypeError): - desmi = DESMI(alpha=alpha) - desmi.fit(X, y) - - -@pytest.mark.parametrize('pct_accuracy', [-1.0, -0.5, 0.0, 1.01]) -def test_check_pct_accuracy_value(pct_accuracy, create_X_y): - X, y = create_X_y - with pytest.raises(ValueError): - desmi = DESMI(pct_accuracy=pct_accuracy) - desmi.fit(X, y) - - -# Test if the class is raising an error when the base classifiers do not -# implements the predict_proba method. -# In this case the test should not raise an error since this class does not -# require base classifiers that can estimate probabilities -def test_require_proba(): - X = np.random.randn(5, 5) - y = np.array([0, 1, 0, 0, 0]) - clf1 = Perceptron() - clf1.fit(X, y) - DESMI([clf1, clf1, clf1]) - - -def test_select_single_sample(): - des_mi = DESMI(pct_accuracy=0.7) - des_mi.N_ = 2 - competences = np.array([0.7, 0.2, 1.0]) - selected_clf = des_mi.select(competences) - expected = np.array([0, 2]) - assert np.array_equal(np.unique(selected_clf), np.unique(expected)) - - -def test_select_batch_samples(): - n_samples = 10 - des_mi = DESMI(pct_accuracy=0.7) - des_mi.N_ = 2 - competences = np.tile(np.array([0.7, 0.2, 1.0]), (n_samples, 1)) - selected_clf = des_mi.select(competences) - expected = np.tile(np.array([0, 2]), (n_samples, 1)) - assert np.array_equal(np.unique(selected_clf), np.unique(expected)) - - -def test_classify_with_ds_batch_samples(): - n_samples = 10 - - # simulated predictions of the pool of classifiers - predictions = np.tile(np.array([0, 1, 0]), (n_samples, 1)) - - desmi_test = DESMI() - desmi_test.n_classes_ = 2 - desmi_test.estimate_competence = MagicMock( - return_value=(np.ones((n_samples, 3)))) - desmi_test.select = MagicMock( - return_value=np.tile(np.array([[0, 2]]), (n_samples, 1))) - result = desmi_test.classify_with_ds(predictions) - assert np.allclose(result, np.zeros(10)) - - -def test_predict_proba_with_ds_soft(create_pool_classifiers): - expected = np.array([0.61, 0.39]) - DFP_mask = np.ones((1, 6)) - predictions = np.array([[0, 1, 0, 0, 1, 0]]) - probabilities = np.array([[[0.5, 0.5], [1, 0], [0.33, 0.67], - [0.5, 0.5], [1, 0], [0.33, 0.67]]]) - pool_classifiers = create_pool_classifiers + create_pool_classifiers - desmi_test = DESMI(pool_classifiers, DFP=True, voting='soft') - desmi_test.n_classes_ = 2 - selected_indices = np.array([[0, 1, 5]]) - desmi_test.estimate_competence = MagicMock(return_value=np.ones(6)) - desmi_test.select = MagicMock(return_value=selected_indices) - - predicted_proba = desmi_test.predict_proba_with_ds(predictions, - probabilities, - DFP_mask=DFP_mask) - assert np.isclose(predicted_proba, expected, atol=0.01).all() - - -def test_predict_proba_with_ds_hard(create_pool_classifiers): - expected = np.array([0.666, 0.333]) - DFP_mask = np.ones((1, 6)) - predictions = np.array([[0, 1, 0, 0, 1, 0]]) - probabilities = np.array([[[0.5, 0.5], [1, 0], [0.33, 0.67], - [0.5, 0.5], [1, 0], [0.33, 0.67]]]) - pool_classifiers = create_pool_classifiers + create_pool_classifiers - desmi_test = DESMI(pool_classifiers, DFP=True, voting='hard') - desmi_test.n_classes_ = 2 - selected_indices = np.array([[0, 1, 5]]) - desmi_test.estimate_competence = MagicMock(return_value=np.ones(6)) - desmi_test.select = MagicMock(return_value=selected_indices) - - predicted_proba = desmi_test.predict_proba_with_ds(predictions, - probabilities, - DFP_mask=DFP_mask) - assert np.isclose(predicted_proba, expected, atol=0.01).all() - - -def test_soft_voting_no_proba(create_X_y): - from sklearn.linear_model import Perceptron - X, y = create_X_y - clf = Perceptron() - clf.fit(X, y) - with pytest.raises(ValueError): - DESMI([clf, clf, clf, clf], voting='soft').fit(X, y) - - -@pytest.mark.parametrize('voting', [None, 'product', 1]) -def test_wrong_voting_value(voting, create_X_y, create_pool_classifiers): - X, y = create_X_y - pool = create_pool_classifiers - with pytest.raises(ValueError): - DESMI(pool, voting=voting).fit(X, y) diff --git a/deslib/tests/des/test_desp.py b/deslib/tests/des/test_desp.py deleted file mode 100644 index 8cd7fea2..00000000 --- a/deslib/tests/des/test_desp.py +++ /dev/null @@ -1,79 +0,0 @@ -import numpy as np -from sklearn.linear_model import Perceptron -from sklearn.utils.estimator_checks import check_estimator - -from deslib.des.des_p import DESP - - -def test_check_estimator(): - check_estimator(DESP()) - - -# Test the estimate competence method receiving n samples as input -def test_estimate_competence_batch(example_estimate_competence, - create_pool_classifiers): - X, y, neighbors, distances, dsel_processed, _ = example_estimate_competence - - expected = np.array([[0.57142857, 0.4285714, 0.57142857], - [0.71428571, 0.2857142, 0.71428571], - [0.2857142, 0.71428571, 0.2857142]]) - - des_p_test = DESP(create_pool_classifiers) - des_p_test.fit(X, y) - competences = des_p_test.estimate_competence(neighbors, distances) - assert np.allclose(competences, expected, atol=0.01) - - -def test_select_two_classes(): - des_p_test = DESP() - des_p_test.n_classes_ = 2 - expected = np.array([[True, False, True], - [True, False, True], - [False, True, False]]) - - competences = np.array([[0.51, 0.0, 0.51], - [0.51, 0.0, 0.51], - [0.49, 1.0, 0.49]]) - - selected = des_p_test.select(competences) - - assert np.array_equal(selected, expected) - - -# In this example, since the number of classes is 3, the competence level -# expected to be selected is > 0.33 -def test_select_three_classes(): - des_p_test = DESP() - des_p_test.n_classes_ = 3 - expected = np.array([[True, False, True], - [True, False, True], - [False, True, False]]) - - competences = np.array([[0.34, 0.32, 1.0], - [0.50, 0.30, 1.01], - [0.25, 1.0, 0.25]]) - - selected = des_p_test.select(competences) - - assert np.array_equal(selected, expected) - - -def test_select_none_competent(): - n_classifiers = 3 - des_p_test = DESP() - des_p_test.n_classes_ = 2 - competences = np.ones(n_classifiers) * 0.49 - indices = des_p_test.select(competences) - expected = np.array([[True, True, True]]) - assert np.array_equal(expected, indices) - - -# Test if the class is raising an error when the base classifiers do not -# implements the predict_proba method. In this case the test should not raise -# an error since this class does not require base classifiers that -# can estimate probabilities -def test_predict_proba(create_X_y): - X, y = create_X_y - clf1 = Perceptron() - clf1.fit(X, y) - DESP([clf1, clf1]).fit(X, y) diff --git a/deslib/tests/des/test_knop.py b/deslib/tests/des/test_knop.py deleted file mode 100644 index 25e243d7..00000000 --- a/deslib/tests/des/test_knop.py +++ /dev/null @@ -1,81 +0,0 @@ -from unittest.mock import Mock - -import numpy as np -import pytest -from sklearn.linear_model import Perceptron -from sklearn.utils.estimator_checks import check_estimator - -from deslib.des.knop import KNOP - - -def test_check_estimator(): - check_estimator(KNOP()) - - -# Test the estimate competence method receiving n samples as input -def test_estimate_competence_batch(example_estimate_competence, - create_pool_classifiers): - X, y, neighbors, distances, _, _ = example_estimate_competence - query = np.ones((3, 2)) - expected = np.array([[4.0, 3.0, 4.0], - [5.0, 2.0, 5.0], - [2.0, 5.0, 2.0]]) - - knop_test = KNOP(create_pool_classifiers) - knop_test.fit(X, y) - knop_test.neighbors = neighbors - knop_test.distances = distances - - knop_test._get_similar_out_profiles = Mock(return_value=(None, neighbors)) - probabilities = np.zeros((3, 6)) - - competences = knop_test.estimate_competence_from_proba(query, - probabilities) - assert np.allclose(competences, expected, atol=0.01) - - -def test_weights_zero(): - knop_test = KNOP() - competences = np.zeros((1, 3)) - result = knop_test.select(competences) - - assert np.all(result) - - -def test_fit(example_estimate_competence, create_pool_classifiers): - X, y = example_estimate_competence[0:2] - - knop_test = KNOP(create_pool_classifiers) - knop_test.fit(X, y) - expected_scores = np.array([[0.5, 0.5], [1.0, 0.0], [0.33, 0.67]]) - expected_scores = np.tile(expected_scores, (15, 1, 1)) - - assert np.array_equal(expected_scores, knop_test.dsel_scores_) - - # Assert the roc_algorithm_ is fitted to the scores (decision space) - # rather than the features (feature space) - expected_roc_data = knop_test.dsel_scores_[:, :, 0] - assert np.array_equal(knop_test.op_knn_._fit_X, expected_roc_data) - - -# Test if the class is raising an error when the base classifiers do not -# implements the predict_proba method. Should raise an exception when the -# base classifier cannot estimate posterior probabilities (predict_proba) -# Using Perceptron classifier as it does not implements predict_proba. -def test_not_predict_proba(create_X_y): - X, y = create_X_y - - clf1 = Perceptron() - clf1.fit(X, y) - with pytest.raises(ValueError): - knop = KNOP([clf1, clf1]) - knop.fit(X, y) - - -def test_select(): - knop_test = KNOP() - competences = np.ones(3) - competences[0] = 0 - expected = np.atleast_2d([False, True, True]) - selected = knop_test.select(competences) - assert np.array_equal(expected, selected) diff --git a/deslib/tests/des/test_knorae.py b/deslib/tests/des/test_knorae.py deleted file mode 100644 index b7a65b3d..00000000 --- a/deslib/tests/des/test_knorae.py +++ /dev/null @@ -1,68 +0,0 @@ -import numpy as np -import pytest -from sklearn.linear_model import Perceptron -from sklearn.utils.estimator_checks import check_estimator - -from deslib.des.knora_e import KNORAE - - -def test_check_estimator(): - check_estimator(KNORAE()) - - -def test_estimate_competence_batch(example_estimate_competence, - create_pool_classifiers): - X, y, neighbors, distances, _, _ = example_estimate_competence - - expected = np.array([[1.0, 0.0, 1.0], - [2.0, 0.0, 2.0], - [0.0, 3.0, 0.0]]) - - knora_e_test = KNORAE(create_pool_classifiers) - knora_e_test.fit(X, y) - - competences = knora_e_test.estimate_competence(neighbors, - distances=distances) - assert np.allclose(competences, expected) - - -@pytest.mark.parametrize('index, expected', [(0, [[True, False, True]]), - (1, [[True, False, True]]), - (2, [[False, True, False]])]) -def test_select(index, expected, create_pool_classifiers, - example_estimate_competence): - X, y, neighbors, distances, _, _ = example_estimate_competence - - knora_e_test = KNORAE(create_pool_classifiers) - knora_e_test.fit(X, y) - neighbors = neighbors[index, :].reshape(1, -1) - distances = distances[index, :].reshape(1, -1) - competences = knora_e_test.estimate_competence(neighbors, - distances=distances) - selected = knora_e_test.select(competences) - - assert np.array_equal(selected, expected) - - -# No classifier here is selected, since the always predict class 2 where there -# are only samples labeled as class 0 and 1 -# in the region of competence -def test_select_none_competent(): - knora_e_test = KNORAE() - competences = np.zeros(100) - selected = knora_e_test.select(competences) - expected = np.atleast_2d([True] * 100) - - assert np.array_equal(expected, selected) - - -# Test if the class is raising an error when the base classifiers do not -# implements the predict_proba method. In this case the test should not raise -# an error since this class does not require base classifiers that -# can estimate probabilities -def test_predict_proba(create_X_y): - X, y = create_X_y - - clf1 = Perceptron() - clf1.fit(X, y) - KNORAE([clf1, clf1]).fit(X, y) diff --git a/deslib/tests/des/test_knorau.py b/deslib/tests/des/test_knorau.py deleted file mode 100644 index ee83da5a..00000000 --- a/deslib/tests/des/test_knorau.py +++ /dev/null @@ -1,54 +0,0 @@ -import numpy as np -from sklearn.linear_model import Perceptron -from sklearn.utils.estimator_checks import check_estimator - -from deslib.des.knora_u import KNORAU - - -def test_check_estimator(): - check_estimator(KNORAU()) - - -# Test the estimate competence method receiving n samples as input -def test_estimate_competence_batch(example_estimate_competence, - create_pool_classifiers): - - X, y, neighbors = example_estimate_competence[0:3] - - expected = np.array([[4.0, 3.0, 4.0], - [5.0, 2.0, 5.0], - [2.0, 5.0, 2.0]]) - knora_u_test = KNORAU(create_pool_classifiers) - knora_u_test.fit(X, y) - - competences = knora_u_test.estimate_competence(neighbors) - assert np.allclose(competences, expected, atol=0.01) - - -def test_weights_zero(): - knorau_test = KNORAU() - competences = np.zeros((1, 3)) - result = knorau_test.select(competences) - - assert np.all(result) - - -# Test if the class is raising an error when the base classifiers do not -# implements the predict_proba method. In this case the test should not raise -# an error since this class does not require base classifiers that -# can estimate probabilities -def test_predict_proba(create_X_y): - X, y = create_X_y - - clf1 = Perceptron() - clf1.fit(X, y) - KNORAU([clf1, clf1]).fit(X, y) - - -def test_select(): - knorau_test = KNORAU() - competences = np.ones(3) - competences[0] = 0 - expected = np.atleast_2d([False, True, True]) - selected = knorau_test.select(competences) - assert np.array_equal(expected, selected) diff --git a/deslib/tests/des/test_meta_des.py b/deslib/tests/des/test_meta_des.py deleted file mode 100644 index c1769f3f..00000000 --- a/deslib/tests/des/test_meta_des.py +++ /dev/null @@ -1,255 +0,0 @@ -from unittest.mock import MagicMock - -import numpy as np -import pytest -from sklearn.linear_model import Perceptron -from sklearn.naive_bayes import GaussianNB -from sklearn.svm import SVC -from sklearn.utils.estimator_checks import check_estimator -from sklearn.utils.validation import check_is_fitted - -from deslib.des.meta_des import METADES - - -def test_check_estimator(): - check_estimator(METADES(Kp=1)) - - -# ---------------------- Testing Hyper-parameters ----------------------- -@pytest.mark.parametrize('model,', [SVC(), Perceptron()]) -def test_meta_classifier_not_predict_proba(create_pool_classifiers, model): - X = np.random.rand(10, 2) - y = np.ones(10) - y[:5] = 0 - with pytest.raises(ValueError): - meta = METADES(create_pool_classifiers, model) - meta.fit(X, y) - - -def test_meta_classifier_not_none(): - X = np.random.rand(100, 2) - y = np.random.randint(0, 2, 100) - meta = METADES(meta_classifier=GaussianNB()) - meta.fit(X, y) - check_is_fitted(meta.meta_classifier_, "classes_") - assert isinstance(meta.meta_classifier_, GaussianNB) - - -def test_fitted_meta_classifier(): - X = np.random.rand(100, 2) - y = np.random.randint(0, 2, 100) - meta = METADES(meta_classifier=GaussianNB()) - meta.fit(X, y) - - meta2 = METADES(meta_classifier=meta.meta_classifier_) - meta2.fit(X, y) - assert meta.meta_classifier_ == meta2.meta_classifier_ - - -@pytest.mark.parametrize('Hc', ['a', None, 0.2, -1]) -def test_parameter_Hc(Hc, create_pool_classifiers): - X = np.random.rand(10, 2) - y = np.ones(10) - with pytest.raises((ValueError, TypeError)): - meta = METADES(create_pool_classifiers, Hc=Hc) - meta.fit(X, y) - - -@pytest.mark.parametrize('Kp', ['a', None, 0.2, -1, 0, 1]) -def test_parameter_Kp(Kp, create_pool_classifiers): - X = np.random.rand(10, 2) - y = np.ones(10) - with pytest.raises((ValueError, TypeError)): - meta = METADES(create_pool_classifiers, Kp=Kp) - meta.fit(X, y) - - -@pytest.mark.parametrize('selection_threshold', ['a', None, 0, -1, 0.45]) -def test_parameter_gamma(selection_threshold, create_pool_classifiers): - X = np.random.rand(10, 2) - y = np.ones(10) - with pytest.raises((ValueError, TypeError)): - meta = METADES(create_pool_classifiers, - selection_threshold=selection_threshold) - meta.fit(X, y) - - -# -------------------------- Testing Methods ----------------------- -def test_compute_meta_features(example_estimate_competence, - create_pool_classifiers): - - X, y, nn, _, dsel_processed, dsel_scores = example_estimate_competence - - query = np.ones((1, 2)) - pool = create_pool_classifiers - meta_test = METADES(pool_classifiers=[pool[0]]) - meta_test.n_classifiers_ = 1 - meta_test.k_ = 7 - meta_test.Kp_ = 5 - # Considering only one classifier in the pool (index = 0) - meta_test.DSEL_processed_ = dsel_processed[:, 0].reshape(-1, 1) - meta_test.dsel_scores_ = dsel_scores[:, 0, :].reshape(15, 1, 2) - meta_test.DSEL_target_ = y - meta_test.n_classes_ = 2 - - neighbors_op = nn[2, 0:meta_test.Kp] - - # Expected values for each meta feature based on the data of ex1. - expected_f1 = [1.0, 0.0, 1.0, 1.0, 1.0, 0.0, 0.0] - expected_f2 = [1.0, 0.0, 0.0, 1.0, 1.0, 0.0, 1.0] - expected_f3 = [4.0 / 7.0] - expected_f4 = [0.0, 1.0, 1.0, 1.0, 0.0] - expected_f5 = [0.5] - - scores = np.empty( - (query.shape[0], meta_test.n_classifiers_, meta_test.n_classes_)) - for index, clf in enumerate(meta_test.pool_classifiers): - scores[:, index, :] = clf.predict_proba(query) - - meta_features = meta_test.compute_meta_features(scores, nn[0, :], - neighbors_op) - expected = np.asarray( - expected_f1 + expected_f2 + expected_f3 + expected_f4 + expected_f5) - assert np.array_equal(meta_features, expected.reshape(1, -1)) - - -# Test the estimate competence function considering -# 3 base classifiers and 1 test sample -def test_estimate_competence(example_estimate_competence, - create_pool_classifiers): - _, y, nn, _, dsel_processed, dsel_scores = example_estimate_competence - - query = np.ones((1, 2)) - meta_test = METADES(create_pool_classifiers) - meta_test.n_classifiers_ = 3 - meta_test.k_ = 7 - meta_test.Kp_ = 5 - # Set the state of the system which is set by the fit method. - meta_test.DSEL_processed_ = dsel_processed - meta_test.dsel_scores_ = dsel_scores - meta_test.DSEL_target_ = y - meta_test.n_classes_ = 2 - - meta_test.meta_classifier_ = GaussianNB() - - meta_test._get_similar_out_profiles = MagicMock( - return_value=(None, nn[0, 0:meta_test.Kp])) - meta_test.meta_classifier_.predict_proba = MagicMock( - return_value=np.array([[0.2, 0.8], [1.0, 0.0], [0.2, 0.8]])) - - probabilities = [] - for clf in meta_test.pool_classifiers: - probabilities.append(clf.predict_proba(query)) - - probabilities = np.array(probabilities).transpose((1, 0, 2)) - - expected = np.array([[0.8, 0.0, 0.8]]) - competences = meta_test.estimate_competence_from_proba(nn[0, :], - probabilities) - assert np.array_equal(competences, expected) - - -# Test the estimate competence function considering 3 base classifiers -# and 3 test samples. -def test_estimate_competence_batch(example_estimate_competence, - create_pool_classifiers): - _, y, nn, _, dsel_processed, dsel_scores = example_estimate_competence - - query = np.ones((3, 1)) - meta_test = METADES(pool_classifiers=create_pool_classifiers) - meta_test.n_classifiers_ = 3 - n_meta_features = 21 - meta_test.meta_classifier_ = GaussianNB - # Set the state of the system which is set by the fit method. - meta_test.DSEL_processed_ = dsel_processed - meta_test.dsel_scores_ = dsel_scores - meta_test.DSEL_target_ = y - nn = nn - - meta_test._get_similar_out_profiles = MagicMock( - return_value=(None, nn[:, 0:meta_test.Kp])) - meta_test.compute_meta_features = MagicMock( - return_value=np.ones((9, n_meta_features))) - meta_test.meta_classifier_.predict_proba = MagicMock( - return_value=np.tile([0.0, 0.8], (9, 1))) - - probabilities = [] - for clf in meta_test.pool_classifiers: - probabilities.append(clf.predict_proba(query)) - - probabilities = np.array(probabilities).transpose((1, 0, 2)) - - expected = np.ones((3, 3)) * 0.8 - competences = meta_test.estimate_competence_from_proba(query, nn, - probabilities) - assert np.array_equal(competences, expected) - - -# Test select passing a single sample -def test_select(): - meta_test = METADES() - competences = np.asarray( - [0.8, 0.6, 0.7, 0.2, 0.3, 0.4, 0.6, 0.1, 1.0, 0.98]) - expected = np.asarray( - [True, True, True, False, False, False, True, False, True, True]) - selected_matrix = meta_test.select(competences) - assert np.array_equal(selected_matrix, expected.reshape(1, -1)) - - -# test select passing 10 samples -def test_select_batch(): - meta_test = METADES() - competences = np.tile( - np.array([0.8, 0.6, 0.7, 0.2, 0.3, 0.4, 0.6, 0.1, 1.0, 0.98]), (10, 1)) - expected = np.tile( - [True, True, True, False, False, False, True, False, True, True], - (10, 1)) - selected_matrix = meta_test.select(competences) - assert np.array_equal(selected_matrix, expected) - - -# 10 samples, no classifier is selected so the array should return all -# True (10 x 3) -def test_select_no_competent_classifiers_batch(): - meta_test = METADES() - meta_test.n_classifiers_ = 3 - competences = np.zeros((10, meta_test.n_classifiers_)) - selected_matrix = meta_test.select(competences) - assert np.all(selected_matrix) - - -# Test the sample selection mechanism considering 5 test samples and 15 base -# classifiers. The agreement is computed -# for all samples at the same time -def test_sample_selection(): - meta_test = METADES() - meta_test.n_classifiers_ = 15 - meta_test.DSEL_processed_ = np.ones((5, 15)) - meta_test.DSEL_processed_[(1, 3, 4), 5:] = 0 - expected = np.asarray([1, 1 / 3, 1, 1 / 3, 1 / 3]) - value = meta_test._sample_selection_agreement() - assert np.array_equal(value, expected) - - -def test_sample_selection_working(): - meta_test = METADES() - meta_test.n_classifiers_ = 15 - meta_test.DSEL_processed_ = np.ones((5, 15)) - meta_test.DSEL_processed_[(1, 3, 4), 5:] = 0 - expected = np.asarray([1, 1 / 3, 1, 1 / 3, 1 / 3]) - value = meta_test._sample_selection_agreement() - assert np.array_equal(value, expected) - - -# Test if the class is raising an error when the base classifiers do not -# implements the predict_proba method. Should raise an exception when the -# base classifier cannot estimate posterior probabilities (predict_proba) -# Using Perceptron classifier as it does not implements predict_proba. -def test_not_predict_proba(create_X_y): - X, y = create_X_y - - clf1 = Perceptron() - clf1.fit(X, y) - with pytest.raises(ValueError): - meta = METADES([clf1, clf1]) - meta.fit(X, y) diff --git a/deslib/tests/des/test_probabilistic.py b/deslib/tests/des/test_probabilistic.py deleted file mode 100644 index 5ba81e13..00000000 --- a/deslib/tests/des/test_probabilistic.py +++ /dev/null @@ -1,296 +0,0 @@ -import numpy as np -import pytest -from sklearn.linear_model import Perceptron -from sklearn.utils.estimator_checks import check_estimator - -from deslib.des.probabilistic import (BaseProbabilistic, - Logarithmic, - Exponential, - RRC, - DESKL, - MinimumDifference) - - -def test_check_estimator_RRC(): - check_estimator(RRC()) - - -def test_check_estimator_DESKL(): - check_estimator(DESKL()) - - -def test_check_estimator_Logarithmic(): - check_estimator(Logarithmic()) - - -def test_check_estimator_Exponential(): - check_estimator(Exponential()) - - -def test_check_estimator_MinimumDifference(): - check_estimator(MinimumDifference()) - - -# Test if the class is raising an error when the base classifiers do not -# implements the predict_proba method. Should raise an exception when the -# base classifier cannot estimate posterior probabilities (predict_proba) -# Using Perceptron classifier as it does not implements predict_proba. -def test_not_predict_proba(create_X_y): - X, y = create_X_y - - clf1 = Perceptron() - clf1.fit(X, y) - with pytest.raises(ValueError): - BaseProbabilistic([clf1, clf1]).fit(X, y) - - -# Being all ones, all base classifiers are deemed competent -def test_select_all_ones(): - competences = np.ones(100) - probabilistic_test = BaseProbabilistic() - probabilistic_test.n_classes_ = 2 - selected_matrix = probabilistic_test.select(competences) - assert selected_matrix.all() - - -# Being all zeros, no base classifier is deemed competent, so the system -# selects all of them -def test_select_all_zeros(): - competences = np.zeros(100) - probabilistic_test = BaseProbabilistic() - probabilistic_test.n_classes_ = 2 - selected_matrix = probabilistic_test.select(competences) - assert selected_matrix.all() - - -# Being all zeros, no base classifier is deemed competent, so the system -# selects all of them -def test_select_random_classifier(): - competences = np.random.rand(1, 100) - expected = (competences > 0.25) - probabilistic_test = BaseProbabilistic() - probabilistic_test.n_classes_ = 4 - indices = probabilistic_test.select(competences) - assert np.array_equal(indices, expected) - - -# Being all zeros, no base classifier is deemed competent, so the system -# selects all of them -def test_select_threshold(): - competences = np.random.rand(1, 100) - expected = (competences > 0.5) - - probabilistic_test = BaseProbabilistic() - probabilistic_test.selection_threshold = 0.5 - indices = probabilistic_test.select(competences) - assert np.array_equal(indices, expected) - - -# Test the potential function calculation. The return value should be zero in -# this test. -def test_potential_function_zeros(): - dists = np.zeros(10) - value = BaseProbabilistic.potential_func(dists) - assert np.array_equal(value, np.ones(10)) - - -# Test the potential function calculation. Higher values for distances should -# obtain a lower value in the results -def test_potential_function(): - dists = np.array([1.0, 0.5, 2, 0.33]) - value = BaseProbabilistic.potential_func(dists) - assert np.allclose(value, [0.3679, 0.7788, 0.0183, 0.8968], atol=0.001) - - -# Test the potential function calculation. Higher values for distances should -# obtain a lower value in the results -def test_potential_function_batch(): - dists = np.tile([1.0, 0.5, 2, 0.33], (10, 1)) - value = BaseProbabilistic.potential_func(dists) - expected = np.tile([0.3679, 0.7788, 0.0183, 0.8968], (10, 1)) - assert np.allclose(value, expected, atol=0.001) - - -def test_estimate_competence_batch(): - n_samples = 10 - query = np.ones((n_samples, 2)) - probabilistic_test = BaseProbabilistic() - probabilistic_test.k_ = 7 - distances = np.tile([0.5, 1.0, 2.0], (n_samples, 1)) - neighbors = np.tile([0, 1, 2], (n_samples, 1)) - - probabilistic_test.C_src_ = np.array( - [[0.5, 0.2, 0.8], [1.0, 1.0, 1.0], [1.0, 0.6, 0.3]]) - expected = np.tile([0.665, 0.458, 0.855], (n_samples, 1)) - competence = probabilistic_test.estimate_competence( - competence_region=neighbors, - distances=distances) - assert np.allclose(competence, expected, atol=0.01) - - -# Test the estimate competence function when the competence source is equal -# to zero. The competence should also be zero. -def test_estimate_competence_zeros(example_estimate_competence): - distances = example_estimate_competence[3] - query = np.atleast_2d([1, 1]) - probabilistic_test = BaseProbabilistic() - probabilistic_test.k_ = 7 - - distances = distances[0, 0:3].reshape(1, -1) - neighbors = np.array([[0, 2, 1]]) - probabilistic_test.C_src_ = np.zeros((3, 3)) - competence = probabilistic_test.estimate_competence( - competence_region=neighbors, - distances=distances) - assert np.sum(competence) == 0.0 - - -# Test the estimate competence function when the competence source is equal -# to one. The competence should also be ones. -def test_estimate_competence_ones(example_estimate_competence): - distances = example_estimate_competence[3] - query = np.atleast_2d([1, 1]) - probabilistic_test = BaseProbabilistic() - probabilistic_test.k_ = 7 - - distances = distances[0, 0:3].reshape(1, -1) - neighbors = np.array([[0, 2, 1]]) - probabilistic_test.C_src_ = np.ones((3, 3)) - competence = probabilistic_test.estimate_competence(neighbors, - distances) - assert (competence == 1.0).all() - - -""" Test the source_competence using the rrc method. Here we consider the same -values from the example applied in the test_prob_functions.py to assert if the -source_competence function call the ccprmod correctly and fill the competence -source (C_src) with the correct results. - -The scores used are: [[0.3, 0.6, 0.1], [1.0 / 3, 1.0 / 3, 1.0 / 3], - [0.5, 0.2, 0.3], [0.5, 0.2, 0.3]] -The correct labels are: [1, 0, 0, 1] -The expected value should be: an np.array (4,1) with -the values = [[0.7849], [0.3328], [0.6428], [0.1194]] -""" - - -def test_source_competence_rrc(): - rrc_test = RRC() - rrc_test.n_classifiers_ = 1 - rrc_test.dsel_scores_ = np.array([[[0.3, 0.6, 0.1], - [1.0 / 3, 1.0 / 3, 1.0 / 3], - [0.5, 0.2, 0.3], - [0.5, 0.2, 0.3]]]).reshape(4, 1, 3) - rrc_test.DSEL_target_ = [1, 0, 0, 1] - rrc_test.n_classes_ = 3 - rrc_test.n_samples_ = 4 - C_src = rrc_test.source_competence() - expected = np.array([[0.7849], [0.3328], [0.6428], [0.1194]]) - assert np.allclose(C_src, expected, atol=0.01) - - -# Test the source_competence estimation for the Kullback-Leibler method. Here -# we consider the same values applied in the test_prob_functions.py to assert -# if the source_competence function fill the competence source (C_src) with -# the correct results. -# -# The scores used are: [[0.33, 0.33, 0.33], [1.0, 0.0, 0.0], [1.0, 0.0, 0.0]] -# The matrix with correct predictions is: [False, True, False] -# The expected value should be: an np.array (3,1) with -# the values = [[0.0], [1.0], [-1.0]] -def test_source_competence_kl(): - entropy_test = DESKL() - entropy_test.n_classifiers_ = 1 - entropy_test.dsel_scores_ = np.array([[[0.33, 0.33, 0.33], - [1.0, 0.0, 0.0], - [1.0, 0.0, 0.0]]]).reshape(3, 1, 3) - entropy_test.DSEL_processed_ = np.array([[False], [True], [False]]) - entropy_test.n_classes_ = 3 - entropy_test.n_samples_ = 3 - C_src = entropy_test.source_competence() - expected = np.array([[0.0], [1.0], [-1.0]]) - assert np.allclose(C_src, expected, atol=0.01) - - -# Test the source_competence estimation for the Minimum difference method. -# Here we consider the same values applied in the test_prob_functions.py to -# assert if the source_competence function fill the competence source -# (C_src) with the correct results. -# -# The scores used are: [[0.3, 0.6, 0.1], [1.0 / 3, 1.0 / 3, 1.0 / 3], -# [0.5, 0.2, 0.3], [0.5, 0.2, 0.3]] -# The correct labels are: [1, 0, 0, 1] -# The expected value should be: an np.array (4,1) with -# the values = [[0.7849], [0.3328], [0.6428], [0.1194]] -def test_source_competence_minimum_difference(): - md_test = MinimumDifference() - md_test.n_classifiers_ = 1 - md_test.dsel_scores_ = np.array([[[0.3, 0.6, 0.1], - [1.0 / 3, 1.0 / 3, 1.0 / 3], - [0.5, 0.2, 0.3], - [0.5, 0.2, 0.3]]]).reshape(4, 1, 3) - - md_test.DSEL_target_ = [1, 0, 0, 1] - md_test.n_classes_ = 3 - md_test.n_samples_ = 4 - C_src = md_test.source_competence() - expected = np.array([[0.3], [0.0], [0.2], [-0.3]]) - assert np.allclose(C_src, expected, atol=0.01) - - -# Test the source_competence using the logarithmic method. Here we consider -# the same values applied in the test_prob_functions.py to assert if the source -# competence function fill the competence source -# (C_src) with the correct results. -# -# The scores used are: [[0.67, 0.33, 0.0], [1.0, 0.0, 0.0], [0.0, 1.0, 0.0]] -# The correct labels are: [1, 1, 1], so the supports for the correct -# class are: [0.33, 0.0, 1.0] -# The expected value should be: an np.array (3,1) with -# the values = [[0.0], [-1.0], [1.0]]] -def test_source_competence_logarithmic(): - log_test = Logarithmic() - log_test.n_classifiers_ = 1 - log_test.dsel_scores_ = np.array([[[0.67, 0.33, 0.0], - [1.0, 0.0, 0.0], - [0.0, 1.0, 0.0]]]).reshape(3, 1, 3) - - log_test.DSEL_target_ = [1, 1, 1] - log_test.n_classes_ = 3 - log_test.n_samples_ = 3 - C_src = log_test.source_competence() - expected = np.array([[0.0], [-1.0], [1.0]]) - assert np.allclose(C_src, expected, atol=0.01) - - -# Test the source_competence using the exponential method. Here we consider -# the same values applied in the test_prob_functions.py to assert if the -# source_competence function fill the competence source (C_src) with the -# correct results. -# -# Only two classes_ are considered in this example. -# The scores used are: [[0.5, 0.5], [1.0, 0.0], [0.0, 1.0]]. -# The correct labels are: [1, 1, 1], so the supports for the correct -# class are: [0.5, 0.0, 1.0]. -# The expected value should be: an np.array (3,1) with -# the values = [[0.0], [-1.0], [1.0]]]. -def test_source_competence_exponential(): - exp_test = Exponential() - exp_test.n_classifiers_ = 1 - exp_test.dsel_scores_ = np.array([[[0.5, 0.5], - [1.0, 0.0], - [0.0, 1.0]]]).reshape(3, 1, 2) - - exp_test.DSEL_target_ = [1, 1, 1] - exp_test.n_classes_ = 2 - exp_test.n_samples_ = 3 - C_src = exp_test.source_competence() - expected = np.array([[0.0], [-1.0], [1.0]]) - assert np.allclose(C_src, expected, atol=0.01) - - -def test_knne_not_allowed(create_X_y): - X, y = create_X_y - - with pytest.raises(ValueError): - BaseProbabilistic(knn_classifier='knne').fit(X, y) diff --git a/deslib/tests/expected_values/des_clustering_proba_integration.npy b/deslib/tests/expected_values/des_clustering_proba_integration.npy deleted file mode 100644 index 633f3821..00000000 Binary files a/deslib/tests/expected_values/des_clustering_proba_integration.npy and /dev/null differ diff --git a/deslib/tests/expected_values/desknn_proba_integration.npy b/deslib/tests/expected_values/desknn_proba_integration.npy deleted file mode 100644 index c19bf87c..00000000 Binary files a/deslib/tests/expected_values/desknn_proba_integration.npy and /dev/null differ diff --git a/deslib/tests/expected_values/desknn_probas_DFP.npy b/deslib/tests/expected_values/desknn_probas_DFP.npy deleted file mode 100644 index 4df05fc6..00000000 Binary files a/deslib/tests/expected_values/desknn_probas_DFP.npy and /dev/null differ diff --git a/deslib/tests/expected_values/desp_proba_DFP.npy b/deslib/tests/expected_values/desp_proba_DFP.npy deleted file mode 100644 index 4dfaec54..00000000 Binary files a/deslib/tests/expected_values/desp_proba_DFP.npy and /dev/null differ diff --git a/deslib/tests/expected_values/desp_proba_integration.npy b/deslib/tests/expected_values/desp_proba_integration.npy deleted file mode 100644 index d8fb23c6..00000000 Binary files a/deslib/tests/expected_values/desp_proba_integration.npy and /dev/null differ diff --git a/deslib/tests/expected_values/kne_knn_proba_integration.npy b/deslib/tests/expected_values/kne_knn_proba_integration.npy deleted file mode 100644 index 229717f9..00000000 Binary files a/deslib/tests/expected_values/kne_knn_proba_integration.npy and /dev/null differ diff --git a/deslib/tests/expected_values/kne_proba_DFP.npy b/deslib/tests/expected_values/kne_proba_DFP.npy deleted file mode 100644 index 1bdce0b7..00000000 Binary files a/deslib/tests/expected_values/kne_proba_DFP.npy and /dev/null differ diff --git a/deslib/tests/expected_values/kne_proba_integration.npy b/deslib/tests/expected_values/kne_proba_integration.npy deleted file mode 100644 index 438d0734..00000000 Binary files a/deslib/tests/expected_values/kne_proba_integration.npy and /dev/null differ diff --git a/deslib/tests/expected_values/knop_proba_integration.npy b/deslib/tests/expected_values/knop_proba_integration.npy deleted file mode 100644 index 15cc533b..00000000 Binary files a/deslib/tests/expected_values/knop_proba_integration.npy and /dev/null differ diff --git a/deslib/tests/expected_values/mcb_proba_DFP.npy b/deslib/tests/expected_values/mcb_proba_DFP.npy deleted file mode 100644 index 059facfb..00000000 Binary files a/deslib/tests/expected_values/mcb_proba_DFP.npy and /dev/null differ diff --git a/deslib/tests/expected_values/mcb_proba_integration.npy b/deslib/tests/expected_values/mcb_proba_integration.npy deleted file mode 100644 index f4e19772..00000000 Binary files a/deslib/tests/expected_values/mcb_proba_integration.npy and /dev/null differ diff --git a/deslib/tests/expected_values/ola_proba_DFP.npy b/deslib/tests/expected_values/ola_proba_DFP.npy deleted file mode 100644 index f69bc9ee..00000000 Binary files a/deslib/tests/expected_values/ola_proba_DFP.npy and /dev/null differ diff --git a/deslib/tests/expected_values/ola_proba_integration.npy b/deslib/tests/expected_values/ola_proba_integration.npy deleted file mode 100644 index b8888393..00000000 Binary files a/deslib/tests/expected_values/ola_proba_integration.npy and /dev/null differ diff --git a/deslib/tests/static/__init__.py b/deslib/tests/static/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/deslib/tests/static/test_oracle.py b/deslib/tests/static/test_oracle.py deleted file mode 100644 index 7060398e..00000000 --- a/deslib/tests/static/test_oracle.py +++ /dev/null @@ -1,66 +0,0 @@ -import numpy as np -from sklearn.datasets import make_classification -from sklearn.ensemble import RandomForestClassifier - -from deslib.static.oracle import Oracle - - -def test_predict(create_X_y, create_pool_classifiers): - X, y = create_X_y - - oracle_test = Oracle(create_pool_classifiers) - oracle_test.fit(X, y) - predicted_labels = oracle_test.predict(X, y) - assert np.equal(predicted_labels, y).all() - - assert oracle_test.score(X, y) == 1.0 - - -# All classifiers predicts the same label. This test only the samples -# with label == 0 are correctly classified by the Oracle. -# The misclassified samples are set to -1. -def test_predict_all_same(create_X_y, create_pool_all_agree): - X, y = create_X_y - - expected = y - oracle_test = Oracle(create_pool_all_agree) - oracle_test.fit(X, y) - expected[expected == 1] = 0 - predicted_labels = oracle_test.predict(X, y) - assert np.equal(predicted_labels, expected).all() - - -def test_predict_proba_shape(): - n_test_samples = 200 - X, y = make_classification(n_samples=1000) - X_test, y_test = make_classification(n_samples=n_test_samples) - pool = RandomForestClassifier(max_depth=3).fit(X, y) - oracle = Oracle(pool_classifiers=pool).fit(X, y) - - proba = oracle.predict_proba(X_test, y_test) - assert proba.shape == (n_test_samples, 2) - - -def test_predict_proba_right_class(): - n_test_samples = 200 - X, y = make_classification(n_samples=1000) - X_test, y_test = make_classification(n_samples=n_test_samples) - pool = RandomForestClassifier(max_depth=3).fit(X, y) - oracle = Oracle(pool_classifiers=pool).fit(X, y) - - preds = oracle.predict(X_test, y_test) - proba = oracle.predict_proba(X_test, y_test) - probas_max = np.argmax(proba, axis=1) - assert np.allclose(probas_max, preds) - - -def test_label_encoder_base_ensemble(): - from sklearn.ensemble import RandomForestClassifier - X, y = make_classification() - y[y == 1] = 2 - y = y.astype(float) - pool = RandomForestClassifier().fit(X, y) - oracle = Oracle(pool) - oracle.fit(X, y) - pred = oracle.predict(X, y) - assert np.isin(oracle.classes_, pred).all() diff --git a/deslib/tests/static/test_single_best.py b/deslib/tests/static/test_single_best.py deleted file mode 100644 index d5b434ea..00000000 --- a/deslib/tests/static/test_single_best.py +++ /dev/null @@ -1,110 +0,0 @@ -from unittest.mock import MagicMock - -import numpy as np -import pytest -from sklearn.datasets import make_classification -from sklearn.ensemble import AdaBoostClassifier -from sklearn.exceptions import NotFittedError -from sklearn.metrics import roc_auc_score -from sklearn.utils.estimator_checks import check_estimator - -from deslib.static.single_best import SingleBest - - -def test_check_estimator(): - check_estimator(SingleBest()) - - -# Testing if the fit function selects the correct classifier (the one with -# highest classification accuracy). # Note: clf[0] and clf[2] have the -# same accuracy since they always predict the same label. -def test_fit(create_X_y, create_pool_classifiers): - X, y = create_X_y - - pool_classifiers = create_pool_classifiers - single_best_test = SingleBest(pool_classifiers) - single_best_test._estimate_performances = MagicMock( - return_value=[1.0, 0.5, 0.99]) - - single_best_test.fit(X, y) - - assert single_best_test.best_clf_index_ == 0 - - -# The classifier with highest accuracy always predicts 0. So the expected -# prediction should always be equal zero. -def test_predict(create_X_y, create_pool_classifiers): - X, y = create_X_y - - pool_classifiers = create_pool_classifiers - single_best_test = SingleBest(pool_classifiers=pool_classifiers) - single_best_test.fit(X, y) - - predicted_labels = single_best_test.predict(X) - assert np.equal(predicted_labels, 0).all() - - -# The probabilities predicted must always be equals to the probabilities -# predicted by the base classifier with index 0. -def test_predict_proba(create_X_y, create_pool_classifiers): - X, y = create_X_y - - pool_classifiers = create_pool_classifiers - single_best_test = SingleBest(pool_classifiers) - single_best_test.fit(X, y) - - predicted_proba = single_best_test.predict_proba(X) - assert np.equal(predicted_proba, - pool_classifiers[0].predict_proba(X)).all() - - -def test_not_fitted(): - single_best_test = SingleBest() - with pytest.raises(NotFittedError): - single_best_test.predict(np.array([[1, -1]])) - - -# Test calling the predict_proba function with classifiers that do not -# implement the predict_proba -def test_not_predict_proba(create_X_y): - X, y = create_X_y - - classifier = MagicMock() - classifier.predict.return_value = [0] - single_best_test = SingleBest([classifier] * 10) - single_best_test.fit(X, y) - with pytest.raises(ValueError): - single_best_test.predict_proba(X) - - -def test_label_encoder(create_label_encoder_test): - X, y, pool = create_label_encoder_test - sb = SingleBest(pool).fit(X, y) - pred = sb.predict(X) - assert np.array_equal(pred, y) - - -def test_label_encoder_base_ensemble(): - from sklearn.ensemble import RandomForestClassifier - X, y = make_classification() - y[y == 1] = 2 - y = y.astype(float) - pool = RandomForestClassifier().fit(X, y) - sb = SingleBest(pool) - sb.fit(X, y) - pred = sb.predict(X) - assert np.isin(sb.classes_, pred).all() - - -def test_different_scorer(): - X, y = make_classification(n_samples=100, random_state=42) - X_val, y_val = make_classification(n_samples=25, random_state=123) - pool = AdaBoostClassifier(n_estimators=10).fit(X, y) - performances = [] - for clf in pool: - preds = clf.predict_proba(X_val) - performances.append(roc_auc_score(y_val.ravel(), preds[:, -1])) - id_best = np.argmax(performances) - sb = SingleBest(pool_classifiers=pool, scoring='roc_auc') - sb.fit(X_val, y_val) - assert id_best == sb.best_clf_index_ diff --git a/deslib/tests/static/test_stacked.py b/deslib/tests/static/test_stacked.py deleted file mode 100644 index ace89bf0..00000000 --- a/deslib/tests/static/test_stacked.py +++ /dev/null @@ -1,94 +0,0 @@ -import numpy as np -import pytest -from sklearn.datasets import make_classification -from sklearn.linear_model import Perceptron -from sklearn.tree import DecisionTreeClassifier -from sklearn.utils.estimator_checks import check_estimator - -from deslib.static.stacked import StackedClassifier - - -def test_check_estimator(): - check_estimator(StackedClassifier()) - - -# Test if the class is raising an error when the base classifiers do not -# implements the predict_proba method. Should raise an exception when the -# base classifier cannot estimate posterior probabilities (predict_proba) -# Using Perceptron classifier as it does not implements predict_proba. -def test_not_predict_proba(create_X_y): - X, y = create_X_y - - clf1 = Perceptron() - clf1.fit(X, y) - with pytest.raises(ValueError): - StackedClassifier([clf1, clf1]).fit(X, y) - - -# Test if the class is raising an error when the meta classifiers do not -# implements the predict_proba method. Should raise an exception when the -# base classifier cannot estimate posterior probabilities (predict_proba) -# Using Perceptron classifier as it does not implements predict_proba. -def test_not_predict_proba_meta(create_X_y, create_pool_classifiers): - X, y = create_X_y - - pool = create_pool_classifiers - with pytest.raises(ValueError): - meta_clf = StackedClassifier(pool_classifiers=pool, - meta_classifier=Perceptron()) - meta_clf.fit(X, y) - meta_clf.predict_proba(X) - - -def test_label_encoder(): - y = ['one', 'one', 'one', 'zero', 'zero', 'two'] - X = np.random.rand(6, 3) - pool = [DecisionTreeClassifier().fit(X, y) for _ in range(5)] - stacked = StackedClassifier(pool).fit(X, y) - pred = stacked.predict(X) - assert np.array_equal(pred, y) - - -def test_label_encoder_base_ensemble(): - from sklearn.ensemble import RandomForestClassifier - X, y = make_classification() - y[y == 1] = 2 - y = y.astype(float) - pool = RandomForestClassifier().fit(X, y) - st = StackedClassifier(pool) - st.fit(X, y) - pred = st.predict(X) - assert np.isin(st.classes_, pred).all() - - -def test_one_class_meta_dataset(create_X_y): - X, y = create_X_y - pool = [DecisionTreeClassifier().fit(X, y) for _ in range(5)] - stacked = StackedClassifier(pool) - X_meta = np.random.rand(10, 2) - y_meta = np.zeros(10, dtype=int) - with pytest.raises(ValueError): - stacked.fit(X_meta, y_meta) - - -def test_passthrough_true(create_X_y): - X, y = create_X_y - pool = [DecisionTreeClassifier().fit(X, y) for _ in range(5)] - stacked = StackedClassifier(pool, passthrough=True) - stacked.fit(X, y) - assert stacked.meta_classifier_.coef_.shape == (1, 7) - - -def test_passthrough_false(create_X_y): - X, y = create_X_y - pool = [DecisionTreeClassifier().fit(X, y) for _ in range(5)] - stacked = StackedClassifier(pool, passthrough=False) - stacked.fit(X, y) - assert stacked.meta_classifier_.coef_.shape == (1, 5) - - -def test_single_model_pool(create_X_y): - X, y = create_X_y - pool = [DecisionTreeClassifier().fit(X, y)] - with pytest.raises(ValueError): - StackedClassifier(pool_classifiers=pool).fit(X, y) diff --git a/deslib/tests/static/test_static_selection.py b/deslib/tests/static/test_static_selection.py deleted file mode 100644 index d2c3d21a..00000000 --- a/deslib/tests/static/test_static_selection.py +++ /dev/null @@ -1,128 +0,0 @@ -import numpy as np -import pytest -from sklearn.datasets import make_classification -from sklearn.ensemble import AdaBoostClassifier -from sklearn.exceptions import NotFittedError -from sklearn.metrics import log_loss -from sklearn.utils.estimator_checks import check_estimator - -from deslib.static.static_selection import StaticSelection - - -def test_check_estimator(): - check_estimator(StaticSelection()) - - -# Testing if the fit function selects the correct classifiers. -# The 50 last classifiers should be selected. -def test_fit(example_static_selection): - X, y, pool = example_static_selection - static_selection_test = StaticSelection(pool, 0.5) - static_selection_test.fit(X, y) - - assert static_selection_test.n_classifiers_ensemble_ == 50 - assert static_selection_test.n_classifiers_ensemble_ == len( - static_selection_test.clf_indices_) - assert np.array_equal(np.sort(static_selection_test.clf_indices_), - list(range(50, 100))) - - -# The classifier with highest accuracy always predicts 0. So the expected -# prediction should always be equal zero. -def test_predict(example_static_selection, create_pool_classifiers): - X, y, _ = example_static_selection - - static_selection_test = StaticSelection(create_pool_classifiers*10, 0.25) - static_selection_test.fit(X, y) - - predicted_labels = static_selection_test.predict(X) - assert np.equal(predicted_labels, 0).all() - - -# Classifiers predicting different labels are selected -def test_predict_diff(example_static_selection): - X, y, pool = example_static_selection - - static_selection_test = StaticSelection(pool, 0.75) - static_selection_test.fit(X, y) - - predicted_labels = static_selection_test.predict(X) - assert np.equal(predicted_labels, 1).all() - - -def test_not_fitted(): - static_selection_test = StaticSelection() - with pytest.raises(NotFittedError): - static_selection_test.predict(np.array([[1, -1]])) - - -def test_invalid_pct(): - with pytest.raises(TypeError): - test = StaticSelection(pct_classifiers='something') - test.fit(np.random.rand(10, 2), np.ones(10)) - - -def test_invalid_pct2(): - with pytest.raises(ValueError): - test = StaticSelection(pct_classifiers=1.2) - test.fit(np.random.rand(10, 2), np.ones(10)) - - -def test_label_encoder(create_label_encoder_test): - X, y, pool = create_label_encoder_test - static = StaticSelection(pool).fit(X, y) - pred = static.predict(X) - assert np.array_equal(pred, y) - - -def test_label_encoder_base_ensemble(): - from sklearn.ensemble import RandomForestClassifier - X, y = make_classification() - y[y == 1] = 2 - y = y.astype(float) - pool = RandomForestClassifier().fit(X, y) - ss = StaticSelection(pool) - ss.fit(X, y) - pred = ss.predict(X) - assert np.isin(ss.classes_, pred).all() - - -def test_predict_proba(example_static_selection): - X, y, pool = example_static_selection - expected = np.tile([0.52, 0.48], (y.size, 1)) - static_selection_test = StaticSelection(pool, 0.5) - static_selection_test.fit(X, y) - proba = static_selection_test.predict_proba(X) - assert np.allclose(proba, expected) - - -# Test if static_selection can select the best classifier according to a -# metric that needs to be minimized. -def test_different_scorer(): - X, y = make_classification(n_samples=100, random_state=42) - X_val, y_val = make_classification(n_samples=25, random_state=123) - pool = AdaBoostClassifier(n_estimators=10).fit(X, y) - performances = [] - for clf in pool: - preds = clf.predict_proba(X_val) - performances.append(log_loss(y_val.ravel(), preds[:, -1])) - id_best = np.argsort(performances) - ss = StaticSelection(pool_classifiers=pool, scoring='neg_log_loss') - ss.fit(X_val, y_val) - assert (id_best[:ss.n_classifiers_ensemble_] == ss.clf_indices_).all() - - -# Test if static_selection can select the best classifier according to a -# metric that needs to be minimized. -def test_different_scorer(): - X, y = make_classification(n_samples=100, random_state=42) - X_val, y_val = make_classification(n_samples=25, random_state=123) - pool = AdaBoostClassifier(n_estimators=10).fit(X, y) - performances = [] - for clf in pool: - preds = clf.predict_proba(X_val) - performances.append(log_loss(y_val.ravel(), preds[:, -1])) - id_best = np.argsort(performances) - ss = StaticSelection(pool_classifiers=pool, scoring='neg_log_loss') - ss.fit(X_val, y_val) - assert (id_best[:ss.n_classifiers_ensemble_] == ss.clf_indices_).all() diff --git a/deslib/tests/test_base.py b/deslib/tests/test_base.py deleted file mode 100644 index 916fd4f4..00000000 --- a/deslib/tests/test_base.py +++ /dev/null @@ -1,346 +0,0 @@ -import unittest.mock -from unittest.mock import Mock, MagicMock - -import numpy as np -import pytest -from sklearn.exceptions import NotFittedError -from sklearn.neighbors import KNeighborsClassifier - -from deslib.base import BaseDS -from deslib.util.dfp import frienemy_pruning_preprocessed -from .conftest import create_base_classifier - - -def test_all_classifiers_agree(): - # 10 classifiers that return 1 - predictions = np.ones((1, 10)) - - assert np.all(BaseDS._all_classifier_agree(predictions)) - - -def test_not_all_classifiers_agree(): - # 10 classifiers that return 1, and one that returns 2 - predictions = np.ones((10, 11)) - predictions[:, -1] = 2 - - assert not np.all(BaseDS._all_classifier_agree(predictions)) - - -@pytest.mark.parametrize('query', [None, [np.nan, 1.0]]) -def test_predict_value(query): - pool_classifiers = create_classifiers_disagree() - ds = BaseDS(pool_classifiers) - X = np.random.rand(10, 2) - y = np.ones(10) - y[:5] = 0 - ds.fit(X, y) - with pytest.raises(ValueError): - ds.predict(query) - - -@pytest.mark.parametrize('k', [0, 1, -1]) -def test_check_k_value(k): - X = np.random.rand(10, 2) - y = np.ones(10) - with pytest.raises(ValueError): - ds_test = BaseDS(k=k) - ds_test.fit(X, y) - - -@pytest.mark.parametrize('k', ['a', 2.5]) -def test_check_k_type(k): - X = np.random.rand(10, 2) - y = np.ones(10) - - with pytest.raises(TypeError): - ds_test = BaseDS(k=k) - ds_test.fit(X, y) - - -@pytest.mark.parametrize('safe_k', ['a', 2.5]) -def test_check_safe_k_type(safe_k): - X = np.random.rand(10, 2) - y = np.ones(10) - with pytest.raises(TypeError): - ds_test = BaseDS(safe_k=safe_k) - ds_test.fit(X, y) - - -@pytest.mark.parametrize('safe_k', [0, 1, -1]) -def test_check_safe_k_value(safe_k): - X = np.random.rand(10, 2) - y = np.ones(10) - with pytest.raises(ValueError): - ds_test = BaseDS(safe_k=safe_k) - ds_test.fit(X, y) - - -@pytest.mark.parametrize('k, safe_k', [(2, 3), (5, 7)]) -def test_valid_safe_k(k, safe_k): - X = np.random.rand(10, 2) - y = np.ones(10) - with pytest.raises(ValueError): - ds = BaseDS(k=k, safe_k=safe_k) - ds.fit(X, y) - - -def create_classifiers_disagree(): - clf1 = create_base_classifier(return_value=np.ones(1)) - clf_2 = create_base_classifier(return_value=np.zeros(1)) - return [clf1, clf_2] - - -@pytest.mark.parametrize('knn_method,', ['invalidmethod', 1]) -def test_valid_selection_mode(knn_method, create_X_y): - X, y = create_X_y - with pytest.raises(ValueError): - ds = BaseDS(knn_classifier=knn_method) - ds.fit(X, y) - - -def test_import_faiss_mode(): - try: - import sys - sys.modules.pop('deslib.util.faiss_knn_wrapper') - except Exception: - pass - with unittest.mock.patch.dict('sys.modules', {'faiss': None}): - with pytest.raises(ImportError): - BaseDS(knn_classifier="faiss") - - -def test_string_selection_mode(create_X_y): - X, y = create_X_y - ds = BaseDS(knn_classifier="knn") - ds.fit(X, y) - assert (isinstance(ds.roc_algorithm_, KNeighborsClassifier)) - - -# In this test the system was trained for a sample containing 2 features and -# we are passing a sample with 3 as argument. So it should raise a value error. -def test_different_input_shape(create_X_y): - X, y = create_X_y - query = np.array([[1.0, 1.0, 2.0]]) - ds_test = BaseDS() - ds_test.fit(X, y) - with pytest.raises(ValueError): - ds_test.predict(query) - - -def test_empty_pool(create_X_y): - pool_classifiers = [] - X, y = create_X_y - - with pytest.raises(ValueError): - ds = BaseDS(pool_classifiers) - ds.fit(X, y) - - -# Should raise a NotFittedError since the function 'fit' was -# not called before predict -def test_not_fitted_ds(): - query = np.array([[1.0, 1.0]]) - - ds_test = BaseDS() - with pytest.raises(NotFittedError): - ds_test.predict(query) - - -# X has 15 samples while y have 20 labels. So this test should raise an error -def test_input_shape_fit(): - X = np.ones((15, 2)) - y = np.ones(20) - ds_test = BaseDS() - with pytest.raises(ValueError): - ds_test.fit(X, y) - - -@pytest.mark.parametrize('X_test', [None, [[0.1, 0.2], [0.5, np.nan]]]) -def test_bad_input_X(X_test, create_X_y): - X_train, y_train = create_X_y - ds_test = BaseDS() - ds_test.fit(X_train, y_train) - with pytest.raises(ValueError): - ds_test.predict(X_test) - - -def test_preprocess_dsel_scores(create_X_y, create_pool_classifiers): - X, y = create_X_y - ds_test = BaseDS(create_pool_classifiers) - ds_test.fit(X, y) - dsel_scores = ds_test._predict_proba_base(X) - expected = np.array([[0.5, 0.5], [1.0, 0.0], [0.33, 0.67]]) - expected = np.tile(expected, (15, 1, 1)) - assert np.array_equal(dsel_scores, expected) - - -def test_DFP_is_used(example_estimate_competence, create_pool_classifiers): - X, y, neighbors, _, dsel_processed, _ = example_estimate_competence - safe_k = 3 - ds_test = BaseDS(create_pool_classifiers, DFP=True, safe_k=safe_k) - ds_test.fit(X, y) - ds_test.DSEL_processed_ = dsel_processed - - DFP_mask = frienemy_pruning_preprocessed(neighbors[0, :safe_k], - y, dsel_processed) - assert np.array_equal(DFP_mask, np.atleast_2d([1, 1, 0])) - - -def test_IH_is_used(example_estimate_competence, create_pool_classifiers): - X, y, neighbors, distances, dsel_processed, _ = example_estimate_competence - expected = [0, 0, 1] - query = np.ones((3, 2)) - ds_test = BaseDS(create_pool_classifiers, with_IH=True, IH_rate=0.5) - ds_test.fit(X, y) - - ds_test.DSEL_processed_ = dsel_processed - - ds_test.get_competence_region = MagicMock(return_value=(distances, - neighbors)) - predicted = ds_test.predict(query) - - assert np.array_equal(predicted, expected) - - -@pytest.mark.parametrize('IH_rate', [None, -1, 'abc', 0.75, 1]) -def test_input_IH_rate(IH_rate): - X = np.random.rand(10, 2) - y = np.ones(10) - with pytest.raises((ValueError, TypeError)): - ds = BaseDS(with_IH=True, IH_rate=IH_rate) - ds.fit(X, y) - - -def test_predict_proba_all_agree(example_estimate_competence, - create_pool_all_agree): - X, y, _, _, _, dsel_scores = example_estimate_competence - - query = np.atleast_2d([1, 1]) - ds_test = BaseDS(create_pool_all_agree) - ds_test.fit(X, y) - ds_test.DSEL_scores = dsel_scores - proba = ds_test.predict_proba(query) - assert np.allclose(proba, np.atleast_2d([0.61, 0.39])) - - -# In this test, the three neighborhoods have an hardness level lower -# than the parameter IH_rate (0.5). Thus, the KNN -# Should be used to predict probabilities -@pytest.mark.parametrize('index', [0, 1, 2]) -def test_predict_proba_IH_knn(index, - example_estimate_competence, - create_pool_classifiers): - X, y, neighbors, distances, _, dsel_scores = example_estimate_competence - query = np.atleast_2d([1, 1]) - ds_test = BaseDS(create_pool_classifiers, with_IH=True, IH_rate=0.5) - ds_test.fit(X, y) - ds_test.DSEL_scores = dsel_scores - - neighbors = neighbors[index, :] - distances = distances[index, :] - - ds_test.get_competence_region = MagicMock(return_value=(distances, - neighbors)) - - ds_test.roc_algorithm_.predict_proba = MagicMock( - return_value=np.atleast_2d([0.45, 0.55])) - proba = ds_test.predict_proba(query) - assert np.array_equal(proba, np.atleast_2d([0.45, 0.55])) - - -# In this test, the three neighborhoods have an hardness level higher -# than the parameter IH_rate. Thus, the prediction -# should be passed down to the predict_proba_with_ds function. -@pytest.mark.parametrize('index', [0, 1, 2]) -def test_predict_proba_instance_called(index, - example_estimate_competence, - create_pool_classifiers): - X, y, neighbors, distances, _, _ = example_estimate_competence - query = np.atleast_2d([1, 1]) - ds_test = BaseDS(create_pool_classifiers, with_IH=True, IH_rate=0.10) - ds_test.fit(X, y) - - neighbors = neighbors[index, :] - distances = distances[index, :] - - ds_test.get_competence_region = MagicMock(return_value=(distances, - neighbors)) - - ds_test.predict_proba_with_ds = MagicMock( - return_value=np.atleast_2d([0.25, 0.75])) - proba = ds_test.predict_proba(query) - assert np.allclose(proba, np.atleast_2d([0.25, 0.75])) - - -# ------------------------ Testing label encoder------------------------ -def create_pool_classifiers_dog_cat_plane(): - clf_0 = create_base_classifier(return_value=np.array(['cat']), - return_prob=np.atleast_2d([0.5, 0.5])) - - clf_1 = create_base_classifier(return_value=np.array(['dog']), - return_prob=np.atleast_2d([1.0, 0.0])) - - clf_2 = create_base_classifier(return_value=np.array(['plane']), - return_prob=np.atleast_2d([0.33, 0.67])) - - pool_classifiers = [clf_0, clf_1, clf_2] - return pool_classifiers - - -def create_pool_classifiers_dog(): - clf_0 = create_base_classifier(return_value=np.array(['dog']), - return_prob=np.atleast_2d([0.5, 0.5])) - pool_classifiers = [clf_0, clf_0, clf_0] - return pool_classifiers - - -def test_label_encoder_only_dsel_allagree(): - X_dsel_ex1 = np.array([[-1, 1], [-0.75, 0.5], [-1.5, 1.5]]) - y_dsel_ex1 = np.array(['cat', 'dog', 'plane']) - - query = np.atleast_2d([[1, 0], [-1, -1]]) - ds_test = BaseDS(create_pool_classifiers_dog(), k=2) - ds_test.fit(X_dsel_ex1, y_dsel_ex1) - - predictions = ds_test.predict(query) - assert np.array_equal(predictions, ['dog', 'dog']) - - -def test_label_encoder_only_dsel(): - X_dsel_ex1 = np.array([[-1, 1], [-0.75, 0.5], [-1.5, 1.5]]) - y_dsel_ex1 = np.array(['cat', 'dog', 'plane']) - - query = np.atleast_2d([[1, 0], [-1, -1]]) - ds_test = BaseDS(create_pool_classifiers_dog_cat_plane(), k=2) - ds_test.fit(X_dsel_ex1, y_dsel_ex1) - - ds_test.classify_with_ds = Mock() - ds_test.classify_with_ds.return_value = [1, 0] - predictions = ds_test.predict(query) - assert np.array_equal(predictions, ['dog', 'cat']) - - -def test_label_encoder_base(): - from sklearn.linear_model import LogisticRegression - - X_dsel_ex1 = np.array([[-1, 1], [-0.75, 0.5], [-1.5, 1.5]]) - y_dsel_ex1 = np.array(['cat', 'dog', 'plane']) - - x = [[-2, -2], [2, 2]] - y = ['cat', 'dog'] - pool_classifiers = [LogisticRegression().fit(x, y) for _ in range(2)] - - query = np.atleast_2d([[1, 0], [-1, -1]]) - ds_test = BaseDS(pool_classifiers, k=2) - ds_test.fit(X_dsel_ex1, y_dsel_ex1) - predictions = ds_test.predict(query) - - assert np.array_equal(predictions, np.array(['dog', 'cat'])) - - -def test_pool_single_model(create_X_y): - X, y = create_X_y - pool = [create_base_classifier(return_value=0, return_prob=[0.9, 0.1])] - ds_tst = BaseDS(pool_classifiers=pool) - with pytest.raises(ValueError): - ds_tst.fit(X, y) diff --git a/deslib/tests/test_des_integration.py b/deslib/tests/test_des_integration.py deleted file mode 100644 index 5629a2b7..00000000 --- a/deslib/tests/test_des_integration.py +++ /dev/null @@ -1,586 +0,0 @@ -import itertools -import warnings - -import numpy as np -import pytest -from sklearn.cluster import KMeans -from sklearn.datasets import load_breast_cancer -from sklearn.ensemble import AdaBoostClassifier -from sklearn.ensemble import BaggingClassifier -from sklearn.linear_model import LogisticRegression -from sklearn.metrics import accuracy_score -from sklearn.model_selection import GridSearchCV -from sklearn.model_selection import train_test_split -from sklearn.preprocessing import StandardScaler -from sklearn.svm import SVC - -# DCS techniques -from deslib.dcs import APosteriori -from deslib.dcs import APriori -from deslib.dcs import LCA -from deslib.dcs import MCB -from deslib.dcs import MLA -from deslib.dcs import OLA -from deslib.dcs import Rank -# DES techniques -from deslib.des import DESClustering -from deslib.des import DESKNN -from deslib.des import DESP -from deslib.des import KNOP -from deslib.des import KNORAE -from deslib.des import KNORAU -from deslib.des import METADES -from deslib.des import RRC, MinimumDifference, DESKL -# Static techniques -from deslib.static import Oracle -from deslib.static import SingleBest -from deslib.static import StackedClassifier -from deslib.static import StaticSelection -from deslib.util import faiss_knn_wrapper - - -@pytest.mark.skip( - reason='Need to wait for changes on scikit-learn (see issue #89)') -def test_grid_search(): - # This tests if the estimator can be cloned and used in a grid search - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - kne = KNORAE(pool_classifiers) - params = {'k': [1, 3, 5, 7]} - grid = GridSearchCV(kne, params) - grid.fit(X_dsel, y_dsel) - grid.best_estimator_.score(X_test, y_test) - - -knn_methods = [None] -voting = ['hard', 'soft'] - -if faiss_knn_wrapper.is_available(): - # knn_methods.append(faiss_knn_wrapper.FaissKNNClassifier) - knn_methods.append('faiss') -else: - warnings.warn("Not testing FAISS for KNN") - - -def test_label_encoder_integration_list_classifiers(): - rng = np.random.RandomState(123456) - X_dsel, X_test, X_train, y_dsel, y_test, y_train = load_dataset( - encode_labels=['no', 'yes'], rng=rng) - - pool_classifiers = [LogisticRegression(), SVC(probability=True)] - [clf.fit(X_train, y_train) for clf in pool_classifiers] - - knorau = KNORAU(pool_classifiers) - knorau.fit(X_dsel, y_dsel) - - this_score = knorau.score(X_test, y_test) - assert np.isclose(this_score, 0.9787234042553191) - - -def test_label_encoder_integration_sklearn_ensembles(): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers( - encode_labels=['no', 'yes']) - - knorau = KNORAU(pool_classifiers) - knorau.fit(X_dsel, y_dsel) - assert np.isclose(knorau.score(X_test, y_test), 0.9787234042553191) - - -def test_label_encoder_integration_sklearn_ensembles_not_encoding(): - - rng = np.random.RandomState(123456) - X_dsel, X_test, X_train, y_dsel, y_test, y_train = load_dataset( - ['yes', 'no'], rng) - - # Train a pool of using adaboost which has label encoding problems. - pool_classifiers = AdaBoostClassifier(n_estimators=10, random_state=rng) - pool_classifiers.fit(X_train, y_train) - - knorau = KNORAU(pool_classifiers) - knorau.fit(X_dsel, y_dsel) - assert np.isclose(knorau.score(X_test, y_test), 0.9627659574468085) - - -def setup_classifiers(encode_labels=None): - rng = np.random.RandomState(123456) - - X_dsel, X_test, X_train, y_dsel, y_test, y_train = load_dataset( - encode_labels, rng) - model = LogisticRegression(C=1, random_state=rng) - # Train a pool of 100 classifiers - pool_classifiers = BaggingClassifier(model, n_estimators=100, n_jobs=-1, - random_state=rng) - pool_classifiers.fit(X_train, y_train) - return pool_classifiers, X_dsel, y_dsel, X_test, y_test - - -def load_dataset(encode_labels, rng): - # Generate a classification dataset - data = load_breast_cancer() - X = data.data - y = data.target - if encode_labels is not None: - y = np.take(encode_labels, y) - # split the data into training and test data - X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, - random_state=rng) - # Scale the variables to have 0 mean and unit variance - scalar = StandardScaler() - X_train = scalar.fit_transform(X_train) - X_test = scalar.transform(X_test) - # Split the data into training and DSEL for DS techniques - X_dsel, y_dsel = X_train, y_train - # Considering a pool composed of 10 base classifiers - # Calibrating Perceptrons to estimate probabilities - return X_dsel, X_test, X_train, y_dsel, y_test, y_train - - -@pytest.mark.parametrize('knn_methods', knn_methods) -def test_knorau(knn_methods): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - - knorau = KNORAU(pool_classifiers, knn_classifier=knn_methods) - knorau.fit(X_dsel, y_dsel) - assert np.isclose(knorau.score(X_test, y_test), 0.9787234042553191) - - -@pytest.mark.parametrize('knn_methods, voting', - itertools.product(knn_methods, voting)) -def test_kne(knn_methods, voting): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - - kne = KNORAE(pool_classifiers, knn_classifier=knn_methods, voting=voting) - kne.fit(X_dsel, y_dsel) - assert np.isclose(kne.score(X_test, y_test), 0.9787234042553191) - - -@pytest.mark.parametrize('knn_methods, voting', - itertools.product(knn_methods, voting)) -def test_desp(knn_methods, voting): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - - desp = DESP(pool_classifiers, knn_classifier=knn_methods, voting=voting) - desp.fit(X_dsel, y_dsel) - assert np.isclose(desp.score(X_test, y_test), 0.9787234042553191) - - -@pytest.mark.parametrize('knn_methods', knn_methods) -def test_ola(knn_methods): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - - ola = OLA(pool_classifiers, knn_classifier=knn_methods) - ola.fit(X_dsel, y_dsel) - assert np.isclose(ola.score(X_test, y_test), 0.9787234042553191) - - -@pytest.mark.parametrize('knn_methods', knn_methods) -def test_lca(knn_methods): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - - lca = LCA(pool_classifiers, knn_classifier=knn_methods) - lca.fit(X_dsel, y_dsel) - assert np.isclose(lca.score(X_test, y_test), 0.973404255319149) - - -@pytest.mark.parametrize('knn_methods', knn_methods) -def test_MLA(knn_methods): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - - mla = MLA(pool_classifiers, knn_classifier=knn_methods) - mla.fit(X_dsel, y_dsel) - assert np.isclose(mla.score(X_test, y_test), 0.973404255319149) - - -@pytest.mark.parametrize('knn_methods', knn_methods) -def test_mcb(knn_methods): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - rng = np.random.RandomState(123456) - - mcb = MCB(pool_classifiers, random_state=rng, knn_classifier=knn_methods) - mcb.fit(X_dsel, y_dsel) - assert np.isclose(mcb.score(X_test, y_test), 0.9627659574468085) - - -def test_apriori(): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - rng = np.random.RandomState(123456) - - apriori = APriori(pool_classifiers, random_state=rng) - apriori.fit(X_dsel, y_dsel) - assert np.isclose(apriori.score(X_test, y_test), 0.9680851063829787) - - -@pytest.mark.parametrize('knn_methods', knn_methods) -def test_rank(knn_methods): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - - rank = Rank(pool_classifiers, knn_classifier=knn_methods) - rank.fit(X_dsel, y_dsel) - assert np.isclose(rank.score(X_test, y_test), 0.9787234042553191) - - -def test_aposteriori(): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - rng = np.random.RandomState(123456) - - a_posteriori = APosteriori(pool_classifiers, random_state=rng) - a_posteriori.fit(X_dsel, y_dsel) - assert np.isclose(a_posteriori.score(X_test, y_test), 0.9787234042553191) - - -@pytest.mark.parametrize('knn_methods, voting', - itertools.product(knn_methods, voting)) -def test_meta(knn_methods, voting): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - - meta_des = METADES(pool_classifiers, - knn_classifier=knn_methods, voting=voting) - meta_des.fit(X_dsel, y_dsel) - assert np.isclose(meta_des.score(X_test, y_test), 0.9787234042553191) - - -@pytest.mark.parametrize('voting', voting) -def test_rrc(voting): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - - rrc = RRC(pool_classifiers, voting=voting) - rrc.fit(X_dsel, y_dsel) - assert np.isclose(rrc.score(X_test, y_test), 0.9840425531914894) - - -@pytest.mark.parametrize('knn_methods', knn_methods) -def test_deskl(knn_methods): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - - deskl = DESKL(pool_classifiers, knn_classifier=knn_methods) - deskl.fit(X_dsel, y_dsel) - assert np.isclose(deskl.score(X_test, y_test), 0.9787234042553191) - - -@pytest.mark.parametrize('knn_methods', knn_methods) -def test_minimum_diff(knn_methods): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - - minimum_diff = MinimumDifference(pool_classifiers, - knn_classifier=knn_methods) - minimum_diff.fit(X_dsel, y_dsel) - assert np.isclose(minimum_diff.score(X_test, y_test), 0.9787234042553191) - - -@pytest.mark.parametrize('knn_methods, voting', - itertools.product(knn_methods, voting)) -def test_knop(knn_methods, voting): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - - knop = KNOP(pool_classifiers, knn_classifier=knn_methods, voting=voting) - knop.fit(X_dsel, y_dsel) - assert np.isclose(knop.score(X_test, y_test), 0.9787234042553191) - - -@pytest.mark.parametrize('knn_methods, voting', - itertools.product(knn_methods, voting)) -def test_desknn(knn_methods, voting): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - - desknn = DESKNN(pool_classifiers, - knn_classifier=knn_methods, voting=voting) - desknn.fit(X_dsel, y_dsel) - assert np.isclose(desknn.score(X_test, y_test), 0.9787234042553191) - - -@pytest.mark.parametrize('voting', voting) -def test_des_clustering(voting): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - rng = np.random.RandomState(123456) - des_clustering = DESClustering(pool_classifiers, - random_state=rng, voting=voting) - des_clustering.fit(X_dsel, y_dsel) - assert np.isclose(des_clustering.score(X_test, y_test), - 0.973404255319149) - - -def test_oracle(): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - - oracle = Oracle(pool_classifiers) - oracle.fit(X_dsel, y_dsel) - assert np.isclose(oracle.score(X_test, y_test), 0.99468085106382975) - - -def test_single_best(): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - - single_best = SingleBest(pool_classifiers) - single_best.fit(X_dsel, y_dsel) - assert np.isclose(single_best.score(X_test, y_test), 0.973404255319149) - - -def test_static_selection(): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - - static_selection = StaticSelection(pool_classifiers) - static_selection.fit(X_dsel, y_dsel) - assert np.isclose(static_selection.score(X_test, y_test), - 0.9787234042553191) - - -# ------------------------ Testing predict_proba ------------------------------ -@pytest.mark.parametrize('knn_methods', knn_methods) -def test_kne_proba(knn_methods): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - - kne = KNORAE(pool_classifiers, knn_classifier=knn_methods, voting='soft') - kne.fit(X_dsel, y_dsel) - probas = kne.predict_proba(X_test) - expected = np.load( - 'deslib/tests/expected_values/kne_proba_integration.npy') - assert np.allclose(probas, expected, atol=0.01) - - -@pytest.mark.parametrize('knn_methods', knn_methods) -def test_desp_proba(knn_methods): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - desp = DESP(pool_classifiers, knn_classifier=knn_methods, voting='soft') - desp.fit(X_dsel, y_dsel) - probas = desp.predict_proba(X_test) - expected = np.load( - 'deslib/tests/expected_values/desp_proba_integration.npy') - assert np.allclose(probas, expected, atol=0.01) - - -@pytest.mark.parametrize('knn_methods', knn_methods) -def test_ola_proba(knn_methods): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - - ola = OLA(pool_classifiers, knn_classifier=knn_methods) - ola.fit(X_dsel, y_dsel) - probas = ola.predict_proba(X_test) - expected = np.load( - 'deslib/tests/expected_values/ola_proba_integration.npy') - assert np.allclose(probas, expected, atol=0.01) - - -@pytest.mark.parametrize('knn_methods', knn_methods) -def test_mcb_proba(knn_methods): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - rng = np.random.RandomState(123456) - - mcb = MCB(pool_classifiers, random_state=rng, knn_classifier=knn_methods) - - mcb.fit(X_dsel, y_dsel) - probas = mcb.predict_proba(X_test).argmax(axis=1) - expected = mcb.predict(X_test) - assert np.allclose(probas, expected) - - -@pytest.mark.parametrize('knn_methods', knn_methods) -def test_desknn_proba(knn_methods): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - - desknn = DESKNN(pool_classifiers, knn_classifier=knn_methods, - voting='soft') - desknn.fit(X_dsel, y_dsel) - probas = desknn.predict_proba(X_test).argmax(axis=1) - expected = desknn.predict(X_test) - assert np.allclose(probas, expected) - - -def test_des_clustering_proba(): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - rng = np.random.RandomState(123456) - cluster = KMeans(n_clusters=5, random_state=rng) - des_clustering = DESClustering(pool_classifiers, clustering=cluster, - voting='soft') - des_clustering.fit(X_dsel, y_dsel) - probas = des_clustering.predict_proba(X_test).argmax(axis=1) - expected = des_clustering.predict(X_test) - assert np.allclose(probas, expected) - - -@pytest.mark.parametrize('knn_methods', knn_methods) -def test_knop_proba(knn_methods): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - - knop = KNOP(pool_classifiers, knn_classifier=knn_methods, voting='soft') - knop.fit(X_dsel, y_dsel) - probas = knop.predict_proba(X_test) - expected = np.load( - 'deslib/tests/expected_values/knop_proba_integration.npy') - assert np.allclose(probas, expected, atol=0.01) - - -def test_meta_no_pool_of_classifiers(): - rng = np.random.RandomState(123456) - data = load_breast_cancer() - X = data.data - y = data.target - - X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, - random_state=rng) - scalar = StandardScaler() - X_train = scalar.fit_transform(X_train) - X_test = scalar.transform(X_test) - meta_des = METADES(random_state=rng, DSEL_perc=0.5) - meta_des.fit(X_train, y_train) - assert np.isclose(meta_des.score(X_test, y_test), 0.898936170212766) - - -def test_ola_subspaces(): - rng = np.random.RandomState(123456) - X_dsel, X_test, X_train, y_dsel, y_test, y_train = load_dataset( - None, rng) - pool = BaggingClassifier(LogisticRegression(), - bootstrap_features=True, - max_features=0.5, - random_state=rng).fit(X_train, y_train) - - ola = OLA(pool) - ola.fit(X_dsel, y_dsel) - assert np.isclose(ola.score(X_test, y_test), - 0.9680851063829787) - - -def test_knorae_subspaces(): - rng = np.random.RandomState(123456) - X_dsel, X_test, X_train, y_dsel, y_test, y_train = load_dataset( - None, rng) - # split the data into training and test data - pool = BaggingClassifier(LogisticRegression(), - max_features=0.5, - random_state=rng).fit(X_train, y_train) - - knorae = KNORAE(pool) - knorae.fit(X_dsel, y_dsel) - assert np.isclose(knorae.score(X_test, y_test), - 0.9787234042553191) - - -def test_knorae_subspaces(): - rng = np.random.RandomState(123456) - X_dsel, X_test, X_train, y_dsel, y_test, y_train = load_dataset( - None, rng) - pool = BaggingClassifier(LogisticRegression(), - max_features=0.5, - random_state=rng).fit(X_train, y_train) - - knorae = KNORAE(pool) - knorae.fit(X_dsel, y_dsel) - y_pred = knorae.predict_proba(X_test).argmax(axis=1) - assert np.isclose(accuracy_score(y_pred, y_test), - 0.9787234042553191) - - -def test_oracle_subspaces(): - rng = np.random.RandomState(123456) - X_dsel, X_test, X_train, y_dsel, y_test, y_train = load_dataset( - None, rng) - pool = BaggingClassifier(LogisticRegression(), - max_features=0.5, - random_state=rng).fit(X_train, y_train) - - oracle = Oracle(pool) - oracle.fit(X_dsel, y_dsel) - assert np.isclose(oracle.score(X_test, y_test), - 0.9946808510638298) - - -def test_oracle_subspaces_proba(): - rng = np.random.RandomState(123456) - X_dsel, X_test, X_train, y_dsel, y_test, y_train = load_dataset( - None, rng) - pool = BaggingClassifier(LogisticRegression(), - max_features=0.5, - random_state=rng).fit(X_train, y_train) - - oracle = Oracle(pool) - oracle.fit(X_dsel, y_dsel) - y_pred = oracle.predict_proba(X_test, y_test).argmax(axis=1) - assert np.isclose(accuracy_score(y_pred, y_test), - 0.9946808510638298) - - -def test_static_selection_subspaces(): - rng = np.random.RandomState(123456) - X_dsel, X_test, X_train, y_dsel, y_test, y_train = load_dataset( - None, rng) - pool = BaggingClassifier(LogisticRegression(), - max_features=0.5, - random_state=rng).fit(X_train, y_train) - - static = StaticSelection(pool) - static.fit(X_dsel, y_dsel) - assert np.isclose(static.score(X_test, y_test), - 0.9787234042553191) - - -def test_static_selection_subspaces_proba(): - rng = np.random.RandomState(123456) - X_dsel, X_test, X_train, y_dsel, y_test, y_train = load_dataset( - None, rng) - pool = BaggingClassifier(LogisticRegression(), - max_features=0.5, - random_state=rng).fit(X_train, y_train) - - static = StaticSelection(pool) - static.fit(X_dsel, y_dsel) - y_pred_proba = static.predict_proba(X_test).argmax(axis=1) - y_pred = static.predict(X_test) - assert np.isclose(y_pred, y_pred_proba).all() - - -def test_stacked_subspaces(): - rng = np.random.RandomState(123456) - X_dsel, X_test, X_train, y_dsel, y_test, y_train = load_dataset( - None, rng) - pool = BaggingClassifier(LogisticRegression(), - max_features=0.5, - random_state=rng).fit(X_train, y_train) - - stacked = StackedClassifier(pool) - stacked.fit(X_dsel, y_dsel) - assert np.isclose(stacked.score(X_test, y_test), - 0.973404255319149) - - -def test_stacked_subspaces_proba(): - rng = np.random.RandomState(123456) - X_dsel, X_test, X_train, y_dsel, y_test, y_train = load_dataset( - None, rng) - pool = BaggingClassifier(LogisticRegression(), - max_features=0.5, - random_state=rng).fit(X_train, y_train) - - stacked = StackedClassifier(pool) - stacked.fit(X_dsel, y_dsel) - y_pred = stacked.predict_proba(X_test).argmax(axis=1) - assert np.isclose(accuracy_score(y_pred, y_test), - 0.973404255319149) - - -def test_single_best_subspaces(): - rng = np.random.RandomState(123456) - X_dsel, X_test, X_train, y_dsel, y_test, y_train = load_dataset( - None, rng) - pool = BaggingClassifier(LogisticRegression(), - max_features=0.5, - random_state=rng).fit(X_train, y_train) - - single_best = SingleBest(pool) - single_best.fit(X_dsel, y_dsel) - assert np.isclose(single_best.score(X_test, y_test), - 0.9627659574468085) - - -def test_single_best_subspaces_proba(): - rng = np.random.RandomState(123456) - X_dsel, X_test, X_train, y_dsel, y_test, y_train = load_dataset( - None, rng) - pool = BaggingClassifier(LogisticRegression(), - max_features=0.5, - random_state=rng).fit(X_train, y_train) - - single_best = SingleBest(pool) - single_best.fit(X_dsel, y_dsel) - y_pred = single_best.predict_proba(X_test).argmax(axis=1) - - assert np.isclose(accuracy_score(y_pred, y_test), - 0.9627659574468085) diff --git a/deslib/tests/test_des_integration_multiclass.py b/deslib/tests/test_des_integration_multiclass.py deleted file mode 100644 index 30799235..00000000 --- a/deslib/tests/test_des_integration_multiclass.py +++ /dev/null @@ -1,96 +0,0 @@ -import numpy as np -from sklearn.datasets import make_classification -from sklearn.ensemble import AdaBoostClassifier -from sklearn.model_selection import train_test_split -from sklearn.preprocessing import StandardScaler - -# DCS techniques -from deslib.dcs.a_priori import APriori -from deslib.dcs.mcb import MCB -# DES techniques -from deslib.des.des_mi import DESMI -from deslib.des.des_p import DESP -from deslib.des.knop import KNOP -from deslib.des.meta_des import METADES - - -def setup_classifiers(): - rng = np.random.RandomState(123456) - - X_dsel, X_test, X_train, y_dsel, y_test, y_train = load_dataset(rng) - # Train a pool of 100 classifiers - pool_classifiers = AdaBoostClassifier(random_state=rng) - pool_classifiers.fit(X_train, y_train) - return pool_classifiers, X_dsel, y_dsel, X_test, y_test - - -def load_dataset(rng): - # Generate a classification dataset - weights = [0.1, 0.2, 0.7] - X, y = make_classification(n_classes=3, n_samples=2000, n_informative=3, - random_state=rng, weights=weights) - - # split the data into training and test data - X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, - random_state=rng) - # Scale the variables to have 0 mean and unit variance - scalar = StandardScaler() - X_train = scalar.fit_transform(X_train) - X_test = scalar.transform(X_test) - # Split the data into training and DSEL for DS techniques - X_train, X_dsel, y_train, y_dsel = train_test_split(X_train, y_train, - test_size=0.5, - random_state=rng) - # Considering a pool composed of 10 base classifiers - # Calibrating Perceptrons to estimate probabilities - return X_dsel, X_test, X_train, y_dsel, y_test, y_train - - -def test_desp(): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - - desp = DESP(pool_classifiers) - desp.fit(X_dsel, y_dsel) - assert np.isclose(desp.score(X_test, y_test), 0.6954545454545454) - - -def test_mcb(): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - rng = np.random.RandomState(123456) - - mcb = MCB(pool_classifiers, random_state=rng) - mcb.fit(X_dsel, y_dsel) - assert np.isclose(mcb.score(X_test, y_test), 0.7196969696969697) - - -def test_apriori(): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - rng = np.random.RandomState(123456) - - apriori = APriori(pool_classifiers, random_state=rng) - apriori.fit(X_dsel, y_dsel) - assert np.isclose(apriori.score(X_test, y_test), 0.6878787878787879) - - -def test_meta(): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - - meta_des = METADES(pool_classifiers) - meta_des.fit(X_dsel, y_dsel) - assert np.isclose(meta_des.score(X_test, y_test), 0.796969696969697) - - -def test_knop(): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - - knop = KNOP(pool_classifiers) - knop.fit(X_dsel, y_dsel) - assert np.isclose(knop.score(X_test, y_test), 0.8106060606060606) - - -def test_mi(): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - - desmi = DESMI(pool_classifiers, alpha=0.9) - desmi.fit(X_dsel, y_dsel) - assert np.isclose(desmi.score(X_test, y_test), 0.3500000000) diff --git a/deslib/tests/test_integration_DFP_IH.py b/deslib/tests/test_integration_DFP_IH.py deleted file mode 100644 index cc418526..00000000 --- a/deslib/tests/test_integration_DFP_IH.py +++ /dev/null @@ -1,78 +0,0 @@ -import numpy as np -from sklearn.calibration import CalibratedClassifierCV -from sklearn.datasets import make_classification -from sklearn.ensemble import BaggingClassifier -from sklearn.linear_model import Perceptron -from sklearn.model_selection import train_test_split -from sklearn.preprocessing import StandardScaler - -# DCS techniques -from deslib.dcs.a_posteriori import APosteriori -from deslib.dcs.mcb import MCB -from deslib.dcs.ola import OLA -from deslib.des import DESClustering -# DES techniques -from deslib.des.des_p import DESP -from deslib.des.knora_u import KNORAU - - -def setup_classifiers(): - rng = np.random.RandomState(654321) - - # Generate a classification dataset - X, y = make_classification(n_classes=2, n_samples=1000, weights=[0.2, 0.8], - random_state=rng) - # split the data into training and test data - X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, - random_state=rng) - - # Scale the variables to have 0 mean and unit variance - scalar = StandardScaler() - X_train = scalar.fit_transform(X_train) - X_test = scalar.transform(X_test) - - # Split the data into training and DSEL for DS techniques - X_train, X_dsel, y_train, y_dsel = train_test_split(X_train, y_train, - test_size=0.5, - random_state=rng) - # Considering a pool composed of 10 base classifiers - model = CalibratedClassifierCV(Perceptron(max_iter=100), cv=5) - - pool_classifiers = BaggingClassifier(model, n_estimators=100, n_jobs=-1, - random_state=rng) - pool_classifiers.fit(X_train, y_train) - return pool_classifiers, X_dsel, y_dsel, X_test, y_test - - -def test_knorau(): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - - knorau = KNORAU(pool_classifiers, DFP=True, with_IH=True, IH_rate=0.1) - knorau.fit(X_dsel, y_dsel) - assert np.isclose(knorau.score(X_test, y_test), 0.9) - - -def test_desp(): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - - desp = DESP(pool_classifiers, DFP=True, with_IH=True, IH_rate=0.1) - desp.fit(X_dsel, y_dsel) - assert np.isclose(desp.score(X_test, y_test), 0.90) - - -def test_ola(): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - - ola = OLA(pool_classifiers, DFP=True, with_IH=True, IH_rate=0.1) - ola.fit(X_dsel, y_dsel) - assert np.isclose(ola.score(X_test, y_test), 0.9030303030303031) - - -def test_mcb(): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - rng = np.random.RandomState(123456) - - mcb = MCB(pool_classifiers, random_state=rng, DFP=True, with_IH=True, - IH_rate=0.1) - mcb.fit(X_dsel, y_dsel) - assert np.isclose(mcb.score(X_test, y_test), 0.8878787878787879) diff --git a/deslib/tests/test_integration_dfp.py b/deslib/tests/test_integration_dfp.py deleted file mode 100644 index 1e53ddbb..00000000 --- a/deslib/tests/test_integration_dfp.py +++ /dev/null @@ -1,225 +0,0 @@ -import numpy as np -import pytest -from sklearn.datasets import make_classification -from sklearn.ensemble import RandomForestClassifier -from sklearn.model_selection import train_test_split -from sklearn.preprocessing import StandardScaler - -# DCS techniques -from deslib.dcs.a_posteriori import APosteriori -from deslib.dcs.a_priori import APriori -from deslib.dcs.lca import LCA -from deslib.dcs.mcb import MCB -from deslib.dcs.mla import MLA -from deslib.dcs.ola import OLA -from deslib.dcs.rank import Rank -from deslib.des import DESKL -# DES techniques -from deslib.des.des_knn import DESKNN -from deslib.des.des_p import DESP -from deslib.des.knop import KNOP -from deslib.des.knora_e import KNORAE -from deslib.des.knora_u import KNORAU -from deslib.des.meta_des import METADES - - -def setup_classifiers(): - rng = np.random.RandomState(654321) - - # Generate a classification dataset - X, y = make_classification(n_classes=2, n_samples=1000, weights=[0.2, 0.8], - random_state=rng) - # split the data into training and test data - X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, - random_state=rng) - - # Scale the variables to have 0 mean and unit variance - scalar = StandardScaler() - X_train = scalar.fit_transform(X_train) - X_test = scalar.transform(X_test) - - # Split the data into training and DSEL for DS techniques - X_train, X_dsel, y_train, y_dsel = train_test_split(X_train, y_train, - test_size=0.5, - random_state=rng) - # Considering a pool composed of 10 base classifiers - pool_classifiers = RandomForestClassifier(n_estimators=10, n_jobs=-1, - random_state=rng, max_depth=10) - pool_classifiers.fit(X_train, y_train) - return pool_classifiers, X_dsel, y_dsel, X_test, y_test - - -def test_knorau(): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - - knorau = KNORAU(pool_classifiers, DFP=True) - knorau.fit(X_dsel, y_dsel) - assert np.isclose(knorau.score(X_test, y_test), 0.90606060606060601) - - -def test_kne(): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - - kne = KNORAE(pool_classifiers, DFP=True) - kne.fit(X_dsel, y_dsel) - assert np.isclose(kne.score(X_test, y_test), 0.896969696969697) - - -@pytest.mark.parametrize('knne, expected', [(False, 0.8939393939393939), - (True, 0.896969696969697)]) -def test_desp(knne, expected): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - - desp = DESP(pool_classifiers, DFP=True, knne=knne) - desp.fit(X_dsel, y_dsel) - assert np.isclose(desp.score(X_test, y_test), expected) - - -def test_ola(): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - - ola = OLA(pool_classifiers, DFP=True) - ola.fit(X_dsel, y_dsel) - assert np.isclose(ola.score(X_test, y_test), 0.88181818181818183) - - -@pytest.mark.parametrize('knne, expected', [(False, 0.88787878787878793), - (True, 0.89393939393939392)]) -def test_lca(knne, expected): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - - lca = LCA(pool_classifiers, DFP=True, knne=knne) - lca.fit(X_dsel, y_dsel) - assert np.isclose(lca.score(X_test, y_test), expected) - - -def test_MLA(): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - - mla = MLA(pool_classifiers, DFP=True) - mla.fit(X_dsel, y_dsel) - assert np.isclose(mla.score(X_test, y_test), 0.88787878787878793) - - -def test_mcb(): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - rng = np.random.RandomState(123456) - - mcb = MCB(pool_classifiers, random_state=rng, DFP=True) - mcb.fit(X_dsel, y_dsel) - assert np.isclose(mcb.score(X_test, y_test), 0.8606060606060606) - - -def test_apriori(): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - rng = np.random.RandomState(123456) - - apriori = APriori(pool_classifiers, random_state=rng, DFP=True) - apriori.fit(X_dsel, y_dsel) - assert np.isclose(apriori.score(X_test, y_test), 0.87272727272727268) - - -def test_rank(): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - - rank = Rank(pool_classifiers, DFP=True) - rank.fit(X_dsel, y_dsel) - assert np.isclose(rank.score(X_test, y_test), 0.8787878787878788) - - -def test_aposteriori(): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - rng = np.random.RandomState(123456) - - a_posteriori = APosteriori(pool_classifiers, random_state=rng, DFP=True) - a_posteriori.fit(X_dsel, y_dsel) - assert np.isclose(a_posteriori.score(X_test, y_test), 0.90000000000000002) - - -@pytest.mark.parametrize('knne, expected', [(False, 0.9090909090909091), - (True, 0.8606060606060606)]) -def test_meta(knne, expected): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - - meta_des = METADES(pool_classifiers, DFP=True, knne=knne) - meta_des.fit(X_dsel, y_dsel) - assert np.isclose(meta_des.score(X_test, y_test), expected) - - -@pytest.mark.parametrize('knne, expected', [(False, 0.9030303030303031), - (True, 0.906060606060606)]) -def test_knop(knne, expected): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - - kne = KNOP(pool_classifiers, DFP=True, knne=knne) - kne.fit(X_dsel, y_dsel) - - assert np.isclose(kne.score(X_test, y_test), expected) - - -def test_desknn(): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - - desknn = DESKNN(pool_classifiers, DFP=True) - desknn.fit(X_dsel, y_dsel) - assert np.isclose(desknn.score(X_test, y_test), 0.89393939393939392) - - -def test_deskl(): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - - deskl = DESKL(pool_classifiers, DFP=True) - deskl.fit(X_dsel, y_dsel) - assert np.isclose(deskl.score(X_test, y_test), 0.90303030303030307) - - -# --------------------- Testing predict_proba --------------------------------- -def test_kne_proba(): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - - kne = KNORAE(pool_classifiers, DFP=True, voting='soft') - kne.fit(X_dsel, y_dsel) - probas = kne.predict_proba(X_test) - expected = np.load('deslib/tests/expected_values/kne_proba_DFP.npy') - assert np.allclose(probas, expected) - - -def test_desp_proba(): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - - desp = DESP(pool_classifiers, DFP=True, voting='soft') - desp.fit(X_dsel, y_dsel) - probas = desp.predict_proba(X_test) - expected = np.load('deslib/tests/expected_values/desp_proba_DFP.npy') - assert np.allclose(probas, expected) - - -def test_ola_proba(): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - - ola = OLA(pool_classifiers, DFP=True) - ola.fit(X_dsel, y_dsel) - probas = ola.predict_proba(X_test) - expected = np.load('deslib/tests/expected_values/ola_proba_DFP.npy') - assert np.allclose(probas, expected) - - -def test_mcb_proba(): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - rng = np.random.RandomState(123456) - - mcb = MCB(pool_classifiers, random_state=rng, DFP=True) - mcb.fit(X_dsel, y_dsel) - probas = mcb.predict_proba(X_test) - expected = np.load('deslib/tests/expected_values/mcb_proba_DFP.npy') - assert np.allclose(probas, expected) - - -def test_desknn_proba(): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - - desknn = DESKNN(pool_classifiers, DFP=True, voting='soft') - desknn.fit(X_dsel, y_dsel) - probas = desknn.predict_proba(X_test) - expected = np.load('deslib/tests/expected_values/desknn_probas_DFP.npy') - assert np.allclose(probas, expected) diff --git a/deslib/tests/test_metric.py b/deslib/tests/test_metric.py deleted file mode 100644 index cdfd5629..00000000 --- a/deslib/tests/test_metric.py +++ /dev/null @@ -1,194 +0,0 @@ -import numpy as np -import pytest -from sklearn.datasets import make_classification -from sklearn.ensemble import RandomForestClassifier -from sklearn.model_selection import train_test_split -from sklearn.preprocessing import StandardScaler - -from deslib.base import BaseDS -# DCS techniques -from deslib.dcs.a_posteriori import APosteriori -from deslib.dcs.mcb import MCB -from deslib.dcs.ola import OLA -from deslib.dcs.rank import Rank -from deslib.des import DESKL -# DES techniques -from deslib.des.des_knn import DESKNN -from deslib.des.des_p import DESP -from deslib.des.knora_e import KNORAE -from deslib.des.knora_u import KNORAU -from deslib.des.meta_des import METADES - - -# ----- Unit tests ----- - -@pytest.mark.parametrize('knn_metric', ['minkowski', 'mahalanobis']) -def test_valid_knn_metric(knn_metric, create_X_y): - X, y = create_X_y - ds = BaseDS(knn_metric=knn_metric) - ds.fit(X, y) - - -@pytest.mark.parametrize('knn_metric', ['invalidValue']) -def test_invalid_knn_metric(knn_metric, create_X_y): - X, y = create_X_y - with pytest.raises(ValueError): - ds = BaseDS(knn_metric=knn_metric) - ds.fit(X, y) - - -def test_minkowski_metric(): - X, y = make_classification() - ds_test = BaseDS(knn_metric='minkowski') - ds_test._set_region_of_competence_algorithm(X) - assert ds_test.roc_algorithm_.metric == 'minkowski' - assert ds_test.roc_algorithm_.metric_params is None - - -def test_mahalanobis_metric(): - X, y = make_classification() - ds_test = BaseDS(knn_metric='mahalanobis') - ds_test._set_region_of_competence_algorithm(X) - assert ds_test.roc_algorithm_.metric == 'mahalanobis' - assert np.array_equal(ds_test.roc_algorithm_.metric_params['VI'], np.cov(X)) - - -# ----- Integration tests ----- - -def setup_classifiers(): - rng = np.random.RandomState(654321) - - # Generate a classification dataset - X, y = make_classification(n_classes=2, n_samples=1000, weights=[0.2, 0.8], - random_state=rng) - # split the data into training and test data - X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, - random_state=rng) - - # Scale the variables to have 0 mean and unit variance - scalar = StandardScaler() - X_train = scalar.fit_transform(X_train) - X_test = scalar.transform(X_test) - - # Split the data into training and DSEL for DS techniques - X_train, X_dsel, y_train, y_dsel = train_test_split(X_train, y_train, - test_size=0.5, - random_state=rng) - # Considering a pool composed of 10 base classifiers - pool_classifiers = RandomForestClassifier(n_estimators=10, n_jobs=-1, - random_state=rng, max_depth=10) - pool_classifiers.fit(X_train, y_train) - return pool_classifiers, X_dsel, y_dsel, X_test, y_test - - -@pytest.mark.parametrize('metric', [ - ['minkowski', 0.9], - ['mahalanobis', 0.8909090909090909] -]) -def test_knorau(metric): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - technique = KNORAU(pool_classifiers, knn_metric=metric[0]) - technique.fit(X_dsel, y_dsel) - assert np.isclose(technique.score(X_test, y_test), metric[1]) - - -@pytest.mark.parametrize('metric', [ - ['minkowski', 0.9], - ['mahalanobis', 0.8818181818181818] -]) -def test_knorae(metric): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - technique = KNORAE(pool_classifiers, knn_metric=metric[0]) - technique.fit(X_dsel, y_dsel) - assert np.isclose(technique.score(X_test, y_test), metric[1]) - - -@pytest.mark.parametrize('metric', [ - ['minkowski', 0.8939393939393939], - ['mahalanobis', 0.8909090909090909] -]) -def test_desp(metric): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - technique = DESP(pool_classifiers, knn_metric=metric[0]) - technique.fit(X_dsel, y_dsel) - assert np.isclose(technique.score(X_test, y_test), metric[1]) - - -@pytest.mark.parametrize('metric', [ - ['minkowski', 0.8818181818181818], - ['mahalanobis', 0.8727272727272727] -]) -def test_ola(metric): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - technique = OLA(pool_classifiers, knn_metric=metric[0]) - technique.fit(X_dsel, y_dsel) - assert np.isclose(technique.score(X_test, y_test), metric[1]) - - -@pytest.mark.parametrize('metric', [ - ['minkowski', 0.8666666666666667], - ['mahalanobis', 0.8606060606060606] -]) -def test_mcb(metric): - rng = np.random.RandomState(123456) - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - technique = MCB(pool_classifiers, random_state=rng, knn_metric=metric[0]) - technique.fit(X_dsel, y_dsel) - assert np.isclose(technique.score(X_test, y_test), metric[1]) - - -@pytest.mark.parametrize('metric', [ - ['minkowski', 0.8787878787878788], - ['mahalanobis', 0.8757575757575757] -]) -def test_rank(metric): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - technique = Rank(pool_classifiers, knn_metric=metric[0]) - technique.fit(X_dsel, y_dsel) - assert np.isclose(technique.score(X_test, y_test), metric[1]) - - -@pytest.mark.parametrize('metric', [ - ['minkowski', 0.9121212121212121], - ['mahalanobis', 0.8515151515151516] -]) -def test_aposteriori(metric): - rng = np.random.RandomState(123456) - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - technique = APosteriori( - pool_classifiers, random_state=rng, knn_metric=metric[0]) - technique.fit(X_dsel, y_dsel) - assert np.isclose(technique.score(X_test, y_test), metric[1]) - - -@pytest.mark.parametrize('metric', [ - ['minkowski', 0.9], - ['mahalanobis', 0.906060606060606] -]) -def test_meta(metric): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - technique = METADES(pool_classifiers, knn_metric=metric[0]) - technique.fit(X_dsel, y_dsel) - assert np.isclose(technique.score(X_test, y_test), metric[1]) - - -@pytest.mark.parametrize('metric', [ - ['minkowski', 0.896969696969697], - ['mahalanobis', 0.8939393939393939] -]) -def test_desknn(metric): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - technique = DESKNN(pool_classifiers, knn_metric=metric[0]) - technique.fit(X_dsel, y_dsel) - assert np.isclose(technique.score(X_test, y_test), metric[1]) - - -@pytest.mark.parametrize('metric', [ - ['minkowski', 0.9030303030303031], - ['mahalanobis', 0.8939393939393939] -]) -def test_deskl(metric): - pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() - technique = DESKL(pool_classifiers, knn_metric=metric[0]) - technique.fit(X_dsel, y_dsel) - assert np.isclose(technique.score(X_test, y_test), metric[1]) diff --git a/deslib/tests/util/__init__.py b/deslib/tests/util/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/deslib/tests/util/test_aggregation.py b/deslib/tests/util/test_aggregation.py deleted file mode 100644 index 20d15653..00000000 --- a/deslib/tests/util/test_aggregation.py +++ /dev/null @@ -1,156 +0,0 @@ -import pytest - -from deslib.util.aggregation import * -from ..conftest import create_base_classifier - - -def test_majority_voting(): - query = np.array([[1, -1], [0, 0], [3, -1]]) - - ensemble_classifiers = [create_base_classifier(return_value=0)] * 10 + [ - create_base_classifier(return_value=1)] * 9 - - predicted = majority_voting(ensemble_classifiers, query) - assert predicted.all() == 0 and predicted.size == 3 - - -def test_majority_voting_multi_class(): - query = np.array([1, -1]) - ensemble_classifiers = ([create_base_classifier(return_value=0)] * 10) + \ - [create_base_classifier(return_value=2)] * 9 + \ - [create_base_classifier(return_value=1)] * 20 - - predicted = majority_voting(ensemble_classifiers, query) - assert predicted.all() == 1 and predicted.size == 1 - - -def test_weighted_majority_voting(): - query = np.array([[1, -1], [0, 0], [3, -1]]) - ensemble_classifiers = ([create_base_classifier(return_value=0)] * 10) + \ - [create_base_classifier(return_value=2)] * 9 - weights = np.array([([0.5] * 10) + ([0.8] * 9), ([0.5] * 10) + ([0.8] * 9), - ([0.5] * 10) + ([0.8] * 9)]) - predicted = weighted_majority_voting(ensemble_classifiers, weights, query) - assert predicted.all() == 1 and predicted.size == 3 - - -def test_weighted_majority_voting_single_sample(): - query = np.array([1, -1]) - clf_1 = create_base_classifier(return_value=1) - clf_2 = create_base_classifier(return_value=1) - clf_3 = create_base_classifier(return_value=2) - ensemble_classifiers = [clf_2, clf_1, clf_3] - weights = np.atleast_2d([0.2, 0.5, 1.0]) - predicted = weighted_majority_voting(ensemble_classifiers, weights, query) - assert predicted == 2 and predicted.size == 1 - - -def test_predict_proba(create_pool_classifiers): - query = np.array([[1, -1]]) - ensemble_classifiers = create_pool_classifiers - predicted_proba = predict_proba_ensemble(ensemble_classifiers, query) - assert np.isclose(predicted_proba, [0.61, 0.39]).all() - - -# This experiment should raise an error since we have 3 base classifiers -# and 4 weights. -def test_wrong_weights_votes(create_pool_classifiers): - query = np.array([[1, -1]]) - ensemble_classifiers = create_pool_classifiers - weights = np.array([1.0, 1.0, 1.0, 1.0]) - with pytest.raises(ValueError): - weighted_majority_voting(ensemble_classifiers, weights, query) - - -# -------Test routines for the ensemble combination methods------- -# These routines calculates the matrix with the supports given for -# each class for each base classifier and them Aggregates the supports - -def test_product_combiner(create_pool_classifiers): - query = np.array([[1, -1]]) - ensemble_classifiers = create_pool_classifiers - expected = 0 - result = product_combiner(ensemble_classifiers, query) - assert np.allclose(expected, result) - - -def test_average_combiner(create_pool_classifiers): - query = np.array([[1, -1]]) - ensemble_classifiers = create_pool_classifiers - expected = 0 - result = average_combiner(ensemble_classifiers, query) - assert result == expected - - -def test_minimum_combiner(create_pool_classifiers): - query = np.array([[1, -1]]) - ensemble_classifiers = create_pool_classifiers - expected = 0 - result = minimum_combiner(ensemble_classifiers, query) - assert np.allclose(expected, result) - - -def test_maximum_combiner(create_pool_classifiers): - query = np.array([[1, -1]]) - ensemble_classifiers = create_pool_classifiers - expected = 0 - result = maximum_combiner(ensemble_classifiers, query) - assert np.allclose(expected, result) - - -def test_median_combiner(create_pool_classifiers): - query = np.array([[1, -1]]) - ensemble_classifiers = create_pool_classifiers - expected = 0 - result = median_combiner(ensemble_classifiers, query) - assert np.allclose(expected, result) - - -def test_check_predictions(): - predictions = example_kuncheva - with pytest.raises(ValueError): - average_rule(predictions) - - -# -------Test routines for the fusion rules receiving prediction directly------ -# These receives the matrix with the supports given for each class and -# returns the class labels (max score) - -# Test example taken from Kuncheva's book: Combining pattern classifiers - - -example_kuncheva = np.array( - [[0.1, 0.5, 0.4], [0.0, 0.0, 1.0], [0.4, 0.3, 0.4], [0.2, 0.7, 0.1], - [0.1, 0.8, 0.2]]) -example_kuncheva_batch = np.expand_dims(example_kuncheva, axis=0) -example_kuncheva_batch = np.repeat(example_kuncheva_batch, 10, axis=0) - - -def test_product_rule(): - expected = 2 - result = product_rule(example_kuncheva_batch) - assert np.allclose(expected, result) - - -def test_average_rule(): - expected = 1 - result = average_rule(example_kuncheva_batch) - assert np.allclose(expected, result) - - -def test_minimum_rule(): - expected = 2 - result = minimum_rule(example_kuncheva_batch) - assert np.allclose(expected, result) - - -def test_maximum_rule(): - expected = 2 - result = maximum_rule(example_kuncheva_batch) - assert np.allclose(expected, result) - - -def test_median_rule(): - expected = 1 - result = median_rule(example_kuncheva_batch) - assert np.allclose(expected, result) diff --git a/deslib/tests/util/test_datasets.py b/deslib/tests/util/test_datasets.py deleted file mode 100644 index 37c894da..00000000 --- a/deslib/tests/util/test_datasets.py +++ /dev/null @@ -1,60 +0,0 @@ -import pytest -import numpy as np -from deslib.util.datasets import make_P2 -from deslib.util.datasets import make_banana -from deslib.util.datasets import make_banana2 -from deslib.util.datasets import make_circle_square -from deslib.util.datasets import make_xor - - -def setup_class_sizes(): - - size_class0 = np.random.randint(1, 1000) - size_class1 = np.random.randint(1, 1000) - return size_class0, size_class1 - - -def test_P2_class_distribution(): - - s0, s1 = setup_class_sizes() - _, y = make_P2(size_classes=[s0, s1]) - assert np.sum(y == 0) == s0 and np.sum(y == 1) == s1 - - -def test_banana2_class_distribution(): - s0, s1 = setup_class_sizes() - _, y = make_banana2(size_classes=[s0, s1]) - assert np.sum(y == 0) == s0 and np.sum(y == 1) == s1 - - -def test_banana_class_distribution(): - s0, s1 = setup_class_sizes() - _, y = make_banana(size_classes=[s0, s1]) - assert np.sum(y == 0) == s0 and np.sum(y == 1) == s1 - - -def test_circle_square_class_distribution(): - s0, s1 = setup_class_sizes() - _, y = make_circle_square(size_classes=[s0, s1]) - assert np.sum(y == 0) == s0 and np.sum(y == 1) == s1 - - -def test_xor_size(): - n_samples = np.random.randint(100, 2000) - X, y = make_xor(n_samples) - assert y.size == n_samples - - -def test_xor(): - n_samples = np.random.randint(100, 2000) - X, y = make_xor(n_samples) - X_0, X_1 = X[y == 0], X[y == 1] - for x in X_0: - assert np.all(x[0] < 0.5 and x[1] < 0.5) or (x[0] > 0.5 and x[1] > 0.5) - - -def test_banana_n_higher_than_one(): - s0, s1 = setup_class_sizes() - na = np.random.rand() + 1 - with pytest.raises(ValueError): - make_banana([s0, s1], na) diff --git a/deslib/tests/util/test_diversity.py b/deslib/tests/util/test_diversity.py deleted file mode 100644 index bdeecd64..00000000 --- a/deslib/tests/util/test_diversity.py +++ /dev/null @@ -1,174 +0,0 @@ -import sys - -import numpy as np -import pytest - -from deslib.util.diversity import (_process_predictions, - double_fault, - Q_statistic, - ratio_errors, - agreement_measure, - disagreement_measure, - correlation_coefficient) - - -@pytest.fixture -def example_diversity(create_X_y): - - y_pred_classifier1 = np.array([0, 0, 0, 1, 0, 1, 0, 0, 0, 0]) - y_pred_classifier2 = np.array([1, 0, 0, 1, 1, 0, 0, 0, 1, 1]) - - y_real = np.array([0, 0, 1, 0, 0, 0, 0, 1, 1, 1]) - - y_ex1 = create_X_y[1] - return y_pred_classifier1, y_pred_classifier2, y_real, y_ex1 - - -@pytest.fixture -def example_diversity_ones_zeros(create_X_y): - - y = create_X_y[1] - y_pred_ones = np.ones(15) - y_pred_zeros = np.zeros(15) - return y, y_pred_ones, y_pred_zeros - - -def test_process_predictions_ones_zeros(example_diversity_ones_zeros): - y, y_pred_ones, y_pred_zeros = example_diversity_ones_zeros - N00, N10, N01, N11 = _process_predictions(y, y_pred_ones, y_pred_zeros) - assert N00 == 0.0 and N11 == 0.0 and N01 == 9.0/15.0 and N10 == 6.0/15.0 - - -def test_process_predictions_zeros_ones(example_diversity_ones_zeros): - y, y_pred_ones, y_pred_zeros = example_diversity_ones_zeros - N00, N10, N01, N11 = _process_predictions(y, y_pred_zeros, y_pred_ones) - assert N00 == 0.0 and N11 == 0.0 and N01 == 6.0/15.0 and N10 == 9.0/15.0 - - -def test_process_predictions_zeros(example_diversity_ones_zeros): - y, y_pred_ones, y_pred_zeros = example_diversity_ones_zeros - N00, N10, N01, N11 = _process_predictions(y, y_pred_zeros, y_pred_zeros) - assert N00 == 6.0/15.0 and N11 == 9.0/15.0 and N01 == 0.0 and N10 == 0.0 - - -def test_process_predictions_ones(example_diversity_ones_zeros): - y, y_pred_ones, y_pred_zeros = example_diversity_ones_zeros - N00, N10, N01, N11 = _process_predictions(y, y_pred_ones, y_pred_ones) - assert N00 == 9.0/15.0 and N11 == 6.0/15.0 and N01 == 0.0 and N10 == 0.0 - - -def test_process_predictions_diff_sizes(example_diversity_ones_zeros): - y_pred1 = np.ones(10) - y_pred2 = np.ones(15) - y, _, _ = example_diversity_ones_zeros - - with pytest.raises(ValueError): - _ = _process_predictions(y, y_pred1, y_pred2) - - -def test_double_fault_ones_zeros(example_diversity_ones_zeros): - y, y_pred_ones, y_pred_zeros = example_diversity_ones_zeros - df = double_fault(y, y_pred_ones, y_pred_zeros) - assert df == 0.0 - - -def test_double_fault_order(example_diversity_ones_zeros): - y, y_pred_ones, y_pred_zeros = example_diversity_ones_zeros - df1 = double_fault(y, y_pred_ones, y_pred_zeros) - df2 = double_fault(y, y_pred_zeros, y_pred_ones) - assert df1 == df2 - - -def test_double_fault_zeros(example_diversity_ones_zeros): - y, y_pred_ones, y_pred_zeros = example_diversity_ones_zeros - df = double_fault(y, y_pred_zeros, y_pred_zeros) - assert df == 6.0/15.0 - - -def test_double_fault_ones(example_diversity_ones_zeros): - y, y_pred_ones, y_pred_zeros = example_diversity_ones_zeros - df = double_fault(y, y_pred_ones, y_pred_ones) - assert df == 9.0/15.0 - - -def test_double_fault(): - labels = np.array([0, 0, 0, 0, 1, 1, 1]) - pred1 = np.array([1, 0, 1, 0, 0, 0, 0]) - pred2 = np.array([1, 0, 0, 0, 1, 0, 0]) - - actual = double_fault(labels, pred1, pred2) - - assert actual == 3. / 7 # three common errors out of 7 predictions - - -def test_q_statistic_ones_zeros(example_diversity_ones_zeros): - y, y_pred_ones, y_pred_zeros = example_diversity_ones_zeros - Q = Q_statistic(y, y_pred_ones, y_pred_zeros) - assert Q == -1.0 - - -def test_q_statistic_ones(example_diversity_ones_zeros): - y, y_pred_ones, y_pred_zeros = example_diversity_ones_zeros - Q = Q_statistic(y, y_pred_ones, y_pred_ones) - assert Q == 1.0 - - -def test_q_statistic_zeros(example_diversity_ones_zeros): - y, y_pred_ones, y_pred_zeros = example_diversity_ones_zeros - Q = Q_statistic(y, y_pred_zeros, y_pred_zeros) - assert Q == 1.0 - - -def test_ratio_errors_zeros(example_diversity_ones_zeros): - y, y_pred_ones, y_pred_zeros = example_diversity_ones_zeros - ratio = ratio_errors(y, y_pred_zeros, y_pred_zeros) - assert ratio == 0.0 - - -def test_ratio_errors_ones(example_diversity_ones_zeros): - y, y_pred_ones, y_pred_zeros = example_diversity_ones_zeros - ratio = ratio_errors(y, y_pred_ones, y_pred_ones) - assert ratio == 0.0 - - -def test_ratio_ones_zeros(example_diversity_ones_zeros): - y, y_pred_ones, y_pred_zeros = example_diversity_ones_zeros - ratio = ratio_errors(y, y_pred_ones, y_pred_zeros) - assert ratio == sys.float_info.max - - -def test_ratio_order(example_diversity_ones_zeros): - y, y_pred_ones, y_pred_zeros = example_diversity_ones_zeros - ratio1 = ratio_errors(y, y_pred_ones, y_pred_zeros) - ratio2 = ratio_errors(y, y_pred_zeros, y_pred_ones) - assert ratio1 == ratio2 - - -def test_ratio_errors_diff_classifiers(example_diversity): - y_pred_classifier1, y_pred_classifier2, y_real, y_ex1 = example_diversity - ratio = ratio_errors(y_real, y_pred_classifier1, y_pred_classifier2) - assert np.isclose(ratio, 1.66, atol=0.01) - - -def test_agreement(example_diversity): - y_pred_classifier1, y_pred_classifier2, y_real, y_ex1 = example_diversity - agreement = agreement_measure(y_real, - y_pred_classifier1, - y_pred_classifier2) - assert np.isclose(agreement, 0.5) - - -def test_disagreement(example_diversity): - y_pred_classifier1, y_pred_classifier2, y_real, y_ex1 = example_diversity - disagreement = disagreement_measure(y_real, - y_pred_classifier1, - y_pred_classifier2) - assert np.isclose(disagreement, 0.5) - - -def test_coefficient_correlation(example_diversity): - y_pred_classifier1, y_pred_classifier2, y_real, y_ex1 = example_diversity - coefficient = correlation_coefficient(y_real, - y_pred_classifier1, - y_pred_classifier2) - assert np.isclose(coefficient, 0.0) diff --git a/deslib/tests/util/test_diversity_batch.py b/deslib/tests/util/test_diversity_batch.py deleted file mode 100644 index f3815661..00000000 --- a/deslib/tests/util/test_diversity_batch.py +++ /dev/null @@ -1,123 +0,0 @@ -import numpy as np -import pytest - -from deslib.util.diversity_batch import (_process_predictions, - double_fault, - Q_statistic, - ratio_errors, - agreement_measure, - disagreement_measure, - correlation_coefficient) - - -@pytest.fixture -def create_X_y(): - # ex1: The distribution of samples of a test example. - X = np.array( - [ - [-1, 1], - [-0.75, 0.5], - [-1.5, 1.5], - [1, 1], - [0.75, 0.5], - [1.5, 1.5], - [1, -1], - [-0.5, 0.5], - [0.5, 0.5], - [0, -1], - [0.75, -0.5], - [0.0, 0.0], - [-1, -1], - [0, -0.5], - [1, -1], - ] - ) - # Labels associated with the samples. This information is used - # by techniques based on a posteriori information. - y = np.array([0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0]) - return X, y - - -@pytest.fixture -def example_diversity(create_X_y): - y_pred_classifier1 = np.array([0, 0, 0, 1, 0, 1, 0, 0, 0, 0]) - y_pred_classifier2 = np.tile(np.array([1, 0, 0, 1, 1, 0, 0, 0, 1, 1]), - (5, 1)) - - y_real = np.array([0, 0, 1, 0, 0, 0, 0, 1, 1, 1]) - - y_ex1 = create_X_y[1] - return y_pred_classifier1, y_pred_classifier2, y_real, y_ex1 - - -@pytest.fixture -def example_diversity_ones_zeros(create_X_y): - y = create_X_y[1] - y_pred_ones = np.ones(15) - y_pred_zeros = np.zeros((5, 15)) - return y, y_pred_ones, y_pred_zeros - - -def test_process_predictions_ones_zeros(example_diversity_ones_zeros): - y, y_pred_ones, y_pred_zeros = example_diversity_ones_zeros - N00, N10, N01, N11 = _process_predictions(y, y_pred_ones, y_pred_zeros) - assert ( - (N00 == np.full((5,), 0.0)).all() and - (N11 == np.full((5,), 0.0)).all() and - (N01 == np.full((5,), 9.0 / 15.0)).all() and - (N10 == np.full((5,), 6.0 / 15.0)).all() - ) - - -def test_double_fault_ones_zeros(example_diversity_ones_zeros): - y, y_pred_ones, y_pred_zeros = example_diversity_ones_zeros - df = double_fault(y, y_pred_ones, y_pred_zeros) - assert (df == np.full((5,), 0)).all() - - -def test_double_fault(): - labels = np.array([0, 0, 0, 0, 1, 1, 1]) - pred1 = np.array([1, 0, 1, 0, 0, 0, 0]) - pred2 = np.tile(np.array([1, 0, 0, 0, 1, 0, 0]), (5, 1)) - - actual = double_fault(labels, pred1, pred2) - - assert ( - actual == np.full((5,), 3.0 / 7) - ).all() # three common errors out of 7 predictions - - -def test_q_statistic_ones_zeros(example_diversity_ones_zeros): - y, y_pred_ones, y_pred_zeros = example_diversity_ones_zeros - Q = Q_statistic(y, y_pred_ones, y_pred_zeros) - assert (Q == np.full((5,), -1.0)).all() - - -def test_ratio_errors_diff_classifiers(example_diversity): - y_pred_classifier1, y_pred_classifier2, y_real, y_ex1 = example_diversity - ratio = ratio_errors(y_real, y_pred_classifier1, y_pred_classifier2) - assert np.isclose(ratio, 1.66, atol=0.01).all() - - -def test_agreement(example_diversity): - y_pred_classifier1, y_pred_classifier2, y_real, y_ex1 = example_diversity - agreement = agreement_measure(y_real, - y_pred_classifier1, - y_pred_classifier2) - assert np.isclose(agreement, 0.5).all() - - -def test_disagreement(example_diversity): - y_pred_classifier1, y_pred_classifier2, y_real, y_ex1 = example_diversity - disagreement = disagreement_measure(y_real, - y_pred_classifier1, - y_pred_classifier2) - assert np.isclose(disagreement, 0.5).all() - - -def test_coefficient_correlation(example_diversity): - y_pred_classifier1, y_pred_classifier2, y_real, y_ex1 = example_diversity - coefficient = correlation_coefficient( - y_real, y_pred_classifier1, y_pred_classifier2 - ) - assert np.isclose(coefficient, 0.0).all() diff --git a/deslib/tests/util/test_faiss.py b/deslib/tests/util/test_faiss.py deleted file mode 100644 index d87e710a..00000000 --- a/deslib/tests/util/test_faiss.py +++ /dev/null @@ -1,24 +0,0 @@ -import pytest -import numpy as np -from sklearn.neighbors import KNeighborsClassifier -from deslib.tests.test_des_integration import load_dataset -from deslib.util import faiss_knn_wrapper - - -@pytest.mark.skipif(not faiss_knn_wrapper.is_available(), - reason="requires the faiss library") -def test_faiss_predict(): - rng = np.random.RandomState(123456) - _, X_test, X_train, _, _, y_train = load_dataset(None, rng) - k = 7 - X_train = X_train.astype(np.float32) - X_test = X_test.astype(np.float32) - f_knn_test = faiss_knn_wrapper.FaissKNNClassifier(n_neighbors=k) - f_knn_test.fit(X_train, y_train) - f_knn_preds = f_knn_test.predict(X_test) - - knn_test = KNeighborsClassifier(n_neighbors=k) - knn_test.fit(X_train, y_train) - knn_preds = knn_test.predict(X_test) - - assert ((f_knn_preds - knn_preds).sum() == 0) diff --git a/deslib/tests/util/test_fire.py b/deslib/tests/util/test_fire.py deleted file mode 100644 index ae5b526c..00000000 --- a/deslib/tests/util/test_fire.py +++ /dev/null @@ -1,58 +0,0 @@ -import numpy as np - -from deslib.util.dfp import frienemy_pruning -from deslib.util.dfp import frienemy_pruning_preprocessed -from ..conftest import create_base_classifier - - -# Since no classifier crosses the region of competence, -# all of them must be selected -def test_frienemy_no_classifier_crosses(example_estimate_competence): - _, y, neighbors = example_estimate_competence[0:3] - n_classifiers = 3 - predictions = np.zeros((y.size, n_classifiers)) - mask = frienemy_pruning_preprocessed(neighbors, y, predictions) - assert mask.all() - - -# In this example, all base classifier should be considered crossing the -# region of competence since they always predicts the correct label for -# the samples in DSEL. -def test_frienemy_all_classifiers_crosses(example_all_ones): - X, y, neighbors, _, dsel_processed, _ = example_all_ones - result = frienemy_pruning_preprocessed(neighbors, y, dsel_processed) - assert result.all() - - -# Check if the batch processing is working by passing multiple -# samples at the same time. -def test_frienemy_not_all_classifiers_crosses(example_estimate_competence): - expected = np.array([[1, 1, 0], [0, 1, 0], [1, 1, 1]]) - _, y, neighbors, _, dsel_processed, _ = example_estimate_competence - - # passing three samples to compute the DFP at the same time - result = frienemy_pruning_preprocessed(neighbors[:, :3], y, dsel_processed) - assert np.array_equal(result, expected) - - -# Test the case where the sample is located in a safe region -# (i.e., all neighbors comes from the same class) -def test_frienemy_safe_region(example_estimate_competence): - X, y, _, _, dsel_processed, _ = example_estimate_competence - neighbors = np.tile(np.array([0, 1, 2, 6, 7, 8, 14]), (10, 1)) - - result = frienemy_pruning_preprocessed(neighbors, y, dsel_processed) - assert result.all() - - -def test_frienemy_not_processed(): - X = np.random.rand(5, 2) - y = np.array([0, 0, 0, 1, 1]) - X_query = np.random.rand(1, 2) - clf1 = create_base_classifier(return_value=[0, 1, 0, 0, 1]) - clf2 = create_base_classifier(return_value=[1, 1, 1, 1, 1]) - clf3 = create_base_classifier(return_value=[0, 0, 0, 0, 0]) - clf4 = create_base_classifier(return_value=[0, 0, 0, 1, 1]) - pool = [clf1, clf2, clf3, clf4] - dfp_mask = frienemy_pruning(X_query, X, y, pool, 5) - assert np.array_equal(dfp_mask, np.array([[1, 0, 0, 1]])) diff --git a/deslib/tests/util/test_instance_hardness.py b/deslib/tests/util/test_instance_hardness.py deleted file mode 100644 index 400ceda0..00000000 --- a/deslib/tests/util/test_instance_hardness.py +++ /dev/null @@ -1,43 +0,0 @@ -import numpy as np -import pytest - -from deslib.util.instance_hardness import hardness_region_competence -from deslib.util.instance_hardness import kdn_score - - -# ------Test routines for the Instance Hardness calculation------------------ -@pytest.mark.parametrize('index, expected', [(0, 0.42), - (1, 0.28), - (2, 0.28)]) -def test_instance_hardness_region(index, - expected, - example_estimate_competence): - y, neighbors = example_estimate_competence[1:3] - k = 7 - neighbors = neighbors[index, :] - IH = hardness_region_competence(neighbors, y, k) - assert np.isclose(IH, expected, atol=0.01) - - -def test_instance_hardness_region_batch(example_estimate_competence): - expected = np.array([0.42, 0.28, 0.28]) - y, neighbors = example_estimate_competence[1:3] - - k = 7 - IH = hardness_region_competence(neighbors, y, k) - assert np.allclose(IH, expected, atol=0.01) - - -def test_instance_hardness_region_all_same(example_estimate_competence): - y = example_estimate_competence[1] - k = 7 - neighbors = np.array([0, 1, 2, 6, 7, 8, 13]) - IH = hardness_region_competence(neighbors, y, k) - assert IH == 0.0 - - -def test_kdn_score(example_estimate_competence): - X, y, neigh, dist, _, _ = example_estimate_competence - X, y = X[0:6, :], y[0:6] - score, _ = kdn_score(X, y, 3) - assert np.allclose(score, 0.3333333) diff --git a/deslib/tests/util/test_knne.py b/deslib/tests/util/test_knne.py deleted file mode 100644 index eb5415fc..00000000 --- a/deslib/tests/util/test_knne.py +++ /dev/null @@ -1,98 +0,0 @@ -import pytest -import numpy as np -from deslib.util import faiss_knn_wrapper -from deslib.util import KNNE - - -def setup_test(n_neighbors, knn_classifier='sklearn'): - X = np.tile(np.arange(15).reshape(-1, 1), 3) - y = np.array(5 * [0] + 5 * [1] + 5 * [2]) - knne = KNNE(n_neighbors=n_neighbors, knn_classifier=knn_classifier) - knne.fit(X, y) - return X, y, knne - - -def test_equal_classes(): - X, y, knne = setup_test(n_neighbors=6) - dist, inds = knne.kneighbors() - classes = y[inds] - b = np.apply_along_axis(np.bincount, 1, classes) - assert np.equal(b, 2).all() - - -def test_uneven_k(): - X, y, knne = setup_test(n_neighbors=7) - test = X[0, :].reshape(1, -1) - dist, inds = knne.kneighbors(test, 7) - assert np.allclose([0, 1, 5, 6, 10, 11, 2], inds) - - -def test_predict(): - X, y, knne = setup_test(n_neighbors=6) - prediction = knne.predict(X) - assert np.equal(prediction, y).all() - - -def test_predict_proba(): - X, y, knne = setup_test(n_neighbors=6) - probas = knne.predict_proba(X) - assert np.all(probas[0:5, 0] > probas[0:5, 1]) - assert np.all(probas[5:-1, 1] > probas[5:-1, 0]) - - -def test_labels_not_encoded(): - X = np.tile(np.arange(10).reshape(-1, 1), 3) - y = np.array(5 * ['cat'] + 5 * ['dog']) - knne = KNNE(n_neighbors=6) - knne.fit(X, y) - dist, inds = knne.kneighbors() - classes = y[inds] - a = np.sum(classes == 'dog') - b = np.sum(classes == 'cat') - assert np.equal(b, a).all() and a == 30 - - -def test_n_neighbors_none(): - X = np.tile(np.arange(10).reshape(-1, 1), 3) - y = np.array(5 * ['cat'] + 5 * ['dog']) - knne = KNNE(n_neighbors=None) - with pytest.raises(ValueError): - knne.fit(X, y) - - -def test_return_indices_only(): - X, y, knne = setup_test(n_neighbors=6) - inds = knne.kneighbors(X, return_distance=False) - assert inds.shape == (15, 6) - - -def test_n_neighbors_less_n_classes(): - with pytest.raises(ValueError): - setup_test(n_neighbors=2) - - -def test_n_neighbors_not_integer(): - with pytest.raises(TypeError): - setup_test(n_neighbors=5.5) - - -def test_n_neighbors_not_multiple_raise_warning(): - with pytest.warns(Warning): - setup_test(n_neighbors=7) - - -# ------Tests using KNNE using faiss for similarity search------------------ -@pytest.mark.skipif(not faiss_knn_wrapper.is_available(), - reason="requires the faiss library") -def test_faiss_knne(): - X, y, knne = setup_test(n_neighbors=6, knn_classifier='faiss') - y_pred = knne.predict(X) - assert np.allclose(y, y_pred) - - -@pytest.mark.skipif(not faiss_knn_wrapper.is_available(), - reason="requires the faiss library") -def test_faiss_knne_inds(): - X, y, knne = setup_test(n_neighbors=6, knn_classifier='faiss') - inds = knne.kneighbors(X, return_distance=False) - assert inds.shape == (15, 6) diff --git a/deslib/tests/util/test_prob_functions.py b/deslib/tests/util/test_prob_functions.py deleted file mode 100644 index eb0c0d9a..00000000 --- a/deslib/tests/util/test_prob_functions.py +++ /dev/null @@ -1,145 +0,0 @@ -import numpy as np -import pytest - -from deslib.util.prob_functions import (ccprmod, - log_func, - min_difference, - softmax, - exponential_func, - entropy_func) - - -# Example from the original paper "A probabilistic model of classifier -# competence for dynamic ensemble selection" -@pytest.mark.parametrize("supports, idx_correct_label, expected", - [([[0.3, 0.6, 0.1], [1.0 / 3, 1.0 / 3, 1.0 / 3]], - [1, 0], [0.784953394056843, 0.332872292262951]), - ([[0.5, 0.2, 0.3], [0.5, 0.2, 0.3]], [0, 1], - [0.6428, 0.1194])]) -def test_ccprmod_return_value(supports, idx_correct_label, expected): - value = ccprmod(supports, idx_correct_label) - assert np.isclose(value, expected, atol=0.001).all() - - -@pytest.mark.parametrize('B', [0, -1, None, 0.55]) -def test_valid_ccprmod_beta(B): - supports = [0.3, 0.6, 0.1] - idx_correct_label = [1] - - with pytest.raises((ValueError, TypeError)): - ccprmod(supports, idx_correct_label, B) - - -def test_ccprmod_zero_support(): - supports = [[0.0, 0.0, 1.0], [0.0, 1.0, 0.0], [0.0, 0.8, 0.2]] - idx_correct_label = [0, 2, 0] - assert np.isclose(ccprmod(supports, idx_correct_label), 0, atol=0.01).all() - - -def test_ccprmod_one_support(): - supports = [[0.0, 0.0, 1.0], [0.0, 1.0, 0.0]] - idx_correct_label = [2, 1] - assert np.isclose(ccprmod(supports, idx_correct_label), 1, atol=0.01).all() - - -def test_softmax_sum_to_one(): - test = np.random.rand(10) - assert np.allclose(np.sum(softmax(test)), 1.0, atol=0.001) - - -@pytest.mark.parametrize('vector, expected', - [([0, 1, -0.5, 0.5], - [0.1674, 0.4551, 0.1015, 0.2760]), - ([3.20, 5, 7.856, 9.65], - [0.0013, 0.0081, 0.1412, 0.8493])]) -def test_softmax(vector, expected): - assert np.isclose(softmax(vector), expected, atol=0.001).all() - - -@pytest.mark.parametrize('supports_correct, expected', - [(np.array([0.5]), 0), - (np.array([0.0]), -1.0), - (np.array([1.0]), 1.0)]) -def test_log_func_two_classes(supports_correct, expected): - n_classes = 2 - result = log_func(n_classes, supports_correct) - assert np.isclose(result, expected) - - -@pytest.mark.parametrize('supports_correct, expected', - [(np.array([0.33]), 0), - (np.array([0.0]), -1.0), - (np.array([1.0]), 1.0)]) -def test_log_func_multi_class(supports_correct, expected): - n_classes = 3 - result = log_func(n_classes, supports_correct) - assert np.isclose(result, expected, atol=0.01) - - -# Example from the paper "A probabilistic model of classifier competence for -# dynamic ensemble selection" -@pytest.mark.parametrize("supports, idx_correct_label, expected", - [(np.array([[0.3, 0.6, 0.1], [0.33, 0.33, 0.33]]), - [1, 0], [0.3, 0.0]), - (np.array([[0.5, 0.2, 0.3], [0.5, 0.2, 0.3]]), - [0, 1], [0.2, -0.3])]) -def test_min_difference(supports, idx_correct_label, expected): - result = min_difference(supports, idx_correct_label) - assert np.isclose(result, expected, atol=0.01).all() - - -@pytest.mark.parametrize('supports_correct, expected', - [(np.array([0.33]), -0.01), - (np.array([0.0]), -1.0), - (np.array([1.0]), 1.0)]) -def test_exponential_func_multi_class(supports_correct, expected): - n_classes = 3 - result = exponential_func(n_classes, supports_correct) - assert np.isclose(result, expected, atol=0.01).all() - - -def test_exponential_func_multi_class_batch(): - supports_correct = np.array([0.33, 0.0, 1.0]) - expected = [-0.01, -1.0, 1.0] - n_classes = 3 - result = exponential_func(n_classes, supports_correct) - assert np.allclose(result, expected, atol=0.01) - - -@pytest.mark.parametrize('supports_correct, expected', - [(np.array([0.5]), 0.00), - (np.array([0.0]), -1.0), - (np.array([1.0]), 1.0)]) -def test_exponential_func_two_class(supports_correct, expected): - n_classes = 2 - result = exponential_func(n_classes, supports_correct) - assert np.isclose(result, expected, atol=0.01).all() - - -def test_exponential_func(): - n_classes = 2 - result1 = exponential_func(n_classes, np.array([0.2])) - assert result1 < 0.0 - - result2 = exponential_func(n_classes, np.array([0.8])) - assert result2 > 0.0 - - assert result2 > result1 - - result3 = exponential_func(n_classes, np.array([1.0])) - result4 = exponential_func(n_classes, np.array([0.9])) - assert result3 > result4 > result2 > result1 - - -def test_entropy_func_three_classes(): - n_classes = 3 - supports = np.array([[0.33, 0.33, 0.33], [1.0, 0.0, 0.0], [1.0, 0.0, 0.0]]) - is_correct = np.array([0, 1, 0]) - expected = [0.0, 1.0, -1.0] - result = entropy_func(n_classes, supports, is_correct) - assert np.isclose(result, expected, atol=0.01).all() - - -def test_entropy_func_parameter_shape(): - with pytest.raises(ValueError): - entropy_func(2, np.array([0, 1]), np.array([0])) diff --git a/deslib/util/__init__.py b/deslib/util/__init__.py deleted file mode 100644 index 43f0e319..00000000 --- a/deslib/util/__init__.py +++ /dev/null @@ -1,34 +0,0 @@ -""" -The :mod:`deslib.util` This module includes various utilities. They are divided -into four parts: - -deslib.util.aggregation - Implementation of aggregation functions such as -majority voting and averaging. Such functions can be applied to any list of -classifiers. - -deslib.util.diversity - Implementation of different measures of diversity -between classifiers. - -deslib.util.prob_functions - Functions to estimate the competence of a base -classifier based on the -probability estimates. - -deslib.util.instance_hardness - Functions to measure the hardness level of a -given instance - -deslib.util.faiss_knn_wrapper - Wrapper for Facebook AI fast similarity search -on GPU - -deslib.util.datasets - Provides methods to generate synthetic data. - -deslib.util.knne - Implementation of the K-Nearest Neighbors Equality -technique -""" - -from .aggregation import * -from .diversity import * -from .instance_hardness import * -from .prob_functions import * -from .datasets import * -from .knne import KNNE -from .faiss_knn_wrapper import FaissKNNClassifier diff --git a/deslib/util/aggregation.py b/deslib/util/aggregation.py deleted file mode 100644 index bd492565..00000000 --- a/deslib/util/aggregation.py +++ /dev/null @@ -1,470 +0,0 @@ -# coding=utf-8 -# Author: Rafael Menelau Oliveira e Cruz -# -# License: BSD 3 clause - -import numpy as np -from scipy.stats.mstats import mode -from sklearn.utils.validation import check_array - -from deslib.util.prob_functions import softmax - -""" -This file contains the implementation of different aggregation functions to -combine the outputs of the base -classifiers to give the final decision. - -References ----------- -Kuncheva, Ludmila I. Combining pattern classifiers: methods and algorithms. -John Wiley & Sons, 2004. - -J. Kittler, M. Hatef, R. P. W. Duin, J. Matas, On combining classifiers, IEEE -Transactions on Pattern Analysis and Machine Intelligence 20 (1998) 226–239. -""" - - -def majority_voting(classifier_ensemble, X): - """Apply the majority voting rule to predict the label of each sample in X. - - Parameters - ---------- - classifier_ensemble : list of shape = [n_classifiers] - Containing the ensemble of classifiers used in the - aggregation scheme. - - X : array of shape (n_samples, n_features) - The input data. - - Returns - ------- - predicted_label : array of shape (n_samples) - The label of each query sample predicted using the majority voting rule - """ - votes = _get_ensemble_votes(classifier_ensemble, X) - predicted_label = majority_voting_rule(votes) - - return predicted_label - - -def weighted_majority_voting(classifier_ensemble, weights, X): - """Apply the weighted majority voting rule to predict the label of each - sample in X. The size of the weights vector should be equal to the size of - the ensemble. - - Parameters - ---------- - classifier_ensemble : list of shape = [n_classifiers] - Containing the ensemble of classifiers used in the aggregation scheme. - - weights : array of shape (n_samples, n_classifiers) - Weights associated to each base classifier for each sample - - - X : array of shape (n_samples, n_features) - The input data. - - Returns - ------- - predicted_label : array of shape (n_samples) - The label of each query sample predicted using the majority voting rule - """ - votes = _get_ensemble_votes(classifier_ensemble, X) - predicted_label = weighted_majority_voting_rule(votes, weights) - return predicted_label - - -def _get_ensemble_votes(classifier_ensemble, X): - """Calculates the votes obtained by each based classifier in the ensemble - for sample in X - - Parameters - ---------- - classifier_ensemble : list of shape = [n_classifiers] - Containing the ensemble of classifiers used in the aggregation scheme. - - X : array of shape (n_samples, n_features) - The input data. - - Returns - ------- - votes : array of shape (n_samples, n_classifiers) - The votes obtained by each base classifier - """ - # Check if a single sample was passed down to the function. In this case - # the sample must be converted to a 2D array. - X = check_array(X, ensure_2d=False) - if X.ndim == 1: - X = np.atleast_2d(X) - - n_samples = X.shape[0] - votes = np.zeros((n_samples, len(classifier_ensemble))) - for clf_index, clf in enumerate(classifier_ensemble): - votes[:, clf_index] = clf.predict(X) - - return votes - - -def majority_voting_rule(votes): - """Applies the majority voting rule to the estimated votes. - - Parameters - ---------- - votes : array of shape (n_samples, n_classifiers), - The votes obtained by each classifier for each sample. - - Returns - ------- - predicted_label : array of shape (n_samples) - The label of each query sample predicted using the majority voting rule - """ - # Omitting nan value in the predictions as they comes from removed - # classifiers - return mode(votes, axis=1)[0][:, 0] - - -def weighted_majority_voting_rule(votes, weights, labels_set=None): - """Applies the weighted majority voting rule based on the votes obtained by - each base classifier and their - respective weights. - - Parameters - ---------- - votes : array of shape (n_samples, n_classifiers), - The votes obtained by each classifier for each sample. - - weights : array of shape (n_samples, n_classifiers) - Weights associated to each base classifier for each sample - - labels_set : (Default=None) set with the possible classes in the problem. - - Returns - ------- - predicted_label : array of shape (n_samples) - The label of each query sample predicted using the majority voting rule - """ - w_votes, labels_set = get_weighted_votes(votes, weights, labels_set) - predicted_label = labels_set[np.argmax(w_votes, axis=1)] - return predicted_label - - -def get_weighted_votes(votes, weights, labels_set=None): - if weights.shape != votes.shape: - raise ValueError( - 'The shape of the arrays votes and weights should be the ' - 'same. weights = {} ' - 'while votes = {}'.format(weights.shape, votes.shape)) - if labels_set is None: - labels_set = np.unique(votes.astype(int)) - - n_samples = votes.shape[0] - w_votes = np.zeros((len(labels_set), n_samples)) - ma_weights = weights.view(np.ma.MaskedArray) - - for ind, label in enumerate(labels_set): - ma_weights.mask = votes != label - w_votes[ind, :] = ma_weights.sum(axis=1) - - return w_votes.T, labels_set - - -def sum_votes_per_class(predictions, n_classes): - """Sum the number of votes for each class. Accepts masked arrays as input. - - Parameters - ---------- - predictions : array of shape (n_samples, n_classifiers), - The votes obtained by each classifier for each sample. Can be a masked - array. - - n_classes : int - Number of classes. - - Returns - ------- - summed_votes : array of shape (n_samples, n_classes) - Summation of votes for each class - """ - votes = np.zeros((predictions.shape[0], n_classes), dtype=int) - for label in range(n_classes): - votes[:, label] = np.sum(predictions == label, axis=1) - return votes - - -def _get_ensemble_probabilities(classifier_ensemble, X, - estimator_features=None): - """Get the probabilities estimate for each base classifier in the ensemble - - Parameters - ---------- - classifier_ensemble : list of shape = [n_classifiers] - Containing the ensemble of classifiers used in the aggregation scheme. - - X : array of shape (n_samples, n_features) - The input data. - - estimator_features : array of shape (n_classifiers, n_selected_features) - Indices containing the features used by each classifier. - - Returns - ------- - list_proba : array of shape (n_samples, n_classifiers, n_classes) - Probabilities predicted by each base classifier in the ensemble for all - samples in X. - """ - list_proba = [] - if estimator_features is None: - for idx, clf in enumerate(classifier_ensemble): - list_proba.append(clf.predict_proba(X)) - else: - for idx, clf in enumerate(classifier_ensemble): - list_proba.append(clf.predict_proba(X[:, estimator_features[idx]])) - # transpose the array to have the - # shape = [n_samples, n_classifiers, n_classes] - return np.array(list_proba).transpose((1, 0, 2)) - - -def predict_proba_ensemble(classifier_ensemble, X, estimator_features=None): - """Estimates the posterior probabilities of the give ensemble for each - sample in X. - - Parameters - ---------- - classifier_ensemble : list of shape = [n_classifiers] - Containing the ensemble of classifiers used in the aggregation scheme. - - X : array of shape (n_samples, n_features) - The input data. - - estimator_features : array of shape (n_classifiers, n_selected_features) - Indices containing the features used by each classifier. - - Returns - ------- - predicted_proba : array of shape (n_samples, n_classes) - Posterior probabilities estimates for each samples in X. - """ - ensemble_proba = _get_ensemble_probabilities(classifier_ensemble, - X, - estimator_features) - n_classifiers = ensemble_proba.shape[1] - predicted_proba = np.sum(ensemble_proba, axis=1) / n_classifiers - return predicted_proba - - -def aggregate_proba_ensemble_weighted(ensemble_proba, weights): - predicted_proba = ensemble_proba * np.expand_dims(weights, axis=2) - predicted_proba = predicted_proba.mean(axis=1) - - return softmax(predicted_proba) - - -def average_combiner(classifier_ensemble, X): - """Ensemble combination using the Average rule. - - Parameters - ---------- - classifier_ensemble : list of shape = [n_classifiers] - Containing the ensemble of classifiers used in the aggregation scheme. - - X : array of shape (n_samples, n_features) - The input data. - - Returns - ------- - predicted_label : array of shape (n_samples) - The label of each query sample predicted using the majority voting rule - """ - ensemble_proba = _get_ensemble_probabilities(classifier_ensemble, X) - return average_rule(ensemble_proba) - - -def product_combiner(classifier_ensemble, X): - """Ensemble combination using the Product rule. - - Parameters - ---------- - classifier_ensemble : list of shape = [n_classifiers] - Containing the ensemble of classifiers used in the aggregation scheme. - - X : array of shape (n_samples, n_features) - The input data. - - Returns - ------- - predicted_label : array of shape = [n_classifiers, n_samples, n_classes] - Probabilities predicted by each base classifier in the ensemble for all - samples in X. - """ - ensemble_proba = _get_ensemble_probabilities(classifier_ensemble, X) - return product_rule(ensemble_proba) - - -def maximum_combiner(classifier_ensemble, X): - """Ensemble combination using the Maximum rule. - - Parameters - ---------- - classifier_ensemble : list of shape = [n_classifiers] - Containing the ensemble of classifiers used in the aggregation scheme. - - X : array of shape (n_samples, n_features) - The input data. - - Returns - ------- - predicted_label : array of shape (n_samples) - The label of each query sample predicted using the majority voting rule - """ - ensemble_proba = _get_ensemble_probabilities(classifier_ensemble, X) - return maximum_rule(ensemble_proba) - - -def minimum_combiner(classifier_ensemble, X): - """Ensemble combination using the Minimum rule. - - Parameters - ---------- - classifier_ensemble : list of shape = [n_classifiers] - Containing the ensemble of classifiers used in the aggregation scheme. - - X : array of shape (n_samples, n_features) - The input data. - - Returns - ------- - predicted_label : array of shape (n_samples) - The label of each query sample predicted using the majority voting rule - """ - ensemble_proba = _get_ensemble_probabilities(classifier_ensemble, X) - return minimum_rule(ensemble_proba) - - -def median_combiner(classifier_ensemble, X): - """Ensemble combination using the Median rule. - - Parameters - ---------- - classifier_ensemble : list of shape = [n_classifiers] - Containing the ensemble of classifiers used in the aggregation scheme. - - X : array of shape (n_samples, n_features) - The input data. - - Returns - ------- - predicted_label : array of shape (n_samples) - The label of each query sample predicted using the majority voting rule - """ - ensemble_proba = _get_ensemble_probabilities(classifier_ensemble, X) - return median_rule(ensemble_proba) - - -def average_rule(predictions): - """Apply the average fusion rule to the predicted vector of class supports - (predictions). - - Parameters - ---------- - predictions : np array of shape (n_samples, n_classifiers, n_classes) - Vector of class supports predicted by each base classifier for sample - - Returns - ------- - predicted_label : array of shape (n_samples) - The label of each query sample predicted using the majority voting rule - """ - _check_predictions(predictions) - average_predictions = np.mean(predictions, axis=1) - return np.argmax(average_predictions, axis=1) - - -def product_rule(predictions): - """Apply the product fusion rule to the predicted vector of class supports - (predictions). - - Parameters - ---------- - predictions : array of shape (n_samples, n_classifiers, n_classes) - Vector of class supports predicted by each base classifier for sample - - Returns - ------- - predicted_label : array of shape (n_samples) - The label of each query sample predicted using the majority voting rule - """ - _check_predictions(predictions) - prod_predictions = np.prod(predictions, axis=1) - return np.argmax(prod_predictions, axis=1) - - -def median_rule(predictions): - """Apply the product fusion rule to the predicted vector of class supports - (predictions). - - Parameters - ---------- - predictions : np array of shape (n_samples, n_classifiers, n_classes) - Vector of class supports predicted by each base classifier for sample - - Returns - ------- - predicted_label : array of shape (n_samples) - The label of each query sample predicted using the majority voting rule - """ - _check_predictions(predictions) - median_predictions = np.median(predictions, axis=1) - return np.argmax(median_predictions, axis=1) - - -def maximum_rule(predictions): - """Apply the product fusion rule to the predicted vector of class supports - (predictions). - - Parameters - ---------- - predictions : np array of shape (n_samples, n_classifiers, n_classes) - Vector of class supports predicted by each base classifier for sample - - Returns - ------- - predicted_label : array of shape (n_samples) - The label of each query sample predicted using the majority voting rule - """ - _check_predictions(predictions) - max_predictions = np.max(predictions, axis=1) - return np.argmax(max_predictions, axis=1) - - -def minimum_rule(predictions): - """Apply the product fusion rule to the predicted vector of class supports - (predictions). - - Parameters - ---------- - predictions : np array of shape (n_samples, n_classifiers, n_classes) - Vector of class supports predicted by each base classifier for sample - - Returns - ------- - list_proba : array of shape = [n_classifiers, n_samples, n_classes] - Probabilities predicted by each base classifier in the ensemble for all - samples in X. - """ - _check_predictions(predictions) - min_predictions = np.min(predictions, axis=1) - return np.argmax(min_predictions, axis=1) - - -def _check_predictions(predictions): - """Check if the predictions array has the correct size. - - Raises a value error if the array do not contain exactly 3 dimensions: - [n_samples, n_classifiers, n_classes] - - """ - if predictions.ndim != 3: - raise ValueError( - 'predictions must contain 3 dimensions: ' - '[n_samples, n_classifiers, n_classes]. Currently' - 'predictions has {} dimensions'.format(predictions.ndim)) diff --git a/deslib/util/datasets.py b/deslib/util/datasets.py deleted file mode 100644 index 6f531960..00000000 --- a/deslib/util/datasets.py +++ /dev/null @@ -1,293 +0,0 @@ -# coding=utf-8 - -# Author: Rafael Menelau Oliveira e Cruz -# -# License: BSD 3 clause - -import numpy as np -from sklearn.utils.validation import check_random_state - -""" -This file contains routines to generate 2D classification datasets -that can be used to test the performance of different machine learning -algorithms. - -Datasets: - -- P2 Dataset -- Circle and Square -- Banana -- Banana 2 -- XOR - -""" - - -def make_P2(size_classes, random_state=None): - """Generate the P2 Dataset: - - The P2 is a two-class problem, presented by Valentini[1], in which each - class is defined in multiple decision regions delimited by polynomial - and trigonometric functions (E1, E2, E3 and E4): - - .. math:: \\begin{eqnarray} - \\label{eq:problem1} - E1(x) = sin(x) + 5 \\\\ - \\label{eq:problem2} - E2(x) = (x - 2)^{2} + 1 \\\\ - \\label{eq:problem3} - E3(x) = -0.1 \\cdot x^{2} + 0.6sin(4x) + 8 \\\\ - \\label{eq:problem4} - E4(x) = \\frac{(x - 10)^{2}}{2} + 7.902 - \\end{eqnarray} - - Parameters - ---------- - size_classes : list with the number of samples for each class. - - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. - - returns - ------- - X : array of shape = [size_classes, 2] - The generated data points. - - y : array of shape = [size_classes] - Class labels associated with each class. - - References - ---------- - G. Valentini, An experimental bias-variance analysis of svm ensembles - based on resampling techniques, IEEE Transactions on Systems, Man, - and Cybernetics, Part B 35 (2005) 1252–1271. - - """ - rng = check_random_state(random_state) - n_samples = sum(size_classes) * 6 - class_1 = np.zeros((size_classes[0], 2)) - class_2 = np.zeros((size_classes[1], 2)) - size_class1 = 0 - size_class2 = 0 - data = rng.rand(n_samples, 2) - for x in data: - if (size_class1 + size_class2) >= sum(size_classes): - break - - # This if is very complex. Need to simplify that somehow - if x[1] > (-0.1 * (x[0] * 10) ** 2 + 0.6 * np.sin( - 4 * x[0] * 10) + 8.) / 10. and x[1] > ( - (x[0] * 10 - 2) ** 2 + 1) / 10 or \ - x[1] < (2 * np.sin(x[0] * 10) + 5) / 10 and x[1] > ( - (x[0] * 10 - 2) ** 2 + 1) / 10 or \ - x[1] < (-0.1 * (x[0] * 10) ** 2 + 0.6 * np.sin( - 4 * x[0] * 10) + 8) / 10 and x[1] < ( - (x[0] * 10 - 2) ** 2 + 1) / 10 and \ - x[1] > (2 * np.sin(x[0] * 10) + 5) / 10 or \ - x[1] > (-0.1 * (x[0] * 10) ** 2 + 0.6 * np.sin( - 4 * x[0] * 10) + 8) / 10 and x[1] < ( - 2 * np.sin(x[0] * 10) + 5) / 10 or \ - x[1] > (((x[0] * 10 - 10) ** 2) / 2 + 7.902) / 10.: - - if size_class1 < size_classes[0]: - class_1[size_class1] = x - size_class1 += 1 - elif size_class2 < size_classes[1]: - class_2[size_class2] = x - size_class2 += 1 - - y = np.hstack((np.zeros(size_class1), np.ones(size_class2))) - X = np.vstack((class_1, class_2)) - - return X, y - - -def make_circle_square(size_classes, random_state=None): - """Generate the circle square dataset. - - Parameters - ---------- - size_classes : list with the number of samples for each class. - - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. - - returns - ------- - X : array of shape = [size_classes, 2] - The generated data points. - - y : array of shape = [size_classes] - Class labels associated with each class. - - References - ---------- - P. Henniges, E. Granger, R. Sabourin, Factors of overtraining - with fuzzy artmap neural networks, International Joint Conference - on Neural Networks (2005) 1075–1080. - - """ - rng = check_random_state(random_state) - n_samples = sum(size_classes) - class_1 = np.zeros((size_classes[0], 2)) - class_2 = np.zeros((size_classes[1], 2)) - size_class1 = 0 - size_class2 = 0 - data = rng.rand(n_samples * 10, 2) - r = 0.398942 - for x in data: - test_class = ((x[0] - 0.5) ** 2) + ((x[1] - 0.5) ** 2) - if test_class < (r ** 2): - if size_class1 < size_classes[0]: - class_1[size_class1] = x - size_class1 += 1 - - elif size_class2 < size_classes[1]: - class_2[size_class2] = x - size_class2 += 1 - - if size_class2 + size_class1 >= n_samples: - break - - y = np.hstack((np.zeros(size_class1), np.ones(size_class2))) - X = np.vstack((class_1, class_2)) - - return X, y - - -def make_banana(size_classes, na=0.1, random_state=None): - """Generate the Banana dataset. - - Parameters - ---------- - size_classes : list with the number of samples for each class. - - na : float (Default = 0.2), - Noise amplitude. It must be < 1.0 - - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. - - Returns - ------- - X : array of shape = [size_classes, 2] - The generated data points. - - y : array of shape = [size_classes] - Class labels associated with each class. - - References - ---------- - Kuncheva, Ludmila I. Combining pattern classifiers: methods and algorithms. - John Wiley & Sons, 2004. - - """ - rng = check_random_state(random_state) - if not isinstance(na, float) or na > 1.: - raise ValueError( - 'Parameter na must be a float lower than 1. na = {}'.format(na)) - - t1 = np.transpose(-np.linspace(-np.pi / 4, np.pi, size_classes[0])) - z1 = np.transpose((np.sin(t1), np.cos(t1))) - class_1 = 1.5 * z1 + rng.randn(size_classes[0], 2) * na - - t2 = np.transpose(-np.linspace(-np.pi / 4, np.pi, size_classes[1])) - z2 = np.transpose((np.sin(t2), np.cos(t2))) - class_2 = z2 - rng.randn(size_classes[1], 2) * na - y = np.hstack((np.zeros(size_classes[0]), np.ones(size_classes[1]))) - X = np.vstack((class_1, class_2)) - - return X, y - - -def make_banana2(size_classes, sigma=1, random_state=None): - """Generate the Banana dataset similar to the Matlab PRTools toolbox. - - Parameters - ---------- - size_classes : list with the number of samples for each class. - - sigma : float (Default = 1), - variance of the normal distribution - - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. - - Returns - ------- - X : array of shape = [size_classes, 2] - The generated data points. - - y : array of shape = [size_classes] - Class labels associated with each class. - - References - ---------- - R.P.W. Duin, P. Juszczak, D.de Ridder, P. Paclik, E. Pekalska, D.M.Tax, - Prtools, a matlab toolbox for - pattern recognition, 2004. URL 〈http://www.prtools.org〉. - - """ - rng = check_random_state(random_state) - banana_size = 5 - region_class_1 = 0.125 * np.pi + rng.rand( - size_classes[0]) * 1.25 * np.pi - data_class_1 = banana_size * np.transpose( - [np.sin(region_class_1), np.cos(region_class_1)]) + \ - rng.randn(size_classes[0], 2) * sigma - - region_class_2 = 0.375 * np.pi - rng.rand( - size_classes[1]) * 1.25 * np.pi - - tmp = np.transpose( - [np.sin(region_class_2), np.cos(region_class_2)]) * banana_size - - data_class_2 = (tmp + rng.randn(size_classes[1], 2) * sigma) + ( - np.ones((size_classes[1], 2)) * (-0.75 * banana_size)) - - X = np.vstack((data_class_1, data_class_2)) - y = np.hstack((np.zeros(size_classes[0]), np.ones(size_classes[1]))) - - return X, y - - -def make_xor(n_samples, random_state=None): - """Generate the exclusive-or (XOR) dataset. - - Parameters - ---------- - n_samples : int - Number of generated data points. - - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. - - Returns - ------- - X : array of shape = [size_classes, 2] - The generated data points. - - y : array of shape = [size_classes] - Class labels associated with each class. - - """ - rng = check_random_state(random_state) - X = rng.uniform(low=0, high=1, size=(n_samples, 2)) - y = np.logical_xor(X[:, 0] > 0.5, X[:, 1] > 0.5) - - return X, y diff --git a/deslib/util/dfp.py b/deslib/util/dfp.py deleted file mode 100644 index 82dd981e..00000000 --- a/deslib/util/dfp.py +++ /dev/null @@ -1,113 +0,0 @@ -"""Implementation of the Dynamic Frienemy Pruning (DFP) algorithm for online -pruning of base classifiers. - -References ----------- -Oliveira, D.V.R., Cavalcanti, G.D.C. and Sabourin, R., Online Pruning -of Base Classifiers for Dynamic Ensemble Selection, -Pattern Recognition, vol. 72, December 2017, pp 44-58. - -Cruz, Rafael MO, Dayvid VR Oliveira, George DC Cavalcanti, and Robert Sabourin. -"FIRE-DES++: Enhanced online pruning of base classifiers for dynamic ensemble -selection." Pattern Recognition 85 (2019): 149-160. -""" - -# coding=utf-8 - -# Author: Rafael Menelau Oliveira e Cruz -# -# License: BSD 3 clause - - -import numpy as np -from sklearn.neighbors import KNeighborsClassifier - - -def frienemy_pruning(X_query, X_dsel, y_dsel, ensemble, k): - """Implements the Online Pruning method (frienemy) which prunes base - classifiers that do not cross the region of competence of a given instance. - A classifier crosses the region of competence if it correctly - classify at least one sample for each different class in the region. - - Parameters - ---------- - X_query : array-like of shape (n_samples, n_features) - Test set. - X_dsel : array-like of shape (n_samples, n_features) - Dynamic selection set. - y_dsel : array-like of shape (n_samples,) - The target values (Dynamic selection set). - ensemble : list of shape = [n_classifiers] - The ensemble of classifiers to be pruned. - k : int - Number of neighbors used to compute the regions of competence. - - Returns - ------- - DFP_mask : array-like of shape = [n_samples, n_classifiers] - Mask containing 1 for the selected base classifier and 0 - otherwise. - - """ - predictions = np.zeros((X_dsel.shape[0], len(ensemble)), - dtype=np.intp) - for index, clf in enumerate(ensemble): - predictions[:, index] = clf.predict(X_dsel) - hit_miss = predictions == y_dsel[:, np.newaxis] - competence_region = KNeighborsClassifier(n_neighbors=k).fit(X_dsel, y_dsel) - neighbors = competence_region.kneighbors(X_query, return_distance=False) - return frienemy_pruning_preprocessed(neighbors, y_dsel, hit_miss) - - -def frienemy_pruning_preprocessed(neighbors, y_val, hit_miss): - """Implements the Online Pruning method (frienemy) which prunes base - classifiers that do not cross the region of competence of a given instance. - A classifier crosses the region of competence if it correctly - classify at least one sample for each different class in the region. - - Notes - ----- - This implementation assumes the regions of competence of each query example - (neighbors) and the predictions for the dynamic selection data (hit_miss) - were already pre-computed. - - Parameters - ---------- - neighbors : array-like of shape (n_samples, n_neighbors) - Indices of the k nearest neighbors. - y_val : array-like of shape (n_samples,) - The target values (class labels). - hit_miss : array-like of shape (n_samples, n_classifiers) - Matrix containing 1 when the base classifier made the correct - prediction, 0 otherwise. - - Returns - ------- - DFP_mask : array-like of shape = [n_samples, n_classifiers] - Mask containing 1 for the selected base classifier and 0 - otherwise. - """ - if neighbors.ndim < 2: - neighbors = neighbors.reshape(1, -1) - - n_samples = neighbors.shape[0] - n_classifiers = hit_miss.shape[1] - dfp_mask = np.zeros((n_samples, n_classifiers)) - - # TODO: vectorize this code? - for sample_idx in range(n_samples): - curr_neighbors = neighbors[sample_idx] - neighbors_y = y_val[curr_neighbors] - if len(set(neighbors_y)) > 1: - # Indecision region. Check if the base classifier predict the - # correct label for a sample belonging to each class. - for clf_index in range(n_classifiers): - [mask] = np.where(hit_miss[curr_neighbors, clf_index]) - if len(set(neighbors_y[mask])) > 1: - dfp_mask[sample_idx, clf_index] = 1.0 - else: - # Safe region. - dfp_mask[sample_idx, :] = 1.0 - # rows that all classifiers were pruned are set to 1.0 - dfp_mask[np.all(dfp_mask == 0, axis=1)] = 1.0 - return dfp_mask diff --git a/deslib/util/diversity.py b/deslib/util/diversity.py deleted file mode 100644 index 9d89c6bb..00000000 --- a/deslib/util/diversity.py +++ /dev/null @@ -1,325 +0,0 @@ -# coding=utf-8 - -# Author: Rafael Menelau Oliveira e Cruz -# -# License: BSD 3 clause - -import sys - -import numpy as np - -""" -This file contains the implementation of key diversity measures found in the -ensemble literature: - -- Double Fault -- Negative Double fault -- Q-statistics -- Ratio of errors -- Agreement/Disagreement -- Classifier Correlation - -The implementation are made according to the specifications from the book -"Combining Pattern Classifiers" based on Oracle outputs, i.e., taking into -account if the pair of classifiers made the correct/incorrect prediction: - -N00 : represents samples that both classifiers made a wrong prediction - -N10 : represents samples that only classifier 2 predicts the wrong label. - -N10 : represents samples that only classifier 1 predicts the wrong label. - -N11 : represents samples that both classifiers predicts the correct label. - -References ----------- -Kuncheva, Ludmila I. Combining pattern classifiers: methods and algorithms. -John Wiley & Sons, 2004. - -Shipp, Catherine A., and Ludmila I. Kuncheva. "Relationships between -combination methods and measures of diversity in combining classifiers." -Information fusion 3.2 (2002): 135-148. - -Giacinto, Giorgio, and Fabio Roli. "Design of effective neural network -ensembles for image classification purposes." -Image and Vision Computing 19.9 (2001): 699-707. - -Aksela, Matti. "Comparison of classifier selection methods for improving -committee performance." -Multiple Classifier Systems (2003): 159-159. -""" - - -def _process_predictions(y, y_pred1, y_pred2): - """Pre-process the predictions of a pair of base classifiers for the - computation of the diversity measures - - Parameters - ---------- - y : array of shape (n_samples): - class labels of each sample. - - y_pred1 : array of shape (n_samples): - predicted class labels by the classifier 1 for each sample. - - y_pred2 : array of shape (n_samples): - predicted class labels by the classifier 2 for each sample. - - Returns - ------- - N00 : Percentage of samples that both classifiers predict the wrong label - - N10 : Percentage of samples that only classifier 2 predicts the wrong label - - N10 : Percentage of samples that only classifier 1 predicts the wrong label - - N11 : Percentage of samples that both classifiers predict the correct label - """ - size_y = len(y) - if size_y != len(y_pred1) or size_y != len(y_pred2): - raise ValueError( - 'The vector with class labels must have the same size.') - - N00, N10, N01, N11 = 0.0, 0.0, 0.0, 0.0 - for index in range(size_y): - if y_pred1[index] == y[index] and y_pred2[index] == y[index]: - N11 += 1.0 - elif y_pred1[index] == y[index] and y_pred2[index] != y[index]: - N10 += 1.0 - elif y_pred1[index] != y[index] and y_pred2[index] == y[index]: - N01 += 1.0 - else: - N00 += 1.0 - - return N00 / size_y, N10 / size_y, N01 / size_y, N11 / size_y - - -def double_fault(y, y_pred1, y_pred2): - """Calculates the double fault (df) measure. This measure represents the - probability that both classifiers makes the wrong prediction. A lower value - of df means the base classifiers are less likely to make the same error. - This measure must be minimized to increase diversity. - - Parameters - ---------- - y : array of shape (n_samples): - class labels of each sample. - - y_pred1 : array of shape (n_samples): - predicted class labels by the classifier 1 for each sample. - - y_pred2 : array of shape (n_samples): - predicted class labels by the classifier 2 for each sample. - - Returns - ------- - df : The double fault measure between two classifiers - - References - ---------- - Giacinto, Giorgio, and Fabio Roli. "Design of effective neural network - ensembles for image classification purposes." - Image and Vision Computing 19.9 (2001): 699-707. - """ - N00, _, _, _ = _process_predictions(y, y_pred1, y_pred2) - df = N00 - return df - - -def negative_double_fault(y, y_pred1, y_pred2): - """The negative of the double fault measure. This measure should be - maximized for a higher diversity. - - Parameters - ---------- - y : array of shape (n_samples): - class labels of each sample. - - y_pred1 : array of shape (n_samples): - predicted class labels by the classifier 1 for each sample. - - y_pred2 : array of shape (n_samples): - predicted class labels by the classifier 2 for each sample. - - Returns - ------- - df : The negative double fault measure between two classifiers - - References - ---------- - Giacinto, Giorgio, and Fabio Roli. "Design of effective neural network - ensembles for image classification purposes." - Image and Vision Computing 19.9 (2001): 699-707. - """ - return -double_fault(y, y_pred1, y_pred2) - - -def Q_statistic(y, y_pred1, y_pred2): - """Calculates the Q-statistics diversity measure between a pair of - classifiers. The Q value is in a range [-1, 1]. Classifiers that tend to - classify the same object correctly will have positive values of Q, and - Q = 0 for two independent classifiers. - - Parameters - ---------- - y : array of shape (n_samples): - class labels of each sample. - - y_pred1 : array of shape (n_samples): - predicted class labels by the classifier 1 for each sample. - - y_pred2 : array of shape (n_samples): - predicted class labels by the classifier 2 for each sample. - - Returns - ------- - Q : The q-statistic measure between two classifiers - """ - N00, N10, N01, N11 = _process_predictions(y, y_pred1, y_pred2) - Q = ((N11 * N00) - (N01 * N10)) / ((N11 * N00) + (N01 * N10)) - return Q - - -def ratio_errors(y, y_pred1, y_pred2): - """Calculates Ratio of errors diversity measure between a pair of - classifiers. A higher value means that the base classifiers are less likely - to make the same errors. The ratio must be maximized for a higher diversity - - Parameters - ---------- - y : array of shape (n_samples): - class labels of each sample. - - y_pred1 : array of shape (n_samples): - predicted class labels by the classifier 1 for each sample. - - y_pred2 : array of shape (n_samples): - predicted class labels by the classifier 2 for each sample. - - Returns - ------- - ratio : The q-statistic measure between two classifiers - - References - ---------- - Aksela, Matti. "Comparison of classifier selection methods for improving - committee performance." - Multiple Classifier Systems (2003): 159-159. - """ - N00, N10, N01, N11 = _process_predictions(y, y_pred1, y_pred2) - if N00 == 0: - ratio = sys.float_info.max - else: - ratio = (N01 + N10) / N00 - return ratio - - -def disagreement_measure(y, y_pred1, y_pred2): - """Calculates the disagreement measure between a pair of classifiers. This - measure is calculated by the frequency that only one classifier makes the - correct prediction. - - Parameters - ---------- - y : array of shape (n_samples): - class labels of each sample. - - y_pred1 : array of shape (n_samples): - predicted class labels by the classifier 1 for each sample. - - y_pred2 : array of shape (n_samples): - predicted class labels by the classifier 2 for each sample. - - Returns - ------- - disagreement : The frequency at which both classifiers disagrees - """ - _, N10, N01, _ = _process_predictions(y, y_pred1, y_pred2) - disagreement = N10 + N01 - return disagreement - - -def agreement_measure(y, y_pred1, y_pred2): - """Calculates the agreement measure between a pair of classifiers. This - measure is calculated by the frequency that both classifiers either - obtained the correct or incorrect prediction for any given sample - - Parameters - ---------- - y : array of shape (n_samples): - class labels of each sample. - - y_pred1 : array of shape (n_samples): - predicted class labels by the classifier 1 for each sample. - - y_pred2 : array of shape (n_samples): - predicted class labels by the classifier 2 for each sample. - - Returns - ------- - agreement : The frequency at which both classifiers agrees - """ - N00, _, _, N11 = _process_predictions(y, y_pred1, y_pred2) - agreement = N00 + N11 - return agreement - - -def correlation_coefficient(y, y_pred1, y_pred2): - """Calculates the correlation between two classifiers using oracle - outputs. Coefficient is a value in a range [-1, 1]. - - Parameters - ---------- - y : array of shape (n_samples): - class labels of each sample. - - y_pred1 : array of shape (n_samples): - predicted class labels by the classifier 1 for each sample. - - y_pred2 : array of shape (n_samples): - predicted class labels by the classifier 2 for each sample. - - Returns - ------- - rho : The correlation coefficient measured between two classifiers - """ - N00, N10, N01, N11 = _process_predictions(y, y_pred1, y_pred2) - tmp = (N11 * N00) - (N10 * N01) - rho = tmp / np.sqrt((N11 + N01) * (N10 + N00) * (N11 + N10) * (N01 + N00)) - return rho - - -def compute_pairwise_diversity(targets, prediction_matrix, diversity_func): - """Computes the pairwise diversity matrix. - - Parameters - ---------- - targets : array of shape (n_samples): - Class labels of each sample in X. - - prediction_matrix : array of shape (n_samples, n_classifiers): - Predicted class labels for each classifier in the pool - - diversity_func : Function - Function used to estimate the pairwise diversity - - Returns - ------- - diversity : array of shape = [n_classifiers] - The average pairwise diversity matrix calculated for the pool of - classifiers - - """ - n_classifiers = prediction_matrix.shape[1] - diversity = np.zeros(n_classifiers) - - for clf_index in range(n_classifiers): - for clf_index2 in range(clf_index + 1, n_classifiers): - this_diversity = diversity_func(targets, - prediction_matrix[:, clf_index], - prediction_matrix[:, clf_index2]) - - diversity[clf_index] += this_diversity - diversity[clf_index2] += this_diversity - - return diversity diff --git a/deslib/util/diversity_batch.py b/deslib/util/diversity_batch.py deleted file mode 100644 index 456faa51..00000000 --- a/deslib/util/diversity_batch.py +++ /dev/null @@ -1,362 +0,0 @@ -import sys - -import numpy as np - -""" -This file contains the implementation of key diversity measures found in the -ensemble literature: - -- Double Fault -- Negative Double fault -- Q-statistics -- Ratio of errors -- Agreement/Disagreement -- Classifier Correlation - -The implementation are made according to the specifications from the book -"Combining Pattern Classifiers" based on Oracle outputs, i.e., taking into -account if the pair of classifiers made the correct/incorrect prediction: - -N00 : represents samples that both classifiers made a wrong prediction - -N10 : represents samples that only classifier 2 predicts the wrong label. - -N10 : represents samples that only classifier 1 predicts the wrong label. - -N11 : represents samples that both classifiers predicts the correct label. - -References ----------- -Kuncheva, Ludmila I. Combining pattern classifiers: methods and algorithms. -John Wiley & Sons, 2004. - -Shipp, Catherine A., and Ludmila I. Kuncheva. "Relationships between -combination methods and measures of diversity in combining classifiers." -Information fusion 3.2 (2002): 135-148. - -Giacinto, Giorgio, and Fabio Roli. "Design of effective neural network -ensembles for image classification purposes." -Image and Vision Computing 19.9 (2001): 699-707. - -Aksela, Matti. "Comparison of classifier selection methods for improving -committee performance." -Multiple Classifier Systems (2003): 159-159. -""" - - -def _process_predictions(y: np.array, y_pred1: np.array, - y_pred2: np.array) -> np.array: - """Pre-process the predictions of a pair of base classifiers for the - computation of the diversity measures - - Parameters - ---------- - y : array of shape (n_samples,): - class labels of each sample. - - y_pred1 : array of shape (n_samples,): - predicted class labels by the classifier 1 for each sample. - - - y_pred2 : array of shape (n_classifiers, n_samples): - predicted class labels by the classifier 2 for each sample. - - Returns - ------- - N00 : Array of shape (n_samples,) - Percentage of samples that both classifiers predict the wrong label - - N10 : Array of shape (n_samples,) - Percentage of samples that only classifier 2 predicts the wrong label - - N01 : Array of shape (n_samples,) - Percentage of samples that only classifier 1 predicts the wrong label - - N11 : Array of shape (n_samples,) - Percentage of samples that both classifiers predict the correct label - """ - - _, n_samples = y_pred2.shape - - if n_samples == 0: - raise ValueError("Need at least one sample.") - if n_samples != len(y_pred1) or n_samples != len(y): - raise ValueError( - "Inconsistent number of samples between " - "class labels and predictions." - ) - - classifier_1_votes = y != y_pred1 - classifier_2_votes = y != y_pred2 - - mismatch = classifier_1_votes != classifier_2_votes - - classifier_1_mismatch_error = np.logical_and(classifier_1_votes, mismatch) - classifier_2_mismatch_error = np.logical_and(classifier_2_votes, mismatch) - - N01 = np.sum(classifier_1_mismatch_error, axis=1) / n_samples - N10 = np.sum(classifier_2_mismatch_error, axis=1) / n_samples - - matching = np.invert(mismatch) - n_matching = np.sum(matching, axis=1) - - both_incorrect = np.logical_and(classifier_1_votes, matching).sum(axis=1) - - N00 = both_incorrect / n_samples - N11 = (n_matching - both_incorrect) / n_samples - - return N00, N10, N01, N11 - - -def double_fault(y: np.array, y_pred1: np.array, - y_pred2: np.array) -> np.array: - """Calculates the double fault (df) measure. This measure represents the - probability that both classifiers makes the wrong prediction. A lower value - of df means the base classifiers are less likely to make the same error. - This measure must be minimized to increase diversity. - - Parameters - ---------- - y : array of shape (n_samples,): - class labels of each sample. - - y_pred1 : array of shape (n_samples,): - predicted class labels by the classifier 1 for each sample. - - - y_pred2 : array of shape (n_classifiers, n_samples): - predicted class labels by the classifier 2 for each sample. - - Returns - ------- - df : The double fault measure between two classifiers - - References - ---------- - Giacinto, Giorgio, and Fabio Roli. "Design of effective neural network - ensembles for image classification purposes." - Image and Vision Computing 19.9 (2001): 699-707. - """ - N00, _, _, _ = _process_predictions(y, y_pred1, y_pred2) - df = N00 - return df - - -def negative_double_fault( - y: np.array, y_pred1: np.array, y_pred2: np.array -) -> np.array: - """The negative of the double fault measure. This measure should be - maximized for a higher diversity. - - Parameters - ---------- - y : array of shape (n_samples,): - class labels of each sample. - - y_pred1 : array of shape (n_samples,): - predicted class labels by the classifier 1 for each sample. - - - y_pred2 : array of shape (n_classifiers, n_samples): - predicted class labels by the classifier 2 for each sample. - - Returns - ------- - df : The negative double fault measure between two classifiers - - References - ---------- - Giacinto, Giorgio, and Fabio Roli. "Design of effective neural network - ensembles for image classification purposes." - Image and Vision Computing 19.9 (2001): 699-707. - """ - return -double_fault(y, y_pred1, y_pred2) - - -def Q_statistic(y: np.array, y_pred1: np.array, y_pred2: np.array) -> np.array: - """Calculates the Q-statistics diversity measure between a pair of - classifiers. The Q value is in a range [-1, 1]. Classifiers that tend to - classify the same object correctly will have positive values of Q, and - Q = 0 for two independent classifiers. - - Parameters - ---------- - y : array of shape (n_samples,): - class labels of each sample. - - y_pred1 : array of shape (n_samples,): - predicted class labels by the classifier 1 for each sample. - - - y_pred2 : array of shape (n_classifiers, n_samples): - predicted class labels by the classifier 2 for each sample. - - Returns - ------- - Q : The q-statistic measure between two classifiers - """ - N00, N10, N01, N11 = _process_predictions(y, y_pred1, y_pred2) - a = (N11 * N00) - (N01 * N10) - b = (N11 * N00) + (N01 * N10) - Q = np.divide(a, b, out=np.zeros_like(a), where=b != 0) - return Q - - -def ratio_errors(y: np.array, y_pred1: np.array, - y_pred2: np.array) -> np.array: - """Calculates Ratio of errors diversity measure between a pair of - classifiers. A higher value means that the base classifiers are less likely - to make the same errors. The ratio must be maximized for a higher diversity - - Parameters - ---------- - y : array of shape (n_samples,): - class labels of each sample. - - y_pred1 : array of shape (n_samples,): - predicted class labels by the classifier 1 for each sample. - - - y_pred2 : array of shape (n_classifiers, n_samples): - predicted class labels by the classifier 2 for each sample. - - Returns - ------- - ratio : The q-statistic measure between two classifiers - - References - ---------- - Aksela, Matti. "Comparison of classifier selection methods for improving - committee performance." - Multiple Classifier Systems (2003): 159-159. - """ - N00, N10, N01, N11 = _process_predictions(y, y_pred1, y_pred2) - N00[N00 == 0] = sys.float_info.max - a = N01 + N10 - b = N00 - ratio = np.divide(a, b, out=np.zeros_like(a), where=b != 0) - return ratio - - -def disagreement_measure(y: np.array, y_pred1: np.array, - y_pred2: np.array) -> np.array: - """Calculates the disagreement measure between a pair of classifiers. This - measure is calculated by the frequency that only one classifier makes the - correct prediction. - - Parameters - ---------- - y : array of shape (n_samples,): - class labels of each sample. - - y_pred1 : array of shape (n_samples,): - predicted class labels by the classifier 1 for each sample. - - - y_pred2 : array of shape (n_classifiers, n_samples): - predicted class labels by the classifier 2 for each sample. - - Returns - ------- - disagreement : The frequency at which both classifiers disagrees - """ - _, N10, N01, _ = _process_predictions(y, y_pred1, y_pred2) - disagreement = N10 + N01 - return disagreement - - -def agreement_measure(y: np.array, y_pred1: np.array, - y_pred2: np.array) -> np.array: - """Calculates the agreement measure between a pair of classifiers. This - measure is calculated by the frequency that both classifiers either - obtained the correct or incorrect prediction for any given sample - - Parameters - ---------- - y : array of shape (n_samples,): - class labels of each sample. - - y_pred1 : array of shape (n_samples,): - predicted class labels by the classifier 1 for each sample. - - - y_pred2 : array of shape (n_classifiers, n_samples): - predicted class labels by the classifier 2 for each sample. - - Returns - ------- - agreement : The frequency at which both classifiers agrees - """ - N00, _, _, N11 = _process_predictions(y, y_pred1, y_pred2) - agreement = N00 + N11 - return agreement - - -def correlation_coefficient( - y: np.array, y_pred1: np.array, y_pred2: np.array -) -> np.array: - """Calculates the correlation between two classifiers using oracle - outputs. Coefficient is a value in a range [-1, 1]. - - Parameters - ---------- - y : array of shape (n_samples,): - class labels of each sample. - - y_pred1 : array of shape (n_samples,): - predicted class labels by the classifier 1 for each sample. - - - y_pred2 : array of shape (n_classifiers, n_samples): - predicted class labels by the classifier 2 for each sample. - - Returns - ------- - rho : The correlation coefficient measured between two classifiers - """ - N00, N10, N01, N11 = _process_predictions(y, y_pred1, y_pred2) - tmp = (N11 * N00) - (N10 * N01) - b = np.sqrt((N11 + N01) * (N10 + N00) * (N11 + N10) * (N01 + N00)) - rho = np.divide(tmp, b, out=np.zeros_like(tmp), where=b != 0) - return rho - - -def compute_pairwise_diversity( - targets: np.array, prediction_matrix: np.array, - diversity_func: np.array -) -> np.array: - """Computes the pairwise diversity matrix. - - Parameters - ---------- - targets : array of shape (n_samples): - Class labels of each sample in X. - - prediction_matrix : array of shape (n_samples, n_classifiers): - Predicted class labels for each classifier in the pool - - diversity_func : Function - Function used to estimate the pairwise diversity - - Returns - ------- - diversity : array of shape = [n_classifiers] - The average pairwise diversity matrix calculated for the pool of - classifiers - - """ - n_classifiers = prediction_matrix.shape[1] - diversity = np.zeros(n_classifiers) - - for clf_index in range(n_classifiers): - for clf_index2 in range(clf_index + 1, n_classifiers): - this_diversity = diversity_func( - targets, - prediction_matrix[:, clf_index], - prediction_matrix[:, clf_index2], - ) - - diversity[clf_index] += this_diversity - diversity[clf_index2] += this_diversity - - return diversity diff --git a/deslib/util/faiss_knn_wrapper.py b/deslib/util/faiss_knn_wrapper.py deleted file mode 100644 index c9ed9434..00000000 --- a/deslib/util/faiss_knn_wrapper.py +++ /dev/null @@ -1,203 +0,0 @@ -# coding=utf-8 - -# Author: Le Thanh Nguyen-Meidine -# -# License: BSD 3 clause - -import numpy as np -from sklearn.utils.validation import check_is_fitted - - -def is_available(): - try: - import faiss - return True - except ImportError: - return False - - -class FaissKNNClassifier: - """ Scikit-learn wrapper interface for Faiss KNN. - - Parameters - ---------- - n_neighbors : int (Default = 5) - Number of neighbors used in the nearest neighbor search. - - n_jobs : int (Default = None) - The number of jobs to run in parallel for both fit and predict. - If -1, then the number of jobs is set to the number of cores. - - algorithm : {'brute', 'voronoi'} (Default = 'brute') - - Algorithm used to compute the nearest neighbors: - - - 'brute' will use the :class: `IndexFlatL2` class from faiss. - - 'voronoi' will use :class:`IndexIVFFlat` class from faiss. - - 'hierarchical' will use :class:`IndexHNSWFlat` class from faiss. - - Note that selecting 'voronoi' the system takes more time during - training, however it can significantly improve the search time - on inference. 'hierarchical' produce very fast and accurate indexes, - however it has a higher memory requirement. It's recommended when - you have a lots of RAM or the dataset is small. - - For more information see: https://github.com/facebookresearch/faiss/wiki/Guidelines-to-choose-an-index - - n_cells : int (Default = 100) - Number of voronoi cells. Only used when algorithm=='voronoi'. - - n_probes : int (Default = 1) - Number of cells that are visited to perform the search. Note that the - search time roughly increases linearly with the number of probes. - Only used when algorithm=='voronoi'. - - References - ---------- - Johnson Jeff, Matthijs Douze, and Hervé Jégou. "Billion-scale similarity - search with gpus." arXiv preprint arXiv:1702.08734 (2017). - """ - - def __init__(self, - n_neighbors=5, - n_jobs=None, - algorithm='brute', - n_cells=100, - n_probes=1): - - self.n_neighbors = n_neighbors - self.n_jobs = n_jobs - self.algorithm = algorithm - self.n_cells = n_cells - self.n_probes = n_probes - - import faiss - self.faiss = faiss - - def predict(self, X): - """Predict the class label for each sample in X. - - Parameters - ---------- - - X : array of shape (n_samples, n_features) - The input data. - - Returns - ------- - preds : array, shape (n_samples,) - Class labels for samples in X. - """ - idx = self.kneighbors(X, self.n_neighbors, return_distance=False) - class_idx = self.y_[idx] - counts = np.apply_along_axis( - lambda x: np.bincount(x, minlength=self.n_classes_), axis=1, - arr=class_idx.astype(np.int16)) - preds = np.argmax(counts, axis=1) - return preds - - def kneighbors(self, X, n_neighbors=None, return_distance=True): - """Finds the K-neighbors of a point. - - Parameters - ---------- - - X : array of shape (n_samples, n_features) - The input data. - - n_neighbors : int - Number of neighbors to get (default is the value passed to the - constructor). - - return_distance : boolean, optional. Defaults to True. - If False, distances will not be returned - - Returns - ------- - dists : list of shape = [n_samples, k] - The distances between the query and each sample in the region of - competence. The vector is ordered in an ascending fashion. - - idx : list of shape = [n_samples, k] - Indices of the instances belonging to the region of competence of - the given query sample. - """ - if n_neighbors is None: - n_neighbors = self.n_neighbors - - elif n_neighbors <= 0: - raise ValueError("Expected n_neighbors > 0." - " Got {}" .format(n_neighbors)) - else: - if not np.issubdtype(type(n_neighbors), np.integer): - raise TypeError( - "n_neighbors does not take {} value, " - "enter integer value" .format(type(n_neighbors))) - - check_is_fitted(self, 'index_') - - X = np.atleast_2d(X).astype(np.float32) - dist, idx = self.index_.search(X, n_neighbors) - if return_distance: - return dist, idx - else: - return idx - - def predict_proba(self, X): - """Estimates the posterior probabilities for sample in X. - - Parameters - ---------- - X : array of shape (n_samples, n_features) - The input data. - - Returns - ------- - preds_proba : array of shape (n_samples, n_classes) - Probabilities estimates for each sample in X. - """ - idx = self.kneighbors(X, self.n_neighbors, return_distance=False) - class_idx = self.y_[idx] - counts = np.apply_along_axis( - lambda x: np.bincount(x, minlength=self.n_classes_), axis=1, - arr=class_idx.astype(np.int16)) - - preds_proba = counts / self.n_neighbors - - return preds_proba - - def fit(self, X, y): - """Fit the model according to the given training data. - - Parameters - ---------- - X : array of shape (n_samples, n_features) - Data used to fit the model. - - y : array of shape (n_samples) - class labels of each example in X. - """ - X = np.atleast_2d(X).astype(np.float32) - X = np.ascontiguousarray(X) - d = X.shape[1] # dimensionality of the feature vector - self._prepare_knn_algorithm(X, d) - self.index_.add(X) - self.y_ = y - self.n_classes_ = np.unique(y).size - return self - - def _prepare_knn_algorithm(self, X, d): - if self.algorithm == 'brute': - self.index_ = self.faiss.IndexFlatL2(d) - elif self.algorithm == 'voronoi': - quantizer = self.faiss.IndexFlatL2(d) - self.index_ = self.faiss.IndexIVFFlat(quantizer, d, self.n_cells) - self.index_.train(X) - self.index_.nprobe = self.n_probes - elif self.algorithm == 'hierarchical': - self.index_ = self.faiss.IndexHNSWFlat(d, 32) - self.index_.hnsw.efConstruction = 40 - else: - raise ValueError("Invalid algorithm option." - " Expected ['brute', 'voronoi', 'hierarchical'], " - "got {}" .format(self.algorithm)) diff --git a/deslib/util/instance_hardness.py b/deslib/util/instance_hardness.py deleted file mode 100644 index fadd504a..00000000 --- a/deslib/util/instance_hardness.py +++ /dev/null @@ -1,105 +0,0 @@ -# coding=utf-8 - -# Author: Rafael Menelau Oliveira e Cruz -# -# License: BSD 3 clause - -import numpy as np -from scipy.stats import mode -from sklearn.neighbors import NearestNeighbors - -""" -This file contains the implementation of different functions to measure -instance hardness. Instance hardness can be defined as the likelihood that a -given sample will be misclassified by different learning algorithms. - -References ----------- -Smith, M.R., Martinez, T. and Giraud-Carrier, C., 2014. An instance level -analysis of data complexity. -Machine learning, 95(2), pp.225-256 -""" - - -def hardness_region_competence(neighbors_idx, labels, safe_k): - """Calculate the Instance hardness of the sample based on its neighborhood. - The sample is deemed hard to classify when there is overlap between - different classes in the region of competence. This method does not - takes into account the target label of the test sample - - This hardness measure is used to select whether use DS or use the KNN for - the classification of a given query sample - - Parameters - ---------- - neighbors_idx : array of shape = [n_samples_test, k] - Indices of the nearest neighbors for each considered sample - - labels : array of shape = [n_samples_train] - labels associated with each training sample - - safe_k : int - Number of neighbors used to estimate the hardness of the corresponding - region - - Returns - ------- - hardness : array of shape = [n_samples_test] - The Hardness level associated with each example. - - References - ---------- - Smith, M.R., Martinez, T. and Giraud-Carrier, C., 2014. An instance level - analysis of data complexity. - Machine learning, 95(2), pp.225-256 - """ - if neighbors_idx.ndim < 2: - neighbors_idx = np.atleast_2d(neighbors_idx) - - neighbors_y = labels[neighbors_idx[:, :safe_k]] - _, num_majority_class = mode(neighbors_y, axis=1) - hardness = ((safe_k - num_majority_class) / safe_k).reshape(-1, ) - - return hardness - - -def kdn_score(X, y, k): - """ - Calculates the K-Disagreeing Neighbors score (KDN) of each sample in the - input dataset. - - Parameters - ---------- - X : array of shape (n_samples, n_features) - The input data. - - y : array of shape (n_samples) - class labels of each example in X. - - k : int - Neighborhood size for calculating the KDN score. - - Returns - ------- - - score : array of shape = [n_samples,1] - KDN score of each sample in X. - - neighbors : array of shape = [n_samples,k] - Indexes of the k neighbors of each sample in X. - - - References - ---------- - M. R. Smith, T. Martinez, C. Giraud-Carrier, An instance level analysis of - data complexity, - Machine Learning 95 (2) (2014) 225-256. - - """ - - nbrs = NearestNeighbors(n_neighbors=k + 1, algorithm='kd_tree').fit(X) - _, indices = nbrs.kneighbors(X) - neighbors = indices[:, 1:] - diff_class = np.tile(y, (k, 1)).transpose() != y[neighbors] - score = np.sum(diff_class, axis=1) / k - return score, neighbors diff --git a/deslib/util/knne.py b/deslib/util/knne.py deleted file mode 100644 index 70f74486..00000000 --- a/deslib/util/knne.py +++ /dev/null @@ -1,250 +0,0 @@ -import warnings - -import numpy as np -from sklearn.base import BaseEstimator -from sklearn.neighbors import KNeighborsClassifier -from sklearn.utils import check_X_y -from sklearn.utils import check_array - -from deslib.util import faiss_knn_wrapper -from deslib.util.prob_functions import softmax - - -class KNNE(BaseEstimator): - """" - Implementation of the K-Nearest Neighbors-Equality technique. - - This implementation fits a different KNN method for each class, and search - on each class for the nearest examples. - - Parameters - ---------- - - n_neighbors : int, (default = 7) - Number of neighbors to use by default for :meth:`kneighbors` queries. - - algorithm : str = ['knn', 'faiss]', (default = 'knn') - Whether to use scikit-learn or faiss for nearest neighbors estimation. - - References - ---------- - Sierra, Basilio, Elena Lazkano, Itziar Irigoien, Ekaitz Jauregi, - and Iñigo Mendialdua. "K nearest neighbor equality: giving equal chance - to all existing classes." - Information Sciences 181, no. 23 (2011): 5158-5168. - - Mendialdua, Iñigo, José María Martínez-Otzeta, I. Rodriguez-Rodriguez, - T. Ruiz-Vazquez, and Basilio Sierra. "Dynamic selection of the best base - classifier in one versus one." Knowledge-Based Systems 85 (2015): 298-306. - - Cruz, Rafael MO, Dayvid VR Oliveira, George DC Cavalcanti, - and Robert Sabourin. "FIRE-DES++: Enhanced online pruning of base - classifiers for dynamic ensemble selection." - Pattern Recognition 85 (2019): 149-160. - """ - - def __init__(self, n_neighbors=7, knn_classifier='sklearn', **kwargs): - - self.n_neighbors = n_neighbors - self.knn_classifier = knn_classifier - self.kwargs = kwargs - - def fit(self, X, y): - """Fit the model according to the given training data. - - Parameters - ---------- - X : array of shape (n_samples, n_features) - Data used to fit the model. - - y : array of shape (n_samples) - class labels of each example in X. - """ - X, y = check_X_y(X, y) - - self.knns_ = {} - self.classes_indexes_ = {} - self.fit_X_ = X - self.fit_y_ = y - self.classes_ = np.unique(y) - self.n_classes_ = self.classes_.size - - # Checking inputs - self._check_n_neighbors(self.n_neighbors) - self._set_knn_type() - - tmp = self._handle_n_neighbors(self.n_neighbors) - self._mdc, self._mod, self._neighbors_per_class = tmp - for class_ in self.classes_: - self.classes_indexes_[class_] = np.argwhere( - np.array(y) == class_).ravel() - y_c = y[self.classes_indexes_[class_]] - X_c = X[self.classes_indexes_[class_], :] - knn = self.knn_type_(n_neighbors=self._neighbors_per_class, - **self.kwargs) - self.knns_[class_] = knn.fit(X_c, y_c) - - return self - - def kneighbors(self, X=None, n_neighbors=None, return_distance=True): - """Finds the K-neighbors of a point. - Returns indices of and distances to the neighbors of each point. - - Parameters - ---------- - X : array-like, shape (n_query, n_features), \ - or (n_query, n_indexed) if metric == 'precomputed' - The query point or points. - If not provided, neighbors of each indexed point are returned. - In this case, the query point is not considered its own neighbor. - - n_neighbors : int - Number of neighbors to get (default is the value - passed to the constructor). - - return_distance : boolean, optional. Defaults to True. - If False, distances will not be returned - - Returns - ------- - dist : array - Array representing the lengths to points, only present if - return_distance=True - - ind : array - Indices of the nearest points in the population matrix. - """ - if n_neighbors is None: - neighbors_per_class = self._neighbors_per_class - mdc = self._mdc - mod = self._mod - else: - mdc, mod, neighbors_per_class = self._handle_n_neighbors( - n_neighbors) - - if X is None: - X = self.fit_X_ - - dists = [] - inds = [] - mod_dists = [] - mod_inds = [] - for class_, knn in self.knns_.items(): - dist_c, ind_c = knn.kneighbors(X, neighbors_per_class) - real_ind_c = self.classes_indexes_[class_].take(ind_c) - dists.append(dist_c[:, 0:mdc]) - inds.append(real_ind_c[:, 0:mdc]) - if mod > 0: - mod_dists.append(dist_c[:, -1].reshape(-1, 1)) - mod_inds.append(real_ind_c[:, -1].reshape(-1, 1)) - - dists, inds = self._organize_neighbors(dists, inds) - - if mod > 0: - mod_dists, mod_inds = self._organize_neighbors(mod_dists, mod_inds) - dists = np.hstack((dists, mod_dists[:, 0:mod])) - inds = np.hstack((inds, mod_inds[:, 0:mod])) - - if return_distance: - return dists, inds - else: - return inds - - def predict(self, X): - """Predict the class label for each sample in X. - - Parameters - ---------- - X : array of shape (n_samples, n_features) - The input data. - - Returns - ------- - preds : array, shape (n_samples,) - Class labels for samples in X. - """ - proba = self.predict_proba(X) - return self.classes_.take(np.argmax(proba, axis=1), axis=0) - - def predict_proba(self, X): - """Return probability estimates for the test data X. - - Parameters - ---------- - X : array-like, shape (n_query, n_features), \ - or (n_query, n_indexed) if metric == 'precomputed' - Test samples. - - Returns - ------- - proba : array of shape (n_samples, n_classes), or a list of n_outputs - of such arrays if n_outputs > 1. - The class probabilities of the input samples. Classes are ordered - by lexicographic order. - """ - X = check_array(X, accept_sparse='csr') - - dists, inds = self.kneighbors(X, return_distance=True) - classes = self.fit_y_[inds] - dists_array = np.empty((X.shape[0], self.n_classes_)) - # TODO: check if a more efficient implementation can be done - for c in self.classes_: - dists_array[:, c] = np.ma.MaskedArray(dists, classes != c).mean( - axis=1) - probas = softmax(1. / dists_array) - return probas - - def _set_knn_type(self): - - if self.knn_classifier is None or self.knn_classifier in ['knn', - 'sklearn']: - self.knn_type_ = KNeighborsClassifier - - elif self.knn_classifier == 'faiss': - if not faiss_knn_wrapper.is_available(): - raise ImportError( - 'Using knn_classifier="faiss" requires that the FAISS ' - 'library be installed.Please check the Installation ' - 'Guide.') - self.knn_type_ = faiss_knn_wrapper.FaissKNNClassifier - - elif callable(self.knn_classifier): - self.knn_type_ = self.knn_classifier - else: - raise ValueError('"knn_classifier" should be one of the following ' - '["knn", "faiss", None] or an estimator class.') - - def _organize_neighbors(self, dists, inds): - inds = np.concatenate(inds, axis=1) - dists = np.concatenate(dists, axis=1) - b = dists.argsort(axis=1) - a = np.tile(np.arange(b.shape[0]).reshape(-1, 1), (1, b.shape[1])) - dists, inds = dists[a, b], inds[a, b] - return dists, inds - - def _check_n_neighbors(self, n_neighbors): - if n_neighbors is None: - raise ValueError('"n_neighbors" is required for the KNN-E model.') - - if n_neighbors < self.n_classes_: - raise ValueError('"n_neighbors" must be equals or higher than ' - 'the number of classes. Got {}.' - .format(n_neighbors)) - - if not np.issubdtype(type(n_neighbors), np.integer): - raise TypeError( - "n_neighbors does not take {} value, " - "enter integer value".format(type(n_neighbors))) - - def _handle_n_neighbors(self, n_neighbors): - mdc = int(n_neighbors / self.n_classes_) - mod = n_neighbors % self.n_classes_ - if mod > 0: - warnings.warn('"n_neighbors" is not a multiple of "n_classes". Got' - '{} and {}.One or more classes will have one less' - ' instance.'.format(n_neighbors, - self.n_classes_)) - n_per_class = mdc + 1 - else: - n_per_class = mdc - return mdc, mod, n_per_class diff --git a/deslib/util/prob_functions.py b/deslib/util/prob_functions.py deleted file mode 100644 index 9a52ca00..00000000 --- a/deslib/util/prob_functions.py +++ /dev/null @@ -1,291 +0,0 @@ -# coding=utf-8 - -# Author: Rafael Menelau Oliveira e Cruz -# -# License: BSD 3 clause -import numpy as np -from scipy.special import betainc -from scipy.stats import entropy - -"""This file contains the implementation of several functions used to estimate -the competence level of a base classifiers based on posterior probabilities -predicted for each class. - -Reference ----------- -T.Woloszynski, M. Kurzynski, A probabilistic model of classifier competence for -dynamic ensemble selection, -Pattern Recognition 44 (2011) 2656–2668. - -R. M. O. Cruz, R. Sabourin, and G. D. Cavalcanti, “Dynamic classifier -selection: Recent advances and perspectives,” -Information Fusion, vol. 41, pp. 195 – 216, 2018. - -B. Antosik, M. Kurzynski, New measures of classifier competence – heuristics -and application to the design of multiple classifier systems., in: Computer -recognition systems 4., 2011, pp. 197–206. -""" - - -def exponential_func(n_classes, support_correct): - """Calculate the exponential function based on the support obtained by - the base classifier for the correct class label. - - Parameters - ---------- - n_classes : int - The number of classes in the problem - - support_correct: array of shape (n_samples) - containing the supports obtained by the base classifier for the correct - class - - Returns - ------- - C_src : array of shape (n_samples) - Representing the classifier competences at each data point - """ - support_correct[support_correct <= 0.0] = 0.0 - C_src = np.zeros(support_correct.size) - - # Special case where the support to the correct class is equal to one. - support_one_indices = support_correct >= 1 - C_src[np.where(support_one_indices)[0]] = 1 - - # Apply the competence formula when the support is less than one - indices_not_one = np.where(~support_one_indices)[0] - - temp = (1.0 - ((n_classes - 1.0) * support_correct[indices_not_one]) / ( - 1.0 - support_correct[indices_not_one])) - C_src[indices_not_one] = (1.0 - (2 ** temp)) - - return C_src - - -def log_func(n_classes, support_correct): - """Calculate the logarithm in the support obtained by - the base classifier. - - Parameters - ---------- - n_classes : int - The number of classes in the problem - - support_correct: array of shape (n_samples) - Containing the supports obtained by the base classifier for the correct - class - - Returns - ------- - C_src : array of shape (n_samples) - representing the classifier competences at each data point - - References - ---------- - T.Woloszynski, M. Kurzynski, A measure of competence based on randomized - reference classifier for dynamic ensemble selection, in: International - Conference on Pattern Recognition (ICPR), 2010, pp. 4194–4197. - """ - - support_correct[support_correct > 1] = 1 - support_correct[support_correct < 0] = 0 - - if n_classes == 2: - C_src = (2 * support_correct) - 1 - else: - temp = np.log(2) / np.log(n_classes) - C_src = (2 * (support_correct ** temp)) - 1 - - return C_src - - -def entropy_func(n_classes, supports, is_correct): - """Calculate the entropy in the support obtained by - the base classifier. The value of the source competence is inverse - proportional to the normalized entropy of its supports vector and the sign - of competence is simply determined by the correct/incorrect classification - - Parameters - ---------- - n_classes : int - The number of classes in the problem - - supports: array of shape (n_samples, n_classes) - Containing the supports obtained by the base classifier for each class. - - is_correct: array of shape (n_samples) - Array with 1 whether the base classifier predicted the correct label - and -1 otherwise - - Returns - ------- - C_src : array of shape (n_samples) - Representing the classifier competences at each data point - - References - ---------- - B. Antosik, M. Kurzynski, New measures of classifier competence – - heuristics and application to the design of multiple classifier systems., - in: Computer recognition systems 4., 2011, pp. 197–206. - """ - n_samples = is_correct.shape[0] - if n_samples != supports.shape[0]: - raise ValueError("The number of samples in X and y must be the same" - "n_samples X = {}, n_samples y = {} ".format( - n_samples, supports.shape[0])) - - supports[supports > 1.0] = 1.0 - supports[supports < 0.0] = 0.0 - - C_src = (1.0 / np.log(n_classes)) * (entropy(supports, axis=1)) - C_src += ((2 * is_correct) - 1) - return C_src - - -def ccprmod(supports, idx_correct_label, B=20): - """Python implementation of the ccprmod.m (Classifier competence based on - probabilistic modelling) - function. Matlab code is available at: - http://www.mathworks.com/matlabcentral/mlc-downloads/downloads/submissions/28391/versions/6/previews/ccprmod.m/index.html - - Parameters - ---------- - supports: array of shape (n_samples, n_classes) - Containing the supports obtained by the base classifier for each class. - - idx_correct_label: array of shape (n_samples) - containing the index of the correct class. - - B : int (Default = 20) - number of points used in the calculation of the competence, higher - values result in a more accurate estimation. - - Returns - ------- - C_src : array of shape (n_samples) - representing the classifier competences at each data point - - Examples - -------- - >>> supports = [[0.3, 0.6, 0.1],[1.0/3, 1.0/3, 1.0/3]] - >>> idx_correct_label = [1,0] - >>> ccprmod(supports,idx_correct_label) - ans = [0.784953394056843, 0.332872292262951] - - References - ---------- - T.Woloszynski, M. Kurzynski, A probabilistic model of classifier competence - for dynamic ensemble selection, - Pattern Recognition 44 (2011) 2656–2668. - """ - if not isinstance(B, int): - raise TypeError( - 'Parameter B should be an integer. ' - 'Currently B is {0}'.format(type(B))) - - if B <= 0 or B is None: - raise ValueError( - 'The parameter B should be higher than 0. ' - 'Currently B is {0}'.format(B)) - - supports = np.asarray(supports) - idx_correct_label = np.array(idx_correct_label) - supports[supports > 1] = 1 - - N, C = supports.shape - - x = np.linspace(0, 1, B) - x = np.tile(x, (N, C)) - - a = np.zeros(x.shape) - - for c in range(C): - a[:, c * B:(c + 1) * B] = C * supports[:, c:c + 1] - - b = C - a - - # For extreme cases, with a or b equal to 0, add a small constant: - eps = 1e-20 - a[a <= eps] = eps - b[b <= eps] = eps - betaincj = betainc(a, b, x) - - C_src = np.zeros(N) - for n in range(N): - t = range((idx_correct_label[n]) * B, (idx_correct_label[n] + 1) * B) - bc = betaincj[n, t].reshape(1, len(t)) - bi = betaincj[n, list(set(range(0, (C * B))) - set(t))] - bi = np.transpose(np.reshape(bi, (B, C - 1), order='F')) - C_src[n] = np.sum(np.multiply((bc[0, 1:] - bc[0, 0:-1]), - np.prod((bi[:, 0:-1] + bi[:, 1:]) / 2, - 0))) - - return C_src - - -def min_difference(supports, idx_correct_label): - """The minimum difference between the supports obtained for the correct - class and the vector of class supports. The value of the source competence - is negative if the sample is misclassified and positive otherwise. - - Parameters - ---------- - supports: array of shape (n_samples, n_classes) - Containing the supports obtained by the base classifier for each class - - idx_correct_label: array of shape (n_samples) - Containing the index of the correct class - - Returns - ------- - C_src : array of shape (n_samples) - Representing the classifier competences at each data point - - References - ---------- - B. Antosik, M. Kurzynski, New measures of classifier competence – - heuristics and application to the design of multiple classifier systems., - in: Computer recognition systems 4., 2011, pp. 197–206. - """ - n_samples = len(idx_correct_label) - # Boolean mask for the correct class - mask = np.zeros(supports.shape, dtype=bool) - mask[np.arange(n_samples), idx_correct_label] = True - # Get supports for the correct class - supports_correct = supports[mask] - # Get supports for the other classes - supports_others = supports[~mask] - - if len(supports_others) == 0: - # Corner case where there is a single class in y_train - supports_others = np.zeros_like(supports_correct) - - difference = supports_correct.reshape(-1, 1) - supports_others.reshape( - supports_correct.size, -1) - C_src = np.sort(difference, axis=1)[:, 0] - return C_src - - -def softmax(w, theta=1.0): - """Takes an vector w of S N-element and returns a vectors where each column - of the vector sums to 1, with elements exponentially proportional to the - respective elements in N. - - Parameters - ---------- - w : array of shape = [N, M] - - theta : float (default = 1.0) - used as a multiplier prior to exponentiation. - - Returns - ------- - dist : array of shape = [N, M] - Which the sum of each row sums to 1 and the elements are exponentially - proportional to the respective elements in N - - """ - w = np.atleast_2d(w) - e = np.exp(np.array(w) / theta) - dist = e / np.sum(e, axis=1).reshape(-1, 1) - return dist diff --git a/docs/.gitignore b/docs/.gitignore deleted file mode 100644 index e35d8850..00000000 --- a/docs/.gitignore +++ /dev/null @@ -1 +0,0 @@ -_build diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index 3bc518ab..00000000 --- a/docs/Makefile +++ /dev/null @@ -1,192 +0,0 @@ -# Makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -PAPER = -BUILDDIR = _build - -# User-friendly check for sphinx-build -ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) -$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) -endif - -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -# the i18n builder cannot share the environment and doctrees with the others -I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . - -.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext - -help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " singlehtml to make a single large HTML file" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " applehelp to make an Apple Help Book" - @echo " devhelp to make HTML files and a Devhelp project" - @echo " epub to make an epub" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " latexpdf to make LaTeX files and run them through pdflatex" - @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" - @echo " text to make text files" - @echo " man to make manual pages" - @echo " texinfo to make Texinfo files" - @echo " info to make Texinfo files and run them through makeinfo" - @echo " gettext to make PO message catalogs" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " xml to make Docutils-native XML files" - @echo " pseudoxml to make pseudoxml-XML files for display purposes" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - @echo " coverage to run coverage check of the documentation (if enabled)" - -clean: - rm -rf $(BUILDDIR)/* - -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -singlehtml: - $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml - @echo - @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." - -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/deslib.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/deslib.qhc" - -applehelp: - $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp - @echo - @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." - @echo "N.B. You won't be able to view it unless you put it in" \ - "~/Library/Documentation/Help or install it in your application" \ - "bundle." - -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp - @echo - @echo "Build finished." - @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/deslib" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/deslib" - @echo "# devhelp" - -epub: - $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub - @echo - @echo "Build finished. The epub file is in $(BUILDDIR)/epub." - -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make' in that directory to run these through (pdf)latex" \ - "(use \`make latexpdf' here to do that automatically)." - -latexpdf: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through pdflatex..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -latexpdfja: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through platex and dvipdfmx..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -text: - $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text - @echo - @echo "Build finished. The text files are in $(BUILDDIR)/text." - -man: - $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man - @echo - @echo "Build finished. The manual pages are in $(BUILDDIR)/man." - -texinfo: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo - @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." - @echo "Run \`make' in that directory to run these through makeinfo" \ - "(use \`make info' here to do that automatically)." - -info: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo "Running Texinfo files through makeinfo..." - make -C $(BUILDDIR)/texinfo info - @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." - -gettext: - $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale - @echo - @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." - -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." - -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." - -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." - -coverage: - $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage - @echo "Testing of coverage in the sources finished, look at the " \ - "results in $(BUILDDIR)/coverage/python.txt." - -xml: - $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml - @echo - @echo "Build finished. The XML files are in $(BUILDDIR)/xml." - -pseudoxml: - $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml - @echo - @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." diff --git a/docs/_static/.keep b/docs/_static/.keep deleted file mode 100644 index e69de29b..00000000 diff --git a/docs/api.rst b/docs/api.rst deleted file mode 100644 index 8639cdd8..00000000 --- a/docs/api.rst +++ /dev/null @@ -1,82 +0,0 @@ -.. _api-reference: - -###################### -API Reference -###################### - -This is the full API documentation of the `DESlib`. Currently the library is divided into four modules: - -Dynamic Classifier Selection (DCS) ------------------------------------ - -This module contains the implementation of techniques in which only the base -classifier that attained the highest competence level is selected for the classification of the query. - -.. automodule:: deslib.dcs - -.. toctree:: - :maxdepth: 3 - - modules/dcs/a_posteriori - modules/dcs/a_priori - modules/dcs/lca - modules/dcs/mcb - modules/dcs/mla - modules/dcs/ola - modules/dcs/rank - -Dynamic Ensemble Selection (DES) ------------------------------------ - -Dynamic ensemble selection strategies refer to techniques that select an ensemble of classifier rather than a single one. -All base classifiers that attain a minimum competence level are selected to compose the ensemble of classifiers. - -.. automodule:: deslib.des - -.. toctree:: - :maxdepth: 3 - - modules/des/meta_des - modules/des/des_clustering - modules/des/des_p - modules/des/ds_knn - modules/des/knop - modules/des/knora_e - modules/des/knora_u - modules/des/desmi - modules/des/probabilistic - -Static ensembles ------------------------------------ - -This module provides the implementation of static ensemble techniques that are usually used as a baseline for the -comparison of DS methods: Single Best (SB), Static Selection (SS), Stacked classifier and Oracle. - - -.. automodule:: deslib.static - -.. toctree:: - :maxdepth: 3 - - modules/static/oracle - modules/static/single_best - modules/static/static_selection - modules/static/stacked - -Utils ------------------------------------ -Utility functions for ensemble methods such as diversity and aggregation methods. - -.. automodule:: deslib.util - -.. toctree:: - :maxdepth: 3 - - modules/util/diversity - modules/util/aggregation - modules/util/prob_functions - modules/util/instance_hardness - modules/util/dfp - modules/util/knne - modules/util/faiss_knn_wrapper - modules/util/datasets diff --git a/docs/conf.py b/docs/conf.py deleted file mode 100644 index 4cdd84b0..00000000 --- a/docs/conf.py +++ /dev/null @@ -1,332 +0,0 @@ -# -*- coding: utf-8 -*- -# -# deslib documentation build configuration file, created by -# sphinx-quickstart on Wed Dec 13 10:07:40 2017. -# -# This file is execfile()d with the current directory set to its -# containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -import sys -import os -import deslib -import sphinx_gallery - -version = '.'.join(deslib.__version__.split('.', 2)[:2]) -release = deslib.__version__ - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.insert(0, os.path.abspath('.')) -sys.path.insert(1, os.path.abspath('..')) - -# -- General configuration ------------------------------------------------ - -# If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.autosummary', - 'sphinx.ext.mathjax', - 'sphinx.ext.linkcode', # link to github, see linkcode_resolve() below - 'numpydoc', - 'sphinx_gallery.gen_gallery', -] - -numpydoc_show_class_members = False - -# See https://github.com/rtfd/readthedocs.org/issues/283 -mathjax_path = ('https://cdn.mathjax.org/mathjax/latest/MathJax.js?' - 'config=TeX-AMS-MML_HTMLorMML') - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix(es) of source filenames. -# You can specify multiple suffix as a list of string: -# source_suffix = ['.rst', '.md'] -source_suffix = '.rst' - -# The encoding of source files. -#source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = u'deslib' -copyright = u'2018, Rafael M. O. Cruz' -author = u'Rafael M. O. Cruz' - -import deslib - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# -# This is also used if you do content translation via gettext catalogs. -# Usually you set "language" from the command line for these cases. -language = None - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -#today = '' -# Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -exclude_patterns = ['_build'] - -# The reST default role (used for this markup: `text`) to use for all -# documents. -#default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -#add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -#show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' - -# A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] - -# If true, keep warnings as "system message" paragraphs in the built documents. -#keep_warnings = False - -# If true, `todo` and `todoList` produce output, else they produce nothing. -todo_include_todos = False - - -# -- Options for HTML output ---------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -#html_theme = 'alabaster' -html_theme = 'sphinx_rtd_theme' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -#html_theme_options = {} - -# Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -#html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -#html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -#html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] - -# Add any extra paths that contain custom files (such as robots.txt or -# .htaccess) here, relative to this directory. These files are copied -# directly to the root of the documentation. -#html_extra_path = [] - -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -#html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -#html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -#html_additional_pages = {} - -# If false, no module index is generated. -#html_domain_indices = True - -# If false, no index is generated. -#html_use_index = True - -# If true, the index is split into individual pages for each letter. -#html_split_index = False - -# If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -#html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None - -# Language to be used for generating the HTML full-text search index. -# Sphinx supports the following languages: -# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' -# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' -#html_search_language = 'en' - -# A dictionary with options for the search language support, empty by default. -# Now only 'ja' uses this config value -#html_search_options = {'type': 'default'} - -# The name of a javascript file (relative to the configuration directory) that -# implements a search results scorer. If empty, the default will be used. -#html_search_scorer = 'scorer.js' - -# Output file base name for HTML help builder. -htmlhelp_basename = 'deslibdoc' - -# -- Options for LaTeX output --------------------------------------------- - -latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', - -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', - -# Additional stuff for the LaTeX preamble. -#'preamble': '', - -# Latex figure (float) alignment -#'figure_align': 'htbp', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - (master_doc, 'deslib.tex', u'deslib Documentation', - u'Rafael M. O. Cruz', 'manual'), -] - -# The name of an image file (relative to this directory) to place at the top of -# the title page. -#latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -#latex_use_parts = False - -# If true, show page references after internal links. -#latex_show_pagerefs = False - -# If true, show URL addresses after external links. -#latex_show_urls = False - -# Documents to append as an appendix to all manuals. -#latex_appendices = [] - -# If false, no module index is generated. -#latex_domain_indices = True - -# sphinx-gallery configuration -sphinx_gallery_conf = { - 'doc_module': 'deslib', - 'backreferences_dir': os.path.join('generated'), - 'reference_url': { - 'deslib': None} -} - -# -- Options for manual page output --------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'deslib', u'deslib Documentation', - [author], 1) -] - -# If true, show URL addresses after external links. -#man_show_urls = False - - -# -- Options for Texinfo output ------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - (master_doc, 'deslib', u'deslib Documentation', - author, 'deslib', 'One line description of project.', - 'Miscellaneous'), -] - -# Documents to append as an appendix to all manuals. -#texinfo_appendices = [] - -# If false, no module index is generated. -#texinfo_domain_indices = True - -# How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' - -# If true, do not generate a @detailmenu in the "Top" node's menu. -#texinfo_no_detailmenu = False - - -# Resolve function for the linkcode extension. -def linkcode_resolve(domain, info): - def find_source(): - # try to find the file and line number, based on code from numpy: - # https://github.com/numpy/numpy/blob/master/doc/source/conf.py#L286 - obj = sys.modules[info['module']] - for part in info['fullname'].split('.'): - obj = getattr(obj, part) - import inspect - import os - fn = inspect.getsourcefile(obj) - fn = os.path.relpath(fn, start=os.path.dirname(deslib.__file__)) - source, lineno = inspect.getsourcelines(obj) - return fn, lineno, lineno + len(source) - 1 - - if domain != 'py' or not info['module']: - return None - try: - filename = 'deslib/%s#L%d-L%d' % find_source() - except Exception: - filename = info['module'].replace('.', '/') + '.py' - tag = 'master' if 'dev' in release else ('v' + release) - return "https://github.com/scikit-learn-contrib/DESlib/blob/%s/%s" % (tag, filename) diff --git a/docs/index.rst b/docs/index.rst deleted file mode 100644 index a26a472d..00000000 --- a/docs/index.rst +++ /dev/null @@ -1,111 +0,0 @@ -Welcome to DESlib documentation! -================================================= - -DESlib is an ensemble learning library focusing the implementation of the state-of-the-art techniques for dynamic classifier -and ensemble selection. - -DESlib is a work in progress. Contributions are welcomed through its GitHub page: https://github.com/scikit-learn-contrib/DESlib. - -Introduction --------------- -Dynamic Selection (DS) refers to techniques in which the base classifiers are selected -on the fly, according to each new sample to be classified. Only the most competent, or an ensemble containing the most competent classifiers is selected to predict -the label of a specific test sample. The rationale for such techniques is that not every classifier in -the pool is an expert in classifying all unknown samples; rather, each base classifier is an expert in -a different local region of the feature space. - -DS is one of the most promising MCS approaches due to the fact that -more and more works are reporting the superior performance of such techniques over static combination methods. Such techniques -have achieved better classification performance especially when dealing with small-sized and imbalanced datasets. A -comprehensive review of dynamic selection can be found in the following papers [1]_ [2]_ - -Philosophy ------------ -DESlib was developed with two objectives in mind: to make it easy to integrate Dynamic Selection algorithms to -machine learning projects, and to facilitate research on this topic, by providing implementations of the main -DES and DCS methods, as well as the commonly used baseline methods. Each algorithm implements the main methods -in the scikit-learn_ API **scikit-learn**: **fit(X, y)**, **predict(X)**, **predict_proba(X)** -and **score(X, y)**. - -The implementation of the DS methods is modular, following a taxonomy defined in [1]_. -This taxonomy considers the main characteristics of DS methods, that are centered in three components: - -1. the methodology used to define the local region, in which the competence level of the base classifiers are estimated (region of competence); -2. the source of information used to estimate the competence level of the base classifiers. -3. the selection approach to define the best classifier (for DCS) or the best set of classifiers (for DES). - -This modular approach makes it easy for researchers to implement new DS methods, in many cases requiring only the -implementation of the method **estimate_competence**, that is, how the local competence of the base classifier is measured. - -`API Reference `_ ----------------------------- - -If you are looking for information on a specific function, class or -method, this part of the documentation is for you. - -.. toctree:: - :hidden: - - user_guide - api - auto_examples/index - news - - -`Example `_ ----------------------------------------- - -Here we present an example of the KNORA-E techniques using a random forest to generate the pool of classifiers: - -.. code-block:: python - - from sklearn.ensemble import RandomForestClassifier - from deslib.des.knora_e import KNORAE - - # Train a pool of 10 classifiers - pool_classifiers = RandomForestClassifier(n_estimators=10) - pool_classifiers.fit(X_train, y_train) - - # Initialize the DES model - knorae = KNORAE(pool_classifiers) - - # Preprocess the Dynamic Selection dataset (DSEL) - knorae.fit(X_dsel, y_dsel) - - # Predict new examples: - knorae.predict(X_test) - -The library accepts any list of classifiers (from scikit-learn) as input, including a list containing different classifier models (heterogeneous ensembles). -More examples to use the API can be found in the `examples page `_. - - -Citation -================== - -If you use DESLib in a scientific paper, please consider citing the following paper: - -Rafael M. O. Cruz, Luiz G. Hafemann, Robert Sabourin and George D. C. Cavalcanti **DESlib: A Dynamic ensemble selection library in Python.** arXiv preprint arXiv:1802.04967 (2018). - -.. code-block:: text - - @article{JMLR:v21:18-144, - author = {Rafael M. O. Cruz and Luiz G. Hafemann and Robert Sabourin and George D. C. Cavalcanti}, - title = {DESlib: A Dynamic ensemble selection library in Python}, - journal = {Journal of Machine Learning Research}, - year = {2020}, - volume = {21}, - number = {8}, - pages = {1-5}, - url = {http://jmlr.org/papers/v21/18-144.html} - } - - -References ------------ -.. [1] : R. M. O. Cruz, R. Sabourin, and G. D. Cavalcanti, “Dynamic classifier selection: Recent advances and perspectives,” Information Fusion, vol. 41, pp. 195 – 216, 2018. - -.. [2] : A. S. Britto, R. Sabourin, L. E. S. de Oliveira, Dynamic selection of classifiers - A comprehensive review, Pattern Recognition 47 (11) (2014) 3665–3680. - -.. _scikit-learn: http://scikit-learn.org/stable/ - -.. _GitHub: https://github.com/scikit-learn-contrib/DESlib diff --git a/docs/make.bat b/docs/make.bat deleted file mode 100644 index d361c82b..00000000 --- a/docs/make.bat +++ /dev/null @@ -1,263 +0,0 @@ -@ECHO OFF - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set BUILDDIR=_build -set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . -set I18NSPHINXOPTS=%SPHINXOPTS% . -if NOT "%PAPER%" == "" ( - set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% - set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% -) - -if "%1" == "" goto help - -if "%1" == "help" ( - :help - echo.Please use `make ^` where ^ is one of - echo. html to make standalone HTML files - echo. dirhtml to make HTML files named index.html in directories - echo. singlehtml to make a single large HTML file - echo. pickle to make pickle files - echo. json to make JSON files - echo. htmlhelp to make HTML files and a HTML help project - echo. qthelp to make HTML files and a qthelp project - echo. devhelp to make HTML files and a Devhelp project - echo. epub to make an epub - echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter - echo. text to make text files - echo. man to make manual pages - echo. texinfo to make Texinfo files - echo. gettext to make PO message catalogs - echo. changes to make an overview over all changed/added/deprecated items - echo. xml to make Docutils-native XML files - echo. pseudoxml to make pseudoxml-XML files for display purposes - echo. linkcheck to check all external links for integrity - echo. doctest to run all doctests embedded in the documentation if enabled - echo. coverage to run coverage check of the documentation if enabled - goto end -) - -if "%1" == "clean" ( - for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i - del /q /s %BUILDDIR%\* - goto end -) - - -REM Check if sphinx-build is available and fallback to Python version if any -%SPHINXBUILD% 2> nul -if errorlevel 9009 goto sphinx_python -goto sphinx_ok - -:sphinx_python - -set SPHINXBUILD=python -m sphinx.__init__ -%SPHINXBUILD% 2> nul -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - exit /b 1 -) - -:sphinx_ok - - -if "%1" == "html" ( - %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/html. - goto end -) - -if "%1" == "dirhtml" ( - %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. - goto end -) - -if "%1" == "singlehtml" ( - %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. - goto end -) - -if "%1" == "pickle" ( - %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the pickle files. - goto end -) - -if "%1" == "json" ( - %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the JSON files. - goto end -) - -if "%1" == "htmlhelp" ( - %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run HTML Help Workshop with the ^ -.hhp project file in %BUILDDIR%/htmlhelp. - goto end -) - -if "%1" == "qthelp" ( - %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run "qcollectiongenerator" with the ^ -.qhcp project file in %BUILDDIR%/qthelp, like this: - echo.^> qcollectiongenerator %BUILDDIR%\qthelp\deslib.qhcp - echo.To view the help file: - echo.^> assistant -collectionFile %BUILDDIR%\qthelp\deslib.ghc - goto end -) - -if "%1" == "devhelp" ( - %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. - goto end -) - -if "%1" == "epub" ( - %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The epub file is in %BUILDDIR%/epub. - goto end -) - -if "%1" == "latex" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "latexpdf" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - cd %BUILDDIR%/latex - make all-pdf - cd %~dp0 - echo. - echo.Build finished; the PDF files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "latexpdfja" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - cd %BUILDDIR%/latex - make all-pdf-ja - cd %~dp0 - echo. - echo.Build finished; the PDF files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "text" ( - %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The text files are in %BUILDDIR%/text. - goto end -) - -if "%1" == "man" ( - %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The manual pages are in %BUILDDIR%/man. - goto end -) - -if "%1" == "texinfo" ( - %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. - goto end -) - -if "%1" == "gettext" ( - %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The message catalogs are in %BUILDDIR%/locale. - goto end -) - -if "%1" == "changes" ( - %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes - if errorlevel 1 exit /b 1 - echo. - echo.The overview file is in %BUILDDIR%/changes. - goto end -) - -if "%1" == "linkcheck" ( - %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck - if errorlevel 1 exit /b 1 - echo. - echo.Link check complete; look for any errors in the above output ^ -or in %BUILDDIR%/linkcheck/output.txt. - goto end -) - -if "%1" == "doctest" ( - %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest - if errorlevel 1 exit /b 1 - echo. - echo.Testing of doctests in the sources finished, look at the ^ -results in %BUILDDIR%/doctest/output.txt. - goto end -) - -if "%1" == "coverage" ( - %SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage - if errorlevel 1 exit /b 1 - echo. - echo.Testing of coverage in the sources finished, look at the ^ -results in %BUILDDIR%/coverage/python.txt. - goto end -) - -if "%1" == "xml" ( - %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The XML files are in %BUILDDIR%/xml. - goto end -) - -if "%1" == "pseudoxml" ( - %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. - goto end -) - -:end diff --git a/docs/modules/dcs/a_posteriori.rst b/docs/modules/dcs/a_posteriori.rst deleted file mode 100644 index b724db0d..00000000 --- a/docs/modules/dcs/a_posteriori.rst +++ /dev/null @@ -1,7 +0,0 @@ -A posteriori ------------- - -.. automodule:: deslib.dcs.a_posteriori - -.. autoclass:: APosteriori - :members: fit, predict, predict_proba, score, estimate_competence, select diff --git a/docs/modules/dcs/a_priori.rst b/docs/modules/dcs/a_priori.rst deleted file mode 100644 index ef49b5e0..00000000 --- a/docs/modules/dcs/a_priori.rst +++ /dev/null @@ -1,7 +0,0 @@ -A Priori ----------- - -.. automodule:: deslib.dcs.a_priori - -.. autoclass:: APriori - :members: fit, predict, predict_proba, score, estimate_competence, select diff --git a/docs/modules/dcs/lca.rst b/docs/modules/dcs/lca.rst deleted file mode 100644 index 823bc67d..00000000 --- a/docs/modules/dcs/lca.rst +++ /dev/null @@ -1,7 +0,0 @@ -Local Class Accuracy (LCA) --------------------------- - -.. automodule:: deslib.dcs.lca - -.. autoclass:: LCA - :members: fit, predict, predict_proba, score, estimate_competence, select diff --git a/docs/modules/dcs/mcb.rst b/docs/modules/dcs/mcb.rst deleted file mode 100644 index f72f9a4a..00000000 --- a/docs/modules/dcs/mcb.rst +++ /dev/null @@ -1,7 +0,0 @@ -Multiple Classifier Behaviour (MCB) ------------------------------------ - -.. automodule:: deslib.dcs.mcb - -.. autoclass:: MCB - :members: fit, predict, predict_proba, score, estimate_competence, select diff --git a/docs/modules/dcs/mla.rst b/docs/modules/dcs/mla.rst deleted file mode 100644 index 689d8c92..00000000 --- a/docs/modules/dcs/mla.rst +++ /dev/null @@ -1,7 +0,0 @@ -Modified Local Accuracy (MLA) ------------------------------ - -.. automodule:: deslib.dcs.mla - -.. autoclass:: MLA - :members: fit, predict, predict_proba, score, estimate_competence, select diff --git a/docs/modules/dcs/ola.rst b/docs/modules/dcs/ola.rst deleted file mode 100644 index b54e6446..00000000 --- a/docs/modules/dcs/ola.rst +++ /dev/null @@ -1,7 +0,0 @@ -Overall Local Accuracy (OLA) ------------------------------ - -.. automodule:: deslib.dcs.ola - -.. autoclass:: OLA - :members: fit, predict, predict_proba, score, estimate_competence, select diff --git a/docs/modules/dcs/rank.rst b/docs/modules/dcs/rank.rst deleted file mode 100644 index 15e961a4..00000000 --- a/docs/modules/dcs/rank.rst +++ /dev/null @@ -1,8 +0,0 @@ -Modified Rank ---------------- - -.. automodule:: deslib.dcs.rank - -.. autoclass:: Rank - :members: fit, predict, predict_proba, score, estimate_competence, select - diff --git a/docs/modules/des/des_clustering.rst b/docs/modules/des/des_clustering.rst deleted file mode 100644 index 38542d11..00000000 --- a/docs/modules/des/des_clustering.rst +++ /dev/null @@ -1,7 +0,0 @@ -DES Clustering --------------- - -.. automodule:: deslib.des.des_clustering - -.. autoclass:: DESClustering - :members: fit, predict, predict_proba, score, estimate_competence, select diff --git a/docs/modules/des/des_p.rst b/docs/modules/des/des_p.rst deleted file mode 100644 index cb61e4ff..00000000 --- a/docs/modules/des/des_p.rst +++ /dev/null @@ -1,7 +0,0 @@ -Dynamic Ensemble Selection performance (DES-P) ----------------------------------------------- - -.. automodule:: deslib.des.des_p - -.. autoclass:: DESP - :members: fit, predict, predict_proba, score, estimate_competence, select diff --git a/docs/modules/des/deskl.rst b/docs/modules/des/deskl.rst deleted file mode 100644 index 71a0ab53..00000000 --- a/docs/modules/des/deskl.rst +++ /dev/null @@ -1,7 +0,0 @@ -DES-Kullback Leibler ---------------------- - -.. automodule:: deslib.des.probabilistic - -.. autoclass:: DESKL - :members: source_competence, fit, predict, predict_proba, score, estimate_competence, select diff --git a/docs/modules/des/desmi.rst b/docs/modules/des/desmi.rst deleted file mode 100644 index 9c1c71a8..00000000 --- a/docs/modules/des/desmi.rst +++ /dev/null @@ -1,7 +0,0 @@ -DES Multiclass Imbalance (DES-MI) ----------------------------------- - -.. automodule:: deslib.des.des_mi - -.. autoclass:: DESMI - :members: fit, predict, predict_proba, score, estimate_competence, select diff --git a/docs/modules/des/ds_knn.rst b/docs/modules/des/ds_knn.rst deleted file mode 100644 index b3c6703f..00000000 --- a/docs/modules/des/ds_knn.rst +++ /dev/null @@ -1,7 +0,0 @@ -DES-KNN --------------- - -.. automodule:: deslib.des.des_knn - -.. autoclass:: DESKNN - :members: fit, predict, predict_proba, score, estimate_competence, select diff --git a/docs/modules/des/exponential.rst b/docs/modules/des/exponential.rst deleted file mode 100644 index 5eae90dc..00000000 --- a/docs/modules/des/exponential.rst +++ /dev/null @@ -1,7 +0,0 @@ -DES-Exponential ----------------- - -.. automodule:: deslib.des.probabilistic - -.. autoclass:: Exponential - :members: source_competence, fit, predict, predict_proba, score, estimate_competence, select \ No newline at end of file diff --git a/docs/modules/des/knop.rst b/docs/modules/des/knop.rst deleted file mode 100644 index f678d26d..00000000 --- a/docs/modules/des/knop.rst +++ /dev/null @@ -1,7 +0,0 @@ -k-Nearest Output Profiles (KNOP) --------------------------------- - -.. automodule:: deslib.des.knop - -.. autoclass:: KNOP - :members: fit, predict, predict_proba, score, estimate_competence_from_proba, select diff --git a/docs/modules/des/knora_e.rst b/docs/modules/des/knora_e.rst deleted file mode 100644 index 87204a9d..00000000 --- a/docs/modules/des/knora_e.rst +++ /dev/null @@ -1,7 +0,0 @@ -k-Nearest Oracle-Eliminate (KNORA-E) --------------------------------------- - -.. automodule:: deslib.des.knora_e - -.. autoclass:: KNORAE - :members: fit, predict, predict_proba, score, estimate_competence, select diff --git a/docs/modules/des/knora_u.rst b/docs/modules/des/knora_u.rst deleted file mode 100644 index a29b3d7a..00000000 --- a/docs/modules/des/knora_u.rst +++ /dev/null @@ -1,7 +0,0 @@ -k-Nearest Oracle Union (KNORA-U) --------------------------------- - -.. automodule:: deslib.des.knora_u - -.. autoclass:: KNORAU - :members: fit, predict, predict_proba, score, estimate_competence, select diff --git a/docs/modules/des/logarithmic.rst b/docs/modules/des/logarithmic.rst deleted file mode 100644 index a254a71c..00000000 --- a/docs/modules/des/logarithmic.rst +++ /dev/null @@ -1,7 +0,0 @@ -DES-Logarithmic ---------------- - -.. automodule:: deslib.des.probabilistic - -.. autoclass:: Logarithmic - :members: source_competence, fit, predict, predict_proba, score, estimate_competence, select \ No newline at end of file diff --git a/docs/modules/des/meta_des.rst b/docs/modules/des/meta_des.rst deleted file mode 100644 index e8838724..00000000 --- a/docs/modules/des/meta_des.rst +++ /dev/null @@ -1,7 +0,0 @@ -META-DES -======== - -.. automodule:: deslib.des.meta_des - -.. autoclass:: METADES - :members: fit, predict, predict_proba, score, estimate_competence_from_proba, select diff --git a/docs/modules/des/minimum_difference.rst b/docs/modules/des/minimum_difference.rst deleted file mode 100644 index 5bc8d7b2..00000000 --- a/docs/modules/des/minimum_difference.rst +++ /dev/null @@ -1,7 +0,0 @@ -DES-Minimum Difference ----------------------- - -.. automodule:: deslib.des.probabilistic - -.. autoclass:: MinimumDifference - :members: source_competence, fit, predict, predict_proba, score, estimate_competence, select \ No newline at end of file diff --git a/docs/modules/des/probabilistic.rst b/docs/modules/des/probabilistic.rst deleted file mode 100644 index a3310c82..00000000 --- a/docs/modules/des/probabilistic.rst +++ /dev/null @@ -1,14 +0,0 @@ -Probabilistic -------------- - -.. automodule:: deslib.des.probabilistic - -.. autoclass:: BaseProbabilistic - :members: - -.. toctree:: - rrc - deskl - minimum_difference - exponential - logarithmic diff --git a/docs/modules/des/rrc.rst b/docs/modules/des/rrc.rst deleted file mode 100644 index b1a78160..00000000 --- a/docs/modules/des/rrc.rst +++ /dev/null @@ -1,7 +0,0 @@ -Randomized Reference Classifier (RRC) -------------------------------------- - -.. automodule:: deslib.des.probabilistic - -.. autoclass:: RRC - :members: source_competence, fit, predict, predict_proba, score, estimate_competence, select \ No newline at end of file diff --git a/docs/modules/static/oracle.rst b/docs/modules/static/oracle.rst deleted file mode 100644 index 5870ed7c..00000000 --- a/docs/modules/static/oracle.rst +++ /dev/null @@ -1,8 +0,0 @@ -Oracle --------------- - -.. automodule:: deslib.static.oracle - -.. autoclass:: Oracle - :members: - diff --git a/docs/modules/static/single_best.rst b/docs/modules/static/single_best.rst deleted file mode 100644 index 344dcade..00000000 --- a/docs/modules/static/single_best.rst +++ /dev/null @@ -1,8 +0,0 @@ -Single Best --------------- - -.. automodule:: deslib.static.single_best - -.. autoclass:: SingleBest - :members: fit, predict, predict_proba, score - diff --git a/docs/modules/static/stacked.rst b/docs/modules/static/stacked.rst deleted file mode 100644 index 9ad85a77..00000000 --- a/docs/modules/static/stacked.rst +++ /dev/null @@ -1,7 +0,0 @@ -Stacked Classifier ------------------- - -.. automodule:: deslib.static.stacked - -.. autoclass:: StackedClassifier - :members: fit, predict, predict_proba, score diff --git a/docs/modules/static/static_selection.rst b/docs/modules/static/static_selection.rst deleted file mode 100644 index eed213f1..00000000 --- a/docs/modules/static/static_selection.rst +++ /dev/null @@ -1,7 +0,0 @@ -Static Selection ----------------- - -.. automodule:: deslib.static.static_selection - -.. autoclass:: StaticSelection - :members: fit, predict, predict_proba, score diff --git a/docs/modules/util/aggregation.rst b/docs/modules/util/aggregation.rst deleted file mode 100644 index 1d0d4777..00000000 --- a/docs/modules/util/aggregation.rst +++ /dev/null @@ -1,8 +0,0 @@ -Aggregation --------------- - -This file contains the implementation of different aggregation functions to combine the outputs of the base -classifiers to give the final decision. - -.. automodule:: deslib.util.aggregation - :members: diff --git a/docs/modules/util/datasets.rst b/docs/modules/util/datasets.rst deleted file mode 100644 index 5fe0388f..00000000 --- a/docs/modules/util/datasets.rst +++ /dev/null @@ -1,15 +0,0 @@ -Datasets --------- - -This file contains routines to generate 2D classification datasets -that can be used to test the performance of different machine learning -algorithms. - -- P2 Dataset -- Circle and Square -- Banana -- Banana 2 - - -.. automodule:: deslib.util.datasets - :members: diff --git a/docs/modules/util/dfp.rst b/docs/modules/util/dfp.rst deleted file mode 100644 index 25844744..00000000 --- a/docs/modules/util/dfp.rst +++ /dev/null @@ -1,5 +0,0 @@ -Frienemy Pruning ----------------- - -.. automodule:: deslib.util.dfp - :members: \ No newline at end of file diff --git a/docs/modules/util/diversity.rst b/docs/modules/util/diversity.rst deleted file mode 100644 index 364aabe2..00000000 --- a/docs/modules/util/diversity.rst +++ /dev/null @@ -1,15 +0,0 @@ -Diversity --------------- - -This file contains the implementation of key diversity measures found in the ensemble literature: - -- Double Fault -- Negative Double fault -- Q-statistics -- Ratio of errors - -The implementation are made according to the specifications from the book "Combining Pattern Classifiers". - - -.. automodule:: deslib.util.diversity - :members: diff --git a/docs/modules/util/faiss_knn_wrapper.rst b/docs/modules/util/faiss_knn_wrapper.rst deleted file mode 100644 index 5a27f6e1..00000000 --- a/docs/modules/util/faiss_knn_wrapper.rst +++ /dev/null @@ -1,7 +0,0 @@ -FAISS Wrapper --------------- - -.. automodule:: deslib.util.faiss_knn_wrapper - -.. autoclass:: FaissKNNClassifier - :members: fit, predict, predict_proba, kneighbors diff --git a/docs/modules/util/instance_hardness.rst b/docs/modules/util/instance_hardness.rst deleted file mode 100644 index cb0c56cf..00000000 --- a/docs/modules/util/instance_hardness.rst +++ /dev/null @@ -1,7 +0,0 @@ -Instance Hardness -------------------- - -This file contains the implementation of different measures of instance hardness. - -.. automodule:: deslib.util.instance_hardness - :members: diff --git a/docs/modules/util/knne.rst b/docs/modules/util/knne.rst deleted file mode 100644 index c48a4754..00000000 --- a/docs/modules/util/knne.rst +++ /dev/null @@ -1,7 +0,0 @@ -KNN-Equality ------------- - -.. automodule:: deslib.util.knne - -.. autoclass:: KNNE - :members: fit, predict, predict_proba, kneighbors diff --git a/docs/modules/util/prob_functions.rst b/docs/modules/util/prob_functions.rst deleted file mode 100644 index 0415ca04..00000000 --- a/docs/modules/util/prob_functions.rst +++ /dev/null @@ -1,7 +0,0 @@ -Probabilistic Functions ------------------------- -This file contains the implementation of several functions used to estimate the competence -level of a base classifiers based on posterior probabilities predicted for each class. - -.. automodule:: deslib.util.prob_functions - :members: diff --git a/docs/news.rst b/docs/news.rst deleted file mode 100644 index 7df2de35..00000000 --- a/docs/news.rst +++ /dev/null @@ -1,18 +0,0 @@ -.. currentmodule:: deslib - -=============== -Release history -=============== - -.. include:: news/v0.3.rst - -.. include:: news/v0.2.rst - -.. include:: news/v0.1.rst - -.. toctree:: - :hidden: - - news/v0.1 - news/v0.2 - news/v0.3 \ No newline at end of file diff --git a/docs/news/v0.1.rst b/docs/news/v0.1.rst deleted file mode 100644 index 990d9246..00000000 --- a/docs/news/v0.1.rst +++ /dev/null @@ -1,45 +0,0 @@ -Version 0.1 -=========== - -API -~~~ - -- First release of the stable API. By `Rafael M O Cruz`_ and `Luiz G Hafemann`_. - -Implemented methods: -~~~~~~~~~~~~~~~~~~~~~ - -* DES techniques currently available are: - 1. META-DES - 2. K-Nearest-Oracle-Eliminate (KNORA-E) - 3. K-Nearest-Oracle-Union (KNORA-U) - 4. Dynamic Ensemble Selection-Performance(DES-P) - 5. K-Nearest-Output Profiles (KNOP) - 6. Randomized Reference Classifier (DES-RRC) - 7. DES Kullback-Leibler Divergence (DES-KL) - 8. DES-Exponential - 9. DES-Logarithmic - 10. DES-Minimum Difference - 11. DES-Clustering - 12. DES-KNN - -* DCS techniques: - 1. Modified Classifier Rank (Rank) - 2. Overall Locall Accuracy (OLA) - 3. Local Class Accuracy (LCA) - 4. Modified Local Accuracy (MLA) - 5. Multiple Classifier Behaviour (MCB) - 6. A Priori Selection (A Priori) - 7. A Posteriori Selection (A Posteriori) - -* Baseline methods: - 1. Oracle - 2. Single Best - 3. Static Selection - -* Dynamic Frienemy Prunning (DFP) -* Diversity measures -* Aggregation functions - -.. _Rafael M O Cruz: https://github.com/Menelau -.. _Luiz G Hafemann: https://github.com/luizgh diff --git a/docs/news/v0.2.rst b/docs/news/v0.2.rst deleted file mode 100644 index f7003e2d..00000000 --- a/docs/news/v0.2.rst +++ /dev/null @@ -1,17 +0,0 @@ -Version 0.2 -=========== - -- Second release of the stable API. By `Rafael M O Cruz`_ and `Luiz G Hafemann`_. - -Changes -~~~~~~~~~~~~~~~~~~~~~ - -* Implemented Label Encoding: labels are no longer required to be integers starting from 0. Categorical (strings) and non-sequential integers are supported (similarly to scikit-learn). -* Batch processing: Vectorized implementation of predictions. Large speed-up in computation time (100x faster in some cases). -* Predict proba: only required (in the base estimators) if using methods that rely on probabilities (or if requesting probabilities from the ensemble). -* Improved documentation: Included additional examples, a step-by-step tutorial on how to use the library. -* New integration tests: Now covering predict_proba, IH and DFP. -* Bug fixes on 1) predict_proba 2) KNOP with DFP. - -.. _Rafael M O Cruz: https://github.com/Menelau -.. _Luiz G Hafemann: https://github.com/luizgh diff --git a/docs/news/v0.3.5.rst b/docs/news/v0.3.5.rst deleted file mode 100644 index 0d3ad2a8..00000000 --- a/docs/news/v0.3.5.rst +++ /dev/null @@ -1,39 +0,0 @@ -Version 0.3.5 -============= - -- Fourth release of the stable API. By `Rafael M O Cruz`_ and `Luiz G Hafemann`_. This release was mainly focused on compatibility with newer scikit-learn versions, performance improvement and bug fixes. - - -Changes -~~~~~~~~~~~~~~~~~~~~~ -* Update tests according to the new scikit-learn standards. -* Added n_jobs parameter for parallelization. -* Refactored FIRE-DES for faster processing. -* Added new approximated KNN methods using Facebook FAISS search. -* Added passtrhough features for StackedClassifier. -* Added different scoring methods (e.g., AUC, F1-Score) for Single best and Static Selection methods -* Added different scoring methods for DESClustering -* Added predict_proba for the Oracle method. -* Added batch processing for probabilistic methods. -* Added KNearest Neighbors equality option and 'n_neighbors parameter. -* Improved weighted majority voting performance with bath processing. -* Removal of redundant information in documentation. -* Update reference article. - -Bug Fixes -~~~~~~~~~~~~ - -* Fixed randomness with APosteriori and APriori methods during test. -* Fixed error with label encoder for the Oracle and static combination methods -* Methods do not allow a pool containing a single classifier mdoel. -* Removal of Collinear features in stacked classifier. -* Fixed meta-classfier when passing a classifier model to the META-DES technique. -* Fixed DCS-OLA documentation. -* Fixed bug when support given to a class is very small but not zero. -* Fixed FAISS batch processing mode. - - - -.. _Rafael M O Cruz: https://github.com/Menelau -.. _Luiz G Hafemann: https://github.com/luizgh - diff --git a/docs/news/v0.3.rst b/docs/news/v0.3.rst deleted file mode 100644 index bb2ff167..00000000 --- a/docs/news/v0.3.rst +++ /dev/null @@ -1,41 +0,0 @@ -Version 0.3 -=========== - -- Third release of the stable API. By `Rafael M O Cruz`_ and `Luiz G Hafemann`_ - -Changes -~~~~~~~~~~~~~~~~~~~~~ -* All techniques are now sklearn estimators and passes the check_estimator tests. -* All techniques can now be instantiated without a trained pool of classifiers. -* Pool of classifiers can now be fitted together with the ensemble techniques. See `simple example `_. -* Added support for Faiss (Facebook AI Similarity Search) for fast region of competence estimation on GPU. -* Added DES Multi-class Imbalance method :class:`deslib.des.des_mi.DESMI`. -* Added stacked classifier model, :class:`deslib.static.stacked.StackedClassifier` to the static ensemble module. -* Added a new Instance Hardness measure :func:`utils.instance_hardness.kdn_score`. -* Added Instance Hardness support when using DES-Clustering. -* Added label encoder for the :mod:`static` module. -* Added a script :mod:`utils.datasets` with routines to generate synthetic datasets (e.g., the P2 and XOR datasets). -* Changed name of base classes (Adding Base to their following scikit-learn standards). -* Removal of **DFP_mask**, **neighbors** and **distances** as class variables. -* Changed signature of methods **estimate_competence**, **predict_with_ds**, **predict_proba_with_ds**. They now require the neighbors and distances to be passed as input arguments. -* Added random_state parameter to all methods in order to have reproducible results. -* Added Python 3.7 support. -* New and updated `examples `_. -* Added performance tests comparing the speed of Faiss vs sklearn KNN. - -Bug Fixes -~~~~~~~~~~~~ - -* Fixed bug with META-DES when checking if the meta-classifier was already fitted. -* Fixed bug with random state on DCS techniques. -* Fixed high memory consumption on DES probabilistic methods. -* Fixed bug on Heterogeneous ensembles example and notebooks examples. -* Fixed bug on :class:`deslib.des.probabilistic.MinimumDifference` when only samples from a single class are provided. -* Fixed problem with DS methods when the number of training examples was lower than the k value. -* Fixed division by zero problems with :class:`APosteriori` :class:`APriori` :class:`MLA` when the distance is equal to zero. -* Fixed bug on :func:`deslib.utils.prob_functions.exponential_func` when the support obtained for the correct class was equal to one. - - -.. _Rafael M O Cruz: https://github.com/Menelau -.. _Luiz G Hafemann: https://github.com/luizgh - diff --git a/docs/user_guide.rst b/docs/user_guide.rst deleted file mode 100644 index 298f0d3f..00000000 --- a/docs/user_guide.rst +++ /dev/null @@ -1,18 +0,0 @@ -.. _user_guide: - -###################### -User guide -###################### - -This user guide explains how to install DESlib, how to contribute to the library and -presents a step-by-step tutorial to fit and predict new instances using several dynamic selection techniques. - -.. toctree:: - :maxdepth: 2 - - user_guide/installation - user_guide/development - user_guide/tutorial - user_guide/known_issues - user_guide/packaging - diff --git a/docs/user_guide/development.rst b/docs/user_guide/development.rst deleted file mode 100644 index e2e441d7..00000000 --- a/docs/user_guide/development.rst +++ /dev/null @@ -1,119 +0,0 @@ -.. _development: - -Development -=========== - -DESlib was started by Rafael M. O. Cruz as a way to facilitate research in this topic by providing other researchers -a toolbox with everything that is required to easily develop and compare different dynamic ensemble techniques. - -The library is a work in progress. As an open-source project, any type of contribution is welcomed and encouraged! - - -Contributing to DESlib ----------------------- - -You can contribute to the project in several ways: - -- Reporting bugs -- Requesting features -- Improving the documentation -- Adding examples to use the library -- Implementing new features and fixing bugs - -Reporting Bugs and requesting features ---------------------------------------- - -We use Github issues to track all bugs and feature requests; feel free to -open an issue if you have found a bug or wish to see a new feature implemented. -Before opening a new issue, please check if the issue is not being currently addressed: -[Issues](https://github.com/scikit-learn-contrib/DESlib/issues) - -For reporting bugs: - -- Include information of your working environment. This information - can be found by running the following code snippet: - -.. code-block:: python - - import platform; print(platform.platform()) - import sys; print("Python", sys.version) - import numpy; print("NumPy", numpy.__version__) - import scipy; print("SciPy", scipy.__version__) - import sklearn; print("Scikit-Learn", sklearn.__version__) - -- Include a [reproducible](https://stackoverflow.com/help/mcve) code snippet - or link to a [gist](https://gist.github.com). If an exception is raised, - please provide the traceback. - -Documentation --------------- - -We are glad to accept any sort of documentation: function docstrings, -reStructuredText documents (like this one), tutorials, etc. -reStructuredText documents live in the source code repository under the -doc/ directory. - -You can edit the documentation using any text editor and then generate -the HTML output by typing ``make html`` from the doc/ directory. -Alternatively, ``make`` can be used to quickly generate the -documentation without the example gallery. The resulting HTML files will -be placed in _build/html/ and are viewable in a web browser. See the -README file in the doc/ directory for more information. - -For building the documentation, you will need to install sphinx and sphinx_rtd_theme. This -can be easily done by installing the requirements for development using the following command: - -.. code-block:: bash - - pip install -r requirements-dev.txt - -Contributing with code ------------------------ - -The preferred way to contribute is to fork the main repository to your account: - -1. Fork the [project repository](https://github.com/scikit-learn-contrib/DESlib): - click on the 'Fork' button near the top of the page. This creates - a copy of the code under your account on the GitHub server. - -2. Clone this copy to your local disk: - -.. code-block:: bash - - git clone git@github.com:YourLogin/DESlib.git - cd DESlib - -3. Install all requirements for development: - -.. code-block:: bash - - pip install -r requirements-dev.txt - pip install --editable . - -4. Create a branch to hold your changes: - -.. code-block:: bash - - git checkout -b branch_name - -Where ``branch_name`` is the new feature or bug to be fixed. Do not work directly on the ``master`` branch. - -5. Work on this copy on your computer using Git to do the version - control. To record your changes in Git, then push them to GitHub with: - -.. code-block:: bash - - git push -u origin branch_name - -It is important to assert your code is well covered by test routines (coverage of at least 90%), well documented and -follows PEP8 guidelines. - -6. Create a 'Pull request' to send your changes for review. - - If your pull request addresses an issue, please use the title to describe - the issue and mention the issue number in the pull request description to - ensure a link is created to the original issue. - -.. _GitHub: https://github.com/scikit-learn-contrib/DESlib. - -.. _scikit-learn: http://scikit-learn.org/stable/ \ No newline at end of file diff --git a/docs/user_guide/installation.rst b/docs/user_guide/installation.rst deleted file mode 100644 index ce00ba0e..00000000 --- a/docs/user_guide/installation.rst +++ /dev/null @@ -1,34 +0,0 @@ -.. _installation: - -============ -Installation -============ - -The library can be installed using pip: - -Stable version: - -.. code-block:: bash - - pip install deslib - -Latest version (under development): - -.. code-block:: bash - - pip install git+https://github.com/scikit-learn-contrib/DESlib - -DESlib is tested to work with Python 3.5, 3.6 and 3.7. The dependency requirements are: - -* scipy(>=0.13.3) -* numpy(>=1.10.4) -* scikit-learn(>=0.19.0) - -These dependencies are automatically installed using the pip commands above. - -Optional dependencies -===================== -To use Faiss (Fair AI Similarity Search), a fast implementation of KNN that can use GPUs, follow the instructions below: -https://github.com/facebookresearch/faiss/blob/master/INSTALL.md - -Note that Faiss is only available on Linux and MacOS. \ No newline at end of file diff --git a/docs/user_guide/known_issues.rst b/docs/user_guide/known_issues.rst deleted file mode 100644 index 0b8c399b..00000000 --- a/docs/user_guide/known_issues.rst +++ /dev/null @@ -1,24 +0,0 @@ -.. _known_issues: - -Known Issues -============ - -The estimators in this library are not compatible with scikit-learn's GridSearch, and other CV methods. That is, the following is not supported: - -.. code-block:: python - - from deslib.des.knora_e import KNORAE - from sklearn.model_selection import GridSearchCV - - # (...) initialize a pool of classifiers - kne = KNORAE(pool_classifiers) - - # Do a grid search on KNORAE's "k" parameter - params = {'k': [1, 3, 5, 7]} - - grid = GridSearchCV(kne, params) - grid.fit(X_dsel, y_dsel) # Raises an error - -This is due to a limitation of a scikit-learn method (sklearn.base.clone), under discussion in this issue_ - -.. _issue: https://github.com/scikit-learn/scikit-learn/issues/8370 \ No newline at end of file diff --git a/docs/user_guide/packaging.rst b/docs/user_guide/packaging.rst deleted file mode 100644 index bfc2ecb1..00000000 --- a/docs/user_guide/packaging.rst +++ /dev/null @@ -1,77 +0,0 @@ -.. _packaging: - -Releasing a new version -======================= - -Publishing new version involves: - -1) Updating the version numbers and creating a new tag in git (which also updates the "stable" version of the documentation) -2) Creating the distribution (.tar.gz and wheel files), and uploading them to pypi - -Some important things to have in mind: - * Read the "Packaging and Distributing Projects" guide: https://packaging.python.org/tutorials/distributing-packages/ - * The version numbers (in setup.py and __init__.py) are used as metadata for pypi and for the readthedocs documentation - pay attention to them or some things can break. In general, you should be working on a version such as "0.2.dev". You then rename it to "0.2" and create a tag "v0.2". After you finish everything, you update the version to "0.3.dev" to indicate that new developments are being made for the next version. - - -Step-by-step process --------------------- - - -* Create an account in PyPi production: https://pypi.org/ and test: https://test.pypi.org/ -* Make sure you have twine installed: - - .. code-block:: bash - - pip install twine - -* Update version on setup.py (e.g. "0.1") -* Update version on deslib/__init__.py -* Create tag: :code:`git tag ` (example: "git tag 'v0.1'") -* Push the tag :code:`git push origin ` -* Create the source and wheels distributions - - .. code-block:: bash - - python setup.py sdist # source distribution - python setup.py bdist_wheel # wheel distribution for current python version - -* Upload to test pypi and check - - - uploading the package: - - .. code-block:: bash - - twine upload --repository-url https://test.pypi.org/legacy/ dist/* - - - Note: if you do this multiple times (e.g. to fix an issue), you will need to rename the files under the "dist" folder: a filename can only be submitted once to pypi. You may also need to manually delete the "source" version of the distribution, since there can only be one source file per version of the software - - - Test an installation from the testing pypi environment. - - .. code-block:: bash - - conda create -y -n testdes python=3 - source activate testdes - pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple deslib - conda remove -y --name testdes --all #remove temporary environment - -* Upload to production pypi - - .. code-block:: bash - - twine upload dist/* - -* Mark the new stable version to be built on readthedocs: - - - Go to https://readthedocs.org/projects/deslib/versions/, find the new tag and click "Edit". Mark the "active" checkbox and save. - -* Update version on setup.py and __init.py__ to mention the new version in development (e.g. "0.2.dev") - - -Note #1: Read the docs is automatically updated: - -* When a new commit is done in master (this updates the "master" version) -* When a new tag is pushed to github (this updates the "stable" version) -> This seems to not aways work - it is better to check - -Note #2: The documentation automatically links to source files for the methods/classes. This only works if the tag is pushed to github, and matches the __version__ variable in __init.py__. Example: -__version__ = "0.1" and the tag being: -git tag "v0.1" diff --git a/docs/user_guide/tutorial.rst b/docs/user_guide/tutorial.rst deleted file mode 100644 index 33baac74..00000000 --- a/docs/user_guide/tutorial.rst +++ /dev/null @@ -1,279 +0,0 @@ -.. _tutorial: - -========= -Tutorial -========= - -This tutorial will walk you through generating a pool of classifiers and applying several dynamic selection techniques -for the classification of unknown samples. The tutorial assumes that you are already familiar with the `Python language`_ -and the `scikit-learn`_ library. Users not familiar with either Python and scikit-learn can start by checking out their tutorials. - -Running Dynamic selection with Bagging -====================================== - -In this first tutorial, we do a step-by-step run of the example_bagging.py, that is included in the examples part of the DESlib. -This example uses the Wisconsin breast cancer dataset available on sklearn.datasets package. - -The first step is to run the example to check if everything is working as intended: - -.. code-block:: bash - - cd examples - python example_bagging.py - -This script run six different dynamic selection models: Three DCS (OLA, A-Priori, MCB) and four DES (KNORA-Union, -KNORA-Eliminate, DES-P and META-DES) - -The example outputs the classification accuracy of each dataset: - -.. code-block:: text - - Evaluating DS techniques: - Classification accuracy KNORA-Union: 0.973404255319 - Classification accuracy KNORA-Eliminate: 0.968085106383 - Classification accuracy DESP: 0.973404255319 - Classification accuracy OLA: 0.968085106383 - Classification accuracy A priori: 0.973404255319 - Classification accuracy MCB: 0.968085106383 - Classification accuracy META-DES: 0.973404255319 - -Code analysis: -============== - -The code starts by importing the corresponding DCS and DES algorithms that are tested as well as the other required -libraries: - -.. code-block:: python - - from sklearn.datasets import load_breast_cancer - from sklearn.model_selection import train_test_split - from sklearn.preprocessing import StandardScaler - from sklearn.linear_model import Perceptron - from sklearn.calibration import CalibratedClassifierCV - from sklearn.ensemble import BaggingClassifier - - #importing DCS techniques from DESlib - from deslib.dcs.ola import OLA - from deslib.dcs.a_priori import APriori - from deslib.dcs.mcb import MCB - - #import DES techniques from DESlib - from deslib.des.des_p import DESP - from deslib.des.knora_u import KNORAU - from deslib.des.knora_e import KNORAE - from deslib.des.meta_des import METADES - -As DESlib is built on top of scikit-learn_, code will usually required the import of routines from it. - -Preparing the dataset: ------------------------ - -The next step is loading the data and dividing it into three partitions: Training, validation and test. In the dynamic -selection literature [1]_ the validation set is usually called the dynamic selection dataset (DSEL), since -this partition is used by the dynamic selection techniques in order to select the base classifiers, so in this -library we use the same terminology. The training set (X_train, y_train) is used to fit the pool of classifiers, -the validation (X_DSEL, y_DSEL) set is used to fit the dynamic selection models. The performance of the system -is then evaluated on the test set (X_test, y_test). - -.. code-block:: python - - data = load_breast_cancer() - X = data.data - y = data.target - # split the data into training and test data - X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33) - - # Scale the variables to have 0 mean and unit variance - scalar = StandardScaler() - X_train = scalar.fit_transform(X_train) - X_test = scalar.transform(X_test) - - # Split the data into training and DSEL for DS techniques - X_train, X_dsel, y_train, y_dsel = train_test_split(X_train, y_train, test_size=0.5) - - -Another important aspect is to normalize the data so that it has -zero mean and unit variance, which is a common requirement for many machine learning algorithms. -This step can be easily done using the StandardScaler class from scikit-learn_. Note that the StandardScaler transform -should be fitted using the training and DSEL data only. Then, it can be applied for the test data. - -An important point here is that in case of small datasets or when the base classifier models in the pool -are weak estimators such as Decision Stumps or Perceptrons, an overlap between the training data and DSEL -may be beneficial for achieving better performance. - -Training a pool of classifiers: -------------------------------- - -The next step is to generate a pool of classifiers. This list can be either -homogeneous (i.e., all base classifiers are of the same type) or heterogeneous (base classifiers of different types). -The library supports any type of base classifiers that is compatible with the scikit-learn library. - -In this example, we generate a pool composed of 10 Perceptron classifiers -using the Bagging technique. It is important to mention that some DS techniques require that the base classifiers are capable of -estimating probabilities (i.e., implements the predict_proba function). - -For the Perceptron model, this can be achieved -by calibrating the outputs of the base classifiers using the CalibratedClassifierCV class from scikit-learn. - -.. code-block:: python - - model = CalibratedClassifierCV(Perceptron(max_iter=10)) - - # Train a pool of 10 classifiers - pool_classifiers = BaggingClassifier(model, n_estimators=10) - pool_classifiers.fit(X_train, y_train) - - -Building the DS models ----------------------- - -Three DCS and four DES techniques are considered in this example: - -- Overal Local Accuracy (OLA) -- Multiple-Classifier Behavior (MCB) -- A Priori selection -- K-Nearest Oracles-Union (KNU) -- K-Nearest Oracles-Eliminate (KNE) -- META-DES - -**NEW:** Since version 0.3, DESlib does not require a trained pool of classifiers for instantiating its estimators. All estimator -can now be instantiated without specifying a pool of classifiers: - -.. code-block:: python - - # DCS techniques - ola = OLA() - mcb = MCB() - apriori = APriori() - - # DES techniques - knorau = KNORAU() - kne = KNORAE() - desp = DESP() - meta = METADES() - -When the pool of classifiers is not specified, a standard :class:`BaggingClassifier` from sklearn is used, which generates -a pool composed of 10 decision trees. The parameter **DSEL_perc** controls the percentage of the input data that is used for fitting -DSEL. The remaining data will be used to fit the pool of classifiers. Note that this parameter is only taken into account if -the pool is either equals to None (when it was not informed) or still unfitted (when the base classifiers were not fitted) - -However, since we already trained a pool of classifiers in the previous step we will continue this tutorial by instantiating the dynamic selection methods with an already fitted pool. -For more information on using DESlib estimators without specifying a trained pool of classifiers -see the `examples page `_. - -.. code-block:: python - - # DCS techniques - ola = OLA(pool_classifiers) - mcb = MCB(pool_classifiers) - apriori = APriori(pool_classifiers) - - # DES techniques - knorau = KNORAU(pool_classifiers) - kne = KNORAE(pool_classifiers) - desp = DESP(pool_classifiers) - meta = METADES(pool_classifiers) - - -Fitting the DS techniques: ---------------------------- - -The next step is to fit the DS model. We call the function fit to prepare the DS techniques for the -classification of new data by pre-processing the information required to apply the DS techniques, such as, -fitting the algorithm used to estimate the region of competence (k-NN, k-Means) and calculating the source of competence -of the base classifiers for each sample in the dynamic selection dataset. - -.. code-block:: python - - knorau.fit(X_dsel, y_dsel) - kne.fit(X_dsel, y_dsel) - desp.fit(X_dsel, y_dsel) - ola.fit(X_dsel, y_dsel) - mcb.fit(X_dsel, y_dsel) - apriori.fit(X_dsel, y_dsel) - meta.fit(X_dsel, y_dsel) - -Note that if the pool of classifiers is still unfitted, this step will also fit the base classifiers in the pool. - -Estimating classification accuracy: ------------------------------------- -Estimating the classification accuracy of each method is very easy. Each DS technique implements the function score -from scikit-learn in order to estimate the classification accuracy. - -.. code-block:: python - - print('Classification accuracy OLA: ', ola.score(X_test, y_test)) - print('Classification accuracy A priori: ', apriori.score(X_test, y_test)) - print('Classification accuracy KNORA-Union: ', knorau.score(X_test, y_test)) - print('Classification accuracy KNORA-Eliminate: ', kne.score(X_test, y_test)) - print('Classification accuracy DESP: ', desp.score(X_test, y_test)) - print('Classification accuracy META-DES: ', apriori.score(X_test, y_test)) - -However, you may need to calculate the predictions of the model or the estimation of probabilities instead of only computing the accuracy. -Class labels and posterior probabilities can be easily calculated using the **predict** and **predict_proba** methods: - -.. code-block:: python - - metades.predict(X_test) - metades.predict_proba(X_test) - -Changing parameters -------------------- - -Changing the hyper-parameters is very easy. We just need to pass its value when instantiating a new method. For example, -we can change the size of the neighborhood used to estimate the competence level by setting the k value. - -.. code-block:: python - - # DES techniques - knorau = KNORAU(pool_classifiers, k=5) - kne = KNORAE(pool_classifiers, k=5) - -Also, we can change the mode DES algorithm works (dynamic selection, dynamic weighting or hybrid) by setting its mode: -.. code-block:: python - - meta = METADES(pool_classifiers, Hc=0.8, k=5, mode='hybrid') - -In this code block, we change the size of the neighborhood (k=5), the value of the sample selection mechanism (Hc=0.8) and -also, state that the META-DES algorithm should work in a hybrid dynamic selection with and weighting mode. -The library accepts the change of several hyper-parameters. A list containing each one for all technique available -as well as its impact in the algorithm is presented in the `API Reference `_. - -Applying the Dynamic Frienemy Pruning (DFP) -------------------------------------------- - -The library also implements the Dynamic Frienemy Pruning (DFP) proposed in [2]_. So any dynamic selection technique can be -applied using the FIRE (Frienemy Indecision Region Dynamic Ensemble Selection) framework. That is easily done by setting the -DFP to true when initializing a DS technique. In this example, we show how to start the FIRE-KNORA-U, FIRE-KNORA-E and FIRE-MCB techniques. - -.. code-block:: python - - fire_knorau = KNORAU(pool_classifiers, DFP=True) - fire_kne = KNORAE(pool_classifiers, DFP=True) - fire_mcb = MCB(pool_classifiers, DFP=True) - -We can also set the size of the neighborhood that is used to decide whether the query sample is located in a safe region or -in an indecision region (safe_k): - -.. code-block:: python - - fire_knorau = KNORAU(pool_classifiers, DFP=True, safe_k=3) - fire_kne = KNORAE(pool_classifiers, DFP=True, safe_k=5) - fire_mcb = MCB(pool_classifiers, DFP=True, safe_k=7) - -So, the fire_knorau will use a neighborhood composed of 3 samples, fire_knorae of 5 and fire_mcb of 7 in order to compute whether a given sample -is located in a indecision or safe region. - -More tutorials on how to use different aspects of the library can be found in `examples page `_ - -.. _Python language: https://docs.python.org/3.5/tutorial/ -.. _scikit-learn: http://scikit-learn.org/stable/tutorial/index.html - - -References ------------ - -.. [1] : R. M. O. Cruz, R. Sabourin, and G. D. Cavalcanti, “Dynamic classifier selection: Recent advances and perspectives,” Information Fusion, vol. 41, pp. 195 – 216, 2018. - -.. [2] : Oliveira, D.V.R., Cavalcanti, G.D.C. and Sabourin, R., Online Pruning of Base Classifiers for Dynamic Ensemble Selection, Pattern Recognition, vol. 72, December 2017, pp 44-58. - diff --git a/examples/README.txt b/examples/README.txt deleted file mode 100644 index c4be9323..00000000 --- a/examples/README.txt +++ /dev/null @@ -1,6 +0,0 @@ -.. _general_examples: - -General examples ----------------- - -Examples showing how to use different aspect of the library \ No newline at end of file diff --git a/examples/example_calibrating_classifiers.py b/examples/example_calibrating_classifiers.py deleted file mode 100644 index 54daceca..00000000 --- a/examples/example_calibrating_classifiers.py +++ /dev/null @@ -1,131 +0,0 @@ -# coding=utf-8 - -# Author: Rafael Menelau Oliveira e Cruz -# -# License: BSD 3 clause -""" -==================================================================== -Calibrating base classifiers to estimate probabilities -==================================================================== - -In this example we show how to apply different DCS and DES techniques for a -classification dataset. - -A very important aspect in dynamic selection is the generation of a pool -of classifiers. A common practice in the dynamic selection literature is to -use the Bagging (Bootstrap Aggregating) method to generate a pool containing -base classifiers that are both diverse and informative. - -In this example we generate a pool of classifiers using the Bagging technique -implemented on the Scikit-learn library. Then, we compare the results obtained -by combining this pool of classifiers using the standard Bagging combination -approach versus the application of dynamic selection technique to select the -set of most competent classifiers -""" - - -import numpy as np -from sklearn.calibration import CalibratedClassifierCV -from sklearn.datasets import load_breast_cancer -from sklearn.ensemble import BaggingClassifier -from sklearn.linear_model import Perceptron -from sklearn.model_selection import train_test_split -from sklearn.preprocessing import StandardScaler - -from deslib.dcs.a_priori import APriori -from deslib.dcs.mcb import MCB -from deslib.dcs.ola import OLA -from deslib.des.des_p import DESP -from deslib.des.knora_e import KNORAE -from deslib.des.knora_u import KNORAU -from deslib.des.meta_des import METADES - -############################################################################### -# Preparing the dataset -# --------------------- -# In this part we load the breast cancer dataset from scikit-learn and -# preprocess it in order to pass to the DS models. An important point here is -# to normalize the data so that it has zero mean and unit variance, which is -# a common requirement for many machine learning algorithms. -# This step can be easily done using the StandardScaler class. - -rng = np.random.RandomState(123) -data = load_breast_cancer() -X = data.data -y = data.target -# split the data into training and test data -X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, - random_state=rng) - -# Scale the variables to have 0 mean and unit variance -scaler = StandardScaler() -X_train = scaler.fit_transform(X_train) -X_test = scaler.transform(X_test) - -# Split the data into training and DSEL for DS techniques -X_train, X_dsel, y_train, y_dsel = train_test_split(X_train, y_train, - test_size=0.5, - random_state=rng) - -# Train a pool of 100 base classifiers -pool_classifiers = BaggingClassifier(Perceptron(max_iter=10), - n_estimators=100, random_state=rng) -pool_classifiers.fit(X_train, y_train) - -# Initialize the DS techniques -knorau = KNORAU(pool_classifiers) -kne = KNORAE(pool_classifiers) -desp = DESP(pool_classifiers) -ola = OLA(pool_classifiers) -mcb = MCB(pool_classifiers, random_state=rng) - -############################################################################### -# Calibrating base classifiers -# ----------------------------- -# Some dynamic selection techniques requires that the base classifiers estimate -# probabilities in order to estimate its competence level. Since the Perceptron -# model is not a probabilistic classifier (does not implements the -# predict_proba method, it needs to be calibrated for -# probability estimation before being used by such DS techniques. This step can -# be conducted using the CalibrateClassifierCV class from scikit-learn. Note -# that in this example we pass a prefited pool of classifiers to the -# calibration method in order to use exactly the same pool used in the other -# DS methods. -calibrated_pool = [] -for clf in pool_classifiers: - calibrated = CalibratedClassifierCV(estimator=clf, cv='prefit') - calibrated.fit(X_dsel, y_dsel) - calibrated_pool.append(calibrated) - -apriori = APriori(calibrated_pool, random_state=rng) -meta = METADES(calibrated_pool) - - -knorau.fit(X_dsel, y_dsel) -kne.fit(X_dsel, y_dsel) -desp.fit(X_dsel, y_dsel) -ola.fit(X_dsel, y_dsel) -mcb.fit(X_dsel, y_dsel) -apriori.fit(X_dsel, y_dsel) -meta.fit(X_dsel, y_dsel) - -############################################################################### -# Evaluating the methods -# ----------------------- -# Let's now evaluate the methods on the test set. We also use the performance -# of Bagging (pool of classifiers without any selection) as a baseline -# comparison. We can see that the majority of DS methods achieve higher -# classification accuracy. - -print('Evaluating DS techniques:') -print('Classification accuracy KNORA-Union: ', - knorau.score(X_test, y_test)) -print('Classification accuracy KNORA-Eliminate: ', - kne.score(X_test, y_test)) -print('Classification accuracy DESP: ', desp.score(X_test, y_test)) -print('Classification accuracy OLA: ', ola.score(X_test, y_test)) -print('Classification accuracy A priori: ', apriori.score(X_test, y_test)) -print('Classification accuracy MCB: ', mcb.score(X_test, y_test)) -print('Classification accuracy META-DES: ', meta.score(X_test, y_test)) -print('Classification accuracy Bagging: ', - pool_classifiers.score(X_test, y_test)) diff --git a/examples/example_heterogeneous.py b/examples/example_heterogeneous.py deleted file mode 100644 index 2d4d4fca..00000000 --- a/examples/example_heterogeneous.py +++ /dev/null @@ -1,121 +0,0 @@ -""" -==================================================================== -Example using heterogeneous ensemble -==================================================================== -DESlib accepts different classifier models in the pool of classifiers. -Such pool of classifiers is called Heterogeneous. - -In this example, we consider a pool of classifiers composed of a -Gaussian Naive Bayes, Perceptron, k-NN, Decision tree and Gaussian SVM. We -also compare the result of DS methods with the voting classifier from sklearn. -""" -import numpy as np -from sklearn.calibration import CalibratedClassifierCV -# Importing dataset and preprocessing routines -from sklearn.datasets import fetch_openml -from sklearn.ensemble import VotingClassifier -# Base classifier models: -from sklearn.linear_model import Perceptron -from sklearn.model_selection import train_test_split -from sklearn.naive_bayes import GaussianNB -from sklearn.neighbors import KNeighborsClassifier -from sklearn.preprocessing import StandardScaler -from sklearn.svm import SVC -from sklearn.tree import DecisionTreeClassifier - -from deslib.dcs import MCB -# Example of DCS techniques -from deslib.dcs import OLA -from deslib.des import DESP -# Example of DES techniques -from deslib.des import KNORAE -from deslib.des import KNORAU -from deslib.des import METADES -from deslib.static import StackedClassifier - -rng = np.random.RandomState(42) -data = fetch_openml(name='phoneme', cache=False, as_frame=False) -X = data.data -y = data.target - -# split the data into training and test data -X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, - random_state=rng) - -scaler = StandardScaler() -X_train = scaler.fit_transform(X_train) -X_test = scaler.transform(X_test) - -# Split the data into training and DSEL for DS techniques -X_train, X_dsel, y_train, y_dsel = train_test_split(X_train, y_train, - test_size=0.5, - random_state=rng) - -model_perceptron = CalibratedClassifierCV(Perceptron(max_iter=100, - random_state=rng), - cv=3) - -model_perceptron.fit(X_train, y_train) -model_svc = SVC(probability=True, gamma='auto', - random_state=rng).fit(X_train, y_train) -model_bayes = GaussianNB().fit(X_train, y_train) -model_tree = DecisionTreeClassifier(random_state=rng, - max_depth=10).fit(X_train, y_train) -model_knn = KNeighborsClassifier(n_neighbors=7).fit(X_train, y_train) - -pool_classifiers = [model_perceptron, - model_svc, - model_bayes, - model_tree, - model_knn] - -voting_classifiers = [("perceptron", model_perceptron), - ("svc", model_svc), - ("bayes", model_bayes), - ("tree", model_tree), - ("knn", model_knn)] - -model_voting = VotingClassifier(estimators=voting_classifiers).fit( - X_train, y_train) - -# Initializing the techniques -knorau = KNORAU(pool_classifiers) -kne = KNORAE(pool_classifiers) -desp = DESP(pool_classifiers) -metades = METADES(pool_classifiers) -# DCS techniques -ola = OLA(pool_classifiers) -mcb = MCB(pool_classifiers) - -############################################################################## -# Adding stacked classifier as baseline comparison. Stacked classifier can -# be found in the static module. In this experiment we consider two types -# of stacking: one using logistic regression as meta-classifier -# (default configuration) and the other using a Decision Tree. -stacked_lr = StackedClassifier(pool_classifiers, random_state=rng) -stacked_dt = StackedClassifier(pool_classifiers, - random_state=rng, - meta_classifier=DecisionTreeClassifier()) -# Fitting the DS techniques -knorau.fit(X_dsel, y_dsel) -kne.fit(X_dsel, y_dsel) -desp.fit(X_dsel, y_dsel) -metades.fit(X_dsel, y_dsel) -ola.fit(X_dsel, y_dsel) -mcb.fit(X_dsel, y_dsel) - -# Fitting the tacking models -stacked_lr.fit(X_dsel, y_dsel) -stacked_dt.fit(X_dsel, y_dsel) - -# Calculate classification accuracy of each technique -print('Evaluating DS techniques:') -print('Classification accuracy of Majority voting the pool: ', - model_voting.score(X_test, y_test)) -print('Classification accuracy of KNORA-U: ', knorau.score(X_test, y_test)) -print('Classification accuracy of KNORA-E: ', kne.score(X_test, y_test)) -print('Classification accuracy of DESP: ', desp.score(X_test, y_test)) -print('Classification accuracy of META-DES: ', metades.score(X_test, y_test)) -print('Classification accuracy of OLA: ', ola.score(X_test, y_test)) -print('Classification accuracy Stacking LR', stacked_lr.score(X_test, y_test)) -print('Classification accuracy Stacking DT', stacked_dt.score(X_test, y_test)) diff --git a/examples/plot_comparing_dynamic_static.py b/examples/plot_comparing_dynamic_static.py deleted file mode 100644 index 935512e5..00000000 --- a/examples/plot_comparing_dynamic_static.py +++ /dev/null @@ -1,169 +0,0 @@ -# coding=utf-8 - -# Author: Rafael Menelau Oliveira e Cruz -# -# License: BSD 3 clause -""" -==================================================================== -Comparing dynamic selection with baseline static methods -==================================================================== - -In this example we compare the performance of DS techinques with the -static ensemble methods. DESlib offer the implementation of static ensemble -methods in the `deslib.static` module. The following techniques are -considered: - -Static methods used as baseline comparison are in the `deslib.static` module. -They are: - -**Majority Voting**: The outputs of all base classifiers in the pool are combined -using the majority voting rule - -**Static Selection**: A fraction of the best performing classifiers (based on the -validation data, is selected to compose the ensemble). - -**Single Best**: The base classifier with the highest classification accuracy in -the validation set is selected for classification - -**Stacked classifier**: The outputs of all base classifiers are passed down to -a meta-estimator which combines the . The meta-estimator is trained based -on the outputs of the base classifiers on the training data. - -These techniques are used in the dynamic selection literature as a baseline -comparison (for more information see references [1] and [2]) - -At the end we also present the result of the **Oracle**, which is an abastract -model which always selects the base classifier that predicted the correct label -if such classifier exists. From the dynamic selection point of view, the Oracle -is seen as the upper limit performance that can be achieved with the given -pool of classifiers. - -References ----------- -[1] Britto, Alceu S., Robert Sabourin, and Luiz ES Oliveira. "Dynamic selection -of classifiers—a comprehensive review." Pattern Recognition 47.11 -(2014): 3665-3680. - -[2] R. M. O. Cruz, R. Sabourin, and G. D. Cavalcanti, “Dynamic classifier -selection: Recent advances and perspectives,” Information Fusion, -vol. 41, pp. 195 – 216, 2018. - -[3] Kuncheva, Ludmila I. "A theoretical study on six classifier fusion - strategies." IEEE Transactions on Pattern Analysis & Machine Intelligence, - (2002): 281-286. -""" - -import matplotlib.pyplot as plt -from matplotlib.ticker import FuncFormatter -from matplotlib.cm import get_cmap -import numpy as np - -# Example of a dcs techniques -from deslib.dcs import OLA -from deslib.dcs import MCB -from deslib.des import DESP -from deslib.des import KNORAU -from deslib.des.knora_e import KNORAE -from deslib.des import KNOP -from deslib.des import METADES - -from sklearn.datasets import make_classification -from sklearn.ensemble import BaggingClassifier -from sklearn.model_selection import train_test_split -from sklearn.tree import DecisionTreeClassifier -# Example of a des techniques - -# Example of stacked model -from deslib.static import (StackedClassifier, - SingleBest, - StaticSelection, - Oracle) - - -rng = np.random.RandomState(123) - -# Generate a classification dataset -X, y = make_classification(n_samples=2000, - n_classes=3, - n_informative=6, - random_state=rng) - -# split the data into training and test data -X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, - random_state=rng) - -X_train, X_dsel, y_train, y_dsel = train_test_split(X_train, y_train, - test_size=0.50, - random_state=rng) - -pool_classifiers = BaggingClassifier(base_estimator=DecisionTreeClassifier(), - n_estimators=100, - random_state=rng) -pool_classifiers.fit(X_train, y_train) - -# Setting up static methods. -stacked = StackedClassifier(pool_classifiers) -static_selection = StaticSelection(pool_classifiers) -single_best = SingleBest(pool_classifiers) - -# Initialize a DS technique. Here we specify the size of -# the region of competence (5 neighbors) -knorau = KNORAU(pool_classifiers, random_state=rng) -kne = KNORAE(pool_classifiers, random_state=rng) -desp = DESP(pool_classifiers, random_state=rng) -ola = OLA(pool_classifiers, random_state=rng) -mcb = MCB(pool_classifiers, random_state=rng) -knop = KNOP(pool_classifiers, random_state=rng) -meta = METADES(pool_classifiers, random_state=rng) - -names = ['Single Best', 'Static Selection', 'Stacked', - 'KNORA-U', 'KNORA-E', 'DES-P', 'OLA', 'MCB', 'KNOP', 'META-DES'] - -methods = [single_best, static_selection, stacked, - knorau, kne, desp, ola, mcb, knop, meta] - -# Fit the DS techniques -scores = [] -for method, name in zip(methods, names): - method.fit(X_dsel, y_dsel) - scores.append(method.score(X_test, y_test)) - print("Classification accuracy {} = {}" - .format(name, method.score(X_test, y_test))) - - -############################################################################### -# Plotting the results -# ----------------------- -# Let's now evaluate the methods on the test set. -# -cmap = get_cmap('Dark2') -colors = [cmap(i) for i in np.linspace(0, 1, 10)] -fig, ax = plt.subplots(figsize=(8, 6.5)) -pct_formatter = FuncFormatter(lambda x, pos: '{:.1f}'.format(x * 100)) -ax.bar(np.arange(len(methods)), - scores, - color=colors, - tick_label=names, - edgecolor='k') - -ax.set_ylim(0.70, 0.86) -ax.set_ylabel('Accuracy on the test set (%)', fontsize=13) -ax.yaxis.set_major_formatter(pct_formatter) -for tick in ax.get_xticklabels(): - tick.set_rotation(60) -plt.subplots_adjust(bottom=0.18) - -plt.show() - -############################################################################### -# The Oracle results -# ----------------------- -# OracleAbstract method that always selects the base classifier that predicts -# the correct label if such classifier exists. This method is often used to -# measure the upper-limit performance that can be achieved by a dynamic -# classifier selection technique. It is used as a benchmark by several -# dynamic selection algorithms. We can see the Oracle performance is close -# to 100%, which is an almost 15% gap to the best performing method. - -oracle = Oracle(pool_classifiers).fit(X_train, y_train) -print('Oracle result: {}' .format(oracle.score(X_test, y_test))) diff --git a/examples/plot_example_DFP.py b/examples/plot_example_DFP.py deleted file mode 100644 index 551cfe3b..00000000 --- a/examples/plot_example_DFP.py +++ /dev/null @@ -1,120 +0,0 @@ -# coding=utf-8 - -# Author: Rafael Menelau Oliveira e Cruz -# -# License: BSD 3 clause -""" -==================================================================== -Using the Dynamic Frienemy Pruning (DFP) -==================================================================== - -In this example we show how to apply the dynamic frienemy pruning (DFP) to -different dynamic selection techniques. - -The DFP method is an online pruning model which analyzes the region -of competence to know if it is composed of samples from different classes -(indecision region). Then, it remove the base classifiers that do not correctly -classifies at least a pair of samples coming from different classes, i.e., the -base classifiers that cannot separate the classes in the local region. -More information on this method can be found in refs [1] and [2]. - -DES techniques using the DFP algorithm are called FIRE-DES (Frienemy Indecision -REgion Dynamic Ensemble Selection). -The FIRE-DES is shown to significantly improve the performance of several -dynamic selection algorithms when dealing with imbalanced classification -problems as it avoids the classifiers that are biased towards the majority -class in predicting the label for the query. - -References ----------- -[1] Oliveira, D.V.R., Cavalcanti, G.D.C. and Sabourin, R., "Online Pruning -of Base Classifiers for Dynamic Ensemble Selection", Pattern Recognition, -vol. 72, 2017, pp 44-58. - -[2] Cruz, R.M.O., Oliveira, D.V.R., Cavalcanti, G.D.C. and Sabourin, R., -"FIRE-DES++: Enhanced online pruning of base classifiers for dynamic ensemble -selection"., Pattern Recognition, vol. 85, 2019, pp 149-160. -""" - -import numpy as np -from sklearn.datasets import make_classification -from sklearn.ensemble import RandomForestClassifier -from sklearn.model_selection import train_test_split -from sklearn.metrics import roc_auc_score -import matplotlib.pyplot as plt -from deslib.dcs import APosteriori -from deslib.dcs import APriori -from deslib.dcs import LCA -from deslib.dcs import OLA -from deslib.des import DESP -from deslib.des import METADES - -rng = np.random.RandomState(654321) - -# Generate an imbalanced classification dataset -X, y = make_classification(n_classes=2, n_samples=2000, weights=[0.05, 0.95], - random_state=rng) -# split the data into training and test data -X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, - random_state=rng) - -# Split the data into training and DSEL for DS techniques -X_train, X_dsel, y_train, y_dsel = train_test_split(X_train, y_train, - test_size=0.5, - random_state=rng) -# Considering a pool composed of 10 base classifiers -pool_classifiers = RandomForestClassifier(n_estimators=10, random_state=rng, - max_depth=10) -pool_classifiers.fit(X_train, y_train) - -ds_names = ['A Priori', 'A Posteriori', 'OLA', 'LCA', 'DES-P', 'META-DES'] - -# DS techniques without DFP -apriori = APriori(pool_classifiers, random_state=rng) -aposteriori = APosteriori(pool_classifiers, random_state=rng) -ola = OLA(pool_classifiers) -lca = LCA(pool_classifiers) -desp = DESP(pool_classifiers) -meta = METADES(pool_classifiers) - -# FIRE-DS techniques (with DFP) -fire_apriori = APriori(pool_classifiers, DFP=True, random_state=rng) -fire_aposteriori = APosteriori(pool_classifiers, DFP=True, random_state=rng) -fire_ola = OLA(pool_classifiers, DFP=True) -fire_lca = LCA(pool_classifiers, DFP=True) -fire_desp = DESP(pool_classifiers, DFP=True) -fire_meta = METADES(pool_classifiers, DFP=True) - -list_ds = [apriori, aposteriori, ola, lca, desp, meta] -list_fire_ds = [fire_apriori, fire_aposteriori, fire_ola, - fire_lca, fire_desp, fire_meta] - -scores_ds = [] -for ds in list_ds: - ds.fit(X_dsel, y_dsel) - scores_ds.append(roc_auc_score(y_test, ds.predict(X_test))) - -scores_fire_ds = [] -for fire_ds in list_fire_ds: - fire_ds.fit(X_dsel, y_dsel) - scores_fire_ds.append(roc_auc_score(y_test, fire_ds.predict(X_test))) - -############################################################################### -# Comparing DS techniques with FIRE-DES techniques -# ------------------------------------------------ -# Let's now evaluate the DES methods on the test set. Since we are dealing with -# imbalanced data, we use the area under the roc curve (AUC) as performance -# metric instead of classification accuracy. The AUC can be easily calculated -# using the `sklearn.metrics.roc_auc_score` function from scikit-learn. - -width = 0.35 -ind = np.arange(len(ds_names)) -plt.bar(ind, scores_ds, width, label='DES', edgecolor='k') -plt.bar(ind + width, scores_fire_ds, width, label='FIRE-DES', edgecolor='k') - -plt.ylabel('Area under the roc curve (AUC)') -plt.title('AUC Performance: DS vs FIRE-DES') -plt.ylim((0.60, 0.81)) -plt.xticks(ind + width / 2, ds_names) -plt.legend(loc='best') -plt.show() diff --git a/examples/plot_example_P2.py b/examples/plot_example_P2.py deleted file mode 100644 index 7f9b4665..00000000 --- a/examples/plot_example_P2.py +++ /dev/null @@ -1,215 +0,0 @@ -# coding: utf-8 -""" -==================================================================== -Visualizing decision boundaries on the P2 problem -==================================================================== - -This example shows the power of dynamic selection (DS) techniques which can -solve complex non-linear classification near classifiers. It also compares -the performance of DS techniques with some baseline classification methods -such as Random Forests, AdaBoost and SVMs. - -The P2 is a two-class problem, presented by Valentini, in which each class -is defined in multiple decision regions delimited by polynomial and -trigonometric functions: - - .. math:: \\begin{eqnarray} - \\label{eq:problem1} - E1(x) = sin(x) + 5 \\\\ - \\label{eq:problem2} - E2(x) = (x - 2)^{2} + 1 \\\\ - \\label{eq:problem3} - E3(x) = -0.1 \\cdot x^{2} + 0.6sin(4x) + 8 \\\\ - \\label{eq:problem4} - E4(x) = \\frac{(x - 10)^{2}}{2} + 7.902 - \\end{eqnarray} - -It is impossible to solve this problem -using a single linear classifier. The performance of the best possible -linear classifier is around 50\\%. - -""" - -############################################################################### -# Let's start by importing all required modules, and defining helper functions -# to facilitate plotting the decision boundaries: - -import matplotlib.pyplot as plt -import numpy as np -from sklearn.ensemble import AdaBoostClassifier, RandomForestClassifier -from sklearn.model_selection import train_test_split -from sklearn.neural_network import MLPClassifier -from sklearn.svm import SVC -from sklearn.tree import DecisionTreeClassifier - -# Importing DS techniques -from deslib.dcs.ola import OLA -from deslib.dcs.rank import Rank -from deslib.des.des_p import DESP -from deslib.des.knora_e import KNORAE -from deslib.static import StackedClassifier -from deslib.util.datasets import make_P2 - - -# Plotting-related functions -def make_grid(x, y, h=.02): - x_min, x_max = x.min() - 1, x.max() + 1 - y_min, y_max = y.min() - 1, y.max() + 1 - xx, yy = np.meshgrid(np.arange(x_min, x_max, h), - np.arange(y_min, y_max, h)) - return xx, yy - - -def plot_classifier_decision(ax, clf, X, mode='line', **params): - xx, yy = make_grid(X[:, 0], X[:, 1]) - - Z = clf.predict(np.c_[xx.ravel(), yy.ravel()]) - Z = Z.reshape(xx.shape) - if mode == 'line': - ax.contour(xx, yy, Z, **params) - else: - ax.contourf(xx, yy, Z, **params) - ax.set_xlim((np.min(X[:, 0]), np.max(X[:, 0]))) - ax.set_ylim((np.min(X[:, 1]), np.max(X[:, 0]))) - - -def plot_dataset(X, y, ax=None, title=None, **params): - if ax is None: - ax = plt.gca() - ax.scatter(X[:, 0], X[:, 1], marker='o', c=y, s=25, - edgecolor='k', **params) - ax.set_xlabel('Feature 1') - ax.set_ylabel('Feature 2') - if title is not None: - ax.set_title(title) - return ax - - -############################################################################### -# Visualizing the dataset -# ----------------------- -# Now let's generate and plot the dataset: - - -# Generating and plotting the P2 Dataset: -rng = np.random.RandomState(1234) -X, y = make_P2([1000, 1000], random_state=rng) -X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, - random_state=rng) -fig, axs = plt.subplots(1, 2, figsize=(15, 5)) -plt.subplots_adjust(wspace=0.4, hspace=0.4) -plot_dataset(X_train, y_train, ax=axs[0], title='P2 Training set') -plot_dataset(X_test, y_test, ax=axs[1], title='P2 Test set') - -############################################################################### -# Evaluating the performance of dynamic selection methods -# ------------------------------------------------------- -# -# We will now generate a pool composed of 5 Decision Stumps using AdaBoost. -# -# These are weak linear models. Each base classifier -# has a classification performance close to 50%. -pool_classifiers = AdaBoostClassifier(DecisionTreeClassifier(max_depth=1), - n_estimators=5, random_state=rng) -pool_classifiers.fit(X_train, y_train) - -ax = plot_dataset(X_train, y_train, title='Five Decision Stumps generated') -for clf in pool_classifiers: - plot_classifier_decision(ax, clf, X_train) - ax.set_xlim((0, 1)) - ax.set_ylim((0, 1)) - -plt.show() -plt.tight_layout() - -############################################################################### -# Comparison with Dynamic Selection techniques -# -------------------------------------------- -# -# We will now consider four DS methods: k-Nearest Oracle-Eliminate (KNORA-E), -# Dynamic Ensemble Selection performance (DES-P), Overall Local Accuracy (OLA) -# and Rank. Let's train the classifiers and plot their decision boundaries: - - -knora_e = KNORAE(pool_classifiers).fit(X_train, y_train) -desp = DESP(pool_classifiers).fit(X_train, y_train) -ola = OLA(pool_classifiers).fit(X_train, y_train) -rank = Rank(pool_classifiers).fit(X_train, y_train) - -# Plotting the Decision Border of the DS methods. -fig2, sub = plt.subplots(2, 2, figsize=(15, 10)) -plt.subplots_adjust(wspace=0.4, hspace=0.4) -titles = ['KNORA-Eliminate', 'DES-P', 'Overall Local Accuracy (OLA)', - 'Modified Rank'] - -classifiers = [knora_e, desp, ola, rank] -for clf, ax, title in zip(classifiers, sub.flatten(), titles): - plot_classifier_decision(ax, clf, X_train, mode='filled', alpha=0.4) - plot_dataset(X_test, y_test, ax=ax) - ax.set_xlim(np.min(X[:, 0]), np.max(X[:, 0])) - ax.set_ylim(np.min(X[:, 1]), np.max(X[:, 1])) - ax.set_title(title, fontsize=15) - -# Setting figure to show -# sphinx_gallery_thumbnail_number = 3 - -plt.show() -plt.tight_layout() - -############################################################################### -# Comparison to baselines -# ----------------------- -# -# Let's now compare the results with four baselines: Support Vector Machine -# (SVM) with an RBF kernel; Multi-Layer Perceptron (MLP), Random Forest, -# Adaboost, and Stacking. - -# Setting a baseline using standard classification methods -svm = SVC(gamma='scale', random_state=rng).fit(X_train, y_train) -mlp = MLPClassifier(max_iter=10000, random_state=rng).fit(X_train, y_train) -forest = RandomForestClassifier(n_estimators=10, - random_state=rng).fit(X_train, y_train) -boosting = AdaBoostClassifier(random_state=rng).fit(X_train, y_train) -stacked_lr = StackedClassifier(pool_classifiers=pool_classifiers, - random_state=rng) -stacked_lr.fit(X_train, y_train) - -stacked_dt = StackedClassifier(pool_classifiers=pool_classifiers, - random_state=rng, - meta_classifier=DecisionTreeClassifier()) -stacked_dt.fit(X_train, y_train) - -############################################################################### - -fig2, sub = plt.subplots(2, 3, figsize=(15, 7)) -plt.subplots_adjust(wspace=0.4, hspace=0.4) -titles = ['SVM decision', 'MLP decision', 'RF decision', - 'Boosting decision', 'Stacked LR', 'Stacked Decision Tree'] -classifiers = [svm, mlp, forest, boosting, stacked_lr, stacked_dt] -for clf, ax, title in zip(classifiers, sub.flatten(), titles): - plot_classifier_decision(ax, clf, X_test, mode='filled', alpha=0.4) - plot_dataset(X_test, y_test, ax=ax) - ax.set_xlim(np.min(X[:, 0]), np.max(X[:, 0])) - ax.set_ylim(np.min(X[:, 1]), np.max(X[:, 1])) - ax.set_title(title, fontsize=15) - -plt.show() -plt.tight_layout() - -############################################################################### -# Evaluation on the test set -# -------------------------- -# -# Finally, let's evaluate the baselines and the Dynamic Selection methods on -# the test set: - -print('KNORAE score = {}'.format(knora_e.score(X_test, y_test))) -print('DESP score = {}'.format(desp.score(X_test, y_test))) -print('OLA score = {}'.format(ola.score(X_test, y_test))) -print('Rank score = {}'.format(rank.score(X_test, y_test))) -print('SVM score = {}'.format(svm.score(X_test, y_test))) -print('MLP score = {}'.format(mlp.score(X_test, y_test))) -print('RF score = {}'.format(forest.score(X_test, y_test))) -print('Boosting score = {}'.format(boosting.score(X_test, y_test))) -print('Stacking LR score = {}' .format(stacked_lr.score(X_test, y_test))) -print('Staking Decision Tree = {}' .format(stacked_dt.score(X_test, y_test))) diff --git a/examples/plot_influence_k_value.py b/examples/plot_influence_k_value.py deleted file mode 100644 index 95aa027a..00000000 --- a/examples/plot_influence_k_value.py +++ /dev/null @@ -1,93 +0,0 @@ -# coding=utf-8 - -# Author: Rafael Menelau Oliveira e Cruz -# -# License: BSD 3 clause -""" -==================================================================== -Measuring the influence of the region of competence -==================================================================== - -This example shows how the size of the region of competence (parameter k) -can influence the final performance of DS techniques. - -In this example we vary the value of the parameter k from 3 to 15 and measure -the performance of 7 different dynamic selection technique using the same -pool of classifiers. - -""" - -############################################################################### -# Let's start by importing all required modules. In this example we use the -# new sklearn-OpenML interface to fetch the diabetes classification problem. - -import matplotlib.pyplot as plt -import numpy as np -from sklearn.datasets import fetch_openml -from sklearn.ensemble import BaggingClassifier -from sklearn.linear_model import Perceptron -from sklearn.model_selection import train_test_split -from sklearn.preprocessing import StandardScaler - -from deslib.dcs import LCA -# DCS techniques -from deslib.dcs import MCB -from deslib.dcs import OLA -from deslib.dcs import Rank -# DES techniques -from deslib.des import DESP -from deslib.des import KNORAE -from deslib.des import KNORAU - -rng = np.random.RandomState(123456) - -data = fetch_openml(name='diabetes', cache=False, as_frame=False) -X = data.data -y = data.target -X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=rng) - -# Normalizing the dataset to have 0 mean and unit variance. -scaler = StandardScaler() -X_train = scaler.fit_transform(X_train) -X_test = scaler.transform(X_test) - -pool_classifiers = BaggingClassifier(Perceptron(max_iter=100), - random_state=rng) -pool_classifiers.fit(X_train, y_train) - -# Setting with_IH -mcb = MCB(pool_classifiers, random_state=rng) -ola = OLA(pool_classifiers) -des_p = DESP(pool_classifiers) -knu = KNORAU(pool_classifiers) -lca = LCA(pool_classifiers) -kne = KNORAE(pool_classifiers) -rank = Rank(pool_classifiers) -list_ds_methods = [mcb, ola, des_p, knu, lca, kne, rank] -names = ['MCB', 'OLA', 'DES-P', 'KNORA-U', 'LCA', 'KNORA-E', 'Rank'] - -k_value_list = range(3, 16) - -############################################################################### -# Plot accuracy x region of competence size. -# ------------------------------------------- -# We can see the this parameter can have a huge influence in the performance -# of certain DS techniques. The main exception being the KNORA-E and Rank -# which have built-in mechanism to automatically adjust the region -# of competence size during the competence level estimation. - -fig, ax = plt.subplots() -for ds_method, name in zip(list_ds_methods, names): - accuracy = [] - for k in k_value_list: - ds_method.k = k - ds_method.fit(X_train, y_train) - accuracy.append(ds_method.score(X_test, y_test)) - ax.plot(k_value_list, accuracy, label=name) - -plt.xticks(k_value_list) -ax.set_ylim(0.60, 0.80) -ax.set_xlabel('Region of competence size (K value)', fontsize=13) -ax.set_ylabel('Accuracy on the test set (%)', fontsize=13) -ax.legend(loc='lower right') -plt.show() diff --git a/examples/plot_random_forest.py b/examples/plot_random_forest.py deleted file mode 100644 index 7cc35fbb..00000000 --- a/examples/plot_random_forest.py +++ /dev/null @@ -1,126 +0,0 @@ -# coding=utf-8 - -# Author: Rafael Menelau Oliveira e Cruz -# -# License: BSD 3 clause -""" -==================================================================== -Comparing dynamic selection with Random Forest -==================================================================== - -In this example we use a pool of classifiers generated using the Random Forest -method rather than Bagging. We also show how to change the size of the region -of competence, used to estimate the local competence of the base classifiers. - -This demonstrates that the library accepts any kind of base classifiers as -long as they implement the predict and predict proba functions. Moreover, -any ensemble generation method such as Boosting or Rotation Trees can be used -to generate a pool containing diverse base classifiers. We also included the -performance of the RandomForest classifier as a baseline comparison. -""" - -import matplotlib.pyplot as plt -import numpy as np -from matplotlib.cm import get_cmap -from matplotlib.ticker import FuncFormatter -from sklearn.datasets import fetch_openml -# Pool of base classifiers -from sklearn.ensemble import RandomForestClassifier -from sklearn.linear_model import LogisticRegression -from sklearn.model_selection import train_test_split - -from deslib.dcs.mcb import MCB -# Example of a dcs techniques -from deslib.dcs.ola import OLA -# Example of a des techniques -from deslib.des.des_p import DESP -from deslib.des.knora_e import KNORAE -from deslib.des.knora_u import KNORAU -from deslib.des.meta_des import METADES -# Example of stacked model -from deslib.static.stacked import StackedClassifier - -rng = np.random.RandomState(42) - -# Fetch a classification dataset from OpenML -data = fetch_openml(name='phoneme', version=1, - cache=False, as_frame=False) -X = data.data -y = data.target -# split the data into training and test data -X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, - random_state=rng) - -# Training a random forest to be used as the pool of classifiers. -# We set the maximum depth of the tree so that it -# can estimate probabilities -RF = RandomForestClassifier(random_state=rng, n_estimators=10) -RF.fit(X_train, y_train) - -X_train, X_dsel, y_train, y_dsel = train_test_split(X_train, y_train, - test_size=0.750, - random_state=rng) - -stacked = StackedClassifier(RF, LogisticRegression()) -stacked.fit(X_dsel, y_dsel) - -# Initialize a DS technique. Here we specify the size of -# the region of competence (5 neighbors) -knorau = KNORAU(RF, k=5, random_state=rng) -kne = KNORAE(RF, k=5, random_state=rng) -desp = DESP(RF, k=5, random_state=rng) -ola = OLA(RF, k=5, random_state=rng) -mcb = MCB(RF, k=5, random_state=rng) -meta = METADES(RF, k=5, random_state=rng) - -# Fit the DS techniques -knorau.fit(X_dsel, y_dsel) -kne.fit(X_dsel, y_dsel) -desp.fit(X_dsel, y_dsel) -meta.fit(X_dsel, y_dsel) -ola.fit(X_dsel, y_dsel) -mcb.fit(X_dsel, y_dsel) - -############################################################################### -# Plotting the results -# ----------------------- -# Let's now evaluate the methods on the test set. - -rf_score = RF.score(X_test, y_test) -stacked_score = stacked.score(X_test, y_test) -knorau_score = knorau.score(X_test, y_test) -kne_score = kne.score(X_test, y_test) -desp_score = desp.score(X_test, y_test) -ola_score = ola.score(X_test, y_test) -mcb_score = mcb.score(X_test, y_test) -meta_score = meta.score(X_test, y_test) -print('Classification accuracy RF: ', rf_score) -print('Classification accuracy Stacked: ', stacked_score) -print('Evaluating DS techniques:') -print('Classification accuracy KNORA-U: ', knorau_score) -print('Classification accuracy KNORA-E: ', kne_score) -print('Classification accuracy DESP: ', desp_score) -print('Classification accuracy OLA: ', ola_score) -print('Classification accuracy MCB: ', mcb_score) -print('Classification accuracy META-DES: ', meta_score) - -cmap = get_cmap('Dark2') -colors = [cmap(i) for i in np.linspace(0, 1, 7)] -labels = ['RF', 'Stacked', 'KNORA-U', 'KNORA-E', 'DESP', 'OLA', 'MCB', - 'META-DES'] - -fig, ax = plt.subplots() -pct_formatter = FuncFormatter(lambda x, pos: '{:.1f}'.format(x * 100)) -ax.bar(np.arange(8), - [rf_score, stacked_score, knorau_score, kne_score, desp_score, - ola_score, mcb_score, meta_score], - color=colors, - tick_label=labels) -ax.set_ylim(0.65, 0.80) -ax.set_xlabel('Method', fontsize=13) -ax.set_ylabel('Accuracy on the test set (%)', fontsize=13) -ax.yaxis.set_major_formatter(pct_formatter) -for tick in ax.get_xticklabels(): - tick.set_rotation(45) -plt.subplots_adjust(bottom=0.15) -plt.show() diff --git a/examples/plot_using_instance_hardness.py b/examples/plot_using_instance_hardness.py deleted file mode 100644 index 24e5b9bb..00000000 --- a/examples/plot_using_instance_hardness.py +++ /dev/null @@ -1,105 +0,0 @@ -# coding=utf-8 - -# Author: Rafael Menelau Oliveira e Cruz -# -# License: BSD 3 clause -""" -==================================================================== -Dynamic selection vs K-NN: Using instance hardness -==================================================================== - -One aspect about dynamic selection techniques is that it can better deal with -the classification of test examples associated with high degree of instance -hardness. Such examples are often found close to the border of the classes, -with the majority of its neighbors belonging to different classes. -On the other hand, the KNN method, which is often used to estimate the region -of competence in DS methods works better in the classification of examples -associated with low instance hardness [1]. - -DESlib already implements a switch mechanism between DS techniques and the KNN -classifier according to the hardness level of an instance. This example -varies the threshold in which KNN is used for classification instead of DS -methods. It also compares the classification results with the standard KNN -as a baseline. - -The switch mechanism also reduces the computational cost involved since only -part of the test samples are classified by the DS method. - -References ----------- -[1] Cruz, Rafael MO, et al. "Dynamic Ensemble Selection VS K-NN: why and -when Dynamic Selection obtains higher classification performance?." -arXiv preprint arXiv:1804.07882 (2018). -""" - -############################################################################### -# Let's start by importing all required modules. In this example we use the -# new sklearn-OpenML interface to fetch the diabetes classification problem. - -import matplotlib.pyplot as plt -import numpy as np -from sklearn.datasets import fetch_openml -from sklearn.ensemble import BaggingClassifier -from sklearn.model_selection import train_test_split -from sklearn.preprocessing import StandardScaler -from sklearn.tree import DecisionTreeClassifier - -from deslib.dcs import MCB -from deslib.dcs import OLA -from deslib.dcs import Rank -from deslib.des import DESP -from deslib.des import KNORAE -from deslib.des import KNORAU - -rng = np.random.RandomState(123456) - -data = fetch_openml(name='diabetes', version=1, cache=False, as_frame=False) -X = data.data -y = data.target -X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=rng) - -# Normalizing the dataset to have 0 mean and unit variance. -scaler = StandardScaler() -X_train = scaler.fit_transform(X_train) -X_test = scaler.transform(X_test) - -# Training a pool of classifiers using the bagging technique. -pool_classifiers = BaggingClassifier(DecisionTreeClassifier(random_state=rng), - random_state=rng) -pool_classifiers.fit(X_train, y_train) - -############################################################################### -# Setting DS method to use the switch mechanism -# ---------------------------------------------- -# In order to activate the functionality to switch between DS and KNN according -# to the instance hardness level we need to set the DS techniques to use this -# information. This is done by setting the hyperparameter `with_IH` to True. -# In this example we consider four different values for te threshold -mcb = MCB(pool_classifiers, with_IH=True, random_state=rng) -ola = OLA(pool_classifiers, with_IH=True, random_state=rng) -rank = Rank(pool_classifiers, with_IH=True, random_state=rng) -des_p = DESP(pool_classifiers, with_IH=True, random_state=rng) -kne = KNORAE(pool_classifiers, with_IH=True, random_state=rng) -knu = KNORAU(pool_classifiers, with_IH=True, random_state=rng) -list_ih_values = [0.0, 1./7., 2./7., 3./7.] - -list_ds_methods = [method.fit(X_train, y_train) for method in - [mcb, ola, rank, des_p, kne, knu]] -names = ['MCB', 'OLA', 'Mod. Rank', 'DES-P', 'KNORA-E', 'KNORA-U'] - -# Plot accuracy x IH -fig, ax = plt.subplots() -for ds_method, name in zip(list_ds_methods, names): - accuracy = [] - for idx_ih, ih_rate in enumerate([0.0, 0.14, 0.28, 0.42]): - ds_method.IH_rate = ih_rate - accuracy.append(ds_method.score(X_test, y_test)) - ax.plot(list_ih_values, accuracy, label=name) - -plt.xticks(list_ih_values) -ax.set_ylim(0.65, 0.80) -ax.set_xlabel('IH value', fontsize=13) -ax.set_ylabel('Accuracy on the test set (%)', fontsize=13) -ax.legend() - -plt.show() diff --git a/examples/plot_xor_example.py b/examples/plot_xor_example.py deleted file mode 100644 index ff1e9dd0..00000000 --- a/examples/plot_xor_example.py +++ /dev/null @@ -1,165 +0,0 @@ -# coding=utf-8 - -# Author: Rafael Menelau Oliveira e Cruz -# -# License: BSD 3 clause -""" -==================================================================== -Dynamic selection with linear classifiers: XOR example -==================================================================== - -This example shows that DS can deal with non-linear problem (XOR) using -a combination of a few linear base classifiers. - -- 10 dynamic selection methods (5 DES and 5 DCS) are evaluated with - a pool composed of Decision stumps. - -- Since we use Bagging to generate the base classifiers, we also - included its performance as a baseline comparison. - -""" - -import matplotlib.pyplot as plt -import numpy as np -from sklearn.ensemble import BaggingClassifier -from sklearn.model_selection import train_test_split -from sklearn.tree import DecisionTreeClassifier - -from deslib.dcs import LCA -from deslib.dcs import MLA -from deslib.dcs import OLA -from deslib.dcs import MCB -from deslib.dcs import Rank - -from deslib.des import DESKNN -from deslib.des import KNORAE -from deslib.des import KNORAU -from deslib.des import KNOP -from deslib.des import METADES -from deslib.util.datasets import make_xor - - -############################################################################### -# Defining helper functions to facilitate plotting the decision boundaries: - -def plot_classifier_decision(ax, clf, X, mode='line', **params): - - xx, yy = make_grid(X[:, 0], X[:, 1]) - - Z = clf.predict(np.c_[xx.ravel(), yy.ravel()]) - Z = Z.reshape(xx.shape) - if mode == 'line': - ax.contour(xx, yy, Z, **params) - else: - ax.contourf(xx, yy, Z, **params) - ax.set_xlim((np.min(X[:, 0]), np.max(X[:, 0]))) - ax.set_ylim((np.min(X[:, 1]), np.max(X[:, 0]))) - - -def plot_dataset(X, y, ax=None, title=None, **params): - - if ax is None: - ax = plt.gca() - ax.scatter(X[:, 0], X[:, 1], marker='o', c=y, s=25, - edgecolor='k', **params) - ax.set_xlabel('Feature 1') - ax.set_ylabel('Feature 2') - if title is not None: - ax.set_title(title) - return ax - - -def make_grid(x, y, h=.02): - - x_min, x_max = x.min() - 1, x.max() + 1 - y_min, y_max = y.min() - 1, y.max() + 1 - xx, yy = np.meshgrid(np.arange(x_min, x_max, h), - np.arange(y_min, y_max, h)) - return xx, yy - - -# Prepare the DS techniques. Changing k value to 5. -def initialize_ds(pool_classifiers, X, y, k=5): - knorau = KNORAU(pool_classifiers, k=k) - kne = KNORAE(pool_classifiers, k=k) - desknn = DESKNN(pool_classifiers, k=k) - ola = OLA(pool_classifiers, k=k) - lca = LCA(pool_classifiers, k=k) - mla = MLA(pool_classifiers, k=k) - mcb = MCB(pool_classifiers, k=k) - rank = Rank(pool_classifiers, k=k) - knop = KNOP(pool_classifiers, k=k) - meta = METADES(pool_classifiers, k=k) - - list_ds = [knorau, kne, ola, lca, mla, desknn, mcb, rank, knop, meta] - names = ['KNORA-U', 'KNORA-E', 'OLA', 'LCA', 'MLA', 'DESKNN', 'MCB', - 'RANK', 'KNOP', 'META-DES'] - # fit the ds techniques - for ds in list_ds: - ds.fit(X, y) - - return list_ds, names - - -############################################################################### -# Generating the dataset and training the pool of classifiers. -# -rng = np.random.RandomState(1234) -X, y = make_xor(1000, random_state=rng) -X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, - random_state=rng) -X_DSEL, X_test, y_DSEL, y_test = train_test_split(X_train, y_train, - test_size=0.5, - random_state=rng) - -pool_classifiers = BaggingClassifier(DecisionTreeClassifier(max_depth=1), - n_estimators=10, - random_state=rng) -pool_classifiers.fit(X_train, y_train) - -############################################################################### -# Merging training and validation data to compose DSEL -# ----------------------------------------------------- -# In this example merge the training data with the validation, to create a -# DSEL having more examples for the competence estimation. Using the training -# data for dynamic selection can be beneficial when dealing with small sample -# size datasets. However, in this case we need to have a pool composed of weak -# classifier so that the base classifiers are not able to memorize the -# training data (overfit). - -X_DSEL = np.vstack((X_DSEL, X_train)) -y_DSEL = np.hstack((y_DSEL, y_train)) -list_ds, names = initialize_ds(pool_classifiers, X_DSEL, y_DSEL, k=7) - -fig, sub = plt.subplots(4, 3, figsize=(13, 10)) -plt.subplots_adjust(wspace=0.4, hspace=0.4) - -ax_data = sub.flatten()[0] -ax_bagging = sub.flatten()[1] -plot_dataset(X_train, y_train, ax=ax_data, title="Training data") - -plot_dataset(X_train, y_train, ax=ax_bagging) -plot_classifier_decision(ax_bagging, pool_classifiers, - X_train, mode='filled', alpha=0.4) -ax_bagging.set_title("Bagging") - -# Plotting the decision border of the DS methods -for ds, name, ax in zip(list_ds, names, sub.flatten()[2:]): - plot_dataset(X_train, y_train, ax=ax) - plot_classifier_decision(ax, ds, X_train, mode='filled', alpha=0.4) - ax.set_xlim((np.min(X_train[:, 0]) - 0.1, np.max(X_train[:, 0] + 0.1))) - ax.set_ylim((np.min(X_train[:, 1]) - 0.1, np.max(X_train[:, 1] + 0.1))) - ax.set_title(name) -plt.show() -plt.tight_layout() - -############################################################################### -# Evaluation on the test set -# -------------------------- -# -# Finally, let's evaluate the classification accuracy of DS techniques and -# Bagging on the test set: - -for ds, name in zip(list_ds, names): - print('Accuracy ' + name + ': ' + str(ds.score(X_test, y_test))) -print('Accuracy Bagging: ' + str(pool_classifiers.score(X_test, y_test))) diff --git a/examples/simple_example.py b/examples/simple_example.py deleted file mode 100644 index c80e6647..00000000 --- a/examples/simple_example.py +++ /dev/null @@ -1,50 +0,0 @@ -# coding=utf-8 - -# Author: Rafael Menelau Oliveira e Cruz -# -# License: BSD 3 clause -""" -==================================================================== -Simple example -==================================================================== - -In this example we show how to apply different DCS and DES techniques for a -classification dataset. - -""" -import numpy as np -from sklearn.datasets import make_classification -from sklearn.model_selection import train_test_split -from deslib.des import METADES -from deslib.des import KNORAE - - -# Setting up the random state to have consistent results -rng = np.random.RandomState(42) - -# Generate a classification dataset -X, y = make_classification(n_samples=1000, random_state=rng) -# split the data into training and test data -X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, - random_state=rng) - -# Split the data into training and DSEL for DS techniques -X_train, X_dsel, y_train, y_dsel = train_test_split(X_train, y_train, - test_size=0.5, - random_state=rng) - -# Initialize the DS techniques. DS methods can be initialized without -# specifying a single input parameter. In this example, we just pass the random -# state in order to always have the same result. -kne = KNORAE(random_state=rng) -meta = METADES(random_state=rng) - -# Fitting the des techniques -kne.fit(X_dsel, y_dsel) -meta.fit(X_dsel, y_dsel) - -# Calculate classification accuracy of each technique -print('Evaluating DS techniques:') -print('Classification accuracy KNORA-Eliminate: ', - kne.score(X_test, y_test)) -print('Classification accuracy META-DES: ', meta.score(X_test, y_test)) diff --git a/index.html b/index.html new file mode 100644 index 00000000..4753a87b --- /dev/null +++ b/index.html @@ -0,0 +1,7 @@ + + + DESlib documentation + + + + diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index abd2a919..00000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,13 +0,0 @@ -numpy>=1.17.0 -scipy>=1.4.0 -scikit-learn>=1.0.2 -sphinx -sphinx_rtd_theme -numpydoc -pytest -coverage -pytest-cov -pillow -sphinx_gallery -matplotlib>=2 -nose diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index f74f4a20..00000000 --- a/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -scipy>=1.4.0 -numpy>=1.17.0 -scikit-learn>=1.0.2 diff --git a/setup.py b/setup.py deleted file mode 100644 index 59dfdd47..00000000 --- a/setup.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python - -import codecs -import os -from distutils.core import setup - -from setuptools import find_packages - -setup_path = os.path.abspath(os.path.dirname(__file__)) -with codecs.open(os.path.join(setup_path, 'README.rst'), encoding='utf-8-sig') as f: - README = f.read() - -setup(name='DESlib', - version='0.3.7', - url='https://github.com/Menelau/DESlib', - maintainer='Rafael M. O. Cruz, L. G. Hafemann', - maintainer_email='rafaelmenelau@gmail.com', - description='Implementation of Dynamic Ensemble Selection methods', - long_description=README, - author='Rafael M. O. Cruz', - author_email='rafaelmenelau@gmail.com', - license='BSD 3-clause "New" or "Revised License"', - - classifiers=[ - 'Development Status :: 3 - Alpha', - 'Intended Audience :: Developers', - 'Intended Audience :: Science/Research', - 'License :: OSI Approved :: BSD License', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.12', - 'Topic :: Scientific/Engineering :: Artificial Intelligence', - ], - install_requires=[ - 'scikit-learn>=1.0.2', - 'numpy>=1.17.0', - 'scipy>=1.4.0', - ], - python_requires='>=3', - - packages=find_packages()) -