diff --git a/.gitignore b/.gitignore index 999cc9e..ab971ff 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,10 @@ *pyc +*.vscode .idea .ipynb_checkpoints *~ *# __pycache__ build/ +dist/ +modAL.egg-info/ diff --git a/.travis.yml b/.travis.yml index 9c9ae7e..b3c7647 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,7 @@ after_success: matrix: include: install: - - pip install numpy==1.13 scikit-learn==0.18 scipy==0.18 + - pip install numpy==1.20 scikit-learn==0.18 scipy==0.18 torch==1.8.1 - pip install codecov - pip install coverage - pip install . diff --git a/README.md b/README.md index 4652ea1..4a41633 100644 --- a/README.md +++ b/README.md @@ -100,12 +100,11 @@ import numpy as np X = np.random.choice(np.linspace(0, 20, 10000), size=200, replace=False).reshape(-1, 1) y = np.sin(X) + np.random.normal(scale=0.3, size=X.shape) ``` -For active learning, we shall define a custom query strategy tailored to Gaussian processes. In a nutshell, a *query stategy* in modAL is a function taking (at least) two arguments (an estimator object and a pool of examples), outputting the index of the queried instance and the instance itself. In our case, the arguments are ```regressor``` and ```X```. +For active learning, we shall define a custom query strategy tailored to Gaussian processes. In a nutshell, a *query stategy* in modAL is a function taking (at least) two arguments (an estimator object and a pool of examples), outputting the index of the queried instance. In our case, the arguments are ```regressor``` and ```X```. ```python def GP_regression_std(regressor, X): _, std = regressor.predict(X, return_std=True) - query_idx = np.argmax(std) - return query_idx, X[query_idx] + return np.argmax(std) ``` After setting up the query strategy and the data, the active learner can be initialized. ```python @@ -165,7 +164,7 @@ modAL requires You can install modAL directly with pip: ``` -pip install modAL +pip install modAL-python ``` Alternatively, you can install modAL directly from source: ``` @@ -181,7 +180,7 @@ If you use modAL in your projects, you can cite it as @article{modAL2018, title={mod{AL}: {A} modular active learning framework for {P}ython}, author={Tivadar Danka and Peter Horvath}, - url={https://github.com/cosmic-cortex/modAL}, + url={https://github.com/modAL-python/modAL}, note={available on arXiv at \url{https://arxiv.org/abs/1805.00979}} } ``` diff --git a/docs/source/content/examples/Pytorch_integration.ipynb b/docs/source/content/examples/Pytorch_integration.ipynb new file mode 100644 index 0000000..bacb824 --- /dev/null +++ b/docs/source/content/examples/Pytorch_integration.ipynb @@ -0,0 +1,277 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Pytorch models in modAL workflows\n", + "=============================\n", + "\n", + "Thanks to Skorch API, you can seamlessly integrate Pytorch models into your modAL workflow. In this tutorial, we shall quickly introduce how to use Skorch API of Keras and we are going to see how to do active learning with it. More details on the Keras scikit-learn API [can be found here](https://skorch.readthedocs.io/en/stable/).\n", + "\n", + "The executable script for this example can be [found here](https://github.com/cosmic-cortex/modAL/blob/master/examples/pytorch_integration.py)!" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Skorch API\n", + "-----------------------\n", + "\n", + "By default, a Pytorch model's interface differs from what is used for scikit-learn estimators. However, with the use of Skorch wrapper, it is possible to adapt your model." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "from torch import nn\n", + "from skorch import NeuralNetClassifier\n", + "\n", + "# build class for the skorch API\n", + "class Torch_Model(nn.Module):\n", + " def __init__(self,):\n", + " super(Torch_Model, self).__init__()\n", + " self.convs = nn.Sequential(\n", + " nn.Conv2d(1,32,3),\n", + " nn.ReLU(),\n", + " nn.Conv2d(32,64,3),\n", + " nn.ReLU(),\n", + " nn.MaxPool2d(2),\n", + " nn.Dropout(0.25)\n", + " )\n", + " self.fcs = nn.Sequential(\n", + " nn.Linear(12*12*64,128),\n", + " nn.ReLU(),\n", + " nn.Dropout(0.5),\n", + " nn.Linear(128,10),\n", + " )\n", + "\n", + " def forward(self, x):\n", + " out = x\n", + " out = self.convs(out)\n", + " out = out.view(-1,12*12*64)\n", + " out = self.fcs(out)\n", + " return out" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For our purposes, the ``classifier`` which we will initialize now acts just like any scikit-learn estimator." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "# create the classifier\n", + "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", + "classifier = NeuralNetClassifier(Torch_Model,\n", + " criterion=nn.CrossEntropyLoss,\n", + " optimizer=torch.optim.Adam,\n", + " train_split=None,\n", + " verbose=1,\n", + " device=device)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Active learning with Pytorch\n", + "---------------------------------------\n", + "\n", + "In this example, we are going to use the famous MNIST dataset, which is available as a built-in for PyTorch." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + "0it [00:00, ?it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz to ./MNIST/raw/train-images-idx3-ubyte.gz\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 97%|█████████▋| 9584640/9912422 [00:15<00:00, 1777143.52it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Extracting ./MNIST/raw/train-images-idx3-ubyte.gz\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + "0it [00:00, ?it/s]\u001b[A" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz to ./MNIST/raw/train-labels-idx1-ubyte.gz\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + " 0%| | 0/28881 [00:00 Queries are selectively drawn from the pool, which is usually assumed to be closed (i.e., static or non-changing), although this is not strictly necessary. Typically, instances are queried in a greedy fashion, according to an informativeness measure used to evaluate all instances in the pool (or, perhaps if $\\mathcal{U}$ is very large, some subsample thereof).\n", "\n", diff --git a/docs/source/content/overview/Extending-modAL.ipynb b/docs/source/content/overview/Extending-modAL.ipynb index bd2f794..641ca98 100644 --- a/docs/source/content/overview/Extending-modAL.ipynb +++ b/docs/source/content/overview/Extending-modAL.ipynb @@ -27,11 +27,8 @@ " # measure the utility of each instance in the pool\n", " utility = utility_measure(classifier, X)\n", "\n", - " # select the indices of the instances to be queried\n", - " query_idx = select_instances(utility)\n", - "\n", - " # return the indices and the instances\n", - " return query_idx, X[query_idx]" + " # select and return the indices of the instances to be queried\n", + " return select_instances(utility)" ] }, { @@ -213,8 +210,7 @@ "# classifier uncertainty and classifier margin\n", "def custom_query_strategy(classifier, X, n_instances=1):\n", " utility = linear_combination(classifier, X)\n", - " query_idx = multi_argmax(utility, n_instances=n_instances)\n", - " return query_idx, X[query_idx]\n", + " return multi_argmax(utility, n_instances=n_instances)\n", "\n", "custom_query_learner = ActiveLearner(\n", " estimator=GaussianProcessClassifier(1.0 * RBF(1.0)),\n", @@ -299,4 +295,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/docs/source/content/overview/Installation.rst b/docs/source/content/overview/Installation.rst index 16a209a..76d4471 100644 --- a/docs/source/content/overview/Installation.rst +++ b/docs/source/content/overview/Installation.rst @@ -5,13 +5,13 @@ modAL requires * Python >= 3.5 * NumPy >= 1.13 * SciPy >= 0.18 - * scikit-learn >= 0.18 + * scikit-learn >= 0.22 You can install modAL directly with pip: :: - pip install modAL + pip install modAL-python Alternatively, you can install modAL directly from source: diff --git a/docs/source/content/overview/modAL-in-a-nutshell.rst b/docs/source/content/overview/modAL-in-a-nutshell.rst index d0435e9..99dda7b 100644 --- a/docs/source/content/overview/modAL-in-a-nutshell.rst +++ b/docs/source/content/overview/modAL-in-a-nutshell.rst @@ -118,15 +118,13 @@ the *noisy sine* function: For active learning, we shall define a custom query strategy tailored to Gaussian processes. In a nutshell, a *query stategy* in modAL is a function taking (at least) two arguments (an estimator object and a pool -of examples), outputting the index of the queried instance and the -instance itself. In our case, the arguments are ``regressor`` and ``X``. +of examples), outputting the index of the queried instance. In our case, the arguments are ``regressor`` and ``X``. .. code:: python def GP_regression_std(regressor, X): _, std = regressor.predict(X, return_std=True) - query_idx = np.argmax(std) - return query_idx, X[query_idx] + return np.argmax(std) After setting up the query strategy and the data, the active learner can be initialized. diff --git a/docs/source/content/query_strategies/Acquisition-functions.rst b/docs/source/content/query_strategies/Acquisition-functions.rst index df28c39..c3a5725 100644 --- a/docs/source/content/query_strategies/Acquisition-functions.rst +++ b/docs/source/content/query_strategies/Acquisition-functions.rst @@ -3,7 +3,7 @@ Acquisition functions ===================== -In Bayesian optimization, a so-called *acquisition funciton* is used instead of the uncertainty based utility measures of active learning. In modAL, Bayesian optimization algorithms are implemented in the ``modAL.models.BayesianOptimizer`` class. Currently, there are three available acquisition funcions: probability of improvement, expected improvement and upper confidence bound. +In Bayesian optimization, a so-called *acquisition funciton* is used instead of the uncertainty based utility measures of active learning. In modAL, Bayesian optimization algorithms are implemented in the ``modAL.models.BayesianOptimizer`` class. Currently, there are three available acquisition functions: probability of improvement, expected improvement and upper confidence bound. Probability of improvement -------------------------- diff --git a/docs/source/content/query_strategies/Disagreement-sampling.rst b/docs/source/content/query_strategies/Disagreement-sampling.rst index b8deeb8..6786212 100644 --- a/docs/source/content/query_strategies/Disagreement-sampling.rst +++ b/docs/source/content/query_strategies/Disagreement-sampling.rst @@ -3,7 +3,7 @@ Disagreement sampling ===================== -When you have several hypothesis about your data, selecting the next instances to label can be done by measuring the disagreement between the hypotheses. Naturally, there are many ways to do that. In modAL, there are three built-in disagreement measures and query strategies: *vote entropy*, *consensus entropy* and *maximum disagreement*. In this quick tutorial, we are going to review them. For more details, see Section 3.4 of the awesome book `Active learning by Burr Settles `__. +When you have several hypotheses about your data, selecting the next instances to label can be done by measuring the disagreement between the hypotheses. Naturally, there are many ways to do that. In modAL, there are three built-in disagreement measures and query strategies: *vote entropy*, *consensus entropy* and *maximum disagreement*. In this quick tutorial, we are going to review them. For more details, see Section 3.4 of the awesome book `Active learning by Burr Settles `__. Disagreement sampling for classifiers ------------------------------------- @@ -52,7 +52,7 @@ Instead of calculating the distribution of the votes, the *consensus entropy* disagreement measure first calculates the average of the class probabilities of each classifier. This is called the consensus probability. Then the entropy of the consensus probability is calculated -and the instance with largest consensus entropy is selected. +and the instance with the largest consensus entropy is selected. For an example, let's suppose that we continue the previous example with three classifiers, classes ``[0, 1, 2]`` and five instances to classify. @@ -100,7 +100,7 @@ Even though the votes for the second instance are ``[1, 1, 2]``, since the class Max disagreement ^^^^^^^^^^^^^^^^ -The disagreement measures so far take the actual *disagreement* into account in a weak way. Instead of this, it is possible to to measure each learner's disagreement with the consensus probabilities and query the instance where the disagreement is largest for some learner. This is called *max disagreement sampling*. Continuing our example, if the vote probabilities for each learner and the consensus probabilities are given, we can calculate the `Kullback-Leibler divergence `__ of each learner to the consensus prediction and then for each instance, select the largest value. +The disagreement measures so far take the actual *disagreement* into account in a weak way. Instead of this, it is possible to measure each learner's disagreement with the consensus probabilities and query the instance where the disagreement is largest for some learner. This is called *max disagreement sampling*. Continuing our example, if the vote probabilities for each learner and the consensus probabilities are given, we can calculate the `Kullback-Leibler divergence `__ of each learner to the consensus prediction and then for each instance, select the largest value. .. code:: python @@ -123,7 +123,7 @@ In this case, one of the learner highly disagrees with the others in the class o Disagreement sampling for regressors ------------------------------------ -Since regressors in general don't provide a way to calculate prediction probabilities, disagreement measures for classifiers may not work with regressors. Despite of this, ensemble regression models can be always used in an active learning scenario, because the standard deviation of the predictions at a given point can be thought of as a measure of disagreement. +Since regressors, in general, don't provide a way to calculate prediction probabilities, disagreement measures for classifiers may not work with regressors. Despite this, ensemble regression models can be always used in an active learning scenario, because the standard deviation of the predictions at a given point can be thought of as a measure of disagreement. Standard deviation sampling ^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -131,7 +131,7 @@ Standard deviation sampling .. figure:: img/er-initial.png :align: center -When a committee of regressors is available, uncertainty of predictions can be estimated by calculating the standard deviation of predictions. This is done by the ``modAL.disagreement.max_std_sampling`` function. +When a committee of regressors is available, the uncertainty of predictions can be estimated by calculating the standard deviation of predictions. This is done by the ``modAL.disagreement.max_std_sampling`` function. Disagreement measures in action ------------------------------- @@ -151,7 +151,7 @@ The consensus predictions of these learners are .. figure:: img/dis-consensus.png :align: center -In this case, the disagreement measures from left to right are vote entropy, consensus entropy and max disagreement. +In this case, the disagreement measures from left to right are vote entropy, consensus entropy, and max disagreement. .. figure:: img/dis-measures.png :align: center diff --git a/docs/source/content/query_strategies/ranked_batch_mode.ipynb b/docs/source/content/query_strategies/ranked_batch_mode.ipynb index c8fe902..f3633b0 100644 --- a/docs/source/content/query_strategies/ranked_batch_mode.ipynb +++ b/docs/source/content/query_strategies/ranked_batch_mode.ipynb @@ -130,7 +130,7 @@ "distance_scores = pairwise_distances(X_pool, X_training, metric='euclidean').min(axis=1)\n", "similarity_scores = 1 / (1 + distance_scores)\n", "\n", - "alpha = len(X_training)/len(X_raw)\n", + "alpha = len(X_pool)/len(X_raw)\n", "\n", "scores = alpha * (1 - similarity_scores) + (1 - alpha) * uncertainty" ] diff --git a/docs/source/index.rst b/docs/source/index.rst index d3adafb..2963c01 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -67,6 +67,7 @@ Currently supported active learning strategies are content/examples/query_by_committee content/examples/bootstrapping_and_bagging content/examples/Keras_integration + content/examples/Pytorch_integration .. toctree:: :glob: diff --git a/examples/active_regression.py b/examples/active_regression.py index f4d1b4a..abe3fe9 100644 --- a/examples/active_regression.py +++ b/examples/active_regression.py @@ -2,18 +2,17 @@ Active regression example with Gaussian processes. """ -import numpy as np import matplotlib.pyplot as plt -from sklearn.gaussian_process import GaussianProcessRegressor -from sklearn.gaussian_process.kernels import WhiteKernel, RBF +import numpy as np from modAL.models import ActiveLearner +from sklearn.gaussian_process import GaussianProcessRegressor +from sklearn.gaussian_process.kernels import RBF, WhiteKernel # query strategy for regression def GP_regression_std(regressor, X): _, std = regressor.predict(X, return_std=True) - query_idx = np.argmax(std) - return query_idx, X[query_idx] + return np.argmax(std) # generating the data diff --git a/examples/bagging.py b/examples/bagging.py index 2d61040..868a10b 100644 --- a/examples/bagging.py +++ b/examples/bagging.py @@ -2,11 +2,12 @@ This example shows how to build models with bagging using the Committee model. """ -import numpy as np from itertools import product + +import numpy as np from matplotlib import pyplot as plt -from sklearn.neighbors import KNeighborsClassifier from modAL.models import ActiveLearner, Committee +from sklearn.neighbors import KNeighborsClassifier # creating the dataset im_width = 500 @@ -90,4 +91,4 @@ plt.subplot(1, n_learners, learner_idx+1) plt.imshow(learner.predict(X_pool).reshape(im_height, im_width)) plt.title('Learner no. %d after refitting' % (learner_idx + 1)) - plt.show() \ No newline at end of file + plt.show() diff --git a/examples/bayesian_optimization.py b/examples/bayesian_optimization.py index 3caa0b4..0d62ea3 100644 --- a/examples/bayesian_optimization.py +++ b/examples/bayesian_optimization.py @@ -1,11 +1,12 @@ -import numpy as np -import matplotlib.pyplot as plt from functools import partial + +import matplotlib.pyplot as plt +import numpy as np +from modAL.acquisition import (max_EI, max_PI, max_UCB, optimizer_EI, + optimizer_PI, optimizer_UCB) +from modAL.models import BayesianOptimizer from sklearn.gaussian_process import GaussianProcessRegressor from sklearn.gaussian_process.kernels import Matern -from modAL.models import BayesianOptimizer -from modAL.acquisition import optimizer_PI, optimizer_EI, optimizer_UCB, max_PI, max_EI, max_UCB - # generating the data X = np.linspace(0, 20, 1000).reshape(-1, 1) diff --git a/examples/bayesian_optimization_multidim.py b/examples/bayesian_optimization_multidim.py new file mode 100644 index 0000000..9c74219 --- /dev/null +++ b/examples/bayesian_optimization_multidim.py @@ -0,0 +1,25 @@ +import numpy as np +from modAL.acquisition import max_EI +from modAL.models import BayesianOptimizer +from sklearn.gaussian_process import GaussianProcessRegressor +from sklearn.gaussian_process.kernels import Matern + +# generating the data +x1, x2 = np.linspace(0, 10, 11).reshape(-1, 1), np.linspace(0, 10, 11).reshape(-1, 1) +x1, x2 = np.meshgrid(x1, x2) +X = np.concatenate((x1.reshape(-1, 1), x2.reshape(-1, 1)), axis=1) + +y = np.sin(np.linalg.norm(X, axis=1))/2 - ((10 - np.linalg.norm(X, axis=1))**2)/50 + 2 + +# assembling initial training set +X_initial, y_initial = X[:10], y[:10] + +# defining the kernel for the Gaussian process +kernel = Matern(length_scale=1.0) + +optimizer = BayesianOptimizer(estimator=GaussianProcessRegressor(kernel=kernel), + X_training=X_initial, y_training=y_initial, + query_strategy=max_EI) + +query_idx, query_inst = optimizer.query(X) +optimizer.teach(X[query_idx].reshape(1, -1), y[query_idx]) diff --git a/examples/cost_effective_active_learning.py b/examples/cost_effective_active_learning.py new file mode 100644 index 0000000..1feac04 --- /dev/null +++ b/examples/cost_effective_active_learning.py @@ -0,0 +1,102 @@ +""" +This is a modified implementation of the algorithm Cost Effective Active Learning +(Pl. refer - https://arxiv.org/abs/1701.03551). This version not only picks up the +top K uncertain samples but also picks up the top N highly confident samples that +may represent information and diversity. It is different than the original implementation +as it does not involve tuning the confidence threshold parameter for every dataset. +""" + +from keras.datasets import mnist +import numpy as np +from modAL.models import ActiveLearner +from sklearn.ensemble import RandomForestClassifier +from scipy.special import entr + + +(X_train, y_train), (X_test, y_test) = mnist.load_data() + +X_train = X_train / 255 +X_test = X_test / 255 +y_train = y_train.astype(np.uint8) +y_test = y_test.astype(np.uint8) + +X_train = X_train.reshape(-1, 784) +X_test = X_test.reshape(-1, 784) + +model = RandomForestClassifier(n_estimators=100) + +INITIAL_SET_SIZE = 32 + +U_x = np.copy(X_train) +U_y = np.copy(y_train) + +ind = np.random.choice(range(len(U_x)), size=INITIAL_SET_SIZE) + +X_initial = U_x[ind] +y_initial = U_y[ind] + +U_x = np.delete(U_x, ind, axis=0) +U_y = np.delete(U_y, ind, axis=0) + + +def assign_pseudo_labels(active_learner, X, confidence_idx): + conf_samples = X[confidence_idx] + labels = active_learner.predict(conf_samples) + return labels + + +def max_entropy(active_learner, X, K=16, N=16): + + class_prob = active_learner.predict_proba(X) + entropy = entr(class_prob).sum(axis=1) + uncertain_idx = np.argpartition(entropy, -K)[-K:] + + """ + Original Implementation -- Pick most confident samples with + entropy less than a threshold. Threshold is decayed in every + iteration. + + Different than original -- Pick top n most confident samples. + """ + + confidence_idx = np.argpartition(entropy, N)[:N] + + return np.concatenate((uncertain_idx, confidence_idx), axis=0) + + +active_learner = ActiveLearner( + estimator=model, + X_training=X_initial, + y_training=y_initial, + query_strategy=max_entropy +) + +N_QUERIES = 20 + +K_MAX_ENTROPY = 16 +N_MIN_ENTROPY = 16 + +scores = [active_learner.score(X_test, y_test)] + +for index in range(N_QUERIES): + + query_idx, query_instance = active_learner.query(U_x, K_MAX_ENTROPY, N_MIN_ENTROPY) + + uncertain_idx = query_idx[:K_MAX_ENTROPY] + confidence_idx = query_idx[K_MAX_ENTROPY:] + + conf_labels = assign_pseudo_labels(active_learner, U_x, confidence_idx) + + L_x = U_x[query_idx] + L_y = np.concatenate((U_y[uncertain_idx], conf_labels), axis=0) + + active_learner.teach(L_x, L_y) + + U_x = np.delete(U_x, query_idx, axis=0) + U_y = np.delete(U_y, query_idx, axis=0) + + acc = active_learner.score(X_test, y_test) + + print(F'Query {index+1}: Test Accuracy: {acc}') + + scores.append(acc) \ No newline at end of file diff --git a/examples/custom_query_strategies.py b/examples/custom_query_strategies.py index ec65d9b..d969643 100644 --- a/examples/custom_query_strategies.py +++ b/examples/custom_query_strategies.py @@ -5,18 +5,16 @@ The first two arguments of a query strategy function is always the estimator and the pool of instances to be queried from. Additional arguments are accepted as keyword arguments. -A valid query strategy function always returns a tuple of the indices of the queried -instances and the instances themselves. +A valid query strategy function always returns indices of the queried +instances. def custom_query_strategy(classifier, X, a_keyword_argument=42): # measure the utility of each instance in the pool utility = utility_measure(classifier, X) - # select the indices of the instances to be queried - query_idx = select_instances(utility) + # select and return the indices of the instances to be queried + return select_instances(utility) - # return the indices and the instances - return query_idx, X[query_idx] This function can be used in the active learning workflow. @@ -27,18 +25,16 @@ def custom_query_strategy(classifier, X, a_keyword_argument=42): and classifier margin. """ -import numpy as np import matplotlib.pyplot as plt - +import numpy as np +from modAL.models import ActiveLearner +from modAL.uncertainty import classifier_margin, classifier_uncertainty from modAL.utils.combination import make_linear_combination, make_product from modAL.utils.selection import multi_argmax -from modAL.uncertainty import classifier_uncertainty, classifier_margin -from modAL.models import ActiveLearner from sklearn.datasets import make_blobs from sklearn.gaussian_process import GaussianProcessClassifier from sklearn.gaussian_process.kernels import RBF - # generating the data centers = np.asarray([[-2, 3], [0.5, 5], [1, 1.5]]) X, y = make_blobs( @@ -97,8 +93,7 @@ def custom_query_strategy(classifier, X, a_keyword_argument=42): # classifier uncertainty and classifier margin def custom_query_strategy(classifier, X, n_instances=1): utility = linear_combination(classifier, X) - query_idx = multi_argmax(utility, n_instances=n_instances) - return query_idx, X[query_idx] + return multi_argmax(utility, n_instances=n_instances) custom_query_learner = ActiveLearner( estimator=GaussianProcessClassifier(1.0 * RBF(1.0)), diff --git a/examples/deep_bayesian_active_learning.py b/examples/deep_bayesian_active_learning.py new file mode 100644 index 0000000..355549b --- /dev/null +++ b/examples/deep_bayesian_active_learning.py @@ -0,0 +1,96 @@ +import keras +import numpy as np +from keras import backend as K +from keras.datasets import mnist +from keras.layers import (Activation, Conv2D, Dense, Dropout, Flatten, + MaxPooling2D) +from keras.models import Sequential +from keras.regularizers import l2 +from keras.wrappers.scikit_learn import KerasClassifier +from modAL.models import ActiveLearner + + +def create_keras_model(): + model = Sequential() + model.add(Conv2D(32, (4, 4), activation='relu')) + model.add(Conv2D(32, (4, 4), activation='relu')) + model.add(MaxPooling2D(pool_size=(2, 2))) + model.add(Dropout(0.25)) + model.add(Flatten()) + model.add(Dense(128, activation='relu')) + model.add(Dropout(0.5)) + model.add(Dense(10, activation='softmax')) + model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=["accuracy"]) + return model + + +# create the classifier +classifier = KerasClassifier(create_keras_model) + +# read training data +(X_train, y_train), (X_test, y_test) = mnist.load_data() + + +# assemble initial data +initial_idx = np.array([],dtype=np.int) +for i in range(10): + idx = np.random.choice(np.where(y_train==i)[0], size=2, replace=False) + initial_idx = np.concatenate((initial_idx, idx)) + +# Preprocessing +X_train = X_train.reshape(60000, 28, 28, 1).astype('float32') / 255. +X_test = X_test.reshape(10000, 28, 28, 1).astype('float32') / 255. +y_train = keras.utils.to_categorical(y_train, 10) +y_test = keras.utils.to_categorical(y_test, 10) + +X_initial = X_train[initial_idx] +y_initial = y_train[initial_idx] + +# remove the initial data from the pool of unlabelled examples +X_pool = np.delete(X_train, initial_idx, axis=0) +y_pool = np.delete(y_train, initial_idx, axis=0) + +""" +Query Strategy +""" + +def max_entropy(learner, X, n_instances=1, T=100): + random_subset = np.random.choice(X.shape[0], 2000, replace=False) + MC_output = K.function([learner.estimator.model.layers[0].input, K.learning_phase()], + [learner.estimator.model.layers[-1].output]) + learning_phase = True + MC_samples = [MC_output([X[random_subset], learning_phase])[0] for _ in range(T)] + MC_samples = np.array(MC_samples) # [#samples x batch size x #classes] + expected_p = np.mean(MC_samples, axis=0) + acquisition = - np.sum(expected_p * np.log(expected_p + 1e-10), axis=-1) # [batch size] + idx = (-acquisition).argsort()[:n_instances] + return random_subset[idx] + +def uniform(learner, X, n_instances=1): + return np.random.choice(range(len(X)), size=n_instances, replace=False) + +""" +Training the ActiveLearner +""" + +# initialize ActiveLearner +learner = ActiveLearner( + estimator=classifier, + X_training=X_initial, + y_training=y_initial, + query_strategy=max_entropy, + verbose=0 +) + +# the active learning loop +n_queries = 100 +perf_hist = [learner.score(X_test, y_test, verbose=0)] +for index in range(n_queries): + query_idx, query_instance = learner.query(X_pool, n_instances=10) + learner.teach(X_pool[query_idx], y_pool[query_idx], epochs=50, batch_size=128, verbose=0) + # remove queried instance from pool + X_pool = np.delete(X_pool, query_idx, axis=0) + y_pool = np.delete(y_pool, query_idx, axis=0) + model_accuracy = learner.score(X_test, y_test, verbose=0) + print('Accuracy after query {n}: {acc:0.4f}'.format(n=index + 1, acc=model_accuracy)) + perf_hist.append(model_accuracy) diff --git a/examples/ensemble.py b/examples/ensemble.py index bd621f4..d4c6791 100644 --- a/examples/ensemble.py +++ b/examples/ensemble.py @@ -1,8 +1,9 @@ -import numpy as np from itertools import product + +import numpy as np from matplotlib import pyplot as plt -from sklearn.ensemble import RandomForestClassifier from modAL.models import ActiveLearner, Committee +from sklearn.ensemble import RandomForestClassifier # creating the dataset im_width = 500 diff --git a/examples/ensemble_regression.py b/examples/ensemble_regression.py index bdb2276..e6d845f 100644 --- a/examples/ensemble_regression.py +++ b/examples/ensemble_regression.py @@ -1,9 +1,9 @@ -import numpy as np import matplotlib.pyplot as plt -from sklearn.gaussian_process import GaussianProcessRegressor -from sklearn.gaussian_process.kernels import WhiteKernel, RBF -from modAL.models import ActiveLearner, CommitteeRegressor +import numpy as np from modAL.disagreement import max_std_sampling +from modAL.models import ActiveLearner, CommitteeRegressor +from sklearn.gaussian_process import GaussianProcessRegressor +from sklearn.gaussian_process.kernels import RBF, WhiteKernel # generating the data X = np.concatenate((np.random.rand(100)-1, np.random.rand(100))) diff --git a/examples/information_density.py b/examples/information_density.py index c091061..a5d8ad1 100644 --- a/examples/information_density.py +++ b/examples/information_density.py @@ -1,5 +1,4 @@ import matplotlib.pyplot as plt - from modAL.density import information_density from sklearn.datasets import make_blobs diff --git a/examples/keras_integration.py b/examples/keras_integration.py index 0e27b75..abf4dca 100644 --- a/examples/keras_integration.py +++ b/examples/keras_integration.py @@ -6,8 +6,8 @@ import keras import numpy as np from keras.datasets import mnist +from keras.layers import Conv2D, Dense, Dropout, Flatten, MaxPooling2D from keras.models import Sequential -from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPooling2D from keras.wrappers.scikit_learn import KerasClassifier from modAL.models import ActiveLearner diff --git a/examples/multilabel_svm.py b/examples/multilabel_svm.py index ab4eb3c..7d34ddf 100644 --- a/examples/multilabel_svm.py +++ b/examples/multilabel_svm.py @@ -1,9 +1,7 @@ -import numpy as np import matplotlib.pyplot as plt - +import numpy as np from modAL.models import ActiveLearner from modAL.multilabel import * - from sklearn.multiclass import OneVsRestClassifier from sklearn.svm import SVC @@ -33,4 +31,4 @@ ) query_idx, query_inst = learner.query(X_pool) -learner.teach(X_pool[query_idx], y_pool[query_idx]) \ No newline at end of file +learner.teach(X_pool[query_idx], y_pool[query_idx]) diff --git a/examples/pool-based_sampling.py b/examples/pool-based_sampling.py index cc89c4b..aedae0a 100644 --- a/examples/pool-based_sampling.py +++ b/examples/pool-based_sampling.py @@ -4,12 +4,12 @@ For its scikit-learn interface, see http://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_iris.html """ -import numpy as np import matplotlib.pyplot as plt -from sklearn.decomposition import PCA +import numpy as np +from modAL.models import ActiveLearner from sklearn.datasets import load_iris +from sklearn.decomposition import PCA from sklearn.neighbors import KNeighborsClassifier -from modAL.models import ActiveLearner # loading the iris dataset iris = load_iris() @@ -65,4 +65,4 @@ prediction = learner.predict(iris['data']) plt.scatter(x=pca[:, 0], y=pca[:, 1], c=prediction, cmap='viridis', s=50) plt.title('Classification accuracy after %i queries: %f' % (n_queries, learner.score(iris['data'], iris['target']))) - plt.show() \ No newline at end of file + plt.show() diff --git a/examples/pytorch_integration.py b/examples/pytorch_integration.py new file mode 100644 index 0000000..5ee4e9e --- /dev/null +++ b/examples/pytorch_integration.py @@ -0,0 +1,101 @@ +""" +This example demonstrates how to use the active learning interface with Pytorch. +The example uses Skorch, a scikit learn wrapper of Pytorch. +For more info, see https://skorch.readthedocs.io/en/stable/ +""" + +import numpy as np +import torch +from modAL.models import ActiveLearner +from skorch import NeuralNetClassifier +from torch import nn +from torch.utils.data import DataLoader +from torchvision.datasets import MNIST +from torchvision.transforms import ToTensor + + +# build class for the skorch API +class Torch_Model(nn.Module): + def __init__(self,): + super(Torch_Model, self).__init__() + self.convs = nn.Sequential( + nn.Conv2d(1,32,3), + nn.ReLU(), + nn.Conv2d(32,64,3), + nn.ReLU(), + nn.MaxPool2d(2), + nn.Dropout(0.25) + ) + self.fcs = nn.Sequential( + nn.Linear(12*12*64,128), + nn.ReLU(), + nn.Dropout(0.5), + nn.Linear(128,10), + ) + + def forward(self, x): + out = x + out = self.convs(out) + out = out.view(-1,12*12*64) + out = self.fcs(out) + return out + + +# create the classifier +device = "cuda" if torch.cuda.is_available() else "cpu" +classifier = NeuralNetClassifier(Torch_Model, + # max_epochs=100, + criterion=nn.CrossEntropyLoss, + optimizer=torch.optim.Adam, + train_split=None, + verbose=1, + device=device) + +""" +Data wrangling +1. Reading data from torchvision +2. Assembling initial training data for ActiveLearner +3. Generating the pool +""" + +mnist_data = MNIST('.', download=True, transform=ToTensor()) +dataloader = DataLoader(mnist_data, shuffle=True, batch_size=60000) +X, y = next(iter(dataloader)) + +# read training data +X_train, X_test, y_train, y_test = X[:50000], X[50000:], y[:50000], y[50000:] +X_train = X_train.reshape(50000, 1, 28, 28) +X_test = X_test.reshape(10000, 1, 28, 28) + +# assemble initial data +n_initial = 1000 +initial_idx = np.random.choice(range(len(X_train)), size=n_initial, replace=False) +X_initial = X_train[initial_idx] +y_initial = y_train[initial_idx] + +# generate the pool +# remove the initial data from the training dataset +X_pool = np.delete(X_train, initial_idx, axis=0) +y_pool = np.delete(y_train, initial_idx, axis=0) + +""" +Training the ActiveLearner +""" + +# initialize ActiveLearner +learner = ActiveLearner( + estimator=classifier, + X_training=X_initial, y_training=y_initial, +) + +# the active learning loop +n_queries = 10 +for idx in range(n_queries): + query_idx, query_instance = learner.query(X_pool, n_instances=100) + learner.teach(X_pool[query_idx], y_pool[query_idx], only_new=True) + # remove queried instance from pool + X_pool = np.delete(X_pool, query_idx, axis=0) + y_pool = np.delete(y_pool, query_idx, axis=0) + +# the final accuracy score +print(learner.score(X_test, y_test)) diff --git a/examples/pytorch_mc_dropout.py b/examples/pytorch_mc_dropout.py new file mode 100644 index 0000000..dab92d5 --- /dev/null +++ b/examples/pytorch_mc_dropout.py @@ -0,0 +1,122 @@ +""" +In this file the basic ModAL PyTorch DeepActiveLearner workflow is explained +through an example on the MNIST dataset and the MC-Dropout-Bald query strategy. +""" +import numpy as np +import torch +# import of query strategies +from modAL.dropout import mc_dropout_bald +from modAL.models import DeepActiveLearner +from skorch import NeuralNetClassifier +from torch import nn +from torch.utils.data import DataLoader +from torchvision.datasets import MNIST +from torchvision.transforms import ToTensor + + +# Standard Pytorch Model (Visit the PyTorch documentation for more details) +class Torch_Model(nn.Module): + def __init__(self,): + super(Torch_Model, self).__init__() + self.convs = nn.Sequential( + nn.Conv2d(1, 32, 3), + nn.ReLU(), + nn.Conv2d(32, 64, 3), + nn.ReLU(), + nn.MaxPool2d(2), + nn.Dropout(0.25) + ) + self.fcs = nn.Sequential( + nn.Linear(12*12*64, 128), + nn.ReLU(), + nn.Dropout(0.5), + nn.Linear(128, 10), + ) + + def forward(self, x): + out = x + out = self.convs(out) + out = out.view(-1, 12*12*64) + out = self.fcs(out) + return out + + +torch_model = Torch_Model() +""" +You can acquire from the layer_list the dropout_layer_indexes, which can then be passed on +to the query strategies to decide which dropout layers should be active for the predictions. +When no dropout_layer_indexes are passed, all dropout layers will be activated on default. +""" +layer_list = list(torch_model.modules()) + +device = "cuda" if torch.cuda.is_available() else "cpu" + +# Use the NeuralNetClassifier from skorch to wrap the Pytorch model to the scikit-learn API +classifier = NeuralNetClassifier(Torch_Model, + criterion=torch.nn.CrossEntropyLoss, + optimizer=torch.optim.Adam, + train_split=None, + verbose=1, + device=device) + + +# Load the Dataset +mnist_data = MNIST('.', download=True, transform=ToTensor()) +dataloader = DataLoader(mnist_data, shuffle=True, batch_size=60000) +X, y = next(iter(dataloader)) + +# read training data +X_train, X_test, y_train, y_test = X[:50000], X[50000:], y[:50000], y[50000:] +X_train = X_train.reshape(50000, 1, 28, 28) +X_test = X_test.reshape(10000, 1, 28, 28) + +# assemble initial data +n_initial = 1000 +initial_idx = np.random.choice( + range(len(X_train)), size=n_initial, replace=False) +X_initial = X_train[initial_idx] +y_initial = y_train[initial_idx] + + +# generate the pool +# remove the initial data from the training dataset +X_pool = np.delete(X_train, initial_idx, axis=0)[:5000] +y_pool = np.delete(y_train, initial_idx, axis=0)[:5000] + + +# initialize ActiveLearner (Pass to him the skorch wrapped PyTorch model & the Query strategy) +learner = DeepActiveLearner( + estimator=classifier, + query_strategy=mc_dropout_bald, +) +# initial teaching if desired (not necessary) +learner.teach(X_initial, y_initial) + +print("Score from sklearn: {}".format(learner.score(X_pool, y_pool))) + + +# the active learning loop +n_queries = 10 +X_teach = X_initial +y_teach = y_initial + + +for idx in range(n_queries): + print('Query no. %d' % (idx + 1)) + """ + Query new data (num_cycles are the number of dropout forward passes that should be performed) + --> check the documentation of mc_dropout_bald in modAL/dropout.py to see all available parameters + """ + query_idx, metric_values = learner.query( + X_pool, n_instances=100, dropout_layer_indexes=[7, 11], num_cycles=10) + # Add queried instances + X_teach = torch.cat((X_teach, X_pool[query_idx])) + y_teach = torch.cat((y_teach, y_pool[query_idx])) + learner.teach(X_teach, y_teach) + + # remove queried instance from pool + X_pool = np.delete(X_pool, query_idx, axis=0) + y_pool = np.delete(y_pool, query_idx, axis=0) + + # give us the model performance + print("Model score: {}".format(learner.score(X_test, y_test))) diff --git a/examples/query_by_committee.py b/examples/query_by_committee.py index 746df00..439076d 100644 --- a/examples/query_by_committee.py +++ b/examples/query_by_committee.py @@ -1,10 +1,11 @@ -import numpy as np -import matplotlib.pyplot as plt from copy import deepcopy -from sklearn.decomposition import PCA + +import matplotlib.pyplot as plt +import numpy as np +from modAL.models import ActiveLearner, Committee from sklearn.datasets import load_iris +from sklearn.decomposition import PCA from sklearn.ensemble import RandomForestClassifier -from modAL.models import ActiveLearner, Committee # loading the iris dataset iris = load_iris() diff --git a/examples/ranked_batch_mode.py b/examples/ranked_batch_mode.py index e9256d5..d9e4ca8 100644 --- a/examples/ranked_batch_mode.py +++ b/examples/ranked_batch_mode.py @@ -1,13 +1,13 @@ -import numpy as np +from functools import partial + import matplotlib as mpl import matplotlib.pyplot as plt +import numpy as np +from modAL.batch import uncertainty_batch_sampling +from modAL.models import ActiveLearner from sklearn.datasets import load_iris from sklearn.decomposition import PCA from sklearn.neighbors import KNeighborsClassifier -from functools import partial - -from modAL.batch import uncertainty_batch_sampling -from modAL.models import ActiveLearner # Set our RNG for reproducibility. RANDOM_STATE_SEED = 123 @@ -161,4 +161,4 @@ )) ax.legend(loc='lower right') - plt.show() \ No newline at end of file + plt.show() diff --git a/examples/runtime_comparison.py b/examples/runtime_comparison.py index f6fdf13..551396e 100644 --- a/examples/runtime_comparison.py +++ b/examples/runtime_comparison.py @@ -1,25 +1,20 @@ -import numpy as np - from time import time -from sklearn.datasets import load_iris - +import numpy as np from acton.acton import main as acton_main - -from alp.active_learning.active_learning import ActiveLearner as ActiveLearnerALP - +from alp.active_learning.active_learning import \ + ActiveLearner as ActiveLearnerALP from libact.base.dataset import Dataset from libact.labelers import IdealLabeler -from libact.query_strategies import UncertaintySampling, QueryByCommittee +from libact.models.logistic_regression import \ + LogisticRegression as LogisticRegressionLibact +from libact.query_strategies import QueryByCommittee, UncertaintySampling from libact.query_strategies.multiclass.expected_error_reduction import EER -from libact.models.logistic_regression import LogisticRegression as LogisticRegressionLibact - -from modAL.models import ActiveLearner, Committee from modAL.expected_error import expected_error_reduction - +from modAL.models import ActiveLearner, Committee +from sklearn.datasets import load_iris from sklearn.linear_model import LogisticRegression - runtime = {} diff --git a/examples/shape_learning.py b/examples/shape_learning.py index c190e24..87b44dd 100644 --- a/examples/shape_learning.py +++ b/examples/shape_learning.py @@ -5,11 +5,12 @@ the scikit-learn implementation of the kNN classifier algorithm. """ -import numpy as np -import matplotlib.pyplot as plt from copy import deepcopy -from sklearn.ensemble import RandomForestClassifier + +import matplotlib.pyplot as plt +import numpy as np from modAL.models import ActiveLearner +from sklearn.ensemble import RandomForestClassifier # creating the image im_width = 500 @@ -57,8 +58,7 @@ def random_sampling(classsifier, X): - query_idx = np.random.rand(range(len(X))) - return query_idx, X[query_idx] + return np.random.randint(len(X)) X_pool = deepcopy(X_full) @@ -100,4 +100,4 @@ def random_sampling(classsifier, X): plt.plot(list(range(len(uncertainty_sampling_accuracy))), uncertainty_sampling_accuracy, label="uncertainty sampling") plt.plot(list(range(len(random_sampling_accuracy))), random_sampling_accuracy, label="random sampling") plt.legend() - plt.show() \ No newline at end of file + plt.show() diff --git a/examples/sklearn_workflow.py b/examples/sklearn_workflow.py index 12b175b..247ad1c 100644 --- a/examples/sklearn_workflow.py +++ b/examples/sklearn_workflow.py @@ -1,9 +1,9 @@ from modAL.models import ActiveLearner -from sklearn.ensemble import RandomForestClassifier from sklearn.datasets import load_iris +from sklearn.ensemble import RandomForestClassifier from sklearn.model_selection import cross_val_score X_train, y_train = load_iris().data, load_iris().target learner = ActiveLearner(estimator=RandomForestClassifier()) -scores = cross_val_score(learner, X_train, y_train, cv=10) \ No newline at end of file +scores = cross_val_score(learner, X_train, y_train, cv=10) diff --git a/examples/stream-based_sampling.py b/examples/stream-based_sampling.py index 95a8082..7a6dd77 100644 --- a/examples/stream-based_sampling.py +++ b/examples/stream-based_sampling.py @@ -1,12 +1,12 @@ """ -In this example the use of ActiveLearner is demonstrated in a pool-based sampling setting. +In this example the use of ActiveLearner is demonstrated in a stream-based sampling setting. """ -import numpy as np import matplotlib.pyplot as plt -from sklearn.ensemble import RandomForestClassifier +import numpy as np from modAL.models import ActiveLearner from modAL.uncertainty import classifier_uncertainty +from sklearn.ensemble import RandomForestClassifier # creating the image im_width = 500 @@ -67,5 +67,5 @@ plt.figure(figsize=(7, 7)) prediction = learner.predict_proba(X_full)[:, 1] plt.imshow(prediction.reshape(im_width, im_height)) - plt.title('Initial prediction accuracy: %f' % learner.score(X_full, y_full)) + plt.title('Final prediction accuracy: %f' % learner.score(X_full, y_full)) plt.show() diff --git a/modAL/__init__.py b/modAL/__init__.py index b34800b..4231550 100644 --- a/modAL/__init__.py +++ b/modAL/__init__.py @@ -1,3 +1,3 @@ from .models import ActiveLearner, Committee, CommitteeRegressor -__all__ = ['ActiveLearner', 'Committee', 'CommitteeRegressor'] \ No newline at end of file +__all__ = ['ActiveLearner', 'Committee', 'CommitteeRegressor'] diff --git a/modAL/acquisition.py b/modAL/acquisition.py index 54792d3..8aa1fac 100644 --- a/modAL/acquisition.py +++ b/modAL/acquisition.py @@ -1,16 +1,15 @@ """ Acquisition functions for Bayesian optimization. """ -from typing import Tuple import numpy as np -from scipy.stats import norm from scipy.special import ndtr +from scipy.stats import norm from sklearn.exceptions import NotFittedError -from modAL.utils.selection import multi_argmax -from modAL.utils.data import modALinput from modAL.models.base import BaseLearner +from modAL.utils.data import modALinput +from modAL.utils.selection import multi_argmax def PI(mean, std, max_val, tradeoff): @@ -47,7 +46,7 @@ def optimizer_PI(optimizer: BaseLearner, X: modALinput, tradeoff: float = 0) -> """ try: mean, std = optimizer.predict(X, return_std=True) - std = std.reshape(-1, 1) + mean, std = mean.reshape(-1, ), std.reshape(-1, ) except NotFittedError: mean, std = np.zeros(shape=(X.shape[0], 1)), np.ones(shape=(X.shape[0], 1)) @@ -68,7 +67,7 @@ def optimizer_EI(optimizer: BaseLearner, X: modALinput, tradeoff: float = 0) -> """ try: mean, std = optimizer.predict(X, return_std=True) - std = std.reshape(-1, 1) + mean, std = mean.reshape(-1, ), std.reshape(-1, ) except NotFittedError: mean, std = np.zeros(shape=(X.shape[0], 1)), np.ones(shape=(X.shape[0], 1)) @@ -89,7 +88,7 @@ def optimizer_UCB(optimizer: BaseLearner, X: modALinput, beta: float = 1) -> np. """ try: mean, std = optimizer.predict(X, return_std=True) - std = std.reshape(-1, 1) + mean, std = mean.reshape(-1, ), std.reshape(-1, ) except NotFittedError: mean, std = np.zeros(shape=(X.shape[0], 1)), np.ones(shape=(X.shape[0], 1)) @@ -104,7 +103,7 @@ def optimizer_UCB(optimizer: BaseLearner, X: modALinput, beta: float = 1) -> np. def max_PI(optimizer: BaseLearner, X: modALinput, tradeoff: float = 0, - n_instances: int = 1) -> Tuple[np.ndarray, modALinput]: + n_instances: int = 1) -> np.ndarray: """ Maximum PI query strategy. Selects the instance with highest probability of improvement. @@ -115,16 +114,16 @@ def max_PI(optimizer: BaseLearner, X: modALinput, tradeoff: float = 0, n_instances: Number of samples to be queried. Returns: - The indices of the instances from X chosen to be labelled; the instances from X chosen to be labelled. + The indices of the instances from X chosen to be labelled. + The pi metric of the chosen instances. + """ pi = optimizer_PI(optimizer, X, tradeoff=tradeoff) - query_idx = multi_argmax(pi, n_instances=n_instances) - - return query_idx, X[query_idx] + return multi_argmax(pi, n_instances=n_instances) def max_EI(optimizer: BaseLearner, X: modALinput, tradeoff: float = 0, - n_instances: int = 1) -> Tuple[np.ndarray, modALinput]: + n_instances: int = 1) -> np.ndarray: """ Maximum EI query strategy. Selects the instance with highest expected improvement. @@ -135,16 +134,16 @@ def max_EI(optimizer: BaseLearner, X: modALinput, tradeoff: float = 0, n_instances: Number of samples to be queried. Returns: - The indices of the instances from X chosen to be labelled; the instances from X chosen to be labelled. + The indices of the instances from X chosen to be labelled. + The ei metric of the chosen instances. + """ ei = optimizer_EI(optimizer, X, tradeoff=tradeoff) - query_idx = multi_argmax(ei, n_instances=n_instances) - - return query_idx, X[query_idx] + return multi_argmax(ei, n_instances=n_instances) def max_UCB(optimizer: BaseLearner, X: modALinput, beta: float = 1, - n_instances: int = 1) -> Tuple[np.ndarray, modALinput]: + n_instances: int = 1) -> np.ndarray: """ Maximum UCB query strategy. Selects the instance with highest upper confidence bound. @@ -155,9 +154,9 @@ def max_UCB(optimizer: BaseLearner, X: modALinput, beta: float = 1, n_instances: Number of samples to be queried. Returns: - The indices of the instances from X chosen to be labelled; the instances from X chosen to be labelled. + The indices of the instances from X chosen to be labelled. + The ucb metric of the chosen instances. + """ ucb = optimizer_UCB(optimizer, X, beta=beta) - query_idx = multi_argmax(ucb, n_instances=n_instances) - - return query_idx, X[query_idx] + return multi_argmax(ucb, n_instances=n_instances) diff --git a/modAL/batch.py b/modAL/batch.py index be931bc..d85afed 100644 --- a/modAL/batch.py +++ b/modAL/batch.py @@ -6,16 +6,17 @@ import numpy as np import scipy.sparse as sp -from sklearn.metrics.pairwise import pairwise_distances, pairwise_distances_argmin_min +from sklearn.metrics.pairwise import (pairwise_distances, + pairwise_distances_argmin_min) -from modAL.utils.data import data_vstack, modALinput from modAL.models.base import BaseCommittee, BaseLearner from modAL.uncertainty import classifier_uncertainty +from modAL.utils.data import data_shape, data_vstack, modALinput def select_cold_start_instance(X: modALinput, metric: Union[str, Callable], - n_jobs: Union[int, None]) -> modALinput: + n_jobs: Union[int, None]) -> Tuple[int, modALinput]: """ Define what to do if our batch-mode sampling doesn't have any labeled data -- a cold start. @@ -35,7 +36,8 @@ def select_cold_start_instance(X: modALinput, n_jobs: This parameter is passed to :func:`~sklearn.metrics.pairwise.pairwise_distances`. Returns: - Best instance for cold-start. + Index of the best cold-start instance from `X` chosen to be labelled; record of the best cold-start instance + from `X` chosen to be labelled. """ # Compute all pairwise distances in our unlabeled data and obtain the row-wise average for each of our records in X. n_jobs = n_jobs if n_jobs else 1 @@ -43,7 +45,7 @@ def select_cold_start_instance(X: modALinput, # Isolate and return our best instance for labeling as the record with the least average distance. best_coldstart_instance_index = np.argmin(average_distances) - return X[best_coldstart_instance_index].reshape(1, -1) + return best_coldstart_instance_index, X[best_coldstart_instance_index].reshape(1, -1) def select_instance( @@ -78,9 +80,11 @@ def select_instance( Index of the best index from X chosen to be labelled; a single record from our unlabeled set that is considered the most optimal incremental record for including in our query set. """ + X_pool_masked = X_pool[mask] + # Extract the number of labeled and unlabeled records. - n_labeled_records, _ = X_training.shape - n_unlabeled, _ = X_pool[mask].shape + n_labeled_records, *rest = X_training.shape + n_unlabeled, *rest = X_pool_masked.shape # Determine our alpha parameter as |U| / (|U| + |D|). Note that because we # append to X_training and remove from X_pool within `ranked_batch`, @@ -89,10 +93,15 @@ def select_instance( # Compute pairwise distance (and then similarity) scores from every unlabeled record # to every record in X_training. The result is an array of shape (n_samples, ). + if n_jobs == 1 or n_jobs is None: - _, distance_scores = pairwise_distances_argmin_min(X_pool[mask], X_training, metric=metric) + _, distance_scores = pairwise_distances_argmin_min(X_pool_masked.reshape(n_unlabeled, -1), + X_training.reshape(n_labeled_records, -1), + metric=metric) else: - distance_scores = pairwise_distances(X_pool[mask], X_training, metric=metric, n_jobs=n_jobs).min(axis=1) + distance_scores = pairwise_distances(X_pool_masked.reshape(n_unlabeled, -1), + X_training.reshape(n_labeled_records, -1), + metric=metric, n_jobs=n_jobs).min(axis=1) similarity_scores = 1 / (1 + distance_scores) @@ -101,9 +110,12 @@ def select_instance( scores = alpha * (1 - similarity_scores) + (1 - alpha) * X_uncertainty[mask] # Isolate and return our best instance for labeling as the one with the largest score. - best_instance_index = np.argmax(scores) + best_instance_index_in_unlabeled = np.argmax(scores) + n_pool, *rest = X_pool.shape + unlabeled_indices = [i for i in range(n_pool) if mask[i]] + best_instance_index = unlabeled_indices[best_instance_index_in_unlabeled] mask[best_instance_index] = 0 - return best_instance_index, X_pool[best_instance_index].reshape(1, -1), mask + return best_instance_index, X_pool[[best_instance_index]], mask def ranked_batch(classifier: Union[BaseLearner, BaseCommittee], @@ -128,19 +140,30 @@ def ranked_batch(classifier: Union[BaseLearner, BaseCommittee], Returns: The indices of the top n_instances ranked unlabelled samples. + The uncertainty scores of the chosen instances. + """ # Make a local copy of our classifier's training data. - if classifier.X_training is None: - labeled = select_cold_start_instance(X=unlabeled, metric=metric, n_jobs=n_jobs) - elif classifier.X_training.shape[0] > 0: - labeled = classifier.X_training[:] + # Define our record container and record the best cold start instance in the case of cold start. - # Define our record container and the maximum number of records to sample. - instance_index_ranking = [] - ceiling = np.minimum(unlabeled.shape[0], n_instances) + # transform unlabeled data if needed + if classifier.on_transformed: + unlabeled = classifier.transform_without_estimating(unlabeled) + + if classifier.X_training is None: + best_coldstart_instance_index, labeled = select_cold_start_instance(X=unlabeled, metric=metric, n_jobs=n_jobs) + instance_index_ranking = [best_coldstart_instance_index] + elif data_shape(classifier.X_training)[0] > 0: + labeled = classifier.transform_without_estimating( + classifier.X_training + ) if classifier.on_transformed else classifier.X_training[:] + instance_index_ranking = [] + + # The maximum number of records to sample. + ceiling = np.minimum(unlabeled.shape[0], n_instances) - len(instance_index_ranking) # mask for unlabeled initialized as transparent - mask = np.ones(unlabeled.shape[0], np.bool) + mask = np.ones(unlabeled.shape[0], bool) for _ in range(ceiling): @@ -158,7 +181,7 @@ def ranked_batch(classifier: Union[BaseLearner, BaseCommittee], instance_index_ranking.append(instance_index) # Return numpy array, not a list. - return np.array(instance_index_ranking) + return np.array(instance_index_ranking), uncertainty_scores[np.array(instance_index_ranking)] def uncertainty_batch_sampling(classifier: Union[BaseLearner, BaseCommittee], @@ -167,7 +190,7 @@ def uncertainty_batch_sampling(classifier: Union[BaseLearner, BaseCommittee], metric: Union[str, Callable] = 'euclidean', n_jobs: Optional[int] = None, **uncertainty_measure_kwargs - ) -> Tuple[np.ndarray, Union[np.ndarray, sp.csr_matrix]]: + ) -> np.ndarray: """ Batch sampling query strategy. Selects the least sure instances for labelling. @@ -190,9 +213,12 @@ def uncertainty_batch_sampling(classifier: Union[BaseLearner, BaseCommittee], **uncertainty_measure_kwargs: Keyword arguments to be passed for the :meth:`predict_proba` of the classifier. Returns: - Indices of the instances from `X` chosen to be labelled; records from `X` chosen to be labelled. + Indices of the instances from `X` chosen to be labelled + Records from `X` chosen to be labelled. + The uncertainty scores of the chosen instances. + """ uncertainty = classifier_uncertainty(classifier, X, **uncertainty_measure_kwargs) - query_indices = ranked_batch(classifier, unlabeled=X, uncertainty_scores=uncertainty, + return ranked_batch(classifier, unlabeled=X, uncertainty_scores=uncertainty, n_instances=n_instances, metric=metric, n_jobs=n_jobs) - return query_indices, X[query_indices] + diff --git a/modAL/disagreement.py b/modAL/disagreement.py index 04e1f12..22430b4 100644 --- a/modAL/disagreement.py +++ b/modAL/disagreement.py @@ -2,16 +2,15 @@ Disagreement measures and disagreement based query strategies for the Committee model. """ from collections import Counter -from typing import Tuple import numpy as np from scipy.stats import entropy -from sklearn.exceptions import NotFittedError from sklearn.base import BaseEstimator +from sklearn.exceptions import NotFittedError +from modAL.models.base import BaseCommittee from modAL.utils.data import modALinput from modAL.utils.selection import multi_argmax, shuffled_argmax -from modAL.models.base import BaseCommittee def vote_entropy(committee: BaseCommittee, X: modALinput, **predict_proba_kwargs) -> np.ndarray: @@ -35,7 +34,6 @@ def vote_entropy(committee: BaseCommittee, X: modALinput, **predict_proba_kwargs return np.zeros(shape=(X.shape[0],)) p_vote = np.zeros(shape=(X.shape[0], len(committee.classes_))) - entr = np.zeros(shape=(X.shape[0],)) for vote_idx, vote in enumerate(votes): vote_counter = Counter(vote) @@ -43,8 +41,7 @@ def vote_entropy(committee: BaseCommittee, X: modALinput, **predict_proba_kwargs for class_idx, class_label in enumerate(committee.classes_): p_vote[vote_idx, class_idx] = vote_counter[class_label]/n_learners - entr[vote_idx] = entropy(p_vote[vote_idx]) - + entr = entropy(p_vote, axis=1) return entr @@ -104,7 +101,7 @@ def KL_max_disagreement(committee: BaseCommittee, X: modALinput, **predict_proba def vote_entropy_sampling(committee: BaseCommittee, X: modALinput, n_instances: int = 1, random_tie_break=False, - **disagreement_measure_kwargs) -> Tuple[np.ndarray, modALinput]: + **disagreement_measure_kwargs) -> np.ndarray: """ Vote entropy sampling strategy. @@ -118,22 +115,21 @@ def vote_entropy_sampling(committee: BaseCommittee, X: modALinput, measure function. Returns: - The indices of the instances from X chosen to be labelled; - the instances from X chosen to be labelled. + The indices of the instances from X chosen to be labelled. + The disagrerment metric of the chosen instances. + """ disagreement = vote_entropy(committee, X, **disagreement_measure_kwargs) if not random_tie_break: - query_idx = multi_argmax(disagreement, n_instances=n_instances) - else: - query_idx = shuffled_argmax(disagreement, n_instances=n_instances) + return multi_argmax(disagreement, n_instances=n_instances) - return query_idx, X[query_idx] + return shuffled_argmax(disagreement, n_instances=n_instances) def consensus_entropy_sampling(committee: BaseCommittee, X: modALinput, n_instances: int = 1, random_tie_break=False, - **disagreement_measure_kwargs) -> Tuple[np.ndarray, modALinput]: + **disagreement_measure_kwargs) -> np.ndarray: """ Consensus entropy sampling strategy. @@ -147,22 +143,21 @@ def consensus_entropy_sampling(committee: BaseCommittee, X: modALinput, measure function. Returns: - The indices of the instances from X chosen to be labelled; - the instances from X chosen to be labelled. + The indices of the instances from X chosen to be labelled. + The disagrerment metric of the chosen instances. + """ disagreement = consensus_entropy(committee, X, **disagreement_measure_kwargs) if not random_tie_break: - query_idx = multi_argmax(disagreement, n_instances=n_instances) - else: - query_idx = shuffled_argmax(disagreement, n_instances=n_instances) + return multi_argmax(disagreement, n_instances=n_instances) - return query_idx, X[query_idx] + return shuffled_argmax(disagreement, n_instances=n_instances) def max_disagreement_sampling(committee: BaseCommittee, X: modALinput, n_instances: int = 1, random_tie_break=False, - **disagreement_measure_kwargs) -> Tuple[np.ndarray, modALinput]: + **disagreement_measure_kwargs) -> np.ndarray: """ Maximum disagreement sampling strategy. @@ -176,22 +171,21 @@ def max_disagreement_sampling(committee: BaseCommittee, X: modALinput, measure function. Returns: - The indices of the instances from X chosen to be labelled; - the instances from X chosen to be labelled. + The indices of the instances from X chosen to be labelled. + The disagrerment metric of the chosen instances. + """ disagreement = KL_max_disagreement(committee, X, **disagreement_measure_kwargs) if not random_tie_break: - query_idx = multi_argmax(disagreement, n_instances=n_instances) - else: - query_idx = shuffled_argmax(disagreement, n_instances=n_instances) + return multi_argmax(disagreement, n_instances=n_instances) - return query_idx, X[query_idx] + return shuffled_argmax(disagreement, n_instances=n_instances) def max_std_sampling(regressor: BaseEstimator, X: modALinput, n_instances: int = 1, random_tie_break=False, - **predict_kwargs) -> Tuple[np.ndarray, modALinput]: + **predict_kwargs) -> np.ndarray: """ Regressor standard deviation sampling strategy. @@ -204,15 +198,14 @@ def max_std_sampling(regressor: BaseEstimator, X: modALinput, **predict_kwargs: Keyword arguments to be passed to :meth:`predict` of the CommiteeRegressor. Returns: - The indices of the instances from X chosen to be labelled; - the instances from X chosen to be labelled. + The indices of the instances from X chosen to be labelled. + The standard deviation of the chosen instances. + """ _, std = regressor.predict(X, return_std=True, **predict_kwargs) std = std.reshape(X.shape[0], ) if not random_tie_break: - query_idx = multi_argmax(std, n_instances=n_instances) - else: - query_idx = shuffled_argmax(std, n_instances=n_instances) + return multi_argmax(std, n_instances=n_instances) - return query_idx, X[query_idx] + return shuffled_argmax(std, n_instances=n_instances) diff --git a/modAL/dropout.py b/modAL/dropout.py new file mode 100644 index 0000000..c6c9cb1 --- /dev/null +++ b/modAL/dropout.py @@ -0,0 +1,436 @@ +from collections.abc import Mapping +from typing import Callable + +import numpy as np +import torch +from scipy.special import entr +from sklearn.base import BaseEstimator +from skorch.utils import to_numpy + +from modAL.utils.data import modALinput +from modAL.utils.selection import multi_argmax, shuffled_argmax + + +def default_logits_adaptor(input_tensor: torch.tensor, samples: modALinput): + # default Callable parameter for get_predictions + return input_tensor + + +def mc_dropout_bald(classifier: BaseEstimator, X: modALinput, n_instances: int = 1, + random_tie_break: bool = False, dropout_layer_indexes: list = [], + num_cycles: int = 50, sample_per_forward_pass: int = 1000, + logits_adaptor: Callable[[ + torch.tensor, modALinput], torch.tensor] = default_logits_adaptor, + **mc_dropout_kwargs,) -> np.ndarray: + """ + Mc-Dropout bald query strategy. Returns the indexes of the instances with the largest BALD + (Bayesian Active Learning by Disagreement) score calculated through the dropout cycles + and the corresponding bald score. + + Based on the work of: + Deep Bayesian Active Learning with Image Data. + (Yarin Gal, Riashat Islam, and Zoubin Ghahramani. 2017.) + Dropout as a Bayesian Approximation: Representing Model Uncer- tainty in Deep Learning. + (Yarin Gal and Zoubin Ghahramani. 2016.) + Bayesian Active Learning for Classification and Preference Learning. + (NeilHoulsby,FerencHusza ́r,ZoubinGhahramani,andMa ́te ́Lengyel. 2011.) + + Args: + classifier: The classifier for which the labels are to be queried. + X: The pool of samples to query from. + n_instances: Number of samples to be queried. + random_tie_break: If True, shuffles utility scores to randomize the order. This + can be used to break the tie when the highest utility score is not unique. + dropout_layer_indexes: Indexes of the dropout layers which should be activated + Choose indices from : list(torch_model.modules()) + num_cycles: Number of forward passes with activated dropout + sample_per_forward_pass: max. sample number for each forward pass. + The allocated RAM does mainly depend on this. + Small number --> small RAM allocation + logits_adaptor: Callable which can be used to adapt the output of a forward pass + to the required vector format for the vectorised metric functions + **uncertainty_measure_kwargs: Keyword arguments to be passed for the uncertainty + measure function. + + Returns: + The indices of the instances from X chosen to be labelled; + The mc-dropout metric of the chosen instances; + """ + predictions = get_predictions( + classifier, X, dropout_layer_indexes, num_cycles, sample_per_forward_pass, logits_adaptor) + # calculate BALD (Bayesian active learning divergence)) + + bald_scores = _bald_divergence(predictions) + + if not random_tie_break: + return multi_argmax(bald_scores, n_instances=n_instances) + + return shuffled_argmax(bald_scores, n_instances=n_instances) + + +def mc_dropout_mean_st(classifier: BaseEstimator, X: modALinput, n_instances: int = 1, + random_tie_break: bool = False, dropout_layer_indexes: list = [], + num_cycles: int = 50, sample_per_forward_pass: int = 1000, + logits_adaptor: Callable[[ + torch.tensor, modALinput], torch.tensor] = default_logits_adaptor, + **mc_dropout_kwargs) -> np.ndarray: + """ + Mc-Dropout mean standard deviation query strategy. Returns the indexes of the instances + with the largest mean of the per class calculated standard deviations over multiple dropout cycles + and the corresponding metric. + + Based on the equations of: + Deep Bayesian Active Learning with Image Data. + (Yarin Gal, Riashat Islam, and Zoubin Ghahramani. 2017.) + + Args: + classifier: The classifier for which the labels are to be queried. + X: The pool of samples to query from. + n_instances: Number of samples to be queried. + random_tie_break: If True, shuffles utility scores to randomize the order. This + can be used to break the tie when the highest utility score is not unique. + dropout_layer_indexes: Indexes of the dropout layers which should be activated + Choose indices from : list(torch_model.modules()) + num_cycles: Number of forward passes with activated dropout + sample_per_forward_pass: max. sample number for each forward pass. + The allocated RAM does mainly depend on this. + Small number --> small RAM allocation + logits_adaptor: Callable which can be used to adapt the output of a forward pass + to the required vector format for the vectorised metric functions + **uncertainty_measure_kwargs: Keyword arguments to be passed for the uncertainty + measure function. + + Returns: + The indices of the instances from X chosen to be labelled; + The mc-dropout metric of the chosen instances; + """ + + # set dropout layers to train mode + predictions = get_predictions( + classifier, X, dropout_layer_indexes, num_cycles, sample_per_forward_pass, logits_adaptor) + + mean_standard_deviations = _mean_standard_deviation(predictions) + + if not random_tie_break: + return multi_argmax(mean_standard_deviations, n_instances=n_instances) + + return shuffled_argmax(mean_standard_deviations, n_instances=n_instances) + + +def mc_dropout_max_entropy(classifier: BaseEstimator, X: modALinput, n_instances: int = 1, + random_tie_break: bool = False, dropout_layer_indexes: list = [], + num_cycles: int = 50, sample_per_forward_pass: int = 1000, + logits_adaptor: Callable[[ + torch.tensor, modALinput], torch.tensor] = default_logits_adaptor, + **mc_dropout_kwargs) -> np.ndarray: + """ + Mc-Dropout maximum entropy query strategy. Returns the indexes of the instances + with the largest entropy of the per class calculated entropies over multiple dropout cycles + and the corresponding metric. + + Based on the equations of: + Deep Bayesian Active Learning with Image Data. + (Yarin Gal, Riashat Islam, and Zoubin Ghahramani. 2017.) + + Args: + classifier: The classifier for which the labels are to be queried. + X: The pool of samples to query from. + n_instances: Number of samples to be queried. + random_tie_break: If True, shuffles utility scores to randomize the order. This + can be used to break the tie when the highest utility score is not unique. + dropout_layer_indexes: Indexes of the dropout layers which should be activated + Choose indices from : list(torch_model.modules()) + num_cycles: Number of forward passes with activated dropout + sample_per_forward_pass: max. sample number for each forward pass. + The allocated RAM does mainly depend on this. + Small number --> small RAM allocation + logits_adaptor: Callable which can be used to adapt the output of a forward pass + to the required vector format for the vectorised metric functions + **uncertainty_measure_kwargs: Keyword arguments to be passed for the uncertainty + measure function. + + Returns: + The indices of the instances from X chosen to be labelled; + The mc-dropout metric of the chosen instances; + """ + predictions = get_predictions( + classifier, X, dropout_layer_indexes, num_cycles, sample_per_forward_pass, logits_adaptor) + + # get entropy values for predictions + entropy = _entropy(predictions) + + if not random_tie_break: + return multi_argmax(entropy, n_instances=n_instances) + + return shuffled_argmax(entropy, n_instances=n_instances) + + +def mc_dropout_max_variationRatios(classifier: BaseEstimator, X: modALinput, n_instances: int = 1, + random_tie_break: bool = False, dropout_layer_indexes: list = [], + num_cycles: int = 50, sample_per_forward_pass: int = 1000, + logits_adaptor: Callable[[ + torch.tensor, modALinput], torch.tensor] = default_logits_adaptor, + **mc_dropout_kwargs) -> np.ndarray: + """ + Mc-Dropout maximum variation ratios query strategy. Returns the indexes of the instances + with the largest variation ratios over multiple dropout cycles + and the corresponding metric. + + Based on the equations of: + Deep Bayesian Active Learning with Image Data. + (Yarin Gal, Riashat Islam, and Zoubin Ghahramani. 2017.) + + Args: + classifier: The classifier for which the labels are to be queried. + X: The pool of samples to query from. + n_instances: Number of samples to be queried. + random_tie_break: If True, shuffles utility scores to randomize the order. This + can be used to break the tie when the highest utility score is not unique. + dropout_layer_indexes: Indexes of the dropout layers which should be activated + Choose indices from : list(torch_model.modules()) + num_cycles: Number of forward passes with activated dropout + sample_per_forward_pass: max. sample number for each forward pass. + The allocated RAM does mainly depend on this. + Small number --> small RAM allocation + logits_adaptor: Callable which can be used to adapt the output of a forward pass + to the required vector format for the vectorised metric functions + **uncertainty_measure_kwargs: Keyword arguments to be passed for the uncertainty + measure function. + + Returns: + The indices of the instances from X chosen to be labelled; + The mc-dropout metric of the chosen instances; + """ + predictions = get_predictions( + classifier, X, dropout_layer_indexes, num_cycles, sample_per_forward_pass, logits_adaptor) + + # get variation ratios values for predictions + variationRatios = _variation_ratios(predictions) + + if not random_tie_break: + return multi_argmax(variationRatios, n_instances=n_instances) + + return shuffled_argmax(variationRatios, n_instances=n_instances) + + +def get_predictions(classifier: BaseEstimator, X: modALinput, dropout_layer_indexes: list = [], + num_predictions: int = 50, sample_per_forward_pass: int = 1000, + logits_adaptor: Callable[[torch.tensor, modALinput], torch.tensor] = default_logits_adaptor): + """ + Runs num_predictions times the prediction of the classifier on the input X + and puts the predictions in a list. + + Args: + classifier: The classifier for which the labels are to be queried. + X: The pool of samples to query from. + dropout_layer_indexes: Indexes of the dropout layers which should be activated + Choose indices from : list(torch_model.modules()) + num_predictions: Number of predictions which should be made + sample_per_forward_pass: max. sample number for each forward pass. + The allocated RAM does mainly depend on this. + Small number --> small RAM allocation + logits_adaptor: Callable which can be used to adapt the output of a forward pass + to the required vector format for the vectorised metric functions + Return: + prediction: list with all predictions + """ + + assert num_predictions > 0, 'num_predictions must be larger than zero' + assert sample_per_forward_pass > 0, 'sample_per_forward_pass must be larger than zero' + + predictions = [] + # set dropout layers to train mode + set_dropout_mode(classifier.estimator.module_, + dropout_layer_indexes, train_mode=True) + + split_args = [] + + if isinstance(X, Mapping): # check for dict + for k, v in X.items(): + + v.detach() + split_v = torch.split(v, sample_per_forward_pass) + # create sub-dictionary split for each forward pass with same keys&values + for split_idx, split in enumerate(split_v): + if len(split_args) <= split_idx: + split_args.append({}) + split_args[split_idx][k] = split + + elif torch.is_tensor(X): # check for tensor + X.detach() + split_args = torch.split(X, sample_per_forward_pass) + else: + raise RuntimeError( + "Error in model data type, only dict or tensors supported") + + for i in range(num_predictions): + + probas = [] + + for samples in split_args: + # call Skorch infer function to perform model forward pass + # In comparison to: predict(), predict_proba() the infer() + # does not change train/eval mode of other layers + with torch.no_grad(): + logits = classifier.estimator.infer(samples) + prediction = logits_adaptor(logits, samples) + mask = ~prediction.isnan() + prediction[mask] = prediction[mask].softmax(-1) + probas.append(prediction) + + probas = torch.cat(probas) + predictions.append(to_numpy(probas)) + + # set dropout layers to eval + set_dropout_mode(classifier.estimator.module_, + dropout_layer_indexes, train_mode=False) + + return predictions + + +def entropy_sum(values: np.array, axis: int = -1): + # sum Scipy basic entropy function: entr() + entropy = entr(values) + return np.sum(entropy, where=~np.isnan(entropy), axis=axis) + + +def _mean_standard_deviation(proba: list) -> np.ndarray: + """ + Calculates the mean of the per class calculated standard deviations. + + As it is explicitly formulated in: + Deep Bayesian Active Learning with Image Data. + (Yarin Gal, Riashat Islam, and Zoubin Ghahramani. 2017.) + + Args: + proba: list with the predictions over the dropout cycles + mask: mask to detect the padded classes (must be of same shape as elements in proba) + Return: + Returns the mean standard deviation of the dropout cycles over all classes. + """ + + proba_stacked = np.stack(proba, axis=len(proba[0].shape)) + + standard_deviation_class_vise = np.std(proba_stacked, axis=-1) + mean_standard_deviation = np.mean(standard_deviation_class_vise, where=~np.isnan( + standard_deviation_class_vise), axis=-1) + + return mean_standard_deviation + + +def _entropy(proba: list) -> np.ndarray: + """ + Calculates the entropy per class over dropout cycles + + As it is explicitly formulated in: + Deep Bayesian Active Learning with Image Data. + (Yarin Gal, Riashat Islam, and Zoubin Ghahramani. 2017.) + + Args: + proba: list with the predictions over the dropout cycles + mask: mask to detect the padded classes (must be of same shape as elements in proba) + Return: + Returns the entropy of the dropout cycles over all classes. + """ + + proba_stacked = np.stack(proba, axis=len(proba[0].shape)) + + # calculate entropy per class and sum along dropout cycles + entropy_classes = entropy_sum(proba_stacked, axis=-1) + entropy = np.mean(entropy_classes, where=~ + np.isnan(entropy_classes), axis=-1) + return entropy + + +def _variation_ratios(proba: list) -> np.ndarray: + """ + Calculates the variation ratios over dropout cycles + + As it is explicitly formulated in: + Deep Bayesian Active Learning with Image Data. + (Yarin Gal, Riashat Islam, and Zoubin Ghahramani. 2017.) + + Args: + proba: list with the predictions over the dropout cycles + mask: mask to detect the padded classes (must be of same shape as elements in proba) + Return: + Returns the variation ratios of the dropout cycles. + """ + proba_stacked = np.stack(proba, axis=len(proba[0].shape)) + + # Calculate the variation ratios over the mean of dropout cycles + valuesDCMean = np.mean(proba_stacked, axis=-1) + return 1 - np.amax(valuesDCMean, initial=0, where=~np.isnan(valuesDCMean), axis=-1) + + +def _bald_divergence(proba: list) -> np.ndarray: + """ + Calculates the bald divergence for each instance + + As it is explicitly formulated in: + Deep Bayesian Active Learning with Image Data. + (Yarin Gal, Riashat Islam, and Zoubin Ghahramani. 2017.) + + Args: + proba: list with the predictions over the dropout cycles + mask: mask to detect the padded classes (must be of same shape as elements in proba) + Return: + Returns the mean standard deviation of the dropout cycles over all classes. + """ + proba_stacked = np.stack(proba, axis=len(proba[0].shape)) + + # entropy along dropout cycles + accumulated_entropy = entropy_sum(proba_stacked, axis=-1) + f_x = accumulated_entropy/len(proba) + + # score sums along dropout cycles + accumulated_score = np.sum(proba_stacked, axis=-1) + average_score = accumulated_score/len(proba) + # expand dimension w/o data for entropy calculation + average_score = np.expand_dims(average_score, axis=-1) + + # entropy over average prediction score + g_x = entropy_sum(average_score, axis=-1) + + # entropy differences + diff = np.subtract(g_x, f_x) + + # sum all dimensions of diff besides first dim (instances) + shaped = np.reshape(diff, (diff.shape[0], -1)) + + bald = np.sum(shaped, where=~np.isnan(shaped), axis=-1) + return bald + + +def set_dropout_mode(model, dropout_layer_indexes: list, train_mode: bool): + """ + Function to change the mode of the dropout layers (bool: train_mode -> train or evaluation) + + Args: + model: Pytorch model + dropout_layer_indexes: Indexes of the dropout layers which should be activated + Choose indices from : list(torch_model.modules()) + train_mode: boolean, true <=> train_mode, false <=> evaluation_mode + """ + + modules = list(model.modules()) # list of all modules in the network. + + if len(dropout_layer_indexes) != 0: + for index in dropout_layer_indexes: + layer = modules[index] + if layer.__class__.__name__.startswith('Dropout'): + if True == train_mode: + layer.train() + elif False == train_mode: + layer.eval() + else: + raise KeyError( + "The passed index: {} is not a Dropout layer".format(index)) + + else: + for module in modules: + if module.__class__.__name__.startswith('Dropout'): + if True == train_mode: + module.train() + elif False == train_mode: + module.eval() diff --git a/modAL/expected_error.py b/modAL/expected_error.py index df596f5..d7b3611 100644 --- a/modAL/expected_error.py +++ b/modAL/expected_error.py @@ -5,19 +5,19 @@ from typing import Tuple import numpy as np - from sklearn.base import clone from sklearn.exceptions import NotFittedError from modAL.models import ActiveLearner -from modAL.utils.data import modALinput, data_vstack -from modAL.utils.selection import multi_argmax, shuffled_argmax -from modAL.uncertainty import _proba_uncertainty, _proba_entropy +from modAL.uncertainty import _proba_entropy, _proba_uncertainty +from modAL.utils.data import (add_row, data_shape, data_vstack, drop_rows, + enumerate_data, modALinput) +from modAL.utils.selection import multi_argmin, shuffled_argmin def expected_error_reduction(learner: ActiveLearner, X: modALinput, loss: str = 'binary', p_subsample: np.float = 1.0, n_instances: int = 1, - random_tie_break: bool = False) -> Tuple[np.ndarray, modALinput]: + random_tie_break: bool = False) -> np.ndarray: """ Expected error reduction query strategy. @@ -38,47 +38,46 @@ def expected_error_reduction(learner: ActiveLearner, X: modALinput, loss: str = Returns: - The indices of the instances from X chosen to be labelled; - the instances from X chosen to be labelled. + The indices of the instances from X chosen to be labelled. + The expected error metric of the chosen instances; """ assert 0.0 <= p_subsample <= 1.0, 'p_subsample subsampling keep ratio must be between 0.0 and 1.0' assert loss in ['binary', 'log'], 'loss must be \'binary\' or \'log\'' - expected_error = np.zeros(shape=(len(X), )) + expected_error = np.zeros(shape=(data_shape(X)[0],)) possible_labels = np.unique(learner.y_training) try: X_proba = learner.predict_proba(X) except NotFittedError: # TODO: implement a proper cold-start - return 0, X[0] + return np.array([0]) cloned_estimator = clone(learner.estimator) - for x_idx, x in enumerate(X): + for x_idx, x in enumerate_data(X): # subsample the data if needed if np.random.rand() <= p_subsample: + X_reduced = drop_rows(X, x_idx) # estimate the expected error for y_idx, y in enumerate(possible_labels): - X_new = data_vstack((learner.X_training, x.reshape(1, -1))) - y_new = data_vstack((learner.y_training, np.array(y).reshape(1, ))) + X_new = add_row(learner.X_training, x) + y_new = data_vstack((learner.y_training, np.array(y).reshape(1,))) cloned_estimator.fit(X_new, y_new) - refitted_proba = cloned_estimator.predict_proba(X) + refitted_proba = cloned_estimator.predict_proba(X_reduced) if loss is 'binary': - loss = _proba_uncertainty(refitted_proba) + nloss = _proba_uncertainty(refitted_proba) elif loss is 'log': - loss = _proba_entropy(refitted_proba) + nloss = _proba_entropy(refitted_proba) - expected_error[x_idx] += np.sum(loss)*X_proba[x_idx, y_idx] + expected_error[x_idx] += np.sum(nloss)*X_proba[x_idx, y_idx] else: expected_error[x_idx] = np.inf if not random_tie_break: - query_idx = multi_argmax(expected_error, n_instances) - else: - query_idx = shuffled_argmax(expected_error, n_instances) + return multi_argmin(expected_error, n_instances) - return query_idx, X[query_idx] + return shuffled_argmin(expected_error, n_instances) diff --git a/modAL/models/__init__.py b/modAL/models/__init__.py index f96b37f..347716e 100644 --- a/modAL/models/__init__.py +++ b/modAL/models/__init__.py @@ -1,6 +1,7 @@ -from .learners import ActiveLearner, BayesianOptimizer, Committee, CommitteeRegressor +from .learners import (ActiveLearner, BayesianOptimizer, Committee, + CommitteeRegressor, DeepActiveLearner) __all__ = [ - 'ActiveLearner', 'BayesianOptimizer', + 'ActiveLearner', 'DeepActiveLearner', 'BayesianOptimizer', 'Committee', 'CommitteeRegressor' -] \ No newline at end of file +] diff --git a/modAL/models/base.py b/modAL/models/base.py index c4703a1..57c8b81 100644 --- a/modAL/models/base.py +++ b/modAL/models/base.py @@ -5,14 +5,15 @@ import abc import sys -from typing import Union, Callable, Optional, Tuple, List, Iterator, Any +import warnings +from typing import Any, Callable, Iterator, List, Tuple, Union import numpy as np +import scipy.sparse as sp +from modAL.utils.data import data_hstack, modALinput, retrieve_rows from sklearn.base import BaseEstimator -from sklearn.utils import check_X_y - -from modAL.utils.data import data_vstack, modALinput - +from sklearn.ensemble._base import _BaseHeterogeneousEnsemble +from sklearn.pipeline import Pipeline if sys.version_info >= (3, 4): ABC = abc.ABC @@ -28,81 +29,75 @@ class BaseLearner(ABC, BaseEstimator): estimator: The estimator to be used in the active learning loop. query_strategy: Function providing the query strategy for the active learning loop, for instance, modAL.uncertainty.uncertainty_sampling. - X_training: Initial training samples, if available. - y_training: Initial training labels corresponding to initial training samples. - bootstrap_init: If initial training data is available, bootstrapping can be done during the first training. - Useful when building Committee models with bagging. + force_all_finite: When True, forces all values of the data finite. + When False, accepts np.nan and np.inf values. + on_transformed: Whether to transform samples with the pipeline defined by the estimator + when applying the query strategy. **fit_kwargs: keyword arguments. Attributes: estimator: The estimator to be used in the active learning loop. query_strategy: Function providing the query strategy for the active learning loop. - X_training: If the model hasn't been fitted yet it is None, otherwise it contains the samples - which the model has been trained on. - y_training: The labels corresponding to X_training. """ + def __init__(self, estimator: BaseEstimator, query_strategy: Callable, - X_training: Optional[modALinput] = None, - y_training: Optional[modALinput] = None, - bootstrap_init: bool = False, + on_transformed: bool = False, + force_all_finite: bool = True, **fit_kwargs ) -> None: assert callable(query_strategy), 'query_strategy must be callable' self.estimator = estimator self.query_strategy = query_strategy + self.on_transformed = on_transformed - self.X_training = X_training - self.y_training = y_training - if X_training is not None: - self._fit_to_known(bootstrap=bootstrap_init, **fit_kwargs) - - def _add_training_data(self, X: modALinput, y: modALinput) -> None: - """ - Adds the new data and label to the known data, but does not retrain the model. + assert isinstance(force_all_finite, + bool), 'force_all_finite must be a bool' + self.force_all_finite = force_all_finite - Args: - X: The new samples for which the labels are supplied by the expert. - y: Labels corresponding to the new instances in X. - - Note: - If the classifier has been fitted, the features in X have to agree with the training samples which the - classifier has seen. - """ - check_X_y(X, y, accept_sparse=True, ensure_2d=False, allow_nd=True, multi_output=True) - - if self.X_training is None: - self.X_training = X - self.y_training = y - else: - try: - self.X_training = data_vstack((self.X_training, X)) - self.y_training = data_vstack((self.y_training, y)) - except ValueError: - raise ValueError('the dimensions of the new training data and label must' - 'agree with the training data and labels provided so far') - - def _fit_to_known(self, bootstrap: bool = False, **fit_kwargs) -> 'BaseLearner': + def transform_without_estimating(self, X: modALinput) -> Union[np.ndarray, sp.csr_matrix]: """ - Fits self.estimator to the training data and labels provided to it so far. + Transforms the data as supplied to the estimator. + * In case the estimator is an skearn pipeline, it applies all pipeline components but the last one. + * In case the estimator is an ensemble, it concatenates the transformations for each classfier + (pipeline) in the ensemble. + * Otherwise returns the non-transformed dataset X Args: - bootstrap: If True, the method trains the model on a set bootstrapped from the known training instances. - **fit_kwargs: Keyword arguments to be passed to the fit method of the predictor. + X: dataset to be transformed Returns: - self + Transformed data set """ - if not bootstrap: - self.estimator.fit(self.X_training, self.y_training, **fit_kwargs) - else: - n_instances = self.X_training.shape[0] - bootstrap_idx = np.random.choice(range(n_instances), n_instances, replace=True) - self.estimator.fit(self.X_training[bootstrap_idx], self.y_training[bootstrap_idx], **fit_kwargs) - - return self + Xt = [] + pipes = [self.estimator] + + if isinstance(self.estimator, _BaseHeterogeneousEnsemble): + pipes = self.estimator.estimators_ + + ################################ + # transform data with pipelines used by estimator + for pipe in pipes: + if isinstance(pipe, Pipeline): + # NOTE: The used pipeline class might be an extension to sklearn's! + # Create a new instance of the used pipeline class with all + # components but the final estimator, which is replaced by an empty (passthrough) component. + # This prevents any special handling of the final transformation pipe, which is usually + # expected to be an estimator. + transformation_pipe = pipe.__class__( + steps=[*pipe.steps[:-1], ('passthrough', 'passthrough')]) + Xt.append(transformation_pipe.transform(X)) + + # in case no transformation pipelines are used by the estimator, + # return the original, non-transfored data + if not Xt: + return X + + ################################ + # concatenate all transformations and return + return data_hstack(Xt) def _fit_on_new(self, X: modALinput, y: modALinput, bootstrap: bool = False, **fit_kwargs) -> 'BaseLearner': """ @@ -117,38 +112,19 @@ def _fit_on_new(self, X: modALinput, y: modALinput, bootstrap: bool = False, **f Returns: self """ - check_X_y(X, y, accept_sparse=True, ensure_2d=False, allow_nd=True, multi_output=True) if not bootstrap: self.estimator.fit(X, y, **fit_kwargs) else: - bootstrap_idx = np.random.choice(range(X.shape[0]), X.shape[0], replace=True) + bootstrap_idx = np.random.choice( + range(X.shape[0]), X.shape[0], replace=True) self.estimator.fit(X[bootstrap_idx], y[bootstrap_idx]) return self - def fit(self, X: modALinput, y: modALinput, bootstrap: bool = False, **fit_kwargs) -> 'BaseLearner': - """ - Interface for the fit method of the predictor. Fits the predictor to the supplied data, then stores it - internally for the active learning loop. - - Args: - X: The samples to be fitted. - y: The corresponding labels. - bootstrap: If true, trains the estimator on a set bootstrapped from X. - Useful for building Committee models with bagging. - **fit_kwargs: Keyword arguments to be passed to the fit method of the predictor. - - Note: - When using scikit-learn estimators, calling this method will make the ActiveLearner forget all training data - it has seen! - - Returns: - self - """ - check_X_y(X, y, accept_sparse=True, ensure_2d=False, allow_nd=True, multi_output=True) - self.X_training, self.y_training = X, y - return self._fit_to_known(bootstrap=bootstrap, **fit_kwargs) + @abc.abstractmethod + def fit(self, *args, **kwargs) -> None: + pass def predict(self, X: modALinput, **predict_kwargs) -> Any: """ @@ -176,11 +152,13 @@ def predict_proba(self, X: modALinput, **predict_proba_kwargs) -> Any: """ return self.estimator.predict_proba(X, **predict_proba_kwargs) - def query(self, *query_args, **query_kwargs) -> Union[Tuple, modALinput]: + def query(self, X_pool, *query_args, return_metrics: bool = False, **query_kwargs) -> Union[Tuple, modALinput]: """ Finds the n_instances most informative point in the data provided by calling the query_strategy function. Args: + X_pool: Pool of unlabeled instances to retrieve most informative instances from + return_metrics: boolean to indicate, if the corresponding query metrics should be (not) returned *query_args: The arguments for the query strategy. For instance, in the case of :func:`~modAL.uncertainty.uncertainty_sampling`, it is the pool of samples from which the query strategy should choose instances to request labels. @@ -190,9 +168,25 @@ def query(self, *query_args, **query_kwargs) -> Union[Tuple, modALinput]: Value of the query_strategy function. Should be the indices of the instances from the pool chosen to be labelled and the instances themselves. Can be different in other cases, for instance only the instance to be labelled upon query synthesis. + query_metrics: returns also the corresponding metrics, if return_metrics == True """ - query_result = self.query_strategy(self, *query_args, **query_kwargs) - return query_result + + try: + query_result, query_metrics = self.query_strategy( + self, X_pool, *query_args, **query_kwargs) + + except: + query_metrics = None + query_result = self.query_strategy( + self, X_pool, *query_args, **query_kwargs) + + if return_metrics: + if query_metrics is None: + warnings.warn( + "The selected query strategy doesn't support return_metrics") + return query_result, retrieve_rows(X_pool, query_result), query_metrics + else: + return query_result, retrieve_rows(X_pool, query_result) def score(self, X: modALinput, y: modALinput, **score_kwargs) -> Any: """ @@ -216,16 +210,20 @@ def teach(self, *args, **kwargs) -> None: class BaseCommittee(ABC, BaseEstimator): """ Base class for query-by-committee setup. - Args: learner_list: List of ActiveLearner objects to form committee. query_strategy: Function to query labels. + on_transformed: Whether to transform samples with the pipeline defined by each learner's estimator + when applying the query strategy. """ - def __init__(self, learner_list: List[BaseLearner], query_strategy: Callable) -> None: + def __init__(self, learner_list: List[BaseLearner], query_strategy: Callable, on_transformed: bool = False) -> None: assert type(learner_list) == list, 'learners must be supplied in a list' self.learner_list = learner_list self.query_strategy = query_strategy + self.on_transformed = on_transformed + # TODO: update training data when using fit() and teach() methods + self.X_training = None def __iter__(self) -> Iterator[BaseLearner]: for learner in self.learner_list: @@ -237,11 +235,9 @@ def __len__(self) -> int: def _add_training_data(self, X: modALinput, y: modALinput) -> None: """ Adds the new data and label to the known data for each learner, but does not retrain the model. - Args: X: The new samples for which the labels are supplied by the expert. y: Labels corresponding to the new instances in X. - Note: If the learners have been fitted, the features in X have to agree with the training samples which the classifier has seen. @@ -252,7 +248,6 @@ def _add_training_data(self, X: modALinput, y: modALinput) -> None: def _fit_to_known(self, bootstrap: bool = False, **fit_kwargs) -> None: """ Fits all learners to the training data and labels provided to it so far. - Args: bootstrap: If True, each estimator is trained on a bootstrapped dataset. Useful when using bagging to build the ensemble. @@ -264,7 +259,6 @@ def _fit_to_known(self, bootstrap: bool = False, **fit_kwargs) -> None: def _fit_on_new(self, X: modALinput, y: modALinput, bootstrap: bool = False, **fit_kwargs) -> None: """ Fits all learners to the given data and labels. - Args: X: The new samples for which the labels are supplied by the expert. y: Labels corresponding to the new instances in X. @@ -279,9 +273,7 @@ def fit(self, X: modALinput, y: modALinput, **fit_kwargs) -> 'BaseCommittee': Fits every learner to a subset sampled with replacement from X. Calling this method makes the learner forget the data it has seen up until this point and replaces it with X! If you would like to perform bootstrapping on each learner using the data it has seen, use the method .rebag()! - Calling this method makes the learner forget the data it has seen up until this point and replaces it with X! - Args: X: The samples to be fitted on. y: The corresponding labels. @@ -292,11 +284,23 @@ def fit(self, X: modALinput, y: modALinput, **fit_kwargs) -> 'BaseCommittee': return self - def query(self, *query_args, **query_kwargs) -> Union[Tuple, modALinput]: + def transform_without_estimating(self, X: modALinput) -> Union[np.ndarray, sp.csr_matrix]: + """ + Transforms the data as supplied to each learner's estimator and concatenates transformations. + Args: + X: dataset to be transformed + Returns: + Transformed data set + """ + return data_hstack([learner.transform_without_estimating(X) for learner in self.learner_list]) + + def query(self, X_pool, return_metrics: bool = False, *query_args, **query_kwargs) -> Union[Tuple, modALinput]: """ Finds the n_instances most informative point in the data provided by calling the query_strategy function. Args: + X_pool: Pool of unlabeled instances to retrieve most informative instances from + return_metrics: boolean to indicate, if the corresponding query metrics should be (not) returned *query_args: The arguments for the query strategy. For instance, in the case of :func:`~modAL.disagreement.max_disagreement_sampling`, it is the pool of samples from which the query. strategy should choose instances to request labels. @@ -306,18 +310,32 @@ def query(self, *query_args, **query_kwargs) -> Union[Tuple, modALinput]: Return value of the query_strategy function. Should be the indices of the instances from the pool chosen to be labelled and the instances themselves. Can be different in other cases, for instance only the instance to be labelled upon query synthesis. + query_metrics: returns also the corresponding metrics, if return_metrics == True """ - query_result = self.query_strategy(self, *query_args, **query_kwargs) - return query_result + + try: + query_result, query_metrics = self.query_strategy( + self, X_pool, *query_args, **query_kwargs) + + except: + query_metrics = None + query_result = self.query_strategy( + self, X_pool, *query_args, **query_kwargs) + + if return_metrics: + if query_metrics is None: + warnings.warn( + "The selected query strategy doesn't support return_metrics") + return query_result, retrieve_rows(X_pool, query_result), query_metrics + else: + return query_result, retrieve_rows(X_pool, query_result) def rebag(self, **fit_kwargs) -> None: """ Refits every learner with a dataset bootstrapped from its training instances. Contrary to .bag(), it bootstraps the training data for each learner based on its own examples. - Todo: Where is .bag()? - Args: **fit_kwargs: Keyword arguments to be passed to the fit method of the predictor. """ @@ -326,7 +344,6 @@ def rebag(self, **fit_kwargs) -> None: def teach(self, X: modALinput, y: modALinput, bootstrap: bool = False, only_new: bool = False, **fit_kwargs) -> None: """ Adds X and y to the known training data for each learner and retrains learners with the augmented dataset. - Args: X: The new samples for which the labels are supplied by the expert. y: Labels corresponding to the new instances in X. @@ -346,4 +363,5 @@ def predict(self, X: modALinput) -> Any: @abc.abstractmethod def vote(self, X: modALinput) -> Any: # TODO: clarify typing - pass \ No newline at end of file + pass + diff --git a/modAL/models/learners.py b/modAL/models/learners.py index 7fdcf2c..b7dac72 100644 --- a/modAL/models/learners.py +++ b/modAL/models/learners.py @@ -1,16 +1,15 @@ -import numpy as np - -from typing import Callable, Optional, Tuple, List, Any +from typing import Any, Callable, List, Optional, Tuple +import numpy as np +from modAL.acquisition import max_EI +from modAL.disagreement import max_std_sampling, vote_entropy_sampling +from modAL.models.base import BaseCommittee, BaseLearner +from modAL.uncertainty import uncertainty_sampling +from modAL.utils.data import data_vstack, modALinput, retrieve_rows +from modAL.utils.validation import check_class_labels, check_class_proba from sklearn.base import BaseEstimator from sklearn.metrics import accuracy_score - -from modAL.models.base import BaseLearner, BaseCommittee -from modAL.utils.validation import check_class_labels, check_class_proba -from modAL.utils.data import modALinput -from modAL.uncertainty import uncertainty_sampling -from modAL.disagreement import vote_entropy_sampling, max_std_sampling -from modAL.acquisition import max_EI +from sklearn.utils import check_X_y """ Classes for active learning algorithms @@ -20,7 +19,7 @@ class ActiveLearner(BaseLearner): """ - This class is an abstract model of a general active learning algorithm. + This class is an model of a general classic (machine learning) active learning algorithm. Args: estimator: The estimator to be used in the active learning loop. @@ -30,13 +29,15 @@ class ActiveLearner(BaseLearner): y_training: Initial training labels corresponding to initial training samples. bootstrap_init: If initial training data is available, bootstrapping can be done during the first training. Useful when building Committee models with bagging. + on_transformed: Whether to transform samples with the pipeline defined by the estimator + when applying the query strategy. **fit_kwargs: keyword arguments. Attributes: estimator: The estimator to be used in the active learning loop. query_strategy: Function providing the query strategy for the active learning loop. X_training: If the model hasn't been fitted yet it is None, otherwise it contains the samples - which the model has been trained on. + which the model has been trained on. If provided, the method fit() of estimator is called during __init__() y_training: The labels corresponding to X_training. Examples: @@ -73,10 +74,88 @@ def __init__(self, X_training: Optional[modALinput] = None, y_training: Optional[modALinput] = None, bootstrap_init: bool = False, + on_transformed: bool = False, **fit_kwargs ) -> None: - super().__init__(estimator, query_strategy, - X_training, y_training, bootstrap_init, **fit_kwargs) + super().__init__(estimator, query_strategy, on_transformed, **fit_kwargs) + + self.X_training = X_training + self.y_training = y_training + + if X_training is not None: + self._fit_to_known(bootstrap=bootstrap_init, **fit_kwargs) + + def _add_training_data(self, X: modALinput, y: modALinput) -> None: + """ + Adds the new data and label to the known data, but does not retrain the model. + + Args: + X: The new samples for which the labels are supplied by the expert. + y: Labels corresponding to the new instances in X. + + Note: + If the classifier has been fitted, the features in X have to agree with the training samples which the + classifier has seen. + """ + check_X_y(X, y, accept_sparse=True, ensure_2d=False, allow_nd=True, multi_output=True, dtype=None, + force_all_finite=self.force_all_finite) + + if self.X_training is None: + self.X_training = X + self.y_training = y + else: + try: + self.X_training = data_vstack((self.X_training, X)) + self.y_training = data_vstack((self.y_training, y)) + except ValueError: + raise ValueError('the dimensions of the new training data and label must' + 'agree with the training data and labels provided so far') + + def _fit_to_known(self, bootstrap: bool = False, **fit_kwargs) -> 'BaseLearner': + """ + Fits self.estimator to the training data and labels provided to it so far. + + Args: + bootstrap: If True, the method trains the model on a set bootstrapped from the known training instances. + **fit_kwargs: Keyword arguments to be passed to the fit method of the predictor. + + Returns: + self + """ + if not bootstrap: + self.estimator.fit(self.X_training, self.y_training, **fit_kwargs) + else: + n_instances = self.X_training.shape[0] + bootstrap_idx = np.random.choice( + range(n_instances), n_instances, replace=True) + self.estimator.fit( + self.X_training[bootstrap_idx], self.y_training[bootstrap_idx], **fit_kwargs) + + return self + + def fit(self, X: modALinput, y: modALinput, bootstrap: bool = False, **fit_kwargs) -> 'BaseLearner': + """ + Interface for the fit method of the predictor. Fits the predictor to the supplied data, then stores it + internally for the active learning loop. + + Args: + X: The samples to be fitted. + y: The corresponding labels. + bootstrap: If true, trains the estimator on a set bootstrapped from X. + Useful for building Committee models with bagging. + **fit_kwargs: Keyword arguments to be passed to the fit method of the predictor. + + Note: + When using scikit-learn estimators, calling this method will make the ActiveLearner forget all training data + it has seen! + + Returns: + self + """ + check_X_y(X, y, accept_sparse=True, ensure_2d=False, allow_nd=True, multi_output=True, dtype=None, + force_all_finite=self.force_all_finite) + self.X_training, self.y_training = X, y + return self._fit_to_known(bootstrap=bootstrap, **fit_kwargs) def teach(self, X: modALinput, y: modALinput, bootstrap: bool = False, only_new: bool = False, **fit_kwargs) -> None: """ @@ -92,20 +171,138 @@ def teach(self, X: modALinput, y: modALinput, bootstrap: bool = False, only_new: tensorflow or keras). **fit_kwargs: Keyword arguments to be passed to the fit method of the predictor. """ - self._add_training_data(X, y) if not only_new: + self._add_training_data(X, y) self._fit_to_known(bootstrap=bootstrap, **fit_kwargs) else: + check_X_y(X, y, accept_sparse=True, ensure_2d=False, allow_nd=True, multi_output=True, dtype=None, + force_all_finite=self.force_all_finite) self._fit_on_new(X, y, bootstrap=bootstrap, **fit_kwargs) +class DeepActiveLearner(BaseLearner): + """ + This class is an model of a general deep active learning algorithm. + Differences to the classical ActiveLearner are: + - Data is no member variable of the DeepActiveLearner class + - Misses the initial add/train data methods, therefore always trains on new data + - Uses different interfaces to sklearn in some functions + + Args: + estimator: The estimator to be used in the active learning loop. + query_strategy: Function providing the query strategy for the active learning loop, + for instance, modAL.uncertainty.uncertainty_sampling. + on_transformed: Whether to transform samples with the pipeline defined by the estimator + when applying the query strategy. + **fit_kwargs: keyword arguments. + + Attributes: + estimator: The estimator to be used in the active learning loop. + query_strategy: Function providing the query strategy for the active learning loop. + """ + + def __init__(self, + estimator: BaseEstimator, + query_strategy: Callable = uncertainty_sampling, + on_transformed: bool = False, + **fit_kwargs + ) -> None: + # TODO: Check if given query strategy works for Deep Learning + super().__init__(estimator, query_strategy, on_transformed, **fit_kwargs) + + self.estimator.initialize() + + def fit(self, X: modALinput, y: modALinput, bootstrap: bool = False, **fit_kwargs) -> 'BaseLearner': + """ + Interface for the fit method of the predictor. Fits the predictor to the supplied data. + + Args: + X: The samples to be fitted. + y: The corresponding labels. + bootstrap: If true, trains the estimator on a set bootstrapped from X. + Useful for building Committee models with bagging. + **fit_kwargs: Keyword arguments to be passed to the fit method of the predictor. + + Returns: + self + """ + return self._fit_on_new(X, y, bootstrap=bootstrap, **fit_kwargs) + + def teach(self, X: modALinput, y: modALinput, warm_start: bool = True, bootstrap: bool = False, **fit_kwargs) -> None: + """ + Trains the predictor with the passed data (warm_start decides if params are resetted or not). + + Args: + X: The new samples for which the labels are supplied by the expert. + y: Labels corresponding to the new instances in X. + warm_start: If False, the model parameters are resetted and the training starts from zero, + otherwise the pre trained model is kept and further trained. + bootstrap: If True, training is done on a bootstrapped dataset. Useful for building Committee models + with bagging. + **fit_kwargs: Keyword arguments to be passed to the fit method of the predictor. + """ + + if warm_start: + if not bootstrap: + self.estimator.partial_fit(X, y, **fit_kwargs) + else: + bootstrap_idx = np.random.choice( + range(X.shape[0]), X.shape[0], replace=True) + self.estimator.partial_fit( + X[bootstrap_idx], y[bootstrap_idx], **fit_kwargs) + else: + self._fit_on_new(X, y, bootstrap=bootstrap, **fit_kwargs) + + @property + def num_epochs(self): + """ + Returns the number of epochs of a single fit cycle. + """ + return self.estimator.max_epochs + + @num_epochs.setter + def num_epochs(self, value): + """ + Sets the number of epochs of a single fit cycle. The number of epochs + can be changed at any time, even after the model was trained. + """ + if isinstance(value, int): + if 0 < value: + self.estimator.max_epochs = value + else: + raise ValueError("num_epochs must be larger than zero") + else: + raise TypeError("num_epochs must be of type integer!") + + @property + def batch_size(self): + """ + Returns the batch size of a single forward pass. + """ + return self.estimator.batch_size + + @batch_size.setter + def batch_size(self, value): + """ + Sets the batch size of a single forward pass. The batch size + can be changed at any time, even after the model was trained. + """ + if isinstance(value, int): + if 0 < value: + self.estimator.batch_size = value + else: + raise ValueError("batch size must be larger than 0") + else: + raise TypeError("batch size must be of type integer!") + + """ Classes for Bayesian optimization --------------------------------- """ -class BayesianOptimizer(BaseLearner): +class BayesianOptimizer(ActiveLearner): """ This class is an abstract model of a Bayesian optimizer algorithm. @@ -171,19 +368,21 @@ class BayesianOptimizer(BaseLearner): ... query_idx, query_inst = optimizer.query(X) ... optimizer.teach(X[query_idx].reshape(1, -1), y[query_idx].reshape(1, -1)) """ + def __init__(self, estimator: BaseEstimator, query_strategy: Callable = max_EI, X_training: Optional[modALinput] = None, y_training: Optional[modALinput] = None, bootstrap_init: bool = False, + on_transformed: bool = False, **fit_kwargs) -> None: super(BayesianOptimizer, self).__init__(estimator, query_strategy, - X_training, y_training, bootstrap_init, **fit_kwargs) + X_training, y_training, bootstrap_init, on_transformed, **fit_kwargs) # setting the maximum value if self.y_training is not None: max_idx = np.argmax(self.y_training) - self.X_max = self.X_training[max_idx] + self.X_max = retrieve_rows(self.X_training, max_idx) self.y_max = self.y_training[max_idx] else: self.X_max = None @@ -194,7 +393,7 @@ def _set_max(self, X: modALinput, y: modALinput) -> None: y_max = y[max_idx] if y_max > self.y_max: self.y_max = y_max - self.X_max = X[max_idx] + self.X_max = retrieve_rows(X, max_idx) def get_max(self) -> Tuple: """ @@ -239,18 +438,16 @@ def teach(self, X: modALinput, y: modALinput, bootstrap: bool = False, only_new: class Committee(BaseCommittee): """ This class is an abstract model of a committee-based active learning algorithm. - Args: learner_list: A list of ActiveLearners forming the Committee. query_strategy: Query strategy function. Committee supports disagreement-based query strategies from :mod:`modAL.disagreement`, but uncertainty-based ones from :mod:`modAL.uncertainty` are also supported. - + on_transformed: Whether to transform samples with the pipeline defined by each learner's estimator + when applying the query strategy. Attributes: classes_: Class labels known by the Committee. n_classes_: Number of classes known by the Committee. - Examples: - >>> from sklearn.datasets import load_iris >>> from sklearn.neighbors import KNeighborsClassifier >>> from sklearn.ensemble import RandomForestClassifier @@ -284,8 +481,9 @@ class Committee(BaseCommittee): ... y=iris['target'][query_idx].reshape(1, ) ... ) """ - def __init__(self, learner_list: List[ActiveLearner], query_strategy: Callable = vote_entropy_sampling) -> None: - super().__init__(learner_list, query_strategy) + def __init__(self, learner_list: List[ActiveLearner], query_strategy: Callable = vote_entropy_sampling, + on_transformed: bool = False) -> None: + super().__init__(learner_list, query_strategy, on_transformed) self._set_classes() def _set_classes(self): @@ -311,16 +509,40 @@ def _set_classes(self): def _add_training_data(self, X: modALinput, y: modALinput): super()._add_training_data(X, y) + + def fit(self, X: modALinput, y: modALinput, **fit_kwargs) -> 'BaseCommittee': + """ + Fits every learner to a subset sampled with replacement from X. Calling this method makes the learner forget the + data it has seen up until this point and replaces it with X! If you would like to perform bootstrapping on each + learner using the data it has seen, use the method .rebag()! + Calling this method makes the learner forget the data it has seen up until this point and replaces it with X! + Args: + X: The samples to be fitted on. + y: The corresponding labels. + **fit_kwargs: Keyword arguments to be passed to the fit method of the predictor. + """ + super().fit(X, y, **fit_kwargs) + self._set_classes() + + def teach(self, X: modALinput, y: modALinput, bootstrap: bool = False, only_new: bool = False, **fit_kwargs) -> None: + """ + Adds X and y to the known training data for each learner and retrains learners with the augmented dataset. + Args: + X: The new samples for which the labels are supplied by the expert. + y: Labels corresponding to the new instances in X. + bootstrap: If True, trains each learner on a bootstrapped set. Useful when building the ensemble by bagging. + only_new: If True, the model is retrained using only X and y, ignoring the previously provided examples. + **fit_kwargs: Keyword arguments to be passed to the fit method of the predictor. + """ + super().teach(X, y, bootstrap=bootstrap, only_new=only_new, **fit_kwargs) self._set_classes() def predict(self, X: modALinput, **predict_proba_kwargs) -> Any: """ Predicts the class of the samples by picking the consensus prediction. - Args: X: The samples to be predicted. **predict_proba_kwargs: Keyword arguments to be passed to the :meth:`predict_proba` of the Committee. - Returns: The predicted class labels for X. """ @@ -334,11 +556,9 @@ def predict(self, X: modALinput, **predict_proba_kwargs) -> Any: def predict_proba(self, X: modALinput, **predict_proba_kwargs) -> Any: """ Consensus probabilities of the Committee. - Args: X: The samples for which the class probabilities are to be predicted. **predict_proba_kwargs: Keyword arguments to be passed to the :meth:`predict_proba` of the Committee. - Returns: Class probabilities for X. """ @@ -347,15 +567,12 @@ def predict_proba(self, X: modALinput, **predict_proba_kwargs) -> Any: def score(self, X: modALinput, y: modALinput, sample_weight: List[float] = None) -> Any: """ Returns the mean accuracy on the given test data and labels. - Todo: Why accuracy? - Args: X: The samples to score. y: Ground truth labels corresponding to X. sample_weight: Sample weights. - Returns: Mean accuracy of the classifiers. """ @@ -365,11 +582,9 @@ def score(self, X: modALinput, y: modALinput, sample_weight: List[float] = None) def vote(self, X: modALinput, **predict_kwargs) -> Any: """ Predicts the labels for the supplied data for each learner in the Committee. - Args: X: The samples to cast votes. **predict_kwargs: Keyword arguments to be passed to the :meth:`predict` of the learners. - Returns: The predicted class for each learner in the Committee and each sample in X. """ @@ -383,11 +598,9 @@ def vote(self, X: modALinput, **predict_kwargs) -> Any: def vote_proba(self, X: modALinput, **predict_proba_kwargs) -> Any: """ Predicts the probabilities of the classes for each sample and each learner. - Args: X: The samples for which class probabilities are to be calculated. **predict_proba_kwargs: Keyword arguments for the :meth:`predict_proba` of the learners. - Returns: Probabilities of each class for each learner and each instance. """ @@ -419,13 +632,12 @@ def vote_proba(self, X: modALinput, **predict_proba_kwargs) -> Any: class CommitteeRegressor(BaseCommittee): """ This class is an abstract model of a committee-based active learning regression. - Args: learner_list: A list of ActiveLearners forming the CommitteeRegressor. query_strategy: Query strategy function. - + on_transformed: Whether to transform samples with the pipeline defined by each learner's estimator + when applying the query strategy. Examples: - >>> import numpy as np >>> import matplotlib.pyplot as plt >>> from sklearn.gaussian_process import GaussianProcessRegressor @@ -452,8 +664,7 @@ class CommitteeRegressor(BaseCommittee): >>> # query strategy for regression >>> def ensemble_regression_std(regressor, X): ... _, std = regressor.predict(X, return_std=True) - ... query_idx = np.argmax(std) - ... return query_idx, X[query_idx] + ... return np.argmax(std) >>> >>> # initializing the CommitteeRegressor >>> committee = CommitteeRegressor( @@ -467,17 +678,16 @@ class CommitteeRegressor(BaseCommittee): ... query_idx, query_instance = committee.query(X.reshape(-1, 1)) ... committee.teach(X[query_idx].reshape(-1, 1), y[query_idx].reshape(-1, 1)) """ - def __init__(self, learner_list: List[ActiveLearner], query_strategy: Callable = max_std_sampling) -> None: - super().__init__(learner_list, query_strategy) + def __init__(self, learner_list: List[ActiveLearner], query_strategy: Callable = max_std_sampling, + on_transformed: bool = False) -> None: + super().__init__(learner_list, query_strategy, on_transformed) def predict(self, X: modALinput, return_std: bool = False, **predict_kwargs) -> Any: """ Predicts the values of the samples by averaging the prediction of each regressor. - Args: X: The samples to be predicted. **predict_kwargs: Keyword arguments to be passed to the :meth:`vote` method of the CommitteeRegressor. - Returns: The predicted class labels for X. """ @@ -490,11 +700,9 @@ def predict(self, X: modALinput, return_std: bool = False, **predict_kwargs) -> def vote(self, X: modALinput, **predict_kwargs): """ Predicts the values for the supplied data for each regressor in the CommitteeRegressor. - Args: X: The samples to cast votes. **predict_kwargs: Keyword arguments to be passed to :meth:`predict` of the learners. - Returns: The predicted value for each regressor in the CommitteeRegressor and each sample in X. """ @@ -503,4 +711,4 @@ def vote(self, X: modALinput, **predict_kwargs): for learner_idx, learner in enumerate(self.learner_list): prediction[:, learner_idx] = learner.predict(X, **predict_kwargs).reshape(-1, ) - return prediction \ No newline at end of file + return prediction diff --git a/modAL/multilabel.py b/modAL/multilabel.py index 28a7254..c908674 100644 --- a/modAL/multilabel.py +++ b/modAL/multilabel.py @@ -1,13 +1,12 @@ -import numpy as np +from typing import Optional -from sklearn.base import BaseEstimator +import numpy as np from sklearn.multiclass import OneVsRestClassifier from modAL.models import ActiveLearner from modAL.utils.data import modALinput -from modAL.utils.selection import multi_argmax, shuffled_argmax -from typing import Tuple, Optional -from itertools import combinations +from modAL.utils.selection import (multi_argmax, multi_argmin, shuffled_argmax, + shuffled_argmin) def _SVM_loss(multiclass_classifier: ActiveLearner, @@ -43,7 +42,7 @@ def _SVM_loss(multiclass_classifier: ActiveLearner, def SVM_binary_minimum(classifier: ActiveLearner, X_pool: modALinput, - random_tie_break: bool = False) -> Tuple[np.ndarray, modALinput]: + random_tie_break: bool = False) -> np.ndarray: """ SVM binary minimum multilabel active learning strategy. For details see the paper Klaus Brinker, On Active Learning in Multi-label Classification @@ -58,7 +57,8 @@ def SVM_binary_minimum(classifier: ActiveLearner, X_pool: modALinput, Returns: The index of the instance from X_pool chosen to be labelled; - the instance from X_pool chosen to be labelled. + The instance from X_pool chosen to be labelled. + The Minimum absolute distance metric of the chosen instance; """ decision_function = np.array([svm.decision_function(X_pool) @@ -67,15 +67,13 @@ def SVM_binary_minimum(classifier: ActiveLearner, X_pool: modALinput, min_abs_dist = np.min(np.abs(decision_function), axis=1) if not random_tie_break: - query_idx = np.argmin(min_abs_dist) - else: - query_idx = shuffled_argmax(min_abs_dist) + return np.argmin(min_abs_dist) - return query_idx, X_pool[query_idx] + return shuffled_argmax(min_abs_dist) def max_loss(classifier: OneVsRestClassifier, X_pool: modALinput, - n_instances: int = 1, random_tie_break: bool = False) -> Tuple[np.ndarray, modALinput]: + n_instances: int = 1, random_tie_break: bool = False) -> np.ndarray: """ Max Loss query strategy for SVM multilabel classification. @@ -94,7 +92,9 @@ def max_loss(classifier: OneVsRestClassifier, X_pool: modALinput, Returns: The index of the instance from X_pool chosen to be labelled; - the instance from X_pool chosen to be labelled. + The instance from X_pool chosen to be labelled. + The SVM-loss-max metric of the chosen instances; + """ assert len(X_pool) >= n_instances, 'n_instances cannot be larger than len(X_pool)' @@ -103,15 +103,13 @@ def max_loss(classifier: OneVsRestClassifier, X_pool: modALinput, loss = _SVM_loss(classifier, X_pool, most_certain_classes=most_certain_classes) if not random_tie_break: - query_idx = multi_argmax(loss, n_instances) - else: - query_idx = shuffled_argmax(loss, n_instances) + return multi_argmax(loss, n_instances) - return query_idx, X_pool[query_idx] + return shuffled_argmax(loss, n_instances) def mean_max_loss(classifier: OneVsRestClassifier, X_pool: modALinput, - n_instances: int = 1, random_tie_break: bool = False) -> Tuple[np.ndarray, modALinput]: + n_instances: int = 1, random_tie_break: bool = False) -> np.ndarray: """ Mean Max Loss query strategy for SVM multilabel classification. @@ -128,23 +126,22 @@ def mean_max_loss(classifier: OneVsRestClassifier, X_pool: modALinput, can be used to break the tie when the highest utility score is not unique. Returns: - The index of the instance from X_pool chosen to be labelled; - the instance from X_pool chosen to be labelled. + The index of the instance from X_pool chosen to be labelled. + The SVM-loss metric of the chosen instances. + """ assert len(X_pool) >= n_instances, 'n_instances cannot be larger than len(X_pool)' loss = _SVM_loss(classifier, X_pool) if not random_tie_break: - query_idx = multi_argmax(loss, n_instances) - else: - query_idx = shuffled_argmax(loss, n_instances) + return multi_argmax(loss, n_instances) - return query_idx, X_pool[query_idx] + return shuffled_argmax(loss, n_instances) def min_confidence(classifier: OneVsRestClassifier, X_pool: modALinput, - n_instances: int = 1, random_tie_break: bool = False) -> Tuple[np.ndarray, modALinput]: + n_instances: int = 1, random_tie_break: bool = False) -> np.ndarray: """ MinConfidence query strategy for multilabel classification. @@ -159,23 +156,22 @@ def min_confidence(classifier: OneVsRestClassifier, X_pool: modALinput, can be used to break the tie when the highest utility score is not unique. Returns: - The index of the instance from X_pool chosen to be labelled; - the instance from X_pool chosen to be labelled. + The index of the instance from X_pool chosen to be labelled. + The minimal confidence metric of the chosen instance. + """ classwise_confidence = classifier.predict_proba(X_pool) classwise_min = np.min(classwise_confidence, axis=1) if not random_tie_break: - query_idx = multi_argmax(-classwise_min, n_instances) - else: - query_idx = shuffled_argmax(-classwise_min, n_instances) + return multi_argmin(classwise_min, n_instances) - return query_idx, X_pool[query_idx] + return shuffled_argmin(classwise_min, n_instances) def avg_confidence(classifier: OneVsRestClassifier, X_pool: modALinput, - n_instances: int = 1, random_tie_break: bool = False) -> Tuple[np.ndarray, modALinput]: + n_instances: int = 1, random_tie_break: bool = False) -> np.ndarray: """ AvgConfidence query strategy for multilabel classification. @@ -190,23 +186,22 @@ def avg_confidence(classifier: OneVsRestClassifier, X_pool: modALinput, can be used to break the tie when the highest utility score is not unique. Returns: - The index of the instance from X_pool chosen to be labelled; - the instance from X_pool chosen to be labelled. + The index of the instance from X_pool chosen to be labelled. + The average confidence metric of the chosen instances. + """ classwise_confidence = classifier.predict_proba(X_pool) classwise_mean = np.mean(classwise_confidence, axis=1) if not random_tie_break: - query_idx = multi_argmax(classwise_mean, n_instances) - else: - query_idx = shuffled_argmax(classwise_mean, n_instances) + return multi_argmax(classwise_mean, n_instances) - return query_idx, X_pool[query_idx] + return shuffled_argmax(classwise_mean, n_instances) def max_score(classifier: OneVsRestClassifier, X_pool: modALinput, - n_instances: int = 1, random_tie_break: bool = 1) -> Tuple[np.ndarray, modALinput]: + n_instances: int = 1, random_tie_break: bool = 1) -> np.ndarray: """ MaxScore query strategy for multilabel classification. @@ -221,8 +216,9 @@ def max_score(classifier: OneVsRestClassifier, X_pool: modALinput, can be used to break the tie when the highest utility score is not unique. Returns: - The index of the instance from X_pool chosen to be labelled; - the instance from X_pool chosen to be labelled. + The index of the instance from X_pool chosen to be labelled. + The classwise maximum metric of the chosen instances. + """ classwise_confidence = classifier.predict_proba(X_pool) @@ -231,15 +227,13 @@ def max_score(classifier: OneVsRestClassifier, X_pool: modALinput, classwise_max = np.max(classwise_scores, axis=1) if not random_tie_break: - query_idx = multi_argmax(classwise_max, n_instances) - else: - query_idx = shuffled_argmax(classwise_max, n_instances) + return multi_argmax(classwise_max, n_instances) - return query_idx, X_pool[query_idx] + return shuffled_argmax(classwise_max, n_instances) def avg_score(classifier: OneVsRestClassifier, X_pool: modALinput, - n_instances: int = 1, random_tie_break: bool = False) -> Tuple[np.ndarray, modALinput]: + n_instances: int = 1, random_tie_break: bool = False) -> np.ndarray: """ AvgScore query strategy for multilabel classification. @@ -254,8 +248,9 @@ def avg_score(classifier: OneVsRestClassifier, X_pool: modALinput, can be used to break the tie when the highest utility score is not unique. Returns: - The index of the instance from X_pool chosen to be labelled; - the instance from X_pool chosen to be labelled. + The index of the instance from X_pool chosen to be labelled. + The classwise mean metric of the chosen instances. + """ classwise_confidence = classifier.predict_proba(X_pool) @@ -264,8 +259,6 @@ def avg_score(classifier: OneVsRestClassifier, X_pool: modALinput, classwise_mean = np.mean(classwise_scores, axis=1) if not random_tie_break: - query_idx = multi_argmax(classwise_mean, n_instances) - else: - query_idx = shuffled_argmax(classwise_mean, n_instances) + return multi_argmax(classwise_mean, n_instances) - return query_idx, X_pool[query_idx] + return shuffled_argmax(classwise_mean, n_instances) diff --git a/modAL/uncertainty.py b/modAL/uncertainty.py index c11de43..d0f7b37 100644 --- a/modAL/uncertainty.py +++ b/modAL/uncertainty.py @@ -1,15 +1,15 @@ """ Uncertainty measures and uncertainty based sampling strategies for the active learning models. """ -from typing import Tuple import numpy as np from scipy.stats import entropy -from sklearn.exceptions import NotFittedError from sklearn.base import BaseEstimator +from sklearn.exceptions import NotFittedError from modAL.utils.data import modALinput -from modAL.utils.selection import multi_argmax, shuffled_argmax +from modAL.utils.selection import (multi_argmax, multi_argmin, shuffled_argmax, + shuffled_argmin) def _proba_uncertainty(proba: np.ndarray) -> np.ndarray: @@ -132,7 +132,7 @@ def classifier_entropy(classifier: BaseEstimator, X: modALinput, **predict_proba def uncertainty_sampling(classifier: BaseEstimator, X: modALinput, n_instances: int = 1, random_tie_break: bool = False, - **uncertainty_measure_kwargs) -> Tuple[np.ndarray, modALinput]: + **uncertainty_measure_kwargs) -> np.ndarray: """ Uncertainty sampling query strategy. Selects the least sure instances for labelling. @@ -146,22 +146,20 @@ def uncertainty_sampling(classifier: BaseEstimator, X: modALinput, measure function. Returns: - The indices of the instances from X chosen to be labelled; - the instances from X chosen to be labelled. + The indices of the instances from X chosen to be labelled. + The uncertainty metric of the chosen instances. """ uncertainty = classifier_uncertainty(classifier, X, **uncertainty_measure_kwargs) if not random_tie_break: - query_idx = multi_argmax(uncertainty, n_instances=n_instances) - else: - query_idx = shuffled_argmax(uncertainty, n_instances=n_instances) + return multi_argmax(uncertainty, n_instances=n_instances) - return query_idx, X[query_idx] + return shuffled_argmax(uncertainty, n_instances=n_instances) def margin_sampling(classifier: BaseEstimator, X: modALinput, n_instances: int = 1, random_tie_break: bool = False, - **uncertainty_measure_kwargs) -> Tuple[np.ndarray, modALinput]: + **uncertainty_measure_kwargs) -> np.ndarray: """ Margin sampling query strategy. Selects the instances where the difference between the first most likely and second most likely classes are the smallest. @@ -174,22 +172,20 @@ def margin_sampling(classifier: BaseEstimator, X: modALinput, **uncertainty_measure_kwargs: Keyword arguments to be passed for the uncertainty measure function. Returns: - The indices of the instances from X chosen to be labelled; - the instances from X chosen to be labelled. + The indices of the instances from X chosen to be labelled. + The margin metric of the chosen instances. """ margin = classifier_margin(classifier, X, **uncertainty_measure_kwargs) if not random_tie_break: - query_idx = multi_argmax(-margin, n_instances=n_instances) - else: - query_idx = shuffled_argmax(-margin, n_instances=n_instances) + return multi_argmin(margin, n_instances=n_instances) - return query_idx, X[query_idx] + return shuffled_argmin(margin, n_instances=n_instances) def entropy_sampling(classifier: BaseEstimator, X: modALinput, n_instances: int = 1, random_tie_break: bool = False, - **uncertainty_measure_kwargs) -> Tuple[np.ndarray, modALinput]: + **uncertainty_measure_kwargs) -> np.ndarray: """ Entropy sampling query strategy. Selects the instances where the class probabilities have the largest entropy. @@ -204,14 +200,12 @@ def entropy_sampling(classifier: BaseEstimator, X: modALinput, measure function. Returns: - The indices of the instances from X chosen to be labelled; - the instances from X chosen to be labelled. + The indices of the instances from X chosen to be labelled. + The entropy metric of the chosen instances. """ entropy = classifier_entropy(classifier, X, **uncertainty_measure_kwargs) if not random_tie_break: - query_idx = multi_argmax(entropy, n_instances=n_instances) - else: - query_idx = shuffled_argmax(entropy, n_instances=n_instances) + return multi_argmax(entropy, n_instances=n_instances) - return query_idx, X[query_idx] + return shuffled_argmax(entropy, n_instances=n_instances) diff --git a/modAL/utils/__init__.py b/modAL/utils/__init__.py index 3b6501c..2f3bc12 100644 --- a/modAL/utils/__init__.py +++ b/modAL/utils/__init__.py @@ -1,4 +1,5 @@ -from .combination import make_linear_combination, make_product, make_query_strategy +from .combination import (make_linear_combination, make_product, + make_query_strategy) from .data import data_vstack from .selection import multi_argmax, weighted_random from .validation import check_class_labels, check_class_proba @@ -8,4 +9,4 @@ 'data_vstack', 'multi_argmax', 'weighted_random', 'check_class_labels', 'check_class_proba' -] \ No newline at end of file +] diff --git a/modAL/utils/combination.py b/modAL/utils/combination.py index 98974ca..eb2b4d2 100644 --- a/modAL/utils/combination.py +++ b/modAL/utils/combination.py @@ -1,9 +1,8 @@ from typing import Callable, Optional, Sequence, Tuple import numpy as np -from sklearn.base import BaseEstimator - from modAL.utils.data import modALinput +from sklearn.base import BaseEstimator def make_linear_combination(*functions: Callable, weights: Optional[Sequence] = None) -> Callable: @@ -78,7 +77,6 @@ def make_query_strategy(utility_measure: Callable, selector: Callable) -> Callab """ def query_strategy(classifier: BaseEstimator, X: modALinput) -> Tuple: utility = utility_measure(classifier, X) - query_idx = selector(utility) - return query_idx, X[query_idx] + return selector(utility) return query_strategy diff --git a/modAL/utils/data.py b/modAL/utils/data.py index 32976e4..3e707ff 100644 --- a/modAL/utils/data.py +++ b/modAL/utils/data.py @@ -1,16 +1,21 @@ -from typing import Union, Container -from itertools import chain +from typing import List, Sequence, Union import numpy as np +import pandas as pd import scipy.sparse as sp +try: + import torch +except: + pass -modALinput = Union[list, np.ndarray, sp.csr_matrix] +modALinput = Union[sp.csr_matrix, pd.DataFrame, np.ndarray, list] -def data_vstack(blocks: Container) -> modALinput: + +def data_vstack(blocks: Sequence[modALinput]) -> modALinput: """ - Stack vertically both sparse and dense arrays. + Stack vertically sparse/dense arrays and pandas data frames. Args: blocks: Sequence of modALinput objects. @@ -18,11 +23,161 @@ def data_vstack(blocks: Container) -> modALinput: Returns: New sequence of vertically stacked elements. """ - if isinstance(blocks[0], np.ndarray): + if any([sp.issparse(b) for b in blocks]): + return sp.vstack(blocks) + elif isinstance(blocks[0], pd.DataFrame): + return blocks[0].append(blocks[1:]) + elif isinstance(blocks[0], np.ndarray): return np.concatenate(blocks) elif isinstance(blocks[0], list): - return list(chain(blocks)) - elif sp.issparse(blocks[0]): - return sp.vstack(blocks) - else: - raise TypeError('%s datatype is not supported' % type(blocks[0])) + return np.concatenate(blocks).tolist() + + try: + if torch.is_tensor(blocks[0]): + return torch.cat(blocks) + except: + pass + + raise TypeError("%s datatype is not supported" % type(blocks[0])) + + +def data_hstack(blocks: Sequence[modALinput]) -> modALinput: + """ + Stack horizontally sparse/dense arrays and pandas data frames. + + Args: + blocks: Sequence of modALinput objects. + + Returns: + New sequence of horizontally stacked elements. + """ + if any([sp.issparse(b) for b in blocks]): + return sp.hstack(blocks) + elif isinstance(blocks[0], pd.DataFrame): + pd.concat(blocks, axis=1) + elif isinstance(blocks[0], np.ndarray): + return np.hstack(blocks) + elif isinstance(blocks[0], list): + return np.hstack(blocks).tolist() + + try: + if torch.is_tensor(blocks[0]): + return torch.cat(blocks, dim=1) + except: + pass + + TypeError("%s datatype is not supported" % type(blocks[0])) + + +def add_row(X: modALinput, row: modALinput): + """ + Returns X' = + + [X + + row] """ + if isinstance(X, np.ndarray): + return np.vstack((X, row)) + elif isinstance(X, list): + return np.vstack((X, row)).tolist() + + # data_vstack readily supports stacking of matrix as first argument + # and row as second for the other data types + return data_vstack([X, row]) + + +def retrieve_rows( + X: modALinput, I: Union[int, List[int], np.ndarray] +) -> Union[sp.csc_matrix, np.ndarray, pd.DataFrame]: + """ + Returns the rows I from the data set X + + For a single index, the result is as follows: + * 1xM matrix in case of scipy sparse NxM matrix X + * pandas series in case of a pandas data frame + * row in case of list or numpy format + """ + + try: + return X[I] + except: + if sp.issparse(X): + # Out of the sparse matrix formats (sp.csc_matrix, sp.csr_matrix, sp.bsr_matrix, + # sp.lil_matrix, sp.dok_matrix, sp.coo_matrix, sp.dia_matrix), only sp.bsr_matrix, sp.coo_matrix + # and sp.dia_matrix don't support indexing and need to be converted to a sparse format + # that does support indexing. It seems conversion to CSR is currently most efficient. + + sp_format = X.getformat() + return X.tocsr()[I].asformat(sp_format) + elif isinstance(X, pd.DataFrame): + return X.iloc[I] + elif isinstance(X, list): + return np.array(X)[I].tolist() + elif isinstance(X, dict): + X_return = {} + for key, value in X.items(): + X_return[key] = retrieve_rows(value, I) + return X_return + + raise TypeError("%s datatype is not supported" % type(X)) + + +def drop_rows( + X: modALinput, I: Union[int, List[int], np.ndarray] +) -> Union[sp.csc_matrix, np.ndarray, pd.DataFrame]: + """ + Returns X without the row(s) at index/indices I + """ + if sp.issparse(X): + mask = np.ones(X.shape[0], dtype=bool) + mask[I] = False + return retrieve_rows(X, mask) + elif isinstance(X, pd.DataFrame): + return X.drop(I, axis=0) + elif isinstance(X, np.ndarray): + return np.delete(X, I, axis=0) + elif isinstance(X, list): + return np.delete(X, I, axis=0).tolist() + + try: + if torch.is_tensor(blocks[0]): + return torch.cat(blocks) + except: + X[[True if row not in I else False for row in range(X.size(0))]] + + raise TypeError("%s datatype is not supported" % type(X)) + + +def enumerate_data(X: modALinput): + """ + for i, x in enumerate_data(X): + + Depending on the data type of X, returns: + + * A 1xM matrix in case of scipy sparse NxM matrix X + * pandas series in case of a pandas data frame X + * row in case of list or numpy format + """ + if sp.issparse(X): + return enumerate(X.tocsr()) + elif isinstance(X, pd.DataFrame): + return X.iterrows() + elif isinstance(X, np.ndarray) or isinstance(X, list): + # numpy arrays and lists can readily be enumerated + return enumerate(X) + + raise TypeError("%s datatype is not supported" % type(X)) + + +def data_shape(X: modALinput): + """ + Returns the shape of the data set X + """ + try: + # scipy.sparse, torch, pandas and numpy all support .shape + return X.shape + except: + if isinstance(X, list): + return np.array(X).shape + + raise TypeError("%s datatype is not supported" % type(X)) diff --git a/modAL/utils/selection.py b/modAL/utils/selection.py index 73700b8..6c9c2d9 100644 --- a/modAL/utils/selection.py +++ b/modAL/utils/selection.py @@ -11,17 +11,13 @@ def shuffled_argmax(values: np.ndarray, n_instances: int = 1) -> np.ndarray: the tie when the highest utility score is not unique. The shuffle randomizes order, which is preserved by the mergesort algorithm. - Args: - values: - n_instances: - Args: values: Contains the values to be selected from. - n_instances: Specifies how many indices to return. - + n_instances: Specifies how many indices and values to return. Returns: - The indices of the n_instances largest values. + The indices and values of the n_instances largest values. """ + assert n_instances <= values.shape[0], 'n_instances must be less or equal than the size of utility' # shuffling indices and corresponding values shuffled_idx = np.random.permutation(len(values)) @@ -29,28 +25,62 @@ def shuffled_argmax(values: np.ndarray, n_instances: int = 1) -> np.ndarray: # getting the n_instances best instance # since mergesort is used, the shuffled order is preserved - sorted_query_idx = np.argsort(shuffled_values, kind='mergesort')[:n_instances] + sorted_query_idx = np.argsort(shuffled_values, kind='mergesort')[ + len(shuffled_values)-n_instances:] # inverting the shuffle query_idx = shuffled_idx[sorted_query_idx] - return query_idx + return query_idx, values[query_idx] -def multi_argmax(values: np.ndarray, n_instances: int = 1) -> np.ndarray: + +def shuffled_argmin(values: np.ndarray, n_instances: int = 1) -> np.ndarray: """ - Selects the indices of the n_instances highest values. + Shuffles the values and sorts them afterwards. This can be used to break + the tie when the highest utility score is not unique. The shuffle randomizes + order, which is preserved by the mergesort algorithm. Args: values: Contains the values to be selected from. - n_instances: Specifies how many indices to return. + n_instances: Specifies how many indices and values to return. + Returns: + The indices and values of the n_instances smallest values. + """ + + indexes, index_values = shuffled_argmax(-values, n_instances) + + return indexes, -index_values + +def multi_argmax(values: np.ndarray, n_instances: int = 1) -> np.ndarray: + """ + return the indices and values of the n_instances highest values. + + Args: + values: Contains the values to be selected from. + n_instances: Specifies how many indices and values to return. Returns: - The indices of the n_instances largest values. + The indices and values of the n_instances largest values. """ assert n_instances <= values.shape[0], 'n_instances must be less or equal than the size of utility' max_idx = np.argpartition(-values, n_instances-1, axis=0)[:n_instances] - return max_idx + + return max_idx, values[max_idx] + + +def multi_argmin(values: np.ndarray, n_instances: int = 1) -> np.ndarray: + """ + return the indices and values of the n_instances smallest values. + + Args: + values: Contains the values to be selected from. + n_instances: Specifies how many indices and values to return. + Returns: + The indices and values of the n_instances smallest values. + """ + indexes, index_values = multi_argmax(-values, n_instances) + return indexes, -index_values def weighted_random(weights: np.ndarray, n_instances: int = 1) -> np.ndarray: @@ -68,5 +98,6 @@ def weighted_random(weights: np.ndarray, n_instances: int = 1) -> np.ndarray: weight_sum = np.sum(weights) assert weight_sum > 0, 'the sum of weights must be larger than zero' - random_idx = np.random.choice(range(len(weights)), size=n_instances, p=weights/weight_sum, replace=False) + random_idx = np.random.choice( + range(len(weights)), size=n_instances, p=weights/weight_sum, replace=False) return random_idx diff --git a/modAL/utils/validation.py b/modAL/utils/validation.py index e5763de..93667db 100644 --- a/modAL/utils/validation.py +++ b/modAL/utils/validation.py @@ -1,8 +1,8 @@ from typing import Sequence import numpy as np -from sklearn.exceptions import NotFittedError from sklearn.base import BaseEstimator +from sklearn.exceptions import NotFittedError def check_class_labels(*args: BaseEstimator) -> bool: diff --git a/rtd_requirements.txt b/rtd_requirements.txt index 7d9fe89..db0bd81 100644 --- a/rtd_requirements.txt +++ b/rtd_requirements.txt @@ -1,5 +1,7 @@ -numpy +numpy==1.20.0 scipy scikit-learn ipykernel nbsphinx +pandas +skorch diff --git a/setup.py b/setup.py index be59e7b..3905e35 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,8 @@ -from setuptools import setup, find_packages +from setuptools import find_packages, setup setup( - name='modAL', - version='0.3.4', + name='modAL-python', + version='0.4.2', author='Tivadar Danka', author_email='85a5187a@opayq.com', description='A modular active learning framework for Python3', @@ -10,5 +10,6 @@ url='https://modAL-python.github.io/', packages=['modAL', 'modAL.models', 'modAL.utils'], classifiers=['Development Status :: 4 - Beta'], - install_requires=['numpy>=1.13', 'scikit-learn>=0.18', 'scipy>=0.18'], + install_requires=['numpy', 'scikit-learn>=0.18', + 'scipy>=0.18', 'pandas>=1.1.0', 'skorch==0.9.0'], ) diff --git a/tests/core_tests.py b/tests/core_tests.py index 1374ab1..e3113c4 100644 --- a/tests/core_tests.py +++ b/tests/core_tests.py @@ -1,35 +1,42 @@ import random import unittest -import numpy as np +from collections import namedtuple +from copy import deepcopy +from itertools import chain, product +from unittest.mock import MagicMock -import mock -import modAL.models.base -import modAL.models.learners -import modAL.utils.selection -import modAL.utils.validation -import modAL.utils.combination import modAL.acquisition import modAL.batch import modAL.density import modAL.disagreement +import modAL.dropout import modAL.expected_error +import modAL.models.base +import modAL.models.learners import modAL.multilabel import modAL.uncertainty - -from copy import deepcopy -from itertools import chain, product -from collections import namedtuple - +import modAL.utils.combination +import modAL.utils.selection +import modAL.utils.validation +import numpy as np +import pandas as pd +import torch +from scipy import sparse as sp +from scipy.special import ndtr +from scipy.stats import entropy, norm from sklearn.ensemble import RandomForestClassifier -from sklearn.gaussian_process import GaussianProcessRegressor from sklearn.exceptions import NotFittedError +from sklearn.feature_extraction.text import CountVectorizer +from sklearn.gaussian_process import GaussianProcessRegressor from sklearn.metrics import confusion_matrix -from sklearn.svm import SVC from sklearn.multiclass import OneVsRestClassifier -from scipy.stats import entropy, norm -from scipy.special import ndtr -from scipy import sparse as sp +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import FunctionTransformer +from sklearn.svm import SVC +from skorch import NeuralNetClassifier +from torch import nn +import mock Test = namedtuple('Test', ['input', 'output']) @@ -46,18 +53,27 @@ def test_check_class_labels(self): for n_learners in range(1, 10): # 1. test fitted estimators labels = np.random.randint(10, size=n_labels) - different_labels = np.random.randint(10, 20, size=np.random.randint(1, 10)) - learner_list_1 = [mock.MockEstimator(classes_=labels) for _ in range(n_learners)] - learner_list_2 = [mock.MockEstimator(classes_=different_labels) for _ in range(np.random.randint(1, 5))] - shuffled_learners = random.sample(learner_list_1 + learner_list_2, len(learner_list_1 + learner_list_2)) - self.assertTrue(modAL.utils.validation.check_class_labels(*learner_list_1)) - self.assertFalse(modAL.utils.validation.check_class_labels(*shuffled_learners)) + different_labels = np.random.randint( + 10, 20, size=np.random.randint(1, 10)) + learner_list_1 = [mock.MockEstimator( + classes_=labels) for _ in range(n_learners)] + learner_list_2 = [mock.MockEstimator( + classes_=different_labels) for _ in range(np.random.randint(1, 5))] + shuffled_learners = random.sample( + learner_list_1 + learner_list_2, len(learner_list_1 + learner_list_2)) + self.assertTrue( + modAL.utils.validation.check_class_labels(*learner_list_1)) + self.assertFalse( + modAL.utils.validation.check_class_labels(*shuffled_learners)) # 2. test unfitted estimators - unfitted_learner_list = [mock.MockEstimator(classes_=labels) for _ in range(n_learners)] + unfitted_learner_list = [mock.MockEstimator( + classes_=labels) for _ in range(n_learners)] idx = np.random.randint(0, n_learners) - unfitted_learner_list.insert(idx, mock.MockEstimator(fitted=False)) - self.assertRaises(NotFittedError, modAL.utils.validation.check_class_labels, *unfitted_learner_list) + unfitted_learner_list.insert( + idx, mock.MockEstimator(fitted=False)) + self.assertRaises( + NotFittedError, modAL.utils.validation.check_class_labels, *unfitted_learner_list) def test_check_class_proba(self): for n_labels in range(2, 20): @@ -65,16 +81,19 @@ def test_check_class_proba(self): proba = np.random.rand(100, n_labels) class_labels = list(range(n_labels)) np.testing.assert_almost_equal( - modAL.utils.check_class_proba(proba, known_labels=class_labels, all_labels=class_labels), + modAL.utils.check_class_proba( + proba, known_labels=class_labels, all_labels=class_labels), proba ) for unknown_idx in range(n_labels): all_labels = list(range(n_labels)) known_labels = deepcopy(all_labels) known_labels.remove(unknown_idx) - aug_proba = np.insert(proba[:, known_labels], unknown_idx, np.zeros(len(proba)), axis=1) + aug_proba = np.insert( + proba[:, known_labels], unknown_idx, np.zeros(len(proba)), axis=1) np.testing.assert_almost_equal( - modAL.utils.check_class_proba(proba[:, known_labels], known_labels=known_labels, all_labels=all_labels), + modAL.utils.check_class_proba( + proba[:, known_labels], known_labels=known_labels, all_labels=all_labels), aug_proba ) @@ -87,7 +106,8 @@ def dummy_function(X_in): for n_features in range(1, 10): for n_functions in range(2, 10): functions = [dummy_function for _ in range(n_functions)] - linear_combination = modAL.utils.combination.make_linear_combination(*functions) + linear_combination = modAL.utils.combination.make_linear_combination( + *functions) X_in = np.random.rand(n_samples, n_features) if n_samples == 1: @@ -96,7 +116,8 @@ def dummy_function(X_in): true_result = n_functions*np.ones(shape=(n_samples, 1)) try: - np.testing.assert_almost_equal(linear_combination(X_in), true_result) + np.testing.assert_almost_equal( + linear_combination(X_in), true_result) except: linear_combination(X_in) @@ -115,7 +136,8 @@ def test_product(self): # linear combination with weights exponents = np.random.rand(n_functions) - exp_product = modAL.utils.combination.make_product(*functions, exponents=exponents) + exp_product = modAL.utils.combination.make_product( + *functions, exponents=exponents) np.testing.assert_almost_equal( exp_product(X_in), np.prod([X_in**exponent for exponent in exponents], axis=0) @@ -140,13 +162,13 @@ def test_make_query_strategy(self): query_1 = query_strategy(learner, X) query_2 = modAL.uncertainty.uncertainty_sampling(learner, X) - np.testing.assert_equal(query_1[0], query_2[0]) - np.testing.assert_almost_equal(query_1[1], query_2[1]) + np.testing.assert_equal(query_1, query_2) def test_data_vstack(self): for n_samples, n_features in product(range(1, 10), range(1, 10)): # numpy arrays - a, b = np.random.rand(n_samples, n_features), np.random.rand(n_samples, n_features) + a, b = np.random.rand(n_samples, n_features), np.random.rand( + n_samples, n_features) np.testing.assert_almost_equal( modAL.utils.data.data_vstack((a, b)), np.concatenate((a, b)) @@ -154,33 +176,55 @@ def test_data_vstack(self): # sparse matrices for format in ['lil', 'csc', 'csr']: - a, b = sp.random(n_samples, n_features, format=format), sp.random(n_samples, n_features, format=format) - self.assertEqual((modAL.utils.data.data_vstack((a, b)) != sp.vstack((a, b))).sum(), 0) + a, b = sp.random(n_samples, n_features, format=format), sp.random( + n_samples, n_features, format=format) + self.assertEqual((modAL.utils.data.data_vstack( + (a, b)) != sp.vstack((a, b))).sum(), 0) + + # lists + a, b = np.random.rand(n_samples, n_features).tolist(), np.random.rand( + n_samples, n_features).tolist() + np.testing.assert_almost_equal( + modAL.utils.data.data_vstack((a, b)), + np.concatenate((a, b)) + ) + + # torch.Tensors + a, b = torch.ones(2, 2), torch.ones(2, 2) + torch.testing.assert_allclose( + modAL.utils.data.data_vstack((a, b)), + torch.cat((a, b)) + ) # not supported formats self.assertRaises(TypeError, modAL.utils.data.data_vstack, (1, 1)) - # functions from modAL.utils.selection + # functions from modALu.tils.selection def test_multi_argmax(self): for n_pool in range(2, 100): - for n_instances in range(1, n_pool): + for n_instances in range(1, n_pool+1): utility = np.zeros(n_pool) - max_idx = np.random.choice(range(n_pool), size=n_instances, replace=False) + max_idx = np.random.choice( + range(n_pool), size=n_instances, replace=False) utility[max_idx] = 1e-10 + np.random.rand(n_instances, ) np.testing.assert_equal( - np.sort(modAL.utils.selection.multi_argmax(utility, n_instances)), - np.sort(max_idx) + np.sort(modAL.utils.selection.multi_argmax( + utility, n_instances)), + (np.sort(max_idx), np.sort(utility) + [len(utility)-n_instances:]) ) def test_shuffled_argmax(self): for n_pool in range(1, 100): for n_instances in range(1, n_pool+1): values = np.random.permutation(n_pool) - true_query_idx = np.argsort(values)[:n_instances] + true_query_idx = np.argsort(values)[len(values)-n_instances:] + true_values = np.sort(values, axis=None)[ + len(values)-n_instances:] np.testing.assert_equal( - true_query_idx, + (true_query_idx, true_values), modAL.utils.selection.shuffled_argmax(values, n_instances) ) @@ -188,11 +232,13 @@ def test_weighted_random(self): for n_pool in range(2, 100): for n_instances in range(1, n_pool): utility = np.ones(n_pool) - query_idx = modAL.utils.selection.weighted_random(utility, n_instances) + query_idx = modAL.utils.selection.weighted_random( + utility, n_instances) # testing for correct number of returned indices np.testing.assert_equal(len(query_idx), n_instances) # testing for uniqueness of each query index - np.testing.assert_equal(len(query_idx), len(np.unique(query_idx))) + np.testing.assert_equal( + len(query_idx), len(np.unique(query_idx))) class TestAcquisitionFunctions(unittest.TestCase): @@ -210,37 +256,42 @@ def test_acquisition_functions(self): def test_optimizer_PI(self): for n_samples in range(1, 100): - mean = np.random.rand(n_samples, 1) - std = np.random.rand(n_samples, 1) + mean = np.random.rand(n_samples, ) + std = np.random.rand(n_samples, ) tradeoff = np.random.rand() max_val = np.random.rand() # 1. fitted estimator mock_estimator = mock.MockEstimator(predict_return=(mean, std)) - optimizer = modAL.models.learners.BayesianOptimizer(estimator=mock_estimator) + optimizer = modAL.models.learners.BayesianOptimizer( + estimator=mock_estimator) optimizer._set_max([0], [max_val]) true_PI = ndtr((mean - max_val - tradeoff)/std) np.testing.assert_almost_equal( true_PI, - modAL.acquisition.optimizer_PI(optimizer, np.random.rand(n_samples, 2), tradeoff) + modAL.acquisition.optimizer_PI( + optimizer, np.random.rand(n_samples, 2), tradeoff) ) # 2. unfitted estimator mock_estimator = mock.MockEstimator(fitted=False) - optimizer = modAL.models.learners.BayesianOptimizer(estimator=mock_estimator) + optimizer = modAL.models.learners.BayesianOptimizer( + estimator=mock_estimator) optimizer._set_max([0], [max_val]) - true_PI = ndtr((np.zeros(shape=(len(mean), 1)) - max_val - tradeoff) / np.ones(shape=(len(mean), 1))) + true_PI = ndtr((np.zeros(shape=(len(mean), 1)) - + max_val - tradeoff) / np.ones(shape=(len(mean), 1))) np.testing.assert_almost_equal( true_PI, - modAL.acquisition.optimizer_PI(optimizer, np.random.rand(n_samples, 2), tradeoff) + modAL.acquisition.optimizer_PI( + optimizer, np.random.rand(n_samples, 2), tradeoff) ) def test_optimizer_EI(self): for n_samples in range(1, 100): - mean = np.random.rand(n_samples, 1) - std = np.random.rand(n_samples, 1) + mean = np.random.rand(n_samples, ) + std = np.random.rand(n_samples, ) tradeoff = np.random.rand() max_val = np.random.rand() @@ -248,54 +299,64 @@ def test_optimizer_EI(self): mock_estimator = mock.MockEstimator( predict_return=(mean, std) ) - optimizer = modAL.models.learners.BayesianOptimizer(estimator=mock_estimator) + optimizer = modAL.models.learners.BayesianOptimizer( + estimator=mock_estimator) optimizer._set_max([0], [max_val]) true_EI = (mean - optimizer.y_max - tradeoff) * ndtr((mean - optimizer.y_max - tradeoff) / std) \ - + std * norm.pdf((mean - optimizer.y_max - tradeoff) / std) + + std * norm.pdf((mean - optimizer.y_max - tradeoff) / std) np.testing.assert_almost_equal( true_EI, - modAL.acquisition.optimizer_EI(optimizer, np.random.rand(n_samples, 2), tradeoff) + modAL.acquisition.optimizer_EI( + optimizer, np.random.rand(n_samples, 2), tradeoff) ) # 2. unfitted estimator mock_estimator = mock.MockEstimator(fitted=False) - optimizer = modAL.models.learners.BayesianOptimizer(estimator=mock_estimator) + optimizer = modAL.models.learners.BayesianOptimizer( + estimator=mock_estimator) optimizer._set_max([0], [max_val]) true_EI = (np.zeros(shape=(len(mean), 1)) - optimizer.y_max - tradeoff) * ndtr((np.zeros(shape=(len(mean), 1)) - optimizer.y_max - tradeoff) / np.ones(shape=(len(mean), 1))) \ - + np.ones(shape=(len(mean), 1)) * norm.pdf((np.zeros(shape=(len(mean), 1)) - optimizer.y_max - tradeoff) / np.ones(shape=(len(mean), 1))) + + np.ones(shape=(len(mean), 1)) * norm.pdf((np.zeros(shape=(len(mean), 1) + ) - optimizer.y_max - tradeoff) / np.ones(shape=(len(mean), 1))) np.testing.assert_almost_equal( true_EI, - modAL.acquisition.optimizer_EI(optimizer, np.random.rand(n_samples, 2), tradeoff) + modAL.acquisition.optimizer_EI( + optimizer, np.random.rand(n_samples, 2), tradeoff) ) def test_optimizer_UCB(self): for n_samples in range(1, 100): - mean = np.random.rand(n_samples, 1) - std = np.random.rand(n_samples, 1) + mean = np.random.rand(n_samples, ) + std = np.random.rand(n_samples, ) beta = np.random.rand() # 1. fitted estimator mock_estimator = mock.MockEstimator( predict_return=(mean, std) ) - optimizer = modAL.models.learners.BayesianOptimizer(estimator=mock_estimator) + optimizer = modAL.models.learners.BayesianOptimizer( + estimator=mock_estimator) true_UCB = mean + beta*std np.testing.assert_almost_equal( true_UCB, - modAL.acquisition.optimizer_UCB(optimizer, np.random.rand(n_samples, 2), beta) + modAL.acquisition.optimizer_UCB( + optimizer, np.random.rand(n_samples, 2), beta) ) # 2. unfitted estimator mock_estimator = mock.MockEstimator(fitted=False) - optimizer = modAL.models.learners.BayesianOptimizer(estimator=mock_estimator) - true_UCB = np.zeros(shape=(len(mean), 1)) + beta * np.ones(shape=(len(mean), 1)) + optimizer = modAL.models.learners.BayesianOptimizer( + estimator=mock_estimator) + true_UCB = np.zeros(shape=(len(mean), 1)) + \ + beta * np.ones(shape=(len(mean), 1)) np.testing.assert_almost_equal( true_UCB, - modAL.acquisition.optimizer_UCB(optimizer, np.random.rand(n_samples, 2), beta) + modAL.acquisition.optimizer_UCB( + optimizer, np.random.rand(n_samples, 2), beta) ) def test_selection(self): @@ -310,12 +371,16 @@ def test_selection(self): predict_return=(mean, std) ) - optimizer = modAL.models.learners.BayesianOptimizer(estimator=mock_estimator) + optimizer = modAL.models.learners.BayesianOptimizer( + estimator=mock_estimator) optimizer._set_max([0], [max_val]) - modAL.acquisition.max_PI(optimizer, X, tradeoff=np.random.rand(), n_instances=n_instances) - modAL.acquisition.max_EI(optimizer, X, tradeoff=np.random.rand(), n_instances=n_instances) - modAL.acquisition.max_UCB(optimizer, X, beta=np.random.rand(), n_instances=n_instances) + modAL.acquisition.max_PI( + optimizer, X, tradeoff=np.random.rand(), n_instances=n_instances) + modAL.acquisition.max_EI( + optimizer, X, tradeoff=np.random.rand(), n_instances=n_instances) + modAL.acquisition.max_UCB( + optimizer, X, beta=np.random.rand(), n_instances=n_instances) class TestDensity(unittest.TestCase): @@ -346,15 +411,20 @@ def test_vote_entropy(self): for n_classes in range(1, 10): for true_query_idx in range(n_samples): # 1. fitted committee - vote_return = np.zeros(shape=(n_samples, n_classes), dtype=np.int16) - vote_return[true_query_idx] = np.asarray(range(n_classes), dtype=np.int16) - committee = mock.MockCommittee(classes_=np.asarray(range(n_classes)), vote_return=vote_return) + vote_return = np.zeros( + shape=(n_samples, n_classes), dtype=np.int16) + vote_return[true_query_idx] = np.asarray( + range(n_classes), dtype=np.int16) + committee = mock.MockCommittee(classes_=np.asarray( + range(n_classes)), vote_return=vote_return) vote_entr = modAL.disagreement.vote_entropy( committee, np.random.rand(n_samples, n_classes) ) true_entropy = np.zeros(shape=(n_samples, )) - true_entropy[true_query_idx] = entropy(np.ones(n_classes)/n_classes) - np.testing.assert_array_almost_equal(vote_entr, true_entropy) + true_entropy[true_query_idx] = entropy( + np.ones(n_classes)/n_classes) + np.testing.assert_array_almost_equal( + vote_entr, true_entropy) # 2. unfitted committee committee = mock.MockCommittee(fitted=False) @@ -377,8 +447,10 @@ def test_consensus_entropy(self): committee, np.random.rand(n_samples, n_classes) ) true_entropy = np.zeros(shape=(n_samples,)) - true_entropy[true_query_idx] = entropy(np.ones(n_classes) / n_classes) - np.testing.assert_array_almost_equal(consensus_entropy, true_entropy) + true_entropy[true_query_idx] = entropy( + np.ones(n_classes) / n_classes) + np.testing.assert_array_almost_equal( + consensus_entropy, true_entropy) # 2. unfitted committee committee = mock.MockCommittee(fitted=False) @@ -386,14 +458,16 @@ def test_consensus_entropy(self): consensus_entropy = modAL.disagreement.consensus_entropy( committee, np.random.rand(n_samples, n_classes) ) - np.testing.assert_almost_equal(consensus_entropy, true_entropy) + np.testing.assert_almost_equal( + consensus_entropy, true_entropy) def test_KL_max_disagreement(self): for n_samples in range(1, 10): for n_classes in range(2, 10): - for n_learners in range (2, 10): + for n_learners in range(2, 10): # 1. fitted committee - vote_proba = np.zeros(shape=(n_samples, n_learners, n_classes)) + vote_proba = np.zeros( + shape=(n_samples, n_learners, n_classes)) vote_proba[:, :, 0] = 1.0 committee = mock.MockCommittee( n_learners=n_learners, classes_=range(n_classes), @@ -405,10 +479,12 @@ def test_KL_max_disagreement(self): try: np.testing.assert_array_almost_equal( true_KL_disagreement, - modAL.disagreement.KL_max_disagreement(committee, np.random.rand(n_samples, 1)) + modAL.disagreement.KL_max_disagreement( + committee, np.random.rand(n_samples, 1)) ) except: - modAL.disagreement.KL_max_disagreement(committee, np.random.rand(n_samples, 1)) + modAL.disagreement.KL_max_disagreement( + committee, np.random.rand(n_samples, 1)) # 2. unfitted committee committee = mock.MockCommittee(fitted=False) @@ -416,20 +492,24 @@ def test_KL_max_disagreement(self): returned_KL_disagreement = modAL.disagreement.KL_max_disagreement( committee, np.random.rand(n_samples, n_classes) ) - np.testing.assert_almost_equal(returned_KL_disagreement, true_KL_disagreement) + np.testing.assert_almost_equal( + returned_KL_disagreement, true_KL_disagreement) def test_vote_entropy_sampling(self): for n_samples, n_features, n_classes in product(range(1, 10), range(1, 10), range(1, 10)): committee = mock.MockCommittee(classes_=np.asarray(range(n_classes)), vote_return=np.zeros(shape=(n_samples, n_classes), dtype=np.int16)) - modAL.disagreement.vote_entropy_sampling(committee, np.random.rand(n_samples, n_features)) + modAL.disagreement.vote_entropy_sampling( + committee, np.random.rand(n_samples, n_features)) modAL.disagreement.vote_entropy_sampling(committee, np.random.rand(n_samples, n_features), random_tie_break=True) def test_consensus_entropy_sampling(self): for n_samples, n_features, n_classes in product(range(1, 10), range(1, 10), range(1, 10)): - committee = mock.MockCommittee(predict_proba_return=np.random.rand(n_samples, n_classes)) - modAL.disagreement.consensus_entropy_sampling(committee, np.random.rand(n_samples, n_features)) + committee = mock.MockCommittee( + predict_proba_return=np.random.rand(n_samples, n_classes)) + modAL.disagreement.consensus_entropy_sampling( + committee, np.random.rand(n_samples, n_features)) modAL.disagreement.consensus_entropy_sampling(committee, np.random.rand(n_samples, n_features), random_tie_break=True) @@ -437,17 +517,21 @@ def test_max_disagreement_sampling(self): for n_samples, n_features, n_classes, n_learners in product(range(1, 10), range(1, 10), range(1, 10), range(2, 5)): committee = mock.MockCommittee( n_learners=n_learners, classes_=range(n_classes), - vote_proba_return=np.zeros(shape=(n_samples, n_learners, n_classes)) + vote_proba_return=np.zeros( + shape=(n_samples, n_learners, n_classes)) ) - modAL.disagreement.max_disagreement_sampling(committee, np.random.rand(n_samples, n_features)) + modAL.disagreement.max_disagreement_sampling( + committee, np.random.rand(n_samples, n_features)) modAL.disagreement.max_disagreement_sampling(committee, np.random.rand(n_samples, n_features), random_tie_break=True) def test_max_std_sampling(self): for n_samples, n_features in product(range(1, 10), range(1, 10)): regressor = GaussianProcessRegressor() - regressor.fit(np.random.rand(n_samples, n_features), np.random.rand(n_samples)) - modAL.disagreement.max_std_sampling(regressor, np.random.rand(n_samples, n_features)) + regressor.fit(np.random.rand(n_samples, n_features), + np.random.rand(n_samples)) + modAL.disagreement.max_std_sampling( + regressor, np.random.rand(n_samples, n_features)) modAL.disagreement.max_std_sampling(regressor, np.random.rand(n_samples, n_features), random_tie_break=True) @@ -455,21 +539,30 @@ def test_max_std_sampling(self): class TestEER(unittest.TestCase): def test_eer(self): for n_pool, n_features, n_classes in product(range(5, 10), range(1, 5), range(2, 5)): - X_training, y_training = np.random.rand(10, n_features), np.random.randint(0, n_classes, size=10) - X_pool, y_pool = np.random.rand(n_pool, n_features), np.random.randint(0, n_classes+1, size=n_pool) - - learner = modAL.models.ActiveLearner(RandomForestClassifier(n_estimators=2), - X_training=X_training, y_training=y_training) - - modAL.expected_error.expected_error_reduction(learner, X_pool) - modAL.expected_error.expected_error_reduction(learner, X_pool, random_tie_break=True) - modAL.expected_error.expected_error_reduction(learner, X_pool, p_subsample=0.1) - modAL.expected_error.expected_error_reduction(learner, X_pool, loss='binary') - modAL.expected_error.expected_error_reduction(learner, X_pool, p_subsample=0.1, loss='log') - self.assertRaises(AssertionError, modAL.expected_error.expected_error_reduction, - learner, X_pool, p_subsample=1.5) - self.assertRaises(AssertionError, modAL.expected_error.expected_error_reduction, - learner, X_pool, loss=42) + X_training_, y_training = np.random.rand( + 10, n_features).tolist(), np.random.randint(0, n_classes, size=10) + X_pool_, y_pool = np.random.rand(n_pool, n_features).tolist( + ), np.random.randint(0, n_classes+1, size=n_pool) + + for data_type in (sp.csr_matrix, pd.DataFrame, np.array, list): + X_training, X_pool = data_type(X_training_), data_type(X_pool_) + + learner = modAL.models.ActiveLearner(RandomForestClassifier(n_estimators=2), + X_training=X_training, y_training=y_training) + + modAL.expected_error.expected_error_reduction(learner, X_pool) + modAL.expected_error.expected_error_reduction( + learner, X_pool, random_tie_break=True) + modAL.expected_error.expected_error_reduction( + learner, X_pool, p_subsample=0.1) + modAL.expected_error.expected_error_reduction( + learner, X_pool, loss='binary') + modAL.expected_error.expected_error_reduction( + learner, X_pool, p_subsample=0.1, loss='log') + self.assertRaises(AssertionError, modAL.expected_error.expected_error_reduction, + learner, X_pool, p_subsample=1.5) + self.assertRaises(AssertionError, modAL.expected_error.expected_error_reduction, + learner, X_pool, loss=42) class TestUncertainties(unittest.TestCase): @@ -485,24 +578,27 @@ def test_classifier_uncertainty(self): ) # fitted estimator - fitted_estimator = mock.MockEstimator(predict_proba_return=case.input) + fitted_estimator = mock.MockEstimator( + predict_proba_return=case.input) np.testing.assert_almost_equal( - modAL.uncertainty.classifier_uncertainty(fitted_estimator, np.random.rand(10)), + modAL.uncertainty.classifier_uncertainty( + fitted_estimator, np.random.rand(10)), case.output ) # not fitted estimator not_fitted_estimator = mock.MockEstimator(fitted=False) np.testing.assert_almost_equal( - modAL.uncertainty.classifier_uncertainty(not_fitted_estimator, case.input), + modAL.uncertainty.classifier_uncertainty( + not_fitted_estimator, case.input), np.ones(shape=(len(case.output))) ) def test_classifier_margin(self): test_cases_1 = (Test(p * np.ones(shape=(k, l)), np.zeros(shape=(k,))) - for k in range(1, 100) for l in range(1, 10) for p in np.linspace(0, 1, 11)) + for k in range(1, 100) for l in range(1, 10) for p in np.linspace(0, 1, 11)) test_cases_2 = (Test(p * np.tile(np.asarray(range(k))+1.0, l).reshape(l, k), - p * np.ones(shape=(l, ))*int(k!=1)) + p * np.ones(shape=(l, ))*int(k != 1)) for k in range(1, 10) for l in range(1, 100) for p in np.linspace(0, 1, 11)) for case in chain(test_cases_1, test_cases_2): # _proba_margin @@ -512,16 +608,19 @@ def test_classifier_margin(self): ) # fitted estimator - fitted_estimator = mock.MockEstimator(predict_proba_return=case.input) + fitted_estimator = mock.MockEstimator( + predict_proba_return=case.input) np.testing.assert_almost_equal( - modAL.uncertainty.classifier_margin(fitted_estimator, np.random.rand(10)), + modAL.uncertainty.classifier_margin( + fitted_estimator, np.random.rand(10)), case.output ) # not fitted estimator not_fitted_estimator = mock.MockEstimator(fitted=False) np.testing.assert_almost_equal( - modAL.uncertainty.classifier_margin(not_fitted_estimator, case.input), + modAL.uncertainty.classifier_margin( + not_fitted_estimator, case.input), np.zeros(shape=(len(case.output))) ) @@ -539,16 +638,19 @@ def test_classifier_entropy(self): ) # fitted estimator - fitted_estimator = mock.MockEstimator(predict_proba_return=proba) + fitted_estimator = mock.MockEstimator( + predict_proba_return=proba) np.testing.assert_equal( - modAL.uncertainty.classifier_entropy(fitted_estimator, np.random.rand(n_samples, 1)), + modAL.uncertainty.classifier_entropy( + fitted_estimator, np.random.rand(n_samples, 1)), np.zeros(shape=(n_samples, )) ) # not fitted estimator not_fitted_estimator = mock.MockEstimator(fitted=False) np.testing.assert_almost_equal( - modAL.uncertainty.classifier_entropy(not_fitted_estimator, np.random.rand(n_samples, 1)), + modAL.uncertainty.classifier_entropy( + not_fitted_estimator, np.random.rand(n_samples, 1)), np.zeros(shape=(n_samples, )) ) @@ -559,15 +661,18 @@ def test_uncertainty_sampling(self): for true_query_idx in range(n_samples): predict_proba = np.random.rand(n_samples, n_classes) predict_proba[true_query_idx] = max_proba - classifier = mock.MockEstimator(predict_proba_return=predict_proba) - query_idx, query_instance = modAL.uncertainty.uncertainty_sampling( + classifier = mock.MockEstimator( + predict_proba_return=predict_proba) + query_idx, query_metric = modAL.uncertainty.uncertainty_sampling( classifier, np.random.rand(n_samples, n_classes) ) - shuffled_query_idx, shuffled_query_instance = modAL.uncertainty.uncertainty_sampling( + shuffled_query_idx, shuffled_query_metric = modAL.uncertainty.uncertainty_sampling( classifier, np.random.rand(n_samples, n_classes), random_tie_break=True ) np.testing.assert_array_equal(query_idx, true_query_idx) + np.testing.assert_array_equal( + shuffled_query_idx, true_query_idx) def test_margin_sampling(self): for n_samples in range(1, 10): @@ -576,15 +681,19 @@ def test_margin_sampling(self): predict_proba = np.zeros(shape=(n_samples, n_classes)) predict_proba[:, 0] = 1.0 predict_proba[true_query_idx, 0] = 0.0 - classifier = mock.MockEstimator(predict_proba_return=predict_proba) - query_idx, query_instance = modAL.uncertainty.margin_sampling( + classifier = mock.MockEstimator( + predict_proba_return=predict_proba) + + query_idx, query_metric = modAL.uncertainty.margin_sampling( classifier, np.random.rand(n_samples, n_classes) ) - shuffled_query_idx, shuffled_query_instance = modAL.uncertainty.margin_sampling( + shuffled_query_idx, shuffled_query_metric = modAL.uncertainty.margin_sampling( classifier, np.random.rand(n_samples, n_classes), random_tie_break=True ) np.testing.assert_array_equal(query_idx, true_query_idx) + np.testing.assert_array_equal( + shuffled_query_idx, true_query_idx) def test_entropy_sampling(self): for n_samples in range(1, 10): @@ -594,15 +703,224 @@ def test_entropy_sampling(self): predict_proba = np.zeros(shape=(n_samples, n_classes)) predict_proba[:, 0] = 1.0 predict_proba[true_query_idx] = max_proba - classifier = mock.MockEstimator(predict_proba_return=predict_proba) - query_idx, query_instance = modAL.uncertainty.entropy_sampling( + classifier = mock.MockEstimator( + predict_proba_return=predict_proba) + + query_idx, query_metric = modAL.uncertainty.entropy_sampling( classifier, np.random.rand(n_samples, n_classes) ) - shuffled_query_idx, shuffled_query_instance = modAL.uncertainty.entropy_sampling( + shuffled_query_idx, shuffled_query_metric = modAL.uncertainty.entropy_sampling( classifier, np.random.rand(n_samples, n_classes), random_tie_break=True ) np.testing.assert_array_equal(query_idx, true_query_idx) + np.testing.assert_array_equal( + shuffled_query_idx, true_query_idx) + + +# PyTorch model for test cases --> Do not change the layers +class Torch_Model(nn.Module): + def __init__(self,): + super(Torch_Model, self).__init__() + self.convs = nn.Sequential( + nn.Conv2d(1, 32, 3), + nn.ReLU(), + nn.Conv2d(32, 64, 3), + nn.ReLU(), + nn.MaxPool2d(2), + nn.Dropout(0.25) + ) + self.fcs = nn.Sequential( + nn.Linear(12*12*64, 128), + nn.ReLU(), + nn.Dropout(0.5), + nn.Linear(128, 10), + ) + + def forward(self, x): + return x + + +class TestDropout(unittest.TestCase): + def setUp(self): + self.skorch_classifier = NeuralNetClassifier(Torch_Model, + criterion=torch.nn.CrossEntropyLoss, + optimizer=torch.optim.Adam, + train_split=None, + verbose=1) + + def test_mc_dropout_bald(self): + learner = modAL.models.learners.DeepActiveLearner( + estimator=self.skorch_classifier, + query_strategy=modAL.dropout.mc_dropout_bald, + ) + for random_tie_break in [True, False]: + for num_cycles, sample_per_forward_pass in product(range(1, 5), range(1, 5)): + for n_samples, n_classes in product(range(1, 5), range(1, 5)): + for n_instances in range(1, n_samples): + X_pool = torch.randn(n_samples, n_classes) + modAL.dropout.mc_dropout_bald(learner, X_pool, n_instances, random_tie_break, [], + num_cycles, sample_per_forward_pass) + + def test_mc_dropout_mean_st(self): + learner = modAL.models.learners.DeepActiveLearner( + estimator=self.skorch_classifier, + query_strategy=modAL.dropout.mc_dropout_mean_st, + ) + for random_tie_break in [True, False]: + for num_cycles, sample_per_forward_pass in product(range(1, 5), range(1, 5)): + for n_samples, n_classes in product(range(1, 5), range(1, 5)): + for n_instances in range(1, n_samples): + X_pool = torch.randn(n_samples, n_classes) + modAL.dropout.mc_dropout_mean_st(learner, X_pool, n_instances, random_tie_break, [], + num_cycles, sample_per_forward_pass) + + def test_mc_dropout_max_entropy(self): + learner = modAL.models.learners.DeepActiveLearner( + estimator=self.skorch_classifier, + query_strategy=modAL.dropout.mc_dropout_max_entropy, + ) + for random_tie_break in [True, False]: + for num_cycles, sample_per_forward_pass in product(range(1, 5), range(1, 5)): + for n_samples, n_classes in product(range(1, 5), range(1, 5)): + for n_instances in range(1, n_samples): + X_pool = torch.randn(n_samples, n_classes) + modAL.dropout.mc_dropout_max_entropy(learner, X_pool, n_instances, random_tie_break, [], + num_cycles, sample_per_forward_pass) + + def test_mc_dropout_max_variationRatios(self): + learner = modAL.models.learners.DeepActiveLearner( + estimator=self.skorch_classifier, + query_strategy=modAL.dropout.mc_dropout_max_variationRatios, + ) + for random_tie_break in [True, False]: + for num_cycles, sample_per_forward_pass in product(range(1, 5), range(1, 5)): + for n_samples, n_classes in product(range(1, 5), range(1, 5)): + for n_instances in range(1, n_samples): + X_pool = torch.randn(n_samples, n_classes) + modAL.dropout.mc_dropout_max_variationRatios(learner, X_pool, n_instances, random_tie_break, [], + num_cycles, sample_per_forward_pass) + + def test_get_predictions(self): + X = torch.randn(100, 1) + + learner = modAL.models.learners.DeepActiveLearner( + estimator=self.skorch_classifier, + query_strategy=mock.MockFunction(return_val=None), + ) + + # num predictions tests + for num_predictions in range(1, 20): + for samples_per_forward_pass in range(1, 10): + + predictions = modAL.dropout.get_predictions( + learner, X, dropout_layer_indexes=[], + num_predictions=num_predictions, + sample_per_forward_pass=samples_per_forward_pass) + + self.assertEqual(len(predictions), num_predictions) + + self.assertRaises(AssertionError, modAL.dropout.get_predictions, + learner, X, dropout_layer_indexes=[], + num_predictions=-1, + sample_per_forward_pass=0) + + self.assertRaises(AssertionError, modAL.dropout.get_predictions, + learner, X, dropout_layer_indexes=[], + num_predictions=10, + sample_per_forward_pass=-5) + + # logits adapter function test + for samples, classes, subclasses in product(range(1, 10), range(1, 10), range(1, 10)): + input_shape = (samples, classes, subclasses) + desired_shape = (input_shape[0], np.prod(input_shape[1:])) + X_adaption_needed = torch.randn(input_shape) + + def logits_adaptor(input_tensor, data): return torch.flatten( + input_tensor, start_dim=1) + + predictions = modAL.dropout.get_predictions( + learner, X_adaption_needed, dropout_layer_indexes=[], + num_predictions=num_predictions, + sample_per_forward_pass=samples_per_forward_pass, + logits_adaptor=logits_adaptor) + + self.assertEqual(predictions[0].shape, desired_shape) + + def test_set_dropout_mode(self): + # set dropmout mode for all dropout layers + for train_mode in [True, False]: + model = Torch_Model() + modules = list(model.modules()) + + for module in modules: + self.assertEqual(module.training, True) + + modAL.dropout.set_dropout_mode(model, [], train_mode) + + self.assertEqual(modules[7].training, train_mode) + self.assertEqual(modules[11].training, train_mode) + + # set dropout mode only for special layers: + for train_mode in [True, False]: + model = Torch_Model() + modules = list(model.modules()) + modAL.dropout.set_dropout_mode(model, [7], train_mode) + self.assertEqual(modules[7].training, train_mode) + self.assertEqual(modules[11].training, True) + + modAL.dropout.set_dropout_mode(model, [], True) + modAL.dropout.set_dropout_mode(model, [11], train_mode) + self.assertEqual(modules[11].training, train_mode) + self.assertEqual(modules[7].training, True) + + # No Dropout Layer + self.assertRaises(KeyError, modAL.dropout.set_dropout_mode, + model, [5], train_mode) + + +class TestDeepActiveLearner(unittest.TestCase): + """ + Tests for the base class methods of the BaseLearner (base.py) are provided in + the TestActiveLearner. + """ + + def setUp(self): + self.mock_deep_estimator = mock.MockEstimator() + # Add methods that can not be autospecced (because of the wrapper) + self.mock_deep_estimator.initialize = MagicMock(name='initialize') + self.mock_deep_estimator.partial_fit = MagicMock(name='partial_fit') + + def test_teach(self): + + for bootstrap, warm_start in product([True, False], [True, False]): + for n_samples in range(1, 10): + X = torch.randn(n_samples, 1) + y = torch.randn(n_samples) + + learner = modAL.models.learners.DeepActiveLearner( + estimator=self.mock_deep_estimator + ) + + learner.teach(X, y, bootstrap=bootstrap, warm_start=warm_start) + + def test_batch_size(self): + learner = modAL.models.learners.DeepActiveLearner( + estimator=self.mock_deep_estimator + ) + + for batch_size in range(1, 50): + learner.batch_size = batch_size + self.assertEqual(batch_size, learner.batch_size) + + def test_num_epochs(self): + learner = modAL.models.learners.DeepActiveLearner( + estimator=self.mock_deep_estimator + ) + + for num_epochs in range(1, 50): + learner.num_epochs = num_epochs + self.assertEqual(num_epochs, learner.num_epochs) class TestActiveLearner(unittest.TestCase): @@ -631,8 +949,10 @@ def test_add_training_data(self): np.concatenate((y_initial, y_new)) ) # 2. vector class labels - y_initial = np.random.randint(0, 2, size=(n_samples, n_features+1)) - y_new = np.random.randint(0, 2, size=(n_new_samples, n_features+1)) + y_initial = np.random.randint( + 0, 2, size=(n_samples, n_features+1)) + y_new = np.random.randint( + 0, 2, size=(n_new_samples, n_features+1)) learner = modAL.models.learners.ActiveLearner( estimator=mock.MockEstimator(), X_training=X_initial, y_training=y_initial @@ -653,24 +973,25 @@ def test_add_training_data(self): y_new = np.random.randint(0, 2, size=(n_new_samples,)) learner._add_training_data(X_new, y_new) - - # testing for invalid cases # 1. len(X_new) != len(y_new) X_new = np.random.rand(n_new_samples, n_features) y_new = np.random.randint(0, 2, size=(2*n_new_samples,)) - self.assertRaises(ValueError, learner._add_training_data, X_new, y_new) + self.assertRaises( + ValueError, learner._add_training_data, X_new, y_new) # 2. X_new has wrong dimensions X_new = np.random.rand(n_new_samples, 2*n_features) y_new = np.random.randint(0, 2, size=(n_new_samples,)) - self.assertRaises(ValueError, learner._add_training_data, X_new, y_new) + self.assertRaises( + ValueError, learner._add_training_data, X_new, y_new) def test_predict(self): for n_samples in range(1, 100): for n_features in range(1, 10): X = np.random.rand(n_samples, n_features) predict_return = np.random.randint(0, 2, size=(n_samples, )) - mock_classifier = mock.MockEstimator(predict_return=predict_return) + mock_classifier = mock.MockEstimator( + predict_return=predict_return) learner = modAL.models.learners.ActiveLearner( estimator=mock_classifier ) @@ -683,8 +1004,10 @@ def test_predict_proba(self): for n_samples in range(1, 100): for n_features in range(1, 10): X = np.random.rand(n_samples, n_features) - predict_proba_return = np.random.randint(0, 2, size=(n_samples,)) - mock_classifier = mock.MockEstimator(predict_proba_return=predict_proba_return) + predict_proba_return = np.random.randint( + 0, 2, size=(n_samples,)) + mock_classifier = mock.MockEstimator( + predict_proba_return=predict_proba_return) learner = modAL.models.learners.ActiveLearner( estimator=mock_classifier ) @@ -698,7 +1021,9 @@ def test_query(self): for n_features in range(1, 10): X = np.random.rand(n_samples, n_features) query_idx = np.random.randint(0, n_samples) - mock_query = mock.MockFunction(return_val=(query_idx, X[query_idx])) + query_metrics = np.random.randint(0, n_samples) + mock_query = mock.MockFunction( + return_val=(query_idx, query_metrics)) learner = modAL.models.learners.ActiveLearner( estimator=None, query_strategy=mock_query @@ -707,12 +1032,17 @@ def test_query(self): learner.query(X), (query_idx, X[query_idx]) ) + np.testing.assert_equal( + learner.query(X, return_metrics=True), + (query_idx, X[query_idx], query_metrics) + ) def test_score(self): test_cases = (np.random.rand() for _ in range(10)) for score_return in test_cases: mock_classifier = mock.MockEstimator(score_return=score_return) - learner = modAL.models.learners.ActiveLearner(mock_classifier, mock.MockFunction(None)) + learner = modAL.models.learners.ActiveLearner( + mock_classifier, mock.MockFunction(None)) np.testing.assert_almost_equal( learner.score(np.random.rand(5, 2), np.random.rand(5, )), score_return @@ -734,6 +1064,25 @@ def test_teach(self): learner.teach(X, y, bootstrap=bootstrap, only_new=only_new) + def test_nan(self): + X_training_nan = np.ones(shape=(10, 2)) * np.nan + X_training_inf = np.ones(shape=(10, 2)) * np.inf + y_training = np.random.randint(0, 2, size=10) + + learner = modAL.models.learners.ActiveLearner( + X_training=X_training_nan, y_training=y_training, + estimator=mock.MockEstimator(), + force_all_finite=False + ) + learner.teach(X_training_nan, y_training) + + learner = modAL.models.learners.ActiveLearner( + X_training=X_training_inf, y_training=y_training, + estimator=mock.MockEstimator(), + force_all_finite=False + ) + learner.teach(X_training_inf, y_training) + def test_keras(self): pass @@ -743,7 +1092,8 @@ def test_sklearn(self): X_training=np.random.rand(10, 10), y_training=np.random.randint(0, 2, size=(10,)) ) - learner.fit(np.random.rand(10, 10), np.random.randint(0, 2, size=(10,))) + learner.fit(np.random.rand(10, 10), + np.random.randint(0, 2, size=(10,))) pred = learner.predict(np.random.rand(10, 10)) learner.predict_proba(np.random.rand(10, 10)) confusion_matrix(pred, np.random.randint(0, 2, size=(10,))) @@ -761,7 +1111,8 @@ def test_sparse_matrices(self): for query_strategy, format, n_samples, n_features in product(query_strategies, formats, sample_count, feature_count): X_pool = sp.random(n_samples, n_features, format=format) y_pool = np.random.randint(0, 2, size=(n_samples, )) - initial_idx = np.random.choice(range(n_samples), size=5, replace=False) + initial_idx = np.random.choice( + range(n_samples), size=5, replace=False) learner = modAL.models.learners.ActiveLearner( estimator=RandomForestClassifier(n_estimators=10), query_strategy=query_strategy, @@ -770,6 +1121,107 @@ def test_sparse_matrices(self): query_idx, query_inst = learner.query(X_pool) learner.teach(X_pool[query_idx], y_pool[query_idx]) + def test_on_transformed(self): + n_samples = 10 + n_features = 5 + query_strategies = [ + modAL.batch.uncertainty_batch_sampling + # add further strategies which work with instance representations + # no further ones as of 25.09.2020 + ] + X_pool = np.random.rand(n_samples, n_features) + + # use pandas data frame as X_pool, which will be transformed back to numpy with sklearn pipeline + X_pool = pd.DataFrame(X_pool) + + y_pool = np.random.randint(0, 2, size=(n_samples,)) + train_idx = np.random.choice(range(n_samples), size=2, replace=False) + + for query_strategy in query_strategies: + learner = modAL.models.learners.ActiveLearner( + estimator=make_pipeline( + FunctionTransformer(func=pd.DataFrame.to_numpy), + RandomForestClassifier(n_estimators=10) + ), + query_strategy=query_strategy, + X_training=X_pool.iloc[train_idx], + y_training=y_pool[train_idx], + on_transformed=True + ) + query_idx, query_inst = learner.query(X_pool) + learner.teach(X_pool.iloc[query_idx], y_pool[query_idx]) + + def test_on_transformed_with_variable_transformation(self): + """ + Learnable transformations naturally change after a model is retrained. Make sure this is handled + properly for on_transformed=True query strategies. + """ + query_strategies = [ + modAL.batch.uncertainty_batch_sampling + # add further strategies which work with instance representations + # no further ones as of 09.12.2020 + ] + + X_labeled = ['Dog', 'Cat', 'Tree'] + + # contains unseen in labeled words, training model on those + # will alter CountVectorizer transformations + X_pool = ['Airplane', 'House'] + + y = [0, 1, 1, 0, 1] # irrelevant for test + + for query_strategy in query_strategies: + learner = modAL.models.learners.ActiveLearner( + estimator=make_pipeline( + CountVectorizer(), + RandomForestClassifier(n_estimators=10) + ), + query_strategy=query_strategy, + X_training=X_labeled, y_training=y[:len(X_labeled)], + on_transformed=True, + ) + + for _ in range(len(X_pool)): + query_idx, query_instance = learner.query( + X_pool, n_instances=1) + i = query_idx[0] + + learner.teach( + X=[X_pool[i]], + y=[y[i]] + ) + + def test_old_query_strategy_interface(self): + n_samples = 10 + n_features = 5 + X_pool = np.random.rand(n_samples, n_features) + y_pool = np.random.randint(0, 2, size=(n_samples,)) + + # defining a custom query strategy also returning the selected instance + # make sure even if a query strategy works in some funny way + # (e.g. instance not matching instance index), + # the old interface remains unchanged + query_idx_ = np.random.choice(n_samples, 2) + query_instance_ = X_pool[query_idx_] + + def custom_query_strategy(classifier, X): + return query_idx_, query_instance_ + + train_idx = np.random.choice(range(n_samples), size=2, replace=False) + custom_query_learner = modAL.models.learners.ActiveLearner( + estimator=RandomForestClassifier(n_estimators=10), + query_strategy=custom_query_strategy, + X_training=X_pool[train_idx], y_training=y_pool[train_idx] + ) + + query_idx, query_instance = custom_query_learner.query(X_pool) + custom_query_learner.teach( + X=X_pool[query_idx], + y=y_pool[query_idx] + ) + np.testing.assert_equal(query_idx, query_idx_) + np.testing.assert_equal(query_instance, query_instance_) + class TestBayesianOptimizer(unittest.TestCase): def test_set_max(self): @@ -799,7 +1251,8 @@ def test_set_new_max(self): y = np.random.rand(n_samples) max_idx = np.argmax(y) regressor = mock.MockEstimator() - learner = modAL.models.learners.BayesianOptimizer(estimator=regressor) + learner = modAL.models.learners.BayesianOptimizer( + estimator=regressor) learner._set_max(X, y) np.testing.assert_equal(learner.X_max, X[max_idx]) np.testing.assert_equal(learner.y_max, y[max_idx]) @@ -849,7 +1302,8 @@ def test_get_max(self): y[max_idx] = 10 regressor = mock.MockEstimator() - optimizer = modAL.models.learners.BayesianOptimizer(regressor, X_training=X, y_training=y) + optimizer = modAL.models.learners.BayesianOptimizer( + regressor, X_training=X, y_training=y) X_max, y_max = optimizer.get_max() np.testing.assert_equal(X_max, X[max_idx]) np.testing.assert_equal(y_max, y[max_idx]) @@ -860,7 +1314,8 @@ def test_teach(self): for n_samples in range(1, 100): for n_features in range(1, 100): regressor = mock.MockEstimator() - learner = modAL.models.learners.BayesianOptimizer(estimator=regressor) + learner = modAL.models.learners.BayesianOptimizer( + estimator=regressor) X = np.random.rand(n_samples, 2) y = np.random.rand(n_samples) @@ -879,6 +1334,39 @@ def test_teach(self): ) learner.teach(X, y, bootstrap=bootstrap, only_new=only_new) + def test_on_transformed(self): + n_samples = 10 + n_features = 5 + query_strategies = [ + # TODO remove, added just to make sure on_transformed doesn't break anything + # but it has no influence on this strategy, nothing special tested here + mock.MockFunction(return_val=[np.random.randint(0, n_samples)]) + + # add further strategies which work with instance representations + # no further ones as of 25.09.2020 + ] + X_pool = np.random.rand(n_samples, n_features) + + # use pandas data frame as X_pool, which will be transformed back to numpy with sklearn pipeline + X_pool = pd.DataFrame(X_pool) + + y_pool = np.random.rand(n_samples) + train_idx = np.random.choice(range(n_samples), size=2, replace=False) + + for query_strategy in query_strategies: + learner = modAL.models.learners.BayesianOptimizer( + estimator=make_pipeline( + FunctionTransformer(func=pd.DataFrame.to_numpy), + GaussianProcessRegressor() + ), + query_strategy=query_strategy, + X_training=X_pool.iloc[train_idx], + y_training=y_pool[train_idx], + on_transformed=True + ) + query_idx, query_inst = learner.query(X_pool) + learner.teach(X_pool.iloc[query_idx], y_pool[query_idx]) + class TestCommittee(unittest.TestCase): @@ -887,7 +1375,8 @@ def test_set_classes(self): for n_learners in range(1, 10): learner_list = [modAL.models.learners.ActiveLearner(estimator=mock.MockEstimator(fitted=False)) for idx in range(n_learners)] - committee = modAL.models.learners.Committee(learner_list=learner_list) + committee = modAL.models.learners.Committee( + learner_list=learner_list) self.assertEqual(committee.classes_, None) self.assertEqual(committee.n_classes_, 0) @@ -895,7 +1384,8 @@ def test_set_classes(self): for n_classes in range(1, 10): learner_list = [modAL.models.learners.ActiveLearner(estimator=mock.MockEstimator(classes_=np.asarray([idx]))) for idx in range(n_classes)] - committee = modAL.models.learners.Committee(learner_list=learner_list) + committee = modAL.models.learners.Committee( + learner_list=learner_list) np.testing.assert_equal( committee.classes_, np.unique(range(n_classes)) @@ -904,13 +1394,14 @@ def test_set_classes(self): def test_predict(self): for n_learners in range(1, 10): for n_instances in range(1, 10): - prediction = np.random.randint(10, size=(n_instances, n_learners)) + prediction = np.random.randint( + 10, size=(n_instances, n_learners)) committee = modAL.models.learners.Committee( learner_list=[mock.MockActiveLearner( - mock.MockEstimator(classes_=np.asarray([0])), - predict_return=prediction[:, learner_idx] - ) - for learner_idx in range(n_learners)] + mock.MockEstimator(classes_=np.asarray([0])), + predict_return=prediction[:, learner_idx] + ) + for learner_idx in range(n_learners)] ) np.testing.assert_equal( committee.vote(np.random.rand(n_instances, 5)), @@ -921,13 +1412,17 @@ def test_predict_proba(self): for n_samples in range(1, 100): for n_learners in range(1, 10): for n_classes in range(1, 10): - vote_proba_output = np.random.rand(n_samples, n_learners, n_classes) + vote_proba_output = np.random.rand( + n_samples, n_learners, n_classes) # assembling the mock learners learner_list = [mock.MockActiveLearner( - predict_proba_return=vote_proba_output[:, learner_idx, :], - predictor=mock.MockEstimator(classes_=list(range(n_classes))) + predict_proba_return=vote_proba_output[:, + learner_idx, :], + predictor=mock.MockEstimator( + classes_=list(range(n_classes))) ) for learner_idx in range(n_learners)] - committee = modAL.models.learners.Committee(learner_list=learner_list) + committee = modAL.models.learners.Committee( + learner_list=learner_list) np.testing.assert_almost_equal( committee.predict_proba(np.random.rand(n_samples, 1)), np.mean(vote_proba_output, axis=1) @@ -936,14 +1431,16 @@ def test_predict_proba(self): def test_vote(self): for n_members in range(1, 10): for n_instances in range(1, 100): - vote_output = np.random.randint(0, 2, size=(n_instances, n_members)) + vote_output = np.random.randint( + 0, 2, size=(n_instances, n_members)) # assembling the Committee learner_list = [mock.MockActiveLearner( - predict_return=vote_output[:, member_idx], - predictor=mock.MockEstimator(classes_=[0]) - ) - for member_idx in range(n_members)] - committee = modAL.models.learners.Committee(learner_list=learner_list) + predict_return=vote_output[:, member_idx], + predictor=mock.MockEstimator(classes_=[0]) + ) + for member_idx in range(n_members)] + committee = modAL.models.learners.Committee( + learner_list=learner_list) np.testing.assert_array_almost_equal( committee.vote(np.random.rand(n_instances).reshape(-1, 1)), vote_output @@ -953,13 +1450,17 @@ def test_vote_proba(self): for n_samples in range(1, 100): for n_learners in range(1, 10): for n_classes in range(1, 10): - vote_proba_output = np.random.rand(n_samples, n_learners, n_classes) + vote_proba_output = np.random.rand( + n_samples, n_learners, n_classes) # assembling the mock learners learner_list = [mock.MockActiveLearner( - predict_proba_return=vote_proba_output[:, learner_idx, :], - predictor=mock.MockEstimator(classes_=list(range(n_classes))) + predict_proba_return=vote_proba_output[:, + learner_idx, :], + predictor=mock.MockEstimator( + classes_=list(range(n_classes))) ) for learner_idx in range(n_learners)] - committee = modAL.models.learners.Committee(learner_list=learner_list) + committee = modAL.models.learners.Committee( + learner_list=learner_list) np.testing.assert_almost_equal( committee.vote_proba(np.random.rand(n_samples, 1)), vote_proba_output @@ -989,6 +1490,44 @@ def test_teach(self): committee.teach(X, y, bootstrap=bootstrap, only_new=only_new) + def test_on_transformed(self): + n_samples = 10 + n_features = 5 + query_strategies = [ + modAL.batch.uncertainty_batch_sampling + # add further strategies which work with instance representations + # no further ones as of 25.09.2020 + ] + X_pool = np.random.rand(n_samples, n_features) + + # use pandas data frame as X_pool, which will be transformed back to numpy with sklearn pipeline + X_pool = pd.DataFrame(X_pool) + + y_pool = np.random.randint(0, 2, size=(n_samples,)) + train_idx = np.random.choice(range(n_samples), size=5, replace=False) + + learner_list = [modAL.models.learners.ActiveLearner( + estimator=make_pipeline( + FunctionTransformer(func=pd.DataFrame.to_numpy), + RandomForestClassifier(n_estimators=10) + ), + # committee learners can contain different amounts of + # different instances + X_training=X_pool.iloc[train_idx[( + np.arange(i + 1) + i) % len(train_idx)]], + y_training=y_pool[train_idx[( + np.arange(i + 1) + i) % len(train_idx)]], + ) for i in range(3)] + + for query_strategy in query_strategies: + committee = modAL.models.learners.Committee( + learner_list=learner_list, + query_strategy=query_strategy, + on_transformed=True + ) + query_idx, query_inst = committee.query(X_pool) + committee.teach(X_pool.iloc[query_idx], y_pool[query_idx]) + class TestCommitteeRegressor(unittest.TestCase): @@ -999,13 +1538,16 @@ def test_predict(self): # assembling the Committee learner_list = [mock.MockActiveLearner(predict_return=vote[:, member_idx]) for member_idx in range(n_members)] - committee = modAL.models.learners.CommitteeRegressor(learner_list=learner_list) + committee = modAL.models.learners.CommitteeRegressor( + learner_list=learner_list) np.testing.assert_array_almost_equal( - committee.predict(np.random.rand(n_instances).reshape(-1, 1), return_std=False), + committee.predict(np.random.rand( + n_instances).reshape(-1, 1), return_std=False), np.mean(vote, axis=1) ) np.testing.assert_array_almost_equal( - committee.predict(np.random.rand(n_instances).reshape(-1, 1), return_std=True), + committee.predict(np.random.rand( + n_instances).reshape(-1, 1), return_std=True), (np.mean(vote, axis=1), np.std(vote, axis=1)) ) @@ -1016,22 +1558,66 @@ def test_vote(self): # assembling the Committee learner_list = [mock.MockActiveLearner(predict_return=vote_output[:, member_idx]) for member_idx in range(n_members)] - committee = modAL.models.learners.CommitteeRegressor(learner_list=learner_list) + committee = modAL.models.learners.CommitteeRegressor( + learner_list=learner_list) np.testing.assert_array_almost_equal( committee.vote(np.random.rand(n_instances).reshape(-1, 1)), vote_output ) + def test_on_transformed(self): + n_samples = 10 + n_features = 5 + query_strategies = [ + # TODO remove, added just to make sure on_transformed doesn't break anything + # but it has no influence on this strategy, nothing special tested here + mock.MockFunction(return_val=[np.random.randint(0, n_samples)]) + + # add further strategies which work with instance representations + # no further ones as of 25.09.2020 + ] + X_pool = np.random.rand(n_samples, n_features) + + # use pandas data frame as X_pool, which will be transformed back to numpy with sklearn pipeline + X_pool = pd.DataFrame(X_pool) + + y_pool = np.random.rand(n_samples) + train_idx = np.random.choice(range(n_samples), size=2, replace=False) + + learner_list = [modAL.models.learners.ActiveLearner( + estimator=make_pipeline( + FunctionTransformer(func=pd.DataFrame.to_numpy), + GaussianProcessRegressor() + ), + # committee learners can contain different amounts of + # different instances + X_training=X_pool.iloc[train_idx[( + np.arange(i + 1) + i) % len(train_idx)]], + y_training=y_pool[train_idx[( + np.arange(i + 1) + i) % len(train_idx)]], + ) for i in range(3)] + + for query_strategy in query_strategies: + committee = modAL.models.learners.CommitteeRegressor( + learner_list=learner_list, + query_strategy=query_strategy, + on_transformed=True + ) + query_idx, query_inst = committee.query(X_pool) + committee.teach(X_pool.iloc[query_idx], y_pool[query_idx]) + class TestMultilabel(unittest.TestCase): def test_SVM_loss(self): for n_classes in range(2, 10): for n_instances in range(1, 10): X_training = np.random.rand(n_instances, 5) - y_training = np.random.randint(0, 2, size=(n_instances, n_classes)) + y_training = np.random.randint( + 0, 2, size=(n_instances, n_classes)) X_pool = np.random.rand(n_instances, 5) y_pool = np.random.randint(0, 2, size=(n_instances, n_classes)) - classifier = OneVsRestClassifier(SVC(probability=True, gamma='auto')) + classifier = OneVsRestClassifier( + SVC(probability=True, gamma='auto')) classifier.fit(X_training, y_training) avg_loss = modAL.multilabel._SVM_loss(classifier, X_pool) mcc_loss = modAL.multilabel._SVM_loss(classifier, X_pool, @@ -1044,28 +1630,43 @@ def test_strategies(self): for n_pool_instances in range(1, 10): for n_query_instances in range(1, min(n_pool_instances, 3)): X_training = np.random.rand(n_pool_instances, 5) - y_training = np.random.randint(0, 2, size=(n_pool_instances, n_classes)) + y_training = np.random.randint( + 0, 2, size=(n_pool_instances, n_classes)) X_pool = np.random.rand(n_pool_instances, 5) - classifier = OneVsRestClassifier(SVC(probability=True, gamma='auto')) + classifier = OneVsRestClassifier( + SVC(probability=True, gamma='auto')) classifier.fit(X_training, y_training) active_learner = modAL.models.ActiveLearner(classifier) # no random tie break modAL.multilabel.SVM_binary_minimum(active_learner, X_pool) - modAL.multilabel.mean_max_loss(classifier, X_pool, n_query_instances) - modAL.multilabel.max_loss(classifier, X_pool, n_query_instances) - modAL.multilabel.min_confidence(classifier, X_pool, n_query_instances) - modAL.multilabel.avg_confidence(classifier, X_pool, n_query_instances) - modAL.multilabel.max_score(classifier, X_pool, n_query_instances) - modAL.multilabel.avg_score(classifier, X_pool, n_query_instances) + modAL.multilabel.mean_max_loss( + classifier, X_pool, n_query_instances) + modAL.multilabel.max_loss( + classifier, X_pool, n_query_instances) + modAL.multilabel.min_confidence( + classifier, X_pool, n_query_instances) + modAL.multilabel.avg_confidence( + classifier, X_pool, n_query_instances) + modAL.multilabel.max_score( + classifier, X_pool, n_query_instances) + modAL.multilabel.avg_score( + classifier, X_pool, n_query_instances) # random tie break - modAL.multilabel.SVM_binary_minimum(active_learner, X_pool, random_tie_break=True) - modAL.multilabel.mean_max_loss(classifier, X_pool, n_query_instances, random_tie_break=True) - modAL.multilabel.max_loss(classifier, X_pool, n_query_instances, random_tie_break=True) - modAL.multilabel.min_confidence(classifier, X_pool, n_query_instances, random_tie_break=True) - modAL.multilabel.avg_confidence(classifier, X_pool, n_query_instances, random_tie_break=True) - modAL.multilabel.max_score(classifier, X_pool, n_query_instances, random_tie_break=True) - modAL.multilabel.avg_score(classifier, X_pool, n_query_instances, random_tie_break=True) + modAL.multilabel.SVM_binary_minimum( + active_learner, X_pool, random_tie_break=True) + modAL.multilabel.mean_max_loss( + classifier, X_pool, n_query_instances, random_tie_break=True) + modAL.multilabel.max_loss( + classifier, X_pool, n_query_instances, random_tie_break=True) + modAL.multilabel.min_confidence( + classifier, X_pool, n_query_instances, random_tie_break=True) + modAL.multilabel.avg_confidence( + classifier, X_pool, n_query_instances, random_tie_break=True) + modAL.multilabel.max_score( + classifier, X_pool, n_query_instances, random_tie_break=True) + modAL.multilabel.avg_score( + classifier, X_pool, n_query_instances, random_tie_break=True) class TestExamples(unittest.TestCase): @@ -1073,16 +1674,17 @@ class TestExamples(unittest.TestCase): def test_examples(self): import example_tests.active_regression import example_tests.bagging + import example_tests.bayesian_optimization + import example_tests.custom_query_strategies import example_tests.ensemble import example_tests.ensemble_regression + import example_tests.information_density + import example_tests.multidimensional_data import example_tests.pool_based_sampling import example_tests.query_by_committee + import example_tests.ranked_batch_mode import example_tests.shape_learning import example_tests.stream_based_sampling - import example_tests.custom_query_strategies - import example_tests.information_density - import example_tests.bayesian_optimization - import example_tests.ranked_batch_mode if __name__ == '__main__': diff --git a/tests/example_tests/active_regression.py b/tests/example_tests/active_regression.py index 331a103..4306a3e 100644 --- a/tests/example_tests/active_regression.py +++ b/tests/example_tests/active_regression.py @@ -3,19 +3,13 @@ """ import numpy as np -from sklearn.gaussian_process import GaussianProcessRegressor -from sklearn.gaussian_process.kernels import WhiteKernel, RBF +from modAL.disagreement import max_std_sampling from modAL.models import ActiveLearner +from sklearn.gaussian_process import GaussianProcessRegressor +from sklearn.gaussian_process.kernels import RBF, WhiteKernel np.random.seed(0) - -# query strategy for regression -def GP_regression_std(regressor, X): - _, std = regressor.predict(X, return_std=True) - query_idx = np.argmax(std) - return query_idx, X[query_idx] - # generating the data X = np.random.choice(np.linspace(0, 20, 10000), size=200, replace=False).reshape(-1, 1) y = np.sin(X) + np.random.normal(scale=0.3, size=X.shape) @@ -32,7 +26,7 @@ def GP_regression_std(regressor, X): # initializing the active learner regressor = ActiveLearner( estimator=GaussianProcessRegressor(kernel=kernel), - query_strategy=GP_regression_std, + query_strategy=max_std_sampling, X_training=X_initial.reshape(-1, 1), y_training=y_initial.reshape(-1, 1) ) diff --git a/tests/example_tests/bagging.py b/tests/example_tests/bagging.py index 4830150..55a7d24 100644 --- a/tests/example_tests/bagging.py +++ b/tests/example_tests/bagging.py @@ -2,10 +2,11 @@ This example shows how to build models with bagging using the Committee model. """ -import numpy as np from itertools import product -from sklearn.neighbors import KNeighborsClassifier + +import numpy as np from modAL.models import ActiveLearner, Committee +from sklearn.neighbors import KNeighborsClassifier np.random.seed(0) diff --git a/tests/example_tests/bayesian_optimization.py b/tests/example_tests/bayesian_optimization.py index 981c6b0..8950c52 100644 --- a/tests/example_tests/bayesian_optimization.py +++ b/tests/example_tests/bayesian_optimization.py @@ -1,10 +1,11 @@ -import numpy as np from functools import partial + +import numpy as np +from modAL.acquisition import (max_EI, max_PI, max_UCB, optimizer_EI, + optimizer_PI, optimizer_UCB) +from modAL.models import BayesianOptimizer from sklearn.gaussian_process import GaussianProcessRegressor from sklearn.gaussian_process.kernels import Matern -from modAL.models import BayesianOptimizer -from modAL.acquisition import optimizer_PI, optimizer_EI, optimizer_UCB, max_PI, max_EI, max_UCB - # generating the data X = np.linspace(0, 20, 1000).reshape(-1, 1) diff --git a/tests/example_tests/custom_query_strategies.py b/tests/example_tests/custom_query_strategies.py index 1181bab..c8a94f1 100644 --- a/tests/example_tests/custom_query_strategies.py +++ b/tests/example_tests/custom_query_strategies.py @@ -1,14 +1,12 @@ import numpy as np - +from modAL.models import ActiveLearner +from modAL.uncertainty import classifier_margin, classifier_uncertainty from modAL.utils.combination import make_linear_combination, make_product from modAL.utils.selection import multi_argmax -from modAL.uncertainty import classifier_uncertainty, classifier_margin -from modAL.models import ActiveLearner from sklearn.datasets import make_blobs from sklearn.gaussian_process import GaussianProcessClassifier from sklearn.gaussian_process.kernels import RBF - # generating the data centers = np.asarray([[-2, 3], [0.5, 5], [1, 1.5]]) X, y = make_blobs( @@ -42,8 +40,7 @@ # classifier uncertainty and classifier margin def custom_query_strategy(classifier, X, n_instances=1): utility = linear_combination(classifier, X) - query_idx = multi_argmax(utility, n_instances=n_instances) - return query_idx, X[query_idx] + return multi_argmax(utility, n_instances=n_instances) custom_query_learner = ActiveLearner( estimator=GaussianProcessClassifier(1.0 * RBF(1.0)), diff --git a/tests/example_tests/ensemble.py b/tests/example_tests/ensemble.py index 35c36df..c7e3193 100644 --- a/tests/example_tests/ensemble.py +++ b/tests/example_tests/ensemble.py @@ -1,7 +1,8 @@ -import numpy as np from itertools import product -from sklearn.ensemble import RandomForestClassifier + +import numpy as np from modAL.models import ActiveLearner, Committee +from sklearn.ensemble import RandomForestClassifier np.random.seed(0) diff --git a/tests/example_tests/ensemble_regression.py b/tests/example_tests/ensemble_regression.py index 1e9e1e4..1082fb3 100644 --- a/tests/example_tests/ensemble_regression.py +++ b/tests/example_tests/ensemble_regression.py @@ -1,8 +1,8 @@ import numpy as np -from sklearn.gaussian_process import GaussianProcessRegressor -from sklearn.gaussian_process.kernels import WhiteKernel, RBF -from modAL.models import ActiveLearner, CommitteeRegressor from modAL.disagreement import max_std_sampling +from modAL.models import ActiveLearner, CommitteeRegressor +from sklearn.gaussian_process import GaussianProcessRegressor +from sklearn.gaussian_process.kernels import RBF, WhiteKernel np.random.seed(0) diff --git a/tests/example_tests/information_density.py b/tests/example_tests/information_density.py index 7e82c76..43c8f39 100644 --- a/tests/example_tests/information_density.py +++ b/tests/example_tests/information_density.py @@ -1,6 +1,6 @@ -from modAL.density import similarize_distance, information_density -from sklearn.datasets import make_blobs +from modAL.density import information_density, similarize_distance from scipy.spatial.distance import euclidean +from sklearn.datasets import make_blobs X, y = make_blobs(n_features=2, n_samples=10, centers=3, random_state=0, cluster_std=0.7) diff --git a/tests/example_tests/multidimensional_data.py b/tests/example_tests/multidimensional_data.py new file mode 100644 index 0000000..e491765 --- /dev/null +++ b/tests/example_tests/multidimensional_data.py @@ -0,0 +1,39 @@ +import numpy as np +from modAL.batch import uncertainty_batch_sampling +from modAL.expected_error import expected_error_reduction +from modAL.models import ActiveLearner +from modAL.uncertainty import entropy_sampling, margin_sampling +from sklearn.base import BaseEstimator + + +class MockClassifier(BaseEstimator): + def __init__(self, n_classes=2): + self.n_classes = n_classes + + def fit(self, X, y): + return self + + def predict(self, X): + return np.random.randint(0, self.n_classes, shape=(len(X), 1)) + + def predict_proba(self, X): + return np.ones(shape=(len(X), self.n_classes))/self.n_classes + + +if __name__ == '__main__': + X_train = np.random.rand(10, 5, 5) + y_train = np.random.randint(0, 2, size=10) + X_pool = np.random.rand(10, 5, 5) + y_pool = np.random.randint(0, 2, size=10) + + strategies = [margin_sampling, entropy_sampling, uncertainty_batch_sampling, expected_error_reduction] + + for query_strategy in strategies: + print("testing %s..." % query_strategy.__name__) + # max margin sampling + learner = ActiveLearner( + estimator=MockClassifier(), query_strategy=query_strategy, + X_training=X_train, y_training=y_train + ) + learner.query(X_pool) + learner.teach(X_pool, y_pool) diff --git a/tests/example_tests/multilabel_svm.py b/tests/example_tests/multilabel_svm.py index ea91dae..96ed4bd 100644 --- a/tests/example_tests/multilabel_svm.py +++ b/tests/example_tests/multilabel_svm.py @@ -1,8 +1,6 @@ import numpy as np - from modAL.models import ActiveLearner from modAL.multilabel import SVM_binary_minimum - from sklearn.multiclass import OneVsRestClassifier from sklearn.svm import LinearSVC @@ -25,4 +23,4 @@ for idx in range(n_queries): query_idx, query_inst = learner.query(X_pool) learner.teach(X_pool[query_idx].reshape(1, -1), y_pool[query_idx].reshape(1, -1)) - X_pool, y_pool = np.delete(X_pool, query_idx, axis=0), np.delete(y_pool, query_idx, axis=0) \ No newline at end of file + X_pool, y_pool = np.delete(X_pool, query_idx, axis=0), np.delete(y_pool, query_idx, axis=0) diff --git a/tests/example_tests/pool_based_sampling.py b/tests/example_tests/pool_based_sampling.py index 8dab142..1fa052c 100644 --- a/tests/example_tests/pool_based_sampling.py +++ b/tests/example_tests/pool_based_sampling.py @@ -5,9 +5,9 @@ """ import numpy as np +from modAL.models import ActiveLearner from sklearn.datasets import load_iris from sklearn.neighbors import KNeighborsClassifier -from modAL.models import ActiveLearner np.random.seed(0) diff --git a/tests/example_tests/query_by_committee.py b/tests/example_tests/query_by_committee.py index b974483..e711faf 100644 --- a/tests/example_tests/query_by_committee.py +++ b/tests/example_tests/query_by_committee.py @@ -1,8 +1,9 @@ -import numpy as np from copy import deepcopy + +import numpy as np +from modAL.models import ActiveLearner, Committee from sklearn.datasets import load_iris from sklearn.ensemble import RandomForestClassifier -from modAL.models import ActiveLearner, Committee np.random.seed(0) diff --git a/tests/example_tests/ranked_batch_mode.py b/tests/example_tests/ranked_batch_mode.py index 949957d..48ac2f0 100644 --- a/tests/example_tests/ranked_batch_mode.py +++ b/tests/example_tests/ranked_batch_mode.py @@ -1,11 +1,11 @@ -import numpy as np -from sklearn.datasets import load_iris -from sklearn.decomposition import PCA -from sklearn.neighbors import KNeighborsClassifier from functools import partial +import numpy as np from modAL.batch import uncertainty_batch_sampling from modAL.models import ActiveLearner +from sklearn.datasets import load_iris +from sklearn.decomposition import PCA +from sklearn.neighbors import KNeighborsClassifier # Set our RNG for reproducibility. RANDOM_STATE_SEED = 123 @@ -76,4 +76,4 @@ # Calculate and report our model's accuracy. model_accuracy = learner.score(X_raw, y_raw) -predictions = learner.predict(X_raw) \ No newline at end of file +predictions = learner.predict(X_raw) diff --git a/tests/example_tests/shape_learning.py b/tests/example_tests/shape_learning.py index f76a07a..17dd2cd 100644 --- a/tests/example_tests/shape_learning.py +++ b/tests/example_tests/shape_learning.py @@ -5,10 +5,11 @@ the scikit-learn implementation of the kNN classifier algorithm. """ -import numpy as np from copy import deepcopy -from sklearn.ensemble import RandomForestClassifier + +import numpy as np from modAL.models import ActiveLearner +from sklearn.ensemble import RandomForestClassifier np.random.seed(0) diff --git a/tests/example_tests/stream_based_sampling.py b/tests/example_tests/stream_based_sampling.py index d306f61..5e603d7 100644 --- a/tests/example_tests/stream_based_sampling.py +++ b/tests/example_tests/stream_based_sampling.py @@ -3,9 +3,9 @@ """ import numpy as np -from sklearn.ensemble import RandomForestClassifier from modAL.models import ActiveLearner from modAL.uncertainty import classifier_uncertainty +from sklearn.ensemble import RandomForestClassifier np.random.seed(0)