From ee02e2fd8b304ee4ec0cd16798b1293eb03eae80 Mon Sep 17 00:00:00 2001
From: Ahmed Gad <ahmed.f.gad@gmail.com>
Date: Tue, 9 May 2023 16:17:12 -0400
Subject: [PATCH 1/4] Use logging module

---
 pygad/cnn/cnn.py   | 338 ++++++++++++++++++++++++++++++++++++---------
 pygad/gann/gann.py |   3 +-
 2 files changed, 276 insertions(+), 65 deletions(-)

diff --git a/pygad/cnn/cnn.py b/pygad/cnn/cnn.py
index d0141ae..1bd9033 100644
--- a/pygad/cnn/cnn.py
+++ b/pygad/cnn/cnn.py
@@ -1,5 +1,6 @@
 import numpy
 import functools
+import logging
 
 """
 Convolutional neural network implementation using NumPy
@@ -84,14 +85,18 @@ def layers_weights(model, initial=True):
             elif initial == False:
                 network_weights.append(layer.trained_weights)
             else:
-                raise ValueError(f"Unexpected value to the 'initial' parameter: {initial}.")
+                msg = f"Unexpected value to the 'initial' parameter: {initial}."
+                model.logger.error(msg)
+                raise ValueError(msg)
 
         # Go to the previous layer.
         layer = layer.previous_layer
 
     # If the first layer in the network is not an input layer (i.e. an instance of the Input2D class), raise an error.
     if not (type(layer) is Input2D):
-        raise TypeError("The first layer in the network architecture must be an input layer.")
+        msg = "The first layer in the network architecture must be an input layer."
+        model.logger.error(msg)
+        raise TypeError(msg)
 
     # Currently, the weights of the layers are in the reverse order. In other words, the weights of the first layer are at the last index of the 'network_weights' list while the weights of the last layer are at the first index.
     # Reversing the 'network_weights' list to order the layers' weights according to their location in the network architecture (i.e. the weights of the first layer appears at index 0 of the list).
@@ -131,7 +136,9 @@ def layers_weights_as_matrix(model, vector_weights):
 
     # If the first layer in the network is not an input layer (i.e. an instance of the Input2D class), raise an error.
     if not (type(layer) is Input2D):
-        raise TypeError("The first layer in the network architecture must be an input layer.")
+        msg = "The first layer in the network architecture must be an input layer."
+        model.logger.error(msg)
+        raise TypeError(msg)
 
     # Currently, the weights of the layers are in the reverse order. In other words, the weights of the first layer are at the last index of the 'network_weights' list while the weights of the last layer are at the first index.
     # Reversing the 'network_weights' list to order the layers' weights according to their location in the network architecture (i.e. the weights of the first layer appears at index 0 of the list).
@@ -164,14 +171,18 @@ def layers_weights_as_vector(model, initial=True):
     #            vector = pygad.nn.DenseLayer.to_vector(array=layer.trained_weights)
                 network_weights.extend(vector)
             else:
-                raise ValueError(f"Unexpected value to the 'initial' parameter: {initial}.")
+                msg = f"Unexpected value to the 'initial' parameter: {initial}."
+                model.logger.error(msg)
+                raise ValueError(msg)
 
         # Go to the previous layer.
         layer = layer.previous_layer
 
     # If the first layer in the network is not an input layer (i.e. an instance of the Input2D class), raise an error.
     if not (type(layer) is Input2D):
-        raise TypeError("The first layer in the network architecture must be an input layer.")
+        msg = "The first layer in the network architecture must be an input layer."
+        model.logger.error(msg)
+        raise TypeError(msg)
 
     # Currently, the weights of the layers are in the reverse order. In other words, the weights of the first layer are at the last index of the 'network_weights' list while the weights of the last layer are at the first index.
     # Reversing the 'network_weights' list to order the layers' weights according to their location in the network architecture (i.e. the weights of the first layer appears at index 0 of the list).
@@ -199,40 +210,91 @@ def update_layers_trained_weights(model, final_weights):
         # Go to the previous layer.
         layer = layer.previous_layer
 
-class Input2D:
+
+class CustomLogger:
+
+    def __init__(self):
+        # Create a logger named with the module name.
+        logger = logging.getLogger(__name__)
+        # Set the logger log level to 'DEBUG' to log all kinds of messages.
+        logger.setLevel(logging.DEBUG)
+
+        # Clear any attached handlers to the logger from the previous runs.
+        # If the handlers are not cleared, then the new handler will be appended to the list of handlers.
+        # This makes the single log message be repeated according to the length of the list of handlers.
+        logger.handlers.clear()
+
+        # Create the handlers.
+        stream_handler = logging.StreamHandler()
+        # Set the handler log level to 'DEBUG' to log all kinds of messages received from the logger.
+        stream_handler.setLevel(logging.DEBUG)
+
+        # Create the formatter that just includes the log message.
+        formatter = logging.Formatter('%(message)s')
+
+        # Add the formatter to the handler.
+        stream_handler.setFormatter(formatter)
+
+        # Add the handler to the logger.
+        logger.addHandler(stream_handler)
+
+        # Create the 'self.logger' attribute to hold the logger.
+        # Instead of using 'print()', use 'self.logger.info()'
+        self.logger = logger
+
+class Input2D(CustomLogger):
 
     """
     Implementing the input layer of a CNN.
     The CNN architecture must start with an input layer.
     """
 
-    def __init__(self, input_shape):
+    def __init__(self, 
+                 input_shape,
+                 logger=None):
 
         """
         input_shape: Shape of the input sample to the CNN.
         """
 
+        super().__init__()
+
+        # If logger is None, then the CustomLogger.logger is created.
+        if logger is None:
+            pass
+        else:
+            self.logger = logger
+
         # If the input sample has less than 2 dimensions, then an exception is raised.
         if len(input_shape) < 2:
-            raise ValueError(f"The Input2D class creates an input layer for data inputs with at least 2 dimensions but ({len(input_shape)}) dimensions found.")
+            msg = f"The Input2D class creates an input layer for data inputs with at least 2 dimensions but ({len(input_shape)}) dimensions found."
+            self.logger.error(msg)
+            raise ValueError(msg)
         # If the input sample has exactly 2 dimensions, the third dimension is set to 1.
         elif len(input_shape) == 2:
             input_shape = (input_shape[0], input_shape[1], 1)
 
         for dim_idx, dim in enumerate(input_shape):
             if dim <= 0:
-                raise ValueError("The dimension size of the inputs cannot be <= 0. Please pass a valid value to the 'input_size' parameter.")
+                msg = "The dimension size of the inputs cannot be <= 0. Please pass a valid value to the 'input_size' parameter."
+                self.logger.error(msg)
+                raise ValueError(msg)
 
         self.input_shape = input_shape # Shape of the input sample.
         self.layer_output_size = input_shape # Shape of the output from the current layer. For an input layer, it is the same as the shape of the input sample.
 
-class Conv2D:
+class Conv2D(CustomLogger):
 
     """
     Implementing the convolution layer.
     """
 
-    def __init__(self, num_filters, kernel_size, previous_layer, activation_function=None):
+    def __init__(self, 
+                 num_filters, 
+                 kernel_size, 
+                 previous_layer, 
+                 activation_function=None,
+                 logger=None):
 
         """
         num_filters: Number of filters in the convolution layer.
@@ -241,13 +303,25 @@ def __init__(self, num_filters, kernel_size, previous_layer, activation_function
         activation_function=None: The name of the activation function to be used in the conv layer. If None, then no activation function is applied besides the convolution operation. The activation function can be applied by a separate layer.
         """
 
+        super().__init__()
+
+        # If logger is None, then the CustomLogger.logger is created.
+        if logger is None:
+            pass
+        else:
+            self.logger = logger
+
         if num_filters <= 0:
-            raise ValueError("Number of filters cannot be <= 0. Please pass a valid value to the 'num_filters' parameter.")
+            msg = "Number of filters cannot be <= 0. Please pass a valid value to the 'num_filters' parameter."
+            self.logger.error(msg)
+            raise ValueError(msg)
         # Number of filters in the conv layer.
         self.num_filters = num_filters
 
         if kernel_size <= 0:
-            raise ValueError("The kernel size cannot be <= 0. Please pass a valid value to the 'kernel_size' parameter.")
+            msg = "The kernel size cannot be <= 0. Please pass a valid value to the 'kernel_size' parameter."
+            self.logger.error(msg)
+            raise ValueError(msg)
         # Kernel size of each filter.
         self.kernel_size = kernel_size
 
@@ -259,15 +333,21 @@ def __init__(self, num_filters, kernel_size, previous_layer, activation_function
         elif (activation_function == "sigmoid"):
             self.activation = sigmoid
         elif (activation_function == "softmax"):
-            raise ValueError("The softmax activation function cannot be used in a conv layer.")
+            msg = "The softmax activation function cannot be used in a conv layer."
+            self.logger.error(msg)
+            raise ValueError(msg)
         else:
-            raise ValueError(f"The specified activation function '{activation_function}' is not among the supported activation functions {supported_activation_functions}. Please use one of the supported functions.")
+            msg = f"The specified activation function '{activation_function}' is not among the supported activation functions {supported_activation_functions}. Please use one of the supported functions."
+            self.logger.error(msg)
+            raise ValueError(msg)
 
         # The activation function used in the current layer.
         self.activation_function = activation_function
 
         if previous_layer is None:
-            raise TypeError("The previous layer cannot be of Type 'None'. Please pass a valid layer to the 'previous_layer' parameter.")
+            msg = "The previous layer cannot be of Type 'None'. Please pass a valid layer to the 'previous_layer' parameter."
+            self.logger.error(msg)
+            raise TypeError(msg)
         # A reference to the layer that preceeds the current layer in the network architecture.
         self.previous_layer = previous_layer
         
@@ -353,24 +433,36 @@ def conv(self, input2D):
         """
 
         if len(input2D.shape) != len(self.initial_weights.shape) - 1: # Check if there is a match in the number of dimensions between the image and the filters.
-            raise ValueError("Number of dimensions in the conv filter and the input do not match.")  
+            msg = "Number of dimensions in the conv filter and the input do not match."
+            self.logger.error(msg)
+            raise ValueError(msg)
         if len(input2D.shape) > 2 or len(self.initial_weights.shape) > 3: # Check if number of image channels matches the filter depth.
             if input2D.shape[-1] != self.initial_weights.shape[-1]:
-                raise ValueError("Number of channels in both the input and the filter must match.")
+                msg = "Number of channels in both the input and the filter must match."
+                self.logger.error(msg)
+                raise ValueError(msg)
         if self.initial_weights.shape[1] != self.initial_weights.shape[2]: # Check if filter dimensions are equal.
-            raise ValueError('A filter must be a square matrix. I.e. number of rows and columns must match.')
+            msg = 'A filter must be a square matrix. I.e. number of rows and columns must match.'
+            self.logger.error(msg)
+            raise ValueError(msg)
         if self.initial_weights.shape[1]%2==0: # Check if filter diemnsions are odd.
-            raise ValueError('A filter must have an odd size. I.e. number of rows and columns must be odd.')
+            msg = 'A filter must have an odd size. I.e. number of rows and columns must be odd.'
+            self.logger.error(msg)
+            raise ValueError(msg)
 
         self.layer_output = self.conv_(input2D, self.trained_weights)
 
-class AveragePooling2D:
+class AveragePooling2D(CustomLogger):
 
     """
     Implementing the average pooling layer.
     """
 
-    def __init__(self, pool_size, previous_layer, stride=2):
+    def __init__(self, 
+                 pool_size, 
+                 previous_layer, 
+                 stride=2,
+                 logger=None):
 
         """
         pool_size: Pool size.
@@ -378,19 +470,35 @@ def __init__(self, pool_size, previous_layer, stride=2):
         stride=2: Stride
         """
 
+        super().__init__()
+
+        # If logger is None, then the CustomLogger.logger is created.
+        if logger is None:
+            pass
+        else:
+            self.logger = logger
+
         if not (type(pool_size) is int):
-            raise ValueError("The expected type of the pool_size is int but {pool_size_type} found.".format(pool_size_type=type(pool_size)))
+            msg = "The expected type of the pool_size is int but {pool_size_type} found.".format(pool_size_type=type(pool_size))
+            self.logger.error(msg)
+            raise ValueError(msg)
 
         if pool_size <= 0:
-            raise ValueError("The passed value to the pool_size parameter cannot be <= 0.")
+            msg = "The passed value to the pool_size parameter cannot be <= 0."
+            self.logger.error(msg)
+            raise ValueError(msg)
         self.pool_size = pool_size
 
         if stride <= 0:
-            raise ValueError("The passed value to the stride parameter cannot be <= 0.")
+            msg = "The passed value to the stride parameter cannot be <= 0."
+            self.logger.error(msg)
+            raise ValueError(msg)
         self.stride = stride
 
         if previous_layer is None:
-            raise TypeError("The previous layer cannot be of Type 'None'. Please pass a valid layer to the 'previous_layer' parameter.")
+            msg = "The previous layer cannot be of Type 'None'. Please pass a valid layer to the 'previous_layer' parameter."
+            self.logger.error(msg)
+            raise TypeError(msg)
         # A reference to the layer that preceeds the current layer in the network architecture.
         self.previous_layer = previous_layer
 
@@ -430,33 +538,53 @@ def average_pooling(self, input2D):
 
         self.layer_output = pool_out
 
-class MaxPooling2D:
+class MaxPooling2D(CustomLogger):
 
     """
     Similar to the AveragePooling2D class except that it implements max pooling.
     """
 
-    def __init__(self, pool_size, previous_layer, stride=2):
+    def __init__(self, 
+                 pool_size, 
+                 previous_layer, 
+                 stride=2,
+                 logger=None):
         
         """
         pool_size: Pool size.
         previous_layer: Reference to the previous layer in the CNN architecture.
         stride=2: Stride
         """
-        
+
+        super().__init__()
+
+        # If logger is None, then the CustomLogger.logger is created.
+        if logger is None:
+            pass
+        else:
+            self.logger = logger
+
         if not (type(pool_size) is int):
-            raise ValueError(f"The expected type of the pool_size is int but {type(pool_size)} found.")
+            msg = f"The expected type of the pool_size is int but {type(pool_size)} found."
+            self.logger.error(msg)
+            raise ValueError(msg)
 
         if pool_size <= 0:
-            raise ValueError("The passed value to the pool_size parameter cannot be <= 0.")
+            msg = "The passed value to the pool_size parameter cannot be <= 0."
+            self.logger.error(msg)
+            raise ValueError(msg)
         self.pool_size = pool_size
 
         if stride <= 0:
-            raise ValueError("The passed value to the stride parameter cannot be <= 0.")
+            msg = "The passed value to the stride parameter cannot be <= 0."
+            self.logger.error(msg)
+            raise ValueError(msg)
         self.stride = stride
 
         if previous_layer is None:
-            raise TypeError("The previous layer cannot be of Type 'None'. Please pass a valid layer to the 'previous_layer' parameter.")
+            msg = "The previous layer cannot be of Type 'None'. Please pass a valid layer to the 'previous_layer' parameter."
+            self.logger.error(msg)
+            raise TypeError(msg)
         # A reference to the layer that preceeds the current layer in the network architecture.
         self.previous_layer = previous_layer
 
@@ -496,20 +624,32 @@ def max_pooling(self, input2D):
 
         self.layer_output = pool_out
 
-class ReLU:
+class ReLU(CustomLogger):
 
     """
     Implementing the ReLU layer.
     """
 
-    def __init__(self, previous_layer):
+    def __init__(self, 
+                 previous_layer,
+                 logger=None):
 
         """
         previous_layer: Reference to the previous layer.
         """
 
+        super().__init__()
+
+        # If logger is None, then the CustomLogger.logger is created.
+        if logger is None:
+            pass
+        else:
+            self.logger = logger
+
         if previous_layer is None:
-            raise TypeError("The previous layer cannot be of Type 'None'. Please pass a valid layer to the 'previous_layer' parameter.")
+            msg = "The previous layer cannot be of Type 'None'. Please pass a valid layer to the 'previous_layer' parameter."
+            self.logger.error(msg)
+            raise TypeError(msg)
 
         # A reference to the layer that preceeds the current layer in the network architecture.
         self.previous_layer = previous_layer
@@ -536,20 +676,32 @@ def relu_layer(self, layer_input):
         self.layer_output_size = layer_input.size
         self.layer_output = relu(layer_input)
 
-class Sigmoid:
+class Sigmoid(CustomLogger):
 
     """
     Implementing the sigmoid layer.
     """
 
-    def __init__(self, previous_layer):
+    def __init__(self, 
+                 previous_layer,
+                 logger=None):
 
         """
         previous_layer: Reference to the previous layer.
         """
 
+        super().__init__()
+
+        # If logger is None, then the CustomLogger.logger is created.
+        if logger is None:
+            pass
+        else:
+            self.logger = logger
+
         if previous_layer is None:
-            raise TypeError("The previous layer cannot be of Type 'None'. Please pass a valid layer to the 'previous_layer' parameter.")
+            msg = "The previous layer cannot be of Type 'None'. Please pass a valid layer to the 'previous_layer' parameter."
+            self.logger.error(msg)
+            raise TypeError(msg)
         # A reference to the layer that preceeds the current layer in the network architecture.
         self.previous_layer = previous_layer
 
@@ -575,20 +727,32 @@ def sigmoid_layer(self, layer_input):
         self.layer_output_size = layer_input.size
         self.layer_output = sigmoid(layer_input)
 
-class Flatten:
+class Flatten(CustomLogger):
 
     """
     Implementing the flatten layer.
     """
 
-    def __init__(self, previous_layer):
+    def __init__(self, 
+                 previous_layer,
+                 logger=None):
         
         """
         previous_layer: Reference to the previous layer.
         """
 
+        super().__init__()
+
+        # If logger is None, then the CustomLogger.logger is created.
+        if logger is None:
+            pass
+        else:
+            self.logger = logger
+
         if previous_layer is None:
-            raise TypeError("The previous layer cannot be of Type 'None'. Please pass a valid layer to the 'previous_layer' parameter.")
+            msg = "The previous layer cannot be of Type 'None'. Please pass a valid layer to the 'previous_layer' parameter."
+            self.logger.error(msg)
+            raise TypeError(msg)
         # A reference to the layer that preceeds the current layer in the network architecture.
         self.previous_layer = previous_layer
 
@@ -614,22 +778,37 @@ def flatten(self, input2D):
         self.layer_output_size = input2D.size
         self.layer_output = numpy.ravel(input2D)
 
-class Dense:
+class Dense(CustomLogger):
 
     """
     Implementing the input dense (fully connected) layer of a CNN.
     """
 
-    def __init__(self, num_neurons, previous_layer, activation_function="relu"):
+    def __init__(self, 
+                 num_neurons, 
+                 previous_layer, 
+                 activation_function="relu",
+                 logger=None):
 
         """
         num_neurons: Number of neurons in the dense layer.
         previous_layer: Reference to the previous layer.
         activation_function: Name of the activation function to be used in the current layer.
+        logger=None: Reference to the instance of the logging.Logger class.
         """
 
+        super().__init__()
+
+        # If logger is None, then the CustomLogger.logger is created.
+        if logger is None:
+            pass
+        else:
+            self.logger = logger
+
         if num_neurons <= 0:
-            raise ValueError("Number of neurons cannot be <= 0. Please pass a valid value to the 'num_neurons' parameter.")
+            msg = "Number of neurons cannot be <= 0. Please pass a valid value to the 'num_neurons' parameter."
+            self.logger.error(msg)
+            raise ValueError(msg)
 
         # Number of neurons in the dense layer.
         self.num_neurons = num_neurons
@@ -642,17 +821,23 @@ def __init__(self, num_neurons, previous_layer, activation_function="relu"):
         elif (activation_function == "softmax"):
             self.activation = softmax
         else:
-            raise ValueError(f"The specified activation function '{activation_function}' is not among the supported activation functions {supported_activation_functions}. Please use one of the supported functions.")
+            msg = f"The specified activation function '{activation_function}' is not among the supported activation functions {supported_activation_functions}. Please use one of the supported functions."
+            self.logger.error(msg)
+            raise ValueError(msg)
 
         self.activation_function = activation_function
 
         if previous_layer is None:
-            raise TypeError("The previous layer cannot be of Type 'None'. Please pass a valid layer to the 'previous_layer' parameter.")
+            msg = "The previous layer cannot be of Type 'None'. Please pass a valid layer to the 'previous_layer' parameter."
+            self.logger.error(msg)
+            raise TypeError(msg)
         # A reference to the layer that preceeds the current layer in the network architecture.
         self.previous_layer = previous_layer
-        
+
         if type(self.previous_layer.layer_output_size) in [list, tuple, numpy.ndarray] and len(self.previous_layer.layer_output_size) > 1:
-            raise ValueError(f"The input to the dense layer must be of type int but {type(self.previous_layer.layer_output_size)} found.")
+            msg = f"The input to the dense layer must be of type int but {type(self.previous_layer.layer_output_size)} found."
+            self.logger.error(msg)
+            raise ValueError(msg)
         # Initializing the weights of the layer.
         self.initial_weights = numpy.random.uniform(low=-0.1,
                                                     high=0.1,
@@ -682,26 +867,41 @@ def dense_layer(self, layer_input):
         """
 
         if self.trained_weights is None:
-            raise TypeError("The weights of the dense layer cannot be of Type 'None'.")
+            msg = "The weights of the dense layer cannot be of Type 'None'."
+            self.logger.error(msg)
+            raise TypeError(msg)
 
         sop = numpy.matmul(layer_input, self.trained_weights)
 
         self.layer_output = self.activation(sop)
 
-class Model:
+class Model(CustomLogger):
 
     """
     Creating a CNN model.
     """
 
-    def __init__(self, last_layer, epochs=10, learning_rate=0.01):
+    def __init__(self, 
+                 last_layer, 
+                 epochs=10, 
+                 learning_rate=0.01,
+                 logger=None):
         
         """
         last_layer: A reference to the last layer in the CNN architecture.
         epochs=10: Number of epochs.
         learning_rate=0.01: Learning rate.
+        logger=None: Reference to the instance of the logging.Logger class.
         """
 
+        super().__init__()
+
+        # If logger is None, then the CustomLogger.logger is created.
+        if logger is None:
+            pass
+        else:
+            self.logger = logger
+
         self.last_layer = last_layer
         self.epochs = epochs
         self.learning_rate = learning_rate
@@ -728,7 +928,7 @@ def get_layers(self):
         return network_layers
 
     def train(self, train_inputs, train_outputs):
-        
+
         """
         Trains the CNN model.
         It is important to note that no learning algorithm is used for training the CNN. Just the learning rate is used for making some changes which is better than leaving the weights unchanged.
@@ -738,16 +938,20 @@ def train(self, train_inputs, train_outputs):
         """
         
         if (train_inputs.ndim != 4):
-            raise ValueError("The training data input has {num_dims} but it must have 4 dimensions. The first dimension is the number of training samples, the second & third dimensions represent the width and height of the sample, and the fourth dimension represents the number of channels in the sample.".format(num_dims=train_inputs.ndim))    
+            msg = f"The training data input has {train_inputs.ndim} but it must have 4 dimensions. The first dimension is the number of training samples, the second & third dimensions represent the width and height of the sample, and the fourth dimension represents the number of channels in the sample."
+            self.logger.error(msg)
+            raise ValueError(msg)
 
         if (train_inputs.shape[0] != len(train_outputs)):
-            raise ValueError(f"Mismatch between the number of input samples and number of labels: {train_inputs.shape[0]} != {len(train_outputs)}.")
+            msg = f"Mismatch between the number of input samples and number of labels: {train_inputs.shape[0]} != {len(train_outputs)}."
+            self.logger.error(msg)
+            raise ValueError(msg)
 
         network_predictions = []
         network_error = 0
     
         for epoch in range(self.epochs):
-            print(f"Epoch {epoch}")
+            self.logger.info(f"Epoch {epoch}")
             for sample_idx in range(train_inputs.shape[0]):
                 # print("Sample {sample_idx}".format(sample_idx=sample_idx))
                 self.feed_sample(train_inputs[sample_idx, :])
@@ -755,8 +959,10 @@ def train(self, train_inputs, train_outputs):
                 try:
                     predicted_label = numpy.where(numpy.max(self.last_layer.layer_output) == self.last_layer.layer_output)[0][0]
                 except IndexError:
-                    print(self.last_layer.layer_output)
-                    raise IndexError("Index out of range")
+                    self.logger.info(self.last_layer.layer_output)
+                    msg = "Index out of range"
+                    self.logger.error(msg)
+                    raise IndexError(msg)
                 network_predictions.append(predicted_label)
     
                 network_error = network_error + abs(predicted_label - train_outputs[sample_idx])
@@ -796,8 +1002,10 @@ def feed_sample(self, sample):
             elif type(layer) is Input2D:
                 pass
             else:
-                print("Other")
-                raise TypeError("The layer of type {type(layer)} is not supported yet.")
+                # self.logger.info("Other")
+                msg = "The layer of type {type(layer)} is not supported."
+                self.logger.error(msg)
+                raise TypeError(msg)
 
             last_layer_outputs = layer.layer_output
         return self.network_layers[-1].layer_output
@@ -828,7 +1036,9 @@ def predict(self, data_inputs):
         """
 
         if (data_inputs.ndim != 4):
-            raise ValueError("The data input has {data_inputs.ndim} but it must have 4 dimensions. The first dimension is the number of training samples, the second & third dimensions represent the width and height of the sample, and the fourth dimension represents the number of channels in the sample.")
+            msg = "The data input has {data_inputs.ndim} but it must have 4 dimensions. The first dimension is the number of training samples, the second & third dimensions represent the width and height of the sample, and the fourth dimension represents the number of channels in the sample."
+            self.logger.error(msg)
+            raise ValueError(msg)
 
         predictions = []
         for sample in data_inputs:
@@ -843,7 +1053,7 @@ def summary(self):
         Prints a summary of the CNN architecture.
         """
 
-        print("\n----------Network Architecture----------")
+        self.logger.info("\n----------Network Architecture----------")
         for layer in self.network_layers:
-            print(type(layer))
-        print("----------------------------------------\n")
+            self.logger.info(type(layer))
+        self.logger.info("----------------------------------------\n")
diff --git a/pygad/gann/gann.py b/pygad/gann/gann.py
index 4037e5a..2c3c5a7 100644
--- a/pygad/gann/gann.py
+++ b/pygad/gann/gann.py
@@ -1,4 +1,5 @@
 from ..nn import nn
+import warnings
 
 def validate_network_parameters(num_neurons_input, 
                                 num_neurons_output, 
@@ -81,7 +82,7 @@ def validate_network_parameters(num_neurons_input,
         else:
             raise TypeError(unexpected_activation_type.format(activations_type=type(hidden_activations)))
     else: # In case there are no hidden layers (num_hidden_layers == 0)
-        print("WARNING: There are no hidden layers however a value is assigned to the parameter 'hidden_activations'. It will be reset to [].")
+        warnings.warn("WARNING: There are no hidden layers however a value is assigned to the parameter 'hidden_activations'. It will be reset to [].")
         hidden_activations = []
 
     # If the value passed to the 'hidden_activations' parameter is actually a list, then its elements are checked to make sure the listed name(s) of the activation function(s) are supported.

From 4de21aecec631fda6d48c2703e7de7059f61a88a Mon Sep 17 00:00:00 2001
From: Ahmed Gad <ahmed.f.gad@gmail.com>
Date: Tue, 9 May 2023 17:28:57 -0400
Subject: [PATCH 2/4] Return outputs from callbacks

---
 pygad/pygad.py | 94 ++++++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 84 insertions(+), 10 deletions(-)

diff --git a/pygad/pygad.py b/pygad/pygad.py
index 01dc850..9352169 100644
--- a/pygad/pygad.py
+++ b/pygad/pygad.py
@@ -1693,7 +1693,7 @@ def cal_pop_fitness(self):
             pop_fitness = numpy.array(pop_fitness)
         except Exception as ex:
             self.logger.exception(ex)
-            exit(-1)
+            sys.exit(-1)
         return pop_fitness
 
     def run(self):
@@ -1738,8 +1738,7 @@ def run(self):
             # Measuring the fitness of each chromosome in the population. Save the fitness in the last_generation_fitness attribute.
             self.last_generation_fitness = self.cal_pop_fitness()
 
-            best_solution, best_solution_fitness, best_match_idx = self.best_solution(
-                pop_fitness=self.last_generation_fitness)
+            best_solution, best_solution_fitness, best_match_idx = self.best_solution(pop_fitness=self.last_generation_fitness)
 
             # Appending the best solution in the initial population to the best_solutions list.
             if self.save_best_solutions:
@@ -1747,7 +1746,20 @@ def run(self):
 
             for generation in range(generation_first_idx, generation_last_idx):
                 if not (self.on_fitness is None):
-                    self.on_fitness(self, self.last_generation_fitness)
+                    on_fitness_output = self.on_fitness(self, 
+                                                        self.last_generation_fitness)
+
+                    if on_fitness_output is None:
+                        pass
+                    else:
+                        if type(on_fitness_output) in [tuple, list, numpy.ndarray, range]:
+                            on_fitness_output = numpy.array(on_fitness_output)
+                            if on_fitness_output.shape == self.last_generation_fitness.shape:
+                                self.last_generation_fitness = on_fitness_output
+                            else:
+                                raise ValueError(f"Size mismatch between the output of on_fitness() {on_fitness_output.shape} and the expected fitness output {self.last_generation_fitness.shape}.")
+                        else:
+                            raise ValueError(f"The output of on_fitness() is expected to be tuple/list/range/numpy.ndarray but {type(on_fitness_output)} found.")
 
                 # Appending the fitness value of the best solution in the current generation to the best_solutions_fitness attribute.
                 self.best_solutions_fitness.append(best_solution_fitness)
@@ -1788,7 +1800,45 @@ def run(self):
                     raise ValueError(f"The iterable holding the selected parents indices is expected to have ({self.num_parents_mating}) values but ({len(self.last_generation_parents_indices)}) found.")
 
                 if not (self.on_parents is None):
-                    self.on_parents(self, self.last_generation_parents)
+                    on_parents_output = self.on_parents(self, 
+                                                        self.last_generation_parents)
+                    
+                    if on_parents_output is None:
+                        pass
+                    elif type(on_parents_output) in [list, tuple, numpy.ndarray]:
+                        if len(on_parents_output) == 2:
+                            on_parents_selected_parents, on_parents_selected_parents_indices = on_parents_output
+                        else:
+                            raise ValueError(f"The output of on_parents() is expected to be tuple/list/numpy.ndarray of length 2 but {type(on_parents_output)} of length {len(on_parents_output)} found.")
+
+                        # Validate the parents.
+                        if on_parents_selected_parents is None:
+                            raise ValueError("The returned outputs of on_parents() cannot be None but the first output is None.")
+                        else:
+                            if type(on_parents_selected_parents) in [tuple, list, numpy.ndarray]:
+                                on_parents_selected_parents = numpy.array(on_parents_selected_parents)
+                                if on_parents_selected_parents.shape == self.last_generation_parents.shape:
+                                    self.last_generation_parents = on_parents_selected_parents
+                                else:
+                                    raise ValueError(f"Size mismatch between the parents retrned by on_parents() {on_parents_selected_parents.shape} and the expected parents shape {self.last_generation_parents.shape}.")
+                            else:
+                                raise ValueError(f"The output of on_parents() is expected to be tuple/list/numpy.ndarray but the first output type is {type(on_parents_selected_parents)}.")
+
+                        # Validate the parents indices.
+                        if on_parents_selected_parents_indices is None:
+                            raise ValueError("The returned outputs of on_parents() cannot be None but the second output is None.")
+                        else:
+                            if type(on_parents_selected_parents_indices) in [tuple, list, numpy.ndarray, range]:
+                                on_parents_selected_parents_indices = numpy.array(on_parents_selected_parents_indices)
+                                if on_parents_selected_parents_indices.shape == self.last_generation_parents_indices.shape:
+                                    self.last_generation_parents_indices = on_parents_selected_parents_indices
+                                else:
+                                    raise ValueError(f"Size mismatch between the parents indices returned by on_parents() {on_parents_selected_parents_indices.shape} and the expected crossover output {self.last_generation_parents_indices.shape}.")
+                            else:
+                                raise ValueError(f"The output of on_parents() is expected to be tuple/list/range/numpy.ndarray but the second output type is {type(on_parents_selected_parents_indices)}.")
+
+                    else:
+                        raise TypeError(f"The output of on_parents() is expected to be tuple/list/numpy.ndarray but {type(on_parents_output)} found.")
 
                 # If self.crossover_type=None, then no crossover is applied and thus no offspring will be created in the next generations. The next generation will use the solutions in the current population.
                 if self.crossover_type is None:
@@ -1832,8 +1882,19 @@ def run(self):
 
                 # PyGAD 2.18.2 // The on_crossover() callback function is called even if crossover_type is None.
                 if not (self.on_crossover is None):
-                    self.on_crossover(
-                        self, self.last_generation_offspring_crossover)
+                    on_crossover_output = self.on_crossover(self, 
+                                                            self.last_generation_offspring_crossover)
+                    if on_crossover_output is None:
+                        pass
+                    else:
+                        if type(on_crossover_output) in [tuple, list, numpy.ndarray]:
+                            on_crossover_output = numpy.array(on_crossover_output)
+                            if on_crossover_output.shape == self.last_generation_offspring_crossover.shape:
+                                self.last_generation_offspring_crossover = on_crossover_output
+                            else:
+                                raise ValueError(f"Size mismatch between the output of on_crossover() {on_crossover_output.shape} and the expected crossover output {self.last_generation_offspring_crossover.shape}.")
+                        else:
+                            raise ValueError(f"The output of on_crossover() is expected to be tuple/list/numpy.ndarray but {type(on_crossover_output)} found.")
 
                 # If self.mutation_type=None, then no mutation is applied and thus no changes are applied to the offspring created using the crossover operation. The offspring will be used unchanged in the next generation.
                 if self.mutation_type is None:
@@ -1857,7 +1918,20 @@ def run(self):
 
                 # PyGAD 2.18.2 // The on_mutation() callback function is called even if mutation_type is None.
                 if not (self.on_mutation is None):
-                    self.on_mutation(self, self.last_generation_offspring_mutation)
+                    on_mutation_output = self.on_mutation(self, 
+                                                          self.last_generation_offspring_mutation)
+
+                    if on_mutation_output is None:
+                        pass
+                    else:
+                        if type(on_mutation_output) in [tuple, list, numpy.ndarray]:
+                            on_mutation_output = numpy.array(on_mutation_output)
+                            if on_mutation_output.shape == self.last_generation_offspring_mutation.shape:
+                                self.last_generation_offspring_mutation = on_mutation_output
+                            else:
+                                raise ValueError(f"Size mismatch between the output of on_mutation() {on_mutation_output.shape} and the expected mutation output {self.last_generation_offspring_mutation.shape}.")
+                        else:
+                            raise ValueError(f"The output of on_mutation() is expected to be tuple/list/numpy.ndarray but {type(on_mutation_output)} found.")
 
                 # Update the population attribute according to the offspring generated.
                 if self.keep_elitism == 0:
@@ -1954,7 +2028,7 @@ def run(self):
             # self.solutions = numpy.array(self.solutions)
         except Exception as ex:
             self.logger.exception(ex)
-            exit(-1)
+            sys.exit(-1)
 
     def best_solution(self, pop_fitness=None):
         """
@@ -1989,7 +2063,7 @@ def best_solution(self, pop_fitness=None):
             best_solution_fitness = pop_fitness[best_match_idx]
         except Exception as ex:
             self.logger.exception(ex)
-            exit(-1)
+            sys.exit(-1)
 
         return best_solution, best_solution_fitness, best_match_idx
 

From 63e83122040b2ead0f3a57eaa88e3d418df50628 Mon Sep 17 00:00:00 2001
From: Ahmed Gad <ahmed.f.gad@gmail.com>
Date: Tue, 9 May 2023 17:31:21 -0400
Subject: [PATCH 3/4] Change workflows names

---
 .github/workflows/main_py310.yml | 2 +-
 .github/workflows/main_py311.yml | 2 +-
 .github/workflows/main_py37.yml  | 2 +-
 .github/workflows/main_py38.yml  | 2 +-
 .github/workflows/main_py39.yml  | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/main_py310.yml b/.github/workflows/main_py310.yml
index 7e45fa9..7260720 100644
--- a/.github/workflows/main_py310.yml
+++ b/.github/workflows/main_py310.yml
@@ -1,4 +1,4 @@
-name: Testing PyGAD using PyTest
+name: Python 3.10 Testing PyGAD using PyTest
 
 on:
   push:
diff --git a/.github/workflows/main_py311.yml b/.github/workflows/main_py311.yml
index 37b2085..5c37789 100644
--- a/.github/workflows/main_py311.yml
+++ b/.github/workflows/main_py311.yml
@@ -1,4 +1,4 @@
-name: Testing PyGAD using PyTest
+name: Python 3.11 Testing PyGAD using PyTest
 
 on:
   push:
diff --git a/.github/workflows/main_py37.yml b/.github/workflows/main_py37.yml
index 061ada0..77e5a9f 100644
--- a/.github/workflows/main_py37.yml
+++ b/.github/workflows/main_py37.yml
@@ -1,4 +1,4 @@
-name: Testing PyGAD using PyTest
+name: Python 3.7 Testing PyGAD using PyTest
 
 on:
   push:
diff --git a/.github/workflows/main_py38.yml b/.github/workflows/main_py38.yml
index 908b08e..7839f18 100644
--- a/.github/workflows/main_py38.yml
+++ b/.github/workflows/main_py38.yml
@@ -1,4 +1,4 @@
-name: Testing PyGAD using PyTest
+name: Python 3.8 Testing PyGAD using PyTest
 
 on:
   push:
diff --git a/.github/workflows/main_py39.yml b/.github/workflows/main_py39.yml
index 0ef4f1c..b34d7ba 100644
--- a/.github/workflows/main_py39.yml
+++ b/.github/workflows/main_py39.yml
@@ -1,4 +1,4 @@
-name: Testing PyGAD using PyTest
+name: Python 3.9 Testing PyGAD using PyTest
 
 on:
   push:

From 464bfd619cac2f17d954e087a315b855630dc86d Mon Sep 17 00:00:00 2001
From: Ahmed Gad <ahmed.f.gad@gmail.com>
Date: Wed, 10 May 2023 10:23:17 -0400
Subject: [PATCH 4/4] Batch fitness for adaptive mutation

---
 pygad/pygad.py          | 18 ++++++------------
 pygad/utils/mutation.py | 35 +++++++++++++++++++++++++++++++++--
 2 files changed, 39 insertions(+), 14 deletions(-)

diff --git a/pygad/pygad.py b/pygad/pygad.py
index 9352169..95a59cd 100644
--- a/pygad/pygad.py
+++ b/pygad/pygad.py
@@ -1531,8 +1531,7 @@ def cal_pop_fitness(self):
                     elif (self.keep_elitism > 0) and (self.last_generation_elitism is not None) and (len(self.last_generation_elitism) > 0) and (list(sol) in last_generation_elitism_as_list):
                         # Return the index of the elitism from the elitism array 'self.last_generation_elitism'.
                         # This is not its index within the population. It is just its index in the 'self.last_generation_elitism' array.
-                        elitism_idx = last_generation_elitism_as_list.index(
-                            list(sol))
+                        elitism_idx = last_generation_elitism_as_list.index(list(sol))
                         # Use the returned elitism index to return its index in the last population.
                         elitism_idx = self.last_generation_elitism_indices[elitism_idx]
                         # Use the elitism's index to return its pre-calculated fitness value.
@@ -1544,8 +1543,7 @@ def cal_pop_fitness(self):
                         # Index of the parent in the 'self.last_generation_parents' array.
                         # This is not its index within the population. It is just its index in the 'self.last_generation_parents' array.
                         # parent_idx = numpy.where(numpy.all(self.last_generation_parents == sol, axis=1))[0][0]
-                        parent_idx = last_generation_parents_as_list.index(
-                            list(sol))
+                        parent_idx = last_generation_parents_as_list.index(list(sol))
                         # Use the returned parent index to return its index in the last population.
                         parent_idx = self.last_generation_parents_indices[parent_idx]
                         # Use the parent's index to return its pre-calculated fitness value.
@@ -1573,13 +1571,11 @@ def cal_pop_fitness(self):
                     solutions_indices = numpy.where(
                         numpy.array(pop_fitness) == "undefined")[0]
                     # Number of batches.
-                    num_batches = int(numpy.ceil(
-                        len(solutions_indices) / self.fitness_batch_size))
+                    num_batches = int(numpy.ceil(len(solutions_indices) / self.fitness_batch_size))
                     # For each batch, get its indices and call the fitness function.
                     for batch_idx in range(num_batches):
                         batch_first_index = batch_idx * self.fitness_batch_size
-                        batch_last_index = (batch_idx + 1) * \
-                            self.fitness_batch_size
+                        batch_last_index = (batch_idx + 1) * self.fitness_batch_size
                         batch_indices = solutions_indices[batch_first_index:batch_last_index]
                         batch_solutions = self.population[batch_indices, :]
 
@@ -1660,8 +1656,7 @@ def cal_pop_fitness(self):
                         # Reaching this block means that batch processing is used. The fitness values are calculated in batches.
 
                         # Number of batches.
-                        num_batches = int(numpy.ceil(
-                            len(solutions_to_submit_indices) / self.fitness_batch_size))
+                        num_batches = int(numpy.ceil(len(solutions_to_submit_indices) / self.fitness_batch_size))
                         # Each element of the `batches_solutions` list represents the solutions in one batch.
                         batches_solutions = []
                         # Each element of the `batches_indices` list represents the solutions' indices in one batch.
@@ -1669,8 +1664,7 @@ def cal_pop_fitness(self):
                         # For each batch, get its indices and call the fitness function.
                         for batch_idx in range(num_batches):
                             batch_first_index = batch_idx * self.fitness_batch_size
-                            batch_last_index = (batch_idx + 1) * \
-                                self.fitness_batch_size
+                            batch_last_index = (batch_idx + 1) * self.fitness_batch_size
                             batch_indices = solutions_to_submit_indices[batch_first_index:batch_last_index]
                             batch_solutions = self.population[batch_indices, :]
 
diff --git a/pygad/utils/mutation.py b/pygad/utils/mutation.py
index 326ba6b..39a1a5d 100644
--- a/pygad/utils/mutation.py
+++ b/pygad/utils/mutation.py
@@ -438,8 +438,39 @@ def adaptive_mutation_population_fitness(self, offspring):
 
         fitness[:self.last_generation_parents.shape[0]] = self.last_generation_fitness[self.last_generation_parents_indices]
 
-        for idx in range(len(parents_to_keep), fitness.shape[0]):
-            fitness[idx] = self.fitness_func(self, temp_population[idx], None)
+        first_idx = len(parents_to_keep)
+        last_idx = fitness.shape[0]
+        fitness[first_idx:last_idx] = [0]*(last_idx - first_idx)
+
+        if self.fitness_batch_size in [1, None]:
+            # Calculate the fitness for each individual solution.
+            for idx in range(first_idx, last_idx):
+                fitness[idx] = self.fitness_func(self, 
+                                                 temp_population[idx], 
+                                                 None)
+        else:
+            # Calculate the fitness for batch of solutions.
+
+            # Number of batches.
+            num_batches = int(numpy.ceil((last_idx - first_idx) / self.fitness_batch_size))
+
+            for batch_idx in range(num_batches):
+                # The index of the first solution in the current batch.
+                batch_first_index = first_idx + batch_idx * self.fitness_batch_size
+                # The index of the last solution in the current batch.
+                if batch_idx == (num_batches - 1):
+                    batch_last_index = last_idx
+                else:
+                    batch_last_index = first_idx + (batch_idx + 1) * self.fitness_batch_size
+
+                # Calculate the fitness values for the batch.
+                fitness_temp = self.fitness_func(self, 
+                                                 temp_population[batch_first_index:batch_last_index], 
+                                                 None) 
+                # Insert the fitness of each solution at the proper index.
+                for idx in range(batch_first_index, batch_last_index):
+                    fitness[idx] = fitness_temp[idx - batch_first_index]
+
         average_fitness = numpy.mean(fitness)
 
         return average_fitness, fitness[len(parents_to_keep):]