diff --git a/.gitignore b/.gitignore new file mode 100755 index 0000000..bb00fe2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +*.pyc +neuralnetworks/backpropagation/results/ +genetics/basic/results/ +neuralnetworks/hopfield/results/ +selforganization/pso/results + diff --git a/README.md b/README.md old mode 100644 new mode 100755 index 3d6e8cc..b12664a --- a/README.md +++ b/README.md @@ -1,2 +1,48 @@ -# PyNet -A Simple Hopfield Neural Network written in Python. +# BioPy + +####Overview: +---- +BioPy is a collection (in-progress) of biologically-inspired algorithms written in Python. Some of the algorithms included are more focused on artificial model's of biological computation, such as Hopfield Neural Networks, while others are inherently more biologically-focused, such as the basic genetic programming module included in this project. Use it for whatever you like, and please contribute back to the project by cleaning up code that is here or contributing new code for +applications in biology that you may find interesting to program. + +NOTE: The code is currently messy in some places. If you want to make a Pull Request by tidying up the code, that would certainly be merged. Since most of this was written while in the middle of our (jaredmichaelsmith and davpcunn) Graduate Bio-Inspired computation course, there are some places where the code has diverged into a dark chasm of non-pythonic mess, despite the algorithms still performing very well. Contributions in this area are much appreciated! + +####Dependencies: +---- +- NumPy +- SciPy +- Scikit-Learn +- Matplotlib + +####Categories +---- +Below you will find several categories of applications in this project. + +#####Neural Networks: +---- +- Hopfield Neural Network +- Back Propagation Neural Network + - Tests included: + - XOR + - Two Function Approximations + - MNIST Handwritten Digits Recognition Test + - From scikit-learn package + - Fisher's Iris data set: http://en.wikipedia.org/wiki/Iris_flower_data_set + - From scikit-learn package + - Labelled Faces in the Wild Dataset: http://vis-www.cs.umass.edu/lfw/ + - From scikit-learn package, originally collected by the University of Mass. Amherst + +#####Genetic Programming: +---- +- Basic Genetic Computation Algorithm + - Features: + - "drag-n-drop" fitness functions + - crossover and mutation of genes + - learning ability for offspring of each generation + +#####Self-Organization and Autonomous Agents +---- +- Particle Swarm Optimization + - Features: + - You can configure many of the parameters of the algorithm such as the velocity, acceleration, and number of initial particles, as well as several other parameters. + diff --git a/genetics/README.md b/genetics/README.md new file mode 100644 index 0000000..51f0265 --- /dev/null +++ b/genetics/README.md @@ -0,0 +1,7 @@ +Genetics +===== +- Basic Genetic Computation Algorithm + - Features: + - "drag-n-drop" fitness functions + - crossover and mutation of genes + - learning ability for offspring of each generation diff --git a/genetics/basic/__init__.py b/genetics/basic/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/genetics/basic/driver.py b/genetics/basic/driver.py new file mode 100644 index 0000000..b849c56 --- /dev/null +++ b/genetics/basic/driver.py @@ -0,0 +1,62 @@ +#driver.py +import os + +def run(exp=0, l=20, N=30, G=10, pm=.033, pc=.6, NG=20): + string = 'python .\geneticRun.py'\ + + ' -exp ' + str(exp) \ + + ' --num_bits '+ str(l) \ + + ' --population_size ' + str(N)\ + + ' --num_gens ' + str(G)\ + + ' --pr_mutation ' + str(pm)\ + + ' --pr_crossover ' + str(pc)\ + + ' --num_learning_guesses ' + str(NG) + print "DRIVER: %s" % string + os.system(string) +def learn(exp=0, l=20, N=30, G=10, pm=.033, pc=.6, learn=True, NG=20): + string = 'python .\geneticRun.py'\ + + ' -exp ' + str(exp) \ + + ' --num_bits '+ str(l) \ + + ' --population_size ' + str(N)\ + + ' --num_gens ' + str(G)\ + + ' --pr_mutation ' + str(pm)\ + + ' --pr_crossover ' + str(pc)\ + + ' --learn_offspring ' + str(learn)\ + + ' --num_learning_guesses ' + str(NG) + print "DRIVER: %s" % string; + os.system(string) +def CE(exp=0, l=20, N=30, G=10, pm=.033, pc=.6, learn=False, CE=True, NG=20, nruns=1, seed=123456): + string = 'python .\geneticRun.py'\ + + ' -exp ' + str(exp) \ + + ' --num_bits '+ str(l) \ + + ' --population_size ' + str(N)\ + + ' --num_gens ' + str(G)\ + + ' --pr_mutation ' + str(pm)\ + + ' --pr_crossover ' + str(pc)\ + + ' --learn_offspring ' + str(learn)\ + + ' --change_environment ' + str(CE)\ + + ' --num_learning_guesses ' + str(NG) + print "DRIVER: %s" % string; + os.system(string) + +exp = 0; +pm = .033; +N = 30 +print "DRIVER: EVALUATING POPULATION SIZE AND MUTATION PROBABILITY" +run(exp = exp, N = N, pm = 1/N) + +while (N < 150): + run(exp = exp, N = N, pm = 1/N) + N+=10 + exp+=1 +print "DRIVER: EVALUATING CROSSOVER PROBABLITY" +pc = .3 +while (pc < 1): + run(exp = exp, pc = pc) + pc +=.1 + exp+=1 +print "DRIVER: EVALUATING NUMBER OF GENERATIONS" +G=30 +while (G <= 150): + run(exp = exp, G = G); + G+=10 + exp+=1 \ No newline at end of file diff --git a/genetics/basic/ffs.py b/genetics/basic/ffs.py new file mode 100644 index 0000000..e64eafd --- /dev/null +++ b/genetics/basic/ffs.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +############################################################## +# ffs.py, fitness functions to be used in +# genetic_algorithms.py +# +# Written by Jared Smith for COSC 427/527 +# at the University of Tennessee, Knoxville. +############################################################### + + +def fitness_func_1(bit_sum, l): + return (pow(((bit_sum *1.0)/ pow(2.0, l)), 10)) + + +def fitness_func_2(bit_sum, l): + return (pow(((1.0 - bit_sum) / pow(2.0, l)), 10)) diff --git a/genetics/basic/geneticAlgorithms.py b/genetics/basic/geneticAlgorithms.py new file mode 100644 index 0000000..d84df60 --- /dev/null +++ b/genetics/basic/geneticAlgorithms.py @@ -0,0 +1,414 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +############################################################## +# genetic_algorithms.py, general purpose genetic +# algorithm in Python +# +# Written by Jared Smith and David Cunningham for COSC 427/527 +# at the University of Tennessee, Knoxville. +############################################################### +# TODO: +# - Generalize ff call to fitness function using kwargs argument +# +############################################################### + +import random + +import bitstring as bs +from scipy import stats +import numpy as np + +import ffs + + +class BaseGeneticAlgorithm(object): + + """ + + Basic class for executing a genetic algorithm. + + Parameters: + l is the size of the bit strings (each bit is a gene) + N is the size of the population + G is the number of generations + pr_mutation is the probability of mutation among the genes + pr_crossover is the probability of crossover among the genes + population is a list of bit strings (gene strings) of size N + current_offspring is the current generation of children + nruns is the number of runs to run the algorithm + learn is a bool specifying whether to learn the offspring + NG is the number of guesses to use when learning the offspring + ff is the fitness function to use + ce is a bool specifying whether to inflict a sudden change of + environment on the final population + rs is the seed for the random number generator; if left blank it will default to None. + + """ + + def __init__(self, args, logger): + # Parameters of the algorithm + self.args = args + self.l = args.l + self.N = args.N + self.G = args.G + self.pr_mutation = args.pm + self.pr_crossover = args.pc + random.seed(args.rs); #seed the RNG + self.population = [] + self.current_offspring = [] + self.nruns = args.nruns + self.NG = args.NG + self.learn = args.learn + self.ff = args.ff + self.ce = args.ce + self.max_recovery = 100 + + # Helper objects + self.logger = logger + self.orig_fitness_vals = np.zeros((self.G + self.max_recovery, self.N)) + self.norm_fitness_vals = np.zeros((self.G + self.max_recovery, self.N)) + self.total_fitness_vals = np.zeros((self.G + self.max_recovery, self.N)) + self.parents = [[None, None] for x in xrange(self.G + self.max_recovery)] + self.pr_mut_dist = None + self.pr_cr_dist = None + self.env_state = 0 + + # Statistics Objects + self.avg_fitness_vals = np.zeros((self.nruns, 2, self.G + self.max_recovery)) + self.best_fitness_vals = np.zeros((self.nruns, 2, self.G + self.max_recovery)) + self.num_correct_bits = np.zeros((self.nruns, 2, self.G + self.max_recovery)) + self.recovery_time = 0 + + + def initialize_algorithm(self): + # Initialize the population + self.logger.info("Initializing population...") + self.initialize_pop() + self.logger.info("Initialization Complete.") + + # Generate probablility distributions for mutation and crossover + self.logger.info("Generating probability distributions for mutation and " + + "crossover...") + self.generate_prob_distributions() + self.logger.info("Generated probability distributions for mutation and " + + "crossover.") + + # Initialize the Population + def initialize_pop(self): + # Generate N random bitstrings + self.population = [] + self.current_offspring = [] + for i in range(self.N): + tmp_bitstring = ''.join(random.choice('01') for _ in range(self.l)) + tmp_bitstring = bs.BitArray(bin=tmp_bitstring) + self.population.append(tmp_bitstring) + + # Initialize the genetic environment + def initialize_env(self): + # Get the appropriate fitness function + self.logger.info("Initializing environment...") + if self.env_state != 0: + self.recovery_time = 0 + self.logger.info("Initialized environment.") + + # Generate probability distributions for mutation and crossover + def generate_prob_distributions(self): + # xk is an array of size 2 (0, 1), that represents the possible + # values we can get out of the distribution + xk = np.arange(2) + + # pk1 and pk2 are the probabilities for getting the corresponding + # xk value for mutation and crossover, respectively + pk1 = (1 - self.pr_mutation, self.pr_mutation) + pk2 = (1 - self.pr_crossover, self.pr_crossover) + + # Generate the object that will be used to get random numbers + # according to each distribution. + self.pr_mut_dist = stats.rv_discrete(name='pr_mut_dist', + values=(xk, pk1)) + self.pr_cr_dist = stats.rv_discrete(name='pr_cr_dist', + values=(xk, pk2)) + + # Calculate the fitness of each individual of the population + def fitness(self, g, nrun, ff=ffs.fitness_func_1): + total_fitness = 0 + + self.logger.debug("Getting fitness of generation %d" % g) + + # Step through each bitstring in the population + for i, bitstring in enumerate(self.population): + # Get the integer value of the string + bit_sum = bitstring.uint + self.logger.info("Sum of bitstring at %d of population: %d" % (i, bit_sum)) + fitness_val = self.ff(bit_sum, self.l) + self.orig_fitness_vals[g][i] = fitness_val + self.logger.debug("Fitness Value at index %d in population: %lf" % (i, fitness_val)) + total_fitness += fitness_val + + self.logger.debug("Total fitness from step 1: %lf" % total_fitness) + if total_fitness > 0: + self.norm_fitness_vals[g] = self.orig_fitness_vals[g] / total_fitness + self.logger.debug("Sum of norm fitness vals: %lf" % np.sum(self.norm_fitness_vals[g])) + + prev_norm_fitness_val = 0 + for i in range(self.N): + self.logger.debug("Normalized Fitness Value at index %d in population: %lf" % (i, self.norm_fitness_vals[g][i])) + self.total_fitness_vals[g][i] = ( + self.norm_fitness_vals[g][i] + prev_norm_fitness_val) + prev_norm_fitness_val = self.total_fitness_vals[g][i] + self.logger.debug("Total Fitness Value at index %d in population: %lf" % (i, self.total_fitness_vals[g][i])) + + # Select parents from population + def select(self, g): + rand_nums = np.random.uniform(0, 1, 2) + + # Select the first parent + self.logger.info("Selecting the first parent...") + prev_individual_fit = 0 + j = 0 + while True: + if j >= self.N: + j = 0 + rand_nums = np.random.uniform(0, 1, 2) + + individual_fit = self.total_fitness_vals[g][j] + if rand_nums[0] < self.total_fitness_vals[g][0]: + self.logger.debug("1: Prev_Individual Fit: %lf" % prev_individual_fit) + self.logger.debug("1: Individual Fit: %lf" % individual_fit) + self.logger.debug("1: Rand Num: %lf" % rand_nums[0]) + self.logger.debug("1: Population at %d: %s" % (j, self.population[j].bin)) + self.parents[g][0] = self.population[0] + break + + if j != 0: + self.logger.debug("1: Prev_Individual Fit: %lf" % prev_individual_fit) + self.logger.debug("1: Individual Fit: %lf" % individual_fit) + self.logger.debug("1: Rand Num: %lf" % rand_nums[0]) + self.logger.debug("1: Population at %d: %s" % (j, self.population[j].bin)) + + if (prev_individual_fit <= rand_nums[0] <= individual_fit): + self.logger.debug("1: selected individuval from population at %d: %s" % (j, self.population[j].bin)) + self.parents[g][0] = self.population[j] + self.logger.debug("1: parents[%d][0]: %s" % (g, str(self.parents[g][0]))) + break + prev_individual_fit = individual_fit + j += 1 + self.logger.info("First parent has been selected.") + + # Select the second parent + self.logger.info("Selecting the second parent...") + prev_individual_fit = 0 + j = 0 + cycles = 0 + while True: + if j >= self.N: + cycles += j + if cycles >= 100: + self.parents[g][1] = self.parents[g][0] + break + else: + j = 0 + rand_nums = np.random.uniform(0, 1, 2) + + individual_fit = self.total_fitness_vals[g][j] + if rand_nums[1] < self.total_fitness_vals[g][0]: + self.logger.debug("2: prev_individual fit: %lf" % prev_individual_fit) + self.logger.debug("2: individual fit: %lf" % individual_fit) + self.logger.debug("2: rand num: %lf" % rand_nums[1]) + self.logger.debug("2: population at %d: %s" % (j, self.population[j].bin)) + self.parents[g][1] = self.population[0] + break + + if j != 0: + self.logger.debug("2: prev_individual fit: %lf" % prev_individual_fit) + self.logger.debug("2: individual fit: %lf" % individual_fit) + self.logger.debug("2: rand num: %lf" % rand_nums[1]) + self.logger.debug("2: population at %d: %s" % (j, self.population[j].bin)) + + if (prev_individual_fit <= rand_nums[1] <= individual_fit): + if (self.population[j] != self.parents[g][0]): + self.logger.debug("2: selected individuval from population at %d: %s" % (j, self.population[j].bin)) + self.parents[g][1] = self.population[j] + self.logger.debug("1: parents[%d][0]: %s" % (g, str(self.parents[g][1]))) + break + + prev_individual_fit = individual_fit + j += 1 + + self.logger.info("Second parent has been selected.") + + # Mutate the parents + def mutate(self, g): + + for parent in self.parents[g]: + for index_bit in xrange(0, self.l): + # Determine whether we will perform a mutation on the bit + to_mutate = self.pr_mut_dist.rvs(size=1) + + # Mutate the bit if choice is 1, otherwise skip it + if to_mutate: + parent.invert(index_bit) + + + # Crossover the parents + def crossover(self, g): + to_crossover = self.pr_cr_dist.rvs(size=1) + + # Crossover the parents if to_crossover is 1, otherwise copy the + # parents exactly into the children + if to_crossover: + # Create empty children + c1 = bs.BitArray(length=self.l) + c2 = bs.BitArray(length=self.l) + + # Select the bit at which to crossover the parents + crossover_bit = random.randint(0, self.l) + + # Perform the crossover + c1.overwrite(self.parents[g][0][:crossover_bit], 0) + c1.overwrite(self.parents[g][1][:(self.l - crossover_bit)], crossover_bit) + c2.overwrite(self.parents[g][1][:crossover_bit], 0) + c1.overwrite(self.parents[g][0][:(self.l - crossover_bit)], crossover_bit) + + self.current_offspring.append(c1) + self.current_offspring.append(c2) + else: + self.current_offspring.append(self.parents[g][0]) + self.current_offspring.append(self.parents[g][1]) + + # Learn the children on the fitness function. + def learn_offspring(self, g, ff=ffs.fitness_func_1): + # For every child in the current generation, iterate for NG guesses, + # manipulating the child every time trying to find a best fitness, + # and when the children are exhausted, the children will have been + # fine tuned according to the original fitness function. + for child in self.current_offspring: + for guess in xrange(0, self.NG): + current_child = child.copy() + max_fitness = 0 + + for ibit in xrange(0, self.l): + if random.choice([0, 1]): + if current_child[ibit]: + current_child.set(False, ibit) + elif not current_child[ibit]: + current_child.set(True, ibit) + + bit_sum = current_child.uint + current_fitness = ff(bit_sum, self.l) + max_fitness = max(current_fitness, max_fitness) + + if current_fitness == max_fitness: + child = current_child + + def compute_statistics(self, g, nrun): + # Get the number of correct bits in the best individual + index_bi = self.orig_fitness_vals[g].argmax() + bi_bitstring = self.population[index_bi] + individual_num_correct_bits = bi_bitstring.count(1) + self.num_correct_bits[nrun][self.env_state][g] = individual_num_correct_bits + + # Get the numerical value of the best fitness + self.best_fitness_vals[nrun][self.env_state][g] = self.orig_fitness_vals[g][index_bi] + + # Get the average value of the fitness + self.avg_fitness_vals[nrun][self.env_state][g] = np.average(self.orig_fitness_vals[g]) + + # Logging computed statistics to stdout and to file + self.logger.info("Number of Correct Bits in Best Individual: %d" + % individual_num_correct_bits) + self.logger.info("Fitness Value of Best Individual: %lf" + % self.best_fitness_vals[nrun][self.env_state][g]) + self.logger.info("Average Fitness Value of Generation: %lf" + % self.avg_fitness_vals[nrun][self.env_state][g]) + + # Check if the population has recovered from an environment change + def check_population_recovery(self, g, nrun): + checks = [] + checks.append((self.best_fitness_vals[nrun][1][g] > self.best_fitness_vals[nrun][0][self.G - 1])) + checks.append((self.avg_fitness_vals[nrun][1][g] > self.avg_fitness_vals[nrun][0][self.G - 1])) + for check in checks: + print check + if all(checks): + return True + + # Run one generation + def reproduce(self, nrun, g): + self.logger.info("Running fitness function on generation " + + "%d..." % g) + self.fitness(g, nrun, self.ff) + + # Select the parents of the next generation and generate the + # new offspring. + for i in range(self.N / 2): + self.logger.info("Selecting the parents of generation %d..." + % g) + self.select(g) + self.logger.info("Selection of the parents of generation " + + "%d finished." % g) + + self.logger.info("Performing crossover and mutation of " + + "the parent's offspring from generation " + + "%d..." % g) + self.crossover(g) + self.mutate(g) + self.logger.info("Crossover and mutation of the " + + "the parent's offspring of " + + " generation %d finished.", g) + + # Learn the offspring if specified + if self.learn: + self.logger.info("Performing learning on the offspring" + + " of generation %d..." % g) + self.learn_offspring(g, self.ff) + self.logger.info("Learning on the offspring" + + " of generation %d finished." % g) + + # Compute statistics for this generation + self.logger.info("Computing statistics for Run %d, Generation %d..." % (nrun, g)) + self.compute_statistics(g, nrun) + self.logger.info("Computation of statistics finished for Run %d, Generation %d." % (nrun, g)) + + # Replace the old population with the new population + self.population = self.current_offspring + self.current_offspring = [] + + # Run through the total runs specified + def run(self): + for nrun in range(self.nruns): + self.logger.info("Starting run %d..." % nrun) + self.initialize_algorithm() + + self.env_state = 0 + self.ff = ffs.fitness_func_1 + self.logger.info("Fitness Function before normal generation run: %s" % str(self.ff.__name__)) + + for g in range(self.G): + self.logger.info("Generation %d running..." % g) + self.reproduce(nrun, g) + self.logger.info("Generation %d finished." % g) + + if self.ce: + self.logger.info("Running Sudden change in environment test starting at generation %d..." % g) + self.env_state = 1 + self.initialize_env() + self.ff = ffs.fitness_func_2 + self.logger.info("Fitness Function before changed environment generation run: %s" % str(self.ff.__name__)) + + while True: + g += 1 + self.recovery_time += 1 + self.logger.info("Recovery Time: %d, Max Recovery: %d, Generation %d In Changed Environment" % (self.recovery_time, self.max_recovery, g)) + if self.recovery_time >= self.max_recovery: + self.logger.info("Population has not recovered after %d iterations. Quitting now." % self.recovery_time) + break + self.reproduce(nrun, g) + if self.check_population_recovery(g, nrun): + self.logger.info("Population has recovered after %d iterations." % self.recovery_time) + break + self.logger.info("Population has not recovered...continuing generation.") + + self.logger.info("Finished run %d." % nrun) + + return (self.avg_fitness_vals, self.best_fitness_vals, + self.num_correct_bits) diff --git a/genetics/basic/geneticRun.py b/genetics/basic/geneticRun.py new file mode 100644 index 0000000..b770ba3 --- /dev/null +++ b/genetics/basic/geneticRun.py @@ -0,0 +1,225 @@ +############################################################## +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# genetic.py, general purpose genetic algorithm in Python +# +# Written by Jared Smith and David Cunningham for COSC 427/527 +# at the University of Tennessee, Knoxville +# +############################################################### + +import os +import argparse +import logging +import contextlib +from argparse import RawTextHelpFormatter + +import numpy as np +import matplotlib.pyplot as plt + +from geneticAlgorithms import BaseGeneticAlgorithm +import ffs + +# Function for changing directories safely +@contextlib.contextmanager +def cd(newPath): + savedPath = os.getcwd() + os.chdir(newPath) + yield + os.chdir(savedPath) + + +# Setup the command line parser +def setup_argparser(): + + parser = argparse.ArgumentParser(description='' + + ' PyNet: General Purpose Genetic ' + + ' Algorithm in Python\n' + + ' Written by: Jared Smith and ' + + ' David Cunningham', + version='1.0.0', + formatter_class=RawTextHelpFormatter) + + requiredArguments = parser.add_argument_group('required Arguments') + requiredArguments.add_argument('-exp', dest='experiment_number', required=True, type=str, help="Number of this experiment.") + optionalArguments = parser.add_argument_group('optional Arguments') + optionalArguments.add_argument('--num_bits', dest='l', required=False, type=int, default=20, help="Number of bits (genes) in the genetic string. Default is 20.") + optionalArguments.add_argument('--population_size', dest='N', required=False, type=int, default=30, help="Size of the population. Default is 30.") + optionalArguments.add_argument('--num_gens', dest='G', required=False, type=int, default=10, help="Number of generations. Default is 10.") + optionalArguments.add_argument('--pr_mutation', dest='pm', required=False, type=float, default=0.033, help="Probability of Mutation. Default is 0.033.") + optionalArguments.add_argument('--pr_crossover', dest='pc', required=False, type=float, default=0.6, help="Probability of Crossover. Default is 0.6.") + optionalArguments.add_argument('--learn_offspring', dest='learn', required=False, type=bool, default=False, help="Specify whether to enforce learning on the offspring of each generation. Default is False.") + optionalArguments.add_argument('--change_environment', dest='ce', required=False, type=bool, default=False, help="Specify whether to inflict a sudden change of environment on the final population. Default is False.") + optionalArguments.add_argument('--num_learning_guesses', dest='NG', required=False, type=int, default=20, help="Specify the number of guesses to take when learning with the offspring. Default is 20.") + optionalArguments.add_argument('--fitness_func', dest='ff', required=False, type=callable, default=ffs.fitness_func_1, help="Specify the fitness function to use. Default is fitness_func_1 from ffs.py.") + optionalArguments.add_argument('--plot', dest='plot', required=False, type=bool, default=True, help="Specify if data is to be plotted. Default is True.") + optionalArguments.add_argument('--autoscale', dest='autoscale', required=False, type=bool, default=True, help="Specify plots should be autoscaled to data frame. Default is True.") + optionalArguments.add_argument('--nruns', dest='nruns', required=False, type=int, default=10, help="Specify the number of runs to do of the algorithm. Default is 10.") + optionalArguments.add_argument('--seed', dest='rs', required=False, type=int, default=None, help="seed for RNG. Default is none") + return parser + +def setup_logger(log_path, logger_name, logfile_name): + + logFormatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + rootLogger = logging.getLogger(logger_name) + rootLogger.setLevel(logging.INFO) + + fileHandler = logging.FileHandler("{0}/{1}.log".format(log_path, logfile_name), mode='w') + fileHandler.setFormatter(logFormatter) + rootLogger.addHandler(fileHandler) + + consoleHandler = logging.StreamHandler() + consoleHandler.setFormatter(logFormatter) + rootLogger.addHandler(consoleHandler) + + return rootLogger + + +def plot_results(G, nruns, avg_fitness_vals, best_fitness_vals, num_correct_bits, + autoscale, plot_file_path, experiment_number, ce): + x = np.arange(0, G) + + # Plot average fitness values + fig = plt.figure() + subplt = plt.subplot(111) + + for nrun in range(nruns): + plt.plot(x, avg_fitness_vals[nrun][0][:G]) + plt.xlabel('Generations') + plt.ylabel('Average Fitness Value') + plt.title('Average Fitness Values Over %d Runs with %d Generations' % + (avg_fitness_vals.shape[0], G)) + plt.grid() + if autoscale: + subplt.autoscale_view(True, True, True) + fig.tight_layout() + plot_file_name = "%s/avg-fit-vals-exp-%s.pdf" % (plot_file_path, experiment_number) + plt.savefig(plot_file_name, bbox_inches='tight') + + # Plot the best fitness values + fig = plt.figure() + subplt = plt.subplot(111) + + for nrun in range(nruns): + plt.plot(x, best_fitness_vals[nrun][0][:G]) + plt.xlabel('Generations') + plt.ylabel('Best Fitness Value') + plt.title('Best Fitness Values Over %d Runs with %d Generations' % + (best_fitness_vals.shape[0], G)) + plt.grid() + if autoscale: + subplt.autoscale_view(True, True, True) + fig.tight_layout() + plot_file_name = "%s/best-fit-vals-exp-%s.pdf" % (plot_file_path, experiment_number) + plt.savefig(plot_file_name, bbox_inches='tight') + + # Plot the number of correct bits for the best individual + fig = plt.figure() + subplt = plt.subplot(111) + + for nrun in range(nruns): + plt.plot(x, num_correct_bits[nrun][0][:G]) + plt.xlabel('Generations') + plt.ylabel('Number of Correct Bits') + plt.title('Number of Correct Bits Over %d Runs with %d Generations' % + (num_correct_bits.shape[0], G)) + plt.grid() + if autoscale: + subplt.autoscale_view(True, True, True) + fig.tight_layout() + plot_file_name = "%s/num-correct-bits-exp-%s.pdf" % (plot_file_path, experiment_number) + plt.savefig(plot_file_name, bbox_inches='tight') + + if ce: + x = np.arange(0, G + 30) + + # Plot average fitness values + fig = plt.figure() + subplt = plt.subplot(111) + + for nrun in range(nruns): + plt.plot(x, avg_fitness_vals[nrun][1][:(G + 30)]) + plt.xlabel('Generations') + plt.ylabel('Average Fitness Value') + plt.title('Average Fitness Values Over %d Runs with %d Max Generations' % + (avg_fitness_vals.shape[0], avg_fitness_vals.shape[2])) + plt.grid() + if autoscale: + subplt.autoscale_view(True, True, True) + fig.tight_layout() + plot_file_name = "%s/ce-avg-fit-vals-exp-%s.pdf" % (plot_file_path, experiment_number) + plt.savefig(plot_file_name, bbox_inches='tight') + + # Plot the best fitness values + fig = plt.figure() + subplt = plt.subplot(111) + + for nrun in range(nruns): + plt.plot(x, best_fitness_vals[nrun][1][:(G + 30)]) + plt.xlabel('Generations') + plt.ylabel('Best Fitness Value') + plt.title('Best Fitness Values Over %d Runs with %d Max Generations' % + (best_fitness_vals.shape[0], best_fitness_vals.shape[2])) + plt.grid() + if autoscale: + subplt.autoscale_view(True, True, True) + fig.tight_layout() + plot_file_name = "%s/ce-best-fit-vals-exp-%s.pdf" % (plot_file_path, experiment_number) + plt.savefig(plot_file_name, bbox_inches='tight') + + # Plot the number of correct bits for the best individual + fig = plt.figure() + subplt = plt.subplot(111) + + for nrun in range(nruns): + plt.plot(x, num_correct_bits[nrun][1][:(G + 30)]) + plt.xlabel('Generations') + plt.ylabel('Number of Correct Bits') + plt.title('Number of Correct Bits Over %d Runs with %d Max Generations' % + (num_correct_bits.shape[0], num_correct_bits.shape[2])) + plt.grid() + if autoscale: + subplt.autoscale_view(True, True, True) + fig.tight_layout() + plot_file_name = "%s/ce-num-correct-bits-exp-%s.pdf" % (plot_file_path, experiment_number) + plt.savefig(plot_file_name, bbox_inches='tight') + + +def main(): + parser = setup_argparser() + args = parser.parse_args() + experiment_number = args.experiment_number + + # Setup directories for storing results + if not os.path.exists('results'): + os.makedirs('results') + + with cd('results'): + if not os.path.exists('data'): + os.makedirs('data') + with cd('data'): + if not os.path.exists('Experiment-' + str(experiment_number)): + os.makedirs('Experiment-' + str(experiment_number)) + + logger = setup_logger('results/data/Experiment-' + + str(experiment_number), "__main__", "main") + logger.info("###################################RUNNING EXPERIMENT NUM " + + "%s#########################", str(experiment_number)) + logger.info("Program Arguments:") + args_dict = vars(args) + for key, value in args_dict.iteritems(): + logger.info("%s=%s" % (str(key), str(value))) + + logger.info("Running Base Genetic Algorithm...") + gen_alg = BaseGeneticAlgorithm(args, logger) + avg_fitness_vals, best_fitness_vals, num_correct_bits = gen_alg.run() + logger.info("Finished Base Genetic Algorithm.") + + if args.plot: + plot_file_path = 'results/data/Experiment-%s' % (experiment_number) + plot_results(args.G, args.nruns, avg_fitness_vals, best_fitness_vals, num_correct_bits, + args.autoscale, plot_file_path, experiment_number, args.ce) + +if __name__ == "__main__": + main() diff --git a/neuralnetworks/backpropagation/.old/driver.py b/neuralnetworks/backpropagation/.old/driver.py new file mode 100644 index 0000000..bee6777 --- /dev/null +++ b/neuralnetworks/backpropagation/.old/driver.py @@ -0,0 +1,33 @@ +# run multiple.py +import os + + +def recursive_run(hidden_layers, learning_rate, exp, tests): + if len(hidden_layers) == 11: + return 0 + hlstr = "" + for i in hidden_layers: + hlstr += str(i) + " " + for test in tests: + if test == 'f': + os.system("python run_tests.py -exp " +str(exp) + " -ttype " + test + " -hidden_layers " + hlstr + " --learning_rate " + str(learning_rate) + ' --ftrain training1.txt --ftest testing1.txt --fvalid validation1.txt') + exp += 1 + os.system("python run_tests.py -exp " +str(exp) + " -ttype " + test + " -hidden_layers " + hlstr + " --learning_rate " + str(learning_rate) + ' --ftrain training2.txt --ftest testing2.txt --fvalid validation2.txt') + exp += 1 + else: + os.system("python run_tests.py -exp " +str(exp) + " -ttype " + test + " -hidden_layers " + hlstr + " --learning_rate " + str(learning_rate)) + learning_rate += .1 + + if learning_rate == 1: + learning_rate = .1 + if hidden_layers[len(hidden_layers)-1] == 100: + for i in hidden_layers: + i = 1 + hidden_layers.append(0) + hidden_layers[len(hidden_layers)-1] += 1 + if recursive_run(hidden_layers, learning_rate, exp, tests) == 0: + return 0 + else: + return 1 + +recursive_run([1], .1,0, ['x', 'i', 'd', 'f']) diff --git a/neuralnetworks/backpropagation/.old/face_graphs.py b/neuralnetworks/backpropagation/.old/face_graphs.py new file mode 100644 index 0000000..2f4c085 --- /dev/null +++ b/neuralnetworks/backpropagation/.old/face_graphs.py @@ -0,0 +1,94 @@ +from sklearn.datasets import load_iris, load_digits, fetch_lfw_people, fetch_lfw_pairs +from sklearn.decomposition import RandomizedPCA +from sklearn.cross_validation import train_test_split as sklearn_train_test_split +import numpy as np +import matplotlib.pyplot as plt + + +def plot_learning_rates_versus_epochs(num_image, autoscale, learning_rates, plot_file_path=None): + x1 = np.arange(len(learning_rates)) + y1 = learning_rates + + fig = plt.figure() + subplt = plt.subplot(111) + plt.plot(x1, y1) + plt.xlabel('Epochs') + plt.ylabel('Learning Rate') + plt.title('Learning Rate Per Epoch Over %s Epochs' % str(len(x1))) + plt.grid() + if autoscale: + subplt.autoscale_view(True,True,True) + fig.tight_layout() + + plt.show() + + if plot_file_path is None: + plot_file_name = "learning_rates-faces-%s.pdf" % (str(num_image)) + else: + plot_file_name = "%s/learning_rates-faces-%s.pdf" % (plot_file_path, str(num_image)) + + plt.savefig(plot_file_name, bbox_inches='tight') + + +def plot_gallery(num_image, images, titles, h, w, n_row=3, n_col=4, plot_file_path=None): + """Helper function to plot a gallery of portraits""" + plt.figure(figsize=(1.8 * n_col, 2.4 * n_row)) + plt.subplots_adjust(bottom=0, left=.01, right=.99, top=.90, hspace=.35) + for i in range(n_row * n_col): + plt.subplot(n_row, n_col, i + 1) + plt.imshow(images[i].reshape((h, w)), cmap=plt.cm.gray) + plt.title(titles[i], size=12) + plt.xticks(()) + plt.yticks(()) + + if plot_file_path is None: + plot_file_name = "gallery-image-%s.pdf" % (num_image) + else: + plot_file_name = "%s/gallery-image-%s.pdf" % (plot_file_path, num_image) + + plt.savefig(plot_file_name, bbox_inches='tight') + +def title(y_pred, y_test, target_names, i): + pred_name = target_names[y_pred[i]].rsplit(' ', 1)[-1] + true_name = target_names[y_test[i]].rsplit(' ', 1)[-1] + return 'predicted: %s\ntrue: %s' % (pred_name, true_name) + + +lfw_people = fetch_lfw_people(min_faces_per_person=70, resize=0.4) + +# introspect the images arrays to find the shapes (for plotting) +n_samples, h, w = lfw_people.images.shape + +# for machine learning we use the 2 data directly (as relative pixel +# positions info is ignored by this model) +X = lfw_people.data +n_features = X.shape[1] + +# the label to predict is the id of the person +y = lfw_people.target +# y = self.translate_to_binary_array(y) + +target_names = lfw_people.target_names +n_classes = target_names.shape[0] + +# split into a training and testing set +X_train, X_test, y_train, y_test = sklearn_train_test_split( + X, y, test_size=0.25) + +y_pred = None +y_test = None +with np.load('target-predicted-info-file-npz-exp-1.npz') as data: + y_pred = data['arr_1'] + y_test = data['arr_0'] + +learning_rates = None +with np.load('learning-rates-info-file-npz-exp-1.npz') as data: + learning_rates = data['arr_0'] + +plot_learning_rates_versus_epochs(1, False, learning_rates) + + +prediction_titles = [title(y_pred, y_test, target_names, i) + for i in range(y_pred.shape[0])] + +plot_gallery(1, X_test, prediction_titles, h, w) diff --git a/neuralnetworks/backpropagation/.old/testing1.txt b/neuralnetworks/backpropagation/.old/testing1.txt new file mode 100644 index 0000000..0a8ee2d --- /dev/null +++ b/neuralnetworks/backpropagation/.old/testing1.txt @@ -0,0 +1,100 @@ +-0.894011 1.105065 0.581007 +-1.439937 -1.574976 0.802565 +-0.345833 -0.373237 0.284707 +-0.209296 1.142366 0.535801 +-1.800757 -0.076113 0.347155 +-1.720433 0.716976 0.408574 +0.926664 -1.108691 0.415612 +-1.888890 1.897929 0.585711 +-1.466422 -1.498806 0.762339 +1.159332 1.648505 0.087558 +1.873561 -1.245506 0.462891 +-0.293485 -0.685443 0.394517 +0.689814 0.968724 0.521697 +0.238872 -1.240571 0.432393 +0.388073 -1.041329 0.481430 +-0.785712 -0.761762 0.327482 +-1.154504 -1.835098 0.969156 +0.829942 -1.723989 0.062355 +-1.810994 -1.190410 0.543101 +-1.276121 1.968922 0.953162 +1.099121 1.842483 0.021091 +-1.420573 -0.451002 0.200194 +0.484757 -1.761188 0.179003 +0.016850 0.477684 0.509679 +-1.308691 -1.134243 0.592590 +-0.710343 0.080045 0.054413 +-1.395020 0.925541 0.452529 +-1.188388 -0.244517 0.056578 +-1.083827 -0.055511 0.006211 +-0.478072 0.668368 0.330206 +-0.223811 -0.232511 0.339175 +0.784943 0.346916 0.903415 +-0.891745 -1.168328 0.628784 +0.400291 0.848522 0.569315 +-1.548245 1.539832 0.744308 +1.461891 -1.170512 0.401003 +1.710265 -0.565532 0.638619 +-1.876460 0.246080 0.410693 +-1.187173 1.162253 0.620648 +0.311704 -1.315819 0.388073 +1.779503 0.460370 0.627257 +-0.910216 -0.754686 0.313929 +1.949063 0.353569 0.533958 +1.047154 -1.246141 0.311980 +-1.377071 -0.794386 0.368336 +1.064461 -1.332495 0.251849 +1.586038 -1.622230 0.249054 +1.033156 -1.498690 0.147654 +-1.764242 -0.685863 0.414285 +1.382848 1.625841 0.156900 +-0.801693 1.405344 0.782971 +-1.218746 -1.504872 0.835424 +0.711118 -1.555809 0.155626 +-0.527347 1.491345 0.756939 +1.082711 -1.885725 0.012180 +-1.767399 1.178736 0.549500 +0.211979 0.764774 0.559021 +-0.374174 -0.202070 0.236604 +1.285752 0.033688 0.949841 +-1.892069 -0.583464 0.448656 +-0.385709 0.614842 0.338050 +-0.418979 1.396097 0.678235 +0.793065 0.107214 0.967113 +0.358678 1.579868 0.289035 +0.678150 0.662579 0.721154 +-1.796763 0.895180 0.474279 +0.395828 -0.892840 0.548792 +-1.033810 0.732985 0.296669 +-1.962138 0.018737 0.470292 +-1.576329 0.126668 0.197363 +-1.564678 1.740958 0.790083 +1.852317 -0.678020 0.555692 +1.764440 -1.884955 0.322128 +-1.879247 0.473723 0.430645 +1.968412 -0.848127 0.505860 +1.930345 -0.644890 0.528901 +1.071014 1.750938 0.040652 +-1.023543 -1.282872 0.714781 +-1.320171 -1.245010 0.664471 +0.416889 -0.821338 0.584343 +-0.429515 -0.386016 0.243353 +1.338923 -0.533699 0.788079 +-1.496542 -0.769259 0.373963 +1.775207 -0.648505 0.590690 +0.626824 -0.680093 0.700611 +-1.858980 -0.749748 0.457920 +-1.449604 -1.678734 0.832986 +0.066931 -0.702277 0.523654 +0.316452 -0.022448 0.738283 +-1.415290 -1.605559 0.823477 +-1.020365 -0.035074 0.001014 +-0.884481 -0.696151 0.274086 +0.573382 -0.192693 0.874056 +-1.853709 -0.417486 0.409735 +-1.807077 -1.790662 0.641219 +-0.608573 -1.649642 0.848122 +1.945042 -1.099247 0.493306 +0.249594 0.967685 0.509694 +-0.728726 -0.715864 0.303475 +-0.952055 -0.131154 0.011961 diff --git a/neuralnetworks/backpropagation/.old/testing2.txt b/neuralnetworks/backpropagation/.old/testing2.txt new file mode 100644 index 0000000..9d2bc7d --- /dev/null +++ b/neuralnetworks/backpropagation/.old/testing2.txt @@ -0,0 +1,100 @@ +0.368665 -1.973397 -1.165540 0.393618 +1.084376 1.366979 -0.535144 0.295940 +-0.163830 1.300969 -0.067870 0.133557 +-1.795341 -0.268578 0.280955 0.382016 +-0.114046 -1.131826 1.239461 0.188672 +-0.944945 -1.559992 -1.378464 0.399852 +0.008773 -1.788264 0.468387 0.258657 +-1.413550 -0.192703 -1.109531 0.304432 +0.441451 0.980665 -1.119144 0.168722 +0.234062 -0.731869 -1.499174 0.177189 +-0.679303 1.636796 -1.472570 0.384433 +0.155157 0.721172 1.894409 0.249830 +1.620013 -1.442658 1.195378 0.545355 +-0.447857 -1.237999 -1.073200 0.207487 +1.833097 0.647955 -0.205026 0.422442 +1.072558 1.703010 0.234983 0.359018 +1.694095 -0.288217 0.446719 0.349052 +0.162482 0.298233 -1.745984 0.185761 +1.052951 -1.260316 1.234682 0.338060 +1.933806 0.973746 -1.497187 0.633751 +-1.565367 -1.705557 -1.860391 0.706175 +-1.037938 0.449600 0.860781 0.182601 +-1.143528 0.069612 1.418123 0.267280 +-1.948150 1.621755 -1.819877 0.831307 +-1.021350 1.454852 0.828078 0.322740 +0.773624 0.527410 0.531088 0.106726 +-0.991393 0.221505 -1.757128 0.295306 +1.455326 -1.616014 0.541105 0.462158 +1.709342 1.436937 1.280789 0.590606 +0.944024 1.370743 0.254535 0.251100 +1.446836 1.805376 0.548978 0.509648 +1.586445 -1.232562 -1.001422 0.465120 +0.447226 -0.376090 1.068190 0.099787 +-0.134652 -0.324240 0.689944 0.037642 +0.045472 0.654409 0.144796 0.034391 +-1.126450 -0.571967 -1.327794 0.273289 +1.404639 0.436640 0.893711 0.288401 +1.647511 -0.108034 1.277697 0.408268 +0.188616 -0.398691 0.714634 0.045796 +-0.530595 -1.454668 0.085377 0.195679 +1.723940 1.992169 -0.109247 0.648895 +0.272918 1.578614 0.658192 0.225282 +1.271495 0.025840 -1.717898 0.356854 +0.339685 1.891188 -0.042139 0.288539 +-0.970371 -0.063340 -1.387730 0.220061 +1.174425 0.810210 0.040304 0.209736 +1.846631 0.214849 -1.523056 0.530846 +0.740342 -0.137640 0.368911 0.072552 +0.018039 -1.949025 1.970219 0.516193 +-1.267327 -0.479620 -1.484448 0.330146 +0.818050 -0.755681 -1.492280 0.249618 +-1.291197 1.517237 -1.913666 0.580721 +1.366995 0.788732 0.112174 0.264196 +1.649096 -0.871583 0.003362 0.372227 +-0.393042 0.158046 1.940022 0.236882 +0.219228 -0.667529 0.750232 0.072294 +-1.740468 -0.820898 -1.034919 0.463155 +-1.263524 1.919444 0.827441 0.507115 +1.105387 -0.062517 0.878416 0.185803 +1.075606 0.670155 -1.601204 0.315953 +1.591158 -0.511794 -0.356885 0.319626 +-1.901122 0.197009 -0.839648 0.460690 +-1.814787 -0.435996 1.949084 0.613805 +0.297387 -0.786900 -0.922499 0.106933 +-1.699252 0.820058 1.235547 0.472970 +-1.759230 -0.960714 -1.431983 0.546403 +0.991002 -0.701182 -0.252881 0.154827 +1.956083 0.035294 -0.333437 0.448002 +0.783524 -0.859319 1.604046 0.276078 +-0.338060 -1.783713 0.274201 0.262265 +0.060736 1.807445 1.762407 0.430919 +1.703851 1.906324 -0.040584 0.614613 +-1.135796 -1.908463 1.523419 0.562914 +-1.186712 0.388923 -1.263480 0.266229 +-0.109211 0.689671 1.556578 0.177749 +-0.873664 0.930442 -1.404136 0.268412 +-0.305647 -0.078556 -0.105318 0.011894 +1.441473 -0.122473 1.929976 0.455798 +-0.891964 -1.338948 -0.929343 0.279534 +-1.287918 0.322992 -0.713055 0.228751 +0.986283 -1.616272 -0.905610 0.360505 +0.748689 -1.912421 -0.999286 0.403622 +-1.291895 -1.048217 -0.907749 0.324635 +-1.768475 -0.234929 1.481174 0.491681 +-1.031955 1.655860 0.170845 0.335474 +-1.475378 -1.217804 -0.898713 0.411840 +-0.879514 0.476549 1.022731 0.167069 +1.015168 -0.081978 -1.099742 0.189203 +0.945144 1.026058 -0.438690 0.195160 +-1.984199 1.738139 1.884301 0.891511 +-0.697254 0.724422 -1.731971 0.269525 +0.397136 -0.526888 -1.644391 0.195554 +1.397851 0.181217 -0.692608 0.255661 +-1.509899 0.412741 1.072464 0.342514 +1.971275 1.380786 0.728324 0.625638 +0.142120 1.905408 1.510520 0.413241 +1.243406 -0.974106 -0.012931 0.251392 +0.266137 -1.958938 1.905091 0.512747 +1.166395 0.986206 0.931149 0.281815 +-1.272296 1.002007 0.669288 0.289852 diff --git a/neuralnetworks/backpropagation/.old/training1.txt b/neuralnetworks/backpropagation/.old/training1.txt new file mode 100644 index 0000000..2276e00 --- /dev/null +++ b/neuralnetworks/backpropagation/.old/training1.txt @@ -0,0 +1,200 @@ +-1.703369 -0.762666 0.418181 +0.227541 -1.902611 0.327113 +0.979284 1.152748 0.381244 +-0.256226 1.903823 0.693619 +1.819377 -1.168538 0.463377 +1.501609 1.501653 0.249986 +-0.760456 -0.836796 0.382088 +-1.056793 1.793893 0.972139 +0.555520 -1.136111 0.418736 +-0.567966 0.326900 0.160981 +-1.633415 0.623531 0.348214 +-1.678810 -1.148927 0.556027 +1.722125 1.830357 0.296081 +-1.661324 -0.425869 0.301038 +0.188628 -0.606491 0.584602 +1.002019 -1.104883 0.417999 +0.219282 0.134661 0.665075 +-1.754065 1.077868 0.522985 +-1.595569 -0.366611 0.251151 +-0.095564 1.065423 0.507671 +0.889688 1.432008 0.190850 +-0.887727 1.753198 0.955715 +1.432248 1.475323 0.235728 +-1.095031 1.813999 0.973486 +0.281514 0.002626 0.713963 +-1.510479 -0.995354 0.497462 +0.199181 1.223927 0.446977 +-0.927955 1.469862 0.834274 +1.738802 1.874293 0.304437 +1.269448 -0.221270 0.928620 +-0.873006 -1.331582 0.743873 +-0.391929 -0.219309 0.228224 +0.940221 -0.787061 0.663418 +-1.897273 0.117908 0.421040 +0.839365 -1.600578 0.108043 +0.969511 -1.111057 0.413318 +1.994774 1.088124 0.499433 +1.154698 -1.839831 0.029969 +-0.607509 1.898971 0.902820 +-0.736327 1.168419 0.619683 +-1.799723 -1.704587 0.638356 +0.568890 -0.096515 0.885200 +-1.309300 -1.156294 0.607460 +0.287834 -1.053567 0.481642 +1.698967 1.785798 0.285050 +1.112726 0.755309 0.684552 +1.531131 0.750082 0.628484 +1.173463 -0.095219 0.976178 +0.285658 1.297272 0.402358 +-0.727854 -1.439055 0.789510 +0.037363 -1.238778 0.489256 +1.470999 1.330112 0.316977 +0.685325 -1.979188 0.060083 +1.112590 0.308646 0.935479 +-0.342237 0.007613 0.243987 +-1.397642 -0.879661 0.423786 +-1.564104 -1.348530 0.664602 +1.357557 1.824933 0.092712 +0.439388 0.110591 0.813550 +-0.579812 1.382737 0.723417 +1.569408 -0.579900 0.691870 +0.014041 -1.108902 0.498123 +0.362159 1.576424 0.288108 +1.553982 0.689014 0.651274 +-0.307761 -1.653223 0.698773 +1.616294 -1.050864 0.477378 +-1.704343 -0.614968 0.372660 +1.725103 -1.257411 0.417676 +-0.940671 1.181976 0.640373 +-0.309173 -1.397835 0.636540 +0.883270 -1.828427 0.026128 +-1.163065 0.185614 0.036726 +0.120711 -1.452227 0.438546 +1.425300 -1.898245 0.112490 +1.038534 -0.206006 0.973181 +-1.340618 -0.589712 0.241589 +1.400568 -0.294055 0.861884 +0.793225 -0.568953 0.796884 +0.010816 0.490376 0.506096 +1.338997 -1.818797 0.086561 +-1.950086 1.064473 0.503959 +0.180932 1.901408 0.361480 +-0.796952 0.022119 0.025504 +1.801587 -0.552582 0.599102 +-0.910859 -1.514048 0.857732 +0.723472 -0.854666 0.602649 +-0.693062 -1.454097 0.789869 +-0.724563 1.339128 0.730532 +-0.013476 -0.650056 0.494471 +-0.120413 -1.311059 0.544128 +0.374816 -1.261145 0.389268 +1.041617 0.919787 0.562699 +-0.105967 -1.877165 0.581305 +-0.705778 1.924422 0.944395 +-1.203834 -0.986437 0.489889 +0.092105 1.737035 0.433976 +-1.201668 -0.956027 0.467208 +-0.789497 0.319410 0.085371 +1.053722 -1.694067 0.058210 +1.387317 0.185520 0.892985 +1.902115 -1.439664 0.451219 +1.481386 1.601953 0.205075 +0.962066 -0.504014 0.850694 +-1.535136 0.790208 0.392072 +1.415080 1.586374 0.183542 +0.851249 -0.321521 0.925686 +1.432752 0.476811 0.784797 +-1.439842 1.687315 0.839783 +-0.973308 0.741036 0.302347 +0.843171 0.128353 0.975083 +-0.651411 0.030469 0.073591 +-1.407137 -0.488145 0.211083 +0.756923 -1.526079 0.158747 +-0.956181 -1.061216 0.547890 +1.255136 -1.646136 0.108928 +0.480016 1.205114 0.391618 +-0.558070 0.637866 0.293001 +-0.552182 1.198024 0.616706 +1.773115 -1.775283 0.336308 +0.404345 1.067887 0.468427 +1.221096 -1.583524 0.126921 +-1.401818 -0.990661 0.494078 +-0.422779 1.766263 0.787636 +-1.947755 -1.189919 0.512047 +-0.682522 -1.934783 0.936802 +0.169084 0.545232 0.585980 +-1.300830 1.987163 0.945117 +-0.293933 -0.565019 0.359378 +0.400279 -0.791904 0.594422 +-0.164473 1.612441 0.604795 +1.399129 0.833537 0.604674 +0.993199 1.431718 0.186345 +-1.900876 -0.991061 0.498911 +-0.321030 -0.938816 0.476817 +-1.802400 0.378662 0.373512 +1.070094 -1.452254 0.175907 +-0.852298 -0.753084 0.315964 +-0.136824 0.952984 0.492132 +-0.223326 -0.646737 0.409472 +-0.085112 1.188790 0.519476 +-0.879010 0.587919 0.203913 +-1.256432 -0.418882 0.136047 +-0.624514 -0.319758 0.135799 +-0.226334 1.359212 0.593074 +0.981927 1.556812 0.116500 +-1.744594 0.626906 0.392007 +-1.925884 1.774608 0.554478 +1.879989 -0.362217 0.578938 +-0.879157 1.414457 0.797559 +1.569793 -0.670655 0.654675 +-0.368817 0.450336 0.291944 +0.526109 1.193904 0.389713 +0.852754 -1.430610 0.195340 +-1.811662 0.343056 0.374884 +0.585106 -0.675017 0.694229 +1.360536 -0.419611 0.833555 +-1.316288 -0.345495 0.123605 +1.253804 -0.465505 0.842995 +-1.223120 0.655338 0.257997 +1.653219 0.225130 0.743059 +-0.281652 1.856313 0.708633 +0.371397 0.382422 0.727210 +-1.394436 -0.764823 0.353022 +-1.480027 -0.576485 0.275027 +1.793594 -1.991378 0.340728 +0.624027 1.369158 0.272445 +-1.918633 -1.947130 0.563513 +-0.686686 1.306675 0.704160 +-1.816956 -1.916445 0.640571 +0.467231 1.736774 0.193327 +0.080764 -0.544878 0.541471 +-1.855368 1.826519 0.608465 +-0.597595 -1.567918 0.813995 +1.229567 -1.047945 0.464800 +1.399785 -1.254351 0.342613 +-1.105013 1.369677 0.770575 +1.970577 1.451043 0.484968 +-1.299327 -1.235643 0.661248 +0.965263 -1.052599 0.458798 +1.885191 1.414632 0.445631 +1.030249 -0.504603 0.850592 +0.031620 -0.359971 0.520960 +1.415414 1.042434 0.473540 +0.034385 0.272001 0.524566 +0.076871 -0.328214 0.552399 +0.127979 0.566773 0.562818 +-0.451755 0.537350 0.283548 +-0.978619 1.238022 0.682513 +1.801192 0.203285 0.645854 +0.664558 0.088476 0.928015 +-0.735303 -0.881275 0.415192 +-0.680041 1.150345 0.602519 +-0.798260 0.565759 0.200481 +-0.484397 -1.399857 0.702595 +0.781521 0.677014 0.728764 +-1.206911 -1.195007 0.642880 +0.174333 0.353237 0.614930 +1.006620 1.374618 0.222480 +-1.396735 1.175810 0.610704 +1.147311 -0.159632 0.971454 diff --git a/neuralnetworks/backpropagation/.old/training2.txt b/neuralnetworks/backpropagation/.old/training2.txt new file mode 100644 index 0000000..f97f7fe --- /dev/null +++ b/neuralnetworks/backpropagation/.old/training2.txt @@ -0,0 +1,200 @@ +-1.525921 1.356134 1.617901 0.561150 +-0.692101 0.533831 1.538976 0.213832 +1.047019 1.055351 -0.158266 0.213610 +0.957779 -0.277486 0.716446 0.141383 +0.404400 0.629897 0.431533 0.060134 +1.333057 -1.955025 0.755746 0.532004 +0.025582 -1.967304 1.270728 0.390948 +1.435424 -1.909411 -0.949487 0.570204 +0.058465 1.553375 0.722078 0.216088 +-1.894157 -0.921768 -1.789341 0.664054 +0.266558 -0.447689 1.566794 0.165241 +-0.115542 0.860210 0.100625 0.059045 +-0.576566 -0.092771 -0.844025 0.080118 +1.265168 -1.134991 0.878489 0.328307 +-0.018386 1.269409 -0.491614 0.137936 +-1.586853 0.602466 -0.446639 0.329980 +1.168893 -1.371952 -0.413943 0.312325 +0.439620 -1.936528 -0.323355 0.316805 +1.490133 0.121936 -0.769979 0.291559 +0.212211 0.227780 0.308252 0.014669 +0.422870 -1.505663 1.860563 0.394732 +-0.010336 0.378796 0.720773 0.041022 +-1.909712 1.802230 -1.371997 0.779255 +-0.753736 1.067398 -0.506989 0.168023 +-1.875247 -0.950987 -1.237580 0.563685 +-0.366861 -0.537841 1.364886 0.145257 +1.186499 -1.368948 1.992934 0.535733 +-1.227444 1.070673 -1.943595 0.479957 +0.449201 0.560806 0.178342 0.049310 +1.679221 -1.226983 -1.593878 0.587731 +-0.012526 1.195887 -1.099541 0.179779 +-0.151963 -0.814449 1.279255 0.148103 +-1.431190 -0.724161 1.081485 0.344160 +-0.803188 0.522103 0.148883 0.096683 +0.689823 0.646856 1.197896 0.169879 +1.452244 -1.720005 -1.339944 0.574502 +0.817129 1.466494 -0.708892 0.271465 +0.810063 -1.760951 -1.638219 0.469082 +0.866468 0.688250 0.922587 0.172170 +-0.955190 0.367472 1.695604 0.281532 +-0.549068 -1.645055 0.891491 0.288807 +0.351391 0.202982 -1.922958 0.230749 +-0.369354 0.771791 -0.647119 0.085721 +-1.287869 1.968603 1.874984 0.692306 +0.861015 0.658427 0.521840 0.134599 +0.058911 0.110670 0.801835 0.038435 +0.718966 -1.072200 0.268328 0.152229 +-1.989926 1.737863 0.507378 0.704073 +-1.628145 0.604331 -0.804372 0.371289 +1.294442 1.649142 1.563099 0.543500 +0.990046 -0.899926 1.918044 0.387640 +-0.118464 1.451465 0.121026 0.164522 +-0.041422 -0.917888 -1.107183 0.135729 +1.311459 -0.205757 -1.138579 0.276499 +1.186442 -1.344743 1.519848 0.434789 +-0.291717 0.714168 -0.369482 0.056929 +-1.489883 -0.566866 0.558318 0.298827 +0.778446 -0.556792 0.296181 0.098829 +-0.714176 -0.184937 -1.099488 0.131225 +0.481451 -0.890495 -1.450346 0.209100 +0.044551 -1.900449 -0.350272 0.285131 +-0.037405 -0.018912 -0.898807 0.046796 +-1.916378 1.939665 0.183305 0.715097 +-1.023561 1.251124 1.977547 0.466911 +-0.162140 0.437566 -1.367195 0.125601 +-0.642292 -1.854151 1.346973 0.416726 +0.988226 -1.344033 -1.219893 0.337494 +-0.453456 1.434412 0.223315 0.184875 +1.842725 -1.279764 -1.961622 0.739786 +-1.256763 1.201688 -0.852116 0.335216 +-0.707109 -0.753762 -0.752565 0.134071 +0.942619 1.208834 1.228523 0.302002 +-1.956188 1.292455 1.168188 0.648765 +0.227117 -1.731106 0.419312 0.246613 +0.204664 0.106754 -1.143121 0.081098 +0.837469 1.464462 -0.997272 0.303276 +0.184441 0.452688 -0.341306 0.026409 +0.964548 1.999232 -0.906893 0.462254 +-0.812137 1.841957 -0.186657 0.339099 +-0.773758 -1.414805 -0.984970 0.279027 +0.374126 -0.121914 0.261269 0.021232 +1.621561 -1.179295 -0.529898 0.426579 +0.850084 -1.135483 -1.237442 0.270903 +0.018272 1.091634 -0.968548 0.145825 +-1.562416 -0.703702 1.138206 0.394504 +-0.705537 -1.866233 0.602668 0.346301 +0.297191 0.318208 -0.944644 0.069462 +1.955885 -0.717244 -0.945411 0.532540 +-0.951008 0.470620 -1.103454 0.191640 +0.862334 1.696862 -0.518259 0.322786 +1.877365 0.070987 1.359826 0.513741 +0.138633 -0.307452 -1.819469 0.200477 +1.608736 -1.457368 -0.954952 0.514609 +-1.628707 0.560904 -1.863318 0.530585 +-0.597255 0.998488 -0.567020 0.136399 +-1.459048 -1.707049 -0.433253 0.480618 +1.143620 0.590142 1.884955 0.382681 +-1.801024 0.546026 -0.832289 0.437170 +-0.746435 1.595018 1.638331 0.414841 +0.150111 0.457352 1.335192 0.121540 +1.631851 0.334717 -0.593821 0.336224 +0.991677 -1.526650 1.098728 0.362400 +1.172209 -1.917914 1.641360 0.596927 +-1.782743 -1.546621 0.202264 0.553075 +-1.646061 -0.143875 -0.799248 0.351083 +-0.213081 0.397076 -0.506297 0.032156 +1.353665 -0.459304 -1.916155 0.439485 +1.238620 -0.260328 0.629872 0.205122 +-1.593669 0.993237 0.224890 0.371856 +-1.955339 -0.856652 -1.317758 0.597788 +1.379854 -1.224801 1.016959 0.394753 +-1.213967 1.766876 1.490309 0.538322 +1.884761 0.939085 1.572395 0.620360 +1.526121 1.156342 -1.974226 0.596452 +-0.271615 1.510280 -0.118101 0.184775 +0.929137 -0.702801 -1.721025 0.308486 +-1.577159 -1.349136 -0.180328 0.428900 +-1.493314 1.889484 1.559344 0.672215 +1.136557 -1.704185 0.552581 0.390069 +-0.638553 -1.659524 1.695929 0.424829 +0.043689 1.720330 -1.528872 0.362730 +-0.939353 -1.493637 -1.761996 0.452538 +-1.449044 -1.608876 1.177089 0.521325 +-1.876649 1.917245 0.333430 0.695533 +-1.850875 -0.354371 -0.156289 0.406347 +0.031024 -1.425233 1.140910 0.231461 +0.309999 -1.002393 1.791774 0.273599 +-1.870329 -0.495707 1.681258 0.585607 +1.689015 -1.359149 1.977073 0.696774 +0.241596 0.002297 -1.682451 0.170042 +-0.062475 -1.954014 -1.962122 0.516267 +0.408653 -0.893367 -1.455759 0.202925 +0.646656 -0.342410 -1.064635 0.122660 +-0.176255 -0.219059 -1.147391 0.083228 +-1.842825 -0.069934 0.498239 0.406544 +0.000886 1.961090 1.073005 0.362260 +-0.858204 0.271089 -1.929387 0.305397 +-1.066430 0.400760 -0.425094 0.154004 +-1.385172 0.089775 0.215757 0.224694 +-1.408099 -1.668628 -1.781946 0.626149 +-1.090551 0.268897 -1.735960 0.316648 +-1.052672 -1.322451 -0.629327 0.285238 +-0.508431 1.324206 1.028263 0.225713 +0.426933 -0.852049 -1.190796 0.158684 +1.279543 -0.694874 0.739270 0.257583 +-0.222219 1.306012 0.700359 0.165201 +-1.149213 -1.552191 -1.028551 0.398752 +-1.078601 -0.618621 1.372209 0.272306 +0.496305 -0.003793 -0.538016 0.045122 +-1.287938 0.588108 -0.206644 0.220467 +-1.069884 1.497558 -1.937747 0.521216 +-0.805844 -1.555115 -1.260198 0.352579 +0.564829 -0.063546 -1.935992 0.253357 +-0.406907 -1.636612 -0.788041 0.260971 +0.402296 1.642930 0.517085 0.241732 +-0.858434 -0.579289 -0.176903 0.112647 +1.841925 0.271498 0.270906 0.401368 +-1.186626 1.192897 1.652285 0.429435 +-1.814417 -0.310797 -0.351508 0.394417 +-0.352433 0.401265 -1.763399 0.206116 +1.440923 1.331381 1.734159 0.549419 +1.503176 -1.474463 -1.820956 0.619251 +-1.757021 1.090366 0.115498 0.448430 +-1.693013 -1.316541 0.478886 0.477286 +-0.481054 1.085755 0.121816 0.118240 +-1.963970 -1.772679 1.542527 0.824054 +-0.140873 -1.930753 -0.185975 0.291040 +-1.869967 -1.117379 -0.993078 0.556412 +1.782318 -0.931797 0.696125 0.461283 +-0.569190 0.715771 -0.902611 0.123794 +-0.332589 0.156694 -1.571230 0.157081 +-0.598431 -0.340129 -1.045694 0.113306 +-0.419387 -0.097151 -1.955327 0.241596 +1.696112 0.209836 -1.271869 0.428651 +0.174998 1.728782 1.813887 0.423251 +-1.703187 1.764812 -1.958792 0.795652 +1.839340 -0.376060 -1.889546 0.607228 +-0.346635 -0.246027 -1.006925 0.077014 +0.660287 -0.463709 0.061278 0.067062 +-0.643588 0.967101 -1.222951 0.206023 +0.453801 -1.365488 0.933743 0.217490 +0.882571 0.036082 -1.406386 0.204088 +1.836878 1.616695 0.496463 0.604596 +1.881551 1.312807 -1.293700 0.637620 +-1.390318 -0.512195 -1.564919 0.384504 +-1.576432 -0.215382 -1.800106 0.477260 +-1.535224 -0.376042 -0.176166 0.284619 +-1.424770 1.277324 1.577807 0.503355 +-0.431695 -0.062389 -0.885902 0.067081 +1.629584 1.294023 -1.918801 0.647628 +-1.593367 -0.252175 -1.284288 0.392990 +1.340376 -1.369604 0.751793 0.384202 +1.933990 -1.532726 0.368489 0.620120 +0.430453 -1.651175 -0.318704 0.236961 +1.136752 -1.041493 1.169101 0.311393 +1.571834 -0.617925 -1.046282 0.377604 +1.771728 -0.153149 0.577677 0.383251 +-0.404439 0.422082 -0.145000 0.033791 +-0.826632 1.990387 1.792612 0.568978 +0.287466 1.619971 1.086635 0.279526 diff --git a/neuralnetworks/backpropagation/.old/validation1.txt b/neuralnetworks/backpropagation/.old/validation1.txt new file mode 100644 index 0000000..75257b5 --- /dev/null +++ b/neuralnetworks/backpropagation/.old/validation1.txt @@ -0,0 +1,50 @@ +0.003277 0.848481 0.500607 +-1.427406 1.964000 0.890860 +-1.391789 0.537382 0.228752 +-0.125430 0.683673 0.453343 +-1.507577 0.876596 0.432709 +-1.569817 -1.731977 0.785425 +-0.381805 -1.786936 0.766554 +1.127473 0.462658 0.866206 +-0.065526 1.733933 0.546951 +0.719028 -1.218122 0.348117 +1.898024 0.785155 0.526406 +1.041945 1.357749 0.234160 +1.489717 1.965961 0.141295 +-0.255205 -0.159470 0.310978 +-0.417612 0.332953 0.235798 +-0.370680 0.763136 0.400042 +0.026609 -1.618669 0.482745 +1.324651 1.508803 0.187194 +1.590378 -0.556722 0.692398 +-0.859758 -1.837694 0.972145 +-1.441049 -1.939670 0.883005 +0.826141 1.102275 0.422983 +-0.942669 0.591992 0.202254 +-0.174097 -1.663213 0.616578 +-0.721853 -0.080824 0.050615 +1.606988 1.548495 0.280356 +0.203851 -0.424896 0.623611 +-0.986222 -1.100245 0.578388 +0.286364 -1.509866 0.343910 +1.292883 -0.369624 0.874598 +-0.918720 0.189327 0.025839 +-0.345168 -0.984532 0.493732 +-0.413866 0.072799 0.199344 +-0.516604 1.898701 0.858072 +1.221189 -0.823151 0.628925 +1.291934 -1.216164 0.350675 +0.125816 0.987687 0.501899 +-0.658971 -1.998535 0.929958 +-1.934955 0.287829 0.454126 +-1.767661 -0.419288 0.358870 +-1.438061 0.661992 0.304436 +-1.048132 -1.683175 0.938094 +-1.555927 -0.097041 0.182553 +0.528270 1.386355 0.289613 +-1.050745 0.607544 0.211839 +-0.710586 -0.100522 0.056373 +-0.821931 -1.974705 0.980188 +0.641828 -0.633677 0.730154 +-0.994657 -0.568632 0.186552 +1.858991 -0.336293 0.594873 diff --git a/neuralnetworks/backpropagation/.old/validation2.txt b/neuralnetworks/backpropagation/.old/validation2.txt new file mode 100644 index 0000000..8e50e5c --- /dev/null +++ b/neuralnetworks/backpropagation/.old/validation2.txt @@ -0,0 +1,50 @@ +-1.387994 -1.695247 -0.606290 0.464565 +-1.119965 0.701889 0.866822 0.225974 +-0.764356 0.099740 -0.951961 0.120460 +0.543036 0.589841 1.460780 0.183896 +-0.384500 0.561116 0.841566 0.082137 +-1.656176 -1.296764 0.746974 0.478035 +1.854343 1.946642 1.772868 0.869584 +-0.158587 0.212779 1.813929 0.196212 +-0.253496 -0.620826 0.800135 0.073998 +-1.322347 0.106878 -0.197858 0.204899 +1.346941 0.718884 0.106895 0.249749 +-1.259348 1.598919 -1.191216 0.461517 +1.607474 -1.165437 0.908524 0.450251 +-1.344488 1.377599 -0.501635 0.369075 +-1.883708 -1.006901 -1.940519 0.704661 +0.957858 -0.663077 -1.237284 0.228005 +-0.295168 -0.808734 -1.290642 0.156466 +-0.522300 1.032679 0.922137 0.162567 +-0.708371 -1.220817 -1.698689 0.339018 +-1.908236 -0.543164 0.408189 0.452464 +-0.106094 -1.196223 -0.872928 0.155333 +-1.999199 -0.455571 -1.274009 0.570774 +-1.190415 -0.848097 -0.439446 0.229980 +1.718109 -0.192585 -1.061846 0.408506 +-0.783526 -0.076292 -0.068747 0.071556 +-0.724045 -1.118434 1.268176 0.249497 +0.038671 0.586398 -1.540557 0.163546 +0.748030 -1.935902 1.492122 0.481297 +-0.329833 -0.644273 -1.728695 0.216890 +-0.028523 -0.552509 -0.271859 0.027840 +-1.620334 1.341397 0.531918 0.457675 +-0.493262 1.342198 -1.923653 0.380137 +0.232729 -1.848217 -0.771750 0.303373 +1.793284 1.869892 1.035665 0.701904 +-1.268562 -0.913634 -1.040627 0.312368 +0.662691 0.362321 -0.159061 0.062230 +-0.069132 -1.599007 -1.572663 0.339919 +0.390310 1.149022 -1.508566 0.250430 +-0.117568 -1.180811 -0.152839 0.110198 +0.153737 0.790666 1.294652 0.147515 +1.881878 1.170332 0.636049 0.537330 +0.413795 -1.322930 -0.021753 0.154411 +0.490142 0.909799 0.130030 0.092367 +1.718392 0.703083 -0.000078 0.378741 +0.754057 1.434521 1.086288 0.291982 +1.713430 0.097212 -0.551391 0.357018 +-0.445631 -1.971920 -0.150398 0.323332 +-0.018295 0.418390 -1.001375 0.071355 +0.473139 -1.699178 -0.182186 0.249838 +-1.679700 0.454559 -1.391520 0.453151 diff --git a/neuralnetworks/backpropagation/__init__.py b/neuralnetworks/backpropagation/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/neuralnetworks/backpropagation/analysis.py b/neuralnetworks/backpropagation/analysis.py new file mode 100644 index 0000000..758e7d7 --- /dev/null +++ b/neuralnetworks/backpropagation/analysis.py @@ -0,0 +1,137 @@ +import numpy as np +import matplotlib.pyplot as plt +import sklearn as sk + +from utils import * + +def plot_cost_versus_epochs_colormap(plot_file_name, target_test, Y_pred, cost_list, cost_test_list, learning_rates): + + """ cost_test_list --> target testing error list for each epoch, where cost_test_list[i] is the testing error for epoch i. + cost_list --> training error list for each epoch, where cost_list[i] is the training error for epoch i. + """ + + x = np.arange(len(cost_list)) + y = cost_list + color_metric = cost_test_list + + fig = plt.figure() + ax = fig.add_subplot(111) + + cmhot = plt.cm.get_cmap("hot") + l = ax.scatter(x, y, c=color_metric, cmap=cmhot) + fig.colorbar(l) + plt.show() + + # Save the figure + plt.savefig(plot_file_name, bbox_inches='tight') + +def plot_cost_versus_epochs(autoscale, plot_file_path, experiment_number, cost_list, cost_test_list): + + x1 = np.arange(len(cost_list)) + y1 = cost_list + + x2 = np.arange(len(cost_test_list)) + y2 = cost_test_list + + fig = plt.figure() + subplt = plt.subplot(111) + plt.plot(x1, y1) + plt.xlabel('Epochs') + plt.ylabel('Cost Function') + plt.title('Cost Function Per Epoch Over %s Epochs' % str(len(x1))) + plt.grid() + if autoscale: + subplt.autoscale_view(True, True, True) + fig.tight_layout() + + plot_file_name = "%s/epoch-vs-cost-exp-%s.pdf" % (plot_file_path, experiment_number) + plt.savefig(plot_file_name, bbox_inches='tight') + + fig = plt.figure() + subplt = plt.subplot(111) + plt.plot(x2, y2) + plt.xlabel('Epochs') + plt.ylabel('Cost Function') + plt.title('Cost Function Per Testing Epoch Over %s Epochs' % str(len(x2))) + plt.grid() + if autoscale: + subplt.autoscale_view(True,True,True) + fig.tight_layout() + + plot_file_name = "%s/epoch-vs-testing-cost-exp-%s.pdf" % (plot_file_path, experiment_number) + plt.savefig(plot_file_name, bbox_inches='tight') + + return plot_file_name + +def plot_rmse_versus_epochs(autoscale, plot_file_path, experiment_number, rmse): + + x1 = np.arange(len(rmse)) + y1 = rmse + + fig = plt.figure() + subplt = plt.subplot(111) + plt.plot(x1, y1) + plt.xlabel('Epochs') + plt.ylabel('RMSE') + plt.title('RMSE Per Epoch Over %s Epochs' % str(len(x1))) + plt.grid() + if autoscale: + subplt.autoscale_view(True,True,True) + fig.tight_layout() + + plot_file_name = "%s/epoch-vs-rmse-exp-%s.pdf" % (plot_file_path, experiment_number) + plt.savefig(plot_file_name, bbox_inches='tight') + + return plot_file_name + +def plot_learning_rates_versus_epochs(autoscale, plot_file_path, experiment_number, learning_rates): + x1 = np.arange(len(learning_rates)) + y1 = learning_rates + + fig = plt.figure() + subplt = plt.subplot(111) + plt.plot(x1, y1) + plt.xlabel('Epochs') + plt.ylabel('Learning Rate') + plt.title('Learning Rate Per Epoch Over %s Epochs' % str(len(x1))) + plt.grid() + if autoscale: + subplt.autoscale_view(True,True,True) + fig.tight_layout() + + plot_file_name = "%s/epoch-vs-lr-exp-%s.pdf" % (plot_file_path, experiment_number) + plt.savefig(plot_file_name, bbox_inches='tight') + + return plot_file_name + +def plot_accuracy(plot_file_path, experiment_number, target_test, Y_pred): + + x1 = target_test + y1 = Y_pred + + fig = plt.figure() + plt.scatter(x1, y1, alpha=0.5) + plt.xlabel('Target Values') + plt.ylabel('Predicted Values') + plt.title('Accuracy of Network') + plt.grid() + fig.tight_layout() + + plot_file_name = "%s/accuracy-exp-%s.pdf" % (plot_file_path, experiment_number) + plt.savefig(plot_file_name, bbox_inches='tight') + + return plot_file_name + +def facial_recognition_graphs(): + + prediction_titles = [title(y_pred, y_test, target_names, i) + for i in range(y_pred.shape[0])] + + plot_gallery(X_test, prediction_titles, h, w) + + # plot the gallery of the most significative eigenfaces + + eigenface_titles = ["eigenface %d" % i for i in range(eigenfaces.shape[0])] + plot_gallery(eigenfaces, eigenface_titles, h, w) + + plt.show() \ No newline at end of file diff --git a/neuralnetworks/backpropagation/network.py b/neuralnetworks/backpropagation/network.py new file mode 100644 index 0000000..a0d276f --- /dev/null +++ b/neuralnetworks/backpropagation/network.py @@ -0,0 +1,369 @@ +#!./usr/bin/python +# -*- coding: utf-8 -*-# + +import numpy as np +np.set_printoptions(precision=4, suppress=True) +import math as math +from matplotlib.pyplot import plot +from sklearn.datasets import load_iris, load_digits + +from utils import * + +class BackPropagationNetwork(object): + """ + + Initialize as: + nn = BackPropagationNetwork(n_features, n_classes, hidden_layers, reg_term) + + --> reg_term (i.e. lambda) is the regularization term + nn input and output units determined by training data + + Set nn.hidden_layers to list of integers to create hidden layer architecture + nn.hidden_layers does not include the bias unit for each layer + nn.hidden_layers is list containing number of units in each hidden layer + [4, 4, 2] will create 3 hidden layers of 4 units, 4 units, and 2 units + Entire architecture will be 5 layers with units [n_features, 4, 4, 2, n_classes] + + nn.fit(X, Y, epochs) where X is training data np.array of features, Y is training data of np.array of output classes , epochs is integer specifying the number of training iterations + For multi-class prediction, each observation in Y should be implemented as a vector with length = number of classes where each position represents a class with 1 for the True class and 0 for all other classes + For multi-class prediction, Y will have shape n_observations by n_classes + + nn.predict(X) returns vector of probability of class being true or false + For multi-class prediction, returns a vector for each observation will return a vector where each position in the vector is the probability of a class + + Test a simple XOR problem with nn.XOR_test() + nn.XOR_test() accepts an optional list of integers to determine the hidden layer architecture + """ + + def __init__(self, logger, n_features, n_classes, hidden_layers, reg_term, test_type=None): + self.logger = logger + self.test_type = test_type + self.n_features = n_features + self.n_classes = n_classes + self.hidden_layers = hidden_layers + self.reg_term = reg_term + self.epochs = 2 + self.learning_rate = 0.5 + self.learning_reward = 1.05 + self.learning_penalty = 0.5 + self.momentum_rate = 0.1 + self.Theta_L = [] + + self.initialize_theta() + + def initialize_theta(self): + """ + initialize_theta creates architecture of neural network + Defines self.Theta_L + + Parameters: + hidden_unit_length_list - List of hidden layer units + input_unit_count - integer, number of input units (features) + output_class_count - integer, number of output classes + """ + + unit_count_list = [len(self.n_features[0])] + unit_count_list += self.hidden_layers + unit_count_list.append(len(self.n_classes[0])) + self.Theta_L = [ 2 * (np.random.rand(unit_count, unit_count_list[l-1]+1) - 0.5) for l, unit_count in enumerate(unit_count_list) if l > 0] + + def print_theta(self): + + T = len(self.Theta_L) + + self.logger.info('\n') + self.logger.info('NN ARCHITECTURE') + self.logger.info('%s Layers (%s Hidden)' % ((T + 1), (T-1))) + self.logger.info('%s Thetas' % T) + self.logger.info('%s Input Features' % (self.Theta_L[0].shape[1]-1)) + self.logger.info('%s Output Classes' % self.Theta_L[T-1].shape[0]) + self.logger.info('\n') + + self.logger.info('Units per layer') + for t, theta in enumerate(self.Theta_L): + if t == 0: + self.logger.info(' - Input: %s Units' % (theta.shape[1] - 1)) + if t < T-1: + self.logger.info(' - Hidden %s: %s Units' % ((t+1), theta.shape[0])) + else: + self.logger.info(' - Output: %s Units' % theta.shape[0]) + + self.logger.info('Theta Shapes') + for l, theta in enumerate(self.Theta_L): + self.logger.info('Theta %s: %s' % (l, theta.shape)) + + self.logger.info('Theta Values') + for l, theta in enumerate(self.Theta_L): + self.logger.info('Theta %s:' % l) + self.logger.info("\n" + str(theta)) + self.logger.info('\n') + + def cost_function(self, Y, Y_pred): + """ + cost_function implements cost function + + y is n_observations by n_classes (n_classes = 1 for n_classes <=2) + pred_y is predicted y values and must be same shape as y + + Returns cost - list of cost values + """ + + if Y.shape != Y_pred.shape: + if Y.shape[0] != Y_pred.shape: + raise ValueError,'Wrong number of predictions' + else: + raise ValueError,'Wrong number of prediction classes' + + n_observations = len(Y) + tiny = 1e-6 + # Cost Function + cost = (-1.0 / n_observations)*(Y * np.log(Y_pred + tiny) + ((1-Y) * np.log(1-Y_pred + tiny))).sum() + + return cost + + + def predict(self, X): + """ + predict calculates activations for all layers, returns prediction for Y + + Parameters + X is array of input features dimensions n_observations by n_features + + Returns + a_N is outputs of all units + a_N[L] is array of predicted Y values dimensions n_observations by n_classes + """ + + m = len(X) + T = len(self.Theta_L) + + a_N_predict = [] # List of activations including bias unit for non-output layers + + # Input Layer inputs + a_N_predict.append( X ) + # Loop through each Theta_List theta + # t is Theta for calculating layer t+1 from layer t + for t, theta in enumerate(self.Theta_L): + # Add bias unit + if a_N_predict[t].ndim == 1: + a_N_predict[t].resize(1, a_N_predict[t].shape[0]) + a_N_predict[t] = np.append(np.ones((a_N_predict[t].shape[0],1)), a_N_predict[t], 1) + + # Calculate and Append new z and a arrays to z_N and a_N lists + z = a_N_predict[t].dot(theta.T) + a_N_predict.append(sigmoid(z)) + + return a_N_predict, a_N_predict[T] + + + def back_prop(self, a_N_backprop, Y_train): + """ + a_N - list of layer outputs with dimensions n_observations by n_units + Y_train is n_observations, n_classes + + Returns + Theta_Gradient_L + """ + + T = len(self.Theta_L) + Y_pred = a_N_backprop[T] + n_observations = len(Y_pred) + + # Backprop Error; One list element for each layer + delta_N = [] + + # Get Error for Output Layer + delta = Y_pred - Y_train + if delta.ndim == 1: + delta.resize(1, len(delta)) + delta_N.append( delta ) + + # Get Error for Hidden Layers working backwards (stop before layer 0; no error in input layer) + for t in range(T-1,0,-1): + delta = delta.dot(self.Theta_L[t][:,1:]) * ( a_N_backprop[t][:,1:] * (1 - a_N_backprop[t][:,1:]) ) + delta_N.append( delta ) + # Reverse the list so that delta_N[t] is delta that Theta[t] causes on a_N[t+1] + delta_N.reverse() + + # Calculate Gradient from delta and activation + # t is the Theta from layer t to layer t+1 + Theta_Gradient_L = [] + for t in range(T): + Theta_Gradient_L.append( delta_N[t].T.dot(a_N_backprop[t]) ) + + # Create modified copy of the Theta_L for Regularization + # Coefficient for theta values from bias unit set to 0 so that bias unit is not regularized + regTheta = [np.zeros_like(theta) for theta in self.Theta_L] + for t, theta in enumerate(self.Theta_L): + regTheta[t][:,1:] = theta[:,1:] + + # Average Error + regularization penalty + for t in range(T): + Theta_Gradient_L[t] = Theta_Gradient_L[t] * (1.0/n_observations) + (self.reg_term * regTheta[t]) + + return Theta_Gradient_L + + def fit(self, X_train, Y_train, X_test=None, Y_test=None): + """ + fit() calls the predict and back_prop functions for the + given number of cycles, tracks error and error improvement rates + + Parameters: + X_train - np.array of training data with dimension n_observations by n_features + Y_train - np.array of training classes with dimension n_observations by n_classes + epochs - integer of number of times to update Theta_L + learning_rate + momentum_rate + learning_reward + learning_penalty + X_test - np.array of training data with dimension n_observations by n_features + Y_test - np.array of training classes with dimension n_observations by n_classes + Returns + cost_list - list of result of cost function for each epoch + Learning_rates - list of learning rates used for each epoch + Notes + Training and Test data are assumed to be in random order; mini-batch processing does not need to re-randomize + """ + + # Initial Learning Rate + learning_rates = [] + learning_rates.append( self.learning_rate ) + + # Initial Weight Change Terms + weight_change_L = [np.zeros_like(theta) for theta in self.Theta_L] + + # List of results of cost functions + cost_list = [0] * self.epochs + cost_test_list = [0] * self.epochs + rmse = [0] * self.epochs + # Initial Forward Pass + a_N_train, Y_pred = self.predict(X_train) + # Initial Cost + cost_list[0] = self.cost_function(Y_train, Y_pred) + + # Test Error + if Y_test is not None: + a_N_test, Y_pred_test = self.predict(X_test) + cost_test_list[0] = self.cost_function(Y_test, Y_pred_test) + + for i in range(1, self.epochs): + + # Back Prop to get Theta Gradients + Theta_grad = self.back_prop(a_N_train, Y_train) + + # Update Theta with Momentum + for l, theta_g in enumerate(Theta_grad): + weight_change_L[l] = self.learning_rate * theta_g + (weight_change_L[l] * self.momentum_rate) + self.Theta_L[l] = self.Theta_L[l] - weight_change_L[l] + + # Update Units + a_N_train, Y_pred_new = self.predict(X_train) + + # Check to see if Cost decreased + cost_new = self.cost_function(Y_train, Y_pred_new) + + if cost_new > cost_list[i-1]: + # Reduce learning rate + self.learning_rate = self.learning_rate * self.learning_penalty + # Reverse part of adjustment (add back new learning_rate * Theta_grad); Leave momentum in place + self.Theta_L = [t + (self.learning_rate * tg) for t, tg in zip(self.Theta_L, Theta_grad)] + # Cut prior weight_change as an approximate fix to momentum + weight_change_L = [m * self.learning_penalty for m in weight_change_L] + + a_N_train, Y_pred_new = self.predict(X_train) + cost_new = self.cost_function(Y_train, Y_pred_new) + else: + self.learning_rate = np.min((10, self.learning_rate * self.learning_reward)) + + learning_rates.append(self.learning_rate) + cost_list[i] = cost_new + + if Y_test is not None: + a_N_test, Y_pred_test = self.predict(X_test) + + sum_e = 0 + for j in range(len(Y_test)): + sum_e += pow((Y_test[j] - Y_pred_test[j]), 2) + + if len(sum_e) > 1: + sum_e = np.sum(sum_e) + + rmse_epoch = math.sqrt((1.0 / (2.0 * len(Y_test))) * sum_e) + rmse[i] = rmse_epoch + + cost_test_list[i] = self.cost_function(Y_test, Y_pred_test) + + for t, theta in enumerate(self.Theta_L): + self.logger.info('Theta: %s' % t) + for theta_i in np.round(theta, 2): + self.logger.info("%s" % str(theta_i)) + + #self.logger.info('i: %ld - cost: %ld' % (i, cost_list[i])) + #self.logger.info('i: %ld - cost test: %ld' % (i, cost_test_list[i])) + + return cost_list, learning_rates, cost_test_list, rmse + + def test(self, data_train, target_train, epochs, learning_rate, momentum_rate, learning_reward, learning_penalty, data_test=None, target_test=None, data_val=None, target_val=None): + + self.epochs = epochs + self.learning_rate = learning_rate + self.momentum_rate = momentum_rate + self.learning_reward = learning_reward + self.learning_penalty = learning_penalty + + # Initialize Theta based on selected architecture + # self.initialize_theta(data_train.shape[1], target_train.shape[1], hidden_unit_length_list) + + # Fit + cost_list, learning_rates, cost_test_list, rmse = self.fit(data_train, target_train, X_test=data_test, Y_test=target_test) + + error = 0 + # Predict for test log + plot_vals = [] + a_N, Y_pred = self.predict(data_test) + self.logger.info('###################################Testing Results###################################') + self.logger.info('Given X:') + for x in data_test[:5]: + self.logger.info(x) + for p in zip(target_test[:10], np.round(Y_pred[:10],6)): + plot_vals.append(p) + self.logger.info('Actual Y, Predicted Y:') + for pv in plot_vals: + self.logger.info("%s" % str(pv)) + self.logger.info('Cost Efficiency on Test Set: %s' % str(self.cost_function(target_test , Y_pred))) + sum_e = 0 + for j in range(len(target_test)): + sum_e += pow((target_test[j] - Y_pred[j]), 2) + if len(sum_e) > 1: + sum_e = np.sum(sum_e) + self.logger.info('Final Testing Sum Over Outputs: %s' % str(sum_e)) + rmse_test_final = math.sqrt((1.0 / (2.0 * len(target_test))) * sum_e) + self.logger.info('Final Testing RMSE: %s' % str(rmse_test_final)) + + error = 0 + #Predict for validation results + + if data_val is not None: + plot_vals = [] + va_N, vY_pred = self.predict(data_val) + self.logger.info('###################################Validation Results###############################') + self.logger.info('Given X:') + for x in data_val[:5]: + self.logger.info(x) + for p in zip(target_val[:10], np.round(vY_pred[:10],6)): + plot_vals.append(p) + self.logger.info('Actual Y, Predicted Y:') + for pv in plot_vals: + self.logger.info((pv)) + self.logger.info('Cost Efficiency on Validation Set: %s' % str(self.cost_function(target_val , vY_pred))) + sum_e = 0 + for j in range(len(target_val)): + sum_e += pow((target_val[j] - vY_pred[j]), 2) + if len(sum_e) > 1: + sum_e = np.sum(sum_e) + self.logger.info('Final Validation Sum Over Outputs: %s' % str(sum_e)) + rmse_val_final = math.sqrt((1.0 / (2.0 * len(target_val))) * sum_e) + self.logger.info('Final Validation RMSE: %s' % str(rmse_val_final)) + + return target_test, Y_pred, cost_list, cost_test_list, learning_rates, rmse + diff --git a/neuralnetworks/backpropagation/run_tests.py b/neuralnetworks/backpropagation/run_tests.py new file mode 100644 index 0000000..52e8d4a --- /dev/null +++ b/neuralnetworks/backpropagation/run_tests.py @@ -0,0 +1,151 @@ +# Python standard libraries +import sys +import os +import contextlib +import logging +import tempfile +import argparse +from argparse import RawTextHelpFormatter + +# 3rd-Party libraries +import numpy as np +import sklearn as sk +from sklearn.metrics import confusion_matrix, classification_report +from sklearn.preprocessing import LabelBinarizer +from sklearn.metrics import accuracy_score +import matplotlib +matplotlib.use('TkAgg') +import matplotlib.pyplot as plt + +# Project specific libraries +from network import BackPropagationNetwork +from tests import Tests +from analysis import * + + +# Function for changing directories safely +@contextlib.contextmanager +def cd(newPath): + savedPath = os.getcwd() + os.chdir(newPath) + yield + os.chdir(savedPath) + + +def is_valid_ttype_option(ttype): + options = ['x', 'd', 'i', 'f', 'w'] + if ttype in options: + return ttype + else: + print "Option 'ttype' is invalid. Choose from: " + print options + sys.exit(1) + + +# Setup the command line parser +def setup_argparser(): + + parser = argparse.ArgumentParser(description='' + + ' PyNet: Feed-Forward Back-Propagation Network in Python\n' + + ' Written by: Jared Smith and David Cunningham', + version='1.0.0', formatter_class=RawTextHelpFormatter) + + requiredArguments = parser.add_argument_group('required Arguments') + requiredArguments.add_argument('-exp', dest='experiment_number', required=True, type=str, help="Number of this experiment.") + requiredArguments.add_argument('-ttype', dest='test_type', required=True, type=is_valid_ttype_option, help="Type of test to run. Choose from 'x', 'd', 'i', 'f', or 'w'") + requiredArguments.add_argument('-hidden_layers', dest='hidden_layers', required=True, type=int, nargs='+', help="A list of numbers which represent each hidden layer and the affiliate nodes in that layer.") + optionalArguments = parser.add_argument_group('optional Arguments') + optionalArguments.add_argument('--epochs', dest='epochs', required=False, type=int, default=2500, help="Number of epochs to train on. Default is 2500.") + optionalArguments.add_argument('--learning_rate', dest='learning_rate', required=False, type=float, default=0.5, help="Learning rate, specifies the step width of the gradient descent. Default is 0.5.") + optionalArguments.add_argument('--momentum_rate', dest='momentum_rate', required=False, type=float, default=0.1, help="Momentum rate, specifies the amount of the old weight change (relative to 1) which is added to the current change. Default is 0.1.") + optionalArguments.add_argument('--learning_reward', dest='learning_reward', required=False, type=float, default=1.05, help="Magnitude to scale the learning rate by if cost/error decreases from the previous epoch. Default is 1.05.") + optionalArguments.add_argument('--learning_penalty', dest='learning_penalty', required=False, type=float, default=0.5, help="Magnitude to scale the learning rate by if the cost/error increases from the previous epoch. Default is 0.5.") + optionalArguments.add_argument('--regularization_term', dest='reg_term', required=False, type=float, default=1e-5, help="Regularization term (i.e. lamdba in the equations). Default is 1e-5.") + optionalArguments.add_argument('--ftrain', dest='ftrain', required=False, type=str, default=None, help="Training data file for function approximation test. Default is None.") + optionalArguments.add_argument('--ftest', dest='ftest', required=False, type=str, default=None, help="Testing data file for function approximation test. Default is None.") + optionalArguments.add_argument('--fvalid', dest='fvalid', required=False, type=str, default=None, help="Validation data file for function approximation test. Default is None.") + optionalArguments.add_argument('--plot', dest='plot', required=False, type=bool, default=True, help="Specify if data is to be plotted. Default is True.") + optionalArguments.add_argument('--autoscale', dest='autoscale', required=False, type=bool, default=True, help="Specify plots should be autoscaled to data frame. Default is True.") + return parser + + +def setup_logger(log_path, logger_name, logfile_name): + + logFormatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + rootLogger = logging.getLogger(logger_name) + rootLogger.setLevel(logging.DEBUG) + + fileHandler = logging.FileHandler("{0}/{1}.log".format(log_path, logfile_name)) + fileHandler.setFormatter(logFormatter) + rootLogger.addHandler(fileHandler) + + consoleHandler = logging.StreamHandler() + consoleHandler.setFormatter(logFormatter) + rootLogger.addHandler(consoleHandler) + + return rootLogger + + +if __name__=="__main__": + + graph_list = [] + + parser = setup_argparser() + args = parser.parse_args() + experiment_number = args.experiment_number + + # Setup directories for storing results + if not os.path.exists('results'): + os.makedirs('results') + + with cd('results'): + if not os.path.exists('data'): + os.makedirs('data') + with cd('data'): + if not os.path.exists('Experiment-' + str(experiment_number)): + os.makedirs('Experiment-' + str(experiment_number)) + + logger = setup_logger('results/data/Experiment-' + str(experiment_number), "__main__", "main") + logger.info("###################################RUNNING EXPERIMENT NUM %s#########################", str(experiment_number)) + logger.info("Program Arguments:") + args_dict = vars(args) + for key, value in args_dict.iteritems() : + logger.info("%s=%s" % (str(key), str(value))) + + test_suite = Tests(logger, args) + target_test, Y_pred, cost_list, cost_test_list, learning_rates, rmse = test_suite.run_tests() + + Y_pred_copy = np.copy(Y_pred) + accuracy_score_Y_pred = np.rint(Y_pred_copy).astype(int) + + if args.test_type != 'f': + logger.info('###################################Accuracy Results###############################') + logger.info('Accuracy: ' + str(accuracy_score(target_test, accuracy_score_Y_pred))) + logger.info('\n' + str(classification_report(target_test, accuracy_score_Y_pred))) + else: + logger.info('###################################Accuracy Results###############################') + + target_test_1d = target_test.ravel() + Y_pred_1d = Y_pred.ravel() + distance = 0 + + for i in range(len(target_test_1d)): + distance += abs(Y_pred_1d[i] - target_test_1d[i]) + + avg_distance = distance / len(target_test_1d) + logger.info("Accuracy Score: %s" % (str(avg_distance))) + logger.info("NOTE: Accuracy Score is avg. distance between expected and predicted y-values") + logger.info("NOTE: Computed using the following code:") + logger.info("for i in range(len(target_test_1d)):") + logger.info("\tdistance += abs(Y_pred_1d[i] - target_test_1d[i])") + logger.info("\tavg_distance = distance / len(target_test_1d)") + + save_path = 'results/data/Experiment-%s' % (experiment_number) + save_data(save_path, target_test, Y_pred, cost_list, cost_test_list, learning_rates, rmse, experiment_number) + + # Plotting + if args.plot: + plot_file_path = 'results/data/Experiment-%s' % (experiment_number) + plot_cost_versus_epochs(args.autoscale, plot_file_path, experiment_number, cost_list, cost_test_list) + plot_rmse_versus_epochs(args.autoscale, plot_file_path, experiment_number, rmse) + plot_learning_rates_versus_epochs(args.autoscale, plot_file_path, experiment_number, learning_rates) diff --git a/neuralnetworks/backpropagation/tests.py b/neuralnetworks/backpropagation/tests.py new file mode 100644 index 0000000..f2ae576 --- /dev/null +++ b/neuralnetworks/backpropagation/tests.py @@ -0,0 +1,234 @@ +import sys + +import numpy as np +import math as math +import pylab as plt +from time import time +from sklearn.datasets import load_iris, load_digits, fetch_lfw_people, fetch_lfw_pairs +from sklearn.decomposition import RandomizedPCA +from sklearn.cross_validation import train_test_split as sklearn_train_test_split + + +from network import BackPropagationNetwork +from utils import * + +class Tests(object): + + def __init__(self, logger, args): + + self.logger = logger + self.args = args + + def run_tests(self): + if self.args.test_type == 'x': + self.logger.info("###################################RUNNING XOR TEST##################################") + target_test, Y_pred, cost_list, cost_test_list, learning_rates, rmse = self.XOR_test(self.args.reg_term, self.args.hidden_layers, self.args.epochs, self.args.learning_rate, self.args.momentum_rate, self.args.learning_reward, self.args.learning_penalty) + return (target_test, Y_pred, cost_list, cost_test_list, learning_rates, rmse) + elif self.args.test_type == 'd': + self.logger.info("###################################RUNNING DIGITS TEST###############################") + target_test, Y_pred, cost_list, cost_test_list, learning_rates, rmse = self.digits_test(self.args.reg_term, self.args.hidden_layers, self.args.epochs, self.args.learning_rate, self.args.momentum_rate, self.args.learning_reward, self.args.learning_penalty) + return (target_test, Y_pred, cost_list, cost_test_list, learning_rates, rmse) + elif self.args.test_type == 'i': + self.logger.info("###################################RUNNING IRIS TEST#################################") + target_test, Y_pred, cost_list, cost_test_list, learning_rates, rmse = self.iris_test(self.args.reg_term, self.args.hidden_layers, self.args.epochs, self.args.learning_rate, self.args.momentum_rate, self.args.learning_reward, self.args.learning_penalty) + return (target_test, Y_pred, cost_list, cost_test_list, learning_rates, rmse) + elif self.args.test_type == 'f': + self.logger.info("###################################RUNNING APPROX TEST###############################") + target_test, Y_pred, cost_list, cost_test_list, learning_rates, rmse = self.fnct_aprox(self.args.reg_term, self.args.hidden_layers, self.args.epochs, self.args.learning_rate, self.args.momentum_rate, self.args.learning_reward, self.args.learning_penalty, self.args.ftrain, self.args.ftest, self.args.fvalid) + return (target_test, Y_pred, cost_list, cost_test_list, learning_rates, rmse) + elif self.args.test_type == 'w': + self.logger.info("###################################RUNNING FACES TEST###############################") + target_test, Y_pred, cost_list, cost_test_list, learning_rates, rmse = self.faces_test(self.args.reg_term, self.args.hidden_layers, self.args.epochs, self.args.learning_rate, self.args.momentum_rate, self.args.learning_reward, self.args.learning_penalty) + return (target_test, Y_pred, cost_list, cost_test_list, learning_rates, rmse) + + + def translate_to_binary_array(self, target): + n_obs = len(target) + unique_targets = np.unique(target) + n_unique_targets = len(np.unique(target)) + + # Translation of target values to array indicies + target_translation = dict(zip(unique_targets, range(n_unique_targets))) + + # Create initial target array with all zeros + target_array = np.zeros((n_obs, n_unique_targets)) + + # Set 1 value + for i, val in enumerate(target): + target_array[i][target_translation[val]] = 1 + + return target_array + + def train_test_split(self, data_array, target_array, split=.8): + """ + Split into randomly shuffled train and test sets + Split on Number of records or Percent of records in the training set + if split is <= 1 then split is a percent, else split is the number of records + """ + + n_obs = len(data_array) + + if split <= 1: + train_len = int(split * n_obs) + else: + train_len = int(np.round(split)) + + shuffled_index = range(n_obs) + np.random.shuffle(shuffled_index) + + train_data = data_array[shuffled_index[:train_len]] + test_data = data_array[shuffled_index[train_len:]] + + train_target = target_array[shuffled_index[:train_len]] + test_target = target_array[shuffled_index[train_len:]] + + print train_data.shape + print test_data.shape + + print train_target.shape + print test_target.shape + + self.logger.info('Data Set: %d Observations, %d Features' % (data_array.shape[0], data_array.shape[1])) + self.logger.info('Training Set: %d Observations, %d Features' % (train_data.shape[0], train_data.shape[1])) + self.logger.info('Test Set: %d Observations, %d Features' % (test_data.shape[0], test_data.shape[1])) + self.logger.info('Target Set: %d Observations, %d Classes' % (target_array.shape[0], target_array.shape[1])) + self.logger.info('Training Set: %d Observations, %d Features' % (train_target.shape[0], train_target.shape[1])) + self.logger.info('Test Set: %d Observations, %d Features' % (test_target.shape[0], test_target.shape[1])) + + return train_data, test_data, train_target, test_target + + def iris_test(self, reg_term, hidden_layers, epochs, learning_rate, momentum_rate, learning_acceleration, learning_backup): + + data_set = load_iris() + + data = data_set.data + target = self.translate_to_binary_array(data_set.target) + + # Split into train, test sets + data_train, data_test, target_train, target_test = self.train_test_split(data, target, .75) + + NN = BackPropagationNetwork(self.logger, data_train, target_train, hidden_layers, reg_term) + return BackPropagationNetwork.test(NN, data_train, target_train, epochs, learning_rate, momentum_rate, learning_acceleration, learning_backup, data_test, target_test) + + def digits_test(self, reg_term, hidden_layers, epochs, learning_rate, momentum_rate, learning_acceleration, learning_backup): + + data_set = load_digits() + + data = data_set.data + target = self.translate_to_binary_array(data_set.target) + + # Split into train, test sets + data_train, data_test, target_train, target_test = self.train_test_split(data, target, .75) + + NN = BackPropagationNetwork(self.logger, data_train, target_train, hidden_layers, reg_term) + return BackPropagationNetwork.test(NN, data_train, target_train, epochs, learning_rate, momentum_rate, learning_acceleration, learning_backup, data_test, target_test) + + def XOR_test(self, reg_term, hidden_layers, epochs, learning_rate, momentum_rate, learning_acceleration, learning_backup): + """ + XOR_test is a simple test of the nn self.logger.info(ing the predicted value to std out + Trains on a sample XOR data set + Predicts a single value + Accepts an option parameter to set architecture of hidden layers + """ + + # Set Data for XOR Test + data_train = np.zeros((4,2)) + data_train[0,0] = 1. + data_train[1,1] = 1. + data_train[2,:] = 1. + data_train[3,:] = 0. + + target_train = np.array([1.,1.,0.,0.]).reshape(4,1) # Single Class + + # Test X and Y + data_test = np.array([[1,0],[0,1],[1,1],[0,0]]) + target_test = np.array([[1],[1],[0],[0]]) + + self.logger.info('Training Data: X') + for data_i in data_train: + self.logger.info("%s" % str(data_i)) + self.logger.info('Training Data: Y') + for target_i in target_train: + self.logger.info("%s" % str(target_i)) + + NN = BackPropagationNetwork(self.logger, data_train, target_train, hidden_layers, reg_term) + return BackPropagationNetwork.test(NN, data_train, target_train, epochs, learning_rate, momentum_rate, learning_acceleration, learning_backup, data_test, target_test) + + def fnct_aprox(self, reg_term, hidden_layers, epochs, learning_rate, momentum_rate, learning_acceleration, learning_backup, training_name, testing_name, validation_name): + + #read in train + data_train, target_train = self.parse_file(training_name, 200) + np.random.shuffle(data_train) + np.random.shuffle(target_train) + #read in test + data_test, target_test = self.parse_file(testing_name, 100) + np.random.shuffle(data_test) + np.random.shuffle(target_test) + #read in validation + data_val, target_val = self.parse_file(validation_name, 50) + np.random.shuffle(data_val) + np.random.shuffle(target_val) + + NN = BackPropagationNetwork(self.logger, data_train, target_train, hidden_layers, reg_term) + return BackPropagationNetwork.test(NN, data_train, target_train, epochs, learning_rate, momentum_rate, learning_acceleration, learning_backup, data_test, target_test, data_val = data_val,target_val = target_val) + + def parse_file(self, filename, num_lines): + + data = [] + target = [] + f = open(filename, 'r') + for line in f: + floats = map(float, line.split()) + target.append([floats.pop()]) + data.append(floats) + f.close() + return np.array(data), np.array(target) + + def faces_test(self, reg_term, hidden_layers, epochs, learning_rate, momentum_rate, learning_acceleration, learning_backup): + + lfw_people = fetch_lfw_people(min_faces_per_person=70, resize=0.4) + + # introspect the images arrays to find the shapes (for plotting) + n_samples, h, w = lfw_people.images.shape + + # for machine learning we use the 2 data directly (as relative pixel + # positions info is ignored by this model) + X = lfw_people.data + n_features = X.shape[1] + + # the label to predict is the id of the person + y = lfw_people.target + y = self.translate_to_binary_array(y) + + + target_names = lfw_people.target_names + n_classes = target_names.shape[0] + + self.logger.info("n_samples: {}".format(n_samples)) + self.logger.info("n_features: {}".format(n_features)) + self.logger.info("n_classes: {}".format(n_classes)) + + # split into a training and testing set + X_train, X_test, y_train, y_test = sklearn_train_test_split( + X, y, test_size=0.25) + + # Compute a PCA (eigenfaces) on the face dataset (treated as unlabeled + # dataset): unsupervised feature extraction / dimensionality reduction + n_components = 150 + + self.logger.info("Extracting the top %d eigenfaces from %d faces" + % (n_components, X_train.shape[0])) + t0 = time() + pca = RandomizedPCA(n_components=n_components, whiten=True).fit(X_train) + self.logger.info("done in %0.3fs" % (time() - t0)) + + eigenfaces = pca.components_.reshape((n_components, h, w)) + + self.logger.info("Projecting the input data on the eigenfaces orthonormal basis") + t0 = time() + X_train_pca = pca.transform(X_train) + X_test_pca = pca.transform(X_test) + self.logger.info("done in %0.3fs" % (time() - t0)) + + NN = BackPropagationNetwork(self.logger, X_train_pca, y_train, hidden_layers, reg_term) + return BackPropagationNetwork.test(NN, X_train_pca, y_train, epochs, learning_rate, momentum_rate, learning_acceleration, learning_backup, X_test_pca, y_test) \ No newline at end of file diff --git a/neuralnetworks/backpropagation/utils.py b/neuralnetworks/backpropagation/utils.py new file mode 100644 index 0000000..cd7027f --- /dev/null +++ b/neuralnetworks/backpropagation/utils.py @@ -0,0 +1,106 @@ +import numpy as np + +def sigmoid(z): + """sigmoid is a basic sigmoid function returning values from 0-1""" + return 1.0 / ( 1.0 + np.exp(-z) ) + +def sigmoidGradient(z): + # Not used + return self.sigmoid(z) * ( 1 - self.sigmoid(z) ) + +def format__1(digits,num): + if digits 1: + plt.text(peak_x, peak_y+.1, text_str) + plt.xlabel('B') plt.ylabel('Value') plt.title('Probability Distribution of Basin Sizes Normalized to 1') - #plt.legend(loc=9, bbox_to_anchor=(0.5, -0.1), ncol=10) plt.grid() + plt.ylim(0, 1.3) - # Histogram normalized to p + # Plot the Histogram Normalized to 1 plt.subplot(2, 1, 2) for i in range(1, num_rows + 1): - if i % 2 == 0: - label = 'p = %s' % str(i + 1) - plt.plot(np.arange(0, num_cols), normalize_data(avg_basin_size[i-1][:], i), label=label) + if i % 2 == 1: + text_str = '%d' % ((i + 1)) + n = normalize_data(avg_basin_size[i-1][:], i, i) + peak_x, peak_y = getpeak(n) + plt.plot(np.arange(0, num_cols), n) + + # Label the Curve + if peak_y < 4.3 and peak_x > 1: + plt.text(peak_x, peak_y+.1, text_str) plt.xlabel('B') plt.ylabel('Value') plt.title('Probability Distribution of Basin Sizes Normalized to P') plt.grid() - #plt.legend(loc=9, bbox_to_anchor=(0.5, -0.1), ncol=10) + plt.ylim(0,4.5) - # Save the figure + # Fix layout issues in plot. fig.tight_layout() + + # Save the figure fig.savefig(root_path + '/' + path) return file_path + '/' + root_path + '/' + path +# Handle Invalid weights class InvalidWeightsException(Exception): pass +# Handle Invalid NN Input class InvalidNetworkInputException(Exception): pass +# Class for maintaining all of the data structures used for the simulation class Data(object): + # Initialize Class def __init__(self, args, histo_file=None): self._nnsize = args.nnsize self._npat = args.npat @@ -152,22 +200,27 @@ def __init__(self, args, histo_file=None): self._data_file_name = args.dfn self._histo_data_file_name = histo_file + # Calculate the probablity vectors def calc_prob(self): stable_prob = np.zeros(self._npat) for p in range(self._npat): stable_prob[p] = self._stable[p] / (p+1) self._prunstable[p] = 1 - stable_prob[p] + # Sum each run def sum(self, data): self._stable += data._stable self._prunstable += data._prunstable self._basin_hist += data._basin_hist + # Average all of the runs def avg(self, nruns): self._stable /= nruns self._basin_hist /= nruns self._prunstable /= nruns + # Save the report data as a human readable text file + # Can also be read back into NumPy Arrays or Python Lists def save_report_data(self): with file(self._data_file_name, 'w') as outfile: outfile.write('# Average Stable Probability Data\n') @@ -185,36 +238,47 @@ def save_report_data(self): np.savetxt(outfile, self._basin_hist, fmt='%-7.2f') outfile.write('\n') + # Save Histogram data for use in plotting separately def save_histo_data(self): np.save(self._histo_data_file_name, self._basin_hist) + # Graph each run def graph(self, run): return plot_graph_data(self._exp, self._npat, self._stable, self._prunstable, run) +# Class for the Hopfield Neural Network Model used class HopfieldNetwork(object): + + # Initialize Class def __init__(self, num_inputs): self._num_inputs = num_inputs - #self._weights = np.zeros((num_inputs,num_inputs)) self._weights = np.random.uniform(-1.0, 1.0, (num_inputs,num_inputs)) + # Set the weights of the weight vector def set_weights(self, weights): if weights.shape != (self._num_inputs, self._num_inputs): raise InvalidWeightsException() self._weights = weights + # Get the weights def get_weights(self): return self._weights + # Evaluate the state of the Neural Network def evaluate(self, input_pattern): if input_pattern.shape != (self._num_inputs, ): raise InvalidNetworkInputException() weights = np.copy(self._weights) + + # Compute the sums of the input patterns + # Uses dot product sums = input_pattern.dot(weights) s = np.zeros(self._num_inputs) + # Enumerate the sums of the inputs and calculate new values for i, value in enumerate(sums): if value > 0: s[i] = 1 @@ -223,23 +287,24 @@ def evaluate(self, input_pattern): return s - def run(self, input_pattern, max_iterations=10): + # Run the updating of the Network for a specified iteration count + # Default number of iterations is 10 + def run(self, input_pattern, iterations=10): last_input_pattern = input_pattern - iteration_count = 0 while True: result = self.evaluate(last_input_pattern) iteration_count += 1 - if np.array_equal(result, last_input_pattern) or iteration_count == max_iterations: + if np.array_equal(result, last_input_pattern) or iteration_count == iterations: return result else: last_input_pattern = result +# Imprint the patterns onto the network def imprint_patterns(network, input_patterns, p): num_neurons = network.get_weights().shape[0] - weights = np.zeros((num_neurons, num_neurons)) for i in range(num_neurons): @@ -248,75 +313,76 @@ def imprint_patterns(network, input_patterns, p): for m in range(p): weights[i, j] += input_patterns[m][i] * input_patterns[m][j] - weights *= 1/float(network._num_inputs) + weights *= 1 / float(network._num_inputs) network.set_weights(weights) +# Test the patterns for stability def test_patterns(p, input_patterns, network, data): for k in range(p): pattern = input_patterns[k][:] updated_pattern = np.copy(pattern) updated_pattern = network.run(updated_pattern, 1) if np.array_equal(updated_pattern, pattern): - data._stable[p-1] +=1 + data._stable[p - 1] +=1 + + # Run Basin test data = basin_test(p, pattern, network, data, 5) else: - data._basin_hist[p-1][0] += 1 + data._basin_hist[p - 1][0] += 1 + return data +# Run the basin size tests def basin_test(p, input_pattern, network, data, runs): - #print "p = {}".format(p) basin = 0 + for run in range(runs): converge = True array = np.random.permutation(data._nnsize) - #print "Permuted array: " + str(array) - #print len(array) updated_pattern = np.copy(input_pattern) - for i in range(1, data._npat+1): - #flip bit: + + for i in range(1, data._npat + 1): for j in range (i): updated_pattern[array[j]] *= -1 - #updated pattern 10x + updated_pattern = network.run(updated_pattern) if not np.array_equal(updated_pattern, input_pattern): converge = False - ##print "Did not Converge: Adding %d to basin size." % i basin += i - ##print "New Basin Size: %d" % basin - ##print "Breaking now." break + if converge: - ##print "Converged: adding 50 to basin size." basin += 50 - basin = round((basin/runs), 0) - ##print "Basin Size: %s" % basin - data._basin_hist[p-1][basin] += 1 - return data + basin = round((basin / runs), 0) + data._basin_hist[p - 1][basin] += 1 + return data -def experiment(args): +# Run the experiment +def run_experiment(args): stable = np.zeros((args.npat)) input_patterns = np.random.choice([-1,1], ((args.npat), (args.nnsize))) Hnet = HopfieldNetwork((args.nnsize)) - #imprint weights data = Data(args) + + # Go through all npat patterns and run tests for p in range (1, args.npat + 1): - imprint_patterns(Hnet, input_patterns, p) #imprints the patterns - test_patterns(p, input_patterns, Hnet, data) #run the test vectors + imprint_patterns(Hnet, input_patterns, p) + test_patterns(p, input_patterns, Hnet, data) data.calc_prob() + return data +# Execute on program start if __name__ == '__main__': graph_list = [] - #np.set_printoptions(threshold=np.nan) - parser = setup_argparser() args = parser.parse_args() experiment_number = args.experiment_number - histo_file = 'basin_histo_data.npy' + histo_file = 'results/data/Experiment-%s/%s' % (experiment_number, args.hfn) # Setup directories for storing results if not os.path.exists('results'): @@ -329,28 +395,22 @@ def experiment(args): if not os.path.exists('Experiment-' + str(experiment_number)): os.makedirs('Experiment-' + str(experiment_number)) - # compute program and graph - # initialize avg stability avg_data = Data(args, histo_file) - #do several runs of experiment compute average stability + # Run experiment for nruns number of times for i in range(1, int(args.nruns) + 1): - exp_data = experiment(args) + exp_data = run_experiment(args) graph_list += exp_data.graph(i) avg_data.sum(exp_data) - #avg stable and unstable probs - #print "Averaging ..." avg_data.avg(args.nruns) - print avg_data._stable - print avg_data._prunstable + # Save the average data avg_data.save_report_data() avg_data.save_histo_data() - #graph stable and unstable probs - avg_graph_file = plot_graph_data(experiment_number, int(args.npat), avg_data._stable, avg_data._prunstable, 0) - histo_file = plot_histogram(experiment_number, avg_data._basin_hist) + # Plot the average stability graphs and make the histogram for basin sizes + plot_graph_data(experiment_number, args.npat, avg_data._stable, avg_data._prunstable, 0) + plot_histogram(avg_data._basin_hist, experiment_number) - #create_html_page(experiment_number, graph_list, histo_file, avg_graph_file) diff --git a/test_histo.py b/neuralnetworks/hopfield/testHisto.py old mode 100644 new mode 100755 similarity index 100% rename from test_histo.py rename to neuralnetworks/hopfield/testHisto.py diff --git a/requirements.txt b/requirements.txt old mode 100644 new mode 100755 index eebfb98..02bd0ff --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ -numpy==1.9.2 -matplotlib==1.4.3 -pillow==2.7.0 +numpy +sklearn +scipy +matplotlib + diff --git a/selforganization/README.md b/selforganization/README.md new file mode 100644 index 0000000..6bbcc95 --- /dev/null +++ b/selforganization/README.md @@ -0,0 +1,4 @@ +Self-Organization +==== +- Author: David Cunningham +- Particle Swarm Optimization diff --git a/selforganization/pso/Particle.py b/selforganization/pso/Particle.py new file mode 100644 index 0000000..f7e46b5 --- /dev/null +++ b/selforganization/pso/Particle.py @@ -0,0 +1,83 @@ +import numpy as np +import random +from Problems import * + +class Particle(object): + '''Particle + A representation of a particle in the PSO. + Fields: + pos: The position of the particle in the world array, represented as an array. + The particles actual xyz... cordinate is equal to: pos - maxPos/2 + best: The position visited by this particle that has the best fitness + curFitness: the fitness of the particle at its current position + bestFitness: the best fitness of any position the particle has traveled to. + In other words, the fitness coresponding to the "best" position field. + Q: The fitness equation of the particle. Takes the pos and the max position + velocity: a vector that represents the current velocity and direction of the particle + maxPos: array of he maximum possible position of the particle in any dimension i. + neighbors: a list of other particles considered "neighbors" to the given particle + + Functions: + setVelocity: + parameters: inertia, globalBest, phi + Description: determines the velocity of the particle + scaleVelocity: + parameters: maxvelocity + Description: rescales the particle velocity so that it does not + exceed the global maxvelocity or exit the world range + Move: + parameters: none + Description: updates the particles position according to its curent velocity + updateFitness: + parameters: Fitness + Description: caclulates the fitness of the particle at its current position + updates bestFitness and best fields if necessary + getDistance: + parameters: position s + Description: Finds the distance between particle and point at position s + ''' + #initializes the paritcle + def __init__(self, pos = [0, 0], maxPos = [100, 100]): + self.pos = pos + self.best = pos + self.curFitness = 0 + self.bestFitness = 0 + self.velocity = np.zeros(len(pos)) + self.maxPos = np.array(maxPos) + self.neighborhood = [] + #sets the velocity of the particle accoriing to its topology + def setVelocity(self, inertia, globalBest, phi): + #get best neighbor + best_neighbor = self + for i in self.neighborhood: + if best_neighbor.curFitness < i.curFitness: + best_neighbor = i + for i in range(len(self.pos)): + self.velocity[i] = inertia * self.velocity[i] + self.velocity[i] += phi[0] * random.random() * (self.best[i] - self.pos[i]) + self.velocity[i] += phi[1] * random.random() * (globalBest[i]-self.pos[i]) + self.velocity[i] += phi[2] * random.random() * (best_neighbor.pos[i] - self.pos[i]) + #scales the velocity to the global maxixmum velocity + def scaleVelocity(self, maxvelocity): + change_distance = 0 + for i in self.velocity: #compute sum of velocities in all dimensions velocities + change_distance+=i**2 + change_distance = change_distance**(1.0/2) *1.0 + for i in range(len(self.pos)): #find velocity for ith dimension + if abs(self.velocity[i]) > maxvelocity**2: + self.velocity[i] *= (maxvelocity/change_distance) + #adjusts the position of the swarm based on its current velocity + def Move(self): + for i in range(len(self.pos)): + self.pos[i] = (self.pos[i] + self.velocity[i]) + #evaluate and update fitness of particle + def updateFitness(self, Q): + #Find fitness at the current position, scaled to the range of the map + self.curFitness = Q(self.pos, self.maxPos) + if self.curFitness > self.bestFitness: + self.bestFitness = self.curFitness + #get distance between particle and position s (Wraps around) + def getDistance(self, s): + x = self.pos - s + x = np.minimum(x, (self.maxPos)-x) + return sum((x)**2)**.5 diff --git a/selforganization/pso/Problems.py b/selforganization/pso/Problems.py new file mode 100644 index 0000000..dd719f8 --- /dev/null +++ b/selforganization/pso/Problems.py @@ -0,0 +1,18 @@ +import math +def mdist(maxX, maxY): + return math.sqrt(maxX**2 + maxY**2/2) +def pdist(px, py): + return math.sqrt((px-20)**2 + (py-7)**2) +def ndist(px, py): + return math.sqrt((px+20)**2 + (py+7)**2) +def Problem1(pos, maxes): + return 100*(1-pdist(pos[0], pos[1])/mdist(maxes[0], maxes[1])) +def Problem2(pos, maxes): + pd = pdist(pos[0], pos[1]) + nd = ndist(pos[0], pos[1]) + md = mdist(maxes[0], maxes[1]) + + ret = 9*max(0, 10-pd**2) + ret+=10*(1-pd/md) + ret+=70*(1-nd/md) + return ret \ No newline at end of file diff --git a/selforganization/pso/__init__.py b/selforganization/pso/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/selforganization/pso/driver.py b/selforganization/pso/driver.py new file mode 100644 index 0000000..e69de29 diff --git a/selforganization/pso/psoAlgorithm.py b/selforganization/pso/psoAlgorithm.py new file mode 100644 index 0000000..2982859 --- /dev/null +++ b/selforganization/pso/psoAlgorithm.py @@ -0,0 +1,80 @@ +import numpy as np +import random +from Problems import * +from Particle import * +class PSO(object): + '''Particle Swarm Optimization: + Fields: + inertia: the inertia of the world + inertiaPrime: rate of change of a the inertia (a percentage) + npart: number of particles in the world + phi: The parameters of the world. Conscious, social and a potential 3rd neigborhood parameter + dimensions: an array of n elements each representing the range for a dimension in the world + maxvelocity: the maximum possible velocity of a particle in the world + Q: Fitness function of the swarm + swarm: the swarm of particles + Fuctions: + Update: moves all particles swarms to the next epoch + setClosestNeighbors: for each particle sets as neighbors the k closest particles to it + setEuclidianNeigbhbors: for each particle sets its neighbors as all particles within range + getError: returns the error for the given orientation. + ''' + def __init__(self, npart = 40, inertia=.5, phi = (2, 2, 2), dimensions = (100, 100), maxvelocity=1, Q = Problem1, inertiaPrime = 1): + self._inertia = inertia + self.phi = phi + self.globalBestFitness = 0 + self.maxvelocity = 1 + self.globalBest = None + self.Q = Q + self.inertiaPrime = inertiaPrime + self.swarm = [] + #Create and initialize the swarm + for i in range(npart): + pos = np.zeros(len(dimensions)) + for j in range(len(dimensions)): + pos[j] = random.randint(-1 * dimensions[j]/2, dimensions[j]/2) + particle = Particle(pos, dimensions) + fitness = particle.updateFitness(Q) + if fitness > self.globalBestFitness or self.globalBestFitness == 0: + self.globalBestFitness = fitness + self.globalBest = particle.pos + self.swarm.append(particle) + def setClosestNeighbors(self, k): + if k > 0: + for i in self.swarm: + #sort neighobrs into a stack + sortedSwarm = sorted(self.swarm, key = (lambda other: i.getDistance(other.pos)), reverse = True) + #the closest k particles that are not the target are its neighbors + while len(i.neighborhood) < k: + neighbor = sortedSwarm.pop() + if i != neighbor: + i.neighborhood.append(neighbor) + print i.neighborhood + #neighborhood is all particles within a certain range + def setEuclidianNeigbors(self, radius): + if radius > 0: + for i in self.swarm: + i.neighborhood = filter(lambda other: i != other and i.getDistance(other.pos) < radius, self.swarm) + print i.neighborhood + #updates the positions of all particles + def Update(self): + #update everybody's position + for i in self.swarm: + i.setVelocity(self._inertia, self.globalBest, self.phi) + i.scaleVelocity(self.maxvelocity) + i.Move() + #Update everybody's fitness + for i in self.swarm: + i.updateFitness(self.Q) + if i.bestFitness > self.globalBestFitness: + self.globalBestFitness = i.bestFitness + self.globalBest = i.best + #degreade inertia + self._inertia *= self.inertiaPrime + #return the error of the swarm at the given orientation + def getError(self): + error = np.zeros(len(self.swarm[0].pos)); + for i in self.swarm: + error += (i.pos - self.globalBest)**2 + error = (1.0/(2*len(self.swarm))*error)**(.5) + return error diff --git a/selforganization/pso/psoRun.py b/selforganization/pso/psoRun.py new file mode 100644 index 0000000..f7fedca --- /dev/null +++ b/selforganization/pso/psoRun.py @@ -0,0 +1,230 @@ +############################################################## +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# psoRun.py, runs and performs analysis on the Partical Swarm Optimization algorithm +# +# Written by Jared Smith and David Cunningham for COSC 427/527 +# at the University of Tennessee, Knoxville +# +############################################################### + +import sys +import os +import argparse +import logging +import contextlib +from argparse import RawTextHelpFormatter + +import numpy as np +import matplotlib.pyplot as plt + +from psoAlgorithm import PSO +from Problems import * + +# Function for changing directories safely +@contextlib.contextmanager +def cd(newPath): + savedPath = os.getcwd() + os.chdir(newPath) + yield + os.chdir(savedPath) + + +#distance with wrapping +def Distance(s1, s2): + x = s2.pos - s1 + x = np.minimum(x, (s2.maxPos) - x) + return sum((x)**2)**.5 + + +# Setup the command line parser +def setup_argparser(): + + parser = argparse.ArgumentParser(description='' + + ' BioPy: Particle Swarm Optimization' + + ' Algorithm in Python.\n' + + ' Written by: Jared Smith and' + + ' David Cunningham', + version='1.0.0', + formatter_class=RawTextHelpFormatter) + + requiredArguments = parser.add_argument_group('required Arguments') + requiredArguments.add_argument('-exp', dest='experiment_number', required=True, type=str, help="Number of this experiment.") + optionalArguments = parser.add_argument_group('optional Arguments') + optionalArguments.add_argument('--num_part', dest='npart', required=False, type=int, default=40, help="Number of particles in the world. Default is 40.") + optionalArguments.add_argument('--phi', dest='phi', required=False, type=tuple, default=(2, 2, 2), help="The conscious, social and potential neighboorhood parameters. Defaults are 2, 2, 2.") + optionalArguments.add_argument('--maxVel', dest='maxVel', required=False, type=int, default=20, help="Maximum Velocity of the PSO. Default is 20.") + optionalArguments.add_argument('--dim', dest='dim', required=False, type=tuple, default=(100, 100), help="Ranges of the dimensions of the world. Default is 100x100.") + optionalArguments.add_argument('--inertia', dest='inertia', required=False, type=float, default=1.0, help="Starting inertia of the particles. Default is 1.") + optionalArguments.add_argument('--inerPrime', dest='inerPrime', required=False, type=float, default=0.99, help="Percentage rate of change of inertia each iteration. Default is 0.99.") + optionalArguments.add_argument('--minError', dest='minError', required=False, type=float, default=.01, help="The minimum error the system must reach before it ceases updating. Default is 0.01.") + optionalArguments.add_argument('--maxIter', dest='maxIter', required=False, type=int, default=1000, help="Maximum number of iterations before the system ceases updating. Default is 1000.") + optionalArguments.add_argument('--Q', dest='Q', required=False, type=str, default='Problem1', help="Fitness function to use. Default is Problem1 specified in Problems.py.") + optionalArguments.add_argument('--num_neighbors', dest='k', required=False, type=int, default=0, help="Number of neighbors a particle has, or neighborhood range if Euclid parameter\n" + + "for Euclidean Topology is True. Default is 0.") + optionalArguments.add_argument('--Euclid', dest='Euclid', required=False, type=bool, default=False, help="Whether or not to use the Euclidian Topology. Default is False.") + return parser + + +def setup_logger(log_path, logger_name, logfile_name): + + logFormatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + rootLogger = logging.getLogger(logger_name) + rootLogger.setLevel(logging.INFO) + + fileHandler = logging.FileHandler("{0}/{1}.log".format(log_path, logfile_name), mode='w') + fileHandler.setFormatter(logFormatter) + rootLogger.addHandler(fileHandler) + + consoleHandler = logging.StreamHandler() + consoleHandler.setFormatter(logFormatter) + rootLogger.addHandler(consoleHandler) + + return rootLogger + + +def PlotScatter(swarm, maxes, best, iteration): + #if swarm is 2D plot + #create x and y arrays + x = [] + y = [] + for i in swarm: + x.append(i.pos[0]) + y.append(i.pos[1]) + #Plot figure + fig = plt.figure() + subplt = plt.subplot(111) + subplt.scatter(x, y, marker = "o", c = 'b', alpha=0.5, label = "Particle") + subplt.scatter(np.array([best[0]]), np.array([best[1]]), s=100,marker = 'x', c = "r", label = "Global Best") + plt.xlim([-1*maxes[0]/2, maxes[0]/2]) + plt.ylim([-1*maxes[1]/2, maxes[1]/2]) + plt.title("Locations of Particles after Iteration %d" % iteration) + plt.xlabel("X") + plt.ylabel('Y') + plt.legend(loc="upper left", bbox_to_anchor=[0, 1], ncol=2, shadow=True, title="Legend", fancybox=True) + filename = "Scatter Plots/Scatter-%d.png" % iteration + fig.savefig(filename) + plt.close(fig) + + +def PlotError(dim_error): + fig = plt.figure() + subplt = plt.subplot(111) + for n, val in enumerate(dim_error): + x = np.arange(0,len(val)) + subplt.plot(x, np.array(val), label ="n={}".format(n)) + plt.xlabel('Iteration') + plt.ylabel('Error') + plt.title('Error for %d Epochs for %d Dimensions' % (len(dim_error[0])-1, len(dim_error))) + plt.grid() + plt.legend(loc="upper left", bbox_to_anchor=[0, 1], ncol=2, shadow=True, title="Dimension", fancybox=True) + fig.savefig('Error.png', bbox_inches='tight') + plt.close(fig) + + +def PlotConvergence(vals): + fig = plt.figure() + subplt = plt.subplot(111) + x = np.arange(0,len(vals)) + subplt.plot(x, np.array(vals)) + plt.xlabel('Iteration') + plt.ylabel('Convergencing Particles') + i = len(vals) - 1 + plt.title('Pecentange of Converging Particles over %d Epochs' % i) + plt.grid() + fig.savefig('PercentConvergence.png', bbox_inches='tight') + plt.close(fig) + + +def FindPercentConverge(swarm, best): + filtered = filter(lambda x: Distance(best, x)<.5, swarm) + return 100*((1.0*len(filtered)/len(swarm))) + + +def main(): + parser = setup_argparser() + args = parser.parse_args() + experiment_number = args.experiment_number + + print args.Q + if str(args.Q) == 'Problem1': + args.Q = Problem1 + elif str(args.Q) == 'Problem2': + args.Q = Problem2 + else: + print 'Incorrectly specified argument to Q (fitness function). Must be Problem1 or Problem2.' + sys.exit(1) + + print args.Q + + # Setup directories for storing results + if not os.path.exists('results'): + os.makedirs('results') + os.chdir('results') + if not os.path.exists('data'): + os.makedirs('data') + os.chdir('data') + if not os.path.exists('Experiment-' + str(experiment_number)): + os.makedirs('Experiment-' + str(experiment_number)) + os.chdir('Experiment-' + str(experiment_number)) + if (len(args.dim)) == 2: + if not os.path.exists('Scatter Plots'): + os.makedirs('Scatter Plots') + args_dict = vars(args) + print args.dim + args.dim=tuple(map(lambda x: x+x%2, args.dim)) #adjust dimensions so that they are even. + logger = setup_logger(os.getcwd(), "__main__", "main") + logger.info("Program Arguments:") + for key, value in args_dict.iteritems(): + logger.info("%s=%s" % (str(key), str(value))) + + #####RUN PSO########## + logger.info("\n\n###################################RUNNING EXPERIMENT NUM " + + "%s#########################", str(experiment_number)) + pso = PSO(npart = args.npart, inertia=args.inertia, phi = args.phi, dimensions = args.dim , maxvelocity=args.maxVel, Q = args.Q, inertiaPrime = args.inerPrime) + if args.Euclid is True: + pso.setEuclidianNeigbors(args.k) + else: + pso.setClosestNeighbors(args.k) + + error_plot = [[] for i in range(len(args.dim))] + convergence_plot =[] + ###Run iieratons### + for i in range(args.maxIter+1): + cont = False + logger.info("EPOCH %d:" % i) + logger.info("Error:") + error = pso.getError() + logger.info(error) + #add error dimensions error to the plot + for n, plot in zip(error, error_plot): + plot.append(n) + if n>args.minError: #if all values for every dimension is below min error, we can break + cont = True + #add percent convergence to plot list: + convergence_plot.append(FindPercentConverge(pso.swarm, pso.globalBest)) + #if swarm is 2D make a Scatter Plot + if len(args.dim) == 2: + logger.info("Creating scatter plot for iteration") + PlotScatter(pso.swarm, args.dim, pso.globalBest, i) + logger.info("Finished created scatter plot") + #if we've converged to within a minimum error, break + if cont is False: + logger.info("Errors are below the threshold. Exiting now...") + break + #otherwise update and begin next iteration: + logger.info("Updating PSO:") + pso.Update() + logger.info("\n\n######################ANALYZING DATA######################:") + + logger.info("Plotting error") + PlotError(error_plot) + logger.info("Finished plotting error") + logger.info("Plotting percentage convergence") + PlotConvergence(convergence_plot) + logger.info("Finished plotting percentage convergence") + + logger.info("Experiment complete") +if __name__ == "__main__": + main()