diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d81d41e --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.coverage +*.pyc \ No newline at end of file diff --git a/README b/README index 5590741..70bda44 100644 --- a/README +++ b/README @@ -1,3 +1,5 @@ -This is the new official Pyevolve repository. +This is the fork of pyevolve framework. -The documentation (html rendered) is still hosted at sourceforge.net at http://pyevolve.sourceforge.net/0_6rc1/ +The main puprose is to add a Cartesian Genetic Programming (CGP) genome to the evolution schema. + +Also some fixes and reformation added. diff --git a/examples/data/input.jpg b/examples/data/input.jpg new file mode 100644 index 0000000..7f51549 Binary files /dev/null and b/examples/data/input.jpg differ diff --git a/examples/data/target.jpg b/examples/data/target.jpg new file mode 100644 index 0000000..3900b67 Binary files /dev/null and b/examples/data/target.jpg differ diff --git a/examples/pyevolve_ex23_cgp.py b/examples/pyevolve_ex23_cgp.py new file mode 100644 index 0000000..9c22706 --- /dev/null +++ b/examples/pyevolve_ex23_cgp.py @@ -0,0 +1,162 @@ +from pyevolve import G2DCartesian, GSimpleGA, Consts, Util, Mutators +import random +import os +from datetime import datetime + +PIL_SUPPORT = None +PYDOT_SUPPORT = None + +try: + import numpy as np +except: + raise ImportError("This example needs numpy module.") + +try: + from PIL import Image, ImageDraw, ImageFilter + PIL_SUPPORT = True +except: + PIL_SUPPORT = False + +try: + import pydot + PYDOT_SUPPORT = True +except: + PYDOT_SUPPORT = False + +DIRECTORY = "unities" +INPUT = "./data/input.jpg" +TARGET = "./data/target.jpg" +IMG_WIDTH=608 +IMG_HEIGHT=300 + +def gp_blur(src, params): + return src.filter(ImageFilter.BLUR) + +def gp_contour(src, params): + return src.filter(ImageFilter.CONTOUR) + +def gp_detail(src, params): + return src.filter(ImageFilter.DETAIL) + +def gp_edge_enhance(src, params): + return src.filter(ImageFilter.EDGE_ENHANCE) + +def gp_edge_enhance_more(src, params): + return src.filter(ImageFilter.EDGE_ENHANCE_MORE) + +def gp_emboss(src, params): + return src.filter(ImageFilter.EMBOSS) + +def gp_find_edges(src, params): + return src.filter(ImageFilter.FIND_EDGES) + +def gp_smooth(src, params): + return src.filter(ImageFilter.SMOOTH) + +def gp_smooth_more(src, params): + return src.filter(ImageFilter.SMOOTH_MORE) + +def gp_sharpen(src, params): + return src.filter(ImageFilter.SHARPEN) + +def gp_gaussian_blur(src, params): + return src.filter(ImageFilter.GaussianBlur(params['pos_int'])) + +def gp_unsharp_mask(src, params): + return src.filter(ImageFilter.UnsharpMask(params['pos_int'], + int(params['percent_int']), + int(params['pos_float']))) + +def gp_kernel(src, params): + size = params['kernel_size'][0] * params['kernel_size'][1] + kernel = params['kernel'][:size] + return src.filter(ImageFilter.Kernel(params['kernel_size'], + kernel, + params['pos_float'])) + +def gp_rank_filter(src, params): + rank = int(params['img_size_val'] * params['kernel_size_rad'] + * params['kernel_size_rad']) + return src.filter(ImageFilter.RankFilter(params['kernel_size_rad'], + rank)) + +def gp_mode_filter(src, params): + return src.filter(ImageFilter.ModeFilter(params['pos_int'])) + +def eval_fitness_mean_diff(chromosome): + rmse_accum = Util.VectorErrorAccumulator() + code = chromosome.getCompiledCode() + evaluated = np.array(eval(code[0])) + rmse_accum.append(evaluated, target) + return rmse_accum.getRMSE() + +def store_result(genome, filename): + if PIL_SUPPORT: + code = genome.getCompiledCode() + ev = eval(code[0]) + if not os.path.exists(DIRECTORY): + os.makedirs(DIRECTORY) + + ev.save(os.path.join(DIRECTORY, filename)) + +def store_graph(genome, filename): + if PYDOT_SUPPORT: + graph = pydot.Dot(graph_type='graph') + genome.writeDotGraph(graph) + graph.write_png(os.path.join(DIRECTORY, filename)) + +def step_callback(engine): + step = engine.getCurrentGeneration() + if step % 100 == 0: + best = engine.bestIndividual() + store_result(best, "%s_%s.png" % (step, best.score)) + store_graph(best, "%s_%s_graph.png" % (step, best.score)) + +def main(): + pos_int = """ rand_randint(1,17) """ + pos_float = """ rand_uniform(0.1, 16.0) """ + percent_int = """ rand_gauss(100, 40) """ + img_size_val = """ rand_uniform(0, 1) """ + kernel_size = """ rand_choice([(3, 3), (5, 5)]) """ + kernel = """ [rand_uniform(0.1, 2.5) for x in range(0,25) ] """ + kernel_size_rad = """ rand_choice([3, 5]) """ + + random.seed(13) + + global im, target, tfft + orig = Image.open(TARGET) + target = np.array(orig) + im = Image.open(INPUT) + im.load() + genome = G2DCartesian.G2DCartesian(128, 3, 1, 1) + genome.evaluator += eval_fitness_mean_diff + ga = GSimpleGA.GSimpleGA(genome, seed=13) + genome.mutator.set(Mutators.G2DCartesianMutatorNodeParams) + genome.mutator.add(Mutators.G2DCartesianMutatorNodeInputs) + genome.mutator.add(Mutators.G2DCartesianMutatorNodeFunction) + genome.mutator.add(Mutators.G2DCartesianMutatorNodesOrder) + ga.setPopulationSize(8) + ga.setGenerations(200) + #ga.setMultiThreading(True) + ga.setMinimax(Consts.minimaxType["minimize"]) + ga.setParams(gp_function_prefix = "gp", gp_terminals = ['im'], + gp_args_mapping = { "pos_int" : pos_int, + "pos_float" : pos_float, + "percent_int" : percent_int, + "img_size_val" : img_size_val, + "kernel_size" : kernel_size, + "kernel" : kernel, + "kernel_size_rad" : kernel_size_rad}) + ga.setMutationRate(0.05) + ga.setElitism(True) + ga.setSortType(Consts.sortType["raw"]) + ga.stepCallback.set(step_callback) + + ga(freq_stats=100) + end = datetime.now() + best = ga.bestIndividual() + store_result(best, "best_%s.png" % (best.score)) + store_graph(best, "best_graph_%s.png" % (best.score)) + +if __name__ == "__main__": + main() diff --git a/examples/test.py b/examples/test.py new file mode 100644 index 0000000..02d279a --- /dev/null +++ b/examples/test.py @@ -0,0 +1,55 @@ +from pyevolve import Util +from PIL import Image +import numpy as np +import random +import pywt +import math + +IMG_WIDTH=608 +IMG_HEIGHT=300 + +w=Image.new("L", (4,4), "white") +pix=w.load() + +#for i in range(0,4): +# for j in range(0,4): +# pix[i,j] = (random.randint(0,255), random.randint(0,255), random.randint(0,255)) + +b=Image.new("L", (4,4), "black") +b=np.array(b) +orig = Image.open("./data/est.png") +target = np.array(orig) +input = np.array(Image.open("./data/target.jpg")) +tfft = np.fft.fft2(target) +rmse_accum = Util.VectorErrorAccumulator() + + + +efft = np.fft.fft2(input) +#print abs(efft/len(efft)) +#print abs(tfft/len(tfft)) +rmse_accum.append(abs(tfft/len(tfft)), abs(efft/len(efft))) +print rmse_accum.getMean()/255 + +rmse_accum.reset() +rmse_accum.append(target, input) +print rmse_accum.getRMSE() +score = 0 +for i in range(0,3): + ev = input[:,:,i] + hist = np.histogram2d(ev.ravel(), target[:,:,i].ravel(), [x for x in xrange(0, 256)])[0] + nonzeroInd = np.nonzero(hist) + nonzero = hist[nonzeroInd] + histProb = nonzero/float(608*300) + score += -np.sum(np.log2(histProb)*histProb) +max_entropy = 3*math.log(IMG_WIDTH*IMG_HEIGHT, 2) +print score/max_entropy + +#cA, (cH, cV, cD) = pywt.dwt2(target, 'haar') +#print cA +#print cD +#cA, (cH, cV, cD) = pywt.dwt2(b, 'haar') +#print cA +#print cD +#print pywt.dwt2(b, 'haar') + \ No newline at end of file diff --git a/pyevolve/Consts.py b/pyevolve/Consts.py index f1442cd..428470d 100644 --- a/pyevolve/Consts.py +++ b/pyevolve/Consts.py @@ -373,6 +373,7 @@ import Crossovers import logging from GTree import GTreeGP +from G2DCartesian import G2DCartesian # Required python version 2.5+ CDefPythonRequire = (2, 5) @@ -491,6 +492,11 @@ CDefG2DListInit = Initializators.G2DListInitializatorInteger CDefG2DListCrossUniformProb = 0.5 +# - G2DCartesian defaults +CDefG2DCartesianInit = Initializators.G2DCartesianInitializatorNode +CDefG2DCartesianCrossover = Crossovers.G2DCartesianCrossoverNode +CDefG2DCartesianMutator = Mutators.G2DCartesianMutatorNodeFunction + # Gaussian Gradient CDefGaussianGradientMU = 1.0 CDefGaussianGradientSIGMA = (1.0 / 3.0) # approx. +/- 3-sigma is +/- 10% @@ -525,7 +531,7 @@ CDefBroadcastAddress = "255.255.255.255" nodeType = {"TERMINAL": 0, "NONTERMINAL": 1} -CDefGPGenomes = [GTreeGP] +CDefGPGenomes = [GTreeGP, G2DCartesian] # Migration Consts CDefGenMigrationRate = 20 diff --git a/pyevolve/Crossovers.py b/pyevolve/Crossovers.py index 7584a00..3c0f264 100644 --- a/pyevolve/Crossovers.py +++ b/pyevolve/Crossovers.py @@ -798,3 +798,10 @@ def GTreeGPCrossoverSinglePoint(genome, **args): assert brother.getHeight() <= max_depth return (sister, brother) + +############################################################################# +################# G2DCartesian Crossovers ################################# +############################################################################# + +def G2DCartesianCrossoverNode(genome, **args): + return (args["mom"].clone(), args["dad"].clone()) \ No newline at end of file diff --git a/pyevolve/G2DCartesian.py b/pyevolve/G2DCartesian.py new file mode 100644 index 0000000..a59226c --- /dev/null +++ b/pyevolve/G2DCartesian.py @@ -0,0 +1,342 @@ +""" +:mod:`G2DCartesian` -- the 2D cartesian net chromosome +================================================================ +This is the 2D Cartesian Net representation for Cartesian Genetic Programming +(CGP), this net consist of nodes which hold function or terminals (like in +Genetic Programming). +This chromosome class extends the :class:`GenomeBase.GenomeBase`. +Default Parameters +------------------------------------------------------------- +*Initializator* + :func:`Initializators.G2DCartesianInitializatorNode` + The Node Initializator for G2DCartesian +*Mutator* + :func:`Mutators.G2DCartesianMutatorNodeInputs` + :func:`Mutators.G2DCartesianMutatorNodeParams` + :func:`Mutators.G2DCartesianMutatorNodeFunction` + :func:`Mutators.G2DCartesianMutatorNodesOrder` + The Mutators for G2DCartesian +*Crossover* + :func:`Crossovers.G2DCartesianCrossoverNode` + The Crossover for G2DCartesian +Class +------------------------------------------------------------- +""" + +from GenomeBase import GenomeBase +import Consts +from random import randint as rand_randint, choice as rand_choice +from random import uniform as rand_uniform, gauss as rand_gauss +import pydot +import copy + +class G2DCartesian(GenomeBase): + + """ G2DCartesian Class - The 2D Cartesian Net chromosome representation + Inheritance diagram for :class:`G2DCartesian.G2DCartesian`: + .. inheritance-diagram:: G2DCartesian.G2DCartesian + **Examples** + The instantiation + >>> genome1 = G2DCartesian.G2DCartesian(2, 3, 4, 2) + Compare + >>> genome2 = genome1.clone() + >>> genome2 == genome1 + True + Size, slice, get/set, append + >>> len(genome) + 12 + >>> genome + (...) + [None, None, None, None, None, None, None, None, None, None, None, + None] + >>> genome[1] = 2 + >>> genome + (...) + [None, 2, None, None, None, None, None, None, None, None, None, None] + >>> genome[1] + 2 + :param rows: the number of rows in the net + :param cols: the number of columns in the net + :param inputs: the number of inputs nodes + :param outputs: the number of output nodes + """ + + __slots__ = ["inputs", "outputs", "cols", "rows", "internalNodes", + "inputSlice", "internalSlice", "outputSlice", "nodes", + "reevaluate", "expressionNodes"] + + def __init__(self, rows, cols, inputs, outputs, cloning = False): + """ The initializator of G2DCartesian representation, + rows, cols, inputs and outputs must be specified and none of them can + equal to 0""" + if (rows * cols * inputs * outputs) == 0: + raise ValueError("One of the genome parameter equals 0.") + + super(G2DCartesian, self).__init__() + self.rows = rows + self.cols = cols + self.inputs = inputs + self.outputs = outputs + self.internalNodes = rows*cols + self.nodes = [None]*(rows*cols+outputs+inputs) + self.inputSlice = slice(0,self.inputs) + self.internalSlice = slice(self.inputs, self.inputs+self.internalNodes) + self.outputSlice = slice(self.inputs+self.internalNodes, + self.inputs+self.internalNodes+self.outputs) + self.expressionNodes = {} + self.reevaluate = True + + if not cloning: + self.initializator.set(Consts.CDefG2DCartesianInit) + self.mutator.set(Consts.CDefG2DCartesianMutator) + self.crossover.set(Consts.CDefG2DCartesianCrossover) + + def __eq__(self, other): + """ Compares one chromosome with another """ + cond1 = (self.nodes == other.nodes) + cond2 = (self.rows == other.rows) + cond3 = (self.cols == other.cols) + return True if cond1 and cond2 and cond3 else False + + def __getitem__(self, key): + """ Return the specified node of net (including inputs and outputs)""" + return self.nodes[key] + + def __iter__(self): + """ Iterator support to the nodes """ + return iter(self.nodes) + + def __len__(self): + """ Return the number of all nodes in net """ + return len(self.nodes) + + def __repr__(self): + """ Return a string representation of Genome """ + ret = GenomeBase.__repr__(self) + ret += "- G2DCartesian\n" + ret += "\tList size:\t %s\n" % (len(self.nodes,)) + ret += "\tList:\t\t %s\n\n" % (self.nodes,) + return ret + + def __setitem__(self, key, value): + """ Set the specified node in net """ + self.nodes[key] = value + + def clone(self): + """ Return a new instace copy of the genome + + :rtype: the G2DCartesian clone instance + + """ + newcopy = G2DCartesian(self.rows, self.cols, self.inputs, self.outputs, + True) + self.copy(newcopy) + return newcopy + + def copy(self, g): + """ Copy genome to 'g' + + Example: + >>> genome_origin.copy(genome_destination) + + :param g: the destination G2DCartesian instance + + """ + GenomeBase.copy(self, g) + g.expressionNodes = self.expressionNodes.copy() + g.reevaluate = self.reevaluate + g.nodes = copy.deepcopy(self.nodes) + + def evaluate(self, **args): + """ Overloaded method of GenomeBase. It is performance improvement, + genome score is evaluated only when mutations had influence on nodes + active in the expression path """ + if self.reevaluate: + super(G2DCartesian, self).evaluate(**args) + self.expressionNodes.clear() + for idx, path in enumerate(self.getActiveNodes()): + for node in path: + self.expressionNodes[(node.x, node.y)] = True + out = self.nodes[-idx-1] + self.expressionNodes[(out.x, out.y)] = True + + def getActiveNodes(self): + """ Return list of lists with active paths in net, the size of list + depends on the number of net outputs. It populates list in reverse + direction, from output to input """ + actives = [] + for i in xrange(0, self.outputs): + actives.append([]) + self.nodes[-i-1].getPreviousNodes(actives[i]) + return actives + + def getCompiledCode(self): + """ Returns list of expressions from the genome, size of list depends on + outputs count. Expressions are already a python compile object """ + expr = [None] * self.outputs + for i in xrange(0, self.outputs): + expr[i] = self.nodes[-i-1].getExpression("") + compiled = [] + for e in expr: + compiled.append(compile(e, "", "eval")) + return compiled + + def mutate(self, **args): + """ Overloaded method of GenomeBase. Mutators of G2DCartesian return + list of mutated nodes and if any of them is on the list of active + expression nodes then genome flag to reevaluate is set """ + self.reevaluate = False + mutated_nodes = [] + for it in self.mutator.applyFunctions(self, **args): + mutated_nodes += it + + for node in mutated_nodes: + if self.expressionNodes.has_key((node.x, node.y)): + self.reevaluate = True + return len(mutated_nodes) + + return len(mutated_nodes) + + def writeDotGraph(self, graph): + """ Populates graph for pydot from active expression of genome """ + node_counter = 0 + for out in xrange(0, self.outputs): + node_stack = [] + node_dict = {} + node_stack.append(self.nodes[-1-out]) + while node_stack: + current_node = node_stack.pop() + if current_node.data != "": + node_label = current_node.data + for param in current_node.params.keys(): + node_label += " " + str(current_node.params[param]) + graph.add_node(pydot.Node(str(node_counter), + label = node_label)) + if current_node in node_dict: + graph.add_edge(pydot.Edge(node_dict[current_node], + str(node_counter))) + + for input in current_node.inputs: + node_stack.append(input) + if current_node.data != "": + node_dict[input] = str(node_counter) + if current_node.data != "": + node_counter += 1 + +class CartesianNode(): + + """ CartesianNode Class - The Cartesian Node representation + **Examples** + The instantiation + >>> node = G2DCartesian.CartesianNode(1, 0, + {"func3" : 3}, [self.prev1, self.prev2]) + + Very important thing for future developer is passing correct values for + data_set and previous_nodes. They are slightly different for input, + internal and output nodes in Cartesian Genetic Programming. + + Input node: + >>> node = G2DCartesian.CartesianNode(1, 1, {"b" : 1}, []) + No previous nodes and every function in data_set must have not more than + one argument (for 'args', in fact it is zero arguments). + Internal node: + >>> node = G2DCartesian.CartesianNode(1, 1, {"f1" : 3}, + [self.prev1, self.prev2]) + It should have previous_nodes (at least input nodes!) and there are no + constraints on functions dictionary and their inputs. + Output node: + >>> node = G2DCartesian.CartesianNode(1, 1, {}, + [self.prev1, self.prev2]) + No functions passed to output nodes, they just map net to the world. In + previous nodes they should have all existing nodes except those being + outputs. + + :param position_x: row position in the net + :param position_y: column position in the net + :param data_set: dictionary of functions which can be assigned to this node + and number of their args + :param previous_nodes: list of nodes which can be connected as inputs to + this node + """ + + paramMapping = {} + + def __init__(self, position_x, position_y, data_set = {}, + previous_nodes = []): + """ The initializator of CartesianNode representation, + position_x and position_y must be specified, data_set and previous_nodes + depends on type of the node """ + self.data = None + self.inputs = [] + self.params = {} + self.x = position_x + self.y = position_y + + try: + self.data = rand_choice(data_set.keys()) + except IndexError: + self.data = "" + + try: + inputs_count = data_set[self.data]-1 + except KeyError: + inputs_count = 1 + + if (len(previous_nodes) == 0 and inputs_count > 0): + raise ValueError("Bad data set and previous nodes values " + "combination. If specified data set with input args" + " then previous nodes can not be empty!") + + for i in range(0, inputs_count): + self.inputs.append(rand_choice(previous_nodes)) + + for param in CartesianNode.paramMapping.keys(): + self.params[param] = eval(CartesianNode.paramMapping[param]) + + def __repr__(self): + """ Return a string representation of Genome """ + ret = "\n\tCartesianNode [%s, %s] - " % (self.x, self.y) + ret += "Data: %s" % (self.data) + for i in self.inputs: + ret += " Input: [%s, %s]" % (i.x, i.y) + return ret + + def getData(self): + """ Return tuple with node value and number of its input args """ + return (self.data, len(self.inputs)) + + def getExpression(self, expr): + """ Recursively iterates through input nodes of current node and return + merged expression (function in the node and params) in string format """ + if self.data is not "": + expr += self.data + if self.inputs: + expr += "( " + input_counter = 0 + for idx, input in enumerate(self.inputs): + expr += input.getExpression("") + if idx < len(self.inputs)-1: + expr += ", " + + expr += ", {" + for idx, param in enumerate(self.params.keys()): + expr += "\"" + param + "\"" + " : " + expr += str(self.params[param]) + if idx < len(self.params)-1: + expr += ", " + expr += "} )" + else: + expr += self.inputs[0].getExpression("") + return expr + + def getPreviousNodes(self, nodes): + """ Recursively returns previous, connected nodes in net of this node""" + if len(self.inputs) == 0: + return + elif self.data is not "": + nodes.append(self) + + for i in self.inputs: + i.getPreviousNodes(nodes) + + diff --git a/pyevolve/GPopulation.py b/pyevolve/GPopulation.py index 919ad18..8436996 100644 --- a/pyevolve/GPopulation.py +++ b/pyevolve/GPopulation.py @@ -47,6 +47,15 @@ except ImportError: MULTI_PROCESSING = False logging.debug("You don't have multiprocessing support for your Python version !") + +try: + from threading import Thread + CPU_COUNT = cpu_count() + MULTI_THREADING = True if CPU_COUNT > 1 else False + logging.debug("You have %d CPU cores, so the threading state is %s", CPU_COUNT, MULTI_PROCESSING) +except ImportError: + MULTI_THREADING = False + logging.debug("You don't have threading support for your Python version !") def key_raw_score(individual): @@ -134,6 +143,7 @@ def __init__(self, genome): self.internalParams = genome.internalParams self.multiProcessing = genome.multiProcessing + self.multiThreading = genome.multiThreading self.statted = False self.stats = Statistics() @@ -153,6 +163,7 @@ def __init__(self, genome): self.internalParams = {} self.multiProcessing = (False, False, None) + self.multiThreading = (False, None) # Statistics self.statted = False @@ -179,6 +190,25 @@ def setMultiProcessing(self, flag=True, full_copy=False, max_processes=None): """ self.multiProcessing = (flag, full_copy, max_processes) + + def setMultiThreading(self, flag=True, max_threads=None): + """ Sets the flag to enable/disable the use of python multithreading module. + Use this option when you have more than one core on your CPU and when your + evaluation function is very slow and can use the same non mutuable target + to evaluate fitness (e.g. images). Threading seems to be more efficient + than processing, there is no need of copying resources. Thread just points + the method and data in existing memory to execute but remember that + threads share memory space. + + :param flag: True (default) or False + :param max_threads: None (default) or an integer value + + .. versionadded:: + The `setMultiThreading` method. + + """ + + self.multiThreading = (flag, max_threads) def setMinimax(self, minimax): """ Sets the population minimax @@ -383,8 +413,31 @@ def evaluate(self, **args): :param args: this params are passed to the evaluation function """ + # We have multithreading + if self.multiThreading[0] and MULTI_THREADING: + logging.debug("Evaluating the population using the" + "multithreading method") + current_threads = {} + if self.multiThreading[1] == None: + spawn_size = len(self) + else: + spawn_size = self.multiThreading[1] + + for counter in xrange(0, len(self), spawn_size): + current_threads.clear() + is_overflow = (counter + spawn_size) > len(self) + overflow_size = is_overflow * (spawn_size+counter - len(self)) + start = counter + end = counter + spawn_size - overflow_size + for ind, idx in zip(self.internalPop[start:end], + xrange(0, spawn_size-overflow_size)): + current_threads[idx] = Thread(target=ind.evaluate, args=()) + + [thread.start() for thread in current_threads.values()] + [thread.join() for thread in current_threads.values()] + # We have multiprocessing - if self.multiProcessing[0] and MULTI_PROCESSING: + elif self.multiProcessing[0] and MULTI_PROCESSING: logging.debug("Evaluating the population using the multiprocessing method") proc_pool = Pool(processes=self.multiProcessing[2]) @@ -451,6 +504,7 @@ def copy(self, pop): pop.scaleMethod = self.scaleMethod pop.internalParams = self.internalParams pop.multiProcessing = self.multiProcessing + pop.multiThreading = self.multiThreading def getParam(self, key, nvl=None): """ Gets an internal parameter diff --git a/pyevolve/GSimpleGA.py b/pyevolve/GSimpleGA.py index 44775d5..a0d451f 100644 --- a/pyevolve/GSimpleGA.py +++ b/pyevolve/GSimpleGA.py @@ -427,6 +427,41 @@ def setMultiProcessing(self, flag=True, full_copy=False, max_processes=None): self.internalPop.setMultiProcessing(flag, full_copy, max_processes) + def setMultiThreading(self, flag=True, max_threads=None): + """ Sets the flag to enable/disable the use of python multithreading module. + Use this option when you have more than one core on your CPU and when your + evaluation function is very slow. + + Pyevolve will automaticly check if your Python version has **multithreading** + support and if you have more than one single CPU core. If you don't have support + or have just only one core, Pyevolve will not use the **multithreading** + feature. + + Pyevolve uses the **multithreading** to execute the evaluation function over + the individuals, so the use of this feature will make sense if you have a + truly slow evaluation function (which is commom in GAs). + + Multithreading in general is better than multiprocessing when target data + to compare is big and copying it during process initialization is time + expensive. + + :param flag: True (default) or False + :param max_threads: None (default) or an integer value + + .. warning:: Use this option only when your evaluation function is slow, so you'll + get a good tradeoff between the process communication speed and the + parallel evaluation. The use of the **multithreading** doesn't means + always a better performance. + + .. versionadded:: + The `setMultiThreading` method. + + """ + if type(flag) != BooleanType: + Util.raiseException("Threading option must be True or False", TypeError) + + self.internalPop.setMultiThreading(flag, max_threads) + def setMigrationAdapter(self, migration_adapter=None): """ Sets the Migration Adapter @@ -682,6 +717,7 @@ def step(self): logging.debug("Evaluating the new created population.") newPop.evaluate() + newPop.sort() if self.elitism: logging.debug("Doing elitism.") @@ -689,13 +725,13 @@ def step(self): for i in xrange(self.nElitismReplacement): #re-evaluate before being sure this is the best self.internalPop.bestRaw(i).evaluate() - if self.internalPop.bestRaw(i).score > newPop.bestRaw(i).score: + if self.internalPop.bestRaw(i).score > newPop[len(newPop) - 1 - i].score: newPop[len(newPop) - 1 - i] = self.internalPop.bestRaw(i) elif self.getMinimax() == Consts.minimaxType["minimize"]: for i in xrange(self.nElitismReplacement): #re-evaluate before being sure this is the best self.internalPop.bestRaw(i).evaluate() - if self.internalPop.bestRaw(i).score < newPop.bestRaw(i).score: + if self.internalPop.bestRaw(i).score < newPop[len(newPop) - 1 - i].score: newPop[len(newPop) - 1 - i] = self.internalPop.bestRaw(i) self.internalPop = newPop diff --git a/pyevolve/Initializators.py b/pyevolve/Initializators.py index ff99c8b..f20a00a 100644 --- a/pyevolve/Initializators.py +++ b/pyevolve/Initializators.py @@ -14,8 +14,9 @@ """ -from random import randint as rand_randint, uniform as rand_uniform, choice as rand_choice +from random import randint as rand_randint, uniform as rand_uniform, choice as rand_choice, gauss as rand_gauss import GTree +import G2DCartesian import Util @@ -272,3 +273,78 @@ def GTreeGPInitializator(genome, **args): genome.setRoot(root) genome.processNodes() assert genome.getHeight() <= max_depth + +#################### +## Cartesian GP ## +#################### + +def G2DCartesianInitializatorNode(genome, **args): + """This initializator is for Cartesian Genetic Programming, it uses three + types of "slicers" from genome: inputs, internals and outputs. Every + input get single terminal from engine, internals get a set of possible + functions, their parameters mapping set and previous available nodes + outputs get just previous available nodes. From ga_engine is uses: + + *gp_function_set* + Dict of functions and their arguments counter founded automatically + after defining function prefix in ga_engine. + + *gp_terminals* + List of terminals passed to ga_engine. + + *gp_args_mapping* + Dict of parameters for node with value being a str generating value for + them via eval(), example: + {"param1" : "random.randint(0,10)"} + uses for parameter 'param1' random integer generator. + + .. versionadded:: + The *G2DCartesianInitializatorNode* function. + """ + + if not isinstance(genome, G2DCartesian.G2DCartesian): + raise TypeError("Specified genome unsuitable for this Initializator.") + + ga_engine = args["ga_engine"] + inputs = genome.inputs + outputs = genome.outputs + rows = genome.rows + cols = genome.cols + inputSlice = genome.inputSlice + internalSlice = genome.internalSlice + outputSlice = genome.outputSlice + terminals = ga_engine.getParam("gp_terminals") + functions_set = ga_engine.getParam("gp_function_set") + args_mapping = ga_engine.getParam("gp_args_mapping") + + if terminals is None: + raise AssertionError("Empty terminal set.") + if functions_set is None: + raise AssertionError("Empty function set.") + if args_mapping is None: + raise AssertionError("Empty argument mapping set.") + if not len(terminals) == inputs: + raise AssertionError("Terminal set must be equal with input length.") + + G2DCartesian.CartesianNode.paramMapping = args_mapping + + nodes = [] + for counter, terminal in enumerate(terminals): + nodes.append(G2DCartesian.CartesianNode(-1, counter, {terminal : 1})) + genome[inputSlice] = nodes + + previous_nodes = genome[0:inputs] + nodes = [] + for counter in xrange(0, rows * cols): + nodes.append(G2DCartesian.CartesianNode(counter / cols, counter % cols, + functions_set, previous_nodes)) + start = (counter / cols) * cols + end = (start+cols)*((counter % cols) == (cols - 1)) + previous_nodes += nodes[start:end] + genome[internalSlice] = nodes + + nodes = [] + for counter in xrange(0, outputs): + nodes.append(G2DCartesian.CartesianNode(rows, counter, {}, + previous_nodes)) + genome[outputSlice] = nodes diff --git a/pyevolve/Migration.py b/pyevolve/Migration.py index 3f566c9..07bd88f 100644 --- a/pyevolve/Migration.py +++ b/pyevolve/Migration.py @@ -263,6 +263,8 @@ def exchange(self): if len(pool) <= 0: break choice = rand_choice(pool) + # reevaluate choice, sometimes we use different methods on islands + choice[2].evaluate() pool.remove(choice) # replace the worst @@ -333,6 +335,8 @@ def exchange(self): break choice = rand_choice(pool) + # reevaluate choice, sometimes we use different methods on islands + choice[2].evaluate() pool.remove(choice) # replace the worst diff --git a/pyevolve/Mutators.py b/pyevolve/Mutators.py index 3c757df..87ff94a 100644 --- a/pyevolve/Mutators.py +++ b/pyevolve/Mutators.py @@ -9,9 +9,10 @@ import Util from random import randint as rand_randint, gauss as rand_gauss, uniform as rand_uniform -from random import choice as rand_choice +from random import choice as rand_choice, shuffle as rand_shuffle, gauss as rand_gauss import Consts import GTree +from G2DCartesian import CartesianNode ############################# ## 1D Binary String ## @@ -1132,3 +1133,122 @@ def GTreeGPMutatorSubtree(genome, **args): genome.processNodes() return int(mutations) + +################### +## Cartesian GP ## +################### + +def G2DCartesianMutatorNodeInputs(genome, **args): + """ The mutator of G2DCartesian, Node inputs mutator + + This mutator will change inputs of node using available previous nodes. + + .. versionadded:: + The *G2DCartesianMutatorNodeInputs* function + """ + mutations = args["pmut"] * (genome.rows * genome.cols + genome.outputs + + genome.inputs) + if mutations < 1.0: + mutations = 1 + + mutated = [] + for i in xrange(0, int(mutations)): + choosen = rand_choice(genome.nodes[genome.inputs:]) + mutated.append(choosen) + previous_nodes = genome.nodes[:(genome.cols * choosen.x + + genome.inputs)] + for idx, input in enumerate(choosen.inputs): + choosen.inputs[idx] = rand_choice(previous_nodes) + + return mutated + + +def G2DCartesianMutatorNodeParams(genome, **args): + """ The mutator of G2DCartesian, Node params mutator + + This mutator will generate new values for parameters of node using + parameters mapping. + + .. versionadded:: + The *G2DCartesianMutatorNodeParams* function + """ + mutations = args["pmut"] * (genome.rows * genome.cols + genome.outputs + + genome.inputs) + if mutations < 1.0: + mutations = 1 + + mutated = [] + for i in xrange(0, int(mutations)): + choosen = rand_choice(genome.nodes[genome.inputs:-genome.outputs]) + mutated.append(choosen) + for key in choosen.params.keys(): + choosen.params[key] = eval(CartesianNode.paramMapping[key]) + + return mutated + +def G2DCartesianMutatorNodeFunction(genome, **args): + """ The mutator of G2DCartesian, Node value mutator + + This mutator will change value of node (function) using available function + set. + + .. versionadded:: + The *G2DCartesianMutatorNodeFunction* function + """ + mutations = args["pmut"] * (genome.rows * genome.cols + genome.outputs + + genome.inputs) + ga_engine = args["ga_engine"] + if mutations < 1.0: + mutations = 1 + + function_set = ga_engine.getParam("gp_function_set") + mutated = [] + for i in xrange(0, int(mutations)): + choosen = rand_choice(genome.nodes[genome.inputs:-genome.outputs]) + mutated.append(choosen) + choosen.data = rand_choice(function_set.keys()) + previous_nodes = genome.nodes[:(genome.cols * choosen.x + + genome.inputs)] + + inputs_diff = len(choosen.inputs) - (function_set[choosen.data]-1) + if inputs_diff > 0: + del choosen.inputs[-inputs_diff:] + elif inputs_diff < 0: + for i in xrange(0, -inputs_diff): + choosen.inputs.append(rand_choice(previous_nodes)) + + return mutated + +def G2DCartesianMutatorNodesOrder(genome, **args): + """ The mutator of G2DCartesian, Nodes order in active path mutator + + This mutator will recreate order of nodes in active path, preserving values + of nodes. It can also manipulate inputs of nodes if current inputs state does + not satisfy new node value args. + + .. versionadded:: + The *G2DCartesianMutatorNodesOrder* function + """ + paths = genome.getActiveNodes() + mutated = [] + for path in paths: + shuffled_functions = [] + for node in path: + shuffled_functions.append(node.getData()) + rand_shuffle(shuffled_functions) + + for idx, node in enumerate(path): + new_function = shuffled_functions[idx] + previous_nodes = genome.nodes[:(genome.cols * node.x + + genome.inputs)] + node.data = new_function[0] + inputs_diff = len(node.inputs) - (new_function[1]) + + if inputs_diff > 0: + del node.inputs[-inputs_diff:] + elif inputs_diff < 0: + for i in xrange(0, -inputs_diff): + node.inputs.append(rand_choice(previous_nodes)) + mutated.append(node) + + return mutated diff --git a/pyevolve/Network.py b/pyevolve/Network.py index 726c82e..e6f4477 100644 --- a/pyevolve/Network.py +++ b/pyevolve/Network.py @@ -232,8 +232,16 @@ def send(self, data): """ bytes = -1 for destination in self.target: - bytes = self.sock.sendto(data, destination) - return bytes + totalsent = 0 + data_copy = data[:] + data = format(len(data), "#06x") + data + to_sent = len(data) + while totalsent < to_sent: + bytes = self.sock.sendto(data, destination) + totalsent += bytes + data = data[bytes:] + data = data_copy[:] + return totalsent def run(self): """ Method called when you call *.start()* of the thread """ @@ -354,8 +362,17 @@ def getData(self): :rtype: tuple (sender ip, data) or None when timeout exception """ + received = 0 + length = 1 + data = "" try: - data, sender = self.sock.recvfrom(self.bufferSize) + while received < length: + buffer, sender = self.sock.recvfrom(self.bufferSize) + if received == 0: + length = int(buffer[:6], 16) + buffer = buffer[6:] + received += len(buffer) + data += buffer except socket.timeout: return None return (sender[0], data) diff --git a/pyevolve/Util.py b/pyevolve/Util.py index 4674797..88d56cd 100644 --- a/pyevolve/Util.py +++ b/pyevolve/Util.py @@ -180,8 +180,8 @@ def append(self, target, evaluated): def __iadd__(self, value): """ The same as append, but you must pass a tuple """ self.append(*value) - return self - + return self + def getMean(self): """ Return the mean of the non-squared accumulator """ return self.acc / self.acc_len @@ -214,6 +214,66 @@ def getMSE(self): """ return self.acc_square / float(self.acc_len) +try: + import numpy as np + + class VectorErrorAccumulator(ErrorAccumulator): + """ Vector version of error accumulator, it uses numpy and it is natural + choice for analysing signal data (e.g. images). Appending one by one - + result and target from fitness function, is slower than passing it via + numpy arrays. Numpy also offers some methods usuable for signal + comparing, especially in making confusion matrix. + """ + def __init__(self, confusion = True, non_zeros = True): + super(VectorErrorAccumulator, self).__init__() + self.non_zeros = 0 + self.confusion = np.zeros([2, 2]) + self.calculate_confusion = confusion + self.calculate_non_zeros = non_zeros + + def reset(self): + """ Reset the accumulator """ + super(VectorErrorAccumulator, self).reset() + self.confusion = np.zeros([2,2]) + self.non_zeros = 0 + + def append(self, target, evaluated): + """ Add array value to the accumulator + + :param target: the array target value + :param evaluated: the array evaluated value + """ + diff = target - evaluated + + if self.calculate_confusion: + targetBin = np.ravel(np.where(target > 0, 1, 0)) + evaluatedBin = np.ravel(np.where(evaluated > 0, 1, 0)) + flags=np.logical_not(np.bitwise_xor(targetBin,evaluatedBin)) + self.confusion = np.bincount(2 * flags + evaluatedBin, + minlength=4).reshape(2, 2).astype(np.float64) + + if self.calculate_non_zeros: + self.non_zeros = np.count_nonzero(diff) + + self.acc_square += np.sum((diff)**2) + self.acc += np.sum(np.absolute(diff)) + self.acc_len += target.size + + def getConfusionMatrix(self): + """ Return the confusion matrix of accumulator """ + return self.confusion + + def getNonZeros(self): + """ Return the counter of non zero values in the accumulator """ + return self.non_zeros + + def getZeros(self): + """ Return the counter of zero values in the accumulator """ + return self.acc_len - self.non_zeros + +except: + logging.info('No numpy module found, VectorErrorAccumulator class is ' + 'unavailable.') class Graph(object): """ The Graph class diff --git a/tests/test_crossovers.py b/tests/test_crossovers.py index 1f87a50..ca92258 100644 --- a/tests/test_crossovers.py +++ b/tests/test_crossovers.py @@ -10,6 +10,7 @@ from pyevolve.G2DBinaryString import G2DBinaryString from pyevolve.G2DList import G2DList from pyevolve.GTree import GTree, GTreeNode +from pyevolve.G2DCartesian import G2DCartesian class CrossoverTestCase(unittest.TestCase): @@ -390,3 +391,18 @@ def test_strict_single_point_crossover(self, rand_mock): assertion_name='assetTreesEqual', crossover_extra_kwargs={'count': 2} ) + +class G2DCartesianCrossoverTestCase(CrossoverTestCase): + def setUp(self): + self.mom = G2DCartesian(2, 2, 1, 1) + self.dad = G2DCartesian(2, 2, 1, 1) + + def test_crossover_node(self): + self.assertCrossoverResultsEqual( + Crossovers.G2DCartesianCrossoverNode, + self.mom, + self.dad, + None, + '' + ) + diff --git a/tests/test_genomes.py b/tests/test_genomes.py new file mode 100644 index 0000000..72fa048 --- /dev/null +++ b/tests/test_genomes.py @@ -0,0 +1,251 @@ +import unittest + +from mock import patch, MagicMock +from pyevolve.G2DCartesian import G2DCartesian, CartesianNode +import re +from random import choice, randint + +class G2DCartesianGenomeTestCase(unittest.TestCase): + def setUp(self): + self.rows = 4 + self.cols = 5 + self.ins = 1 + self.outs = 3 + self.genome = G2DCartesian(self.rows, self.cols, self.ins, self.outs) + + def tearDown(self): + self.genome = None + + def test_genome_init(self): + self.assertTrue(self.genome.rows == self.rows) + self.assertTrue(self.genome.cols == self.cols) + self.assertTrue(self.genome.inputs == self.ins) + self.assertTrue(self.genome.outputs == self.outs) + self.assertTrue(self.genome.internalNodes == self.rows * self.cols) + self.assertTrue(self.genome.inputSlice == slice(0,self.ins)) + self.assertTrue(self.genome.internalSlice == slice(self.ins, self.ins + + (self.rows * self.cols))) + self.assertTrue(self.genome.outputSlice == slice(self.ins + + (self.rows * self.cols), + self.ins + + (self.rows * self.cols) + + self.outs)) + + def test_genome_clone(self): + genomeClone = self.genome.clone() + self.assertTrue(genomeClone == self.genome) + + def test_genome_nodes(self): + self.assertTrue(len(self.genome) == self.rows * self.cols + self.ins + + self.outs) + self.assertTrue(self.genome[0] == None) + self.assertTrue(self.genome[self.rows * self.cols + self.ins + + self.outs - 1] == None) + self.assertRaises(IndexError, self.genome.__getitem__, + self.rows*self.cols*self.cols) + + def test_genome_zero_param(self): + self.assertRaises(ValueError, G2DCartesian, 0, 1, 2, 10) + + def test_genome_active_nodes(self): + for key, node in enumerate(self.genome): + mock = MagicMock() + inputs = [] + if key > 0: + for i in xrange(0, randint(1, 3)): + inputs.append(choice(self.genome[0:key])) + + mock.inputs = inputs + self.genome[key] = mock + self.genome[key].getPreviousNodes = MagicMock(return_value = inputs) + paths = self.genome.getActiveNodes() + self.assertTrue(len(paths) == 3) + + def test_genome_compiled_code(self): + from math import sin + expected_exprs = [] + values = [10, 50, 90] + def get_effect(arg): + expected_exprs.append("sin(" + str(values.pop(0)) + ")") + return expected_exprs[-1] + + for key, node in enumerate(self.genome): + mock = MagicMock() + mock.getExpression = MagicMock(side_effect = get_effect) + self.genome[key] = mock + + compiled = self.genome.getCompiledCode() + self.assertTrue(len(compiled) == self.outs) + for comp, expr in zip(compiled, expected_exprs): + self.assertEqual(eval(comp), eval(expr)) + + def test_genome_to_graph(self): + try: + import pydot + except ImportError: + return + + for key, node in enumerate(self.genome): + mock = MagicMock() + mock.params = {"p1" : 1} + if key >= self.rows * self.cols + self.ins: + mock.data = "" + else: + mock.data = str(key) + inputs = [] + if key > 0: + for i in xrange(0, randint(1, 3)): + inputs.append(choice(self.genome[0:key])) + + mock.inputs = inputs + self.genome[key] = mock + + graph = pydot.Dot(graph_type='graph') + self.genome.writeDotGraph(graph) + self.assertTrue(len(graph.get_node_list()) >= self.genome.outputs) + + def test_genome_mutate_expressed(self): + nodes = [] + for i in xrange(0, randint(2, 5)): + node = MagicMock() + node.x = randint(0, 7) + node.y = randint(0, 7) + self.genome.expressionNodes[(node.x, node.y)] = True + nodes.append(node) + + def apply_effect(arg, **args): + mutated = [] + mutated.append([]) + for i in xrange(0, len(self.genome.expressionNodes)): + mutated[0].append(choice(nodes)) + return mutated + + self.genome.mutator.applyFunctions = MagicMock( + side_effect = apply_effect) + ret = self.genome.mutate() + self.assertTrue(ret > 0) + self.assertTrue(self.genome.reevaluate) + + def test_genome_mutate_non_expressed(self): + def apply_effect(arg, **args): + mutated = [] + mutated.append([MagicMock(), MagicMock(), MagicMock()]) + return mutated + + self.genome.mutator.applyFunctions = MagicMock( + side_effect = apply_effect) + ret = self.genome.mutate() + self.assertTrue(ret > 0) + self.assertFalse(self.genome.reevaluate) + + def test_genome_evaluate_not(self): + expected_expression_nodes = {MagicMock() : True, MagicMock() : True, + MagicMock() : True} + self.genome.expressionNodes = expected_expression_nodes + self.genome.reevaluate = False + self.genome.evaluate() + self.assertEqual(expected_expression_nodes, self.genome.expressionNodes) + + def test_genome_evaluate_do(self): + nodes_pool = [] + coords_pool = [] + for i in xrange(0, 10): + node = MagicMock() + node.x = randint(0, 4) + node.y = randint(0, 4) + nodes_pool.append(node) + + def get_effect(arg): + ret = [] + for i in xrange(0, randint(1, 5)): + node = choice(nodes_pool) + ret.append(node) + coords_pool.append((node.x, node.y)) + arg[:] = list(ret) + + for i in xrange(0, self.outs): + self.genome[-1-i] = MagicMock() + self.genome[-1-i].getPreviousNodes = MagicMock( + side_effect = get_effect) + self.genome.evaluator.applyFunctions = MagicMock( + return_value = [0.1 , 3.0]) + self.genome.reevaluate = True + self.genome.evaluate() + self.assertTrue(len(self.genome.expressionNodes) > 0) + for node in self.genome.expressionNodes.keys(): + self.assertTrue(node in coords_pool) + +class CartesianNodeTestCase(unittest.TestCase): + @patch('pyevolve.G2DCartesian.rand_randint') + @patch('pyevolve.G2DCartesian.rand_uniform') + def setUp(self, rand_uni, rand_int): + rand_uni.return_value = 0.1313 + rand_int.return_value = 8 + CartesianNode.paramMapping = {"p1" : "rand_randint(0, 10)", + "p2" : "rand_uniform(0,1)"} + self.prev1 = CartesianNode(0, 1, {"a" : 1}, []) + self.prev2 = CartesianNode(0, 2, {"b" : 1}, []) + self.node = CartesianNode(1, 0, {"func3" : 3}, [self.prev1, self.prev2]) + self.outnode = CartesianNode(2, 0, {}, [self.node]) + + def tearDown(self): + self.node = None + CartesianNode.paramMapping.clear() + + def test_node_init(self): + self.assertFalse(self.node.data == None) + self.assertTrue(len(self.node.params) == + len(CartesianNode.paramMapping)) + + def test_node_init_input_like(self): + node = CartesianNode(1, 1, {"b" : 1}, []) + data = node.getData() + self.assertTrue(data[0] == "b") + self.assertTrue(data[1] == 0) + + def test_node_init_internal_like(self): + node = CartesianNode(1, 1, {"f1" : 3}, [self.prev1, self.prev2]) + data = node.getData() + self.assertTrue(data[0] == "f1") + self.assertTrue(data[1] == 2) + + def test_node_init_output_like(self): + node = CartesianNode(1, 1, {}, [self.prev1, self.prev2]) + data = node.getData() + self.assertTrue(data[0] == "") + self.assertTrue(data[1] == 1) + + def test_node_init_empty_previous_with_bad_function_set(self): + self.assertRaises(ValueError, CartesianNode, 0, 0, {"f2" : 3}) + + def test_node_param_mapping(self): + CartesianNode.paramMapping = {"p1" : "rand_randint(0, 10)"} + node = CartesianNode(1, 1, {"f1" : 3}, [self.prev1, self.prev2]) + self.assertTrue(len(node.params) == len(CartesianNode.paramMapping)) + + def test_node_previous_for_input(self): + node = CartesianNode(1, 1, {"b" : 1}, []) + previous = [] + node.getPreviousNodes(previous) + self.assertTrue(len(previous) == 0) + + def test_node_previous_for_internal(self): + previous = [] + self.node.getPreviousNodes(previous) + self.assertTrue(len(previous) > 0) + + def test_node_previous_for_output(self): + prev = CartesianNode(1, 1, {"f" : 2}, [self.prev1, self.prev2]) + node = CartesianNode(1, 1, {}, [prev]) + previous = [] + node.getPreviousNodes(previous) + self.assertTrue(len(previous) > 0) + + def test_get_expression(self): + expected_expr = "func3( " + expr = self.outnode.getExpression("") + match = re.search('func3\( [a-z], [a-z], \{"[a-z0-9]*" : (\d+\.)?\d+, ' + '"[a-z0-9]*" : (\d+\.)?\d+\} \)', expr) + self.assertEqual(match.group(0), expr) + + \ No newline at end of file diff --git a/tests/test_initializators.py b/tests/test_initializators.py index f7dcef3..be11642 100644 --- a/tests/test_initializators.py +++ b/tests/test_initializators.py @@ -1,10 +1,13 @@ import unittest +from mock import MagicMock from pyevolve.G1DBinaryString import G1DBinaryString from pyevolve import Initializators from pyevolve.G1DList import G1DList from pyevolve.G2DList import G2DList from pyevolve.GTree import GTree +from pyevolve.G2DCartesian import G2DCartesian +from random import randint as rand_randint, uniform as rand_uniform class InitializatorsTestCase(unittest.TestCase): @@ -40,3 +43,168 @@ def test_tree_integer_initializator(self): Initializators.GTreeInitializatorInteger(genome) for gen in genome.getAllNodes(): self.assertTrue(type(gen.getData()) == int) + +class G2DCartesianInitializatorTestCase(unittest.TestCase): + @classmethod + def setUpClass(self): + self.engine = MagicMock(); + def getParams(value): + if value == "gp_terminals": + return ['a', 'b', 'c', 'd'] + elif value == "gp_function_set": + return {"gp1" : 2, "gp2" : 2, "gp3" : 3} + elif value == "gp_args_mapping": + return {"arg1" : "rand_randint(0,10)", + "arg2" : "rand_uniform(2.0,4.2)"} + + self.engine.getParam = MagicMock(side_effect=getParams) + + def setUp(self): + self.genome = G2DCartesian(2, 3, 4, 1) + + def tearDown(self): + self.genome = None + + def test_nodes_creation(self): + Initializators.G2DCartesianInitializatorNode(self.genome, + ga_engine=self.engine) + for node in self.genome: + self.assertFalse(node == None) + + def test_input_nodes_inputs(self): + Initializators.G2DCartesianInitializatorNode(self.genome, + ga_engine=self.engine) + for node in self.genome[self.genome.inputSlice]: + self.assertTrue(len(node.inputs) == 0) + + def test_internal_nodes_inputs(self): + Initializators.G2DCartesianInitializatorNode(self.genome, + ga_engine=self.engine) + for node in self.genome[self.genome.internalSlice]: + self.assertTrue(len(node.inputs) in xrange(1,3)) + for input in node.inputs: + self.assertTrue(input.x < node.x) + + def test_output_nodes_inputs(self): + Initializators.G2DCartesianInitializatorNode(self.genome, + ga_engine=self.engine) + for node in self.genome[self.genome.outputSlice]: + self.assertTrue(len(node.inputs) == 1) + for input in node.inputs: + self.assertTrue(input.x < node.x) + + def test_input_nodes_data_sets(self): + Initializators.G2DCartesianInitializatorNode(self.genome, + ga_engine=self.engine) + for node in self.genome[self.genome.inputSlice]: + data = node.getData() + self.assertTrue(data[0] in self.engine.getParam("gp_terminals")) + + def test_internal_nodes_data_sets(self): + Initializators.G2DCartesianInitializatorNode(self.genome, + ga_engine=self.engine) + for node in self.genome[self.genome.internalSlice]: + data = node.getData() + self.assertTrue(data[0] in self.engine.getParam("gp_function_set")) + + + def test_output_nodes_data_sets(self): + Initializators.G2DCartesianInitializatorNode(self.genome, + ga_engine=self.engine) + for node in self.genome[self.genome.outputSlice]: + data = node.getData() + self.assertTrue(data[0] == "") + + def test_input_nodes_positions(self): + Initializators.G2DCartesianInitializatorNode(self.genome, + ga_engine=self.engine) + for idx, node in enumerate(self.genome[self.genome.inputSlice]): + self.assertTrue(node.x == -1) + self.assertTrue(node.y == idx) + + def test_internal_nodes_positions(self): + Initializators.G2DCartesianInitializatorNode(self.genome, + ga_engine=self.engine) + for idx, node in enumerate(self.genome[self.genome.internalSlice]): + self.assertTrue(node.x == idx / self.genome.cols) + self.assertTrue(node.y == idx % self.genome.cols) + + def test_output_nodes_positions(self): + Initializators.G2DCartesianInitializatorNode(self.genome, + ga_engine=self.engine) + for idx, node in enumerate(self.genome[self.genome.outputSlice]): + self.assertTrue(node.x == self.genome.rows) + self.assertTrue(node.y == idx) + + def test_nodes_params(self): + Initializators.G2DCartesianInitializatorNode(self.genome, + ga_engine=self.engine) + mapping = self.engine.getParam("gp_args_mapping") + for node in self.genome: + for key in mapping.keys(): + self.assertTrue(key in node.params) + self.assertTrue(type(eval(mapping[key])) == + type(node.params[key])) + + def test_empty_functions(self): + eng = MagicMock() + def getParams(value): + if value == "gp_terminals": + return ['a', 'b', 'c', 'd'] + return None + + eng.getParam = MagicMock(side_effect=getParams) + self.assertRaises(AssertionError, + Initializators.G2DCartesianInitializatorNode, + self.genome, ga_engine=eng) + + def test_empty_terminals(self): + eng = MagicMock() + def getParams(value): + if value == "gp_function_set": + return {"gp1" : 2, "gp2" : 2, "gp3" : 3} + return None + + eng.getParam = MagicMock(side_effect=getParams) + self.assertRaises(AssertionError, + Initializators.G2DCartesianInitializatorNode, + self.genome, ga_engine=eng) + + def test_empty_mapping(self): + eng = MagicMock() + def getParams(value): + if value == "gp_terminals": + return ['a', 'b', 'c', 'd'] + elif value == "gp_function_set": + return {"gp1" : 2, "gp2" : 2, "gp3" : 3} + return None + + eng.getParam = MagicMock(side_effect=getParams) + self.assertRaises(AssertionError, + Initializators.G2DCartesianInitializatorNode, + self.genome, ga_engine=eng) + + def test_empty_engine(self): + self.assertRaises(KeyError, + Initializators.G2DCartesianInitializatorNode, + self.genome) + + def test_mismatch_input_with_terminals(self): + eng = MagicMock() + def getParams(value): + if value == "gp_terminals": + return ['a', 'b', 'c'] + if value == "gp_function_set": + return {"gp1" : 2, "gp2" : 2, "gp3" : 3} + + eng.getParam = MagicMock(side_effect=getParams) + self.assertRaises(AssertionError, + Initializators.G2DCartesianInitializatorNode, + self.genome, ga_engine=eng) + + def test_bad_genome(self): + genome = G1DList() + self.assertRaises(TypeError, + Initializators.G2DCartesianInitializatorNode, + genome) + diff --git a/tests/test_mutators.py b/tests/test_mutators.py index 65c4d5d..10653d7 100644 --- a/tests/test_mutators.py +++ b/tests/test_mutators.py @@ -1,10 +1,12 @@ import unittest -from mock import patch +from mock import patch, Mock from pyevolve.G1DBinaryString import G1DBinaryString from pyevolve import Mutators, Consts from pyevolve.G1DList import G1DList +from pyevolve.G2DCartesian import G2DCartesian, CartesianNode +from random import randint class G1DBinaryStringMutatorsTestCase(unittest.TestCase): @@ -177,3 +179,178 @@ def test_binary_mutator_large_pmut(self, rand_mock): expected_result = [1, 2, 3] Mutators.G1DListMutatorIntegerBinary(self.genome, pmut=0.5) self.assertEqual(self.genome.genomeList, expected_result) + +class G2DCartesianMutatorsTestCase(unittest.TestCase): + @classmethod + def setUpClass(cls): + CartesianNode.paramMapping.clear() + + def setUp(self): + self.function_set = {'f1' : 2, 'f3' : 3, 'f4' : 5} + self.genome = G2DCartesian(2, 2, 1, 1) + + self.genome[0] = CartesianNode(0, -1, {'a' : 1}, []) + prevs = [self.genome[0]] + self.genome[1] = CartesianNode(0, 0, {'f1' : 2}, prevs) + self.genome[2] = CartesianNode(1, 0, {'f2' : 2}, prevs) + prevs.append(self.genome[1]) + prevs.append(self.genome[2]) + self.genome[3] = CartesianNode(0, 1, {'f3' : 3}, prevs) + self.genome[4] = CartesianNode(1, 1, {'f4' : 3}, prevs) + prevs.append(self.genome[3]) + prevs.append(self.genome[4]) + self.genome[5] = CartesianNode(0, 2, {}, prevs) + + def side_param(arg): + if arg == "gp_function_set": + return self.function_set + + self.ga_engine = Mock() + self.ga_engine.getParam = side_param + + @patch('pyevolve.Mutators.rand_choice') + def test_cartesian_mutator_inputs_small_pmut(self, rand_mock): + expected_result = [] + values = [3, 0, 2] + def choice_effect(arg): + return self.genome[values.pop(0)] + + rand_mock.side_effect = choice_effect + + for i in values[1:]: + expected_result.append(self.genome[i]) + mutations = Mutators.G2DCartesianMutatorNodeInputs(self.genome, + pmut=0.1) + self.assertEqual(self.genome[3].inputs, expected_result) + self.assertTrue(self.genome[3] in mutations) + + @patch('pyevolve.Mutators.rand_choice') + def test_cartesian_mutator_inputs_large_pmut(self, rand_mock): + expected_result = {} + values = [1, 0, 2, 0 , 3, 1, 0, 4, 1, 2] + def choice_effect(arg): + return self.genome[values.pop(0)] + + rand_mock.side_effect = choice_effect + + idx = 0 + while idx < len(values): + expected_result[values[idx]] = [] + node_idx = idx + for input in self.genome[values[idx]].inputs: + idx = idx+1 + expected_result[values[node_idx]].append( + self.genome[values[idx]]) + idx = idx+1 + + mutations = Mutators.G2DCartesianMutatorNodeInputs(self.genome, + pmut=0.7) + for key in expected_result.keys(): + self.assertEqual(self.genome[key].inputs, expected_result[key]) + self.assertTrue(self.genome[key] in mutations) + + @patch('pyevolve.Mutators.rand_choice') + def test_cartesian_mutator_function_small_pmut(self, rand_mock): + values = [4, "f3"] + def choice_effect(arg): + what = values.pop(0) + if isinstance(what, int): + return self.genome[what] + else: + return what + + rand_mock.side_effect = choice_effect + mutations = Mutators.G2DCartesianMutatorNodeFunction(self.genome, + pmut=0.1, ga_engine = self.ga_engine) + self.assertEqual(self.genome[4].data, "f3") + self.assertEqual(len(self.genome[4].inputs), self.function_set["f3"]-1) + self.assertTrue(self.genome[4] in mutations) + + @patch('pyevolve.Mutators.rand_choice') + def test_cartesian_mutator_function_large_pmut(self, rand_mock): + values = [1, "f3", 0, 2, "f4", 0, 0, 0, 3, "f1", 4, "f4", 0, 0] + def choice_effect(arg): + what = values.pop(0) + if isinstance(what, int): + return self.genome[what] + else: + return what + + rand_mock.side_effect = choice_effect + mutations = Mutators.G2DCartesianMutatorNodeFunction(self.genome, + pmut=0.7, ga_engine = self.ga_engine) + values = [1, "f3", 0, 2, "f4", 0, 0, 0, 3, "f1", 4, "f4", 0, 0] + idx = 0 + while idx >= 0: + values = values[idx:] + node_idx = values[0] + func = values[1] + self.assertEqual(self.genome[node_idx].data, func) + self.assertEqual(len(self.genome[node_idx].inputs), + self.function_set[func]-1) + self.assertTrue(self.genome[node_idx] in mutations) + values = values[2:] + idx = (next((key for key, val in + enumerate(values) if + isinstance(val, str)), 0) - 1) + + @patch('pyevolve.Mutators.rand_choice') + def test_cartesian_mutator_params_small_pmut(self, rand_mock): + CartesianNode.paramMapping = {"p1" : "rand_randint(0,10)", + "p2" : "rand_randint(0,10)"} + value = 3 + for param in CartesianNode.paramMapping.keys(): + self.genome[value].params[param] = -1 + rand_mock.return_value = self.genome[value] + mutations = Mutators.G2DCartesianMutatorNodeParams(self.genome, + pmut=0.1) + for param in self.genome[value].params.values(): + self.assertTrue(param in range(0,11)) + self.assertTrue(self.genome[value] in mutations) + CartesianNode.paramMapping.clear() + + @patch('pyevolve.Mutators.rand_choice') + def test_cartesian_mutator_params_large_pmut(self, rand_mock): + CartesianNode.paramMapping = {"p1" : "rand_randint(0,10)", + "p2" : "rand_randint(0,10)"} + values = [1, 2, 3, 4] + for v in values: + for param in CartesianNode.paramMapping.keys(): + self.genome[v].params[param] = -1 + def choice_effect(arg): + return self.genome[values.pop(0)] + rand_mock.side_effect = choice_effect + mutations = Mutators.G2DCartesianMutatorNodeParams(self.genome, + pmut=0.7) + values = [1, 2, 3, 4] + for v in values: + for param in self.genome[v].params.values(): + self.assertTrue(param in range(0,11)) + self.assertTrue(self.genome[v] in mutations) + CartesianNode.paramMapping.clear() + + @patch('pyevolve.Mutators.rand_shuffle') + def test_cartesian_mutator_order(self, rand_mock): + self.genome[5].inputs = [self.genome[4]] + self.genome[4].inputs = [self.genome[1]] + self.genome[1].inputs = [self.genome[0]] + + expected_order = [("f4", 5), ("f3", 1)] + def shuffle_effect(arg): + arg[:] = list(expected_order) + return + + rand_mock.side_effect = shuffle_effect + mutations = Mutators.G2DCartesianMutatorNodesOrder(self.genome, + pmut=0.2) + + paths = self.genome.getActiveNodes() + shuffled_functions = [] + for node in paths[0]: + shuffled_functions.append(node.getData()) + + for mutation in mutations: + self.assertTrue(mutation in paths[0]) + + for func in expected_order: + self.assertTrue(func in shuffled_functions) \ No newline at end of file diff --git a/tests/test_network.py b/tests/test_network.py new file mode 100644 index 0000000..7698a6e --- /dev/null +++ b/tests/test_network.py @@ -0,0 +1,43 @@ +from unittest import TestCase + +from mock import MagicMock +from pyevolve import Network + +class UDPThreadServerTestCase(TestCase): + def setUp(self): + self.server = Network.UDPThreadServer("localhost", 1777) + self.plain_data = ("living is not enough one must have sunshine freedom" + "and a little flower") + self.data = format(len(self.plain_data), "#06x") + self.plain_data + self.chunk_size = 8 + + def test_get_data_packs(self): + def recv_effect(size): + buf = self.data[:self.chunk_size] + self.data = self.data[self.chunk_size:] + return (buf, ("Butterfly", )) + + self.server.sock = MagicMock() + self.server.sock.recvfrom = MagicMock(side_effect = recv_effect) + + ret = self.server.getData() + self.assertEqual(ret, ("Butterfly", self.plain_data)) + +class UDPThreadUnicastClientTestCase(TestCase): + def setUp(self): + self.client = Network.UDPThreadUnicastClient("localhost", 1777) + self.plain_data = ("living is not enough one must have sunshine freedom" + "and a little flower") + self.client.target = 4*[None] + self.chunk_size = 8 + + def test_send_data_packs(self): + def sendto_effect(data, dest): + buf = data[:self.chunk_size] + return len(buf) + + self.client.sock = MagicMock() + self.client.sock.sendto = MagicMock(side_effect = sendto_effect) + + ret = self.client.send(self.plain_data) + self.assertEqual(ret, len(self.plain_data) + 6) \ No newline at end of file diff --git a/tests/test_population.py b/tests/test_population.py new file mode 100644 index 0000000..59f7cb3 --- /dev/null +++ b/tests/test_population.py @@ -0,0 +1,37 @@ +from unittest import TestCase +from mock import MagicMock +from pyevolve.GPopulation import GPopulation + + +class MultithreadingTestCase(TestCase): + def setUp(self): + + self.scores = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1] + def eval_effect(idx): + self.population.internalPop[idx].score = self.scores[idx] + + self.mockGenome = MagicMock() + self.population = GPopulation(self.mockGenome) + self.population.internalPop = [MagicMock() for _ in xrange(0,11)] + for idx, mock in enumerate(self.population.internalPop): + mock.score = 0.0 + mock.evaluate = MagicMock(side_effect = eval_effect(idx)) + + def test_init(self): + self.assertTrue(self.population.multiThreading == (False, None)) + + def test_set_MT(self): + self.population.setMultiThreading(True, 10) + self.assertTrue(self.population.multiThreading == (True, 10)) + + def test_run_MT_without_constraint(self): + self.population.setMultiThreading(True) + self.population.evaluate() + for ind in self.population.internalPop: + self.assertTrue(ind.score in self.scores) + + def test_run_MT_with_constraints(self): + self.population.setMultiThreading(True, 3) + self.population.evaluate() + for ind in self.population.internalPop: + self.assertTrue(ind.score in self.scores) \ No newline at end of file diff --git a/tests/test_util.py b/tests/test_util.py index dea3a4d..63f57e7 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -16,4 +16,78 @@ def test_randomFlipCoin_border_cases(self): def test_list2DSwapElement(self): _list = [[1, 2, 3], [4, 5, 6]] Util.list2DSwapElement(_list, (0, 1), (1, 1)) - self.assertEqual(_list, [[1, 5, 3], [4, 2, 6]]) \ No newline at end of file + self.assertEqual(_list, [[1, 5, 3], [4, 2, 6]]) + +try: + import numpy as np + + class VectorErrorAccumulatorTestCase(TestCase): + def setUp(self): + np.empty([2, 2]) + self.VEC = Util.VectorErrorAccumulator(False, False) + self.VECC = Util.VectorErrorAccumulator(True, False) + self.VECZ = Util.VectorErrorAccumulator(False, True) + self.VECCZ = Util.VectorErrorAccumulator(True, True) + self.target = np.ones([12, 12]) + self.evaluated = np.zeros([12, 12]) + + def test_init_standard(self): + self.assertTrue(self.VEC.non_zeros == 0) + self.assertTrue((self.VEC.confusion == np.zeros([2, 2])).all()) + + def test_init_without_any(self): + self.assertTrue(self.VEC.calculate_confusion == False) + self.assertTrue(self.VEC.calculate_non_zeros == False) + + def test_init_with_confusion(self): + self.assertTrue(self.VECC.calculate_confusion == True) + self.assertTrue(self.VECC.calculate_non_zeros == False) + + def test_init_with_confusion_and_zeros(self): + self.assertTrue(self.VECCZ.calculate_confusion == True) + self.assertTrue(self.VECCZ.calculate_non_zeros == True) + + def test_init_with_zeros(self): + self.assertTrue(self.VECZ.calculate_confusion == False) + self.assertTrue(self.VECZ.calculate_non_zeros == True) + + + def test_reset_empty(self): + self.VEC.reset() + self.assertTrue(self.VEC.non_zeros == 0) + self.assertTrue((self.VEC.confusion == np.zeros([2, 2])).all()) + + def test_reset_non_empty(self): + self.VEC.non_zeros = 10 + self.VEC.confusion = np.ones([2 ,2]) + self.VEC.reset() + self.assertTrue(self.VEC.non_zeros == 0) + print self.VEC.confusion + self.assertTrue((self.VEC.confusion == np.zeros([2, 2])).all()) + + def test_append_to_empty(self): + self.VEC.append(self.target, self.evaluated) + self.assertTrue(self.VEC.acc_len == 144) + + def test_append_to_non_empty(self): + self.VEC.append(self.target, self.evaluated) + self.VEC.append(self.target, self.evaluated) + self.assertTrue(self.VEC.acc_len == 144*2) + + def test_get_confusion_matrix(self): + self.VECC.append(self.target, self.evaluated) + ret = self.VECC.getConfusionMatrix() + self.assertTrue((np.array([[144, 0], [0, 0]]) == ret).all()) + + def test_get_zeros(self): + self.VECZ.append(self.target, self.evaluated) + ret = self.VECZ.getZeros() + self.assertTrue(ret == 0) + + def test_get_non_zeros(self): + self.VECCZ.append(self.target, self.evaluated) + ret = self.VECCZ.getNonZeros() + self.assertTrue(ret == 144) + +except: + pass \ No newline at end of file