diff --git a/pyevolve/GPopulation.py b/pyevolve/GPopulation.py index a60c3ff..a48d866 100644 --- a/pyevolve/GPopulation.py +++ b/pyevolve/GPopulation.py @@ -133,6 +133,7 @@ def __init__(self, genome): self.internalParams = genome.internalParams self.multiProcessing = genome.multiProcessing + self.proc_pool = genome.proc_pool self.statted = False self.stats = Statistics() @@ -151,13 +152,14 @@ def __init__(self, genome): self.allSlots = [self.scaleMethod] self.internalParams = {} - self.multiProcessing = (False, False) + self.multiProcessing = (False, False, CPU_COUNT) + self.proc_pool = None # Statistics self.statted = False self.stats = Statistics() - def setMultiProcessing(self, flag=True, full_copy=False): + def setMultiProcessing(self, flag=True, full_copy=False, processes=CPU_COUNT): """ Sets the flag to enable/disable the use of python multiprocessing module. Use this option when you have more than one core on your CPU and when your evaluation function is very slow. @@ -167,6 +169,7 @@ def setMultiProcessing(self, flag=True, full_copy=False): :param flag: True (default) or False :param full_copy: True or False (default) + :param proccesses: Number of processes (default=CPU_COUNT) .. warning:: Use this option only when your evaluation function is slow, se you will get a good tradeoff between the process communication speed and the @@ -176,7 +179,10 @@ def setMultiProcessing(self, flag=True, full_copy=False): The `setMultiProcessing` method. """ - self.multiProcessing = (flag, full_copy) + self.multiProcessing = (flag, full_copy, processes) + + if self.multiProcessing[0] and MULTI_PROCESSING: + self.proc_pool = Pool(self.multiProcessing[2]) def setMinimax(self, minimax): """ Sets the population minimax @@ -386,19 +392,17 @@ def evaluate(self, **args): # We have multiprocessing if self.multiProcessing[0] and MULTI_PROCESSING: logging.debug("Evaluating the population using the multiprocessing method") - proc_pool = Pool() + + if self.proc_pool == None: #Should not happen but just in case + self.proc_pool = Pool(self.multiProcessing[2]) # Multiprocessing full_copy parameter if self.multiProcessing[1]: - results = proc_pool.map(multiprocessing_eval_full, self.internalPop) - proc_pool.close() - proc_pool.join() + results = self.proc_pool.map(multiprocessing_eval_full, self.internalPop) for i in xrange(len(self.internalPop)): self.internalPop[i] = results[i] else: - results = proc_pool.map(multiprocessing_eval, self.internalPop) - proc_pool.close() - proc_pool.join() + results = self.proc_pool.map(multiprocessing_eval, self.internalPop) for individual, score in zip(self.internalPop, results): individual.score = score else: @@ -452,6 +456,7 @@ def copy(self, pop): #pop.internalParams = self.internalParams.copy() pop.internalParams = self.internalParams pop.multiProcessing = self.multiProcessing + pop.proc_pool = self.proc_pool def getParam(self, key, nvl=None): """ Gets an internal parameter diff --git a/pyevolve/GSimpleGA.py b/pyevolve/GSimpleGA.py index 379d092..3b4d16e 100644 --- a/pyevolve/GSimpleGA.py +++ b/pyevolve/GSimpleGA.py @@ -72,7 +72,7 @@ import logging from time import time -from types import BooleanType +from types import BooleanType, IntType from sys import platform as sys_platform from sys import stdout as sys_stdout @@ -381,7 +381,7 @@ def __repr__(self): ret+="\n" return ret - def setMultiProcessing(self, flag=True, full_copy=False): + def setMultiProcessing(self, flag=True, full_copy=False, processes=None): """ Sets the flag to enable/disable the use of python multiprocessing module. Use this option when you have more than one core on your CPU and when your evaluation function is very slow. @@ -398,9 +398,18 @@ def setMultiProcessing(self, flag=True, full_copy=False): The parameter "full_copy" defines where the individual data should be copied back after the evaluation or not. This parameter is useful when you change the individual in the evaluation function. + + The parameter "processes" sets the number of processes to be used in the processing + pool. If processes is set to a value less than 1, the number of CPUs will be used. + By default, processes is set to the number of CPUs. In some cases the + user may wish to specify fewer or more processes than this. Fewer processes may be + used to restrict the number of processes spawned. More processes may be used when + the machine has multiple CPUs, each with multiple cores, or with processes that are + not CPU bound. :param flag: True (default) or False :param full_copy: True or False (default) + :param processes: number of processes in the pool (default=CPU_COUNT) .. warning:: Use this option only when your evaluation function is slow, so you'll get a good tradeoff between the process communication speed and the @@ -422,7 +431,13 @@ def setMultiProcessing(self, flag=True, full_copy=False): if type(full_copy) != BooleanType: Util.raiseException("Multiprocessing 'full_copy' option must be True or False", TypeError) - self.internalPop.setMultiProcessing(flag, full_copy) + if processes != None and type(processes) != IntType: + Util.raiseException("Multiprocessing 'processes' option must be an integer") + + if processes != None and processes > 0: + self.internalPop.setMultiProcessing(flag, full_copy, processes) + else: + self.internalPop.setMultiProcessing(flag, full_copy) def setMigrationAdapter(self, migration_adapter=None): """ Sets the Migration Adapter