From d83b79c58db983c502f99ada1aec175cfaa096c8 Mon Sep 17 00:00:00 2001 From: Kyle Feuz Date: Mon, 12 Nov 2012 23:07:25 -0800 Subject: [PATCH 1/2] Added parameter for number of processes --- pyevolve/GPopulation.py | 9 +++++---- pyevolve/GSimpleGA.py | 20 +++++++++++++++++--- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/pyevolve/GPopulation.py b/pyevolve/GPopulation.py index a60c3ff..2f21b76 100644 --- a/pyevolve/GPopulation.py +++ b/pyevolve/GPopulation.py @@ -151,13 +151,13 @@ def __init__(self, genome): self.allSlots = [self.scaleMethod] self.internalParams = {} - self.multiProcessing = (False, False) + self.multiProcessing = (False, False, CPU_COUNT) # Statistics self.statted = False self.stats = Statistics() - def setMultiProcessing(self, flag=True, full_copy=False): + def setMultiProcessing(self, flag=True, full_copy=False, processes=CPU_COUNT): """ Sets the flag to enable/disable the use of python multiprocessing module. Use this option when you have more than one core on your CPU and when your evaluation function is very slow. @@ -167,6 +167,7 @@ def setMultiProcessing(self, flag=True, full_copy=False): :param flag: True (default) or False :param full_copy: True or False (default) + :param proccesses: Number of processes (default=CPU_COUNT) .. warning:: Use this option only when your evaluation function is slow, se you will get a good tradeoff between the process communication speed and the @@ -176,7 +177,7 @@ def setMultiProcessing(self, flag=True, full_copy=False): The `setMultiProcessing` method. """ - self.multiProcessing = (flag, full_copy) + self.multiProcessing = (flag, full_copy, processes) def setMinimax(self, minimax): """ Sets the population minimax @@ -386,7 +387,7 @@ def evaluate(self, **args): # We have multiprocessing if self.multiProcessing[0] and MULTI_PROCESSING: logging.debug("Evaluating the population using the multiprocessing method") - proc_pool = Pool() + proc_pool = Pool(self.multiProcessing[2]) # Multiprocessing full_copy parameter if self.multiProcessing[1]: diff --git a/pyevolve/GSimpleGA.py b/pyevolve/GSimpleGA.py index 379d092..52a1e52 100644 --- a/pyevolve/GSimpleGA.py +++ b/pyevolve/GSimpleGA.py @@ -72,7 +72,7 @@ import logging from time import time -from types import BooleanType +from types import BooleanType, IntType from sys import platform as sys_platform from sys import stdout as sys_stdout @@ -381,7 +381,7 @@ def __repr__(self): ret+="\n" return ret - def setMultiProcessing(self, flag=True, full_copy=False): + def setMultiProcessing(self, flag=True, full_copy=False, processes=None): """ Sets the flag to enable/disable the use of python multiprocessing module. Use this option when you have more than one core on your CPU and when your evaluation function is very slow. @@ -398,6 +398,14 @@ def setMultiProcessing(self, flag=True, full_copy=False): The parameter "full_copy" defines where the individual data should be copied back after the evaluation or not. This parameter is useful when you change the individual in the evaluation function. + + The parameter processes sets the number of processes to be used in the processing + pool. If processes is set to a value less than 1, the number of CPUs will be used. + By default, processes is set to the number of CPUs. In some cases the + user may wish to specify fewer or more processes than this. Fewer processes may be + used to restrict the number of processes spawned. More processes may be used when + the machine has multiple CPUs, each with multiple cores, or with processes that are + not CPU bound. :param flag: True (default) or False :param full_copy: True or False (default) @@ -422,7 +430,13 @@ def setMultiProcessing(self, flag=True, full_copy=False): if type(full_copy) != BooleanType: Util.raiseException("Multiprocessing 'full_copy' option must be True or False", TypeError) - self.internalPop.setMultiProcessing(flag, full_copy) + if processes != None and type(processes) != IntType: + Util.raiseException("Multiprocessing 'processes' option must be an integer") + + if processes != None and processes > 0: + self.internalPop.setMultiProcessing(flag, full_copy, processes) + else: + self.internalPop.setMultiProcessing(flag, full_copy) def setMigrationAdapter(self, migration_adapter=None): """ Sets the Migration Adapter From 4c461bb1c87139c70c261ecfe168e0b8bcd862b6 Mon Sep 17 00:00:00 2001 From: Kyle Feuz Date: Thu, 15 Nov 2012 00:03:48 -0800 Subject: [PATCH 2/2] Made proc_pool a class variable so the pool persists between functions calls. Removed call to close the proc_pool as the pool is now persistent --- pyevolve/GPopulation.py | 18 +++++++++++------- pyevolve/GSimpleGA.py | 3 ++- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/pyevolve/GPopulation.py b/pyevolve/GPopulation.py index 2f21b76..a48d866 100644 --- a/pyevolve/GPopulation.py +++ b/pyevolve/GPopulation.py @@ -133,6 +133,7 @@ def __init__(self, genome): self.internalParams = genome.internalParams self.multiProcessing = genome.multiProcessing + self.proc_pool = genome.proc_pool self.statted = False self.stats = Statistics() @@ -152,6 +153,7 @@ def __init__(self, genome): self.internalParams = {} self.multiProcessing = (False, False, CPU_COUNT) + self.proc_pool = None # Statistics self.statted = False @@ -178,6 +180,9 @@ def setMultiProcessing(self, flag=True, full_copy=False, processes=CPU_COUNT): """ self.multiProcessing = (flag, full_copy, processes) + + if self.multiProcessing[0] and MULTI_PROCESSING: + self.proc_pool = Pool(self.multiProcessing[2]) def setMinimax(self, minimax): """ Sets the population minimax @@ -387,19 +392,17 @@ def evaluate(self, **args): # We have multiprocessing if self.multiProcessing[0] and MULTI_PROCESSING: logging.debug("Evaluating the population using the multiprocessing method") - proc_pool = Pool(self.multiProcessing[2]) + + if self.proc_pool == None: #Should not happen but just in case + self.proc_pool = Pool(self.multiProcessing[2]) # Multiprocessing full_copy parameter if self.multiProcessing[1]: - results = proc_pool.map(multiprocessing_eval_full, self.internalPop) - proc_pool.close() - proc_pool.join() + results = self.proc_pool.map(multiprocessing_eval_full, self.internalPop) for i in xrange(len(self.internalPop)): self.internalPop[i] = results[i] else: - results = proc_pool.map(multiprocessing_eval, self.internalPop) - proc_pool.close() - proc_pool.join() + results = self.proc_pool.map(multiprocessing_eval, self.internalPop) for individual, score in zip(self.internalPop, results): individual.score = score else: @@ -453,6 +456,7 @@ def copy(self, pop): #pop.internalParams = self.internalParams.copy() pop.internalParams = self.internalParams pop.multiProcessing = self.multiProcessing + pop.proc_pool = self.proc_pool def getParam(self, key, nvl=None): """ Gets an internal parameter diff --git a/pyevolve/GSimpleGA.py b/pyevolve/GSimpleGA.py index 52a1e52..3b4d16e 100644 --- a/pyevolve/GSimpleGA.py +++ b/pyevolve/GSimpleGA.py @@ -399,7 +399,7 @@ def setMultiProcessing(self, flag=True, full_copy=False, processes=None): after the evaluation or not. This parameter is useful when you change the individual in the evaluation function. - The parameter processes sets the number of processes to be used in the processing + The parameter "processes" sets the number of processes to be used in the processing pool. If processes is set to a value less than 1, the number of CPUs will be used. By default, processes is set to the number of CPUs. In some cases the user may wish to specify fewer or more processes than this. Fewer processes may be @@ -409,6 +409,7 @@ def setMultiProcessing(self, flag=True, full_copy=False, processes=None): :param flag: True (default) or False :param full_copy: True or False (default) + :param processes: number of processes in the pool (default=CPU_COUNT) .. warning:: Use this option only when your evaluation function is slow, so you'll get a good tradeoff between the process communication speed and the