From 6b84048ff1c03d78961b8acf60a8a552c70670c1 Mon Sep 17 00:00:00 2001 From: David Wolever Date: Fri, 21 Mar 2014 14:44:13 -0400 Subject: [PATCH 1/3] Allow factories to be passed directly to ObjectPool --- objpool/__init__.py | 17 +++++++++++++---- objpool/test/tests.py | 23 ++++++++++++++++++----- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/objpool/__init__.py b/objpool/__init__.py index d1c18e3..0429fbe 100644 --- a/objpool/__init__.py +++ b/objpool/__init__.py @@ -114,7 +114,7 @@ class ObjectPool(object): relies upon will be copied when the new process is being created. """ - def __init__(self, size=None): + def __init__(self, size=None, create=None, verify=None, cleanup=None): self._pool_pid = getpid() try: self.size = int(size) @@ -123,6 +123,10 @@ def __init__(self, size=None): raise ValueError("Invalid size for pool (positive integer " "required): %r" % (size,)) + self._create_func = create + self._verify_func = verify + self._cleanup_func = cleanup + self._semaphore = Semaphore(size) # Pool grows up to size limit self._mutex = Lock() # Protect shared _set oject self._set = set() @@ -237,7 +241,9 @@ def _pool_create(self): Must be thread-safe. """ - raise NotImplementedError + if self._create_func is None: + raise NotImplementedError + return self._create_func() def _pool_verify(self, obj): """Verify an object after getting it from the pool. @@ -249,7 +255,9 @@ def _pool_verify(self, obj): Must be thread-safe. """ - raise NotImplementedError + if self._verify_func is None: + return True + return self._verify_func(obj) def _pool_cleanup(self, obj): """Cleanup an object before being put back into the pool. @@ -259,7 +267,8 @@ def _pool_cleanup(self, obj): Must be thread-safe. """ - raise NotImplementedError + if self._cleanup_func is not None: + return self._cleanup_func(obj) class PooledObject(object): diff --git a/objpool/test/tests.py b/objpool/test/tests.py index 86fe745..77ad306 100644 --- a/objpool/test/tests.py +++ b/objpool/test/tests.py @@ -123,12 +123,25 @@ def test_get_not_implemented(self): """Test pool_get() method not implemented in abstract class""" pool = ObjectPool(100) self.assertRaises(NotImplementedError, pool._pool_create) - self.assertRaises(NotImplementedError, pool._pool_verify, None) - def test_put_not_implemented(self): - """Test pool_put() method not implemented in abstract class""" - pool = ObjectPool(100) - self.assertRaises(NotImplementedError, pool._pool_cleanup, None) + def test_get_with_factory(self): + obj_generator = iter(range(10)).next + pool = ObjectPool(3, create=obj_generator) + self.assertEqual(pool.pool_get(), 0) + self.assertEqual(pool.pool_get(), 1) + self.assertEqual(pool.pool_get(), 2) + + def test_put_with_factory(self): + cleaned_objects = [] + pool = ObjectPool(3, + create=lambda: 0, + verify=lambda o: o % 2 == 0, + cleanup=cleaned_objects.append, + ) + self.assertEqual(pool.pool_get(), 0) + pool.pool_put(0) + self.assertRaises(PoolVerificationError, pool.pool_put, 1) + self.assertEqual(cleaned_objects, [0]) class NumbersPoolTestCase(unittest.TestCase): From d5701b79b347a4e428d86a2a09a52478203edba0 Mon Sep 17 00:00:00 2001 From: David Wolever Date: Fri, 21 Mar 2014 15:16:21 -0400 Subject: [PATCH 2/3] Oops, fix a couple tests --- objpool/test/tests.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/objpool/test/tests.py b/objpool/test/tests.py index 77ad306..adca1d4 100644 --- a/objpool/test/tests.py +++ b/objpool/test/tests.py @@ -134,13 +134,15 @@ def test_get_with_factory(self): def test_put_with_factory(self): cleaned_objects = [] pool = ObjectPool(3, - create=lambda: 0, + create=[2, 1, 0].pop, verify=lambda o: o % 2 == 0, cleanup=cleaned_objects.append, ) self.assertEqual(pool.pool_get(), 0) pool.pool_put(0) - self.assertRaises(PoolVerificationError, pool.pool_put, 1) + self.assertEqual(pool.pool_get(), 0) + self.assertRaises(PoolVerificationError, pool.pool_get) + self.assertEqual(pool.pool_get(), 2) self.assertEqual(cleaned_objects, [0]) From 6abdd291f83b73588d3877dabcdd1f4f388a77d3 Mon Sep 17 00:00:00 2001 From: David Wolever Date: Fri, 21 Mar 2014 14:55:34 -0400 Subject: [PATCH 3/3] Allow unbounded pool sizes --- objpool/__init__.py | 26 ++++++++++++++++++++++---- objpool/test/tests.py | 16 ++++++++++++---- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/objpool/__init__.py b/objpool/__init__.py index 0429fbe..1316dbe 100644 --- a/objpool/__init__.py +++ b/objpool/__init__.py @@ -57,6 +57,16 @@ log = logging.getLogger(__name__) +class DummySemaphore(object): + _Semaphore__value = 0 + + def acquire(self, *args, **kwargs): + return True + + def release(self): + pass + + class ObjectPoolError(Exception): pass @@ -116,18 +126,26 @@ class ObjectPool(object): """ def __init__(self, size=None, create=None, verify=None, cleanup=None): self._pool_pid = getpid() + if size is None: + size = 0 + try: - self.size = int(size) - assert size >= 1 + int(size) + assert size >= 0 except: - raise ValueError("Invalid size for pool (positive integer " + raise ValueError("Invalid size for pool (non-negative integer " "required): %r" % (size,)) self._create_func = create self._verify_func = verify self._cleanup_func = cleanup - self._semaphore = Semaphore(size) # Pool grows up to size limit + self.size = size + if self.size > 0: + self._semaphore = Semaphore(self.size) + else: + self._semaphore = DummySemaphore() + self._mutex = Lock() # Protect shared _set oject self._set = set() log.debug("Initialized pool %r", self) diff --git a/objpool/test/tests.py b/objpool/test/tests.py index adca1d4..6893c63 100644 --- a/objpool/test/tests.py +++ b/objpool/test/tests.py @@ -107,12 +107,20 @@ def _pool_cleanup(self, obj): class ObjectPoolTestCase(unittest.TestCase): - def test_create_pool_requires_size(self): + def test_create_pool_invalid_sizes(self): """Test __init__() requires valid size argument""" - self.assertRaises(ValueError, ObjectPool) - self.assertRaises(ValueError, ObjectPool, size="size10") - self.assertRaises(ValueError, ObjectPool, size=0) self.assertRaises(ValueError, ObjectPool, size=-1) + self.assertRaises(ValueError, ObjectPool, size="size10") + + def test_create_pool_valid_sizes(self): + ObjectPool(size=0) + ObjectPool(size=None) + + def test_unbounded_pool(self): + pool = ObjectPool(size=0, create=[1,2,3].pop) + self.assertEqual(pool.pool_get(), 3) + self.assertEqual(pool.pool_get(), 2) + self.assertEqual(pool.pool_get(), 1) def test_create_pool(self): """Test pool creation works"""