From 0a242083a07b12f5312eea08422c9d81928a7b1f Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Thu, 10 Nov 2011 18:08:50 -0800 Subject: [PATCH] Updated tests, added a priority queue Updated the tests to reflect the functionality added in verion 0.4.0, and then updated the library to actually pass all those tests. Notably, there was a problem with loading and dumping to and from files, but it's been resolved. This version also sees the introduction of a priority queue, with unique elements. While it is potentially problematic to be tied to the uniqueness constraing, one can always inject arbitrary ids for repeated values. Perhaps not the most efficient method in the world, but it exposes one more list type in redis. --- qr.py | 36 +++++--- test/tests.py | 252 ++++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 254 insertions(+), 34 deletions(-) diff --git a/qr.py b/qr.py index cc22a5b..3cb595e 100644 --- a/qr.py +++ b/qr.py @@ -69,7 +69,7 @@ def __len__(self): def __getitem__(self, val): """Get a slice or a particular index.""" try: - return [self._unpack(i) for i in self.redis.lrange(self.key, val.start, val.stop)] + return [self._unpack(i) for i in self.redis.lrange(self.key, val.start, val.stop - 1)] except AttributeError: return self._unpack(self.redis.lindex(self.key, val)) except Exception as e: @@ -96,10 +96,11 @@ def dump(self, fobj): def load(self, fobj): """Load the contents of the provided fobj into the queue""" - next = self.serializer.load(obj) - while next: - self.redis.lpush(self._pack(next)) - next = self.serializer.load(fobj) + try: + while True: + self.redis.lpush(self.key, self._pack(self.serializer.load(fobj))) + except: + return def dumpfname(self, fname, truncate=False): """Destructively dump the contents of the queue into fname""" @@ -128,11 +129,15 @@ def peek(self): def elements(self): """Return all elements as a Python list""" - return self.redis.lrange(self.key, 0, -1) + return [self._unpack(o) for o in self.redis.lrange(self.key, 0, -1)] def elements_as_json(self): """Return all elements as JSON object""" return json.dumps(self.elements) + + def clear(self): + """Removes all the elements in the queue""" + self.redis.delete(self.key) class Deque(BaseQueue): """Implements a double-ended queue""" @@ -191,7 +196,7 @@ def __len__(self): def __getitem__(self, val): """Get a slice or a particular index.""" try: - return [self._unpack(i) for i in self.redis.zrange(self.key, val.start, val.stop)] + return [self._unpack(i) for i in self.redis.zrange(self.key, val.start, val.stop - 1)] except AttributeError: val = self.redis.zrange(self.key, val, val) if val: @@ -206,16 +211,18 @@ def dump(self, fobj): next = self.redis.zrange(self.key, 0, 0, withscores=True) removed = self.redis.zremrangebyrank(self.key, 0, 0) while next: - fobj.write(next[0]) - next = self.redis.zrange(self.key, 0, 0) + self.serializer.dump(next[0], fobj) + next = self.redis.zrange(self.key, 0, 0, withscores=True) removed = self.redis.zremrangebyrank(self.key, 0, 0) def load(self, fobj): """Load the contents of the provided fobj into the queue""" - next = self.serializer.load(fobj) - while next: - self.redis.zadd(self.key, self.serializer._pack(*next)) - next = self.serializer.load(fobj) + try: + while True: + value, score = self.serializer.load(fobj) + self.redis.zadd(self.key, value, score) + except Exception as e: + return def dumpfname(self, fname, truncate=False): """Destructively dump the contents of the queue into fname""" @@ -244,7 +251,7 @@ def peek(self, withscores=False): def elements(self): """Return all elements as a Python list""" - return self.redis.zrange(self.key, 0, -1) + return [self._unpack(o) for o in self.redis.zrange(self.key, 0, -1)] def pop(self, withscores=False): '''Get the element with the lowest score, and pop it off''' @@ -313,4 +320,3 @@ def pop(self): popped = self.redis.lpop(self.key) log.debug('Popped ** %s ** from key ** %s **' % (popped, self.key)) return self._unpack(popped) - \ No newline at end of file diff --git a/test/tests.py b/test/tests.py index c8ec475..6520397 100644 --- a/test/tests.py +++ b/test/tests.py @@ -1,21 +1,22 @@ -import unittest +import os import qr import redis +import unittest r = redis.Redis() class Queue(unittest.TestCase): def setUp(self): - r.delete('qrtestqueue') + r.delete('qrtestqueue') self.q = qr.Queue(key='qrtestqueue') - self.assertEquals(len(self.q.elements()), 0) + self.assertEquals(len(self.q), 0) def test_roundtrip(self): q = self.q q.push('foo') - self.assertEquals(len(q.elements()), 1) + self.assertEquals(len(q), 1) self.assertEquals(q.pop(), 'foo') - self.assertEquals(len(q.elements()), 0) + self.assertEquals(len(q), 0) def test_order(self): q = self.q @@ -31,18 +32,86 @@ def test_order_mixed(self): q.push('bar') self.assertEquals(q.pop(), 'bar') + def test_len(self): + count = 100 + for i in range(count): + self.assertEquals(len(self.q), i) + self.q.push(i) + for i in range(count): + self.assertEquals(len(self.q), count - i) + self.q.pop() + self.q.clear() + + def test_get_item(self): + count = 100 + items = [i for i in range(count)] + self.q.clear() + self.q.extend(items) + items.reverse() + # Get single values + for i in range(count): + self.assertEquals(self.q[i], items[i]) + # Get small ranges + for i in range(count-1): + self.assertEquals(self.q[i:i+1], items[i:i+1]) + # Now get the whole range + self.assertEquals(self.q[0:-1], items[0:-1]) + self.q.clear() + + def test_extend(self): + '''Test extending a queue, including with a generator''' + count = 100 + self.q.extend(i for i in range(count)) + self.assertEquals(len(self.q), count) + self.q.clear() + + self.q.extend([i for i in range(count)]) + self.assertEquals(len(self.q), count) + self.q.clear() + + self.q.extend(range(count)) + self.assertEquals(self.q.elements(), [count - i - 1 for i in range(count)]) + self.q.clear() + + def test_pack_unpack(self): + '''Make sure that it behaves like python-object-in, python-object-out''' + count = 100 + self.q.extend({'key': i} for i in range(count)) + next = self.q.pop() + while next: + self.assertTrue(isinstance(next, dict)) + next = self.q.pop() + + def test_dump_load(self): + # Get a temporary file to dump a queue to that file + count = 100 + self.q.extend(range(count)) + self.assertEquals(self.q.elements(), [count - i - 1for i in range(count)]) + with os.tmpfile() as f: + self.q.dump(f) + # Now, assert that it is empty + self.assertEquals(len(self.q), 0) + # Now, try to load it back in + f.seek(0) + self.q.load(f) + self.assertEquals(len(self.q), count) + self.assertEquals(self.q.elements(), [count - i - 1 for i in range(count)]) + # Now clean up after myself + f.truncate() + self.q.clear() + class CappedCollection(unittest.TestCase): def setUp(self): - r.delete('qrtestcc') - self.aq = qr.CappedCollection(key='qrtestcc', size=3) - self.assertEquals(len(self.aq.elements()), 0) + r.delete('qrtestcc') + self.aq = qr.CappedCollection(key='qrtestcc', size=3) + self.assertEquals(len(self.aq), 0) def test_roundtrip(self): aq = self.aq aq.push('foo') - self.assertEquals(len(aq.elements()), 1) + self.assertEquals(len(aq), 1) self.assertEquals(aq.pop(), 'foo') - self.assertEquals(len(aq.elements()), 0) + self.assertEquals(len(aq), 0) def test_order(self): aq = self.aq @@ -63,26 +132,33 @@ def test_limit(self): aq.push('a') aq.push('b') aq.push('c') - self.assertEquals(len(aq.elements()), 3) + self.assertEquals(len(aq), 3) aq.push('d') aq.push('e') - self.assertEquals(len(aq.elements()), 3) + self.assertEquals(len(aq), 3) self.assertEquals(aq.pop(), 'c') self.assertEquals(aq.pop(), 'd') self.assertEquals(aq.pop(), 'e') - self.assertEquals(len(aq.elements()), 0) + self.assertEquals(len(aq), 0) + + def test_extend(self): + '''Test extending a queue, including with a generator''' + count = 100 + self.aq.extend(i for i in range(count)) + self.assertEquals(len(self.aq), self.aq.size) + self.aq.clear() class Stack(unittest.TestCase): def setUp(self): - r.delete('qrteststack') + r.delete('qrteststack') self.stack = qr.Stack(key='qrteststack') def test_roundtrip(self): stack = self.stack stack.push('foo') - self.assertEquals(len(stack.elements()), 1) + self.assertEquals(len(stack), 1) self.assertEquals(stack.pop(), 'foo') - self.assertEquals(len(stack.elements()), 0) + self.assertEquals(len(stack), 0) def test_order(self): stack = self.stack @@ -98,8 +174,146 @@ def test_order_mixed(self): stack.push('bar') self.assertEquals(stack.pop(), 'bar') -if __name__ == '__main__': - unittest.main() + def test_get_item(self): + count = 100 + items = [i for i in range(count)] + self.stack.extend(items) + items.reverse() + # Get single values + for i in range(count): + self.assertEquals(self.stack[i], items[i]) + # Get small ranges + for i in range(count-1): + self.assertEquals(self.stack[i:i+2], items[i:i+2]) + # Now get the whole range + self.assertEquals(self.stack[0:-1], items[0:-1]) + self.stack.clear() + + def test_extend(self): + '''Test extending a queue, including with a generator''' + count = 100 + self.stack.extend(i for i in range(count)) + self.assertEquals(self.stack.elements(), [count - i - 1 for i in range(count)]) + + # Also, make sure it's still a stack. It should be in reverse order + last = self.stack.pop() + while last != None: + now = self.stack.pop() + self.assertTrue(last > now) + last = now + self.stack.clear() + + def test_dump_load(self): + # Get a temporary file to dump a queue to that file + count = 100 + self.stack.extend(range(count)) + self.assertEquals(self.stack.elements(), [count - i - 1 for i in range(count)]) + with os.tmpfile() as f: + self.stack.dump(f) + # Now, assert that it is empty + self.assertEquals(len(self.stack), 0) + # Now, try to load it back in + f.seek(0) + self.stack.load(f) + self.assertEquals(len(self.stack), count) + self.assertEquals(self.stack.elements(), [count - i - 1 for i in range(count)]) + # Now clean up after myself + f.truncate() + self.stack.clear() + +class PriorityQueue(unittest.TestCase): + def setUp(self): + r.delete('qrpriorityqueue') + self.q = qr.PriorityQueue(key='qrpriorityqueue') + + def test_roundtrip(self): + self.q.push('foo', 1) + self.assertEquals(len(self.q), 1) + self.assertEquals(self.q.pop(), 'foo') + self.assertEquals(len(self.q), 0) + + def test_order(self): + self.q.push('foo', 1) + self.q.push('bar', 0) + self.assertEquals(self.q.pop(), 'bar') + self.assertEquals(self.q.pop(), 'foo') - + def test_get_item(self): + count = 100 + items = [i for i in range(count)] + self.q.extend(zip(items, items)) + # Get single values + for i in range(count): + self.assertEquals(self.q[i], items[i]) + # Get small ranges + for i in range(count-1): + self.assertEquals(self.q[i:i+2], items[i:i+2]) + # Now get the whole range + self.assertEquals(self.q[0:-1], items[0:-1]) + self.q.clear() + def test_extend(self): + '''Test extending a queue, including with a generator''' + count = 100 + items = [i for i in range(count)] + self.q.extend(zip(items, items)) + self.assertEquals(self.q.elements(), items) + self.q.clear() + + def test_pop(self): + '''Test whether or not we can get real values with pop''' + count = 100 + items = [i for i in range(count)] + self.q.extend(zip(items, items)) + next = self.q.pop() + while next: + self.assertTrue(isinstance(next, int)) + next = self.q.pop() + # Now we'll pop with getting the scores as well + items = [i for i in range(count)] + self.q.extend(zip(items, items)) + value, score = self.q.pop(withscores=True) + while value: + self.assertTrue(isinstance(value, int)) + self.assertTrue(isinstance(score, float)) + value, score = self.q.pop(withscores=True) + + def test_push(self): + '''Test whether we can push well''' + count = 100 + for i in range(count): + self.q.push(i, count - i) + value, score = self.q.pop(withscores=True) + while value: + self.assertEqual(value + score, count) + value, score = self.q.pop(withscores=True) + + def test_uniqueness(self): + count = 100 + # Push the same value on with different scores + for i in range(count): + self.q.push(1, i) + self.assertEquals(len(self.q), 1) + self.q.clear() + + def test_dump_load(self): + # Get a temporary file to dump a queue to that file + count = 100 + items = [i for i in range(count)] + self.q.extend(zip(items, items)) + self.assertEquals(self.q.elements(), items) + with os.tmpfile() as f: + self.q.dump(f) + # Now, assert that it is empty + self.assertEquals(len(self.q), 0) + # Now, try to load it back in + f.seek(0) + self.q.load(f) + self.assertEquals(len(self.q), count) + self.assertEquals(self.q.elements(), items) + # Now clean up after myself + f.truncate() + self.q.clear() + +if __name__ == '__main__': + unittest.main()