From 28058fb53d21097947d190bcc47e3609a6794e7a Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Sat, 22 Sep 2012 22:57:00 +1000 Subject: A first implementation of Unpacker.skip() --- msgpack/_msgpack.pyx | 25 +++++++++++++++++-------- msgpack/unpack_template.h | 24 ++++++++++++++---------- 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 80d34ab..c1e3e75 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -200,7 +200,7 @@ cdef extern from "unpack.h": PyObject* key int template_execute(template_context* ctx, const_char_ptr data, - size_t len, size_t* off) except -1 + size_t len, size_t* off, bool construct) except -1 void template_init(template_context* ctx) object template_data(template_context* ctx) @@ -246,7 +246,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, if not PyCallable_Check(list_hook): raise TypeError("list_hook must be a callable.") ctx.user.list_hook = list_hook - ret = template_execute(&ctx, buf, buf_len, &off) + ret = template_execute(&ctx, buf, buf_len, &off, True) if ret == 1: return template_data(&ctx) else: @@ -440,15 +440,12 @@ cdef class Unpacker(object): else: self.file_like = None - cpdef unpack(self): - """unpack one object""" + cpdef _unpack(self, bool construct): cdef int ret while 1: - ret = template_execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) + ret = template_execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head, construct) if ret == 1: - o = template_data(&self.ctx) - template_init(&self.ctx) - return o + return elif ret == 0: if self.file_like is not None: self.fill_buffer() @@ -457,6 +454,18 @@ cdef class Unpacker(object): else: raise ValueError("Unpack failed: error = %d" % (ret,)) + cpdef unpack(self): + """unpack one object""" + self._unpack(True) + o = template_data(&self.ctx) + template_init(&self.ctx) + + + cpdef skip(self): + """read and ignore one object, returning None""" + self._unpack(False) + template_init(&self.ctx) + def __iter__(self): return self diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index b844a24..10e41e1 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -95,7 +95,7 @@ msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context } -msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off) +msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off, bool construct) { assert(len >= *off); @@ -117,14 +117,17 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c int ret; +#define construct_cb(name) \ + construct && msgpack_unpack_callback(name) + #define push_simple_value(func) \ - if(msgpack_unpack_callback(func)(user, &obj) < 0) { goto _failed; } \ + if(construct_cb(func)(user, &obj) < 0) { goto _failed; } \ goto _push #define push_fixed_value(func, arg) \ - if(msgpack_unpack_callback(func)(user, arg, &obj) < 0) { goto _failed; } \ + if(construct_cb(func)(user, arg, &obj) < 0) { goto _failed; } \ goto _push #define push_variable_value(func, base, pos, len) \ - if(msgpack_unpack_callback(func)(user, \ + if(construct_cb(func)(user, \ (const char*)base, (const char*)pos, len, &obj) < 0) { goto _failed; } \ goto _push @@ -140,9 +143,9 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c #define start_container(func, count_, ct_) \ if(top >= MSGPACK_EMBED_STACK_SIZE) { goto _failed; } /* FIXME */ \ - if(msgpack_unpack_callback(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \ + if(construct_cb(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \ if((count_) == 0) { obj = stack[top].obj; \ - msgpack_unpack_callback(func##_end)(user, &obj); \ + construct_cb(func##_end)(user, &obj); \ goto _push; } \ stack[top].ct = ct_; \ stack[top].size = count_; \ @@ -340,10 +343,10 @@ _push: c = &stack[top-1]; switch(c->ct) { case CT_ARRAY_ITEM: - if(msgpack_unpack_callback(_array_item)(user, c->count, &c->obj, obj) < 0) { goto _failed; } + if(construct_cb(_array_item)(user, c->count, &c->obj, obj) < 0) { goto _failed; } if(++c->count == c->size) { obj = c->obj; - msgpack_unpack_callback(_array_end)(user, &obj); + construct_cb(_array_end)(user, &obj); --top; /*printf("stack pop %d\n", top);*/ goto _push; @@ -354,10 +357,10 @@ _push: c->ct = CT_MAP_VALUE; goto _header_again; case CT_MAP_VALUE: - if(msgpack_unpack_callback(_map_item)(user, &c->obj, c->map_key, obj) < 0) { goto _failed; } + if(construct_cb(_map_item)(user, &c->obj, c->map_key, obj) < 0) { goto _failed; } if(++c->count == c->size) { obj = c->obj; - msgpack_unpack_callback(_map_end)(user, &obj); + construct_cb(_map_end)(user, &obj); --top; /*printf("stack pop %d\n", top);*/ goto _push; @@ -399,6 +402,7 @@ _end: *off = p - (const unsigned char*)data; return ret; +#undef construct_cb } -- cgit v1.2.1 From 4d643894a1ab02b0836245b8a456200cac5ae314 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 23 Sep 2012 02:13:32 +0900 Subject: Support packing subclass of dict. --- ChangeLog.rst | 11 +++++++++++ msgpack/_msgpack.pyx | 16 +++++++++++++--- test/test_pack.py | 26 ++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 3 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 4fd5cbc..46b83ee 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,14 @@ +0.2.3 +======= +:release date: in development + +Changes +------- + +Bugs fixed +----------- +* Can't pack subclass of dict. + 0.2.2 ======= :release date: 2012-09-21 diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index c9f5e31..976871e 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -139,11 +139,19 @@ cdef class Packer(object): ret = msgpack_pack_raw(&self.pk, len(o)) if ret == 0: ret = msgpack_pack_raw_body(&self.pk, rawval, len(o)) - elif PyDict_Check(o): + elif PyDict_CheckExact(o): d = o ret = msgpack_pack_map(&self.pk, len(d)) if ret == 0: - for k,v in d.iteritems(): + for k, v in d.iteritems(): + ret = self._pack(k, nest_limit-1) + if ret != 0: break + ret = self._pack(v, nest_limit-1) + if ret != 0: break + elif PyDict_Check(o): + ret = msgpack_pack_map(&self.pk, len(o)) + if ret == 0: + for k, v in o.items(): ret = self._pack(k, nest_limit-1) if ret != 0: break ret = self._pack(v, nest_limit-1) @@ -332,7 +340,9 @@ cdef class Unpacker(object): def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=0, object object_hook=None, object list_hook=None, - encoding=None, unicode_errors='strict', int max_buffer_size=0): + encoding=None, unicode_errors='strict', int max_buffer_size=0, + object object_pairs_hook=None, + ): self.use_list = use_list self.file_like = file_like if file_like: diff --git a/test/test_pack.py b/test/test_pack.py index 85d11a0..b216c46 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -91,5 +91,31 @@ def testPackFloat(): assert_equal(packb(1.0, use_single_float=True), b'\xca' + struct.pack('>f', 1.0)) assert_equal(packb(1.0, use_single_float=False), b'\xcb' + struct.pack('>d', 1.0)) + +class odict(dict): + '''Reimplement OrderedDict to run test on Python 2.6''' + def __init__(self, seq): + self._seq = seq + dict.__init__(self, seq) + + def items(self): + return self._seq[:] + + def iteritems(self): + return iter(self._seq) + + def keys(self): + return [x[0] for x in self._seq] + +def test_odict(): + seq = [(b'one', 1), (b'two', 2), (b'three', 3), (b'four', 4)] + od = odict(seq) + assert_equal(unpackb(packb(od)), dict(seq)) + # After object_pairs_hook is implemented. + #def pair_hook(seq): + # return seq + #assert_equal(unpackb(packb(od), object_pairs_hook=pair_hook), seq) + + if __name__ == '__main__': main() -- cgit v1.2.1 From 8b2959bc0ab086a3dbe47176b3c241dd1a1ecf6c Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 23 Sep 2012 03:39:14 +0900 Subject: pack and packb raises ValueError when extra data passed. --- msgpack/_msgpack.pyx | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 976871e..12ee2ea 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -257,7 +257,10 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, ctx.user.list_hook = list_hook ret = template_execute(&ctx, buf, buf_len, &off) if ret == 1: - return template_data(&ctx) + obj = template_data(&ctx) + if off < buf_len: + raise ValueError("Extra data.") + return obj else: return None @@ -461,7 +464,7 @@ cdef class Unpacker(object): if self.file_like is not None: self.read_from_file() continue - raise StopIteration("No more unpack data.") + raise StopIteration("No more data to unpack.") else: raise ValueError("Unpack failed: error = %d" % (ret,)) -- cgit v1.2.1 From 36b88b407718cb7b96f7ae12cffbb0c8a27e493b Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 23 Sep 2012 03:44:41 +0900 Subject: Add Roadmap. --- ROADMAP.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 ROADMAP.md diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 0000000..5245cc0 --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1,7 @@ +0.2 series +========== +Improve compatibility to simplejson. + +0.3 series +========== +Add features msgpack-ruby has. -- cgit v1.2.1 From e8842efdedb1917a28147aa8ad1bf6f7b729a751 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 23 Sep 2012 08:57:32 +0900 Subject: Add py33 to tox. --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 5e80dd0..214c4c4 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,6 @@ [tox] -envlist = py26,py27,py32 +envlist = py26,py27,py32,py33 + [testenv] deps= nose -- cgit v1.2.1 From eaf9891b4255f3b1ca5cf2ea5b631091523b913d Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 23 Sep 2012 10:00:18 +0900 Subject: clean some cython code. --- msgpack/_msgpack.pyx | 28 +++++++++++++++------------- msgpack/unpack_template.h | 2 +- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 9061d42..c8ee7bb 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -209,7 +209,7 @@ cdef extern from "unpack.h": PyObject* key int template_execute(template_context* ctx, const_char_ptr data, - size_t len, size_t* off, bool construct) except -1 + size_t len, size_t* off, bint construct) except -1 void template_init(template_context* ctx) object template_data(template_context* ctx) @@ -255,7 +255,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, if not PyCallable_Check(list_hook): raise TypeError("list_hook must be a callable.") ctx.user.list_hook = list_hook - ret = template_execute(&ctx, buf, buf_len, &off, True) + ret = template_execute(&ctx, buf, buf_len, &off, 1) if ret == 1: obj = template_data(&ctx) if off < buf_len: @@ -451,12 +451,18 @@ cdef class Unpacker(object): else: self.file_like = None - cpdef _unpack(self, bool construct): + cdef _unpack(self, bint construct): cdef int ret + cdef object obj while 1: ret = template_execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head, construct) if ret == 1: - return + if construct: + obj = template_data(&self.ctx) + else: + obj = None + template_init(&self.ctx) + return obj elif ret == 0: if self.file_like is not None: self.read_from_file() @@ -465,23 +471,19 @@ cdef class Unpacker(object): else: raise ValueError("Unpack failed: error = %d" % (ret,)) - cpdef unpack(self): + def unpack(self): """unpack one object""" - self._unpack(True) - o = template_data(&self.ctx) - template_init(&self.ctx) - + return self._unpack(1) - cpdef skip(self): + def skip(self): """read and ignore one object, returning None""" - self._unpack(False) - template_init(&self.ctx) + return self._unpack(0) def __iter__(self): return self def __next__(self): - return self.unpack() + return self._unpack(1) # for debug. #def _buf(self): diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 10e41e1..5495a51 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -95,7 +95,7 @@ msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context } -msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off, bool construct) +msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off, int construct) { assert(len >= *off); -- cgit v1.2.1 From 7d142d2bef0805a528f1cd84e173579209298380 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 23 Sep 2012 10:02:11 +0900 Subject: Add changelog --- ChangeLog.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ChangeLog.rst b/ChangeLog.rst index 46b83ee..fe5b820 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,12 @@ +0.3.0 +===== +:release date: in development + +Changes +------- +* Add ``.skip()`` method to ``Unpacker`` (thanks to jnothman) + + 0.2.3 ======= :release date: in development -- cgit v1.2.1 From 48d693c1b9613fd976a3bf668f692ec22ad4a520 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 23 Sep 2012 10:09:51 +0900 Subject: Add test for `.skip()` --- msgpack/_msgpack.pyx | 2 +- test/test_sequnpack.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index c8ee7bb..8d37aaa 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -451,7 +451,7 @@ cdef class Unpacker(object): else: self.file_like = None - cdef _unpack(self, bint construct): + cdef object _unpack(self, bint construct): cdef int ret cdef object obj while 1: diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index b1b80b2..aa47d3c 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -28,6 +28,20 @@ def test_foobar(): k += 1 assert k == len(b'foobar') +def test_foobar_skip(): + unpacker = Unpacker(read_size=3) + unpacker.feed(b'foobar') + assert unpacker.unpack() == ord(b'f') + unpacker.skip() + assert unpacker.unpack() == ord(b'o') + unpacker.skip() + assert unpacker.unpack() == ord(b'a') + unpacker.skip() + try: + o = unpacker.unpack() + assert 0, "should raise exception" + except StopIteration: + assert 1, "ok" def test_maxbuffersize(): nose.tools.assert_raises(ValueError, Unpacker, read_size=5, max_buffer_size=3) -- cgit v1.2.1 From c3da8458681fc479233910d4c92dc84374e5efed Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 23 Sep 2012 11:16:59 +0900 Subject: Add docstring about raising ValueError when there are extra bytes. --- msgpack/_msgpack.pyx | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 12ee2ea..0886580 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -217,8 +217,10 @@ cdef extern from "unpack.h": def unpackb(object packed, object object_hook=None, object list_hook=None, bint use_list=0, encoding=None, unicode_errors="strict", ): + """Unpack packed_bytes to object. Returns an unpacked object. + + Raises `ValueError` when `packed` contains extra bytes. """ - Unpack packed_bytes to object. Returns an unpacked object.""" cdef template_context ctx cdef size_t off = 0 cdef int ret @@ -268,14 +270,16 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, def unpack(object stream, object object_hook=None, object list_hook=None, bint use_list=0, encoding=None, unicode_errors="strict", ): - """ - unpack an object from stream. + """Unpack an object from `stream`. + + Raises `ValueError` when `stream` has extra bytes. """ return unpackb(stream.read(), use_list=use_list, object_hook=object_hook, list_hook=list_hook, encoding=encoding, unicode_errors=unicode_errors, ) + cdef class Unpacker(object): """ Streaming unpacker. -- cgit v1.2.1 From b06ed8eb75563111ef88a119f9f7a45e67f61736 Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Sun, 23 Sep 2012 18:11:49 +1000 Subject: Factor context initialisation from unpackb and Unpacker --- msgpack/_msgpack.pyx | 82 ++++++++++++++++++---------------------------------- 1 file changed, 28 insertions(+), 54 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index e0a1043..823ed62 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -213,6 +213,32 @@ cdef extern from "unpack.h": void template_init(template_context* ctx) object template_data(template_context* ctx) +cdef inline init_ctx(template_context *ctx, object object_hook, object list_hook, bint use_list, encoding, unicode_errors): + template_init(ctx) + ctx.user.use_list = use_list + ctx.user.object_hook = ctx.user.list_hook = NULL + if object_hook is not None: + if not PyCallable_Check(object_hook): + raise TypeError("object_hook must be a callable.") + ctx.user.object_hook = object_hook + if list_hook is not None: + if not PyCallable_Check(list_hook): + raise TypeError("list_hook must be a callable.") + ctx.user.list_hook = list_hook + if encoding is None: + ctx.user.encoding = NULL + ctx.user.unicode_errors = NULL + else: + if isinstance(encoding, unicode): + _bencoding = encoding.encode('ascii') + else: + _bencoding = encoding + ctx.user.encoding = PyBytes_AsString(_bencoding) + if isinstance(unicode_errors, unicode): + _berrors = unicode_errors.encode('ascii') + else: + _berrors = unicode_errors + ctx.user.unicode_errors = PyBytes_AsString(_berrors) def unpackb(object packed, object object_hook=None, object list_hook=None, bint use_list=0, encoding=None, unicode_errors="strict", @@ -229,34 +255,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, cdef Py_ssize_t buf_len PyObject_AsReadBuffer(packed, &buf, &buf_len) - if encoding is None: - enc = NULL - err = NULL - else: - if isinstance(encoding, unicode): - bencoding = encoding.encode('ascii') - else: - bencoding = encoding - if isinstance(unicode_errors, unicode): - berrors = unicode_errors.encode('ascii') - else: - berrors = unicode_errors - enc = PyBytes_AsString(bencoding) - err = PyBytes_AsString(berrors) - - template_init(&ctx) - ctx.user.use_list = use_list - ctx.user.object_hook = ctx.user.list_hook = NULL - ctx.user.encoding = enc - ctx.user.unicode_errors = err - if object_hook is not None: - if not PyCallable_Check(object_hook): - raise TypeError("object_hook must be a callable.") - ctx.user.object_hook = object_hook - if list_hook is not None: - if not PyCallable_Check(list_hook): - raise TypeError("list_hook must be a callable.") - ctx.user.list_hook = list_hook + init_ctx(&ctx, object_hook, list_hook, use_list, encoding, unicode_errors) ret = template_execute(&ctx, buf, buf_len, &off, 1) if ret == 1: obj = template_data(&ctx) @@ -348,7 +347,6 @@ cdef class Unpacker(object): def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=0, object object_hook=None, object list_hook=None, encoding=None, unicode_errors='strict', int max_buffer_size=0, - object object_pairs_hook=None, ): self.use_list = use_list self.file_like = file_like @@ -370,31 +368,7 @@ cdef class Unpacker(object): self.buf_size = read_size self.buf_head = 0 self.buf_tail = 0 - template_init(&self.ctx) - self.ctx.user.use_list = use_list - self.ctx.user.object_hook = self.ctx.user.list_hook = NULL - if object_hook is not None: - if not PyCallable_Check(object_hook): - raise TypeError("object_hook must be a callable.") - self.ctx.user.object_hook = object_hook - if list_hook is not None: - if not PyCallable_Check(list_hook): - raise TypeError("list_hook must be a callable.") - self.ctx.user.list_hook = list_hook - if encoding is None: - self.ctx.user.encoding = NULL - self.ctx.user.unicode_errors = NULL - else: - if isinstance(encoding, unicode): - self._bencoding = encoding.encode('ascii') - else: - self._bencoding = encoding - self.ctx.user.encoding = PyBytes_AsString(self._bencoding) - if isinstance(unicode_errors, unicode): - self._berrors = unicode_errors.encode('ascii') - else: - self._berrors = unicode_errors - self.ctx.user.unicode_errors = PyBytes_AsString(self._berrors) + init_ctx(&self.ctx, object_hook, list_hook, use_list, encoding, unicode_errors) def feed(self, object next_bytes): cdef char* buf -- cgit v1.2.1 From 77942514db0c5a80e9f3f9bcb1e1939ecc8705e6 Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Sun, 23 Sep 2012 19:37:28 +1000 Subject: Implement object_pairs_hook --- msgpack/_msgpack.pyx | 34 ++++++++++++++++++++++++++-------- msgpack/unpack.h | 30 +++++++++++++++++++++--------- msgpack/unpack_template.h | 2 +- test/test_obj.py | 10 ++++++++++ test/test_pack.py | 7 +++---- 5 files changed, 61 insertions(+), 22 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 823ed62..b6d8e8b 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -197,6 +197,7 @@ cdef extern from "unpack.h": ctypedef struct msgpack_user: int use_list PyObject* object_hook + bint has_pairs_hook # call object_hook with k-v pairs PyObject* list_hook char *encoding char *unicode_errors @@ -213,18 +214,32 @@ cdef extern from "unpack.h": void template_init(template_context* ctx) object template_data(template_context* ctx) -cdef inline init_ctx(template_context *ctx, object object_hook, object list_hook, bint use_list, encoding, unicode_errors): +cdef inline init_ctx(template_context *ctx, object object_hook, object object_pairs_hook, object list_hook, bint use_list, encoding, unicode_errors): template_init(ctx) ctx.user.use_list = use_list ctx.user.object_hook = ctx.user.list_hook = NULL + + if object_hook is not None and object_pairs_hook is not None: + raise ValueError("object_pairs_hook and object_hook are mutually exclusive.") + if object_hook is not None: if not PyCallable_Check(object_hook): raise TypeError("object_hook must be a callable.") ctx.user.object_hook = object_hook + + if object_pairs_hook is None: + ctx.user.has_pairs_hook = False + else: + if not PyCallable_Check(object_pairs_hook): + raise TypeError("object_pairs_hook must be a callable.") + ctx.user.object_hook = object_pairs_hook + ctx.user.has_pairs_hook = True + if list_hook is not None: if not PyCallable_Check(list_hook): raise TypeError("list_hook must be a callable.") ctx.user.list_hook = list_hook + if encoding is None: ctx.user.encoding = NULL ctx.user.unicode_errors = NULL @@ -240,7 +255,7 @@ cdef inline init_ctx(template_context *ctx, object object_hook, object list_hook _berrors = unicode_errors ctx.user.unicode_errors = PyBytes_AsString(_berrors) -def unpackb(object packed, object object_hook=None, object list_hook=None, +def unpackb(object packed, object object_hook=None, object object_pairs_hook=None, object list_hook=None, bint use_list=0, encoding=None, unicode_errors="strict", ): """Unpack packed_bytes to object. Returns an unpacked object. @@ -255,7 +270,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, cdef Py_ssize_t buf_len PyObject_AsReadBuffer(packed, &buf, &buf_len) - init_ctx(&ctx, object_hook, list_hook, use_list, encoding, unicode_errors) + init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, use_list, encoding, unicode_errors) ret = template_execute(&ctx, buf, buf_len, &off, 1) if ret == 1: obj = template_data(&ctx) @@ -266,7 +281,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, return None -def unpack(object stream, object object_hook=None, object list_hook=None, +def unpack(object stream, object object_hook=None, object object_pairs_hook=None, object list_hook=None, bint use_list=0, encoding=None, unicode_errors="strict", ): """Unpack an object from `stream`. @@ -274,7 +289,7 @@ def unpack(object stream, object object_hook=None, object list_hook=None, Raises `ValueError` when `stream` has extra bytes. """ return unpackb(stream.read(), use_list=use_list, - object_hook=object_hook, list_hook=list_hook, + object_hook=object_hook, object_pairs_hook=object_pairs_hook, list_hook=list_hook, encoding=encoding, unicode_errors=unicode_errors, ) @@ -294,7 +309,10 @@ cdef class Unpacker(object): Otherwise, it is deserialized to Python tuple. (default: False) `object_hook` is same to simplejson. If it is not None, it should be callable - and Unpacker calls it when deserializing key-value. + and Unpacker calls it with a dict argument after deserializing a map. + + `object_pairs_hook` is same to simplejson. If it is not None, it should be callable + and Unpacker calls it with a list of key-value pairs after deserializing a map. `encoding` is encoding used for decoding msgpack bytes. If it is None (default), msgpack bytes is deserialized to Python bytes. @@ -345,7 +363,7 @@ cdef class Unpacker(object): self.buf = NULL def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=0, - object object_hook=None, object list_hook=None, + object object_hook=None, object object_pairs_hook=None, object list_hook=None, encoding=None, unicode_errors='strict', int max_buffer_size=0, ): self.use_list = use_list @@ -368,7 +386,7 @@ cdef class Unpacker(object): self.buf_size = read_size self.buf_head = 0 self.buf_tail = 0 - init_ctx(&self.ctx, object_hook, list_hook, use_list, encoding, unicode_errors) + init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, use_list, encoding, unicode_errors) def feed(self, object next_bytes): cdef char* buf diff --git a/msgpack/unpack.h b/msgpack/unpack.h index a106f9c..7064a1b 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -22,6 +22,7 @@ typedef struct unpack_user { int use_list; PyObject *object_hook; + bool has_pairs_hook; PyObject *list_hook; const char *encoding; const char *unicode_errors; @@ -160,9 +161,7 @@ static inline int template_callback_array_item(unpack_user* u, unsigned int curr static inline int template_callback_array_end(unpack_user* u, msgpack_unpack_object* c) { if (u->list_hook) { - PyObject *arglist = Py_BuildValue("(O)", *c); - PyObject *new_c = PyEval_CallObject(u->list_hook, arglist); - Py_DECREF(arglist); + PyObject *new_c = PyEval_CallFunction(u->list_hook, "(O)", *c); Py_DECREF(*c); *c = new_c; } @@ -171,16 +170,31 @@ static inline int template_callback_array_end(unpack_user* u, msgpack_unpack_obj static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { - PyObject *p = PyDict_New(); + PyObject *p; + if (u->has_pairs_hook) { + p = PyList_New(n); // Or use tuple? + } + else { + p = PyDict_New(); + } if (!p) return -1; *o = p; return 0; } -static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v) +static inline int template_callback_map_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v) { - if (PyDict_SetItem(*c, k, v) == 0) { + if (u->has_pairs_hook) { + msgpack_unpack_object item = PyTuple_Pack(2, k, v); + if (!item) + return -1; + Py_DECREF(k); + Py_DECREF(v); + PyList_SET_ITEM(*c, current, item); + return 0; + } + else if (PyDict_SetItem(*c, k, v) == 0) { Py_DECREF(k); Py_DECREF(v); return 0; @@ -191,9 +205,7 @@ static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_obje static inline int template_callback_map_end(unpack_user* u, msgpack_unpack_object* c) { if (u->object_hook) { - PyObject *arglist = Py_BuildValue("(O)", *c); - PyObject *new_c = PyEval_CallObject(u->object_hook, arglist); - Py_DECREF(arglist); + PyObject *new_c = PyEval_CallFunction(u->object_hook, "(O)", *c); Py_DECREF(*c); *c = new_c; } diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 5495a51..6080a51 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -357,7 +357,7 @@ _push: c->ct = CT_MAP_VALUE; goto _header_again; case CT_MAP_VALUE: - if(construct_cb(_map_item)(user, &c->obj, c->map_key, obj) < 0) { goto _failed; } + if(construct_cb(_map_item)(user, c->count, &c->obj, c->map_key, obj) < 0) { goto _failed; } if(++c->count == c->size) { obj = c->obj; construct_cb(_map_end)(user, &obj); diff --git a/test/test_obj.py b/test/test_obj.py index d155b73..e0d89fc 100644 --- a/test/test_obj.py +++ b/test/test_obj.py @@ -26,6 +26,16 @@ def test_decode_hook(): unpacked = unpackb(packed, object_hook=_decode_complex) eq_(unpacked[1], 1+2j) +def test_decode_pairs_hook(): + packed = packb([3, {1: 2, 3: 4}]) + prod_sum = 1 * 2 + 3 * 4 + unpacked = unpackb(packed, object_pairs_hook=lambda l: sum(k * v for k, v in l)) + eq_(unpacked[1], prod_sum) + +@raises(ValueError) +def test_only_one_obj_hook(): + unpackb('', object_hook=lambda x: x, object_pairs_hook=lambda x: x) + @raises(ValueError) def test_bad_hook(): packed = packb([3, 1+2j], default=lambda o: o) diff --git a/test/test_pack.py b/test/test_pack.py index b216c46..2c99873 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -111,10 +111,9 @@ def test_odict(): seq = [(b'one', 1), (b'two', 2), (b'three', 3), (b'four', 4)] od = odict(seq) assert_equal(unpackb(packb(od)), dict(seq)) - # After object_pairs_hook is implemented. - #def pair_hook(seq): - # return seq - #assert_equal(unpackb(packb(od), object_pairs_hook=pair_hook), seq) + def pair_hook(seq): + return seq + assert_equal(unpackb(packb(od), object_pairs_hook=pair_hook), seq) if __name__ == '__main__': -- cgit v1.2.1 From 60df5eadaf507594b73e5e5a887da1fc52cb3f32 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 24 Sep 2012 02:12:55 +0900 Subject: Warn when use_list is not specified. --- msgpack/_msgpack.pyx | 30 ++++++++++++++++++++++-------- setup.py | 2 +- test/test_buffer.py | 4 ++-- test/test_format.py | 4 ++-- test/test_pack.py | 12 ++++++------ test/test_seq.py | 2 +- test/test_sequnpack.py | 6 +++--- 7 files changed, 37 insertions(+), 23 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index e0a1043..d0c4541 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -1,12 +1,16 @@ # coding: utf-8 #cython: embedsignature=True +import warnings + from cpython cimport * cdef extern from "Python.h": ctypedef char* const_char_ptr "const char*" ctypedef char* const_void_ptr "const void*" ctypedef struct PyObject cdef int PyObject_AsReadBuffer(object o, const_void_ptr* buff, Py_ssize_t* buf_len) except -1 + char* __FILE__ + int __LINE__ from libc.stdlib cimport * from libc.string cimport * @@ -195,7 +199,7 @@ def packb(object o, default=None, encoding='utf-8', unicode_errors='strict', use cdef extern from "unpack.h": ctypedef struct msgpack_user: - int use_list + bint use_list PyObject* object_hook PyObject* list_hook char *encoding @@ -215,7 +219,7 @@ cdef extern from "unpack.h": def unpackb(object packed, object object_hook=None, object list_hook=None, - bint use_list=0, encoding=None, unicode_errors="strict", + use_list=None, encoding=None, unicode_errors="strict", ): """Unpack packed_bytes to object. Returns an unpacked object. @@ -227,6 +231,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, cdef char* buf cdef Py_ssize_t buf_len + PyObject_AsReadBuffer(packed, &buf, &buf_len) if encoding is None: @@ -245,7 +250,11 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, err = PyBytes_AsString(berrors) template_init(&ctx) - ctx.user.use_list = use_list + if use_list is None: + warnings.warn("Set use_list explicitly.", category=DeprecationWarning, stacklevel=1) + ctx.user.use_list = 0 + else: + ctx.user.use_list = use_list ctx.user.object_hook = ctx.user.list_hook = NULL ctx.user.encoding = enc ctx.user.unicode_errors = err @@ -268,12 +277,15 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, def unpack(object stream, object object_hook=None, object list_hook=None, - bint use_list=0, encoding=None, unicode_errors="strict", + use_list=None, encoding=None, unicode_errors="strict", ): """Unpack an object from `stream`. Raises `ValueError` when `stream` has extra bytes. """ + if use_list is None: + warnings.warn("Set use_list explicitly.", category=DeprecationWarning, stacklevel=1) + use_list = 0 return unpackb(stream.read(), use_list=use_list, object_hook=object_hook, list_hook=list_hook, encoding=encoding, unicode_errors=unicode_errors, @@ -292,7 +304,7 @@ cdef class Unpacker(object): (default: min(1024**2, max_buffer_size)) If `use_list` is true, msgpack list is deserialized to Python list. - Otherwise, it is deserialized to Python tuple. (default: False) + Otherwise, it is deserialized to Python tuple. `object_hook` is same to simplejson. If it is not None, it should be callable and Unpacker calls it when deserializing key-value. @@ -330,7 +342,6 @@ cdef class Unpacker(object): cdef object file_like cdef object file_like_read cdef Py_ssize_t read_size - cdef bint use_list cdef object object_hook cdef object _bencoding cdef object _berrors @@ -345,12 +356,15 @@ cdef class Unpacker(object): free(self.buf) self.buf = NULL - def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=0, + def __init__(self, file_like=None, Py_ssize_t read_size=0, use_list=None, object object_hook=None, object list_hook=None, encoding=None, unicode_errors='strict', int max_buffer_size=0, object object_pairs_hook=None, ): - self.use_list = use_list + if use_list is None: + warnings.warn("Set use_list explicitly.", category=DeprecationWarning, stacklevel=1) + use_list = 0 + self.file_like = file_like if file_like: self.file_like_read = file_like.read diff --git a/setup.py b/setup.py index 86b0b34..9f0ce5d 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ except ImportError: def cythonize(src): sys.stderr.write("cythonize: %r\n" % (src,)) - cython_compiler.compile([src]) + cython_compiler.compile([src], emit_linenums=True) def ensure_source(src): pyx = os.path.splitext(src)[0] + '.pyx' diff --git a/test/test_buffer.py b/test/test_buffer.py index 01310a0..785fb60 100644 --- a/test/test_buffer.py +++ b/test/test_buffer.py @@ -9,8 +9,8 @@ def test_unpack_buffer(): from array import array buf = array('b') buf.fromstring(packb(('foo', 'bar'))) - obj = unpackb(buf) - assert_equal((b'foo', b'bar'), obj) + obj = unpackb(buf, use_list=1) + assert_equal([b'foo', b'bar'], obj) if __name__ == '__main__': main() diff --git a/test/test_format.py b/test/test_format.py index c03b3e2..ac08709 100644 --- a/test/test_format.py +++ b/test/test_format.py @@ -5,8 +5,8 @@ from nose import main from nose.tools import * from msgpack import unpackb -def check(src, should): - assert_equal(unpackb(src), should) +def check(src, should, use_list=0): + assert_equal(unpackb(src, use_list=use_list), should) def testSimpleValue(): check(b"\x93\xc0\xc2\xc3", diff --git a/test/test_pack.py b/test/test_pack.py index b216c46..dc77dfe 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -11,8 +11,8 @@ from msgpack import packb, unpackb, Unpacker, Packer from io import BytesIO -def check(data): - re = unpackb(packb(data)) +def check(data, use_list=False): + re = unpackb(packb(data), use_list=use_list) assert_equal(re, data) def testPack(): @@ -34,7 +34,7 @@ def testPackUnicode(): six.u(""), six.u("abcd"), (six.u("defgh"),), six.u("Русский текст"), ] for td in test_data: - re = unpackb(packb(td, encoding='utf-8'), encoding='utf-8') + re = unpackb(packb(td, encoding='utf-8'), use_list=0, encoding='utf-8') assert_equal(re, td) packer = Packer(encoding='utf-8') data = packer.pack(td) @@ -46,11 +46,11 @@ def testPackUTF32(): test_data = [ six.u(""), six.u("abcd"), - (six.u("defgh"),), + [six.u("defgh")], six.u("Русский текст"), ] for td in test_data: - re = unpackb(packb(td, encoding='utf-32'), encoding='utf-32') + re = unpackb(packb(td, encoding='utf-32'), use_list=1, encoding='utf-32') assert_equal(re, td) except LookupError: raise SkipTest @@ -110,7 +110,7 @@ class odict(dict): def test_odict(): seq = [(b'one', 1), (b'two', 2), (b'three', 3), (b'four', 4)] od = odict(seq) - assert_equal(unpackb(packb(od)), dict(seq)) + assert_equal(unpackb(packb(od), use_list=1), dict(seq)) # After object_pairs_hook is implemented. #def pair_hook(seq): # return seq diff --git a/test/test_seq.py b/test/test_seq.py index d0f9ccc..72e935a 100644 --- a/test/test_seq.py +++ b/test/test_seq.py @@ -34,7 +34,7 @@ def test_exceeding_unpacker_read_size(): f = io.BytesIO(dumpf.getvalue()) dumpf.close() - unpacker = msgpack.Unpacker(f, read_size=read_size) + unpacker = msgpack.Unpacker(f, read_size=read_size, use_list=1) read_count = 0 for idx, o in enumerate(unpacker): diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index aa47d3c..dac36a8 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -5,7 +5,7 @@ from msgpack import Unpacker, BufferFull import nose def test_foobar(): - unpacker = Unpacker(read_size=3) + unpacker = Unpacker(read_size=3, use_list=1) unpacker.feed(b'foobar') assert unpacker.unpack() == ord(b'f') assert unpacker.unpack() == ord(b'o') @@ -29,7 +29,7 @@ def test_foobar(): assert k == len(b'foobar') def test_foobar_skip(): - unpacker = Unpacker(read_size=3) + unpacker = Unpacker(read_size=3, use_list=1) unpacker.feed(b'foobar') assert unpacker.unpack() == ord(b'f') unpacker.skip() @@ -45,7 +45,7 @@ def test_foobar_skip(): def test_maxbuffersize(): nose.tools.assert_raises(ValueError, Unpacker, read_size=5, max_buffer_size=3) - unpacker = Unpacker(read_size=3, max_buffer_size=3) + unpacker = Unpacker(read_size=3, max_buffer_size=3, use_list=1) unpacker.feed(b'fo') nose.tools.assert_raises(BufferFull, unpacker.feed, b'ob') unpacker.feed(b'o') -- cgit v1.2.1 From c2a2d417f1e3a95992b41c3b24e6470077c99c6f Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 24 Sep 2012 02:20:53 +0900 Subject: Fix warnings in tests. --- test/test_case.py | 6 +++--- test/test_obj.py | 8 ++++---- test/test_pack.py | 15 +++++++-------- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/test/test_case.py b/test/test_case.py index b88714d..9cbf9bd 100644 --- a/test/test_case.py +++ b/test/test_case.py @@ -9,7 +9,7 @@ from msgpack import packb, unpackb def check(length, obj): v = packb(obj) assert_equal(len(v), length, "%r length should be %r but get %r" % (obj, length, len(v))) - assert_equal(unpackb(v), obj) + assert_equal(unpackb(v, use_list=0), obj) def test_1(): for o in [None, True, False, 0, 1, (1 << 6), (1 << 7) - 1, -1, @@ -71,7 +71,7 @@ def test_array32(): def match(obj, buf): assert_equal(packb(obj), buf) - assert_equal(unpackb(buf), obj) + assert_equal(unpackb(buf, use_list=0), obj) def test_match(): cases = [ @@ -99,7 +99,7 @@ def test_match(): match(v, p) def test_unicode(): - assert_equal(b'foobar', unpackb(packb('foobar'))) + assert_equal(b'foobar', unpackb(packb('foobar'), use_list=1)) if __name__ == '__main__': main() diff --git a/test/test_obj.py b/test/test_obj.py index d155b73..d809093 100644 --- a/test/test_obj.py +++ b/test/test_obj.py @@ -18,25 +18,25 @@ def _encode_complex(obj): def test_encode_hook(): packed = packb([3, 1+2j], default=_encode_complex) - unpacked = unpackb(packed) + unpacked = unpackb(packed, use_list=1) eq_(unpacked[1], {b'__complex__': True, b'real': 1, b'imag': 2}) def test_decode_hook(): packed = packb([3, {b'__complex__': True, b'real': 1, b'imag': 2}]) - unpacked = unpackb(packed, object_hook=_decode_complex) + unpacked = unpackb(packed, object_hook=_decode_complex, use_list=1) eq_(unpacked[1], 1+2j) @raises(ValueError) def test_bad_hook(): packed = packb([3, 1+2j], default=lambda o: o) - unpacked = unpackb(packed) + unpacked = unpackb(packed, use_list=1) def _arr_to_str(arr): return ''.join(str(c) for c in arr) def test_array_hook(): packed = packb([1,2,3]) - unpacked = unpackb(packed, list_hook=_arr_to_str) + unpacked = unpackb(packed, list_hook=_arr_to_str, use_list=1) eq_(unpacked, '123') if __name__ == '__main__': diff --git a/test/test_pack.py b/test/test_pack.py index dc77dfe..9bd2b32 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -31,14 +31,14 @@ def testPack(): def testPackUnicode(): test_data = [ - six.u(""), six.u("abcd"), (six.u("defgh"),), six.u("Русский текст"), + six.u(""), six.u("abcd"), [six.u("defgh")], six.u("Русский текст"), ] for td in test_data: - re = unpackb(packb(td, encoding='utf-8'), use_list=0, encoding='utf-8') + re = unpackb(packb(td, encoding='utf-8'), use_list=1, encoding='utf-8') assert_equal(re, td) packer = Packer(encoding='utf-8') data = packer.pack(td) - re = Unpacker(BytesIO(data), encoding='utf-8').unpack() + re = Unpacker(BytesIO(data), encoding='utf-8', use_list=1).unpack() assert_equal(re, td) def testPackUTF32(): @@ -63,20 +63,19 @@ def testPackBytes(): check(td) def testIgnoreUnicodeErrors(): - re = unpackb(packb(b'abc\xeddef'), - encoding='utf-8', unicode_errors='ignore') + re = unpackb(packb(b'abc\xeddef'), encoding='utf-8', unicode_errors='ignore', use_list=1) assert_equal(re, "abcdef") @raises(UnicodeDecodeError) def testStrictUnicodeUnpack(): - unpackb(packb(b'abc\xeddef'), encoding='utf-8') + unpackb(packb(b'abc\xeddef'), encoding='utf-8', use_list=1) @raises(UnicodeEncodeError) def testStrictUnicodePack(): packb(six.u("abc\xeddef"), encoding='ascii', unicode_errors='strict') def testIgnoreErrorsPack(): - re = unpackb(packb(six.u("abcФФФdef"), encoding='ascii', unicode_errors='ignore'), encoding='utf-8') + re = unpackb(packb(six.u("abcФФФdef"), encoding='ascii', unicode_errors='ignore'), encoding='utf-8', use_list=1) assert_equal(re, six.u("abcdef")) @raises(TypeError) @@ -84,7 +83,7 @@ def testNoEncoding(): packb(six.u("abc"), encoding=None) def testDecodeBinary(): - re = unpackb(packb("abc"), encoding=None) + re = unpackb(packb("abc"), encoding=None, use_list=1) assert_equal(re, b"abc") def testPackFloat(): -- cgit v1.2.1 From d503788e9537498ff2ed0da1f836dc4de6074981 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 24 Sep 2012 02:12:55 +0900 Subject: Warn when use_list is not specified. Conflicts: test/test_sequnpack.py --- msgpack/_msgpack.pyx | 30 ++++++++++++++++++++++-------- setup.py | 2 +- test/test_buffer.py | 4 ++-- test/test_format.py | 4 ++-- test/test_pack.py | 12 ++++++------ test/test_seq.py | 2 +- test/test_sequnpack.py | 5 ++--- 7 files changed, 36 insertions(+), 23 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 0886580..44a50ae 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -1,12 +1,16 @@ # coding: utf-8 #cython: embedsignature=True +import warnings + from cpython cimport * cdef extern from "Python.h": ctypedef char* const_char_ptr "const char*" ctypedef char* const_void_ptr "const void*" ctypedef struct PyObject cdef int PyObject_AsReadBuffer(object o, const_void_ptr* buff, Py_ssize_t* buf_len) except -1 + char* __FILE__ + int __LINE__ from libc.stdlib cimport * from libc.string cimport * @@ -195,7 +199,7 @@ def packb(object o, default=None, encoding='utf-8', unicode_errors='strict', use cdef extern from "unpack.h": ctypedef struct msgpack_user: - int use_list + bint use_list PyObject* object_hook PyObject* list_hook char *encoding @@ -215,7 +219,7 @@ cdef extern from "unpack.h": def unpackb(object packed, object object_hook=None, object list_hook=None, - bint use_list=0, encoding=None, unicode_errors="strict", + use_list=None, encoding=None, unicode_errors="strict", ): """Unpack packed_bytes to object. Returns an unpacked object. @@ -227,6 +231,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, cdef char* buf cdef Py_ssize_t buf_len + PyObject_AsReadBuffer(packed, &buf, &buf_len) if encoding is None: @@ -245,7 +250,11 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, err = PyBytes_AsString(berrors) template_init(&ctx) - ctx.user.use_list = use_list + if use_list is None: + warnings.warn("Set use_list explicitly.", category=DeprecationWarning, stacklevel=1) + ctx.user.use_list = 0 + else: + ctx.user.use_list = use_list ctx.user.object_hook = ctx.user.list_hook = NULL ctx.user.encoding = enc ctx.user.unicode_errors = err @@ -268,12 +277,15 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, def unpack(object stream, object object_hook=None, object list_hook=None, - bint use_list=0, encoding=None, unicode_errors="strict", + use_list=None, encoding=None, unicode_errors="strict", ): """Unpack an object from `stream`. Raises `ValueError` when `stream` has extra bytes. """ + if use_list is None: + warnings.warn("Set use_list explicitly.", category=DeprecationWarning, stacklevel=1) + use_list = 0 return unpackb(stream.read(), use_list=use_list, object_hook=object_hook, list_hook=list_hook, encoding=encoding, unicode_errors=unicode_errors, @@ -292,7 +304,7 @@ cdef class Unpacker(object): (default: min(1024**2, max_buffer_size)) If `use_list` is true, msgpack list is deserialized to Python list. - Otherwise, it is deserialized to Python tuple. (default: False) + Otherwise, it is deserialized to Python tuple. `object_hook` is same to simplejson. If it is not None, it should be callable and Unpacker calls it when deserializing key-value. @@ -330,7 +342,6 @@ cdef class Unpacker(object): cdef object file_like cdef object file_like_read cdef Py_ssize_t read_size - cdef bint use_list cdef object object_hook cdef object _bencoding cdef object _berrors @@ -345,12 +356,15 @@ cdef class Unpacker(object): free(self.buf) self.buf = NULL - def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=0, + def __init__(self, file_like=None, Py_ssize_t read_size=0, use_list=None, object object_hook=None, object list_hook=None, encoding=None, unicode_errors='strict', int max_buffer_size=0, object object_pairs_hook=None, ): - self.use_list = use_list + if use_list is None: + warnings.warn("Set use_list explicitly.", category=DeprecationWarning, stacklevel=1) + use_list = 0 + self.file_like = file_like if file_like: self.file_like_read = file_like.read diff --git a/setup.py b/setup.py index 86b0b34..9f0ce5d 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ except ImportError: def cythonize(src): sys.stderr.write("cythonize: %r\n" % (src,)) - cython_compiler.compile([src]) + cython_compiler.compile([src], emit_linenums=True) def ensure_source(src): pyx = os.path.splitext(src)[0] + '.pyx' diff --git a/test/test_buffer.py b/test/test_buffer.py index 01310a0..785fb60 100644 --- a/test/test_buffer.py +++ b/test/test_buffer.py @@ -9,8 +9,8 @@ def test_unpack_buffer(): from array import array buf = array('b') buf.fromstring(packb(('foo', 'bar'))) - obj = unpackb(buf) - assert_equal((b'foo', b'bar'), obj) + obj = unpackb(buf, use_list=1) + assert_equal([b'foo', b'bar'], obj) if __name__ == '__main__': main() diff --git a/test/test_format.py b/test/test_format.py index c03b3e2..ac08709 100644 --- a/test/test_format.py +++ b/test/test_format.py @@ -5,8 +5,8 @@ from nose import main from nose.tools import * from msgpack import unpackb -def check(src, should): - assert_equal(unpackb(src), should) +def check(src, should, use_list=0): + assert_equal(unpackb(src, use_list=use_list), should) def testSimpleValue(): check(b"\x93\xc0\xc2\xc3", diff --git a/test/test_pack.py b/test/test_pack.py index b216c46..dc77dfe 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -11,8 +11,8 @@ from msgpack import packb, unpackb, Unpacker, Packer from io import BytesIO -def check(data): - re = unpackb(packb(data)) +def check(data, use_list=False): + re = unpackb(packb(data), use_list=use_list) assert_equal(re, data) def testPack(): @@ -34,7 +34,7 @@ def testPackUnicode(): six.u(""), six.u("abcd"), (six.u("defgh"),), six.u("Русский текст"), ] for td in test_data: - re = unpackb(packb(td, encoding='utf-8'), encoding='utf-8') + re = unpackb(packb(td, encoding='utf-8'), use_list=0, encoding='utf-8') assert_equal(re, td) packer = Packer(encoding='utf-8') data = packer.pack(td) @@ -46,11 +46,11 @@ def testPackUTF32(): test_data = [ six.u(""), six.u("abcd"), - (six.u("defgh"),), + [six.u("defgh")], six.u("Русский текст"), ] for td in test_data: - re = unpackb(packb(td, encoding='utf-32'), encoding='utf-32') + re = unpackb(packb(td, encoding='utf-32'), use_list=1, encoding='utf-32') assert_equal(re, td) except LookupError: raise SkipTest @@ -110,7 +110,7 @@ class odict(dict): def test_odict(): seq = [(b'one', 1), (b'two', 2), (b'three', 3), (b'four', 4)] od = odict(seq) - assert_equal(unpackb(packb(od)), dict(seq)) + assert_equal(unpackb(packb(od), use_list=1), dict(seq)) # After object_pairs_hook is implemented. #def pair_hook(seq): # return seq diff --git a/test/test_seq.py b/test/test_seq.py index d0f9ccc..72e935a 100644 --- a/test/test_seq.py +++ b/test/test_seq.py @@ -34,7 +34,7 @@ def test_exceeding_unpacker_read_size(): f = io.BytesIO(dumpf.getvalue()) dumpf.close() - unpacker = msgpack.Unpacker(f, read_size=read_size) + unpacker = msgpack.Unpacker(f, read_size=read_size, use_list=1) read_count = 0 for idx, o in enumerate(unpacker): diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index b1b80b2..21fc3be 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -5,7 +5,7 @@ from msgpack import Unpacker, BufferFull import nose def test_foobar(): - unpacker = Unpacker(read_size=3) + unpacker = Unpacker(read_size=3, use_list=1) unpacker.feed(b'foobar') assert unpacker.unpack() == ord(b'f') assert unpacker.unpack() == ord(b'o') @@ -28,10 +28,9 @@ def test_foobar(): k += 1 assert k == len(b'foobar') - def test_maxbuffersize(): nose.tools.assert_raises(ValueError, Unpacker, read_size=5, max_buffer_size=3) - unpacker = Unpacker(read_size=3, max_buffer_size=3) + unpacker = Unpacker(read_size=3, max_buffer_size=3, use_list=1) unpacker.feed(b'fo') nose.tools.assert_raises(BufferFull, unpacker.feed, b'ob') unpacker.feed(b'o') -- cgit v1.2.1 From c280e589884f2b4afd064cffd08b0f353db93036 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 24 Sep 2012 02:20:53 +0900 Subject: Fix warnings in tests. --- test/test_case.py | 6 +++--- test/test_obj.py | 8 ++++---- test/test_pack.py | 15 +++++++-------- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/test/test_case.py b/test/test_case.py index b88714d..9cbf9bd 100644 --- a/test/test_case.py +++ b/test/test_case.py @@ -9,7 +9,7 @@ from msgpack import packb, unpackb def check(length, obj): v = packb(obj) assert_equal(len(v), length, "%r length should be %r but get %r" % (obj, length, len(v))) - assert_equal(unpackb(v), obj) + assert_equal(unpackb(v, use_list=0), obj) def test_1(): for o in [None, True, False, 0, 1, (1 << 6), (1 << 7) - 1, -1, @@ -71,7 +71,7 @@ def test_array32(): def match(obj, buf): assert_equal(packb(obj), buf) - assert_equal(unpackb(buf), obj) + assert_equal(unpackb(buf, use_list=0), obj) def test_match(): cases = [ @@ -99,7 +99,7 @@ def test_match(): match(v, p) def test_unicode(): - assert_equal(b'foobar', unpackb(packb('foobar'))) + assert_equal(b'foobar', unpackb(packb('foobar'), use_list=1)) if __name__ == '__main__': main() diff --git a/test/test_obj.py b/test/test_obj.py index d155b73..d809093 100644 --- a/test/test_obj.py +++ b/test/test_obj.py @@ -18,25 +18,25 @@ def _encode_complex(obj): def test_encode_hook(): packed = packb([3, 1+2j], default=_encode_complex) - unpacked = unpackb(packed) + unpacked = unpackb(packed, use_list=1) eq_(unpacked[1], {b'__complex__': True, b'real': 1, b'imag': 2}) def test_decode_hook(): packed = packb([3, {b'__complex__': True, b'real': 1, b'imag': 2}]) - unpacked = unpackb(packed, object_hook=_decode_complex) + unpacked = unpackb(packed, object_hook=_decode_complex, use_list=1) eq_(unpacked[1], 1+2j) @raises(ValueError) def test_bad_hook(): packed = packb([3, 1+2j], default=lambda o: o) - unpacked = unpackb(packed) + unpacked = unpackb(packed, use_list=1) def _arr_to_str(arr): return ''.join(str(c) for c in arr) def test_array_hook(): packed = packb([1,2,3]) - unpacked = unpackb(packed, list_hook=_arr_to_str) + unpacked = unpackb(packed, list_hook=_arr_to_str, use_list=1) eq_(unpacked, '123') if __name__ == '__main__': diff --git a/test/test_pack.py b/test/test_pack.py index dc77dfe..9bd2b32 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -31,14 +31,14 @@ def testPack(): def testPackUnicode(): test_data = [ - six.u(""), six.u("abcd"), (six.u("defgh"),), six.u("Русский текст"), + six.u(""), six.u("abcd"), [six.u("defgh")], six.u("Русский текст"), ] for td in test_data: - re = unpackb(packb(td, encoding='utf-8'), use_list=0, encoding='utf-8') + re = unpackb(packb(td, encoding='utf-8'), use_list=1, encoding='utf-8') assert_equal(re, td) packer = Packer(encoding='utf-8') data = packer.pack(td) - re = Unpacker(BytesIO(data), encoding='utf-8').unpack() + re = Unpacker(BytesIO(data), encoding='utf-8', use_list=1).unpack() assert_equal(re, td) def testPackUTF32(): @@ -63,20 +63,19 @@ def testPackBytes(): check(td) def testIgnoreUnicodeErrors(): - re = unpackb(packb(b'abc\xeddef'), - encoding='utf-8', unicode_errors='ignore') + re = unpackb(packb(b'abc\xeddef'), encoding='utf-8', unicode_errors='ignore', use_list=1) assert_equal(re, "abcdef") @raises(UnicodeDecodeError) def testStrictUnicodeUnpack(): - unpackb(packb(b'abc\xeddef'), encoding='utf-8') + unpackb(packb(b'abc\xeddef'), encoding='utf-8', use_list=1) @raises(UnicodeEncodeError) def testStrictUnicodePack(): packb(six.u("abc\xeddef"), encoding='ascii', unicode_errors='strict') def testIgnoreErrorsPack(): - re = unpackb(packb(six.u("abcФФФdef"), encoding='ascii', unicode_errors='ignore'), encoding='utf-8') + re = unpackb(packb(six.u("abcФФФdef"), encoding='ascii', unicode_errors='ignore'), encoding='utf-8', use_list=1) assert_equal(re, six.u("abcdef")) @raises(TypeError) @@ -84,7 +83,7 @@ def testNoEncoding(): packb(six.u("abc"), encoding=None) def testDecodeBinary(): - re = unpackb(packb("abc"), encoding=None) + re = unpackb(packb("abc"), encoding=None, use_list=1) assert_equal(re, b"abc") def testPackFloat(): -- cgit v1.2.1 From 15a46eb143d979c7b55f30d12d8bfe6a846f2a7e Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 24 Sep 2012 02:42:38 +0900 Subject: use_list=1 is default --- msgpack/_msgpack.pyx | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index d0c4541..e932ba9 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -219,7 +219,7 @@ cdef extern from "unpack.h": def unpackb(object packed, object object_hook=None, object list_hook=None, - use_list=None, encoding=None, unicode_errors="strict", + bint use_list=1, encoding=None, unicode_errors="strict", ): """Unpack packed_bytes to object. Returns an unpacked object. @@ -250,11 +250,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, err = PyBytes_AsString(berrors) template_init(&ctx) - if use_list is None: - warnings.warn("Set use_list explicitly.", category=DeprecationWarning, stacklevel=1) - ctx.user.use_list = 0 - else: - ctx.user.use_list = use_list + ctx.user.use_list = use_list ctx.user.object_hook = ctx.user.list_hook = NULL ctx.user.encoding = enc ctx.user.unicode_errors = err @@ -277,15 +273,12 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, def unpack(object stream, object object_hook=None, object list_hook=None, - use_list=None, encoding=None, unicode_errors="strict", + bint use_list=1, encoding=None, unicode_errors="strict", ): """Unpack an object from `stream`. Raises `ValueError` when `stream` has extra bytes. """ - if use_list is None: - warnings.warn("Set use_list explicitly.", category=DeprecationWarning, stacklevel=1) - use_list = 0 return unpackb(stream.read(), use_list=use_list, object_hook=object_hook, list_hook=list_hook, encoding=encoding, unicode_errors=unicode_errors, @@ -356,15 +349,11 @@ cdef class Unpacker(object): free(self.buf) self.buf = NULL - def __init__(self, file_like=None, Py_ssize_t read_size=0, use_list=None, + def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=1, object object_hook=None, object list_hook=None, encoding=None, unicode_errors='strict', int max_buffer_size=0, object object_pairs_hook=None, ): - if use_list is None: - warnings.warn("Set use_list explicitly.", category=DeprecationWarning, stacklevel=1) - use_list = 0 - self.file_like = file_like if file_like: self.file_like_read = file_like.read -- cgit v1.2.1 From ac403ef68da911ac4735407ada88db02015bd520 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 24 Sep 2012 02:45:37 +0900 Subject: Start 0.2.3dev --- msgpack/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/_version.py b/msgpack/_version.py index f343b7a..9bffd02 100644 --- a/msgpack/_version.py +++ b/msgpack/_version.py @@ -1 +1 @@ -version = (0, 2, 2) +version = (0, 2, 3, 'dev1') -- cgit v1.2.1 From 927d29131dc8d2a9f606cf7c881606d47ace557b Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 24 Sep 2012 02:55:50 +0900 Subject: Write about warning in changelog. --- ChangeLog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog.rst b/ChangeLog.rst index 46b83ee..f49b577 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -4,6 +4,7 @@ Changes ------- +* Warn when use_list is not specified. It's default value will be changed in 0.3. Bugs fixed ----------- -- cgit v1.2.1 From 477d3b152f5d36a48a8083b3720def2dd1f5d1a7 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 24 Sep 2012 03:08:13 +0900 Subject: Fix warnings. --- test/test_obj.py | 2 +- test/test_pack.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_obj.py b/test/test_obj.py index 12a149f..881e627 100644 --- a/test/test_obj.py +++ b/test/test_obj.py @@ -29,7 +29,7 @@ def test_decode_hook(): def test_decode_pairs_hook(): packed = packb([3, {1: 2, 3: 4}]) prod_sum = 1 * 2 + 3 * 4 - unpacked = unpackb(packed, object_pairs_hook=lambda l: sum(k * v for k, v in l)) + unpacked = unpackb(packed, object_pairs_hook=lambda l: sum(k * v for k, v in l), use_list=1) eq_(unpacked[1], prod_sum) @raises(ValueError) diff --git a/test/test_pack.py b/test/test_pack.py index 9009d35..6af87fd 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -112,7 +112,7 @@ def test_odict(): assert_equal(unpackb(packb(od), use_list=1), dict(seq)) def pair_hook(seq): return seq - assert_equal(unpackb(packb(od), object_pairs_hook=pair_hook), seq) + assert_equal(unpackb(packb(od), object_pairs_hook=pair_hook, use_list=1), seq) if __name__ == '__main__': -- cgit v1.2.1 From d56e2b2c8aa1005fbac3b584cd003ba0cdece2e2 Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Tue, 25 Sep 2012 00:30:15 +1000 Subject: Use C++ function templating for skip()/construct() --- msgpack/_msgpack.pyx | 23 +++++++++++------------ msgpack/unpack.h | 1 + msgpack/unpack_template.h | 21 +++++++++++++-------- setup.py | 4 ++-- 4 files changed, 27 insertions(+), 22 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index e0a1043..0fc3739 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -208,8 +208,10 @@ cdef extern from "unpack.h": unsigned int ct PyObject* key - int template_execute(template_context* ctx, const_char_ptr data, - size_t len, size_t* off, bint construct) except -1 + ctypedef int (*execute_fn)(template_context* ctx, const_char_ptr data, + size_t len, size_t* off) except -1 + execute_fn template_construct + execute_fn template_skip void template_init(template_context* ctx) object template_data(template_context* ctx) @@ -257,7 +259,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, if not PyCallable_Check(list_hook): raise TypeError("list_hook must be a callable.") ctx.user.list_hook = list_hook - ret = template_execute(&ctx, buf, buf_len, &off, 1) + ret = template_construct(&ctx, buf, buf_len, &off) if ret == 1: obj = template_data(&ctx) if off < buf_len: @@ -455,16 +457,13 @@ cdef class Unpacker(object): else: self.file_like = None - cdef object _unpack(self, bint construct): + cdef object _unpack(self, execute_fn execute): cdef int ret cdef object obj while 1: - ret = template_execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head, construct) + ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) if ret == 1: - if construct: - obj = template_data(&self.ctx) - else: - obj = None + obj = template_data(&self.ctx) template_init(&self.ctx) return obj elif ret == 0: @@ -477,17 +476,17 @@ cdef class Unpacker(object): def unpack(self): """unpack one object""" - return self._unpack(1) + return self._unpack(template_construct) def skip(self): """read and ignore one object, returning None""" - return self._unpack(0) + return self._unpack(template_skip) def __iter__(self): return self def __next__(self): - return self._unpack(1) + return self._unpack(template_construct) # for debug. #def _buf(self): diff --git a/msgpack/unpack.h b/msgpack/unpack.h index a106f9c..3c9d4be 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -41,6 +41,7 @@ typedef struct unpack_user { #define msgpack_unpack_user unpack_user +typedef int (*execute_fn)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off); struct template_context; typedef struct template_context template_context; diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 5495a51..e0cf42e 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -95,7 +95,8 @@ msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context } -msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off, int construct) +template +msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off) { assert(len >= *off); @@ -380,6 +381,8 @@ _header_again: _finish: + if (!construct) + msgpack_unpack_callback(_nil)(user, &obj); stack[0].obj = obj; ++p; ret = 1; @@ -405,13 +408,6 @@ _end: #undef construct_cb } - -#undef msgpack_unpack_func -#undef msgpack_unpack_callback -#undef msgpack_unpack_struct -#undef msgpack_unpack_object -#undef msgpack_unpack_user - #undef push_simple_value #undef push_fixed_value #undef push_variable_value @@ -419,6 +415,15 @@ _end: #undef again_fixed_trail_if_zero #undef start_container +static const execute_fn template_construct = &template_execute; +static const execute_fn template_skip = &template_execute; + +#undef msgpack_unpack_func +#undef msgpack_unpack_callback +#undef msgpack_unpack_struct +#undef msgpack_unpack_object +#undef msgpack_unpack_user + #undef NEXT_CS /* vim: set ts=4 sw=4 noexpandtab */ diff --git a/setup.py b/setup.py index 86b0b34..708fa13 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ except ImportError: def cythonize(src): sys.stderr.write("cythonize: %r\n" % (src,)) - cython_compiler.compile([src]) + cython_compiler.compile([src], cplus=True) def ensure_source(src): pyx = os.path.splitext(src)[0] + '.pyx' @@ -67,7 +67,7 @@ if have_cython: else: Sdist = sdist -sources = ['msgpack/_msgpack.c'] +sources = ['msgpack/_msgpack.cpp'] libraries = [] if sys.platform == 'win32': libraries.append('ws2_32') -- cgit v1.2.1 From 0431a766f4e069d74627441aa3facbc7e64e4511 Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Tue, 25 Sep 2012 01:18:33 +1000 Subject: read_array/map_header functionality --- msgpack/_msgpack.pyx | 10 +++++++ msgpack/unpack_template.h | 63 ++++++++++++++++++++++++++++++++++++++++++++ test/test_read_size.py | 66 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 139 insertions(+) create mode 100644 test/test_read_size.py diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 0fc3739..7131d1f 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -212,6 +212,8 @@ cdef extern from "unpack.h": size_t len, size_t* off) except -1 execute_fn template_construct execute_fn template_skip + execute_fn read_array_header + execute_fn read_map_header void template_init(template_context* ctx) object template_data(template_context* ctx) @@ -482,6 +484,14 @@ cdef class Unpacker(object): """read and ignore one object, returning None""" return self._unpack(template_skip) + def read_array_header(self): + """assuming the next object is an array, return its size n, such that the next n unpack() calls will iterate over its contents.""" + return self._unpack(read_array_header) + + def read_map_header(self): + """assuming the next object is a map, return its size n, such that the next n * 2 unpack() calls will iterate over its key-value pairs.""" + return self._unpack(read_map_header) + def __iter__(self): return self diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index e0cf42e..69ef6e2 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -408,6 +408,10 @@ _end: #undef construct_cb } +#undef SWITCH_RANGE_BEGIN +#undef SWITCH_RANGE +#undef SWITCH_RANGE_DEFAULT +#undef SWITCH_RANGE_END #undef push_simple_value #undef push_fixed_value #undef push_variable_value @@ -415,8 +419,67 @@ _end: #undef again_fixed_trail_if_zero #undef start_container +template +msgpack_unpack_func(int, _container_header)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off) +{ + assert(len >= *off); + uint32_t size; + const unsigned char *const p = (unsigned char*)data + *off; + +#define inc_offset(inc) \ + if (len - *off < inc) \ + return 0; \ + *off += inc; + + switch (*p) { + case var_offset: + inc_offset(3); + size = _msgpack_load16(uint16_t, p + 1); + break; + case var_offset + 1: + inc_offset(5); + size = _msgpack_load32(uint32_t, p + 1); + break; +#ifdef USE_CASE_RANGE + case fixed_offset + 0x0 ... fixed_offset + 0xf: +#else + case fixed_offset + 0x0: + case fixed_offset + 0x1: + case fixed_offset + 0x2: + case fixed_offset + 0x3: + case fixed_offset + 0x4: + case fixed_offset + 0x5: + case fixed_offset + 0x6: + case fixed_offset + 0x7: + case fixed_offset + 0x8: + case fixed_offset + 0x9: + case fixed_offset + 0xa: + case fixed_offset + 0xb: + case fixed_offset + 0xc: + case fixed_offset + 0xd: + case fixed_offset + 0xe: + case fixed_offset + 0xf: +#endif + ++*off; + size = ((unsigned int)*p) & 0x0f; + break; + default: + PyErr_SetString(PyExc_ValueError, "Unexpected type header on stream"); + return -1; + } + msgpack_unpack_callback(_uint32)(&ctx->user, size, &ctx->stack[0].obj); + return 1; +} + +#undef SWITCH_RANGE_BEGIN +#undef SWITCH_RANGE +#undef SWITCH_RANGE_DEFAULT +#undef SWITCH_RANGE_END + static const execute_fn template_construct = &template_execute; static const execute_fn template_skip = &template_execute; +static const execute_fn read_array_header = &template_container_header<0x90, 0xdc>; +static const execute_fn read_map_header = &template_container_header<0x80, 0xde>; #undef msgpack_unpack_func #undef msgpack_unpack_callback diff --git a/test/test_read_size.py b/test/test_read_size.py new file mode 100644 index 0000000..714f963 --- /dev/null +++ b/test/test_read_size.py @@ -0,0 +1,66 @@ +"""Test Unpacker's read_array_header and read_map_header methods""" +from msgpack import packb, Unpacker +UnexpectedTypeException = ValueError + +def test_read_array_header(): + unpacker = Unpacker() + unpacker.feed(packb(['a', 'b', 'c'])) + assert unpacker.read_array_header() == 3 + assert unpacker.unpack() == 'a' + assert unpacker.unpack() == 'b' + assert unpacker.unpack() == 'c' + try: + unpacker.unpack() + assert 0, 'should raise exception' + except StopIteration: + assert 1, 'okay' + + +def test_read_map_header(): + unpacker = Unpacker() + unpacker.feed(packb({'a': 'A'})) + assert unpacker.read_map_header() == 1 + assert unpacker.unpack() == 'a' + assert unpacker.unpack() == 'A' + try: + unpacker.unpack() + assert 0, 'should raise exception' + except StopIteration: + assert 1, 'okay' + +def test_incorrect_type_array(): + unpacker = Unpacker() + unpacker.feed(packb(1)) + try: + unpacker.read_array_header() + assert 0, 'should raise exception' + except UnexpectedTypeException: + assert 1, 'okay' + +def test_incorrect_type_map(): + unpacker = Unpacker() + unpacker.feed(packb(1)) + try: + unpacker.read_map_header() + assert 0, 'should raise exception' + except UnexpectedTypeException: + assert 1, 'okay' + +def test_correct_type_nested_array(): + unpacker = Unpacker() + unpacker.feed(packb({'a': ['b', 'c', 'd']})) + try: + unpacker.read_array_header() + assert 0, 'should raise exception' + except UnexpectedTypeException: + assert 1, 'okay' + +def test_incorrect_type_nested_map(): + unpacker = Unpacker() + unpacker.feed(packb([{'a': 'b'}])) + try: + unpacker.read_map_header() + assert 0, 'should raise exception' + except UnexpectedTypeException: + assert 1, 'okay' + -- cgit v1.2.1 From 9d9c3eecb846c6a927a31aae394dea39fa75aef4 Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Sun, 23 Sep 2012 17:26:16 +1000 Subject: Packer.pack_array/map_header to correspond to read functions --- msgpack/_msgpack.pyx | 11 +++++++++++ test/test_pack.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 7131d1f..18a75ca 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -178,6 +178,17 @@ cdef class Packer(object): self.pk.length = 0 return buf + cpdef pack_array_header(self, size_t size): + msgpack_pack_array(&self.pk, size) + buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) + self.pk.length = 0 + return buf + + cpdef pack_map_header(self, size_t size): + msgpack_pack_map(&self.pk, size) + buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) + self.pk.length = 0 + return buf def pack(object o, object stream, default=None, encoding='utf-8', unicode_errors='strict'): """ diff --git a/test/test_pack.py b/test/test_pack.py index b216c46..937141d 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -91,6 +91,35 @@ def testPackFloat(): assert_equal(packb(1.0, use_single_float=True), b'\xca' + struct.pack('>f', 1.0)) assert_equal(packb(1.0, use_single_float=False), b'\xcb' + struct.pack('>d', 1.0)) +def testArraySize(sizes=[0, 5, 50, 1000]): + bio = six.BytesIO() + packer = Packer() + for size in sizes: + bio.write(packer.pack_array_header(size)) + for i in range(size): + bio.write(packer.pack(i)) + + bio.seek(0) + unpacker = Unpacker(bio) + for size in sizes: + assert unpacker.unpack() == tuple(range(size)) + +def testMapSize(sizes=[0, 5, 50, 1000]): + bio = six.BytesIO() + packer = Packer() + for size in sizes: + bio.write(packer.pack_map_header(size)) + for i in range(size): + bio.write(packer.pack(i)) # key + bio.write(packer.pack(i * 2)) # value + + bio.seek(0) + unpacker = Unpacker(bio) + for size in sizes: + assert unpacker.unpack() == {i: i * 2 for i in range(size)} + + + class odict(dict): '''Reimplement OrderedDict to run test on Python 2.6''' -- cgit v1.2.1 From d5f99959cc2ec393c13fc9e44714351272bac7fc Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 1 Oct 2012 01:34:58 +0900 Subject: Fix some test failure. --- test/test_pack.py | 6 +++--- test/test_read_size.py | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test/test_pack.py b/test/test_pack.py index ff1eeef..21c2bd7 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -99,9 +99,9 @@ def testArraySize(sizes=[0, 5, 50, 1000]): bio.write(packer.pack(i)) bio.seek(0) - unpacker = Unpacker(bio) + unpacker = Unpacker(bio, use_list=1) for size in sizes: - assert unpacker.unpack() == tuple(range(size)) + assert unpacker.unpack() == list(range(size)) def testMapSize(sizes=[0, 5, 50, 1000]): bio = six.BytesIO() @@ -115,7 +115,7 @@ def testMapSize(sizes=[0, 5, 50, 1000]): bio.seek(0) unpacker = Unpacker(bio) for size in sizes: - assert unpacker.unpack() == {i: i * 2 for i in range(size)} + assert unpacker.unpack() == dict((i, i * 2) for i in range(size)) diff --git a/test/test_read_size.py b/test/test_read_size.py index 714f963..e130805 100644 --- a/test/test_read_size.py +++ b/test/test_read_size.py @@ -6,9 +6,9 @@ def test_read_array_header(): unpacker = Unpacker() unpacker.feed(packb(['a', 'b', 'c'])) assert unpacker.read_array_header() == 3 - assert unpacker.unpack() == 'a' - assert unpacker.unpack() == 'b' - assert unpacker.unpack() == 'c' + assert unpacker.unpack() == b'a' + assert unpacker.unpack() == b'b' + assert unpacker.unpack() == b'c' try: unpacker.unpack() assert 0, 'should raise exception' @@ -20,8 +20,8 @@ def test_read_map_header(): unpacker = Unpacker() unpacker.feed(packb({'a': 'A'})) assert unpacker.read_map_header() == 1 - assert unpacker.unpack() == 'a' - assert unpacker.unpack() == 'A' + assert unpacker.unpack() == B'a' + assert unpacker.unpack() == B'A' try: unpacker.unpack() assert 0, 'should raise exception' -- cgit v1.2.1