From d61097511a1caa0e3bc5a70c1d2d92f448bd5025 Mon Sep 17 00:00:00 2001 From: Antonio Cuni Date: Tue, 15 Oct 2013 16:59:43 +0200 Subject: add support for extended types: you can now pack/unpack custom python objects by subclassing Packer and Unpacker --- test/test_extension.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 test/test_extension.py (limited to 'test') diff --git a/test/test_extension.py b/test/test_extension.py new file mode 100644 index 0000000..45e6027 --- /dev/null +++ b/test/test_extension.py @@ -0,0 +1,24 @@ +import array +import msgpack + +def test_extension_type(): + class MyPacker(msgpack.Packer): + def handle_extended_type(self, obj): + if isinstance(obj, array.array): + fmt = "ext 32" + typecode = 123 # application specific typecode + data = obj.tostring() + return fmt, typecode, data + + class MyUnpacker(msgpack.Unpacker): + def handle_extended_type(self, typecode, data): + assert typecode == 123 + obj = array.array('d') + obj.fromstring(data) + return obj + + obj = [42, 'hello', array.array('d', [1.1, 2.2, 3.3])] + s = msgpack.packb(obj, MyPacker) + obj2 = msgpack.unpackb(s, MyUnpacker) + assert obj == obj2 + -- cgit v1.2.1 From 5529dfe59660f3c2fc5058e6fa42b24fe764a255 Mon Sep 17 00:00:00 2001 From: Antonio Cuni Date: Fri, 18 Oct 2013 14:38:52 +0200 Subject: kill some duplicate code from unpack/unpackb and move the logic to Unpacker.unpack_one. By doing this we no longer need to make the module-level pack/unpack parametric on the class, because they contain no logic at all --- test/test_extension.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'test') diff --git a/test/test_extension.py b/test/test_extension.py index 45e6027..0a9c14f 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -18,7 +18,9 @@ def test_extension_type(): return obj obj = [42, 'hello', array.array('d', [1.1, 2.2, 3.3])] - s = msgpack.packb(obj, MyPacker) - obj2 = msgpack.unpackb(s, MyUnpacker) + packer = MyPacker() + unpacker = MyUnpacker(None) + s = packer.pack(obj) + unpacker.feed(s) + obj2 = unpacker.unpack_one() assert obj == obj2 - -- cgit v1.2.1 From 522c4bfc7993c296b78df9c9c91aac5fd40ae8e0 Mon Sep 17 00:00:00 2001 From: Antonio Cuni Date: Fri, 18 Oct 2013 15:03:58 +0200 Subject: slightly change to API --- test/test_extension.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'test') diff --git a/test/test_extension.py b/test/test_extension.py index 0a9c14f..0b26f8e 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -3,15 +3,16 @@ import msgpack def test_extension_type(): class MyPacker(msgpack.Packer): - def handle_extended_type(self, obj): + def handle_unknown_type(self, obj): if isinstance(obj, array.array): fmt = "ext 32" typecode = 123 # application specific typecode data = obj.tostring() - return fmt, typecode, data + self.pack_extended_type(fmt, typecode, data) + return True class MyUnpacker(msgpack.Unpacker): - def handle_extended_type(self, typecode, data): + def read_extended_type(self, typecode, data): assert typecode == 123 obj = array.array('d') obj.fromstring(data) -- cgit v1.2.1 From c727440ba5fe2f77d6cc03171ad7c193a3f481ee Mon Sep 17 00:00:00 2001 From: Antonio Cuni Date: Fri, 18 Oct 2013 15:45:50 +0200 Subject: automatically find the best format to encode extended types --- test/test_extension.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) (limited to 'test') diff --git a/test/test_extension.py b/test/test_extension.py index 0b26f8e..1908fa2 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -1,14 +1,28 @@ import array +import struct import msgpack +def test_pack_extended_type(): + def p(s): + packer = msgpack.Packer() + packer.pack_extended_type(0x42, s) + return packer._buffer.getvalue() + assert p('A') == '\xd4\x42A' # fixext 1 + assert p('AB') == '\xd5\x42AB' # fixext 2 + assert p('ABCD') == '\xd6\x42ABCD' # fixext 4 + assert p('ABCDEFGH') == '\xd7\x42ABCDEFGH' # fixext 8 + assert p('A'*16) == '\xd8\x42' + 'A'*16 # fixext 16 + assert p('ABC') == '\xc7\x03\x42ABC' # ext 8 + assert p('A'*0x0123) == '\xc8\x01\x23\x42' + 'A'*0x0123 # ext 16 + assert p('A'*0x00012345) == '\xc9\x00\x01\x23\x45\x42' + 'A'*0x00012345 # ext 32 + def test_extension_type(): class MyPacker(msgpack.Packer): def handle_unknown_type(self, obj): if isinstance(obj, array.array): - fmt = "ext 32" typecode = 123 # application specific typecode data = obj.tostring() - self.pack_extended_type(fmt, typecode, data) + self.pack_extended_type(typecode, data) return True class MyUnpacker(msgpack.Unpacker): -- cgit v1.2.1 From 5467515065b95496b9f5b9d842ffc73c9ccb806e Mon Sep 17 00:00:00 2001 From: Antonio Cuni Date: Fri, 18 Oct 2013 17:33:54 +0200 Subject: implement Packer.pack_extended_type also in the cython version of the code --- test/test_extension.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'test') diff --git a/test/test_extension.py b/test/test_extension.py index 1908fa2..9ec1153 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -6,7 +6,7 @@ def test_pack_extended_type(): def p(s): packer = msgpack.Packer() packer.pack_extended_type(0x42, s) - return packer._buffer.getvalue() + return packer.bytes() assert p('A') == '\xd4\x42A' # fixext 1 assert p('AB') == '\xd5\x42AB' # fixext 2 assert p('ABCD') == '\xd6\x42ABCD' # fixext 4 -- cgit v1.2.1 From ff858387d37d37ec4472f6b6ac7010d8f2b0744f Mon Sep 17 00:00:00 2001 From: Antonio Cuni Date: Sat, 19 Oct 2013 01:49:03 +0200 Subject: implement unpack_one also for the cython version, and add a test for it --- test/test_sequnpack.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'test') diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index 9db14ca..abc447a 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -1,9 +1,10 @@ #!/usr/bin/env python # coding: utf-8 +import py import six from msgpack import Unpacker, BufferFull -from msgpack.exceptions import OutOfData +from msgpack.exceptions import OutOfData, ExtraData, UnpackValueError from pytest import raises @@ -85,3 +86,15 @@ def test_readbytes(): assert unpacker.unpack() == ord(b'a') assert unpacker.unpack() == ord(b'r') +def test_unpack_one(): + unpacker = Unpacker() + unpacker.feed('\xda\x00\x03abc') + assert unpacker.unpack_one() == 'abc' + # + unpacker = Unpacker() + unpacker.feed('\xda\x00\x03abcd') + py.test.raises(ExtraData, "unpacker.unpack_one()") + # + unpacker = Unpacker() + unpacker.feed('\xda\x00\x03ab') + py.test.raises(UnpackValueError, "unpacker.unpack_one()") -- cgit v1.2.1 From 985d4c1496d8c9186079ebc4e42aee319e67c385 Mon Sep 17 00:00:00 2001 From: Antonio Cuni Date: Sat, 19 Oct 2013 11:34:28 +0200 Subject: add a test for unpacking extended types --- test/test_extension.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) (limited to 'test') diff --git a/test/test_extension.py b/test/test_extension.py index 9ec1153..96944a3 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -16,6 +16,28 @@ def test_pack_extended_type(): assert p('A'*0x0123) == '\xc8\x01\x23\x42' + 'A'*0x0123 # ext 16 assert p('A'*0x00012345) == '\xc9\x00\x01\x23\x45\x42' + 'A'*0x00012345 # ext 32 +def test_unpack_extended_type(): + class MyUnpacker(msgpack.Unpacker): + def read_extended_type(self, typecode, data): + return (typecode, data) + + def u(s): + unpacker = MyUnpacker() + unpacker.feed(s) + return unpacker.unpack_one() + + assert u('\xd4\x42A') == (0x42, 'A') # fixext 1 + assert u('\xd5\x42AB') == (0x42, 'AB') # fixext 2 + assert u('\xd6\x42ABCD') == (0x42, 'ABCD') # fixext 4 + assert u('\xd7\x42ABCDEFGH') == (0x42, 'ABCDEFGH') # fixext 8 + assert u('\xd8\x42' + 'A'*16) == (0x42, 'A'*16) # fixext 16 + assert u('\xc7\x03\x42ABC') == (0x42, 'ABC') # ext 8 + assert (u('\xc8\x01\x23\x42' + 'A'*0x0123) == + (0x42, 'A'*0x0123)) # ext 16 + assert (u('\xc9\x00\x01\x23\x45\x42' + 'A'*0x00012345) == + (0x42, 'A'*0x00012345)) # ext 32 + + def test_extension_type(): class MyPacker(msgpack.Packer): def handle_unknown_type(self, obj): -- cgit v1.2.1 From 56dd1650a42a454027ba335b494100a9f211758e Mon Sep 17 00:00:00 2001 From: Antonio Cuni Date: Sat, 19 Oct 2013 17:27:16 +0200 Subject: implement unpacking for all the fixtext formats --- test/test_extension.py | 1 + 1 file changed, 1 insertion(+) (limited to 'test') diff --git a/test/test_extension.py b/test/test_extension.py index 96944a3..94117e1 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -1,3 +1,4 @@ +import py import array import struct import msgpack -- cgit v1.2.1 From aa68c9b8330b130d600b22ec47d5c3841499b536 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 20 Oct 2013 15:40:20 +0900 Subject: fallback: Support pack_ext_type. --- test/test_extension.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) (limited to 'test') diff --git a/test/test_extension.py b/test/test_extension.py index 94117e1..f2fa363 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -1,21 +1,21 @@ -import py import array -import struct import msgpack -def test_pack_extended_type(): + +def test_pack_ext_type(): def p(s): packer = msgpack.Packer() - packer.pack_extended_type(0x42, s) + packer.pack_ext_type(0x42, s) return packer.bytes() - assert p('A') == '\xd4\x42A' # fixext 1 - assert p('AB') == '\xd5\x42AB' # fixext 2 - assert p('ABCD') == '\xd6\x42ABCD' # fixext 4 - assert p('ABCDEFGH') == '\xd7\x42ABCDEFGH' # fixext 8 - assert p('A'*16) == '\xd8\x42' + 'A'*16 # fixext 16 - assert p('ABC') == '\xc7\x03\x42ABC' # ext 8 - assert p('A'*0x0123) == '\xc8\x01\x23\x42' + 'A'*0x0123 # ext 16 - assert p('A'*0x00012345) == '\xc9\x00\x01\x23\x45\x42' + 'A'*0x00012345 # ext 32 + assert p(b'A') == b'\xd4\x42A' # fixext 1 + assert p(b'AB') == b'\xd5\x42AB' # fixext 2 + assert p(b'ABCD') == b'\xd6\x42ABCD' # fixext 4 + assert p(b'ABCDEFGH') == b'\xd7\x42ABCDEFGH' # fixext 8 + assert p(b'A'*16) == b'\xd8\x42' + 'A'*16 # fixext 16 + assert p(b'ABC') == b'\xc7\x03\x42ABC' # ext 8 + assert p(b'A'*0x0123) == b'\xc8\x01\x23\x42' + b'A'*0x0123 # ext 16 + assert p(b'A'*0x00012345) == b'\xc9\x00\x01\x23\x45\x42' + b'A'*0x00012345 # ext 32 + def test_unpack_extended_type(): class MyUnpacker(msgpack.Unpacker): @@ -45,7 +45,7 @@ def test_extension_type(): if isinstance(obj, array.array): typecode = 123 # application specific typecode data = obj.tostring() - self.pack_extended_type(typecode, data) + self.pack_ext_type(typecode, data) return True class MyUnpacker(msgpack.Unpacker): -- cgit v1.2.1 From 96bcd76f49afd00f5b7def1ff7cfd002a7fa477d Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 20 Oct 2013 20:28:32 +0900 Subject: Packing ExtType and some cleanup --- test/test_extension.py | 75 +++++++++++++++++++++++--------------------------- test/test_sequnpack.py | 13 --------- 2 files changed, 34 insertions(+), 54 deletions(-) (limited to 'test') diff --git a/test/test_extension.py b/test/test_extension.py index f2fa363..2f85ce3 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -1,5 +1,7 @@ +from __future__ import print_function import array import msgpack +from msgpack import ExtType def test_pack_ext_type(): @@ -11,54 +13,45 @@ def test_pack_ext_type(): assert p(b'AB') == b'\xd5\x42AB' # fixext 2 assert p(b'ABCD') == b'\xd6\x42ABCD' # fixext 4 assert p(b'ABCDEFGH') == b'\xd7\x42ABCDEFGH' # fixext 8 - assert p(b'A'*16) == b'\xd8\x42' + 'A'*16 # fixext 16 + assert p(b'A'*16) == b'\xd8\x42' + b'A'*16 # fixext 16 assert p(b'ABC') == b'\xc7\x03\x42ABC' # ext 8 assert p(b'A'*0x0123) == b'\xc8\x01\x23\x42' + b'A'*0x0123 # ext 16 assert p(b'A'*0x00012345) == b'\xc9\x00\x01\x23\x45\x42' + b'A'*0x00012345 # ext 32 -def test_unpack_extended_type(): - class MyUnpacker(msgpack.Unpacker): - def read_extended_type(self, typecode, data): - return (typecode, data) +def test_unpack_ext_type(): + def check(b, expected): + assert msgpack.unpackb(b) == expected - def u(s): - unpacker = MyUnpacker() - unpacker.feed(s) - return unpacker.unpack_one() - - assert u('\xd4\x42A') == (0x42, 'A') # fixext 1 - assert u('\xd5\x42AB') == (0x42, 'AB') # fixext 2 - assert u('\xd6\x42ABCD') == (0x42, 'ABCD') # fixext 4 - assert u('\xd7\x42ABCDEFGH') == (0x42, 'ABCDEFGH') # fixext 8 - assert u('\xd8\x42' + 'A'*16) == (0x42, 'A'*16) # fixext 16 - assert u('\xc7\x03\x42ABC') == (0x42, 'ABC') # ext 8 - assert (u('\xc8\x01\x23\x42' + 'A'*0x0123) == - (0x42, 'A'*0x0123)) # ext 16 - assert (u('\xc9\x00\x01\x23\x45\x42' + 'A'*0x00012345) == - (0x42, 'A'*0x00012345)) # ext 32 + check(b'\xd4\x42A', ExtType(0x42, b'A')) # fixext 1 + check(b'\xd5\x42AB', ExtType(0x42, b'AB')) # fixext 2 + check(b'\xd6\x42ABCD', ExtType(0x42, b'ABCD')) # fixext 4 + check(b'\xd7\x42ABCDEFGH', ExtType(0x42, b'ABCDEFGH')) # fixext 8 + check(b'\xd8\x42' + b'A'*16, ExtType(0x42, b'A'*16)) # fixext 16 + check(b'\xc7\x03\x42ABC', ExtType(0x42, b'ABC')) # ext 8 + check(b'\xc8\x01\x23\x42' + b'A'*0x0123, + ExtType(0x42, b'A'*0x0123)) # ext 16 + check(b'\xc9\x00\x01\x23\x45\x42' + b'A'*0x00012345, + ExtType(0x42, b'A'*0x00012345)) # ext 32 def test_extension_type(): - class MyPacker(msgpack.Packer): - def handle_unknown_type(self, obj): - if isinstance(obj, array.array): - typecode = 123 # application specific typecode - data = obj.tostring() - self.pack_ext_type(typecode, data) - return True - - class MyUnpacker(msgpack.Unpacker): - def read_extended_type(self, typecode, data): - assert typecode == 123 - obj = array.array('d') - obj.fromstring(data) - return obj - - obj = [42, 'hello', array.array('d', [1.1, 2.2, 3.3])] - packer = MyPacker() - unpacker = MyUnpacker(None) - s = packer.pack(obj) - unpacker.feed(s) - obj2 = unpacker.unpack_one() + def default(obj): + print('default called', obj) + if isinstance(obj, array.array): + typecode = 123 # application specific typecode + data = obj.tostring() + return ExtType(typecode, data) + raise TypeError("Unknwon type object %r" % (obj,)) + + def ext_hook(code, data): + print('ext_hook called', code, data) + assert code == 123 + obj = array.array('d') + obj.fromstring(data) + return obj + + obj = [42, b'hello', array.array('d', [1.1, 2.2, 3.3])] + s = msgpack.packb(obj, default=default) + obj2 = msgpack.unpackb(s, ext_hook=ext_hook) assert obj == obj2 diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index abc447a..af66b78 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -85,16 +85,3 @@ def test_readbytes(): assert unpacker.read_bytes(3) == b'oob' assert unpacker.unpack() == ord(b'a') assert unpacker.unpack() == ord(b'r') - -def test_unpack_one(): - unpacker = Unpacker() - unpacker.feed('\xda\x00\x03abc') - assert unpacker.unpack_one() == 'abc' - # - unpacker = Unpacker() - unpacker.feed('\xda\x00\x03abcd') - py.test.raises(ExtraData, "unpacker.unpack_one()") - # - unpacker = Unpacker() - unpacker.feed('\xda\x00\x03ab') - py.test.raises(UnpackValueError, "unpacker.unpack_one()") -- cgit v1.2.1 From 822cce823cfea8e9f7625598a125897718b4ab58 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 20 Oct 2013 22:59:27 +0900 Subject: Support unpacking new types. --- test/test_obj.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'test') diff --git a/test/test_obj.py b/test/test_obj.py index fbf610c..9083218 100644 --- a/test/test_obj.py +++ b/test/test_obj.py @@ -35,7 +35,7 @@ def test_only_one_obj_hook(): unpackb(b'', object_hook=lambda x: x, object_pairs_hook=lambda x: x) def test_bad_hook(): - with raises(ValueError): + with raises(TypeError): packed = packb([3, 1+2j], default=lambda o: o) unpacked = unpackb(packed, use_list=1) -- cgit v1.2.1 From 0d5c58bd517caddd6b62a8931f6833e2a3add283 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 20 Oct 2013 23:06:02 +0900 Subject: cleanup --- test/test_sequnpack.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'test') diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index af66b78..f541207 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -1,10 +1,9 @@ #!/usr/bin/env python # coding: utf-8 -import py import six from msgpack import Unpacker, BufferFull -from msgpack.exceptions import OutOfData, ExtraData, UnpackValueError +from msgpack.exceptions import OutOfData from pytest import raises -- cgit v1.2.1 From 37c2ad63af8a6e5cb6944f80d931fedbc6b49e7d Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 21 Oct 2013 00:29:05 +0900 Subject: Add tests and bugfix. --- test/test_newspec.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) (limited to 'test') diff --git a/test/test_newspec.py b/test/test_newspec.py index 8bc2cfe..ab05029 100644 --- a/test/test_newspec.py +++ b/test/test_newspec.py @@ -1,6 +1,6 @@ # coding: utf-8 -from msgpack import packb, unpackb +from msgpack import packb, unpackb, ExtType def test_str8(): @@ -66,4 +66,23 @@ def test_bin32(): assert b[5:] == data assert unpackb(b) == data - +def test_ext(): + def check(ext, packed): + assert packb(ext) == packed + assert unpackb(packed) == ext + check(ExtType(0x42, b'Z'), b'\xd4\x42Z') # fixext 1 + check(ExtType(0x42, b'ZZ'), b'\xd5\x42ZZ') # fixext 2 + check(ExtType(0x42, b'Z'*4), b'\xd6\x42' + b'Z'*4) # fixext 4 + check(ExtType(0x42, b'Z'*8), b'\xd7\x42' + b'Z'*8) # fixext 8 + check(ExtType(0x42, b'Z'*16), b'\xd8\x42' + b'Z'*16) # fixext 16 + # ext 8 + check(ExtType(0x42, b''), b'\xc7\x00\x42') + check(ExtType(0x42, b'Z'*255), b'\xc7\xff\x42' + b'Z'*255) + # ext 16 + check(ExtType(0x42, b'Z'*256), b'\xc8\x01\x00\x42' + b'Z'*256) + check(ExtType(0x42, b'Z'*0xffff), b'\xc8\xff\xff\x42' + b'Z'*0xffff) + # ext 32 + check(ExtType(0x42, b'Z'*0x10000), b'\xc9\x00\x01\x00\x00\x42' + b'Z'*0x10000) + # needs large memory + #check(ExtType(0x42, b'Z'*0xffffffff), + # b'\xc9\xff\xff\xff\xff\x42' + b'Z'*0xffffffff) -- cgit v1.2.1