From c930f5367b41ef141e80a76506fd91b49fe7d426 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 8 Jun 2009 01:43:50 +0900 Subject: add cythoned source and setup script. --- python/msgpack | 1 + 1 file changed, 1 insertion(+) create mode 120000 python/msgpack (limited to 'python/msgpack') diff --git a/python/msgpack b/python/msgpack new file mode 120000 index 0000000..430db49 --- /dev/null +++ b/python/msgpack @@ -0,0 +1 @@ +../msgpack \ No newline at end of file -- cgit v1.2.1 From a1fb1507d45d27f0c08e8d39054987ed6dcaf9e7 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 10 Jun 2009 10:45:07 +0900 Subject: Refactor include path. --- python/msgpack | 1 - 1 file changed, 1 deletion(-) delete mode 120000 python/msgpack (limited to 'python/msgpack') diff --git a/python/msgpack b/python/msgpack deleted file mode 120000 index 430db49..0000000 --- a/python/msgpack +++ /dev/null @@ -1 +0,0 @@ -../msgpack \ No newline at end of file -- cgit v1.2.1 From 3a9f74e79c3d1912d8c0c1ad4d1478c611caba0a Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 10 Jun 2009 10:58:09 +0900 Subject: Make msgpack package instead of module. --- python/msgpack/__init__.py | 3 + python/msgpack/_msgpack.pyx | 202 ++++++++++++++++++++++++++++++++++++++++++++ python/msgpack/pack.h | 121 ++++++++++++++++++++++++++ python/msgpack/unpack.h | 122 ++++++++++++++++++++++++++ 4 files changed, 448 insertions(+) create mode 100644 python/msgpack/__init__.py create mode 100644 python/msgpack/_msgpack.pyx create mode 100644 python/msgpack/pack.h create mode 100644 python/msgpack/unpack.h (limited to 'python/msgpack') diff --git a/python/msgpack/__init__.py b/python/msgpack/__init__.py new file mode 100644 index 0000000..797b29c --- /dev/null +++ b/python/msgpack/__init__.py @@ -0,0 +1,3 @@ +# coding: utf-8 +from _msgpack import * + diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx new file mode 100644 index 0000000..f24f403 --- /dev/null +++ b/python/msgpack/_msgpack.pyx @@ -0,0 +1,202 @@ +# coding: utf-8 + +from cStringIO import StringIO + +cdef extern from "Python.h": + ctypedef char* const_char_ptr "const char*" + ctypedef struct PyObject + cdef object PyString_FromStringAndSize(const_char_ptr b, Py_ssize_t len) + +cdef extern from "stdlib.h": + void* malloc(int) + void free(void*) + +cdef extern from "string.h": + int memcpy(char*dst, char*src, unsigned int size) + +cdef extern from "pack.h": + ctypedef int (*msgpack_packer_write)(void* data, const_char_ptr buf, unsigned int len) + + struct msgpack_packer: + void *data + msgpack_packer_write callback + + void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback) + void msgpack_pack_int(msgpack_packer* pk, int d) + void msgpack_pack_nil(msgpack_packer* pk) + void msgpack_pack_true(msgpack_packer* pk) + void msgpack_pack_false(msgpack_packer* pk) + void msgpack_pack_long_long(msgpack_packer* pk, long long d) + void msgpack_pack_double(msgpack_packer* pk, double d) + void msgpack_pack_array(msgpack_packer* pk, size_t l) + void msgpack_pack_map(msgpack_packer* pk, size_t l) + void msgpack_pack_raw(msgpack_packer* pk, size_t l) + void msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) + + +cdef int BUFF_SIZE=2*1024 + +cdef class Packer: + """Packer that pack data into strm. + + strm must have `write(bytes)` method. + size specifies local buffer size. + """ + cdef char* buff + cdef unsigned int length + cdef unsigned int allocated + cdef msgpack_packer pk + cdef object strm + + def __init__(self, strm, int size=0): + if size <= 0: + size = BUFF_SIZE + + self.strm = strm + self.buff = malloc(size) + self.allocated = size + self.length = 0 + + msgpack_packer_init(&self.pk, self, _packer_write) + + def __del__(self): + free(self.buff); + + def flush(self): + """Flash local buffer and output stream if it has 'flush()' method.""" + if self.length > 0: + self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) + self.length = 0 + if hasattr(self.strm, 'flush'): + self.strm.flush() + + def pack_list(self, len): + """Start packing sequential objects. + + Example: + + packer.pack_list(2) + packer.pack('foo') + packer.pack('bar') + + This code is same as below code: + + packer.pack(['foo', 'bar']) + """ + msgpack_pack_array(&self.pk, len) + + def pack_dict(self, len): + """Start packing key-value objects. + + Example: + + packer.pack_dict(1) + packer.pack('foo') + packer.pack('bar') + + This code is same as below code: + + packer.pack({'foo', 'bar'}) + """ + msgpack_pack_map(&self.pk, len) + + cdef __pack(self, object o): + cdef long long intval + cdef double fval + cdef char* rawval + + if o is None: + msgpack_pack_nil(&self.pk) + elif o is True: + msgpack_pack_true(&self.pk) + elif o is False: + msgpack_pack_false(&self.pk) + elif isinstance(o, long): + intval = o + msgpack_pack_long_long(&self.pk, intval) + elif isinstance(o, int): + intval = o + msgpack_pack_long_long(&self.pk, intval) + elif isinstance(o, float): + fval = 9 + msgpack_pack_double(&self.pk, fval) + elif isinstance(o, str): + rawval = o + msgpack_pack_raw(&self.pk, len(o)) + msgpack_pack_raw_body(&self.pk, rawval, len(o)) + elif isinstance(o, unicode): + o = o.encode('utf-8') + rawval = o + msgpack_pack_raw(&self.pk, len(o)) + msgpack_pack_raw_body(&self.pk, rawval, len(o)) + elif isinstance(o, dict): + msgpack_pack_map(&self.pk, len(o)) + for k,v in o.iteritems(): + self.pack(k) + self.pack(v) + elif isinstance(o, tuple) or isinstance(o, list): + msgpack_pack_array(&self.pk, len(o)) + for v in o: + self.pack(v) + else: + # TODO: Serialize with defalt() like simplejson. + raise TypeError, "can't serialize %r" % (o,) + + def pack(self, obj, flush=True): + self.__pack(obj) + if flush: + self.flush() + +cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): + if packer.length + l > packer.allocated: + if packer.length > 0: + packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) + if l > 64: + packer.strm.write(PyString_FromStringAndSize(b, l)) + packer.length = 0 + else: + memcpy(packer.buff, b, l) + packer.length = l + else: + memcpy(packer.buff + packer.length, b, l) + packer.length += l + return 0 + +def pack(object o, object stream): + packer = Packer(stream) + packer.pack(o) + packer.flush() + +def packs(object o): + buf = StringIO() + packer = Packer(buf) + packer.pack(o) + packer.flush() + return buf.getvalue() + +cdef extern from "unpack.h": + ctypedef struct template_context: + pass + int template_execute(template_context* ctx, const_char_ptr data, + size_t len, size_t* off) + void template_init(template_context* ctx) + object template_data(template_context* ctx) + + +def unpacks(object packed_bytes): + """Unpack packed_bytes to object. Returns unpacked object.""" + cdef const_char_ptr p = packed_bytes + cdef template_context ctx + cdef size_t off = 0 + template_init(&ctx) + template_execute(&ctx, p, len(packed_bytes), &off) + return template_data(&ctx) + +def unpack(object stream): + """unpack from stream.""" + packed = stream.read() + return unpacks(packed) + +cdef class Unpacker: + """Do nothing. This function is for symmetric to Packer""" + unpack = staticmethod(unpacks) diff --git a/python/msgpack/pack.h b/python/msgpack/pack.h new file mode 100644 index 0000000..f3935fb --- /dev/null +++ b/python/msgpack/pack.h @@ -0,0 +1,121 @@ +/* + * MessagePack for Python packing routine + * + * Copyright (C) 2009 Naoki INADA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#if _MSC_VER +typedef signed char uint8_t; +typedef unsigned char uint8_t; +typedef short int16_t; +typedef unsigned short uint16_t; +typedef int int32_t; +typedef unsigned int uint32_t; +typedef long long int64_t; +typedef unsigned long long uint64_t; +#else +#include +#endif + +#include +#include +#include "msgpack/pack_define.h" + +#ifdef __cplusplus +extern "C" { +#endif + + +typedef int (*msgpack_packer_write)(void* data, const char* buf, unsigned int len); + +typedef struct msgpack_packer { + void* data; + msgpack_packer_write callback; +} msgpack_packer; + +static inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback); + +static inline msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback); +static inline void msgpack_packer_free(msgpack_packer* pk); + +static inline int msgpack_pack_short(msgpack_packer* pk, short d); +static inline int msgpack_pack_int(msgpack_packer* pk, int d); +static inline int msgpack_pack_long(msgpack_packer* pk, long d); +static inline int msgpack_pack_long_long(msgpack_packer* pk, long long d); +static inline int msgpack_pack_unsigned_short(msgpack_packer* pk, unsigned short d); +static inline int msgpack_pack_unsigned_int(msgpack_packer* pk, unsigned int d); +static inline int msgpack_pack_unsigned_long(msgpack_packer* pk, unsigned long d); +static inline int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d); + +static inline int msgpack_pack_uint8(msgpack_packer* pk, uint8_t d); +static inline int msgpack_pack_uint16(msgpack_packer* pk, uint16_t d); +static inline int msgpack_pack_uint32(msgpack_packer* pk, uint32_t d); +static inline int msgpack_pack_uint64(msgpack_packer* pk, uint64_t d); +static inline int msgpack_pack_int8(msgpack_packer* pk, int8_t d); +static inline int msgpack_pack_int16(msgpack_packer* pk, int16_t d); +static inline int msgpack_pack_int32(msgpack_packer* pk, int32_t d); +static inline int msgpack_pack_int64(msgpack_packer* pk, int64_t d); + +static inline int msgpack_pack_float(msgpack_packer* pk, float d); +static inline int msgpack_pack_double(msgpack_packer* pk, double d); + +static inline int msgpack_pack_nil(msgpack_packer* pk); +static inline int msgpack_pack_true(msgpack_packer* pk); +static inline int msgpack_pack_false(msgpack_packer* pk); + +static inline int msgpack_pack_array(msgpack_packer* pk, unsigned int n); + +static inline int msgpack_pack_map(msgpack_packer* pk, unsigned int n); + +static inline int msgpack_pack_raw(msgpack_packer* pk, size_t l); +static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l); + + + +#define msgpack_pack_inline_func(name) \ + static inline int msgpack_pack ## name + +#define msgpack_pack_inline_func_cint(name) \ + static inline int msgpack_pack ## name + +#define msgpack_pack_user msgpack_packer* + +#define msgpack_pack_append_buffer(user, buf, len) \ + return (*(user)->callback)((user)->data, (const char*)buf, len) + +#include "msgpack/pack_template.h" + +static inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback) +{ + pk->data = data; + pk->callback = callback; +} + +static inline msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback) +{ + msgpack_packer* pk = (msgpack_packer*)calloc(1, sizeof(msgpack_packer)); + if(!pk) { return NULL; } + msgpack_packer_init(pk, data, callback); + return pk; +} + +static inline void msgpack_packer_free(msgpack_packer* pk) +{ + free(pk); +} + + +#ifdef __cplusplus +} +#endif diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h new file mode 100644 index 0000000..e51557f --- /dev/null +++ b/python/msgpack/unpack.h @@ -0,0 +1,122 @@ +/* + * MessagePack for Python unpacking routine + * + * Copyright (C) 2009 Naoki INADA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "msgpack/unpack_define.h" + +typedef struct { + int reserved; +} unpack_user; + + +#define msgpack_unpack_struct(name) \ + struct template ## name + +#define msgpack_unpack_func(ret, name) \ + static inline ret template ## name + +#define msgpack_unpack_callback(name) \ + template_callback ## name + +#define msgpack_unpack_object PyObject* + +#define msgpack_unpack_user unpack_user + + +struct template_context; +typedef struct template_context template_context; + +static inline msgpack_unpack_object template_callback_root(unpack_user* u) +{ return NULL; } + +static inline int template_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpack_object* o) +{ *o = PyInt_FromLong((long)d); return 0; } + +static inline int template_callback_uint16(unpack_user* u, uint16_t d, msgpack_unpack_object* o) +{ *o = PyInt_FromLong((long)d); return 0; } + +static inline int template_callback_uint32(unpack_user* u, uint32_t d, msgpack_unpack_object* o) +{ + if (d >= 0x80000000UL) { + *o = PyLong_FromUnsignedLongLong((unsigned long long)d); + } else { + *o = PyInt_FromLong((long)d); + } + return 0; +} + +static inline int template_callback_uint64(unpack_user* u, uint64_t d, msgpack_unpack_object* o) +{ *o = PyLong_FromUnsignedLongLong(d); return 0; } + +static inline int template_callback_int8(unpack_user* u, int8_t d, msgpack_unpack_object* o) +{ *o = PyInt_FromLong(d); return 0; } + +static inline int template_callback_int16(unpack_user* u, int16_t d, msgpack_unpack_object* o) +{ *o = PyInt_FromLong(d); return 0; } + +static inline int template_callback_int32(unpack_user* u, int32_t d, msgpack_unpack_object* o) +{ *o = PyInt_FromLong(d); return 0; } + +static inline int template_callback_int64(unpack_user* u, int64_t d, msgpack_unpack_object* o) +{ *o = PyLong_FromLongLong(d); return 0; } + +static inline int template_callback_float(unpack_user* u, float d, msgpack_unpack_object* o) +{ *o = PyFloat_FromDouble((double)d); return 0; } + +static inline int template_callback_double(unpack_user* u, double d, msgpack_unpack_object* o) +{ *o = PyFloat_FromDouble(d); return 0; } + +static inline int template_callback_nil(unpack_user* u, msgpack_unpack_object* o) +{ Py_INCREF(Py_None); *o = Py_None; return 0; } + +static inline int template_callback_true(unpack_user* u, msgpack_unpack_object* o) +{ Py_INCREF(Py_True); *o = Py_True; return 0; } + +static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* o) +{ Py_INCREF(Py_False); *o = Py_False; return 0; } + +static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) +{ + /* TODO: use PyList_New(n). */ + *o = PyList_New(0); + return 0; +} + +static inline int template_callback_array_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object o) +{ + PyList_Append(*c, o); + return 0; +} + +static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o) +{ + *o = PyDict_New(); + return 0; +} + +static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v) +{ + PyDict_SetItem(*c, k, v); + return 0; +} + +static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) +{ + *o = PyString_FromStringAndSize(p, l); + return 0; +} + +#include "msgpack/unpack_template.h" -- cgit v1.2.1 From e814986b4ec017ab124dcf16496a6fefa65a9876 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 10 Jun 2009 13:33:42 +0900 Subject: Fix document miss. --- python/msgpack/_msgpack.pyx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'python/msgpack') diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index f24f403..7c07cde 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -79,7 +79,7 @@ cdef class Packer: packer.pack('foo') packer.pack('bar') - This code is same as below code: + This is same to: packer.pack(['foo', 'bar']) """ @@ -94,9 +94,9 @@ cdef class Packer: packer.pack('foo') packer.pack('bar') - This code is same as below code: + This is same to: - packer.pack({'foo', 'bar'}) + packer.pack({'foo': 'bar'}) """ msgpack_pack_map(&self.pk, len) -- cgit v1.2.1 From 87f5df1503e70efd2bb9e9565259f1ed37a45933 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 22 Jun 2009 15:59:02 +0900 Subject: Use std::stack. --- python/msgpack/unpack.h | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) (limited to 'python/msgpack') diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h index 694e816..daeb54c 100644 --- a/python/msgpack/unpack.h +++ b/python/msgpack/unpack.h @@ -18,17 +18,21 @@ #include #include +#include #define MSGPACK_MAX_STACK_SIZE (1024) #include "msgpack/unpack_define.h" using namespace std; -typedef struct unpack_user { - struct array_stack_type{unsigned int size, last;}; - array_stack_type array_stack[MSGPACK_MAX_STACK_SIZE]; - int array_current; - +struct array_context { + unsigned int size; + unsigned int last; + stack_item(unsigned int size) : size(size), last(0) + {} +}; +struct unpack_user { + stack array_stack; map str_cache; ~unpack_user() { @@ -38,7 +42,7 @@ typedef struct unpack_user { Py_DECREF(it->second); } } -} unpack_user; +}; #define msgpack_unpack_struct(name) \ @@ -60,7 +64,6 @@ typedef struct template_context template_context; static inline msgpack_unpack_object template_callback_root(unpack_user* u) { - u->array_current = -1; return NULL; } @@ -113,9 +116,7 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { if (n > 0) { - int cur = ++u->array_current; - u->array_stack[cur].size = n; - u->array_stack[cur].last = 0; + u->array_stack.push(stack_item(n)); *o = PyList_New(n); } else { @@ -126,17 +127,13 @@ static inline int template_callback_array(unpack_user* u, unsigned int n, msgpac static inline int template_callback_array_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object o) { - int cur = u->array_current; - int n = u->array_stack[cur].size; - int last = u->array_stack[cur].last; + unsigned int n = u->array_stack.top().size; + unsigned int &last = u->array_stack.top().last; PyList_SetItem(*c, last, o); last++; if (last >= n) { - u->array_current--; - } - else { - u->array_stack[cur].last = last; + u->array_stack.pop(); } return 0; } -- cgit v1.2.1 From 46d7c656214073cde4c458c7cf1eb03e2d33dbd9 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 22 Jun 2009 19:49:02 +0900 Subject: Fix compile error. --- python/msgpack/pack.h | 12 ------------ python/msgpack/unpack.h | 4 ++-- 2 files changed, 2 insertions(+), 14 deletions(-) (limited to 'python/msgpack') diff --git a/python/msgpack/pack.h b/python/msgpack/pack.h index f3935fb..544950b 100644 --- a/python/msgpack/pack.h +++ b/python/msgpack/pack.h @@ -15,18 +15,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#if _MSC_VER -typedef signed char uint8_t; -typedef unsigned char uint8_t; -typedef short int16_t; -typedef unsigned short uint16_t; -typedef int int32_t; -typedef unsigned int uint32_t; -typedef long long int64_t; -typedef unsigned long long uint64_t; -#else -#include -#endif #include #include diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h index daeb54c..b753493 100644 --- a/python/msgpack/unpack.h +++ b/python/msgpack/unpack.h @@ -28,7 +28,7 @@ using namespace std; struct array_context { unsigned int size; unsigned int last; - stack_item(unsigned int size) : size(size), last(0) + array_context(unsigned int size) : size(size), last(0) {} }; struct unpack_user { @@ -116,7 +116,7 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { if (n > 0) { - u->array_stack.push(stack_item(n)); + u->array_stack.push(array_context(n)); *o = PyList_New(n); } else { -- cgit v1.2.1 From dd53b141ef6d2495d0cc14c8949180a74ae3caf6 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 24 Jun 2009 01:13:39 +0900 Subject: Remove unneccessary value. --- python/msgpack/unpack.h | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) (limited to 'python/msgpack') diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h index b753493..12702d8 100644 --- a/python/msgpack/unpack.h +++ b/python/msgpack/unpack.h @@ -25,14 +25,8 @@ using namespace std; -struct array_context { - unsigned int size; - unsigned int last; - array_context(unsigned int size) : size(size), last(0) - {} -}; struct unpack_user { - stack array_stack; + stack array_stack; map str_cache; ~unpack_user() { @@ -116,7 +110,7 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { if (n > 0) { - u->array_stack.push(array_context(n)); + u->array_stack.push(0); *o = PyList_New(n); } else { @@ -127,12 +121,12 @@ static inline int template_callback_array(unpack_user* u, unsigned int n, msgpac static inline int template_callback_array_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object o) { - unsigned int n = u->array_stack.top().size; - unsigned int &last = u->array_stack.top().last; - - PyList_SetItem(*c, last, o); + unsigned int &last = u->array_stack.top(); + PyList_SET_ITEM(*c, last, o); last++; - if (last >= n) { + + Py_ssize_t len = PyList_GET_SIZE(*c); + if (last >= len) { u->array_stack.pop(); } return 0; -- cgit v1.2.1 From 3fd28d07928c51cb563f35d9dddb0c7867ac9875 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 24 Jun 2009 01:38:48 +0900 Subject: Remove duplicated values. --- python/msgpack/pack.h | 4 +- python/msgpack/pack_define.h | 26 ++ python/msgpack/pack_template.h | 741 +++++++++++++++++++++++++++++++++++++++ python/msgpack/unpack.h | 53 +-- python/msgpack/unpack_define.h | 129 +++++++ python/msgpack/unpack_template.h | 363 +++++++++++++++++++ 6 files changed, 1276 insertions(+), 40 deletions(-) create mode 100644 python/msgpack/pack_define.h create mode 100644 python/msgpack/pack_template.h create mode 100644 python/msgpack/unpack_define.h create mode 100644 python/msgpack/unpack_template.h (limited to 'python/msgpack') diff --git a/python/msgpack/pack.h b/python/msgpack/pack.h index 544950b..9bd6b68 100644 --- a/python/msgpack/pack.h +++ b/python/msgpack/pack.h @@ -18,7 +18,7 @@ #include #include -#include "msgpack/pack_define.h" +#include "pack_define.h" #ifdef __cplusplus extern "C" { @@ -82,7 +82,7 @@ static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_ #define msgpack_pack_append_buffer(user, buf, len) \ return (*(user)->callback)((user)->data, (const char*)buf, len) -#include "msgpack/pack_template.h" +#include "pack_template.h" static inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback) { diff --git a/python/msgpack/pack_define.h b/python/msgpack/pack_define.h new file mode 100644 index 0000000..33408e5 --- /dev/null +++ b/python/msgpack/pack_define.h @@ -0,0 +1,26 @@ +/* + * MessagePack unpacking routine template + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_PACK_DEFINE_H__ +#define MSGPACK_PACK_DEFINE_H__ + +#include +#include +#include + +#endif /* msgpack/pack_define.h */ + diff --git a/python/msgpack/pack_template.h b/python/msgpack/pack_template.h new file mode 100644 index 0000000..aa620f5 --- /dev/null +++ b/python/msgpack/pack_template.h @@ -0,0 +1,741 @@ +/* + * MessagePack packing routine template + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) +#if __BYTE_ORDER == __LITTLE_ENDIAN +#define __LITTLE_ENDIAN__ +#elif __BYTE_ORDER == __BIG_ENDIAN +#define __BIG_ENDIAN__ +#endif +#endif + + +#ifdef __LITTLE_ENDIAN__ + +#define STORE8_BE8(d) \ + ((uint8_t*)&d)[0] + + +#define STORE16_BE8(d) \ + ((uint8_t*)&d)[0] + +#define STORE16_BE16(d) \ + ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + + +#define STORE32_BE8(d) \ + ((uint8_t*)&d)[0] + +#define STORE32_BE16(d) \ + ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + +#define STORE32_BE32(d) \ + ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + + +#define STORE64_BE8(d) \ + ((uint8_t*)&d)[0] + +#define STORE64_BE16(d) \ + ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + +#define STORE64_BE32(d) \ + ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + +#define STORE64_BE64(d) \ + ((uint8_t*)&d)[7], ((uint8_t*)&d)[6], ((uint8_t*)&d)[5], ((uint8_t*)&d)[4], \ + ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + + +#elif __BIG_ENDIAN__ + +#define STORE8_BE8(d) \ + ((uint8_t*)&d)[0] + + +#define STORE16_BE8(d) \ + ((uint8_t*)&d)[1] + +#define STORE16_BE16(d) \ + ((uint8_t*)&d)[0], ((uint8_t*)&d)[1] + + +#define STORE32_BE8(d) \ + ((uint8_t*)&d)[3] + +#define STORE32_BE16(d) \ + ((uint8_t*)&d)[2], ((uint8_t*)&d)[3] + +#define STORE32_BE32(d) \ + ((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3] + + +#define STORE64_BE8(d) \ + ((uint8_t*)&d)[7] + +#define STORE64_BE16(d) \ + ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] + +#define STORE64_BE32(d) \ + ((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] + +#define STORE64_BE64(d) \ + ((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3], \ + ((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] + +#endif + +#ifndef msgpack_pack_inline_func +#error msgpack_pack_inline_func template is not defined +#endif + +#ifndef msgpack_pack_user +#error msgpack_pack_user type is not defined +#endif + +#ifndef msgpack_pack_append_buffer +#error msgpack_pack_append_buffer callback is not defined +#endif + + +/* + * Integer + */ + +#define msgpack_pack_real_uint8(x, d) \ +do { \ + if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \ + } else { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE8_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ +} while(0) + +#define msgpack_pack_real_uint16(x, d) \ +do { \ + if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \ + } else if(d < (1<<8)) { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } \ +} while(0) + +#define msgpack_pack_real_uint32(x, d) \ +do { \ + if(d < (1<<8)) { \ + if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \ + } else { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else { \ + if(d < (1<<16)) { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* unsigned 32 */ \ + const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } \ + } \ +} while(0) + +#define msgpack_pack_real_uint64(x, d) \ +do { \ + if(d < (1ULL<<8)) { \ + if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \ + } else { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else { \ + if(d < (1ULL<<16)) { \ + /* signed 16 */ \ + const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else if(d < (1ULL<<32)) { \ + /* signed 32 */ \ + const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } else { \ + /* signed 64 */ \ + const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \ + msgpack_pack_append_buffer(x, buf, 9); \ + } \ + } \ +} while(0) + +#define msgpack_pack_real_int8(x, d) \ +do { \ + if(d < -(1<<5)) { \ + /* signed 8 */ \ + const unsigned char buf[2] = {0xd0, STORE8_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \ + } \ +} while(0) + +#define msgpack_pack_real_int16(x, d) \ +do { \ + if(d < -(1<<5)) { \ + if(d < -(1<<7)) { \ + /* signed 16 */ \ + const unsigned char buf[3] = {0xd1, STORE16_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* signed 8 */ \ + const unsigned char buf[2] = {0xd0, STORE16_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \ + } else { \ + if(d < (1<<8)) { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } \ + } \ +} while(0) + +#define msgpack_pack_real_int32(x, d) \ +do { \ + if(d < -(1<<5)) { \ + if(d < -(1<<15)) { \ + /* signed 32 */ \ + const unsigned char buf[5] = {0xd2, STORE32_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } else if(d < -(1<<7)) { \ + /* signed 16 */ \ + const unsigned char buf[3] = {0xd1, STORE32_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* signed 8 */ \ + const unsigned char buf[2] = {0xd0, STORE32_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \ + } else { \ + if(d < (1<<8)) { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else if(d < (1<<16)) { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* unsigned 32 */ \ + const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } \ + } \ +} while(0) + +#define msgpack_pack_real_int64(x, d) \ +do { \ + if(d < -(1LL<<5)) { \ + if(d < -(1LL<<15)) { \ + if(d < -(1LL<<31)) { \ + /* signed 64 */ \ + const unsigned char buf[9] = {0xd3, STORE64_BE64(d)}; \ + msgpack_pack_append_buffer(x, buf, 9); \ + } else { \ + /* signed 32 */ \ + const unsigned char buf[5] = {0xd2, STORE64_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } \ + } else { \ + if(d < -(1<<7)) { \ + /* signed 16 */ \ + const unsigned char buf[3] = {0xd1, STORE64_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* signed 8 */ \ + const unsigned char buf[2] = {0xd0, STORE64_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } \ + } else if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \ + } else { \ + if(d < (1LL<<16)) { \ + if(d < (1<<8)) { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } \ + } else { \ + if(d < (1LL<<32)) { \ + /* unsigned 32 */ \ + const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } else { \ + /* unsigned 64 */ \ + const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \ + msgpack_pack_append_buffer(x, buf, 9); \ + } \ + } \ + } \ +} while(0) + + +#ifdef msgpack_pack_inline_func_fastint + +msgpack_pack_inline_func_fastint(_uint8)(msgpack_pack_user x, uint8_t d) +{ + const unsigned char buf[2] = {0xcc, STORE8_BE8(d)}; + msgpack_pack_append_buffer(x, buf, 2); +} + +msgpack_pack_inline_func_fastint(_uint16)(msgpack_pack_user x, uint16_t d) +{ + const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; + msgpack_pack_append_buffer(x, buf, 3); +} + +msgpack_pack_inline_func_fastint(_uint32)(msgpack_pack_user x, uint32_t d) +{ + const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; + msgpack_pack_append_buffer(x, buf, 5); +} + +msgpack_pack_inline_func_fastint(_uint64)(msgpack_pack_user x, uint64_t d) +{ + const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; + msgpack_pack_append_buffer(x, buf, 9); +} + +msgpack_pack_inline_func_fastint(_int8)(msgpack_pack_user x, int8_t d) +{ + const unsigned char buf[2] = {0xd0, STORE8_BE8(d)}; + msgpack_pack_append_buffer(x, buf, 2); +} + +msgpack_pack_inline_func_fastint(_int16)(msgpack_pack_user x, int16_t d) +{ + const unsigned char buf[3] = {0xd1, STORE16_BE16(d)}; + msgpack_pack_append_buffer(x, buf, 3); +} + +msgpack_pack_inline_func_fastint(_int32)(msgpack_pack_user x, int32_t d) +{ + const unsigned char buf[5] = {0xd2, STORE32_BE32(d)}; + msgpack_pack_append_buffer(x, buf, 5); +} + +msgpack_pack_inline_func_fastint(_int64)(msgpack_pack_user x, int64_t d) +{ + const unsigned char buf[9] = {0xd3, STORE64_BE64(d)}; + msgpack_pack_append_buffer(x, buf, 9); +} + +#undef msgpack_pack_inline_func_fastint +#endif + + +msgpack_pack_inline_func(_uint8)(msgpack_pack_user x, uint8_t d) +{ + msgpack_pack_real_uint8(x, d); +} + +msgpack_pack_inline_func(_uint16)(msgpack_pack_user x, uint16_t d) +{ + msgpack_pack_real_uint16(x, d); +} + +msgpack_pack_inline_func(_uint32)(msgpack_pack_user x, uint32_t d) +{ + msgpack_pack_real_uint32(x, d); +} + +msgpack_pack_inline_func(_uint64)(msgpack_pack_user x, uint64_t d) +{ + msgpack_pack_real_uint64(x, d); +} + +msgpack_pack_inline_func(_int8)(msgpack_pack_user x, int8_t d) +{ + msgpack_pack_real_int8(x, d); +} + +msgpack_pack_inline_func(_int16)(msgpack_pack_user x, int16_t d) +{ + msgpack_pack_real_int16(x, d); +} + +msgpack_pack_inline_func(_int32)(msgpack_pack_user x, int32_t d) +{ + msgpack_pack_real_int32(x, d); +} + +msgpack_pack_inline_func(_int64)(msgpack_pack_user x, int64_t d) +{ + msgpack_pack_real_int64(x, d); +} + + +#ifdef msgpack_pack_inline_func_cint + +msgpack_pack_inline_func_cint(_short)(msgpack_pack_user x, short d) +{ +#if defined(SIZEOF_SHORT) || defined(SHRT_MAX) +#if SIZEOF_SHORT == 2 || SHRT_MAX == 0x7fff + msgpack_pack_real_int16(x, d); +#elif SIZEOF_SHORT == 4 || SHRT_MAX == 0x7fffffff + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif +#else +if(sizeof(short) == 2) { + msgpack_pack_real_int16(x, d); +} else if(sizeof(short) == 4) { + msgpack_pack_real_int32(x, d); +} else { + msgpack_pack_real_int64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_int)(msgpack_pack_user x, int d) +{ +#if defined(SIZEOF_INT) || defined(INT_MAX) +#if SIZEOF_INT == 2 || INT_MAX == 0x7fff + msgpack_pack_real_int16(x, d); +#elif SIZEOF_INT == 4 || INT_MAX == 0x7fffffff + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif +#else +if(sizeof(int) == 2) { + msgpack_pack_real_int16(x, d); +} else if(sizeof(int) == 4) { + msgpack_pack_real_int32(x, d); +} else { + msgpack_pack_real_int64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_long)(msgpack_pack_user x, long d) +{ +#if defined(SIZEOF_LONG) || defined(LONG_MAX) +#if SIZEOF_LONG == 2 || LONG_MAX == 0x7fffL + msgpack_pack_real_int16(x, d); +#elif SIZEOF_LONG == 4 || LONG_MAX == 0x7fffffffL + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif +#else +if(sizeof(long) == 2) { + msgpack_pack_real_int16(x, d); +} else if(sizeof(long) == 4) { + msgpack_pack_real_int32(x, d); +} else { + msgpack_pack_real_int64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_long_long)(msgpack_pack_user x, long long d) +{ +#if defined(SIZEOF_LONG_LONG) || defined(LLONG_MAX) +#if SIZEOF_LONG_LONG == 2 || LLONG_MAX == 0x7fffL + msgpack_pack_real_int16(x, d); +#elif SIZEOF_LONG_LONG == 4 || LLONG_MAX == 0x7fffffffL + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif +#else +if(sizeof(long long) == 2) { + msgpack_pack_real_int16(x, d); +} else if(sizeof(long long) == 4) { + msgpack_pack_real_int32(x, d); +} else { + msgpack_pack_real_int64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_unsigned_short)(msgpack_pack_user x, unsigned short d) +{ +#if defined(SIZEOF_SHORT) || defined(USHRT_MAX) +#if SIZEOF_SHORT == 2 || USHRT_MAX == 0xffffU + msgpack_pack_real_uint16(x, d); +#elif SIZEOF_SHORT == 4 || USHRT_MAX == 0xffffffffU + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif +#else +if(sizeof(unsigned short) == 2) { + msgpack_pack_real_uint16(x, d); +} else if(sizeof(unsigned short) == 4) { + msgpack_pack_real_uint32(x, d); +} else { + msgpack_pack_real_uint64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_unsigned_int)(msgpack_pack_user x, unsigned int d) +{ +#if defined(SIZEOF_INT) || defined(UINT_MAX) +#if SIZEOF_INT == 2 || UINT_MAX == 0xffffU + msgpack_pack_real_uint16(x, d); +#elif SIZEOF_INT == 4 || UINT_MAX == 0xffffffffU + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif +#else +if(sizeof(unsigned int) == 2) { + msgpack_pack_real_uint16(x, d); +} else if(sizeof(unsigned int) == 4) { + msgpack_pack_real_uint32(x, d); +} else { + msgpack_pack_real_uint64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_unsigned_long)(msgpack_pack_user x, unsigned long d) +{ +#if defined(SIZEOF_LONG) || defined(ULONG_MAX) +#if SIZEOF_LONG == 2 || ULONG_MAX == 0xffffUL + msgpack_pack_real_uint16(x, d); +#elif SIZEOF_LONG == 4 || ULONG_MAX == 0xffffffffUL + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif +#else +if(sizeof(unsigned int) == 2) { + msgpack_pack_real_uint16(x, d); +} else if(sizeof(unsigned int) == 4) { + msgpack_pack_real_uint32(x, d); +} else { + msgpack_pack_real_uint64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_unsigned_long_long)(msgpack_pack_user x, unsigned long long d) +{ +#if defined(SIZEOF_LONG_LONG) || defined(ULLONG_MAX) +#if SIZEOF_LONG_LONG == 2 || ULLONG_MAX == 0xffffUL + msgpack_pack_real_uint16(x, d); +#elif SIZEOF_LONG_LONG == 4 || ULLONG_MAX == 0xffffffffUL + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif +#else +if(sizeof(unsigned long long) == 2) { + msgpack_pack_real_uint16(x, d); +} else if(sizeof(unsigned long long) == 4) { + msgpack_pack_real_uint32(x, d); +} else { + msgpack_pack_real_uint64(x, d); +} +#endif +} + +#undef msgpack_pack_inline_func_cint +#endif + + + +/* + * Float + */ + +msgpack_pack_inline_func(_float)(msgpack_pack_user x, float d) +{ + union { char buf[4]; uint32_t num; } f; + *((float*)&f.buf) = d; // FIXME + const unsigned char buf[5] = {0xca, STORE32_BE32(f.num)}; + msgpack_pack_append_buffer(x, buf, 5); +} + +msgpack_pack_inline_func(_double)(msgpack_pack_user x, double d) +{ + union { char buf[8]; uint64_t num; } f; + *((double*)&f.buf) = d; // FIXME + const unsigned char buf[9] = {0xcb, STORE64_BE64(f.num)}; + msgpack_pack_append_buffer(x, buf, 9); +} + + +/* + * Nil + */ + +msgpack_pack_inline_func(_nil)(msgpack_pack_user x) +{ + static const unsigned char d = 0xc0; + msgpack_pack_append_buffer(x, &d, 1); +} + + +/* + * Boolean + */ + +msgpack_pack_inline_func(_true)(msgpack_pack_user x) +{ + static const unsigned char d = 0xc3; + msgpack_pack_append_buffer(x, &d, 1); +} + +msgpack_pack_inline_func(_false)(msgpack_pack_user x) +{ + static const unsigned char d = 0xc2; + msgpack_pack_append_buffer(x, &d, 1); +} + + +/* + * Array + */ + +msgpack_pack_inline_func(_array)(msgpack_pack_user x, unsigned int n) +{ + if(n < 16) { + unsigned char d = 0x90 | n; + msgpack_pack_append_buffer(x, &d, 1); + } else if(n < 65536) { + uint16_t d = (uint16_t)n; + unsigned char buf[3] = {0xdc, STORE16_BE16(d)}; + msgpack_pack_append_buffer(x, buf, 3); + } else { + uint32_t d = (uint32_t)n; + unsigned char buf[5] = {0xdd, STORE32_BE32(d)}; + msgpack_pack_append_buffer(x, buf, 5); + } +} + + +/* + * Map + */ + +msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n) +{ + if(n < 16) { + unsigned char d = 0x80 | n; + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); + } else if(n < 65536) { + uint16_t d = (uint16_t)n; + unsigned char buf[3] = {0xde, STORE16_BE16(d)}; + msgpack_pack_append_buffer(x, buf, 3); + } else { + uint32_t d = (uint32_t)n; + unsigned char buf[5] = {0xdf, STORE32_BE32(d)}; + msgpack_pack_append_buffer(x, buf, 5); + } +} + + +/* + * Raw + */ + +msgpack_pack_inline_func(_raw)(msgpack_pack_user x, size_t l) +{ + if(l < 32) { + unsigned char d = 0xa0 | l; + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); + } else if(l < 65536) { + uint16_t d = (uint16_t)l; + unsigned char buf[3] = {0xda, STORE16_BE16(d)}; + msgpack_pack_append_buffer(x, buf, 3); + } else { + uint32_t d = (uint32_t)l; + unsigned char buf[5] = {0xdb, STORE32_BE32(d)}; + msgpack_pack_append_buffer(x, buf, 5); + } +} + +msgpack_pack_inline_func(_raw_body)(msgpack_pack_user x, const void* b, size_t l) +{ + msgpack_pack_append_buffer(x, (const unsigned char*)b, l); +} + +#undef msgpack_pack_inline_func +#undef msgpack_pack_user +#undef msgpack_pack_append_buffer + +#undef STORE8_BE8 + +#undef STORE16_BE8 +#undef STORE16_BE16 + +#undef STORE32_BE8 +#undef STORE32_BE16 +#undef STORE32_BE32 + +#undef STORE64_BE8 +#undef STORE64_BE16 +#undef STORE64_BE32 +#undef STORE64_BE64 + +#undef msgpack_pack_real_uint8 +#undef msgpack_pack_real_uint16 +#undef msgpack_pack_real_uint32 +#undef msgpack_pack_real_uint64 +#undef msgpack_pack_real_int8 +#undef msgpack_pack_real_int16 +#undef msgpack_pack_real_int32 +#undef msgpack_pack_real_int64 + diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h index 12702d8..cc48d9c 100644 --- a/python/msgpack/unpack.h +++ b/python/msgpack/unpack.h @@ -21,18 +21,18 @@ #include #define MSGPACK_MAX_STACK_SIZE (1024) -#include "msgpack/unpack_define.h" +#include "unpack_define.h" using namespace std; +typedef map str_cach_t; struct unpack_user { - stack array_stack; - map str_cache; + str_cach_t strcache; ~unpack_user() { - map::iterator it, itend; - itend = str_cache.end(); - for (it = str_cache.begin(); it != itend; ++it) { + str_cach_t::iterator it, itend; + itend = strcache.end(); + for (it = strcache.begin(); it != itend; ++it) { Py_DECREF(it->second); } } @@ -108,35 +108,13 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* { Py_INCREF(Py_False); *o = Py_False; return 0; } static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) -{ - if (n > 0) { - u->array_stack.push(0); - *o = PyList_New(n); - } - else { - *o = PyList_New(0); - } - return 0; -} - -static inline int template_callback_array_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object o) -{ - unsigned int &last = u->array_stack.top(); - PyList_SET_ITEM(*c, last, o); - last++; +{ *o = PyList_New(n); return 0; } - Py_ssize_t len = PyList_GET_SIZE(*c); - if (last >= len) { - u->array_stack.pop(); - } - return 0; -} +static inline int template_callback_array_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object o) +{ PyList_SET_ITEM(*c, current, o); return 0; } static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o) -{ - *o = PyDict_New(); - return 0; -} +{ *o = PyDict_New(); return 0; } static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v) { @@ -150,16 +128,15 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha { if (l < 16) { string s(p, l); - map::iterator it = u->str_cache.find(s); - if (it != u->str_cache.end()) { + str_cach_t ::iterator it = u->strcache.find(s); + if (it != u->strcache.end()) { *o = it->second; - Py_INCREF(*o); } else { *o = PyString_FromStringAndSize(p, l); - Py_INCREF(*o); - u->str_cache[s] = *o; + u->strcache[s] = *o; } + Py_INCREF(*o); } else { *o = PyString_FromStringAndSize(p, l); @@ -167,4 +144,4 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha return 0; } -#include "msgpack/unpack_template.h" +#include "unpack_template.h" diff --git a/python/msgpack/unpack_define.h b/python/msgpack/unpack_define.h new file mode 100644 index 0000000..63668c2 --- /dev/null +++ b/python/msgpack/unpack_define.h @@ -0,0 +1,129 @@ +/* + * MessagePack unpacking routine template + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_UNPACK_DEFINE_H__ +#define MSGPACK_UNPACK_DEFINE_H__ + +#include +#include +#include +#include +#include +#ifndef __WIN32__ +#include +#endif + +#ifdef __cplusplus +extern "C" { +#endif + + +#ifndef MSGPACK_MAX_STACK_SIZE +#define MSGPACK_MAX_STACK_SIZE 16 +#endif + + +#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) +#if __BYTE_ORDER == __LITTLE_ENDIAN +#define __LITTLE_ENDIAN__ +#elif __BYTE_ORDER == __BIG_ENDIAN +#define __BIG_ENDIAN__ +#endif +#endif + +#define msgpack_betoh16(x) ntohs(x) +#define msgpack_betoh32(x) ntohl(x) + +#ifdef __LITTLE_ENDIAN__ +#if defined(__bswap_64) +# define msgpack_betoh64(x) __bswap_64(x) +#elif defined(__DARWIN_OSSwapInt64) +# define msgpack_betoh64(x) __DARWIN_OSSwapInt64(x) +#else +static inline uint64_t msgpack_betoh64(uint64_t x) { + return ((x << 56) & 0xff00000000000000ULL ) | + ((x << 40) & 0x00ff000000000000ULL ) | + ((x << 24) & 0x0000ff0000000000ULL ) | + ((x << 8) & 0x000000ff00000000ULL ) | + ((x >> 8) & 0x00000000ff000000ULL ) | + ((x >> 24) & 0x0000000000ff0000ULL ) | + ((x >> 40) & 0x000000000000ff00ULL ) | + ((x >> 56) & 0x00000000000000ffULL ) ; +} +#endif +#else +#define msgpack_betoh64(x) (x) +#endif + + +typedef enum { + CS_HEADER = 0x00, // nil + + //CS_ = 0x01, + //CS_ = 0x02, // false + //CS_ = 0x03, // true + + //CS_ = 0x04, + //CS_ = 0x05, + //CS_ = 0x06, + //CS_ = 0x07, + + //CS_ = 0x08, + //CS_ = 0x09, + CS_FLOAT = 0x0a, + CS_DOUBLE = 0x0b, + CS_UINT_8 = 0x0c, + CS_UINT_16 = 0x0d, + CS_UINT_32 = 0x0e, + CS_UINT_64 = 0x0f, + CS_INT_8 = 0x10, + CS_INT_16 = 0x11, + CS_INT_32 = 0x12, + CS_INT_64 = 0x13, + + //CS_ = 0x14, + //CS_ = 0x15, + //CS_BIG_INT_16 = 0x16, + //CS_BIG_INT_32 = 0x17, + //CS_BIG_FLOAT_16 = 0x18, + //CS_BIG_FLOAT_32 = 0x19, + CS_RAW_16 = 0x1a, + CS_RAW_32 = 0x1b, + CS_ARRAY_16 = 0x1c, + CS_ARRAY_32 = 0x1d, + CS_MAP_16 = 0x1e, + CS_MAP_32 = 0x1f, + + //ACS_BIG_INT_VALUE, + //ACS_BIG_FLOAT_VALUE, + ACS_RAW_VALUE, +} msgpack_unpack_state; + + +typedef enum { + CT_ARRAY_ITEM, + CT_MAP_KEY, + CT_MAP_VALUE, +} msgpack_container_type; + + +#ifdef __cplusplus +} +#endif + +#endif /* msgpack/unpack_define.h */ + diff --git a/python/msgpack/unpack_template.h b/python/msgpack/unpack_template.h new file mode 100644 index 0000000..db33368 --- /dev/null +++ b/python/msgpack/unpack_template.h @@ -0,0 +1,363 @@ +/* + * MessagePack unpacking routine template + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef msgpack_unpack_func +#error msgpack_unpack_func template is not defined +#endif + +#ifndef msgpack_unpack_callback +#error msgpack_unpack_callback template is not defined +#endif + +#ifndef msgpack_unpack_struct +#error msgpack_unpack_struct template is not defined +#endif + +#ifndef msgpack_unpack_struct_decl +#define msgpack_unpack_struct_decl(name) msgpack_unpack_struct(name) +#endif + +#ifndef msgpack_unpack_object +#error msgpack_unpack_object type is not defined +#endif + +#ifndef msgpack_unpack_user +#error msgpack_unpack_user type is not defined +#endif + + +msgpack_unpack_struct_decl(_stack) { + msgpack_unpack_object obj; + size_t curr; + size_t count; + unsigned int ct; + msgpack_unpack_object map_key; +}; + +msgpack_unpack_struct_decl(_context) { + msgpack_unpack_user user; + unsigned int cs; + unsigned int trail; + unsigned int top; + msgpack_unpack_struct(_stack) stack[MSGPACK_MAX_STACK_SIZE]; +}; + + +msgpack_unpack_func(void, _init)(msgpack_unpack_struct(_context)* ctx) +{ + ctx->cs = CS_HEADER; + ctx->trail = 0; + ctx->top = 0; + ctx->stack[0].obj = msgpack_unpack_callback(_root)(&ctx->user); +} + +msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context)* ctx) +{ + return (ctx)->stack[0].obj; +} + + +msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off) +{ + assert(len >= *off); + + const unsigned char* p = (unsigned char*)data + *off; + const unsigned char* const pe = (unsigned char*)data + len; + const void* n = NULL; + + unsigned int trail = ctx->trail; + unsigned int cs = ctx->cs; + unsigned int top = ctx->top; + msgpack_unpack_struct(_stack)* stack = ctx->stack; + msgpack_unpack_user* user = &ctx->user; + + msgpack_unpack_object obj; + msgpack_unpack_struct(_stack)* c = NULL; + + int ret; + +#define push_simple_value(func) \ + if(msgpack_unpack_callback(func)(user, &obj) < 0) { goto _failed; } \ + goto _push +#define push_fixed_value(func, arg) \ + if(msgpack_unpack_callback(func)(user, arg, &obj) < 0) { goto _failed; } \ + goto _push +#define push_variable_value(func, base, pos, len) \ + if(msgpack_unpack_callback(func)(user, \ + (const char*)base, (const char*)pos, len, &obj) < 0) { goto _failed; } \ + goto _push + +#define again_fixed_trail(_cs, trail_len) \ + trail = trail_len; \ + cs = _cs; \ + goto _fixed_trail_again +#define again_fixed_trail_if_zero(_cs, trail_len, ifzero) \ + trail = trail_len; \ + if(trail == 0) { goto ifzero; } \ + cs = _cs; \ + goto _fixed_trail_again + +#define start_container(func, count_, ct_) \ + if(msgpack_unpack_callback(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \ + if((count_) == 0) { obj = stack[top].obj; goto _push; } \ + if(top >= MSGPACK_MAX_STACK_SIZE) { goto _failed; } \ + stack[top].ct = ct_; \ + stack[top].curr = 0; \ + stack[top].count = count_; \ + /*printf("container %d count %d stack %d\n",stack[top].obj,count_,top);*/ \ + /*printf("stack push %d\n", top);*/ \ + ++top; \ + goto _header_again + +#define NEXT_CS(p) \ + ((unsigned int)*p & 0x1f) + +#define PTR_CAST_8(ptr) (*(uint8_t*)ptr) +#define PTR_CAST_16(ptr) msgpack_betoh16(*(uint16_t*)ptr) +#define PTR_CAST_32(ptr) msgpack_betoh32(*(uint32_t*)ptr) +#define PTR_CAST_64(ptr) msgpack_betoh64(*(uint64_t*)ptr) + + if(p == pe) { goto _out; } + do { + switch(cs) { + case CS_HEADER: + switch(*p) { + case 0x00 ... 0x7f: // Positive Fixnum + push_fixed_value(_uint8, *(uint8_t*)p); + case 0xe0 ... 0xff: // Negative Fixnum + push_fixed_value(_int8, *(int8_t*)p); + case 0xc0 ... 0xdf: // Variable + switch(*p) { + case 0xc0: // nil + push_simple_value(_nil); + //case 0xc1: // string + // again_terminal_trail(NEXT_CS(p), p+1); + case 0xc2: // false + push_simple_value(_false); + case 0xc3: // true + push_simple_value(_true); + //case 0xc4: + //case 0xc5: + //case 0xc6: + //case 0xc7: + //case 0xc8: + //case 0xc9: + case 0xca: // float + case 0xcb: // double + case 0xcc: // unsigned int 8 + case 0xcd: // unsigned int 16 + case 0xce: // unsigned int 32 + case 0xcf: // unsigned int 64 + case 0xd0: // signed int 8 + case 0xd1: // signed int 16 + case 0xd2: // signed int 32 + case 0xd3: // signed int 64 + again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03)); + //case 0xd4: + //case 0xd5: + //case 0xd6: // big integer 16 + //case 0xd7: // big integer 32 + //case 0xd8: // big float 16 + //case 0xd9: // big float 32 + case 0xda: // raw 16 + case 0xdb: // raw 32 + case 0xdc: // array 16 + case 0xdd: // array 32 + case 0xde: // map 16 + case 0xdf: // map 32 + again_fixed_trail(NEXT_CS(p), 2 << (((unsigned int)*p) & 0x01)); + default: + goto _failed; + } + case 0xa0 ... 0xbf: // FixRaw + again_fixed_trail_if_zero(ACS_RAW_VALUE, ((unsigned int)*p & 0x1f), _raw_zero); + case 0x90 ... 0x9f: // FixArray + start_container(_array, ((unsigned int)*p) & 0x0f, CT_ARRAY_ITEM); + case 0x80 ... 0x8f: // FixMap + start_container(_map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY); + + default: + goto _failed; + } + // end CS_HEADER + + + _fixed_trail_again: + ++p; + + default: + if((size_t)(pe - p) < trail) { goto _out; } + n = p; p += trail - 1; + switch(cs) { + //case CS_ + //case CS_ + case CS_FLOAT: { + union { uint32_t num; char buf[4]; } f; + f.num = PTR_CAST_32(n); // FIXME + push_fixed_value(_float, *((float*)f.buf)); } + case CS_DOUBLE: { + union { uint64_t num; char buf[8]; } f; + f.num = PTR_CAST_64(n); // FIXME + push_fixed_value(_double, *((double*)f.buf)); } + case CS_UINT_8: + push_fixed_value(_uint8, (uint8_t)PTR_CAST_8(n)); + case CS_UINT_16: + push_fixed_value(_uint16, (uint16_t)PTR_CAST_16(n)); + case CS_UINT_32: + push_fixed_value(_uint32, (uint32_t)PTR_CAST_32(n)); + case CS_UINT_64: + push_fixed_value(_uint64, (uint64_t)PTR_CAST_64(n)); + + case CS_INT_8: + push_fixed_value(_int8, (int8_t)PTR_CAST_8(n)); + case CS_INT_16: + push_fixed_value(_int16, (int16_t)PTR_CAST_16(n)); + case CS_INT_32: + push_fixed_value(_int32, (int32_t)PTR_CAST_32(n)); + case CS_INT_64: + push_fixed_value(_int64, (int64_t)PTR_CAST_64(n)); + + //case CS_ + //case CS_ + //case CS_BIG_INT_16: + // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, (uint16_t)PTR_CAST_16(n), _big_int_zero); + //case CS_BIG_INT_32: + // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, (uint32_t)PTR_CAST_32(n), _big_int_zero); + //case ACS_BIG_INT_VALUE: + //_big_int_zero: + // // FIXME + // push_variable_value(_big_int, data, n, trail); + + //case CS_BIG_FLOAT_16: + // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint16_t)PTR_CAST_16(n), _big_float_zero); + //case CS_BIG_FLOAT_32: + // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint32_t)PTR_CAST_32(n), _big_float_zero); + //case ACS_BIG_FLOAT_VALUE: + //_big_float_zero: + // // FIXME + // push_variable_value(_big_float, data, n, trail); + + case CS_RAW_16: + again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint16_t)PTR_CAST_16(n), _raw_zero); + case CS_RAW_32: + again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint32_t)PTR_CAST_32(n), _raw_zero); + case ACS_RAW_VALUE: + _raw_zero: + push_variable_value(_raw, data, n, trail); + + case CS_ARRAY_16: + start_container(_array, (uint16_t)PTR_CAST_16(n), CT_ARRAY_ITEM); + case CS_ARRAY_32: + /* FIXME security guard */ + start_container(_array, (uint32_t)PTR_CAST_32(n), CT_ARRAY_ITEM); + + case CS_MAP_16: + start_container(_map, (uint16_t)PTR_CAST_16(n), CT_MAP_KEY); + case CS_MAP_32: + /* FIXME security guard */ + start_container(_map, (uint32_t)PTR_CAST_32(n), CT_MAP_KEY); + + default: + goto _failed; + } + } + +_push: + if(top == 0) { goto _finish; } + c = &stack[top-1]; + switch(c->ct) { + case CT_ARRAY_ITEM: + if(msgpack_unpack_callback(_array_item)(user, c->curr, &c->obj, obj) < 0) { goto _failed; } + if(++c->curr == c->count) { + obj = c->obj; + --top; + /*printf("stack pop %d\n", top);*/ + goto _push; + } + goto _header_again; + case CT_MAP_KEY: + c->map_key = obj; + c->ct = CT_MAP_VALUE; + goto _header_again; + case CT_MAP_VALUE: + if(msgpack_unpack_callback(_map_item)(user, &c->obj, c->map_key, obj) < 0) { goto _failed; } + if(--c->count == 0) { + obj = c->obj; + --top; + /*printf("stack pop %d\n", top);*/ + goto _push; + } + c->ct = CT_MAP_KEY; + goto _header_again; + + default: + goto _failed; + } + +_header_again: + cs = CS_HEADER; + ++p; + } while(p != pe); + goto _out; + + +_finish: + stack[0].obj = obj; + ++p; + ret = 1; + /*printf("-- finish --\n"); */ + goto _end; + +_failed: + /*printf("** FAILED **\n"); */ + ret = -1; + goto _end; + +_out: + ret = 0; + goto _end; + +_end: + ctx->cs = cs; + ctx->trail = trail; + ctx->top = top; + *off = p - (const unsigned char*)data; + + return ret; +} + + +#undef msgpack_unpack_func +#undef msgpack_unpack_callback +#undef msgpack_unpack_struct +#undef msgpack_unpack_object +#undef msgpack_unpack_user + +#undef push_simple_value +#undef push_fixed_value +#undef push_variable_value +#undef again_fixed_trail +#undef again_fixed_trail_if_zero +#undef start_container + +#undef NEXT_CS +#undef PTR_CAST_8 +#undef PTR_CAST_16 +#undef PTR_CAST_32 +#undef PTR_CAST_64 + -- cgit v1.2.1 From f61b282886bb9d7f7932fbcbe2affee06d150a5f Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 24 Jun 2009 01:54:47 +0900 Subject: Reduce memory footprint. --- python/msgpack/unpack.h | 6 ++---- python/msgpack/unpack_template.h | 7 +++++-- 2 files changed, 7 insertions(+), 6 deletions(-) (limited to 'python/msgpack') diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h index cc48d9c..12afb99 100644 --- a/python/msgpack/unpack.h +++ b/python/msgpack/unpack.h @@ -23,9 +23,7 @@ #define MSGPACK_MAX_STACK_SIZE (1024) #include "unpack_define.h" -using namespace std; - -typedef map str_cach_t; +typedef std::map str_cach_t; struct unpack_user { str_cach_t strcache; @@ -127,7 +125,7 @@ static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_obje static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) { if (l < 16) { - string s(p, l); + std::string s(p, l); str_cach_t ::iterator it = u->strcache.find(s); if (it != u->strcache.end()) { *o = it->second; diff --git a/python/msgpack/unpack_template.h b/python/msgpack/unpack_template.h index db33368..c960c3a 100644 --- a/python/msgpack/unpack_template.h +++ b/python/msgpack/unpack_template.h @@ -43,10 +43,13 @@ msgpack_unpack_struct_decl(_stack) { msgpack_unpack_object obj; - size_t curr; size_t count; unsigned int ct; - msgpack_unpack_object map_key; + + union { + size_t curr; + msgpack_unpack_object map_key; + }; }; msgpack_unpack_struct_decl(_context) { -- cgit v1.2.1 From 5d4189306ae0b6ea23e9410d34434b41aee384b3 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 24 Jun 2009 04:25:05 +0900 Subject: Check return value of c-api. --- python/msgpack/unpack.h | 118 +++++++++++++++++++++++++++++++++++++----------- 1 file changed, 92 insertions(+), 26 deletions(-) (limited to 'python/msgpack') diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h index 12afb99..e30b72a 100644 --- a/python/msgpack/unpack.h +++ b/python/msgpack/unpack.h @@ -59,42 +59,84 @@ static inline msgpack_unpack_object template_callback_root(unpack_user* u) return NULL; } +static inline int template_callback_uint16(unpack_user* u, uint16_t d, msgpack_unpack_object* o) +{ + PyObject *p = PyInt_FromLong((long)d); + if (!p) + return -1; + *o = p; + return 0; +} static inline int template_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpack_object* o) -{ *o = PyInt_FromLong((long)d); return 0; } +{ + return template_callback_uint16(u, d, o); +} -static inline int template_callback_uint16(unpack_user* u, uint16_t d, msgpack_unpack_object* o) -{ *o = PyInt_FromLong((long)d); return 0; } static inline int template_callback_uint32(unpack_user* u, uint32_t d, msgpack_unpack_object* o) { + PyObject *p; if (d > LONG_MAX) { - *o = PyLong_FromUnsignedLong((unsigned long)d); + p = PyLong_FromUnsignedLong((unsigned long)d); } else { - *o = PyInt_FromLong((long)d); + p = PyInt_FromLong((long)d); } + if (!p) + return -1; + *o = p; return 0; } static inline int template_callback_uint64(unpack_user* u, uint64_t d, msgpack_unpack_object* o) -{ *o = PyLong_FromUnsignedLongLong(d); return 0; } +{ + PyObject *p = PyLong_FromUnsignedLongLong(d); + if (!p) + return -1; + *o = p; + return 0; +} -static inline int template_callback_int8(unpack_user* u, int8_t d, msgpack_unpack_object* o) -{ *o = PyInt_FromLong(d); return 0; } +static inline int template_callback_int32(unpack_user* u, int32_t d, msgpack_unpack_object* o) +{ + PyObject *p = PyInt_FromLong(d); + if (!p) + return -1; + *o = p; + return 0; +} static inline int template_callback_int16(unpack_user* u, int16_t d, msgpack_unpack_object* o) -{ *o = PyInt_FromLong(d); return 0; } +{ + return template_callback_int32(u, d, o); +} -static inline int template_callback_int32(unpack_user* u, int32_t d, msgpack_unpack_object* o) -{ *o = PyInt_FromLong(d); return 0; } +static inline int template_callback_int8(unpack_user* u, int8_t d, msgpack_unpack_object* o) +{ + return template_callback_int32(u, d, o); +} static inline int template_callback_int64(unpack_user* u, int64_t d, msgpack_unpack_object* o) -{ *o = PyLong_FromLongLong(d); return 0; } - -static inline int template_callback_float(unpack_user* u, float d, msgpack_unpack_object* o) -{ *o = PyFloat_FromDouble((double)d); return 0; } +{ + PyObject *p = PyLong_FromLongLong(d); + if (!p) + return -1; + *o = p; + return 0; +} static inline int template_callback_double(unpack_user* u, double d, msgpack_unpack_object* o) -{ *o = PyFloat_FromDouble(d); return 0; } +{ + PyObject *p = PyFloat_FromDouble(d); + if (!p) + return -1; + *o = p; + return 0; +} + +static inline int template_callback_float(unpack_user* u, float d, msgpack_unpack_object* o) +{ + return template_callback_double(u, d, o); +} static inline int template_callback_nil(unpack_user* u, msgpack_unpack_object* o) { Py_INCREF(Py_None); *o = Py_None; return 0; } @@ -106,40 +148,64 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* { Py_INCREF(Py_False); *o = Py_False; return 0; } static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) -{ *o = PyList_New(n); return 0; } +{ + PyObject *p = PyList_New(n); + if (!p) + return -1; + *o = p; + return 0; +} static inline int template_callback_array_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object o) { PyList_SET_ITEM(*c, current, o); return 0; } static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o) -{ *o = PyDict_New(); return 0; } +{ + PyObject *p = PyDict_New(); + if (!p) + return -1; + *o = p; + return 0; +} static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v) { - PyDict_SetItem(*c, k, v); - Py_DECREF(k); - Py_DECREF(v); - return 0; + if (PyDict_SetItem(*c, k, v) == 0) { + Py_DECREF(k); + Py_DECREF(v); + return 0; + } + return -1; } static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) { + PyObject *py; if (l < 16) { std::string s(p, l); str_cach_t ::iterator it = u->strcache.find(s); if (it != u->strcache.end()) { *o = it->second; + Py_INCREF(*o); + return 0; } else { - *o = PyString_FromStringAndSize(p, l); + py = PyString_FromStringAndSize(p, l); + if (!py) + return -1; + *o = py; + Py_INCREF(*o); u->strcache[s] = *o; + return 0; } - Py_INCREF(*o); } else { - *o = PyString_FromStringAndSize(p, l); + py = PyString_FromStringAndSize(p, l); + if (!py) + return -1; + *o = py; + return 0; } - return 0; } #include "unpack_template.h" -- cgit v1.2.1 From 479263989b02693f0e449ee591ca58ed84fd904b Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 24 Jun 2009 14:58:02 +0900 Subject: Stop unnecessary caching. --- python/msgpack/unpack.h | 44 +++++--------------------------------------- 1 file changed, 5 insertions(+), 39 deletions(-) (limited to 'python/msgpack') diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h index e30b72a..dd23130 100644 --- a/python/msgpack/unpack.h +++ b/python/msgpack/unpack.h @@ -16,24 +16,10 @@ * limitations under the License. */ -#include -#include -#include - #define MSGPACK_MAX_STACK_SIZE (1024) #include "unpack_define.h" -typedef std::map str_cach_t; struct unpack_user { - str_cach_t strcache; - - ~unpack_user() { - str_cach_t::iterator it, itend; - itend = strcache.end(); - for (it = strcache.begin(); it != itend; ++it) { - Py_DECREF(it->second); - } - } }; @@ -181,31 +167,11 @@ static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_obje static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) { PyObject *py; - if (l < 16) { - std::string s(p, l); - str_cach_t ::iterator it = u->strcache.find(s); - if (it != u->strcache.end()) { - *o = it->second; - Py_INCREF(*o); - return 0; - } - else { - py = PyString_FromStringAndSize(p, l); - if (!py) - return -1; - *o = py; - Py_INCREF(*o); - u->strcache[s] = *o; - return 0; - } - } - else { - py = PyString_FromStringAndSize(p, l); - if (!py) - return -1; - *o = py; - return 0; - } + py = PyString_FromStringAndSize(p, l); + if (!py) + return -1; + *o = py; + return 0; } #include "unpack_template.h" -- cgit v1.2.1 From 3c3df3133c6f6320a03ebcaef546ab9c2c53154d Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Fri, 26 Jun 2009 14:10:20 +0900 Subject: Implement streaming deserializer. --- python/msgpack/_msgpack.pyx | 162 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 147 insertions(+), 15 deletions(-) (limited to 'python/msgpack') diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index cbdcfc5..c37f8ba 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -6,13 +6,16 @@ cdef extern from "Python.h": ctypedef char* const_char_ptr "const char*" ctypedef struct PyObject cdef object PyString_FromStringAndSize(const_char_ptr b, Py_ssize_t len) + char* PyString_AsString(object o) cdef extern from "stdlib.h": - void* malloc(int) + void* malloc(size_t) + void* realloc(void*, size_t) void free(void*) cdef extern from "string.h": - int memcpy(char*dst, char*src, unsigned int size) + void* memcpy(char* dst, char* src, size_t size) + void* memmove(char* dst, char* src, size_t size) cdef extern from "pack.h": ctypedef int (*msgpack_packer_write)(void* data, const_char_ptr buf, unsigned int len) @@ -34,8 +37,6 @@ cdef extern from "pack.h": void msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) -cdef int BUFF_SIZE=2*1024 - cdef class Packer: """Packer that pack data into strm. @@ -48,10 +49,7 @@ cdef class Packer: cdef msgpack_packer pk cdef object strm - def __init__(self, strm, int size=0): - if size <= 0: - size = BUFF_SIZE - + def __init__(self, strm, int size=4*1024): self.strm = strm self.buff = malloc(size) self.allocated = size @@ -147,6 +145,8 @@ cdef class Packer: if flush: self.flush() + close = flush + cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): if packer.length + l > packer.allocated: if packer.length > 0: @@ -163,20 +163,28 @@ cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): return 0 def pack(object o, object stream): + u"""pack o and write to stream).""" packer = Packer(stream) packer.pack(o) packer.flush() -def packs(object o): +def packb(object o): + u"""pack o and return packed bytes.""" buf = StringIO() packer = Packer(buf) packer.pack(o) packer.flush() return buf.getvalue() +packs = packb + cdef extern from "unpack.h": ctypedef struct template_context: - pass + PyObject* obj + size_t count + unsigned int ct + PyObject* key + int template_execute(template_context* ctx, const_char_ptr data, size_t len, size_t* off) void template_init(template_context* ctx) @@ -188,15 +196,139 @@ def unpacks(object packed_bytes): cdef const_char_ptr p = packed_bytes cdef template_context ctx cdef size_t off = 0 + cdef int ret template_init(&ctx) - template_execute(&ctx, p, len(packed_bytes), &off) - return template_data(&ctx) + ret = template_execute(&ctx, p, len(packed_bytes), &off) + if ret == 1: + return template_data(&ctx) + else: + return None def unpack(object stream): """unpack from stream.""" packed = stream.read() return unpacks(packed) -cdef class Unpacker: - """Do nothing. This function is for symmetric to Packer""" - unpack = staticmethod(unpacks) +cdef class UnpackIterator(object): + cdef object unpacker + + def __init__(self, unpacker): + self.unpacker = unpacker + + def __next__(self): + return self.unpacker.unpack() + + def __iter__(self): + return self + +cdef class Unpacker(object): + """Unpacker(file_like=None, read_size=4096) + + Streaming unpacker. + file_like must have read(n) method. + read_size is used like file_like.read(read_size) + + If file_like is None, you can feed() bytes. feed() is useful + for unpack from non-blocking stream. + + exsample 1: + unpacker = Unpacker(afile) + for o in unpacker: + do_something(o) + + example 2: + unpacker = Unpacker() + while 1: + buf = astream.read() + unpacker.feed(buf) + for o in unpacker: + do_something(o) + """ + + cdef template_context ctx + cdef char* buf + cdef size_t buf_size, buf_head, buf_tail + cdef object file_like + cdef int read_size + cdef object waiting_bytes + + def __init__(self, file_like=None, int read_size=4096): + self.file_like = file_like + self.read_size = read_size + self.waiting_bytes = [] + self.buf = malloc(read_size) + self.buf_size = read_size + self.buf_head = 0 + self.buf_tail = 0 + template_init(&self.ctx) + + def feed(self, next_bytes): + if not isinstance(next_bytes, str): + raise ValueError, "Argument must be bytes object" + self.waiting_bytes.append(next_bytes) + + cdef append_buffer(self): + cdef char* buf = self.buf + cdef Py_ssize_t tail = self.buf_tail + cdef Py_ssize_t l + + for b in self.waiting_bytes: + l = len(b) + memcpy(buf + tail, PyString_AsString(b), l) + tail += l + self.buf_tail = tail + del self.waiting_bytes[:] + + # prepare self.buf + cdef fill_buffer(self): + cdef Py_ssize_t add_size + + if self.file_like is not None: + self.waiting_bytes.append(self.file_like.read(self.read_size)) + + if not self.waiting_bytes: + return + + add_size = 0 + for b in self.waiting_bytes: + add_size += len(b) + + cdef char* buf = self.buf + cdef size_t head = self.buf_head + cdef size_t tail = self.buf_tail + cdef size_t size = self.buf_size + + if self.buf_tail + add_size <= self.buf_size: + # do nothing. + pass + if self.buf_tail - self.buf_head + add_size < self.buf_size: + # move to front. + memmove(buf, buf + head, tail - head) + tail -= head + head = 0 + else: + # expand buffer + size = tail + add_size + buf = realloc(buf, size) + + self.buf = buf + self.buf_head = head + self.buf_tail = tail + self.buf_size = size + + self.append_buffer() + + cpdef unpack(self): + """unpack one object""" + cdef int ret + self.fill_buffer() + ret = template_execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) + if ret == 1: + return template_data(&self.ctx) + elif ret == 0: + raise StopIteration, "No more unpack data." + else: + raise ValueError, "Unpack failed." + + def __iter__(self): + return UnpackIterator(self) -- cgit v1.2.1 From 1b07b61c048f749c7a5e505617b0e59974f81077 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Sun, 28 Jun 2009 21:24:02 +0900 Subject: Ues more suitable type when packing. --- python/msgpack/_msgpack.pyx | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) (limited to 'python/msgpack') diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index c37f8ba..cde5313 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -7,6 +7,8 @@ cdef extern from "Python.h": ctypedef struct PyObject cdef object PyString_FromStringAndSize(const_char_ptr b, Py_ssize_t len) char* PyString_AsString(object o) + int PyMapping_Check(object o) + int PySequence_Check(object o) cdef extern from "stdlib.h": void* malloc(size_t) @@ -37,7 +39,7 @@ cdef extern from "pack.h": void msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) -cdef class Packer: +cdef class Packer(object): """Packer that pack data into strm. strm must have `write(bytes)` method. @@ -99,7 +101,8 @@ cdef class Packer: msgpack_pack_map(&self.pk, len) cdef __pack(self, object o): - cdef long long intval + cdef long long llval + cdef long longval cdef double fval cdef char* rawval @@ -110,11 +113,11 @@ cdef class Packer: elif o is False: msgpack_pack_false(&self.pk) elif isinstance(o, long): - intval = o - msgpack_pack_long_long(&self.pk, intval) + llval = o + msgpack_pack_long_long(&self.pk, llval) elif isinstance(o, int): - intval = o - msgpack_pack_long_long(&self.pk, intval) + longval = o + msgpack_pack_long_long(&self.pk, longval) elif isinstance(o, float): fval = o msgpack_pack_double(&self.pk, fval) @@ -127,12 +130,12 @@ cdef class Packer: rawval = o msgpack_pack_raw(&self.pk, len(o)) msgpack_pack_raw_body(&self.pk, rawval, len(o)) - elif isinstance(o, dict): + elif PyMapping_Check(o): msgpack_pack_map(&self.pk, len(o)) for k,v in o.iteritems(): self.pack(k) self.pack(v) - elif isinstance(o, tuple) or isinstance(o, list): + elif PySequence_Check(o): msgpack_pack_array(&self.pk, len(o)) for v in o: self.pack(v) -- cgit v1.2.1 From 3e396ef146940ada28fc8b154189ae9bb206babc Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Sun, 28 Jun 2009 21:24:16 +0900 Subject: Don't use C++. --- python/msgpack/unpack.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'python/msgpack') diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h index dd23130..40058d0 100644 --- a/python/msgpack/unpack.h +++ b/python/msgpack/unpack.h @@ -19,8 +19,8 @@ #define MSGPACK_MAX_STACK_SIZE (1024) #include "unpack_define.h" -struct unpack_user { -}; +typedef struct unpack_user { +} unpack_user; #define msgpack_unpack_struct(name) \ -- cgit v1.2.1 From 257270c1ebc5bb6ac6c60e47130e2a577ffb6714 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 29 Jun 2009 08:23:49 +0900 Subject: Refactor packing code. --- python/msgpack/_msgpack.pyx | 14 +++++++------- python/msgpack/pack.h | 17 ----------------- 2 files changed, 7 insertions(+), 24 deletions(-) (limited to 'python/msgpack') diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index cde5313..279fdf5 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -31,6 +31,7 @@ cdef extern from "pack.h": void msgpack_pack_nil(msgpack_packer* pk) void msgpack_pack_true(msgpack_packer* pk) void msgpack_pack_false(msgpack_packer* pk) + void msgpack_pack_long(msgpack_packer* pk, long d) void msgpack_pack_long_long(msgpack_packer* pk, long long d) void msgpack_pack_double(msgpack_packer* pk, double d) void msgpack_pack_array(msgpack_packer* pk, size_t l) @@ -60,7 +61,7 @@ cdef class Packer(object): msgpack_packer_init(&self.pk, self, _packer_write) def __del__(self): - free(self.buff); + free(self.buff) def flush(self): """Flash local buffer and output stream if it has 'flush()' method.""" @@ -117,7 +118,7 @@ cdef class Packer(object): msgpack_pack_long_long(&self.pk, llval) elif isinstance(o, int): longval = o - msgpack_pack_long_long(&self.pk, longval) + msgpack_pack_long(&self.pk, longval) elif isinstance(o, float): fval = o msgpack_pack_double(&self.pk, fval) @@ -133,12 +134,12 @@ cdef class Packer(object): elif PyMapping_Check(o): msgpack_pack_map(&self.pk, len(o)) for k,v in o.iteritems(): - self.pack(k) - self.pack(v) + self.__pack(k) + self.__pack(v) elif PySequence_Check(o): msgpack_pack_array(&self.pk, len(o)) for v in o: - self.pack(v) + self.__pack(v) else: # TODO: Serialize with defalt() like simplejson. raise TypeError, "can't serialize %r" % (o,) @@ -154,7 +155,7 @@ cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): if packer.length + l > packer.allocated: if packer.length > 0: packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) - if l > 64: + if l > packer.allocated/4: packer.strm.write(PyString_FromStringAndSize(b, l)) packer.length = 0 else: @@ -176,7 +177,6 @@ def packb(object o): buf = StringIO() packer = Packer(buf) packer.pack(o) - packer.flush() return buf.getvalue() packs = packb diff --git a/python/msgpack/pack.h b/python/msgpack/pack.h index 9bd6b68..cdac819 100644 --- a/python/msgpack/pack.h +++ b/python/msgpack/pack.h @@ -34,9 +34,6 @@ typedef struct msgpack_packer { static inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback); -static inline msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback); -static inline void msgpack_packer_free(msgpack_packer* pk); - static inline int msgpack_pack_short(msgpack_packer* pk, short d); static inline int msgpack_pack_int(msgpack_packer* pk, int d); static inline int msgpack_pack_long(msgpack_packer* pk, long d); @@ -90,20 +87,6 @@ static inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_p pk->callback = callback; } -static inline msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback) -{ - msgpack_packer* pk = (msgpack_packer*)calloc(1, sizeof(msgpack_packer)); - if(!pk) { return NULL; } - msgpack_packer_init(pk, data, callback); - return pk; -} - -static inline void msgpack_packer_free(msgpack_packer* pk) -{ - free(pk); -} - - #ifdef __cplusplus } #endif -- cgit v1.2.1 From 9015bd4ecfe7c30aef47d7c3c6fbc6fd4219eae0 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 29 Jun 2009 10:09:04 +0900 Subject: Fix error on packing unsigned long long. --- python/msgpack/_msgpack.pyx | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) (limited to 'python/msgpack') diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index 279fdf5..e1c497b 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -9,6 +9,8 @@ cdef extern from "Python.h": char* PyString_AsString(object o) int PyMapping_Check(object o) int PySequence_Check(object o) + long long PyLong_AsLongLong(object o) + unsigned long long PyLong_AsUnsignedLongLong(object o) cdef extern from "stdlib.h": void* malloc(size_t) @@ -27,17 +29,18 @@ cdef extern from "pack.h": msgpack_packer_write callback void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback) - void msgpack_pack_int(msgpack_packer* pk, int d) - void msgpack_pack_nil(msgpack_packer* pk) - void msgpack_pack_true(msgpack_packer* pk) - void msgpack_pack_false(msgpack_packer* pk) - void msgpack_pack_long(msgpack_packer* pk, long d) - void msgpack_pack_long_long(msgpack_packer* pk, long long d) - void msgpack_pack_double(msgpack_packer* pk, double d) - void msgpack_pack_array(msgpack_packer* pk, size_t l) - void msgpack_pack_map(msgpack_packer* pk, size_t l) - void msgpack_pack_raw(msgpack_packer* pk, size_t l) - void msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) + int msgpack_pack_int(msgpack_packer* pk, int d) + int msgpack_pack_nil(msgpack_packer* pk) + int msgpack_pack_true(msgpack_packer* pk) + int msgpack_pack_false(msgpack_packer* pk) + int msgpack_pack_long(msgpack_packer* pk, long d) + int msgpack_pack_long_long(msgpack_packer* pk, long long d) + int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d) + int msgpack_pack_double(msgpack_packer* pk, double d) + int msgpack_pack_array(msgpack_packer* pk, size_t l) + int msgpack_pack_map(msgpack_packer* pk, size_t l) + int msgpack_pack_raw(msgpack_packer* pk, size_t l) + int msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) cdef class Packer(object): @@ -103,6 +106,7 @@ cdef class Packer(object): cdef __pack(self, object o): cdef long long llval + cdef unsigned long long ullval cdef long longval cdef double fval cdef char* rawval @@ -114,8 +118,12 @@ cdef class Packer(object): elif o is False: msgpack_pack_false(&self.pk) elif isinstance(o, long): - llval = o - msgpack_pack_long_long(&self.pk, llval) + if o > 0: + ullval = PyLong_AsUnsignedLongLong(o) + msgpack_pack_unsigned_long_long(&self.pk, ullval) + else: + llval = PyLong_AsLongLong(o) + msgpack_pack_long_long(&self.pk, llval) elif isinstance(o, int): longval = o msgpack_pack_long(&self.pk, longval) -- cgit v1.2.1 From d4317fdc853c7dbe3bc57718dc009cb8b0a2b351 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Tue, 30 Jun 2009 23:03:33 +0900 Subject: Some optimization on packing. --- python/msgpack/_msgpack.pyx | 69 +++++++++++++++------------------------------ python/msgpack/pack.h | 29 ++++++++++--------- 2 files changed, 39 insertions(+), 59 deletions(-) (limited to 'python/msgpack') diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index e1c497b..26222aa 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -1,16 +1,23 @@ # coding: utf-8 -from cStringIO import StringIO +import cStringIO cdef extern from "Python.h": ctypedef char* const_char_ptr "const char*" ctypedef struct PyObject cdef object PyString_FromStringAndSize(const_char_ptr b, Py_ssize_t len) + PyObject* Py_True + PyObject* Py_False char* PyString_AsString(object o) int PyMapping_Check(object o) int PySequence_Check(object o) long long PyLong_AsLongLong(object o) unsigned long long PyLong_AsUnsignedLongLong(object o) + int PyLong_Check(object o) + int PyInt_Check(object o) + int PyFloat_Check(object o) + int PyString_Check(object o) + int PyUnicode_Check(object o) cdef extern from "stdlib.h": void* malloc(size_t) @@ -22,13 +29,9 @@ cdef extern from "string.h": void* memmove(char* dst, char* src, size_t size) cdef extern from "pack.h": - ctypedef int (*msgpack_packer_write)(void* data, const_char_ptr buf, unsigned int len) - struct msgpack_packer: - void *data - msgpack_packer_write callback + PyObject* writer - void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback) int msgpack_pack_int(msgpack_packer* pk, int d) int msgpack_pack_nil(msgpack_packer* pk) int msgpack_pack_true(msgpack_packer* pk) @@ -49,28 +52,17 @@ cdef class Packer(object): strm must have `write(bytes)` method. size specifies local buffer size. """ - cdef char* buff - cdef unsigned int length - cdef unsigned int allocated cdef msgpack_packer pk cdef object strm + cdef object writer - def __init__(self, strm, int size=4*1024): - self.strm = strm - self.buff = malloc(size) - self.allocated = size - self.length = 0 - - msgpack_packer_init(&self.pk, self, _packer_write) - - def __del__(self): - free(self.buff) + def __init__(self, strm_, int size=4*1024): + self.strm = strm_ + self.writer = strm_.write + self.pk.writer = self.writer def flush(self): """Flash local buffer and output stream if it has 'flush()' method.""" - if self.length > 0: - self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) - self.length = 0 if hasattr(self.strm, 'flush'): self.strm.flush() @@ -113,28 +105,28 @@ cdef class Packer(object): if o is None: msgpack_pack_nil(&self.pk) - elif o is True: + elif o == Py_True: msgpack_pack_true(&self.pk) - elif o is False: + elif o == Py_False: msgpack_pack_false(&self.pk) - elif isinstance(o, long): + elif PyLong_Check(o): if o > 0: ullval = PyLong_AsUnsignedLongLong(o) msgpack_pack_unsigned_long_long(&self.pk, ullval) else: llval = PyLong_AsLongLong(o) msgpack_pack_long_long(&self.pk, llval) - elif isinstance(o, int): + elif PyInt_Check(o): longval = o msgpack_pack_long(&self.pk, longval) - elif isinstance(o, float): + elif PyFloat_Check(o): fval = o msgpack_pack_double(&self.pk, fval) - elif isinstance(o, str): + elif PyString_Check(o): rawval = o msgpack_pack_raw(&self.pk, len(o)) msgpack_pack_raw_body(&self.pk, rawval, len(o)) - elif isinstance(o, unicode): + elif PyUnicode_Check(o): o = o.encode('utf-8') rawval = o msgpack_pack_raw(&self.pk, len(o)) @@ -152,28 +144,13 @@ cdef class Packer(object): # TODO: Serialize with defalt() like simplejson. raise TypeError, "can't serialize %r" % (o,) - def pack(self, obj, flush=True): + def pack(self, object obj, flush=True): self.__pack(obj) if flush: self.flush() close = flush -cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): - if packer.length + l > packer.allocated: - if packer.length > 0: - packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) - if l > packer.allocated/4: - packer.strm.write(PyString_FromStringAndSize(b, l)) - packer.length = 0 - else: - memcpy(packer.buff, b, l) - packer.length = l - else: - memcpy(packer.buff + packer.length, b, l) - packer.length += l - return 0 - def pack(object o, object stream): u"""pack o and write to stream).""" packer = Packer(stream) @@ -182,7 +159,7 @@ def pack(object o, object stream): def packb(object o): u"""pack o and return packed bytes.""" - buf = StringIO() + buf = cStringIO.StringIO() packer = Packer(buf) packer.pack(o) return buf.getvalue() diff --git a/python/msgpack/pack.h b/python/msgpack/pack.h index cdac819..ea97601 100644 --- a/python/msgpack/pack.h +++ b/python/msgpack/pack.h @@ -24,15 +24,11 @@ extern "C" { #endif - -typedef int (*msgpack_packer_write)(void* data, const char* buf, unsigned int len); - typedef struct msgpack_packer { - void* data; - msgpack_packer_write callback; + PyObject* writer; } msgpack_packer; -static inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback); +typedef struct Packer Packer; static inline int msgpack_pack_short(msgpack_packer* pk, short d); static inline int msgpack_pack_int(msgpack_packer* pk, int d); @@ -66,7 +62,20 @@ static inline int msgpack_pack_map(msgpack_packer* pk, unsigned int n); static inline int msgpack_pack_raw(msgpack_packer* pk, size_t l); static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l); +static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_t l) +{ + PyObject *buf, *ret; + + buf = PyBuffer_FromMemory((void*)data, l); + //buf = PyString_FromStringAndSize(data, l); + if (buf == NULL) return -1; + ret = PyObject_CallFunctionObjArgs(pk->writer, buf, NULL); + Py_DECREF(buf); + if (ret == NULL) return -1; + Py_DECREF(ret); + return 0; +} #define msgpack_pack_inline_func(name) \ static inline int msgpack_pack ## name @@ -77,16 +86,10 @@ static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_ #define msgpack_pack_user msgpack_packer* #define msgpack_pack_append_buffer(user, buf, len) \ - return (*(user)->callback)((user)->data, (const char*)buf, len) + return msgpack_pack_write(user, (const char*)buf, len) #include "pack_template.h" -static inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback) -{ - pk->data = data; - pk->callback = callback; -} - #ifdef __cplusplus } #endif -- cgit v1.2.1 From 03942a1b9020b00bf47e93b7d5ec606e8160c054 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 1 Jul 2009 00:57:46 +0900 Subject: Major speedup on packing. --- python/msgpack/_msgpack.pyx | 60 ++++++++++++++++----------------------------- python/msgpack/pack.h | 29 +++++++++++++--------- 2 files changed, 39 insertions(+), 50 deletions(-) (limited to 'python/msgpack') diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index 26222aa..9d766c5 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -18,6 +18,7 @@ cdef extern from "Python.h": int PyFloat_Check(object o) int PyString_Check(object o) int PyUnicode_Check(object o) + object PyBuffer_FromMemory(const_char_ptr b, Py_ssize_t len) cdef extern from "stdlib.h": void* malloc(size_t) @@ -30,7 +31,9 @@ cdef extern from "string.h": cdef extern from "pack.h": struct msgpack_packer: - PyObject* writer + char* buf + size_t length + size_t buf_size int msgpack_pack_int(msgpack_packer* pk, int d) int msgpack_pack_nil(msgpack_packer* pk) @@ -56,45 +59,16 @@ cdef class Packer(object): cdef object strm cdef object writer - def __init__(self, strm_, int size=4*1024): - self.strm = strm_ - self.writer = strm_.write - self.pk.writer = self.writer + def __init__(self, strm): + self.strm = strm - def flush(self): - """Flash local buffer and output stream if it has 'flush()' method.""" - if hasattr(self.strm, 'flush'): - self.strm.flush() - - def pack_list(self, len): - """Start packing sequential objects. - - Example: - - packer.pack_list(2) - packer.pack('foo') - packer.pack('bar') - - This is same to: - - packer.pack(['foo', 'bar']) - """ - msgpack_pack_array(&self.pk, len) + cdef int buf_size = 1024*1024 + self.pk.buf = malloc(buf_size); + self.pk.buf_size = buf_size + self.pk.length = 0 - def pack_dict(self, len): - """Start packing key-value objects. - - Example: - - packer.pack_dict(1) - packer.pack('foo') - packer.pack('bar') - - This is same to: - - packer.pack({'foo': 'bar'}) - """ - msgpack_pack_map(&self.pk, len) + def __del__(self): + free(self.pk.buf); cdef __pack(self, object o): cdef long long llval @@ -144,11 +118,19 @@ cdef class Packer(object): # TODO: Serialize with defalt() like simplejson. raise TypeError, "can't serialize %r" % (o,) - def pack(self, object obj, flush=True): + def pack(self, object obj, object flush=True): self.__pack(obj) + buf = PyBuffer_FromMemory(self.pk.buf, self.pk.length) + self.pk.length = 0 + self.strm.write(buf) if flush: self.flush() + def flush(self): + """Flash local buffer and output stream if it has 'flush()' method.""" + if hasattr(self.strm, 'flush'): + self.strm.flush() + close = flush def pack(object o, object stream): diff --git a/python/msgpack/pack.h b/python/msgpack/pack.h index ea97601..d7e0867 100644 --- a/python/msgpack/pack.h +++ b/python/msgpack/pack.h @@ -25,7 +25,9 @@ extern "C" { #endif typedef struct msgpack_packer { - PyObject* writer; + char *buf; + size_t length; + size_t buf_size; } msgpack_packer; typedef struct Packer Packer; @@ -64,16 +66,21 @@ static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_ static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_t l) { - PyObject *buf, *ret; - - buf = PyBuffer_FromMemory((void*)data, l); - //buf = PyString_FromStringAndSize(data, l); - if (buf == NULL) return -1; - - ret = PyObject_CallFunctionObjArgs(pk->writer, buf, NULL); - Py_DECREF(buf); - if (ret == NULL) return -1; - Py_DECREF(ret); + char* buf = pk->buf; + size_t bs = pk->buf_size; + size_t len = pk->length; + + if (len + l > bs) { + bs = (len + l) * 2; + buf = realloc(pk->buf, bs); + if (!buf) return -1; + } + memcpy(buf + len, data, l); + len += l; + + pk->buf = buf; + pk->buf_size = bs; + pk->length = len; return 0; } -- cgit v1.2.1 From 78db826a75d9f091f2175ace5f2cd9cb81c0f115 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 1 Jul 2009 20:55:24 +0900 Subject: Fix memory leak. Remove stream packing feature. Add errorcheck in packing. --- python/msgpack/_msgpack.pyx | 135 +++++++++++++++++++++++--------------------- python/msgpack/pack.h | 2 +- 2 files changed, 73 insertions(+), 64 deletions(-) (limited to 'python/msgpack') diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index 9d766c5..c455394 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -5,20 +5,22 @@ import cStringIO cdef extern from "Python.h": ctypedef char* const_char_ptr "const char*" ctypedef struct PyObject + cdef object PyString_FromStringAndSize(const_char_ptr b, Py_ssize_t len) - PyObject* Py_True - PyObject* Py_False - char* PyString_AsString(object o) - int PyMapping_Check(object o) - int PySequence_Check(object o) - long long PyLong_AsLongLong(object o) - unsigned long long PyLong_AsUnsignedLongLong(object o) - int PyLong_Check(object o) - int PyInt_Check(object o) - int PyFloat_Check(object o) - int PyString_Check(object o) - int PyUnicode_Check(object o) - object PyBuffer_FromMemory(const_char_ptr b, Py_ssize_t len) + cdef PyObject* Py_True + cdef PyObject* Py_False + + cdef char* PyString_AsString(object o) + cdef long long PyLong_AsLongLong(object o) + cdef unsigned long long PyLong_AsUnsignedLongLong(object o) + + cdef int PyMapping_Check(object o) + cdef int PySequence_Check(object o) + cdef int PyLong_Check(object o) + cdef int PyInt_Check(object o) + cdef int PyFloat_Check(object o) + cdef int PyString_Check(object o) + cdef int PyUnicode_Check(object o) cdef extern from "stdlib.h": void* malloc(size_t) @@ -50,101 +52,101 @@ cdef extern from "pack.h": cdef class Packer(object): - """Packer that pack data into strm. + """MessagePack Packer + + usage: - strm must have `write(bytes)` method. - size specifies local buffer size. + packer = Packer() + astream.write(packer.pack(a)) + astream.write(packer.pack(b)) """ cdef msgpack_packer pk - cdef object strm - cdef object writer - - def __init__(self, strm): - self.strm = strm + def __cinit__(self): cdef int buf_size = 1024*1024 self.pk.buf = malloc(buf_size); self.pk.buf_size = buf_size self.pk.length = 0 - def __del__(self): + def __dealloc__(self): free(self.pk.buf); - cdef __pack(self, object o): + cdef int __pack(self, object o): cdef long long llval cdef unsigned long long ullval cdef long longval cdef double fval cdef char* rawval + cdef int ret if o is None: - msgpack_pack_nil(&self.pk) + ret = msgpack_pack_nil(&self.pk) elif o == Py_True: - msgpack_pack_true(&self.pk) + ret = msgpack_pack_true(&self.pk) elif o == Py_False: - msgpack_pack_false(&self.pk) + ret = msgpack_pack_false(&self.pk) elif PyLong_Check(o): if o > 0: ullval = PyLong_AsUnsignedLongLong(o) - msgpack_pack_unsigned_long_long(&self.pk, ullval) + ret = msgpack_pack_unsigned_long_long(&self.pk, ullval) else: llval = PyLong_AsLongLong(o) - msgpack_pack_long_long(&self.pk, llval) + ret = msgpack_pack_long_long(&self.pk, llval) elif PyInt_Check(o): longval = o - msgpack_pack_long(&self.pk, longval) + ret = msgpack_pack_long(&self.pk, longval) elif PyFloat_Check(o): fval = o - msgpack_pack_double(&self.pk, fval) + ret = msgpack_pack_double(&self.pk, fval) elif PyString_Check(o): rawval = o - msgpack_pack_raw(&self.pk, len(o)) - msgpack_pack_raw_body(&self.pk, rawval, len(o)) + ret = msgpack_pack_raw(&self.pk, len(o)) + if ret == 0: + ret = msgpack_pack_raw_body(&self.pk, rawval, len(o)) elif PyUnicode_Check(o): o = o.encode('utf-8') rawval = o - msgpack_pack_raw(&self.pk, len(o)) - msgpack_pack_raw_body(&self.pk, rawval, len(o)) + ret = msgpack_pack_raw(&self.pk, len(o)) + if ret == 0: + ret = msgpack_pack_raw_body(&self.pk, rawval, len(o)) elif PyMapping_Check(o): - msgpack_pack_map(&self.pk, len(o)) - for k,v in o.iteritems(): - self.__pack(k) - self.__pack(v) + ret = msgpack_pack_map(&self.pk, len(o)) + if ret == 0: + for k,v in o.iteritems(): + ret = self.__pack(k) + if ret != 0: break + ret = self.__pack(v) + if ret != 0: break elif PySequence_Check(o): - msgpack_pack_array(&self.pk, len(o)) - for v in o: - self.__pack(v) + ret = msgpack_pack_array(&self.pk, len(o)) + if ret == 0: + for v in o: + ret = self.__pack(v) + if ret != 0: break else: # TODO: Serialize with defalt() like simplejson. raise TypeError, "can't serialize %r" % (o,) + return ret - def pack(self, object obj, object flush=True): - self.__pack(obj) - buf = PyBuffer_FromMemory(self.pk.buf, self.pk.length) + def pack(self, object obj): + cdef int ret + ret = self.__pack(obj) + if ret: + raise TypeError + buf = PyString_FromStringAndSize(self.pk.buf, self.pk.length) self.pk.length = 0 - self.strm.write(buf) - if flush: - self.flush() - - def flush(self): - """Flash local buffer and output stream if it has 'flush()' method.""" - if hasattr(self.strm, 'flush'): - self.strm.flush() + return buf - close = flush def pack(object o, object stream): - u"""pack o and write to stream).""" - packer = Packer(stream) - packer.pack(o) - packer.flush() + """pack a object `o` and write it to stream).""" + packer = Packer() + stream.write(packer.pack(o)) def packb(object o): - u"""pack o and return packed bytes.""" - buf = cStringIO.StringIO() - packer = Packer(buf) - packer.pack(o) - return buf.getvalue() + """pack o and return packed bytes.""" + packer = Packer() + return packer.pack(o) packs = packb @@ -222,7 +224,14 @@ cdef class Unpacker(object): cdef int read_size cdef object waiting_bytes - def __init__(self, file_like=None, int read_size=4096): + def __cinit__(self): + self.buf = NULL + + def __dealloc__(self): + if self.buf: + free(self.buf); + + def __init__(self, file_like=None, int read_size=1024*1024): self.file_like = file_like self.read_size = read_size self.waiting_bytes = [] diff --git a/python/msgpack/pack.h b/python/msgpack/pack.h index d7e0867..58f021e 100644 --- a/python/msgpack/pack.h +++ b/python/msgpack/pack.h @@ -72,7 +72,7 @@ static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_ if (len + l > bs) { bs = (len + l) * 2; - buf = realloc(pk->buf, bs); + buf = realloc(buf, bs); if (!buf) return -1; } memcpy(buf + len, data, l); -- cgit v1.2.1 From 979efbb9501e14d43e2def8a80992494690a9d39 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 9 Jul 2009 13:41:04 +0900 Subject: Support MinGW. --- python/msgpack/unpack_define.h | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'python/msgpack') diff --git a/python/msgpack/unpack_define.h b/python/msgpack/unpack_define.h index 63668c2..d997569 100644 --- a/python/msgpack/unpack_define.h +++ b/python/msgpack/unpack_define.h @@ -45,8 +45,21 @@ extern "C" { #endif #endif +#ifdef __WIN32__ +static inline uint16_t msgpack_betoh16(uint16_t x) { + return ((x << 8) & 0xff00U) | + ((x >> 8) & 0x00ffU); +} +static inline uint32_t msgpack_betoh32(uint32_t x) { + return ((x << 24) & 0xff000000UL ) | + ((x << 8) & 0x00ff0000UL ) | + ((x >> 8) & 0x0000ff00UL ) | + ((x >> 24) & 0x000000ffUL ); +} +#else #define msgpack_betoh16(x) ntohs(x) #define msgpack_betoh32(x) ntohl(x) +#endif #ifdef __LITTLE_ENDIAN__ #if defined(__bswap_64) -- cgit v1.2.1 From 900785e1aa2e031a4496f8c2a30bb95d0c950b9b Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Sun, 12 Jul 2009 09:29:11 +0900 Subject: Clean up --- python/msgpack/_msgpack.pyx | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'python/msgpack') diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index c455394..b407b92 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -163,7 +163,7 @@ cdef extern from "unpack.h": object template_data(template_context* ctx) -def unpacks(object packed_bytes): +def unpackb(object packed_bytes): """Unpack packed_bytes to object. Returns unpacked object.""" cdef const_char_ptr p = packed_bytes cdef template_context ctx @@ -176,10 +176,12 @@ def unpacks(object packed_bytes): else: return None +unpacks = unpackb + def unpack(object stream): """unpack from stream.""" packed = stream.read() - return unpacks(packed) + return unpackb(packed) cdef class UnpackIterator(object): cdef object unpacker -- cgit v1.2.1 From e5c49dae13b26f2155dffffbc4e9915badbcecfb Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Sun, 12 Jul 2009 20:02:21 +0900 Subject: Release 0.1.0 --- python/msgpack/_msgpack.pyx | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'python/msgpack') diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index b407b92..7990a18 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -139,7 +139,7 @@ cdef class Packer(object): def pack(object o, object stream): - """pack a object `o` and write it to stream).""" + """pack an object `o` and write it to stream).""" packer = Packer() stream.write(packer.pack(o)) @@ -164,7 +164,7 @@ cdef extern from "unpack.h": def unpackb(object packed_bytes): - """Unpack packed_bytes to object. Returns unpacked object.""" + """Unpack packed_bytes to object. Returns an unpacked object.""" cdef const_char_ptr p = packed_bytes cdef template_context ctx cdef size_t off = 0 @@ -179,7 +179,7 @@ def unpackb(object packed_bytes): unpacks = unpackb def unpack(object stream): - """unpack from stream.""" + """unpack an object from stream.""" packed = stream.read() return unpackb(packed) @@ -196,14 +196,14 @@ cdef class UnpackIterator(object): return self cdef class Unpacker(object): - """Unpacker(file_like=None, read_size=4096) + """Unpacker(file_like=None, read_size=1024*1024) Streaming unpacker. file_like must have read(n) method. read_size is used like file_like.read(read_size) - If file_like is None, you can feed() bytes. feed() is useful - for unpack from non-blocking stream. + If file_like is None, you can ``feed()`` bytes. ``feed()`` is + useful for unpacking from non-blocking stream. exsample 1: unpacker = Unpacker(afile) -- cgit v1.2.1