From 67391fd60e2285067a6cbd7392ef0a603875cc59 Mon Sep 17 00:00:00 2001 From: Bas Westerbaan Date: Sat, 14 Jun 2014 18:25:34 +0200 Subject: fallback: add missing update of _fb_buf_n --- msgpack/fallback.py | 1 + 1 file changed, 1 insertion(+) (limited to 'msgpack') diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 49323e6..d326885 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -252,6 +252,7 @@ class Unpacker(object): if not tmp: break buffs.append(tmp) + self._fb_buf_n += len(tmp) continue sliced = n - len(ret) ret += buffs[self._fb_buf_i][self._fb_buf_o:self._fb_buf_o + sliced] -- cgit v1.2.1 From 56cf3841593733dafe0f84a0e8cbc16c3dc7250d Mon Sep 17 00:00:00 2001 From: Bas Westerbaan Date: Sat, 14 Jun 2014 18:25:57 +0200 Subject: fallback: set default read_size to 4096 --- msgpack/fallback.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'msgpack') diff --git a/msgpack/fallback.py b/msgpack/fallback.py index d326885..294a97e 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -169,7 +169,7 @@ class Unpacker(object): self._max_buffer_size = max_buffer_size or 2**31-1 if read_size > self._max_buffer_size: raise ValueError("read_size must be smaller than max_buffer_size") - self._read_size = read_size or min(self._max_buffer_size, 2048) + self._read_size = read_size or min(self._max_buffer_size, 4096) self._encoding = encoding self._unicode_errors = unicode_errors self._use_list = use_list -- cgit v1.2.1 From ba8cf1c402fe68f283480c9abbf7707d004a8583 Mon Sep 17 00:00:00 2001 From: Bas Westerbaan Date: Sat, 14 Jun 2014 18:26:30 +0200 Subject: fallback: _fb_consume: improve performance with pypy --- msgpack/fallback.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'msgpack') diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 294a97e..0e0557f 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -202,12 +202,17 @@ class Unpacker(object): self._fb_buffers.append(next_bytes) def _fb_consume(self): - self._fb_buffers = self._fb_buffers[self._fb_buf_i:] + if self._fb_buf_i: + for i in xrange(self._fb_buf_i): + self._fb_buf_n -= len(self._fb_buffers[i]) + self._fb_buffers = self._fb_buffers[self._fb_buf_i:] + self._fb_buf_i = 0 if self._fb_buffers: self._fb_buffers[0] = self._fb_buffers[0][self._fb_buf_o:] + self._fb_buf_n -= self._fb_buf_o + else: + self._fb_buf_n = 0 self._fb_buf_o = 0 - self._fb_buf_i = 0 - self._fb_buf_n = sum(map(len, self._fb_buffers)) def _fb_got_extradata(self): if self._fb_buf_i != len(self._fb_buffers): -- cgit v1.2.1 From 7eb371f8278941fb2323e0c2333ed89c88ab822b Mon Sep 17 00:00:00 2001 From: Bas Westerbaan Date: Sat, 14 Jun 2014 18:30:38 +0200 Subject: fallback: do not reset the buffer completely in between of iterations --- msgpack/fallback.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) (limited to 'msgpack') diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 0e0557f..d838e55 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -201,7 +201,16 @@ class Unpacker(object): self._fb_buf_n += len(next_bytes) self._fb_buffers.append(next_bytes) + def _fb_sloppy_consume(self): + """ Gets rid of some of the used parts of the buffer. """ + if self._fb_buf_i: + for i in xrange(self._fb_buf_i): + self._fb_buf_n -= len(self._fb_buffers[i]) + self._fb_buffers = self._fb_buffers[self._fb_buf_i:] + self._fb_buf_i = 0 + def _fb_consume(self): + """ Gets rid of the used parts of the buffer. """ if self._fb_buf_i: for i in xrange(self._fb_buf_i): self._fb_buf_n -= len(self._fb_buffers[i]) @@ -446,9 +455,10 @@ class Unpacker(object): def next(self): try: ret = self._fb_unpack(EX_CONSTRUCT, None) - self._fb_consume() + self._fb_sloppy_consume() return ret except OutOfData: + self._fb_consume() raise StopIteration __next__ = next -- cgit v1.2.1 From 952eb9fc5331fb7134cc70ce6084b30eff4ccda2 Mon Sep 17 00:00:00 2001 From: Bas Westerbaan Date: Sat, 14 Jun 2014 18:34:17 +0200 Subject: fallback: add some comments to _fb_read --- msgpack/fallback.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'msgpack') diff --git a/msgpack/fallback.py b/msgpack/fallback.py index d838e55..fbd87ba 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -252,11 +252,15 @@ class Unpacker(object): def _fb_read(self, n, write_bytes=None): buffs = self._fb_buffers + # We have a redundant codepath for the most common case, such that + # pypy optimizes it properly. This is the case that the read fits + # in the current buffer. if (write_bytes is None and self._fb_buf_i < len(buffs) and self._fb_buf_o + n < len(buffs[self._fb_buf_i])): self._fb_buf_o += n return buffs[self._fb_buf_i][self._fb_buf_o - n:self._fb_buf_o] + # The remaining cases. ret = b'' while len(ret) != n: if self._fb_buf_i == len(buffs): -- cgit v1.2.1 From b334d441c3ed4e8bcd01bb1c2440dc975c377e56 Mon Sep 17 00:00:00 2001 From: Bas Westerbaan Date: Sat, 14 Jun 2014 18:42:02 +0200 Subject: fallback: _fb_read: do a big read, when we need a big read --- msgpack/fallback.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'msgpack') diff --git a/msgpack/fallback.py b/msgpack/fallback.py index fbd87ba..c422fb5 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -263,16 +263,19 @@ class Unpacker(object): # The remaining cases. ret = b'' while len(ret) != n: + sliced = n - len(ret) if self._fb_buf_i == len(buffs): if self._fb_feeding: break - tmp = self.file_like.read(self._read_size) + to_read = sliced + if self._read_size > to_read: + to_read = self._read_size + tmp = self.file_like.read(to_read) if not tmp: break buffs.append(tmp) self._fb_buf_n += len(tmp) continue - sliced = n - len(ret) ret += buffs[self._fb_buf_i][self._fb_buf_o:self._fb_buf_o + sliced] self._fb_buf_o += sliced if self._fb_buf_o >= len(buffs[self._fb_buf_i]): -- cgit v1.2.1 From 0532ea87fb9b7f8d3b8544ff116fd589ad2a93b2 Mon Sep 17 00:00:00 2001 From: Bas Westerbaan Date: Sun, 15 Jun 2014 22:45:30 +0200 Subject: fallback: fix BufferFull with sloppy consume --- msgpack/fallback.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) (limited to 'msgpack') diff --git a/msgpack/fallback.py b/msgpack/fallback.py index c422fb5..8ab3ad2 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -166,6 +166,11 @@ class Unpacker(object): self._fb_buf_o = 0 self._fb_buf_i = 0 self._fb_buf_n = 0 + # When Unpacker is used as an iterable, between the calls to next(), + # the buffer is not "consumed" completely, for efficiency sake. + # Instead, it is done sloppily. To make sure we raise BufferFull at + # the correct moments, we have to keep track of how sloppy we were. + self._fb_sloppiness = 0 self._max_buffer_size = max_buffer_size or 2**31-1 if read_size > self._max_buffer_size: raise ValueError("read_size must be smaller than max_buffer_size") @@ -196,7 +201,8 @@ class Unpacker(object): elif isinstance(next_bytes, bytearray): next_bytes = bytes(next_bytes) assert self._fb_feeding - if self._fb_buf_n + len(next_bytes) > self._max_buffer_size: + if (self._fb_buf_n + len(next_bytes) - self._fb_sloppiness + > self._max_buffer_size): raise BufferFull self._fb_buf_n += len(next_bytes) self._fb_buffers.append(next_bytes) @@ -208,6 +214,10 @@ class Unpacker(object): self._fb_buf_n -= len(self._fb_buffers[i]) self._fb_buffers = self._fb_buffers[self._fb_buf_i:] self._fb_buf_i = 0 + if self._fb_buffers: + self._fb_sloppiness = self._fb_buf_o + else: + self._fb_sloppiness = 0 def _fb_consume(self): """ Gets rid of the used parts of the buffer. """ @@ -222,6 +232,7 @@ class Unpacker(object): else: self._fb_buf_n = 0 self._fb_buf_o = 0 + self._fb_sloppiness = 0 def _fb_got_extradata(self): if self._fb_buf_i != len(self._fb_buffers): -- cgit v1.2.1