From 729967637c1ead8042bca8e1ece3906f6f6ed6e7 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Mon, 10 Jan 2011 17:15:55 -0600 Subject: Tolerate streams with the \r missing from the chunk length. --- python/subunit/chunked.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'python/subunit/chunked.py') diff --git a/python/subunit/chunked.py b/python/subunit/chunked.py index 82e4b0d..9c51357 100644 --- a/python/subunit/chunked.py +++ b/python/subunit/chunked.py @@ -87,7 +87,7 @@ class Decoder(object): if count_chars[-1][-1] != '\n': return count_str = ''.join(count_chars) - self.body_length = int(count_str[:-2], 16) + self.body_length = int(count_str.rstrip('\n\r'), 16) excess_bytes = len(count_str) while excess_bytes: if excess_bytes >= len(self.buffered_bytes[0]): -- cgit v1.2.1 From 0de847bb324e3bf3345c355f899c939180ecb406 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Tue, 11 Jan 2011 22:49:26 -0600 Subject: Default to (more) strict decoding of chunked parts --- python/subunit/chunked.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'python/subunit/chunked.py') diff --git a/python/subunit/chunked.py b/python/subunit/chunked.py index 9c51357..c15675b 100644 --- a/python/subunit/chunked.py +++ b/python/subunit/chunked.py @@ -1,6 +1,7 @@ # # subunit: extensions to python unittest to get test results from subprocesses. # Copyright (C) 2005 Robert Collins +# Copyright (C) 2011 Martin Pool # # Licensed under either the Apache License, Version 2.0 or the BSD 3-clause # license at the users choice. A copy of both licenses are available in the @@ -19,7 +20,7 @@ class Decoder(object): """Decode chunked content to a byte stream.""" - def __init__(self, output): + def __init__(self, output, strict=True): """Create a decoder decoding to output. :param output: A file-like object. Bytes written to the Decoder are @@ -29,11 +30,18 @@ class Decoder(object): when no more data is available, to detect short streams; the write method will return none-None when the end of a stream is detected. + + :param strict: If True (the default), the decoder will not knowingly + accept input that is not conformant to the HTTP specification. + (This does not imply that it will catch every nonconformance.) + If False, it will accept incorrect input that is still + unambiguous. """ self.output = output self.buffered_bytes = [] self.state = self._read_length self.body_length = 0 + self.strict = strict def close(self): """Close the decoder. @@ -87,6 +95,11 @@ class Decoder(object): if count_chars[-1][-1] != '\n': return count_str = ''.join(count_chars) + if self.strict: + if count_str[-2:] != '\r\n': + raise ValueError("chunk header invalid: %r" % count_str) + if '\r' in count_str[:-2]: + raise ValueError("too many crs in chunk header %r" % count_str) self.body_length = int(count_str.rstrip('\n\r'), 16) excess_bytes = len(count_str) while excess_bytes: -- cgit v1.2.1 From 4720c35d1bfd2af2f27ceddfd039c54553ce2deb Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Fri, 28 Jan 2011 17:36:39 +0000 Subject: CR, not cr. Flakes. Whitespace. --- python/subunit/chunked.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'python/subunit/chunked.py') diff --git a/python/subunit/chunked.py b/python/subunit/chunked.py index c15675b..5f8c6f1 100644 --- a/python/subunit/chunked.py +++ b/python/subunit/chunked.py @@ -7,7 +7,7 @@ # license at the users choice. A copy of both licenses are available in the # project source as Apache-2.0 and BSD. You may not use this file except in # compliance with one of these two licences. -# +# # Unless required by applicable law or agreed to in writing, software # distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -34,7 +34,7 @@ class Decoder(object): :param strict: If True (the default), the decoder will not knowingly accept input that is not conformant to the HTTP specification. (This does not imply that it will catch every nonconformance.) - If False, it will accept incorrect input that is still + If False, it will accept incorrect input that is still unambiguous. """ self.output = output @@ -80,7 +80,6 @@ class Decoder(object): def _read_length(self): """Try to decode a length from the bytes.""" - count = -1 match_chars = "0123456789abcdefABCDEF\r\n" count_chars = [] for bytes in self.buffered_bytes: @@ -99,7 +98,7 @@ class Decoder(object): if count_str[-2:] != '\r\n': raise ValueError("chunk header invalid: %r" % count_str) if '\r' in count_str[:-2]: - raise ValueError("too many crs in chunk header %r" % count_str) + raise ValueError("too many CRs in chunk header %r" % count_str) self.body_length = int(count_str.rstrip('\n\r'), 16) excess_bytes = len(count_str) while excess_bytes: @@ -120,7 +119,7 @@ class Decoder(object): def write(self, bytes): """Decode bytes to the output stream. - + :raises ValueError: If the stream has already seen the end of file marker. :returns: None, or the excess bytes beyond the end of file marker. @@ -146,7 +145,7 @@ class Encoder(object): def flush(self, extra_len=0): """Flush the encoder to the output stream. - + :param extra_len: Increase the size of the chunk by this many bytes to allow for a subsequent write. """ -- cgit v1.2.1 From 86b85e3f83f4030a63026386ffc3d3488ab34d13 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Mon, 25 Apr 2011 16:25:24 +1200 Subject: Nearly done. --- python/subunit/chunked.py | 41 +++++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 16 deletions(-) (limited to 'python/subunit/chunked.py') diff --git a/python/subunit/chunked.py b/python/subunit/chunked.py index 5f8c6f1..b992129 100644 --- a/python/subunit/chunked.py +++ b/python/subunit/chunked.py @@ -17,6 +17,10 @@ """Encoder/decoder for http style chunked encoding.""" +from testtools.compat import _b + +empty = _b('') + class Decoder(object): """Decode chunked content to a byte stream.""" @@ -25,11 +29,11 @@ class Decoder(object): :param output: A file-like object. Bytes written to the Decoder are decoded to strip off the chunking and written to the output. - Up to a full write worth of data or a single control line may be + Up to a full write worth of data or a single control line may be buffered (whichever is larger). The close method should be called when no more data is available, to detect short streams; the write method will return none-None when the end of a stream is - detected. + detected. The output object must accept bytes objects. :param strict: If True (the default), the decoder will not knowingly accept input that is not conformant to the HTTP specification. @@ -42,6 +46,11 @@ class Decoder(object): self.state = self._read_length self.body_length = 0 self.strict = strict + self._match_chars = _b("0123456789abcdefABCDEF\r\n") + self._slash_n = _b('\n') + self._slash_r = _b('\r') + self._slash_rn = _b('\r\n') + self._slash_nr = _b('\n\r') def close(self): """Close the decoder. @@ -56,7 +65,7 @@ class Decoder(object): if self.buffered_bytes: buffered_bytes = self.buffered_bytes self.buffered_bytes = [] - return ''.join(buffered_bytes) + return empty.join(buffered_bytes) else: raise ValueError("stream is finished") @@ -80,26 +89,26 @@ class Decoder(object): def _read_length(self): """Try to decode a length from the bytes.""" - match_chars = "0123456789abcdefABCDEF\r\n" count_chars = [] for bytes in self.buffered_bytes: - for byte in bytes: - if byte not in match_chars: + for pos in range(len(bytes)): + byte = bytes[pos:pos+1] + if byte not in self._match_chars: break count_chars.append(byte) - if byte == '\n': + if byte == self._slash_n: break if not count_chars: return - if count_chars[-1][-1] != '\n': + if count_chars[-1] != self._slash_n: return - count_str = ''.join(count_chars) + count_str = empty.join(count_chars) if self.strict: - if count_str[-2:] != '\r\n': + if count_str[-2:] != self._slash_rn: raise ValueError("chunk header invalid: %r" % count_str) - if '\r' in count_str[:-2]: + if self._slash_r in count_str[:-2]: raise ValueError("too many CRs in chunk header %r" % count_str) - self.body_length = int(count_str.rstrip('\n\r'), 16) + self.body_length = int(count_str.rstrip(self._slash_nr), 16) excess_bytes = len(count_str) while excess_bytes: if excess_bytes >= len(self.buffered_bytes[0]): @@ -112,7 +121,7 @@ class Decoder(object): self.state = self._finished if not self.buffered_bytes: # May not call into self._finished with no buffered data. - return '' + return empty else: self.state = self._read_body return self.state() @@ -155,9 +164,9 @@ class Encoder(object): buffer_size = self.buffer_size self.buffered_bytes = [] self.buffer_size = 0 - self.output.write("%X\r\n" % (buffer_size + extra_len)) + self.output.write(_b("%X\r\n" % (buffer_size + extra_len))) if buffer_size: - self.output.write(''.join(buffered_bytes)) + self.output.write(empty.join(buffered_bytes)) return True def write(self, bytes): @@ -173,4 +182,4 @@ class Encoder(object): def close(self): """Finish the stream. This does not close the output stream.""" self.flush() - self.output.write("0\r\n") + self.output.write(_b("0\r\n")) -- cgit v1.2.1