From 1f8dfd8723dd7aa9610fd9249775dc3b403d7e77 Mon Sep 17 00:00:00 2001 From: quest Date: Sun, 22 Apr 2012 00:27:15 +0200 Subject: Oops; doesnt handle UTF-8 correctly when reading from stream --- sqlparse/lexer.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/lexer.py b/sqlparse/lexer.py index a0ce5d2..dc794ab 100644 --- a/sqlparse/lexer.py +++ b/sqlparse/lexer.py @@ -271,8 +271,9 @@ class Lexer(object): statetokens = tokendefs[statestack[-1]] known_names = {} - text = self._decode(stream.read(self.bufsize)) + text = stream.read(self.bufsize) hasmore = len(text) == self.bufsize + text = self._decode(text) while 1: for rexmatch, action, new_state in statetokens: @@ -317,9 +318,9 @@ class Lexer(object): break else: if hasmore: - buf = self._decode(stream.read(self.bufsize)) + buf = stream.read(self.bufsize) hasmore = len(buf) == self.bufsize - text = text[pos:] + buf + text = text[pos:] + self._decode(buf) pos = 0 continue -- cgit v1.2.1