From 843499915e91e0ee324a0407c78ac6f570806370 Mon Sep 17 00:00:00 2001 From: Oleg Broytman Date: Wed, 31 Aug 2016 16:10:35 +0300 Subject: Decode bytes to unicode in Lexer.get_tokens(). Raise TypeError if the input is neither bytes in a known encoding nor unicode nor a file-like object (file, StringIO). Remove function u(). Add bytes_type to compat. Add tests for non-ascii. --- sqlparse/compat.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) (limited to 'sqlparse/compat.py') diff --git a/sqlparse/compat.py b/sqlparse/compat.py index d6a9144..933e0be 100644 --- a/sqlparse/compat.py +++ b/sqlparse/compat.py @@ -23,14 +23,10 @@ PY3 = sys.version_info[0] == 3 if PY3: - def u(s, encoding=None): - return str(s) - - def unicode_compatible(cls): return cls - + bytes_type = bytes text_type = str string_types = (str,) from io import StringIO @@ -38,20 +34,12 @@ if PY3: elif PY2: - def u(s, encoding=None): - encoding = encoding or 'unicode-escape' - try: - return unicode(s) - except UnicodeDecodeError: - return unicode(s, encoding) - - def unicode_compatible(cls): cls.__unicode__ = cls.__str__ cls.__str__ = lambda x: x.__unicode__().encode('utf-8') return cls - + bytes_type = str text_type = unicode string_types = (str, unicode,) from StringIO import StringIO -- cgit v1.2.1