From 954f48431a0fe8ea1749ba0236da23a705bf3b75 Mon Sep 17 00:00:00 2001
From: Victor Uriarte <victor.m.uriarte@intel.com>
Date: Sat, 4 Jun 2016 15:08:40 -0700
Subject: Redo unicode-encoding on lexer.py

---
 sqlparse/lexer.py | 15 ++++-----------
 1 file changed, 4 insertions(+), 11 deletions(-)

(limited to 'sqlparse')

diff --git a/sqlparse/lexer.py b/sqlparse/lexer.py
index dd15212..0fb8936 100644
--- a/sqlparse/lexer.py
+++ b/sqlparse/lexer.py
@@ -14,7 +14,7 @@
 
 from sqlparse import tokens
 from sqlparse.keywords import SQL_REGEX
-from sqlparse.compat import StringIO, string_types, text_type
+from sqlparse.compat import StringIO, string_types, u
 from sqlparse.utils import consume
 
 
@@ -37,17 +37,10 @@ class Lexer(object):
 
         ``stack`` is the inital stack (default: ``['root']``)
         """
-        encoding = encoding or 'utf-8'
-
         if isinstance(text, string_types):
-            text = StringIO(text)
-
-        text = text.read()
-        if not isinstance(text, text_type):
-            try:
-                text = text.decode(encoding)
-            except UnicodeDecodeError:
-                text = text.decode('unicode-escape')
+            text = u(text, encoding)
+        elif isinstance(text, StringIO):
+            text = u(text.read(), encoding)
 
         iterable = enumerate(text)
         for pos, char in iterable:
-- 
cgit v1.2.1