Migrate to six and get rid of 2to3.

Now we've got really a single code base for both Python 2 and 3.
For now it just adds a dependency to six which should be available
on most systems. However, if it turns out to switch back to
no dependencies it should be fairly easy to replace six by
some conditional imports.

5 files changed, 48 insertions, 41 deletions

diff --git a/sqlparse/__init__.py b/sqlparse/__init__.py
index 3052484..2537d5e 100644
--- a/sqlparse/__init__.py
+++ b/sqlparse/__init__.py
@@ -8,6 +8,8 @@
 
 __version__ = '0.1.14'
 
+import six
+
 
 # Setup namespace
 from sqlparse import engine
@@ -64,7 +66,7 @@ def split(sql, encoding=None):
     """
     stack = engine.FilterStack()
     stack.split_statements = True
-    return [unicode(stmt).strip() for stmt in stack.run(sql, encoding)]
+    return [six.text_type(stmt).strip() for stmt in stack.run(sql, encoding)]
 
 
 from sqlparse.engine.filter import StatementFilter
diff --git a/sqlparse/engine/grouping.py b/sqlparse/engine/grouping.py
index b43ccc7..cebf8dc 100644
--- a/sqlparse/engine/grouping.py
+++ b/sqlparse/engine/grouping.py
@@ -50,7 +50,7 @@ def _group_matching(tlist, start_ttype, start_value, end_ttype, end_value,
                     cls, include_semicolon=False, recurse=False):
     def _find_matching(i, tl, stt, sva, ett, eva):
         depth = 1
-        for n in xrange(i, len(tl.tokens)):
+        for n in range(i, len(tl.tokens)):
             t = tl.tokens[n]
             if t.match(stt, sva):
                 depth += 1
diff --git a/sqlparse/filters.py b/sqlparse/filters.py
index abb7e4f..aaeb15e 100644
--- a/sqlparse/filters.py
+++ b/sqlparse/filters.py
@@ -4,6 +4,8 @@ import re
 
 from os.path import abspath, join
 
+import six
+
 from sqlparse import sql, tokens as T
 from sqlparse.engine import FilterStack
 from sqlparse.pipeline import Pipeline
@@ -24,7 +26,7 @@ class _CaseFilter:
         if case is None:
             case = 'upper'
         assert case in ['lower', 'upper', 'capitalize']
-        self.convert = getattr(unicode, case)
+        self.convert = getattr(six.text_type, case)
 
     def process(self, stack, stream):
         for ttype, value in stream:
@@ -51,20 +53,20 @@ class TruncateStringFilter:
 
     def __init__(self, width, char):
         self.width = max(width, 1)
-        self.char = unicode(char)
+        self.char = six.text_type(char)
 
     def process(self, stack, stream):
         for ttype, value in stream:
             if ttype is T.Literal.String.Single:
                 if value[:2] == '\'\'':
                     inner = value[2:-2]
-                    quote = u'\'\''
+                    quote = six.text_type('\'\'')
                 else:
                     inner = value[1:-1]
-                    quote = u'\''
+                    quote = six.text_type('\'')
                 if len(inner) > self.width:
-                    value = u''.join((quote, inner[:self.width], self.char,
-                                      quote))
+                    value = six.text_type('').join(
+                        (quote, inner[:self.width], self.char, quote))
             yield ttype, value
 
 
@@ -152,13 +154,13 @@ class IncludeStatement:
                         f.close()
 
                     # There was a problem loading the include file
-                    except IOError, err:
+                    except IOError as err:
                         # Raise the exception to the interpreter
                         if self.raiseexceptions:
                             raise
 
                         # Put the exception as a comment on the SQL code
-                        yield Comment, u'-- IOError: %s\n' % err
+                        yield Comment, six.text_type('-- IOError: %s\n' % err)
 
                     else:
                         # Create new FilterStack to parse readed file
@@ -169,13 +171,14 @@ class IncludeStatement:
                                                      self.raiseexceptions)
 
                         # Max recursion limit reached
-                        except ValueError, err:
+                        except ValueError as err:
                             # Raise the exception to the interpreter
                             if self.raiseexceptions:
                                 raise
 
                             # Put the exception as a comment on the SQL code
-                            yield Comment, u'-- ValueError: %s\n' % err
+                            yield Comment, six.text_type(
+                                '-- ValueError: %s\n' % err)
 
                         stack = FilterStack()
                         stack.preprocess.append(filtr)
@@ -294,7 +297,7 @@ class ReindentFilter:
                 raise StopIteration
 
     def _get_offset(self, token):
-        raw = ''.join(map(unicode, self._flatten_up_to_token(token)))
+        raw = ''.join(map(six.text_type, self._flatten_up_to_token(token)))
         line = raw.splitlines()[-1]
         # Now take current offset into account and return relative offset.
         full_offset = len(line) - len(self.char * (self.width * self.indent))
@@ -334,7 +337,7 @@ class ReindentFilter:
             if prev and prev.is_whitespace() and prev not in added:
                 tlist.tokens.pop(tlist.token_index(prev))
                 offset += 1
-            uprev = unicode(prev)
+            uprev = six.text_type(prev)
             if (prev and (uprev.endswith('\n') or uprev.endswith('\r'))):
                 nl = tlist.token_next(token)
             else:
@@ -455,7 +458,7 @@ class ReindentFilter:
         self._process(stmt)
         if isinstance(stmt, sql.Statement):
             if self._last_stmt is not None:
-                if unicode(self._last_stmt).endswith('\n'):
+                if six.text_type(self._last_stmt).endswith('\n'):
                     nl = '\n'
                 else:
                     nl = '\n\n'
@@ -487,7 +490,7 @@ class RightMarginFilter:
                   and token.__class__ not in self.keep_together):
                 token.tokens = self._process(stack, token, token.tokens)
             else:
-                val = unicode(token)
+                val = six.text_type(token)
                 if len(self.line) + len(val) > self.width:
                     match = re.search('^ +', self.line)
                     if match is not None:
@@ -561,7 +564,7 @@ class ColumnsSelect:
 class SerializerUnicode:
 
     def process(self, stack, stmt):
-        raw = unicode(stmt)
+        raw = six.text_type(stmt)
         lines = split_unquoted_newlines(raw)
         res = '\n'.join(line.rstrip() for line in lines)
         return res
@@ -571,7 +574,7 @@ def Tokens2Unicode(stream):
     result = ""
 
     for _, value in stream:
-        result += unicode(value)
+        result += six.text_type(value)
 
     return result
 
@@ -593,7 +596,7 @@ class OutputFilter:
         else:
             varname = self.varname
 
-        has_nl = len(unicode(stmt).strip().splitlines()) > 1
+        has_nl = len(six.text_type(stmt).strip().splitlines()) > 1
         stmt.tokens = self._process(stmt.tokens, varname, has_nl)
         return stmt
 
diff --git a/sqlparse/lexer.py b/sqlparse/lexer.py
index 1eacff5..b2419ac 100644
--- a/sqlparse/lexer.py
+++ b/sqlparse/lexer.py
@@ -15,9 +15,11 @@
 import re
 import sys
 
+import six
+from six import StringIO
+
 from sqlparse import tokens
 from sqlparse.keywords import KEYWORDS, KEYWORDS_COMMON
-from cStringIO import StringIO
 
 
 class include(str):
@@ -81,7 +83,7 @@ class LexerMeta(type):
 
             try:
                 rex = re.compile(tdef[0], rflags).match
-            except Exception, err:
+            except Exception as err:
                 raise ValueError(("uncompilable regex %r in state"
                                   " %r of %r: %s"
                                   % (tdef[0], state, cls, err)))
@@ -151,9 +153,7 @@ class LexerMeta(type):
         return type.__call__(cls, *args, **kwds)
 
 
-class Lexer(object):
-
-    __metaclass__ = LexerMeta
+class Lexer(six.with_metaclass(LexerMeta)):
 
     encoding = 'utf-8'
     stripall = False
@@ -228,8 +228,8 @@ class Lexer(object):
         if self.encoding == 'guess':
             try:
                 text = text.decode('utf-8')
-                if text.startswith(u'\ufeff'):
-                    text = text[len(u'\ufeff'):]
+                if text.startswith(six.text_type('\ufeff')):
+                    text = text[len(six.text_type('\ufeff')):]
             except UnicodeDecodeError:
                 text = text.decode('latin1')
         else:
@@ -251,13 +251,13 @@ class Lexer(object):
         Also preprocess the text, i.e. expand tabs and strip it if
         wanted and applies registered filters.
         """
-        if isinstance(text, basestring):
+        if isinstance(text, six.string_types):
             if self.stripall:
                 text = text.strip()
             elif self.stripnl:
                 text = text.strip('\n')
 
-            if sys.version_info[0] < 3 and isinstance(text, unicode):
+            if six.PY2 and isinstance(text, six.text_type):
                 text = StringIO(text.encode('utf-8'))
                 self.encoding = 'utf-8'
             else:
@@ -330,7 +330,7 @@ class Lexer(object):
                         pos += 1
                         statestack = ['root']
                         statetokens = tokendefs['root']
-                        yield pos, tokens.Text, u'\n'
+                        yield pos, tokens.Text, six.text_type('\n')
                         continue
                     yield pos, tokens.Error, text[pos]
                     pos += 1
diff --git a/sqlparse/sql.py b/sqlparse/sql.py
index 0fe92c7..f3a4820 100644
--- a/sqlparse/sql.py
+++ b/sqlparse/sql.py
@@ -5,6 +5,8 @@
 import re
 import sys
 
+import six
+
 from sqlparse import tokens as T
 
 
@@ -32,7 +34,7 @@ class Token(object):
         if sys.version_info[0] == 3:
             return self.value
         else:
-            return unicode(self).encode('utf-8')
+            return six.text_type(self).encode('utf-8')
 
     def __repr__(self):
         short = self._get_repr_value()
@@ -51,15 +53,15 @@ class Token(object):
         .. deprecated:: 0.1.5
            Use ``unicode(token)`` (for Python 3: ``str(token)``) instead.
         """
-        return unicode(self)
+        return six.text_type(self)
 
     def _get_repr_name(self):
         return str(self.ttype).split('.')[-1]
 
     def _get_repr_value(self):
-        raw = unicode(self)
+        raw = six.text_type(self)
         if len(raw) > 7:
-            raw = raw[:6] + u'...'
+            raw = raw[:6] + six.text_type('...')
         return re.sub('\s+', ' ', raw)
 
     def flatten(self):
@@ -83,7 +85,7 @@ class Token(object):
             return type_matched
 
         if regex:
-            if isinstance(values, basestring):
+            if isinstance(values, six.string_types):
                 values = set([values])
 
             if self.ttype is T.Keyword:
@@ -96,7 +98,7 @@ class Token(object):
                     return True
             return False
 
-        if isinstance(values, basestring):
+        if isinstance(values, six.string_types):
             if self.is_keyword:
                 return values.upper() == self.normalized
             return values == self.value
@@ -172,7 +174,7 @@ class TokenList(Token):
         if sys.version_info[0] == 3:
             return ''.join(x.value for x in self.flatten())
         else:
-            return ''.join(unicode(x) for x in self.flatten())
+            return ''.join(six.text_type(x) for x in self.flatten())
 
     def _get_repr_name(self):
         return self.__class__.__name__
@@ -185,9 +187,9 @@ class TokenList(Token):
                 pre = ' +-'
             else:
                 pre = ' | '
-            print '%s%s%d %s \'%s\'' % (indent, pre, idx,
+            print('%s%s%d %s \'%s\'' % (indent, pre, idx,
                                         token._get_repr_name(),
-                                        token._get_repr_value())
+                                        token._get_repr_value()))
             if (token.is_group() and (max_depth is None or depth < max_depth)):
                 token._pprint_tree(max_depth, depth + 1)
 
@@ -272,7 +274,7 @@ class TokenList(Token):
         if not isinstance(idx, int):
             idx = self.token_index(idx)
 
-        for n in xrange(idx, len(self.tokens)):
+        for n in range(idx, len(self.tokens)):
             token = self.tokens[n]
             if token.match(ttype, value, regex):
                 return token
@@ -396,7 +398,7 @@ class TokenList(Token):
             alias = next_
         if isinstance(alias, Identifier):
             return alias.get_name()
-        return self._remove_quotes(unicode(alias))
+        return self._remove_quotes(six.text_type(alias))
 
     def get_name(self):
         """Returns the name of this identifier.
@@ -485,7 +487,7 @@ class Identifier(TokenList):
         next_ = self.token_next(self.token_index(marker), False)
         if next_ is None:
             return None
-        return unicode(next_)
+        return six.text_type(next_)
 
     def get_ordering(self):
         """Returns the ordering or ``None`` as uppercase string."""