Merge pull request #80 from vthriller/master

py3k fixes

1 files changed, 69 insertions, 9 deletions

diff --git a/extras/py3k/fixes.diff b/extras/py3k/fixes.diff
index 402e0ee..824e537 100644
--- a/extras/py3k/fixes.diff
+++ b/extras/py3k/fixes.diff
@@ -1,6 +1,58 @@
---- sqlparse/sql.py.orig	2012-03-29 12:09:46.660311410 +0200
-+++ sqlparse/sql.py	2012-03-29 12:11:04.924699486 +0200
-@@ -23,17 +23,13 @@
+diff -ru sqlparse.orig/lexer.py sqlparse/lexer.py
+--- sqlparse.orig/lexer.py	2012-11-15 02:15:39.179139370 +0400
++++ sqlparse/lexer.py	2012-11-15 03:04:09.298963427 +0400
+@@ -219,15 +219,16 @@
+         self.filters.append(filter_)
+ 
+     def _decode(self, text):
+-        if self.encoding == 'guess':
+-            try:
+-                text = text.decode('utf-8')
+-                if text.startswith('\ufeff'):
+-                    text = text[len('\ufeff'):]
+-            except UnicodeDecodeError:
+-                text = text.decode('latin1')
+-        else:
+-            text = text.decode(self.encoding)
++        if not isinstance(text, str):
++            if self.encoding == 'guess':
++                try:
++                    text = text.decode('utf-8')
++                    if text.startswith('\ufeff'):
++                        text = text[len('\ufeff'):]
++                except UnicodeDecodeError:
++                    text = text.decode('latin1')
++            else:
++                text = text.decode(self.encoding)
+ 
+         if self.tabsize > 0:
+             text = text.expandtabs(self.tabsize)
+@@ -242,17 +243,17 @@
+         Also preprocess the text, i.e. expand tabs and strip it if
+         wanted and applies registered filters.
+         """
+-        if isinstance(text, str):
++        if isinstance(text, str) or isinstance(text, bytes):
+             if self.stripall:
+                 text = text.strip()
+             elif self.stripnl:
+                 text = text.strip('\n')
+ 
+             if isinstance(text, str):
+-                text = StringIO(text.encode('utf-8'))
+-                self.encoding = 'utf-8'
+-            else:
+                 text = StringIO(text)
++            else:
++                text = StringIO(text.decode('utf-8'))
++                self.encoding = 'utf-8'
+ 
+         def streamer():
+             for i, t, v in self.get_tokens_unprocessed(text):
+diff -ru sqlparse.orig/sql.py sqlparse/sql.py
+--- sqlparse.orig/sql.py	2012-11-15 02:15:39.179139370 +0400
++++ sqlparse/sql.py	2012-11-15 02:27:15.779097255 +0400
+@@ -25,24 +25,13 @@
          self.parent = None
  
      def __str__(self):
@@ -8,7 +60,8 @@
 +        return self.value or ''
  
      def __repr__(self):
-         short = self._get_repr_value()
+-        short = self._get_repr_value().encode('utf-8')
++        short = self._get_repr_value()
          return '<%s \'%s\' at 0x%07x>' % (self._get_repr_name(),
                                            short, id(self))
  
@@ -16,12 +69,19 @@
 -        """Returns a unicode representation of this object."""
 -        return self.value or ''
 -
-     def to_unicode(self):
-         """Returns a unicode representation of this object.
-         
-@@ -139,7 +135,7 @@
+-    def to_unicode(self):
+-        """Returns a unicode representation of this object.
+-
+-        @deprecated: please use __unicode__()
+-        """
+-        return str(self)
+-
+     def _get_repr_name(self):
+         return str(self.ttype).split('.')[-1]
+ 
+@@ -149,7 +138,7 @@
          self.tokens = tokens
-         Token.__init__(self, None, None)
+         Token.__init__(self, None, str(self))
  
 -    def __unicode__(self):
 +    def __str__(self):