Backed out changeset 489299fc3548

3 files changed, 43 insertions, 3 deletions

diff --git a/extras/py3k/Makefile b/extras/py3k/Makefile
index 5c2d1eb..0f5f4b6 100644
--- a/extras/py3k/Makefile
+++ b/extras/py3k/Makefile
@@ -6,6 +6,7 @@ sqlparse:
 	cp -r ../../sqlparse .
 	$(2TO3) sqlparse > sqlparse.diff
 	patch -p0 < sqlparse.diff
+	patch -p0 < fixes.diff
 
 tests:
 	cp -r ../../tests .
@@ -16,4 +17,4 @@ clean:
 	rm -rf sqlparse
 	rm -rf tests
 	rm -f sqlparse.diff
-	rm -f tests.diff
+	rm -f tests.diff
\ No newline at end of file
diff --git a/extras/py3k/fixes.diff b/extras/py3k/fixes.diff
new file mode 100644
index 0000000..10c4f57
--- /dev/null
+++ b/extras/py3k/fixes.diff
@@ -0,0 +1,33 @@
+--- sqlparse/sql.py~	2009-04-30 08:38:16.000000000 +0200
++++ sqlparse/sql.py	2009-04-30 08:39:15.000000000 +0200
+@@ -23,16 +23,13 @@
+         self.ttype = ttype
+ 
+     def __str__(self):
+-        return str(self).encode('utf-8')
++        return self.value or ''
+ 
+     def __repr__(self):
+         short = self._get_repr_value()
+         return '<%s \'%s\' at 0x%07x>' % (self._get_repr_name(),
+                                           short, id(self))
+ 
+-    def __unicode__(self):
+-        return self.value or ''
+-
+     def to_unicode(self):
+         """Returns a unicode representation of this object."""
+         return str(self)
+@@ -105,11 +102,8 @@
+         self.tokens = tokens
+         Token.__init__(self, None, None)
+ 
+-    def __unicode__(self):
+-        return ''.join(str(x) for x in self.flatten())
+-
+     def __str__(self):
+-        return str(self).encode('utf-8')
++        return ''.join(str(x) for x in self.flatten())
+ 
+     def _get_repr_name(self):
+         return self.__class__.__name__
diff --git a/sqlparse/sql.py b/sqlparse/sql.py
index 6bf26e6..a2462bc 100644
--- a/sqlparse/sql.py
+++ b/sqlparse/sql.py
@@ -23,13 +23,16 @@ class Token(object):
         self.parent = None
 
     def __str__(self):
-        return self.value or ''
+        return unicode(self).encode('utf-8')
 
     def __repr__(self):
         short = self._get_repr_value()
         return '<%s \'%s\' at 0x%07x>' % (self._get_repr_name(),
                                           short, id(self))
 
+    def __unicode__(self):
+        return self.value or ''
+
     def to_unicode(self):
         """Returns a unicode representation of this object."""
         return unicode(self)
@@ -132,8 +135,11 @@ class TokenList(Token):
         self.tokens = tokens
         Token.__init__(self, None, None)
 
+    def __unicode__(self):
+        return ''.join(unicode(x) for x in self.flatten())
+
     def __str__(self):
-        return ''.join('%s' % x for x in self.flatten())
+        return unicode(self).encode('utf-8')
 
     def _get_repr_name(self):
         return self.__class__.__name__