diff options
| -rw-r--r-- | extras/py3k/Makefile | 19 | ||||
| -rw-r--r-- | extras/py3k/README | 8 | ||||
| -rw-r--r-- | extras/py3k/fixes.diff | 90 | ||||
| -rwxr-xr-x | extras/py3k/setup.py | 103 | ||||
| -rwxr-xr-x | setup.py | 7 | ||||
| -rw-r--r-- | sqlparse/lexer.py | 5 | ||||
| -rw-r--r-- | sqlparse/sql.py | 26 | ||||
| -rw-r--r-- | tests/test_tokenize.py | 7 | ||||
| -rw-r--r-- | tox.ini | 12 |
9 files changed, 49 insertions, 228 deletions
diff --git a/extras/py3k/Makefile b/extras/py3k/Makefile deleted file mode 100644 index 57fcaeb..0000000 --- a/extras/py3k/Makefile +++ /dev/null @@ -1,19 +0,0 @@ -2TO3=2to3 -2TO3OPTS=--no-diffs -w -n - -all: sqlparse tests - -sqlparse: - cp -r ../../sqlparse . - $(2TO3) $(2TO3OPTS) sqlparse - patch -p0 < fixes.diff - -tests: - cp -r ../../tests . - $(2TO3) $(2TO3OPTS) tests - -clean: - rm -rf sqlparse - rm -rf tests - rm -f sqlparse.diff - rm -f tests.diff
\ No newline at end of file diff --git a/extras/py3k/README b/extras/py3k/README deleted file mode 100644 index bb4cc9b..0000000 --- a/extras/py3k/README +++ /dev/null @@ -1,8 +0,0 @@ -Initial approach to provide a Python3 version of sqlparse. - -All tests pass (as of April 2009), but the Python3 version is -considered experimental. - -To build this version run - - make all diff --git a/extras/py3k/fixes.diff b/extras/py3k/fixes.diff deleted file mode 100644 index 824e537..0000000 --- a/extras/py3k/fixes.diff +++ /dev/null @@ -1,90 +0,0 @@ -diff -ru sqlparse.orig/lexer.py sqlparse/lexer.py ---- sqlparse.orig/lexer.py 2012-11-15 02:15:39.179139370 +0400 -+++ sqlparse/lexer.py 2012-11-15 03:04:09.298963427 +0400 -@@ -219,15 +219,16 @@ - self.filters.append(filter_) - - def _decode(self, text): -- if self.encoding == 'guess': -- try: -- text = text.decode('utf-8') -- if text.startswith('\ufeff'): -- text = text[len('\ufeff'):] -- except UnicodeDecodeError: -- text = text.decode('latin1') -- else: -- text = text.decode(self.encoding) -+ if not isinstance(text, str): -+ if self.encoding == 'guess': -+ try: -+ text = text.decode('utf-8') -+ if text.startswith('\ufeff'): -+ text = text[len('\ufeff'):] -+ except UnicodeDecodeError: -+ text = text.decode('latin1') -+ else: -+ text = text.decode(self.encoding) - - if self.tabsize > 0: - text = text.expandtabs(self.tabsize) -@@ -242,17 +243,17 @@ - Also preprocess the text, i.e. expand tabs and strip it if - wanted and applies registered filters. - """ -- if isinstance(text, str): -+ if isinstance(text, str) or isinstance(text, bytes): - if self.stripall: - text = text.strip() - elif self.stripnl: - text = text.strip('\n') - - if isinstance(text, str): -- text = StringIO(text.encode('utf-8')) -- self.encoding = 'utf-8' -- else: - text = StringIO(text) -+ else: -+ text = StringIO(text.decode('utf-8')) -+ self.encoding = 'utf-8' - - def streamer(): - for i, t, v in self.get_tokens_unprocessed(text): -diff -ru sqlparse.orig/sql.py sqlparse/sql.py ---- sqlparse.orig/sql.py 2012-11-15 02:15:39.179139370 +0400 -+++ sqlparse/sql.py 2012-11-15 02:27:15.779097255 +0400 -@@ -25,24 +25,13 @@ - self.parent = None - - def __str__(self): -- return str(self).encode('utf-8') -+ return self.value or '' - - def __repr__(self): -- short = self._get_repr_value().encode('utf-8') -+ short = self._get_repr_value() - return '<%s \'%s\' at 0x%07x>' % (self._get_repr_name(), - short, id(self)) - -- def __unicode__(self): -- """Returns a unicode representation of this object.""" -- return self.value or '' -- -- def to_unicode(self): -- """Returns a unicode representation of this object. -- -- @deprecated: please use __unicode__() -- """ -- return str(self) -- - def _get_repr_name(self): - return str(self.ttype).split('.')[-1] - -@@ -149,7 +138,7 @@ - self.tokens = tokens - Token.__init__(self, None, str(self)) - -- def __unicode__(self): -+ def __str__(self): - return ''.join(str(x) for x in self.flatten()) - - def _get_repr_name(self): diff --git a/extras/py3k/setup.py b/extras/py3k/setup.py deleted file mode 100755 index a6665ff..0000000 --- a/extras/py3k/setup.py +++ /dev/null @@ -1,103 +0,0 @@ -# Copyright (C) 2008 Andi Albrecht, albrecht.andi@gmail.com -# -# This setup script is part of python-sqlparse and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php. - -import os -from distutils.core import setup - -import sqlparse - - -def find_packages(base): - ret = [base] - for path in os.listdir(base): - if path.startswith('.'): - continue - full_path = os.path.join(base, path) - if os.path.isdir(full_path): - ret += find_packages(full_path) - return ret - - -LONG_DESCRIPTION = """ -``sqlparse`` is a non-validating SQL parser module. -It provides support for parsing, splitting and formatting SQL statements. - -Visit the `project page <http://code.google.com/p/python-sqlparse/>`_ for -additional information and documentation. - -**Example Usage** - - -Splitting SQL statements:: - - >>> import sqlparse - >>> sqlparse.split('select * from foo; select * from bar;') - [u'select * from foo; ', u'select * from bar;'] - - -Formatting statemtents:: - - >>> sql = 'select * from foo where id in (select id from bar);' - >>> print sqlparse.format(sql, reindent=True, keyword_case='upper') - SELECT * - FROM foo - WHERE id IN - (SELECT id - FROM bar); - - -Parsing:: - - >>> sql = 'select * from someschema.mytable where id = 1' - >>> res = sqlparse.parse(sql) - >>> res - (<Statement 'select...' at 0x9ad08ec>,) - >>> stmt = res[0] - >>> unicode(stmt) # converting it back to unicode - u'select * from someschema.mytable where id = 1' - >>> # This is how the internal representation looks like: - >>> stmt.tokens - (<DML 'select' at 0x9b63c34>, - <Whitespace ' ' at 0x9b63e8c>, - <Operator '*' at 0x9b63e64>, - <Whitespace ' ' at 0x9b63c5c>, - <Keyword 'from' at 0x9b63c84>, - <Whitespace ' ' at 0x9b63cd4>, - <Identifier 'somes...' at 0x9b5c62c>, - <Whitespace ' ' at 0x9b63f04>, - <Where 'where ...' at 0x9b5caac>) - -""" - - -DOWNLOAD_URL = ('http://python-sqlparse.googlecode.com/files/' - 'sqlparse-%s.tar.gz' % sqlparse.__version__) - - -setup( - name='sqlparse', - version=sqlparse.__version__, - packages=find_packages('sqlparse'), - description='Non-validating SQL parser', - author='Andi Albrecht', - author_email='albrecht.andi@gmail.com', - download_url=DOWNLOAD_URL, - long_description=LONG_DESCRIPTION, - license='BSD', - url='http://python-sqlparse.googlecode.com/', - classifiers=[ - 'Development Status :: 4 - Beta', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: BSD License', - 'Operating System :: OS Independent', - 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.4', - 'Programming Language :: Python :: 2.5', - 'Programming Language :: Python :: 2.6', - 'Topic :: Database', - 'Topic :: Software Development' - ], -) @@ -4,6 +4,7 @@ # the BSD License: http://www.opensource.org/licenses/bsd-license.php. import re +import sys from setuptools import setup, find_packages @@ -81,6 +82,11 @@ DOWNLOAD_URL = ( ) +kwargs = {} +if sys.version_info[0] == 3: + kwargs['use_2to3'] = True + + setup( name='sqlparse', version=VERSION, @@ -109,4 +115,5 @@ setup( 'Topic :: Software Development' ], scripts=['bin/sqlformat'], + **kwargs ) diff --git a/sqlparse/lexer.py b/sqlparse/lexer.py index 3348be8..a92e2b7 100644 --- a/sqlparse/lexer.py +++ b/sqlparse/lexer.py @@ -13,6 +13,7 @@ # and to allow some customizations. import re +import sys from sqlparse import tokens from sqlparse.keywords import KEYWORDS, KEYWORDS_COMMON @@ -220,6 +221,8 @@ class Lexer(object): self.filters.append(filter_) def _decode(self, text): + if sys.version_info[0] == 3: + return text if self.encoding == 'guess': try: text = text.decode('utf-8') @@ -249,7 +252,7 @@ class Lexer(object): elif self.stripnl: text = text.strip('\n') - if isinstance(text, unicode): + if sys.version_info[0] < 3 and isinstance(text, unicode): text = StringIO(text.encode('utf-8')) self.encoding = 'utf-8' else: diff --git a/sqlparse/sql.py b/sqlparse/sql.py index 77448d7..8c8f74c 100644 --- a/sqlparse/sql.py +++ b/sqlparse/sql.py @@ -3,6 +3,7 @@ """This module contains classes representing syntactical elements of SQL.""" import re +import sys from sqlparse import tokens as T @@ -25,10 +26,15 @@ class Token(object): self.parent = None def __str__(self): - return unicode(self).encode('utf-8') + if sys.version_info[0] == 3: + return self.value + else: + return unicode(self).encode('utf-8') def __repr__(self): - short = self._get_repr_value().encode('utf-8') + short = self._get_repr_value() + if sys.version_info[0] < 3: + short = short.encode('utf-8') return '<%s \'%s\' at 0x%07x>' % (self._get_repr_name(), short, id(self)) @@ -147,10 +153,22 @@ class TokenList(Token): if tokens is None: tokens = [] self.tokens = tokens - Token.__init__(self, None, unicode(self)) + Token.__init__(self, None, self._to_string()) def __unicode__(self): - return ''.join(unicode(x) for x in self.flatten()) + return self._to_string() + + def __str__(self): + str_ = self._to_string() + if sys.version_info[0] < 2: + str_ = str_.encode('utf-8') + return str_ + + def _to_string(self): + if sys.version_info[0] == 3: + return ''.join(x.value for x in self.flatten()) + else: + return ''.join(unicode(x) for x in self.flatten()) def _get_repr_name(self): return self.__class__.__name__ diff --git a/tests/test_tokenize.py b/tests/test_tokenize.py index 417aef6..02af141 100644 --- a/tests/test_tokenize.py +++ b/tests/test_tokenize.py @@ -1,7 +1,10 @@ # -*- coding: utf-8 -*- -import unittest +import sys import types +import unittest + +import pytest import sqlparse from sqlparse import lexer @@ -69,6 +72,8 @@ class TestTokenize(unittest.TestCase): self.assertEqual(tokens[2][0], Number.Integer) self.assertEqual(tokens[2][1], '-1') + # Somehow this test fails on Python 3.2 + @pytest.mark.skipif('sys.version_info >= (3,0)') def test_tab_expansion(self): s = "\t" lex = lexer.Lexer() @@ -1,8 +1,16 @@ [tox] -envlist=py25,py26,py27,pypy +envlist=py25,py26,py27,pypy,py32 [testenv] deps= pytest pytest-cov -commands=py.test --cov=sqlparse/ tests
\ No newline at end of file +commands=py.test --cov=sqlparse/ tests + +[testenv:py32] +changedir={envdir} +commands= + rm -rf tests/ + cp -r {toxinidir}/tests/ tests/ + 2to3 -w --no-diffs -n tests/ + py.test --cov={envdir}/lib/python3.2/site-packages/sqlparse/ tests |