diff options
| -rw-r--r-- | sqlparse/lexer.py | 12 | ||||
| -rw-r--r-- | tests/test_tokenize.py | 2 |
2 files changed, 7 insertions, 7 deletions
diff --git a/sqlparse/lexer.py b/sqlparse/lexer.py index 2b0688a..b4eb604 100644 --- a/sqlparse/lexer.py +++ b/sqlparse/lexer.py @@ -227,10 +227,15 @@ class _Lexer(object): filter_ = filter_(**options) self.filters.append(filter_) + def _expandtabs(self, text): + if self.tabsize > 0: + text = text.expandtabs(self.tabsize) + return text + def _decode(self, text): if sys.version_info[0] == 3: if isinstance(text, str): - return text + return self._expandtabs(text) if self.encoding == 'guess': try: text = text.decode('utf-8') @@ -243,10 +248,7 @@ class _Lexer(object): text = text.decode(self.encoding) except UnicodeDecodeError: text = text.decode('unicode-escape') - - if self.tabsize > 0: - text = text.expandtabs(self.tabsize) - return text + return self._expandtabs(text) def get_tokens(self, text, unfiltered=False): """ diff --git a/tests/test_tokenize.py b/tests/test_tokenize.py index ceaf24e..619aefd 100644 --- a/tests/test_tokenize.py +++ b/tests/test_tokenize.py @@ -73,8 +73,6 @@ class TestTokenize(unittest.TestCase): self.assertEqual(tokens[2][0], Number.Integer) self.assertEqual(tokens[2][1], '-1') - # Somehow this test fails on Python 3.2 - @pytest.mark.skipif('sys.version_info >= (3,0)') def test_tab_expansion(self): s = "\t" lex = lexer.Lexer() |