import types from io import StringIO import pytest import sqlparse from sqlparse import lexer from sqlparse import sql, tokens as T def test_tokenize_simple(): s = 'select * from foo;' stream = lexer.tokenize(s) assert isinstance(stream, types.GeneratorType) tokens = list(stream) assert len(tokens) == 8 assert len(tokens[0]) == 2 assert tokens[0] == (T.Keyword.DML, 'select') assert tokens[-1] == (T.Punctuation, ';') def test_tokenize_backticks(): s = '`foo`.`bar`' tokens = list(lexer.tokenize(s)) assert len(tokens) == 3 assert tokens[0] == (T.Name, '`foo`') @pytest.mark.parametrize('s', ['foo\nbar\n', 'foo\rbar\r', 'foo\r\nbar\r\n', 'foo\r\nbar\n']) def test_tokenize_linebreaks(s): # issue1 tokens = lexer.tokenize(s) assert ''.join(str(x[1]) for x in tokens) == s def test_tokenize_inline_keywords(): # issue 7 s = "create created_foo" tokens = list(lexer.tokenize(s)) assert len(tokens) == 3 assert tokens[0][0] == T.Keyword.DDL assert tokens[2][0] == T.Name assert tokens[2][1] == 'created_foo' s = "enddate" tokens = list(lexer.tokenize(s)) assert len(tokens) == 1 assert tokens[0][0] == T.Name s = "join_col" tokens = list(lexer.tokenize(s)) assert len(tokens) == 1 assert tokens[0][0] == T.Name s = "left join_col" tokens = list(lexer.tokenize(s)) assert len(tokens) == 3 assert tokens[2][0] == T.Name assert tokens[2][1] == 'join_col' def test_tokenize_negative_numbers(): s = "values(-1)" tokens = list(lexer.tokenize(s)) assert len(tokens) == 4 assert tokens[2][0] == T.Number.Integer assert tokens[2][1] == '-1' def test_token_str(): token = sql.Token(None, 'FoO') assert str(token) == 'FoO' def test_token_repr(): token = sql.Token(T.Keyword, 'foo') tst = "