From 50c6fa204286cb1f8b7b7b075dbeaa9c48b5d90d Mon Sep 17 00:00:00 2001 From: Thomas Aglassinger Date: Mon, 21 Mar 2016 08:29:44 +0100 Subject: Added lexer for Transact-SQL as used by Microsoft SQL Server and Sybase. --- tests/test_sql.py | 76 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 tests/test_sql.py (limited to 'tests/test_sql.py') diff --git a/tests/test_sql.py b/tests/test_sql.py new file mode 100644 index 00000000..37a81ff8 --- /dev/null +++ b/tests/test_sql.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- +""" + Pygments SQL lexers tests + ~~~~~~~~~~~~~~~~~~~~~~~~~ + + :copyright: Copyright 2006-2016 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" +import io +import os.path +import unittest + +from pygments.lexers.sql import TransactSqlLexer +from pygments.token import Comment, Error, Name, Number, Whitespace + + +class TransactSqlLexerTest(unittest.TestCase): + + def setUp(self): + self.lexer = TransactSqlLexer() + + def _assertAreTokensOfType(self, examples, expected_token_type): + for test_number, example in enumerate(examples.split(), 1): + token_count = 0 + for token_type, token_value in self.lexer.get_tokens(example): + if token_type != Whitespace: + token_count += 1 + self.assertEqual( + token_type, expected_token_type, + 'token_type #%d for %s is be %s but must be %s' % + (test_number, token_value, token_type, expected_token_type)) + self.assertEqual( + token_count, 1, + '%s must yield exactly 1 token instead of %d' % + (example, token_count)) + + def _assertTokensMatch(self, text, expected_tokens_without_trailing_newline): + actual_tokens = tuple(self.lexer.get_tokens(text)) + if (len(actual_tokens) >= 1) and (actual_tokens[-1] == (Whitespace, '\n')): + actual_tokens = tuple(actual_tokens[:-1]) + self.assertEqual( + expected_tokens_without_trailing_newline, actual_tokens, + 'text must yield expected tokens: %s' % text) + + def test_can_lex_float(self): + self._assertAreTokensOfType( + '1.2 1.2e3 1.2e+3 1.2e-3 1e2', Number.Float) + self._assertTokensMatch( + '1e2.1e2', + ((Number.Float, '1e2'), (Number.Float, '.1e2')) + ) + + def test_can_lex_names(self): + self._assertAreTokensOfType( + u'thingy thingy123 _thingy _ _123 Ähnliches Müll #temp1 ##temp2', Name) + + def test_can_lex_comments(self): + self._assertTokensMatch('--\n', ((Comment.Single, '--\n'),)) + self._assertTokensMatch('/**/', ( + (Comment.Multiline, '/*'), (Comment.Multiline, '*/') + )) + self._assertTokensMatch('/*/**/*/', ( + (Comment.Multiline, '/*'), + (Comment.Multiline, '/*'), + (Comment.Multiline, '*/'), + (Comment.Multiline, '*/'), + )) + + def test_can_lex_example_file(self): + tests_path = os.path.dirname(__file__) + example_path = os.path.join(tests_path, 'examplefiles', 'test_transact-sql.txt') + + with io.open(example_path, 'r', encoding='utf-8') as example_file: + example_code = example_file.read() + for token_type, token_value in self.lexer.get_tokens(example_code): + self.assertNotEqual(Error, token_type, 'token_value=%r' % token_value) -- cgit v1.2.1 From b82317034f4c3c8912beb3a5f5b16ace6c36c201 Mon Sep 17 00:00:00 2001 From: Thomas Aglassinger Date: Tue, 22 Mar 2016 22:52:18 +0100 Subject: Fixed broken test case by properly naming Transact-SQL example file to prevent ResourceLexer from attempting to lex it. --- tests/test_sql.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) (limited to 'tests/test_sql.py') diff --git a/tests/test_sql.py b/tests/test_sql.py index 37a81ff8..f4016918 100644 --- a/tests/test_sql.py +++ b/tests/test_sql.py @@ -6,12 +6,10 @@ :copyright: Copyright 2006-2016 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ -import io -import os.path import unittest from pygments.lexers.sql import TransactSqlLexer -from pygments.token import Comment, Error, Name, Number, Whitespace +from pygments.token import Comment, Name, Number, Whitespace class TransactSqlLexerTest(unittest.TestCase): @@ -65,12 +63,3 @@ class TransactSqlLexerTest(unittest.TestCase): (Comment.Multiline, '*/'), (Comment.Multiline, '*/'), )) - - def test_can_lex_example_file(self): - tests_path = os.path.dirname(__file__) - example_path = os.path.join(tests_path, 'examplefiles', 'test_transact-sql.txt') - - with io.open(example_path, 'r', encoding='utf-8') as example_file: - example_code = example_file.read() - for token_type, token_value in self.lexer.get_tokens(example_code): - self.assertNotEqual(Error, token_type, 'token_value=%r' % token_value) -- cgit v1.2.1 From b9fd619bbc2855c6a4b447cf4f2d5626bef0f034 Mon Sep 17 00:00:00 2001 From: Thomas Aglassinger Date: Wed, 23 Mar 2016 00:25:43 +0100 Subject: Fixed that regex to detect Number.Float shadowed regex for Number.Integer for Transact-SQL. --- tests/test_sql.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'tests/test_sql.py') diff --git a/tests/test_sql.py b/tests/test_sql.py index f4016918..674d0426 100644 --- a/tests/test_sql.py +++ b/tests/test_sql.py @@ -48,6 +48,10 @@ class TransactSqlLexerTest(unittest.TestCase): ((Number.Float, '1e2'), (Number.Float, '.1e2')) ) + def test_can_lex_integer(self): + self._assertAreTokensOfType( + '1 23 456', Number.Integer) + def test_can_lex_names(self): self._assertAreTokensOfType( u'thingy thingy123 _thingy _ _123 Ähnliches Müll #temp1 ##temp2', Name) -- cgit v1.2.1 From 7f0eff568e557dd55b218abea6d44ed66794df6e Mon Sep 17 00:00:00 2001 From: Thomas Aglassinger Date: Thu, 24 Mar 2016 00:41:59 +0100 Subject: Fixed detection of Number.Float in Transact-SQL for corner cases like "1." and "1.e2." --- tests/test_sql.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'tests/test_sql.py') diff --git a/tests/test_sql.py b/tests/test_sql.py index 674d0426..c5f5c758 100644 --- a/tests/test_sql.py +++ b/tests/test_sql.py @@ -9,7 +9,7 @@ import unittest from pygments.lexers.sql import TransactSqlLexer -from pygments.token import Comment, Name, Number, Whitespace +from pygments.token import Comment, Name, Number, Punctuation, Whitespace class TransactSqlLexerTest(unittest.TestCase): @@ -42,12 +42,17 @@ class TransactSqlLexerTest(unittest.TestCase): def test_can_lex_float(self): self._assertAreTokensOfType( - '1.2 1.2e3 1.2e+3 1.2e-3 1e2', Number.Float) + '1. 1.e1 .1 1.2 1.2e3 1.2e+3 1.2e-3 1e2', Number.Float) self._assertTokensMatch( '1e2.1e2', ((Number.Float, '1e2'), (Number.Float, '.1e2')) ) + def test_can_reject_almost_float(self): + self._assertTokensMatch( + '.e1', + ((Punctuation, '.'), (Name, 'e1'))) + def test_can_lex_integer(self): self._assertAreTokensOfType( '1 23 456', Number.Integer) -- cgit v1.2.1