From 6ded9db39463372e5205a36bea72d6de516ece69 Mon Sep 17 00:00:00 2001 From: Christian Hammond Date: Fri, 4 Nov 2016 16:57:38 -0700 Subject: Add support for partials and path segments for Handlebars. This introduces support for some missing features to the Handlebars lexer: Partials and path segments. Partials mostly appeared to work before, but the `>` in `{{> ... }}` would appear as a syntax error, as could other components of the partial. This change introduces support for: * Standard partials: `{{> partialName}}` * Partials with parameters: `{{> partialName varname="value"}}` * Ddynamic partials: `{{> (partialFunc)}}` * Ddynamic partials with lookups: `{{> (lookup ../path "partialName")}}` * Partial blocks: `{{> @partial-block}}` * Inline partials: `{{#*inline}}..{{/inline}}` It also introduces support for path segments, which can reference content in the current context or in a parent context. For instance, `this.name`, `this/name`, `./name`, `../name`, `this/name`, etc. These are all now tracked as variables. --- tests/test_clexer.py | 259 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 259 insertions(+) create mode 100644 tests/test_clexer.py (limited to 'tests/test_clexer.py') diff --git a/tests/test_clexer.py b/tests/test_clexer.py new file mode 100644 index 00000000..fd7f58fc --- /dev/null +++ b/tests/test_clexer.py @@ -0,0 +1,259 @@ +# -*- coding: utf-8 -*- +""" + Basic CLexer Test + ~~~~~~~~~~~~~~~~~ + + :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import unittest +import os +import textwrap + +from pygments.token import Text, Number, Token +from pygments.lexers import CLexer + + +class CLexerTest(unittest.TestCase): + + def setUp(self): + self.lexer = CLexer() + + def testNumbers(self): + code = '42 23.42 23. .42 023 0xdeadbeef 23e+42 42e-23' + wanted = [] + for item in zip([Number.Integer, Number.Float, Number.Float, + Number.Float, Number.Oct, Number.Hex, + Number.Float, Number.Float], code.split()): + wanted.append(item) + wanted.append((Text, ' ')) + wanted = wanted[:-1] + [(Text, '\n')] + self.assertEqual(list(self.lexer.get_tokens(code)), wanted) + + def testSwitch(self): + fragment = u'''\ + int main() + { + switch (0) + { + case 0: + default: + ; + } + } + ''' + tokens = [ + (Token.Keyword.Type, u'int'), + (Token.Text, u' '), + (Token.Name.Function, u'main'), + (Token.Punctuation, u'('), + (Token.Punctuation, u')'), + (Token.Text, u'\n'), + (Token.Punctuation, u'{'), + (Token.Text, u'\n'), + (Token.Text, u' '), + (Token.Keyword, u'switch'), + (Token.Text, u' '), + (Token.Punctuation, u'('), + (Token.Literal.Number.Integer, u'0'), + (Token.Punctuation, u')'), + (Token.Text, u'\n'), + (Token.Text, u' '), + (Token.Punctuation, u'{'), + (Token.Text, u'\n'), + (Token.Text, u' '), + (Token.Keyword, u'case'), + (Token.Text, u' '), + (Token.Literal.Number.Integer, u'0'), + (Token.Operator, u':'), + (Token.Text, u'\n'), + (Token.Text, u' '), + (Token.Keyword, u'default'), + (Token.Operator, u':'), + (Token.Text, u'\n'), + (Token.Text, u' '), + (Token.Punctuation, u';'), + (Token.Text, u'\n'), + (Token.Text, u' '), + (Token.Punctuation, u'}'), + (Token.Text, u'\n'), + (Token.Punctuation, u'}'), + (Token.Text, u'\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(textwrap.dedent(fragment)))) + + def testSwitchSpaceBeforeColon(self): + fragment = u'''\ + int main() + { + switch (0) + { + case 0 : + default : + ; + } + } + ''' + tokens = [ + (Token.Keyword.Type, u'int'), + (Token.Text, u' '), + (Token.Name.Function, u'main'), + (Token.Punctuation, u'('), + (Token.Punctuation, u')'), + (Token.Text, u'\n'), + (Token.Punctuation, u'{'), + (Token.Text, u'\n'), + (Token.Text, u' '), + (Token.Keyword, u'switch'), + (Token.Text, u' '), + (Token.Punctuation, u'('), + (Token.Literal.Number.Integer, u'0'), + (Token.Punctuation, u')'), + (Token.Text, u'\n'), + (Token.Text, u' '), + (Token.Punctuation, u'{'), + (Token.Text, u'\n'), + (Token.Text, u' '), + (Token.Keyword, u'case'), + (Token.Text, u' '), + (Token.Literal.Number.Integer, u'0'), + (Token.Text, u' '), + (Token.Operator, u':'), + (Token.Text, u'\n'), + (Token.Text, u' '), + (Token.Keyword, u'default'), + (Token.Text, u' '), + (Token.Operator, u':'), + (Token.Text, u'\n'), + (Token.Text, u' '), + (Token.Punctuation, u';'), + (Token.Text, u'\n'), + (Token.Text, u' '), + (Token.Punctuation, u'}'), + (Token.Text, u'\n'), + (Token.Punctuation, u'}'), + (Token.Text, u'\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(textwrap.dedent(fragment)))) + + def testLabel(self): + fragment = u'''\ + int main() + { + foo: + goto foo; + } + ''' + tokens = [ + (Token.Keyword.Type, u'int'), + (Token.Text, u' '), + (Token.Name.Function, u'main'), + (Token.Punctuation, u'('), + (Token.Punctuation, u')'), + (Token.Text, u'\n'), + (Token.Punctuation, u'{'), + (Token.Text, u'\n'), + (Token.Name.Label, u'foo'), + (Token.Punctuation, u':'), + (Token.Text, u'\n'), + (Token.Text, u' '), + (Token.Keyword, u'goto'), + (Token.Text, u' '), + (Token.Name, u'foo'), + (Token.Punctuation, u';'), + (Token.Text, u'\n'), + (Token.Punctuation, u'}'), + (Token.Text, u'\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(textwrap.dedent(fragment)))) + + def testLabelSpaceBeforeColon(self): + fragment = u'''\ + int main() + { + foo : + goto foo; + } + ''' + tokens = [ + (Token.Keyword.Type, u'int'), + (Token.Text, u' '), + (Token.Name.Function, u'main'), + (Token.Punctuation, u'('), + (Token.Punctuation, u')'), + (Token.Text, u'\n'), + (Token.Punctuation, u'{'), + (Token.Text, u'\n'), + (Token.Name.Label, u'foo'), + (Token.Text, u' '), + (Token.Punctuation, u':'), + (Token.Text, u'\n'), + (Token.Text, u' '), + (Token.Keyword, u'goto'), + (Token.Text, u' '), + (Token.Name, u'foo'), + (Token.Punctuation, u';'), + (Token.Text, u'\n'), + (Token.Punctuation, u'}'), + (Token.Text, u'\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(textwrap.dedent(fragment)))) + + def testLabelFollowedByStatement(self): + fragment = u'''\ + int main() + { + foo:return 0; + goto foo; + } + ''' + tokens = [ + (Token.Keyword.Type, u'int'), + (Token.Text, u' '), + (Token.Name.Function, u'main'), + (Token.Punctuation, u'('), + (Token.Punctuation, u')'), + (Token.Text, u'\n'), + (Token.Punctuation, u'{'), + (Token.Text, u'\n'), + (Token.Name.Label, u'foo'), + (Token.Punctuation, u':'), + (Token.Keyword, u'return'), + (Token.Text, u' '), + (Token.Literal.Number.Integer, u'0'), + (Token.Punctuation, u';'), + (Token.Text, u'\n'), + (Token.Text, u' '), + (Token.Keyword, u'goto'), + (Token.Text, u' '), + (Token.Name, u'foo'), + (Token.Punctuation, u';'), + (Token.Text, u'\n'), + (Token.Punctuation, u'}'), + (Token.Text, u'\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(textwrap.dedent(fragment)))) + + def testPreprocFile(self): + fragment = u'#include \n' + tokens = [ + (Token.Comment.Preproc, u'#'), + (Token.Comment.Preproc, u'include'), + (Token.Text, u' '), + (Token.Comment.PreprocFile, u''), + (Token.Comment.Preproc, u'\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testPreprocFile2(self): + fragment = u'#include "foo.h"\n' + tokens = [ + (Token.Comment.Preproc, u'#'), + (Token.Comment.Preproc, u'include'), + (Token.Text, u' '), + (Token.Comment.PreprocFile, u'"foo.h"'), + (Token.Comment.Preproc, u'\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + -- cgit v1.2.1