diff options
| author | Matthäus G. Chajdas <dev@anteru.net> | 2020-09-08 20:31:17 +0200 |
|---|---|---|
| committer | Matthäus G. Chajdas <dev@anteru.net> | 2020-09-08 20:31:17 +0200 |
| commit | e09d4e0cf23d7c6069ddc690942ceb4cd23fd556 (patch) | |
| tree | c2bb261ee7a0aa28f47a47cd74ed0834ea9b64db | |
| parent | 98f816ae5ca7d98f388ace349a29b154fa9dc9e1 (diff) | |
| download | pygments-git-e09d4e0cf23d7c6069ddc690942ceb4cd23fd556.tar.gz | |
Skip empty tokens in some cases.
This fixes an empty token appearing in the Angular lexer (and
apparently also in the MSDOS lexer.)
| -rw-r--r-- | pygments/lexer.py | 3 | ||||
| -rw-r--r-- | tests/test_shell.py | 1 | ||||
| -rw-r--r-- | tests/test_template.py | 1 |
3 files changed, 3 insertions, 2 deletions
diff --git a/pygments/lexer.py b/pygments/lexer.py index 3f0df88e..62439b10 100644 --- a/pygments/lexer.py +++ b/pygments/lexer.py @@ -805,7 +805,8 @@ def do_insertions(insertions, tokens): except StopIteration: insleft = False break # not strictly necessary - yield realpos, t, v[oldi:] + if oldi < len(v): + yield realpos, t, v[oldi:] realpos += len(v) - oldi # leftover tokens diff --git a/tests/test_shell.py b/tests/test_shell.py index 79f78b3a..79611f16 100644 --- a/tests/test_shell.py +++ b/tests/test_shell.py @@ -162,7 +162,6 @@ def test_msdos_gt_only(lexer_msdos): (Token.Generic.Prompt, u'>'), (Token.Text, u' '), (Token.Text, u'py'), - (Token.Text, u''), (Token.Text, u'\n'), (Token.Generic.Output, u'hi\n'), ] diff --git a/tests/test_template.py b/tests/test_template.py index 48226f62..9ae87171 100644 --- a/tests/test_template.py +++ b/tests/test_template.py @@ -23,5 +23,6 @@ def testAngularFragment(lexer_ng2): (Token.Punctuation, '*'), (Token.Name.Attribute, '39j5Sq'), (Token.Operator, '='), + (Token.Text, '\n') ] assert list(lexer_ng2.get_tokens(fragment)) == tokens
\ No newline at end of file |