1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
|
"""
Generated lexer tests
~~~~~~~~~~~~~~~~~~~~~
Checks that lexers output the expected tokens for each sample
under lexers/*/test_*.txt.
After making a change, rather than updating the samples manually,
run `pytest --update-goldens tests/lexers`.
To add a new sample, create a new file matching this pattern.
The directory must match the alias of the lexer to be used.
Populate only the input, then just `--update-goldens`.
:copyright: Copyright 2021 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
from pathlib import Path
import pytest
import pygments.lexers
def pytest_collect_file(parent, path):
if path.ext == '.txt':
return LexerTestFile.from_parent(parent, fspath=path)
class LexerTestFile(pytest.File):
def collect(self):
yield LexerTestItem.from_parent(self, name='test')
class LexerTestItem(pytest.Item):
def __init__(self, name, parent):
super().__init__(name, parent)
self.lexer = Path(str(self.fspath)).parent.name
content = self.fspath.read_text('utf-8')
content, _, self.expected = content.partition('\n---tokens---\n')
if content.startswith('---input---\n'):
content = '\n' + content
self.comment, _, self.input = content.rpartition('\n---input---\n')
if not self.input.endswith('\n'):
self.input += '\n'
self.comment = self.comment.strip()
@classmethod
def _prettyprint_tokens(cls, tokens):
for tok, val in tokens:
yield '{!r:<13} {}'.format(val, str(tok)[6:])
if val.endswith('\n'):
yield ''
def runtest(self):
lexer = pygments.lexers.get_lexer_by_name(self.lexer)
tokens = lexer.get_tokens(self.input)
self.actual = '\n'.join(self._prettyprint_tokens(tokens)).rstrip('\n') + '\n'
if not self.config.getoption('--update-goldens'):
assert self.actual == self.expected
def _test_file_rel_path(self):
return Path(str(self.fspath)).relative_to(Path(__file__).parent.parent.parent)
def repr_failure(self, excinfo):
if isinstance(excinfo.value, AssertionError):
message = (
'The tokens produced by the "{}" lexer differ from the '
'expected ones in the file "{}".\n'
'Run `pytest tests/lexers --update-goldens` to update it.'
).format(self.lexer, self._test_file_rel_path())
diff = str(excinfo.value).split('\n', 1)[-1]
return message + '\n\n' + diff
def reportinfo(self):
return self.fspath, None, str(self._test_file_rel_path())
def pytest_runtest_teardown(item, nextitem):
if item.config.getoption('--update-goldens') and isinstance(item, LexerTestItem):
with item.fspath.open('w', encoding='utf-8') as f:
f.write(item.comment)
if item.comment:
f.write('\n\n')
f.write('---input---\n')
f.write(item.input)
f.write('\n---tokens---\n')
f.write(item.actual)
|
