from __future__ import absolute_import import os import re import unittest import shlex import sys import tempfile import textwrap from io import open from functools import partial from .Compiler import Errors from .CodeWriter import CodeWriter from .Compiler.TreeFragment import TreeFragment, strip_common_indent from .Compiler.Visitor import TreeVisitor, VisitorTransform from .Compiler import TreePath class NodeTypeWriter(TreeVisitor): def __init__(self): super(NodeTypeWriter, self).__init__() self._indents = 0 self.result = [] def visit_Node(self, node): if not self.access_path: name = u"(root)" else: tip = self.access_path[-1] if tip[2] is not None: name = u"%s[%d]" % tip[1:3] else: name = tip[1] self.result.append(u" " * self._indents + u"%s: %s" % (name, node.__class__.__name__)) self._indents += 1 self.visitchildren(node) self._indents -= 1 def treetypes(root): """Returns a string representing the tree by class names. There's a leading and trailing whitespace so that it can be compared by simple string comparison while still making test cases look ok.""" w = NodeTypeWriter() w.visit(root) return u"\n".join([u""] + w.result + [u""]) class CythonTest(unittest.TestCase): def setUp(self): Errors.init_thread() def tearDown(self): Errors.init_thread() def assertLines(self, expected, result): "Checks that the given strings or lists of strings are equal line by line" if not isinstance(expected, list): expected = expected.split(u"\n") if not isinstance(result, list): result = result.split(u"\n") for idx, (expected_line, result_line) in enumerate(zip(expected, result)): self.assertEqual(expected_line, result_line, "Line %d:\nExp: %s\nGot: %s" % (idx, expected_line, result_line)) self.assertEqual(len(expected), len(result), "Unmatched lines. Got:\n%s\nExpected:\n%s" % ("\n".join(expected), u"\n".join(result))) def codeToLines(self, tree): writer = CodeWriter() writer.write(tree) return writer.result.lines def codeToString(self, tree): return "\n".join(self.codeToLines(tree)) def assertCode(self, expected, result_tree): result_lines = self.codeToLines(result_tree) expected_lines = strip_common_indent(expected.split("\n")) for idx, (line, expected_line) in enumerate(zip(result_lines, expected_lines)): self.assertEqual(expected_line, line, "Line %d:\nGot: %s\nExp: %s" % (idx, line, expected_line)) self.assertEqual(len(result_lines), len(expected_lines), "Unmatched lines. Got:\n%s\nExpected:\n%s" % ("\n".join(result_lines), expected)) def assertNodeExists(self, path, result_tree): self.assertNotEqual(TreePath.find_first(result_tree, path), None, "Path '%s' not found in result tree" % path) def fragment(self, code, pxds=None, pipeline=None): "Simply create a tree fragment using the name of the test-case in parse errors." if pxds is None: pxds = {} if pipeline is None: pipeline = [] name = self.id() if name.startswith("__main__."): name = name[len("__main__."):] name = name.replace(".", "_") return TreeFragment(code, name, pxds, pipeline=pipeline) def treetypes(self, root): return treetypes(root) def should_fail(self, func, exc_type=Exception): """Calls "func" and fails if it doesn't raise the right exception (any exception by default). Also returns the exception in question. """ try: func() self.fail("Expected an exception of type %r" % exc_type) except exc_type as e: self.assertTrue(isinstance(e, exc_type)) return e def should_not_fail(self, func): """Calls func and succeeds if and only if no exception is raised (i.e. converts exception raising into a failed testcase). Returns the return value of func.""" try: return func() except Exception as exc: self.fail(str(exc)) class TransformTest(CythonTest): """ Utility base class for transform unit tests. It is based around constructing test trees (either explicitly or by parsing a Cython code string); running the transform, serialize it using a customized Cython serializer (with special markup for nodes that cannot be represented in Cython), and do a string-comparison line-by-line of the result. To create a test case: - Call run_pipeline. The pipeline should at least contain the transform you are testing; pyx should be either a string (passed to the parser to create a post-parse tree) or a node representing input to pipeline. The result will be a transformed result. - Check that the tree is correct. If wanted, assertCode can be used, which takes a code string as expected, and a ModuleNode in result_tree (it serializes the ModuleNode to a string and compares line-by-line). All code strings are first stripped for whitespace lines and then common indentation. Plans: One could have a pxd dictionary parameter to run_pipeline. """ def run_pipeline(self, pipeline, pyx, pxds=None): if pxds is None: pxds = {} tree = self.fragment(pyx, pxds).root # Run pipeline for T in pipeline: tree = T(tree) return tree # For the test C code validation, we have to take care that the test directives (and thus # the match strings) do not just appear in (multiline) C code comments containing the original # Cython source code. Thus, we discard the comments before matching. # This seems a prime case for re.VERBOSE, but it seems to match some of the whitespace. _strip_c_comments = partial(re.compile( re.sub(r'\s+', '', r''' /[*] ( (?: [^*\n] | [*][^/] )* [\n] (?: [^*] | [*][^/] )* ) [*]/ ''') ).sub, '') _strip_cython_code_from_html = partial(re.compile( re.sub(r'\s\s+', '', r''' (?:
(?:[^<]|<(?!/pre))+)|(?: ) ''') ).sub, '') class TreeAssertVisitor(VisitorTransform): # actually, a TreeVisitor would be enough, but this needs to run # as part of the compiler pipeline def __init__(self): super(TreeAssertVisitor, self).__init__() self._module_pos = None self._c_patterns = [] self._c_antipatterns = [] def create_c_file_validator(self): patterns, antipatterns = self._c_patterns, self._c_antipatterns def fail(pos, pattern, found, file_path): Errors.error(pos, "Pattern '%s' %s found in %s" %( pattern, 'was' if found else 'was not', file_path, )) def validate_file_content(file_path, content): for pattern in patterns: #print("Searching pattern '%s'" % pattern) if not re.search(pattern, content): fail(self._module_pos, pattern, found=False, file_path=file_path) for antipattern in antipatterns: #print("Searching antipattern '%s'" % antipattern) if re.search(antipattern, content): fail(self._module_pos, antipattern, found=True, file_path=file_path) def validate_c_file(result): c_file = result.c_file if not (patterns or antipatterns): #print("No patterns defined for %s" % c_file) return result with open(c_file, encoding='utf8') as f: content = f.read() content = _strip_c_comments(content) validate_file_content(c_file, content) html_file = os.path.splitext(c_file)[0] + ".html" if os.path.exists(html_file) and os.path.getmtime(c_file) <= os.path.getmtime(html_file): with open(html_file, encoding='utf8') as f: content = f.read() content = _strip_cython_code_from_html(content) validate_file_content(html_file, content) return validate_c_file def _check_directives(self, node): directives = node.directives if 'test_assert_path_exists' in directives: for path in directives['test_assert_path_exists']: if TreePath.find_first(node, path) is None: Errors.error( node.pos, "Expected path '%s' not found in result tree" % path) if 'test_fail_if_path_exists' in directives: for path in directives['test_fail_if_path_exists']: first_node = TreePath.find_first(node, path) if first_node is not None: Errors.error( first_node.pos, "Unexpected path '%s' found in result tree" % path) if 'test_assert_c_code_has' in directives: self._c_patterns.extend(directives['test_assert_c_code_has']) if 'test_fail_if_c_code_has' in directives: self._c_antipatterns.extend(directives['test_fail_if_c_code_has']) def visit_ModuleNode(self, node): self._module_pos = node.pos self._check_directives(node) self.visitchildren(node) return node def visit_CompilerDirectivesNode(self, node): self._check_directives(node) self.visitchildren(node) return node visit_Node = VisitorTransform.recurse_to_children def unpack_source_tree(tree_file, workdir, cython_root): programs = { 'PYTHON': [sys.executable], 'CYTHON': [sys.executable, os.path.join(cython_root, 'cython.py')], 'CYTHONIZE': [sys.executable, os.path.join(cython_root, 'cythonize.py')] } if workdir is None: workdir = tempfile.mkdtemp() header, cur_file = [], None with open(tree_file, 'rb') as f: try: for line in f: if line[:5] == b'#####': filename = line.strip().strip(b'#').strip().decode('utf8').replace('/', os.path.sep) path = os.path.join(workdir, filename) if not os.path.exists(os.path.dirname(path)): os.makedirs(os.path.dirname(path)) if cur_file is not None: to_close, cur_file = cur_file, None to_close.close() cur_file = open(path, 'wb') elif cur_file is not None: cur_file.write(line) elif line.strip() and not line.lstrip().startswith(b'#'): if line.strip() not in (b'"""', b"'''"): command = shlex.split(line.decode('utf8')) if not command: continue # In Python 3: prog, *args = command prog, args = command[0], command[1:] try: header.append(programs[prog]+args) except KeyError: header.append(command) finally: if cur_file is not None: cur_file.close() return workdir, header def write_file(file_path, content, dedent=False, encoding=None): r"""Write some content (text or bytes) to the file at `file_path` without translating `'\n'` into `os.linesep`. The default encoding is `'utf-8'`. """ if isinstance(content, bytes): mode = "wb" # binary mode doesn't take an encoding and newline arguments newline = None default_encoding = None else: mode = "w" # any "\n" characters written are not translated # to the system default line separator, os.linesep newline = "\n" default_encoding = "utf-8" if encoding is None: encoding = default_encoding if dedent: content = textwrap.dedent(content) with open(file_path, mode=mode, encoding=encoding, newline=newline) as f: f.write(content) def write_newer_file(file_path, newer_than, content, dedent=False, encoding=None): r""" Write `content` to the file `file_path` without translating `'\n'` into `os.linesep` and make sure it is newer than the file `newer_than`. The default encoding is `'utf-8'` (same as for `write_file`). """ write_file(file_path, content, dedent=dedent, encoding=encoding) try: other_time = os.path.getmtime(newer_than) except OSError: # Support writing a fresh file (which is always newer than a non-existant one) other_time = None while other_time is None or other_time >= os.path.getmtime(file_path): write_file(file_path, content, dedent=dedent, encoding=encoding)