diff options
Diffstat (limited to 'Lib')
| -rw-r--r-- | Lib/codeop.py | 3 | ||||
| -rw-r--r-- | Lib/compileall.py | 42 | ||||
| -rw-r--r-- | Lib/email/contentmanager.py | 14 | ||||
| -rw-r--r-- | Lib/gettext.py | 8 | ||||
| -rw-r--r-- | Lib/idlelib/zzdummy.py | 2 | ||||
| -rw-r--r-- | Lib/linecache.py | 6 | ||||
| -rw-r--r-- | Lib/test/test__xxsubinterpreters.py | 301 | ||||
| -rw-r--r-- | Lib/test/test_asyncio/test_tasks.py | 27 | ||||
| -rw-r--r-- | Lib/test/test_compileall.py | 224 | ||||
| -rw-r--r-- | Lib/test/test_dictcomps.py | 2 | ||||
| -rw-r--r-- | Lib/test/test_email/test_contentmanager.py | 15 | ||||
| -rw-r--r-- | Lib/test/test_exceptions.py | 2 | ||||
| -rw-r--r-- | Lib/test/test_generators.py | 21 | ||||
| -rw-r--r-- | Lib/test/test_genexps.py | 2 | ||||
| -rw-r--r-- | Lib/test/test_grammar.py | 5 | ||||
| -rw-r--r-- | Lib/test/test_json/test_recursion.py | 2 | ||||
| -rw-r--r-- | Lib/test/test_peg_parser.py | 27 | ||||
| -rw-r--r-- | Lib/test/test_syntax.py | 52 |
18 files changed, 410 insertions, 345 deletions
diff --git a/Lib/codeop.py b/Lib/codeop.py index 082285f94f..835e68c09b 100644 --- a/Lib/codeop.py +++ b/Lib/codeop.py @@ -112,7 +112,8 @@ def compile_command(source, filename="<input>", symbol="single"): source -- the source string; may contain \n characters filename -- optional filename from which source was read; default "<input>" - symbol -- optional grammar start symbol; "single" (default) or "eval" + symbol -- optional grammar start symbol; "single" (default), "exec" + or "eval" Return value / exceptions raised: diff --git a/Lib/compileall.py b/Lib/compileall.py index abe6cffce5..fe7f450c55 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -15,6 +15,7 @@ import sys import importlib.util import py_compile import struct +import filecmp from functools import partial from pathlib import Path @@ -47,7 +48,7 @@ def _walk_dir(dir, maxlevels, quiet=0): def compile_dir(dir, maxlevels=None, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, workers=1, invalidation_mode=None, *, stripdir=None, - prependdir=None, limit_sl_dest=None): + prependdir=None, limit_sl_dest=None, hardlink_dupes=False): """Byte-compile all modules in the given directory tree. Arguments (only dir is required): @@ -70,6 +71,7 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False, after stripdir limit_sl_dest: ignore symlinks if they are pointing outside of the defined path + hardlink_dupes: hardlink duplicated pyc files """ ProcessPoolExecutor = None if ddir is not None and (stripdir is not None or prependdir is not None): @@ -104,7 +106,8 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False, invalidation_mode=invalidation_mode, stripdir=stripdir, prependdir=prependdir, - limit_sl_dest=limit_sl_dest), + limit_sl_dest=limit_sl_dest, + hardlink_dupes=hardlink_dupes), files) success = min(results, default=True) else: @@ -112,14 +115,15 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False, if not compile_file(file, ddir, force, rx, quiet, legacy, optimize, invalidation_mode, stripdir=stripdir, prependdir=prependdir, - limit_sl_dest=limit_sl_dest): + limit_sl_dest=limit_sl_dest, + hardlink_dupes=hardlink_dupes): success = False return success def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, invalidation_mode=None, *, stripdir=None, prependdir=None, - limit_sl_dest=None): + limit_sl_dest=None, hardlink_dupes=False): """Byte-compile one file. Arguments (only fullname is required): @@ -140,6 +144,7 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, after stripdir limit_sl_dest: ignore symlinks if they are pointing outside of the defined path. + hardlink_dupes: hardlink duplicated pyc files """ if ddir is not None and (stripdir is not None or prependdir is not None): @@ -176,6 +181,14 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, if isinstance(optimize, int): optimize = [optimize] + # Use set() to remove duplicates. + # Use sorted() to create pyc files in a deterministic order. + optimize = sorted(set(optimize)) + + if hardlink_dupes and len(optimize) < 2: + raise ValueError("Hardlinking of duplicated bytecode makes sense " + "only for more than one optimization level") + if rx is not None: mo = rx.search(fullname) if mo: @@ -220,10 +233,16 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, if not quiet: print('Compiling {!r}...'.format(fullname)) try: - for opt_level, cfile in opt_cfiles.items(): + for index, opt_level in enumerate(optimize): + cfile = opt_cfiles[opt_level] ok = py_compile.compile(fullname, cfile, dfile, True, optimize=opt_level, invalidation_mode=invalidation_mode) + if index > 0 and hardlink_dupes: + previous_cfile = opt_cfiles[optimize[index - 1]] + if filecmp.cmp(cfile, previous_cfile, shallow=False): + os.unlink(cfile) + os.link(previous_cfile, cfile) except py_compile.PyCompileError as err: success = False if quiet >= 2: @@ -352,6 +371,9 @@ def main(): 'Python interpreter itself (specified by -O).')) parser.add_argument('-e', metavar='DIR', dest='limit_sl_dest', help='Ignore symlinks pointing outsite of the DIR') + parser.add_argument('--hardlink-dupes', action='store_true', + dest='hardlink_dupes', + help='Hardlink duplicated pyc files') args = parser.parse_args() compile_dests = args.compile_dest @@ -371,6 +393,10 @@ def main(): if args.opt_levels is None: args.opt_levels = [-1] + if len(args.opt_levels) == 1 and args.hardlink_dupes: + parser.error(("Hardlinking of duplicated bytecode makes sense " + "only for more than one optimization level.")) + if args.ddir is not None and ( args.stripdir is not None or args.prependdir is not None ): @@ -404,7 +430,8 @@ def main(): stripdir=args.stripdir, prependdir=args.prependdir, optimize=args.opt_levels, - limit_sl_dest=args.limit_sl_dest): + limit_sl_dest=args.limit_sl_dest, + hardlink_dupes=args.hardlink_dupes): success = False else: if not compile_dir(dest, maxlevels, args.ddir, @@ -414,7 +441,8 @@ def main(): stripdir=args.stripdir, prependdir=args.prependdir, optimize=args.opt_levels, - limit_sl_dest=args.limit_sl_dest): + limit_sl_dest=args.limit_sl_dest, + hardlink_dupes=args.hardlink_dupes): success = False return success else: diff --git a/Lib/email/contentmanager.py b/Lib/email/contentmanager.py index b904ded94c..2b4b8757f4 100644 --- a/Lib/email/contentmanager.py +++ b/Lib/email/contentmanager.py @@ -146,13 +146,13 @@ def _encode_text(string, charset, cte, policy): def normal_body(lines): return b'\n'.join(lines) + b'\n' if cte==None: # Use heuristics to decide on the "best" encoding. - try: - return '7bit', normal_body(lines).decode('ascii') - except UnicodeDecodeError: - pass - if (policy.cte_type == '8bit' and - max(len(x) for x in lines) <= policy.max_line_length): - return '8bit', normal_body(lines).decode('ascii', 'surrogateescape') + if max(len(x) for x in lines) <= policy.max_line_length: + try: + return '7bit', normal_body(lines).decode('ascii') + except UnicodeDecodeError: + pass + if policy.cte_type == '8bit': + return '8bit', normal_body(lines).decode('ascii', 'surrogateescape') sniff = embedded_body(lines[:10]) sniff_qp = quoprimime.body_encode(sniff.decode('latin-1'), policy.max_line_length) diff --git a/Lib/gettext.py b/Lib/gettext.py index b98f501884..77b67aef42 100644 --- a/Lib/gettext.py +++ b/Lib/gettext.py @@ -46,7 +46,6 @@ internationalized, to the local language and cultural habits. # find this format documented anywhere. -import locale import os import re import sys @@ -210,6 +209,7 @@ def c2py(plural): def _expand_lang(loc): + import locale loc = locale.normalize(loc) COMPONENT_CODESET = 1 << 0 COMPONENT_TERRITORY = 1 << 1 @@ -278,6 +278,7 @@ class NullTranslations: import warnings warnings.warn('lgettext() is deprecated, use gettext() instead', DeprecationWarning, 2) + import locale if self._fallback: with warnings.catch_warnings(): warnings.filterwarnings('ignore', r'.*\blgettext\b.*', @@ -299,6 +300,7 @@ class NullTranslations: import warnings warnings.warn('lngettext() is deprecated, use ngettext() instead', DeprecationWarning, 2) + import locale if self._fallback: with warnings.catch_warnings(): warnings.filterwarnings('ignore', r'.*\blngettext\b.*', @@ -462,6 +464,7 @@ class GNUTranslations(NullTranslations): import warnings warnings.warn('lgettext() is deprecated, use gettext() instead', DeprecationWarning, 2) + import locale missing = object() tmsg = self._catalog.get(message, missing) if tmsg is missing: @@ -476,6 +479,7 @@ class GNUTranslations(NullTranslations): import warnings warnings.warn('lngettext() is deprecated, use ngettext() instead', DeprecationWarning, 2) + import locale try: tmsg = self._catalog[(msgid1, self.plural(n))] except KeyError: @@ -668,6 +672,7 @@ def ldgettext(domain, message): import warnings warnings.warn('ldgettext() is deprecated, use dgettext() instead', DeprecationWarning, 2) + import locale codeset = _localecodesets.get(domain) try: with warnings.catch_warnings(): @@ -695,6 +700,7 @@ def ldngettext(domain, msgid1, msgid2, n): import warnings warnings.warn('ldngettext() is deprecated, use dngettext() instead', DeprecationWarning, 2) + import locale codeset = _localecodesets.get(domain) try: with warnings.catch_warnings(): diff --git a/Lib/idlelib/zzdummy.py b/Lib/idlelib/zzdummy.py index 8084499646..3c4b1d23b0 100644 --- a/Lib/idlelib/zzdummy.py +++ b/Lib/idlelib/zzdummy.py @@ -28,7 +28,7 @@ class ZzDummy: text = self.text text.undo_block_start() for line in range(1, text.index('end')): - text.insert('%d.0', ztest) + text.insert('%d.0', ztext) text.undo_block_stop() return "break" diff --git a/Lib/linecache.py b/Lib/linecache.py index ddd0abf2cf..fa5dbd09ea 100644 --- a/Lib/linecache.py +++ b/Lib/linecache.py @@ -71,10 +71,10 @@ def checkcache(filename=None): try: stat = os.stat(fullname) except OSError: - del cache[filename] + cache.pop(filename, None) continue if size != stat.st_size or mtime != stat.st_mtime: - del cache[filename] + cache.pop(filename, None) def updatecache(filename, module_globals=None): @@ -84,7 +84,7 @@ def updatecache(filename, module_globals=None): if filename in cache: if len(cache[filename]) != 1: - del cache[filename] + cache.pop(filename, None) if not filename or (filename.startswith('<') and filename.endswith('>')): return [] diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index 039c040ad3..e17bfde2c2 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -1,4 +1,3 @@ -import builtins from collections import namedtuple import contextlib import itertools @@ -867,11 +866,10 @@ class RunStringTests(TestBase): yield if msg is None: self.assertEqual(str(caught.exception).split(':')[0], - exctype.__name__) + str(exctype)) else: self.assertEqual(str(caught.exception), - "{}: {}".format(exctype.__name__, msg)) - self.assertIsInstance(caught.exception.__cause__, exctype) + "{}: {}".format(exctype, msg)) def test_invalid_syntax(self): with self.assert_run_failed(SyntaxError): @@ -1062,301 +1060,6 @@ class RunStringTests(TestBase): self.assertEqual(retcode, 0) -def build_exception(exctype, /, *args, **kwargs): - # XXX Use __qualname__? - name = exctype.__name__ - argreprs = [repr(a) for a in args] - if kwargs: - kwargreprs = [f'{k}={v!r}' for k, v in kwargs.items()] - script = f'{name}({", ".join(argreprs)}, {", ".join(kwargreprs)})' - else: - script = f'{name}({", ".join(argreprs)})' - expected = exctype(*args, **kwargs) - return script, expected - - -def build_exceptions(self, *exctypes, default=None, custom=None, bases=True): - if not exctypes: - raise NotImplementedError - if not default: - default = ((), {}) - elif isinstance(default, str): - default = ((default,), {}) - elif type(default) is not tuple: - raise NotImplementedError - elif len(default) != 2: - default = (default, {}) - elif type(default[0]) is not tuple: - default = (default, {}) - elif type(default[1]) is not dict: - default = (default, {}) - # else leave it alone - - for exctype in exctypes: - customtype = None - values = default - if custom: - if exctype in custom: - customtype = exctype - elif bases: - for customtype in custom: - if issubclass(exctype, customtype): - break - else: - customtype = None - if customtype is not None: - values = custom[customtype] - if values is None: - continue - args, kwargs = values - script, expected = build_exception(exctype, *args, **kwargs) - yield exctype, customtype, script, expected - - -try: - raise Exception -except Exception as exc: - assert exc.__traceback__ is not None - Traceback = type(exc.__traceback__) - - -class RunFailedTests(TestBase): - - BUILTINS = [v - for v in vars(builtins).values() - if (type(v) is type - and issubclass(v, Exception) - #and issubclass(v, BaseException) - ) - ] - BUILTINS_SPECIAL = [ - # These all have extra attributes (i.e. args/kwargs) - SyntaxError, - ImportError, - UnicodeError, - OSError, - SystemExit, - StopIteration, - ] - - @classmethod - def build_exceptions(cls, exctypes=None, default=(), custom=None): - if exctypes is None: - exctypes = cls.BUILTINS - if custom is None: - # Skip the "special" ones. - custom = {et: None for et in cls.BUILTINS_SPECIAL} - yield from build_exceptions(*exctypes, default=default, custom=custom) - - def assertExceptionsEqual(self, exc, expected, *, chained=True): - if type(expected) is type: - self.assertIs(type(exc), expected) - return - elif not isinstance(exc, Exception): - self.assertEqual(exc, expected) - elif not isinstance(expected, Exception): - self.assertEqual(exc, expected) - else: - # Plain equality doesn't work, so we have to compare manually. - self.assertIs(type(exc), type(expected)) - self.assertEqual(exc.args, expected.args) - self.assertEqual(exc.__reduce__(), expected.__reduce__()) - if chained: - self.assertExceptionsEqual(exc.__context__, - expected.__context__) - self.assertExceptionsEqual(exc.__cause__, - expected.__cause__) - self.assertEqual(exc.__suppress_context__, - expected.__suppress_context__) - - def assertTracebacksEqual(self, tb, expected): - if not isinstance(tb, Traceback): - self.assertEqual(tb, expected) - elif not isinstance(expected, Traceback): - self.assertEqual(tb, expected) - else: - self.assertEqual(tb.tb_frame.f_code.co_name, - expected.tb_frame.f_code.co_name) - self.assertEqual(tb.tb_frame.f_code.co_filename, - expected.tb_frame.f_code.co_filename) - self.assertEqual(tb.tb_lineno, expected.tb_lineno) - self.assertTracebacksEqual(tb.tb_next, expected.tb_next) - - # XXX Move this to TestBase? - @contextlib.contextmanager - def expected_run_failure(self, expected): - exctype = expected if type(expected) is type else type(expected) - - with self.assertRaises(interpreters.RunFailedError) as caught: - yield caught - exc = caught.exception - - modname = exctype.__module__ - if modname == 'builtins' or modname == '__main__': - exctypename = exctype.__name__ - else: - exctypename = f'{modname}.{exctype.__name__}' - if exctype is expected: - self.assertEqual(str(exc).split(':')[0], exctypename) - else: - self.assertEqual(str(exc), f'{exctypename}: {expected}') - self.assertExceptionsEqual(exc.__cause__, expected) - if exc.__cause__ is not None: - self.assertIsNotNone(exc.__cause__.__traceback__) - - def test_builtin_exceptions(self): - interpid = interpreters.create() - msg = '<a message>' - for i, info in enumerate(self.build_exceptions( - default=msg, - custom={ - SyntaxError: ((msg, '<stdin>', 1, 3, 'a +?'), {}), - ImportError: ((msg,), {'name': 'spam', 'path': '/x/spam.py'}), - UnicodeError: None, - #UnicodeError: ((), {}), - #OSError: ((), {}), - SystemExit: ((1,), {}), - StopIteration: (('<a value>',), {}), - }, - )): - exctype, _, script, expected = info - testname = f'{i+1} - {script}' - script = f'raise {script}' - - with self.subTest(testname): - with self.expected_run_failure(expected): - interpreters.run_string(interpid, script) - - def test_custom_exception_from___main__(self): - script = dedent(""" - class SpamError(Exception): - def __init__(self, q): - super().__init__(f'got {q}') - self.q = q - raise SpamError('eggs') - """) - expected = Exception(f'SpamError: got {"eggs"}') - - interpid = interpreters.create() - with self.assertRaises(interpreters.RunFailedError) as caught: - interpreters.run_string(interpid, script) - cause = caught.exception.__cause__ - - self.assertExceptionsEqual(cause, expected) - - class SpamError(Exception): - # The normal Exception.__reduce__() produces a funny result - # here. So we have to use a custom __new__(). - def __new__(cls, q): - if type(q) is SpamError: - return q - return super().__new__(cls, q) - def __init__(self, q): - super().__init__(f'got {q}') - self.q = q - - def test_custom_exception(self): - script = dedent(""" - import test.test__xxsubinterpreters - SpamError = test.test__xxsubinterpreters.RunFailedTests.SpamError - raise SpamError('eggs') - """) - try: - ns = {} - exec(script, ns, ns) - except Exception as exc: - expected = exc - - interpid = interpreters.create() - with self.expected_run_failure(expected): - interpreters.run_string(interpid, script) - - class SpamReducedError(Exception): - def __init__(self, q): - super().__init__(f'got {q}') - self.q = q - def __reduce__(self): - return (type(self), (self.q,), {}) - - def test_custom___reduce__(self): - script = dedent(""" - import test.test__xxsubinterpreters - SpamError = test.test__xxsubinterpreters.RunFailedTests.SpamReducedError - raise SpamError('eggs') - """) - try: - exec(script, (ns := {'__name__': '__main__'}), ns) - except Exception as exc: - expected = exc - - interpid = interpreters.create() - with self.expected_run_failure(expected): - interpreters.run_string(interpid, script) - - def test_traceback_propagated(self): - script = dedent(""" - def do_spam(): - raise Exception('uh-oh') - def do_eggs(): - return do_spam() - class Spam: - def do(self): - return do_eggs() - def get_handler(): - def handler(): - return Spam().do() - return handler - go = (lambda: get_handler()()) - def iter_all(): - yield from (go() for _ in [True]) - yield None - def main(): - for v in iter_all(): - pass - main() - """) - try: - ns = {} - exec(script, ns, ns) - except Exception as exc: - expected = exc - expectedtb = exc.__traceback__.tb_next - - interpid = interpreters.create() - with self.expected_run_failure(expected) as caught: - interpreters.run_string(interpid, script) - exc = caught.exception - - self.assertTracebacksEqual(exc.__cause__.__traceback__, - expectedtb) - - def test_chained_exceptions(self): - script = dedent(""" - try: - raise ValueError('msg 1') - except Exception as exc1: - try: - raise TypeError('msg 2') - except Exception as exc2: - try: - raise IndexError('msg 3') from exc2 - except Exception: - raise AttributeError('msg 4') - """) - try: - exec(script, {}, {}) - except Exception as exc: - expected = exc - - interpid = interpreters.create() - with self.expected_run_failure(expected) as caught: - interpreters.run_string(interpid, script) - exc = caught.exception - - # ...just to be sure. - self.assertIs(type(exc.__cause__), AttributeError) - - ################################## # channel tests diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 68f3b8cce9..6eb6b46ec8 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -466,6 +466,33 @@ class BaseTaskTests: t = outer() self.assertEqual(self.loop.run_until_complete(t), 1042) + def test_exception_chaining_after_await(self): + # Test that when awaiting on a task when an exception is already + # active, if the task raises an exception it will be chained + # with the original. + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + + async def raise_error(): + raise ValueError + + async def run(): + try: + raise KeyError(3) + except Exception as exc: + task = self.new_task(loop, raise_error()) + try: + await task + except Exception as exc: + self.assertEqual(type(exc), ValueError) + chained = exc.__context__ + self.assertEqual((type(chained), chained.args), + (KeyError, (3,))) + + task = self.new_task(loop, run()) + loop.run_until_complete(task) + loop.close() + def test_cancel(self): def gen(): diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index 7267894508..b4061b7935 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -1,16 +1,19 @@ -import sys import compileall +import contextlib +import filecmp import importlib.util -import test.test_importlib.util +import io +import itertools import os import pathlib import py_compile import shutil import struct +import sys import tempfile +import test.test_importlib.util import time import unittest -import io from unittest import mock, skipUnless try: @@ -26,6 +29,24 @@ from .test_py_compile import without_source_date_epoch from .test_py_compile import SourceDateEpochTestMeta +def get_pyc(script, opt): + if not opt: + # Replace None and 0 with '' + opt = '' + return importlib.util.cache_from_source(script, optimization=opt) + + +def get_pycs(script): + return [get_pyc(script, opt) for opt in (0, 1, 2)] + + +def is_hardlink(filename1, filename2): + """Returns True if two files have the same inode (hardlink)""" + inode1 = os.stat(filename1).st_ino + inode2 = os.stat(filename2).st_ino + return inode1 == inode2 + + class CompileallTestsBase: def setUp(self): @@ -825,6 +846,32 @@ class CommandLineTestsBase: self.assertTrue(os.path.isfile(allowed_bc)) self.assertFalse(os.path.isfile(prohibited_bc)) + def test_hardlink_bad_args(self): + # Bad arguments combination, hardlink deduplication make sense + # only for more than one optimization level + self.assertRunNotOK(self.directory, "-o 1", "--hardlink-dupes") + + def test_hardlink(self): + # 'a = 0' code produces the same bytecode for the 3 optimization + # levels. All three .pyc files must have the same inode (hardlinks). + # + # If deduplication is disabled, all pyc files must have different + # inodes. + for dedup in (True, False): + with tempfile.TemporaryDirectory() as path: + with self.subTest(dedup=dedup): + script = script_helper.make_script(path, "script", "a = 0") + pycs = get_pycs(script) + + args = ["-q", "-o 0", "-o 1", "-o 2"] + if dedup: + args.append("--hardlink-dupes") + self.assertRunOK(path, *args) + + self.assertEqual(is_hardlink(pycs[0], pycs[1]), dedup) + self.assertEqual(is_hardlink(pycs[1], pycs[2]), dedup) + self.assertEqual(is_hardlink(pycs[0], pycs[2]), dedup) + class CommandLineTestsWithSourceEpoch(CommandLineTestsBase, unittest.TestCase, @@ -841,5 +888,176 @@ class CommandLineTestsNoSourceEpoch(CommandLineTestsBase, +class HardlinkDedupTestsBase: + # Test hardlink_dupes parameter of compileall.compile_dir() + + def setUp(self): + self.path = None + + @contextlib.contextmanager + def temporary_directory(self): + with tempfile.TemporaryDirectory() as path: + self.path = path + yield path + self.path = None + + def make_script(self, code, name="script"): + return script_helper.make_script(self.path, name, code) + + def compile_dir(self, *, dedup=True, optimize=(0, 1, 2), force=False): + compileall.compile_dir(self.path, quiet=True, optimize=optimize, + hardlink_dupes=dedup, force=force) + + def test_bad_args(self): + # Bad arguments combination, hardlink deduplication make sense + # only for more than one optimization level + with self.temporary_directory(): + self.make_script("pass") + with self.assertRaises(ValueError): + compileall.compile_dir(self.path, quiet=True, optimize=0, + hardlink_dupes=True) + with self.assertRaises(ValueError): + # same optimization level specified twice: + # compile_dir() removes duplicates + compileall.compile_dir(self.path, quiet=True, optimize=[0, 0], + hardlink_dupes=True) + + def create_code(self, docstring=False, assertion=False): + lines = [] + if docstring: + lines.append("'module docstring'") + lines.append('x = 1') + if assertion: + lines.append("assert x == 1") + return '\n'.join(lines) + + def iter_codes(self): + for docstring in (False, True): + for assertion in (False, True): + code = self.create_code(docstring=docstring, assertion=assertion) + yield (code, docstring, assertion) + + def test_disabled(self): + # Deduplication disabled, no hardlinks + for code, docstring, assertion in self.iter_codes(): + with self.subTest(docstring=docstring, assertion=assertion): + with self.temporary_directory(): + script = self.make_script(code) + pycs = get_pycs(script) + self.compile_dir(dedup=False) + self.assertFalse(is_hardlink(pycs[0], pycs[1])) + self.assertFalse(is_hardlink(pycs[0], pycs[2])) + self.assertFalse(is_hardlink(pycs[1], pycs[2])) + + def check_hardlinks(self, script, docstring=False, assertion=False): + pycs = get_pycs(script) + self.assertEqual(is_hardlink(pycs[0], pycs[1]), + not assertion) + self.assertEqual(is_hardlink(pycs[0], pycs[2]), + not assertion and not docstring) + self.assertEqual(is_hardlink(pycs[1], pycs[2]), + not docstring) + + def test_hardlink(self): + # Test deduplication on all combinations + for code, docstring, assertion in self.iter_codes(): + with self.subTest(docstring=docstring, assertion=assertion): + with self.temporary_directory(): + script = self.make_script(code) + self.compile_dir() + self.check_hardlinks(script, docstring, assertion) + + def test_only_two_levels(self): + # Don't build the 3 optimization levels, but only 2 + for opts in ((0, 1), (1, 2), (0, 2)): + with self.subTest(opts=opts): + with self.temporary_directory(): + # code with no dostring and no assertion: + # same bytecode for all optimization levels + script = self.make_script(self.create_code()) + self.compile_dir(optimize=opts) + pyc1 = get_pyc(script, opts[0]) + pyc2 = get_pyc(script, opts[1]) + self.assertTrue(is_hardlink(pyc1, pyc2)) + + def test_duplicated_levels(self): + # compile_dir() must not fail if optimize contains duplicated + # optimization levels and/or if optimization levels are not sorted. + with self.temporary_directory(): + # code with no dostring and no assertion: + # same bytecode for all optimization levels + script = self.make_script(self.create_code()) + self.compile_dir(optimize=[1, 0, 1, 0]) + pyc1 = get_pyc(script, 0) + pyc2 = get_pyc(script, 1) + self.assertTrue(is_hardlink(pyc1, pyc2)) + + def test_recompilation(self): + # Test compile_dir() when pyc files already exists and the script + # content changed + with self.temporary_directory(): + script = self.make_script("a = 0") + self.compile_dir() + # All three levels have the same inode + self.check_hardlinks(script) + + pycs = get_pycs(script) + inode = os.stat(pycs[0]).st_ino + + # Change of the module content + script = self.make_script("print(0)") + + # Recompilation without -o 1 + self.compile_dir(optimize=[0, 2], force=True) + + # opt-1.pyc should have the same inode as before and others should not + self.assertEqual(inode, os.stat(pycs[1]).st_ino) + self.assertTrue(is_hardlink(pycs[0], pycs[2])) + self.assertNotEqual(inode, os.stat(pycs[2]).st_ino) + # opt-1.pyc and opt-2.pyc have different content + self.assertFalse(filecmp.cmp(pycs[1], pycs[2], shallow=True)) + + def test_import(self): + # Test that import updates a single pyc file when pyc files already + # exists and the script content changed + with self.temporary_directory(): + script = self.make_script(self.create_code(), name="module") + self.compile_dir() + # All three levels have the same inode + self.check_hardlinks(script) + + pycs = get_pycs(script) + inode = os.stat(pycs[0]).st_ino + + # Change of the module content + script = self.make_script("print(0)", name="module") + + # Import the module in Python with -O (optimization level 1) + script_helper.assert_python_ok( + "-O", "-c", "import module", __isolated=False, PYTHONPATH=self.path + ) + + # Only opt-1.pyc is changed + self.assertEqual(inode, os.stat(pycs[0]).st_ino) + self.assertEqual(inode, os.stat(pycs[2]).st_ino) + self.assertFalse(is_hardlink(pycs[1], pycs[2])) + # opt-1.pyc and opt-2.pyc have different content + self.assertFalse(filecmp.cmp(pycs[1], pycs[2], shallow=True)) + + +class HardlinkDedupTestsWithSourceEpoch(HardlinkDedupTestsBase, + unittest.TestCase, + metaclass=SourceDateEpochTestMeta, + source_date_epoch=True): + pass + + +class HardlinkDedupTestsNoSourceEpoch(HardlinkDedupTestsBase, + unittest.TestCase, + metaclass=SourceDateEpochTestMeta, + source_date_epoch=False): + pass + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_dictcomps.py b/Lib/test/test_dictcomps.py index 16aa651b93..472e3dfa0d 100644 --- a/Lib/test/test_dictcomps.py +++ b/Lib/test/test_dictcomps.py @@ -77,7 +77,7 @@ class DictComprehensionTest(unittest.TestCase): compile("{x: y for y, x in ((1, 2), (3, 4))} = 5", "<test>", "exec") - with self.assertRaisesRegex(SyntaxError, "cannot assign"): + with self.assertRaisesRegex(SyntaxError, "illegal expression"): compile("{x: y for y, x in ((1, 2), (3, 4))} += 5", "<test>", "exec") diff --git a/Lib/test/test_email/test_contentmanager.py b/Lib/test/test_email/test_contentmanager.py index 169058eac8..64dca2d017 100644 --- a/Lib/test/test_email/test_contentmanager.py +++ b/Lib/test/test_email/test_contentmanager.py @@ -329,6 +329,21 @@ class TestRawDataManager(TestEmailBase): self.assertEqual(m.get_payload(decode=True).decode('utf-8'), content) self.assertEqual(m.get_content(), content) + def test_set_text_plain_long_line_heuristics(self): + m = self._make_message() + content = ("Simple but long message that is over 78 characters" + " long to force transfer encoding.\n") + raw_data_manager.set_content(m, content) + self.assertEqual(str(m), textwrap.dedent("""\ + Content-Type: text/plain; charset="utf-8" + Content-Transfer-Encoding: quoted-printable + + Simple but long message that is over 78 characters long to = + force transfer encoding. + """)) + self.assertEqual(m.get_payload(decode=True).decode('utf-8'), content) + self.assertEqual(m.get_content(), content) + def test_set_text_short_line_minimal_non_ascii_heuristics(self): m = self._make_message() content = "et là il est monté sur moi et il commence à m'éto.\n" diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index b689ec7aed..efd77fdbaa 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -228,6 +228,8 @@ class ExceptionTests(unittest.TestCase): def baz(): '''quux''' """, 9, 20) + check("pass\npass\npass\n(1+)\npass\npass\npass", 4, 4) + check("(1+)", 1, 4) # Errors thrown by symtable.c check('x = [(yield i) for i in range(3)]', 1, 5) diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index e047801199..348ae15aa6 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -318,7 +318,7 @@ class ExceptionTest(unittest.TestCase): class GeneratorThrowTest(unittest.TestCase): - def test_exception_context_set(self): + def test_exception_context_with_yield(self): def f(): try: raise KeyError('a') @@ -332,6 +332,23 @@ class GeneratorThrowTest(unittest.TestCase): context = cm.exception.__context__ self.assertEqual((type(context), context.args), (KeyError, ('a',))) + def test_exception_context_with_yield_from(self): + def f(): + yield + + def g(): + try: + raise KeyError('a') + except Exception: + yield from f() + + gen = g() + gen.send(None) + with self.assertRaises(ValueError) as cm: + gen.throw(ValueError) + context = cm.exception.__context__ + self.assertEqual((type(context), context.args), (KeyError, ('a',))) + def test_throw_after_none_exc_type(self): def g(): try: @@ -1904,7 +1921,7 @@ SyntaxError: cannot assign to yield expression >>> def f(): (yield bar) += y Traceback (most recent call last): ... -SyntaxError: cannot assign to yield expression +SyntaxError: 'yield expression' is an illegal expression for augmented assignment Now check some throw() conditions: diff --git a/Lib/test/test_genexps.py b/Lib/test/test_genexps.py index 86e4e195f5..5c1a209b0e 100644 --- a/Lib/test/test_genexps.py +++ b/Lib/test/test_genexps.py @@ -158,7 +158,7 @@ Verify that syntax error's are raised for genexps used as lvalues >>> (y for y in (1,2)) += 10 Traceback (most recent call last): ... - SyntaxError: cannot assign to generator expression + SyntaxError: 'generator expression' is an illegal expression for augmented assignment ########### Tests borrowed from or inspired by test_generators.py ############ diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index 02ba8a8b15..e1a402e2b4 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -1279,7 +1279,7 @@ class GrammarTests(unittest.TestCase): def test_try(self): ### try_stmt: 'try' ':' suite (except_clause ':' suite)+ ['else' ':' suite] ### | 'try' ':' suite 'finally' ':' suite - ### except_clause: 'except' [expr ['as' expr]] + ### except_clause: 'except' [expr ['as' NAME]] try: 1/0 except ZeroDivisionError: @@ -1297,6 +1297,9 @@ class GrammarTests(unittest.TestCase): except (EOFError, TypeError, ZeroDivisionError) as msg: pass try: pass finally: pass + with self.assertRaises(SyntaxError): + compile("try:\n pass\nexcept Exception as a.b:\n pass", "?", "exec") + compile("try:\n pass\nexcept Exception as a[b]:\n pass", "?", "exec") def test_suite(self): # simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT diff --git a/Lib/test/test_json/test_recursion.py b/Lib/test/test_json/test_recursion.py index 877dc448b1..543c62839b 100644 --- a/Lib/test/test_json/test_recursion.py +++ b/Lib/test/test_json/test_recursion.py @@ -52,7 +52,7 @@ class TestRecursion: return [JSONTestObject] else: return 'JSONTestObject' - return pyjson.JSONEncoder.default(o) + return self.json.JSONEncoder.default(o) enc = RecursiveJSONEncoder() self.assertEqual(enc.encode(JSONTestObject), '"JSONTestObject"') diff --git a/Lib/test/test_peg_parser.py b/Lib/test/test_peg_parser.py index df2d46d882..9614e45799 100644 --- a/Lib/test/test_peg_parser.py +++ b/Lib/test/test_peg_parser.py @@ -35,6 +35,9 @@ TEST_CASES = [ ('attribute_simple', 'a.b'), ('attributes_subscript', 'a.b[0]'), ('augmented_assignment', 'x += 42'), + ('augmented_assignment_attribute', 'a.b.c += 42'), + ('augmented_assignment_paren', '(x) += 42'), + ('augmented_assignment_paren_subscript', '(x[0]) -= 42'), ('binop_add', '1 + 1'), ('binop_add_multiple', '1 + 1 + 1 + 1'), ('binop_all', '1 + 2 * 5 + 3 ** 2 - -3'), @@ -547,6 +550,11 @@ TEST_CASES = [ with a as (x, y): pass '''), + ('with_list_target', + ''' + with a as [x, y]: + pass + '''), ('yield', 'yield'), ('yield_expr', 'yield a'), ('yield_from', 'yield from a'), @@ -560,6 +568,9 @@ FAIL_TEST_CASES = [ ("annotation_tuple", "(a,): int"), ("annotation_tuple_without_paren", "a,: int"), ("assignment_keyword", "a = if"), + ("augmented_assignment_list", "[a, b] += 1"), + ("augmented_assignment_tuple", "a, b += 1"), + ("augmented_assignment_tuple_paren", "(a, b) += (1, 2)"), ("comprehension_lambda", "(a for a in lambda: b)"), ("comprehension_else", "(a for a in b if c else d"), ("del_call", "del a()"), @@ -589,6 +600,20 @@ FAIL_TEST_CASES = [ a """), ("not_terminated_string", "a = 'example"), + ("try_except_attribute_target", + """ + try: + pass + except Exception as a.b: + pass + """), + ("try_except_subscript_target", + """ + try: + pass + except Exception as a[0]: + pass + """), ] FAIL_SPECIALIZED_MESSAGE_CASES = [ @@ -600,7 +625,7 @@ FAIL_SPECIALIZED_MESSAGE_CASES = [ ("(a, b): int", "only single target (not tuple) can be annotated"), ("[a, b]: int", "only single target (not list) can be annotated"), ("a(): int", "illegal target for annotation"), - ("1 += 1", "cannot assign to literal"), + ("1 += 1", "'literal' is an illegal expression for augmented assignment"), ("pass\n pass", "unexpected indent"), ("def f():\npass", "expected an indented block"), ("def f(*): pass", "named arguments must follow bare *"), diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index 06636ae8a1..60c7d9fd38 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -100,35 +100,54 @@ expression inside that contain should still cause a syntax error. This test just checks a couple of cases rather than enumerating all of them. -# All of the following also produce different error messages with pegen -# >>> (a, "b", c) = (1, 2, 3) -# Traceback (most recent call last): -# SyntaxError: cannot assign to literal +>>> (a, "b", c) = (1, 2, 3) +Traceback (most recent call last): +SyntaxError: cannot assign to literal -# >>> (a, True, c) = (1, 2, 3) -# Traceback (most recent call last): -# SyntaxError: cannot assign to True +>>> (a, True, c) = (1, 2, 3) +Traceback (most recent call last): +SyntaxError: cannot assign to True >>> (a, __debug__, c) = (1, 2, 3) Traceback (most recent call last): SyntaxError: cannot assign to __debug__ -# >>> (a, *True, c) = (1, 2, 3) -# Traceback (most recent call last): -# SyntaxError: cannot assign to True +>>> (a, *True, c) = (1, 2, 3) +Traceback (most recent call last): +SyntaxError: cannot assign to True >>> (a, *__debug__, c) = (1, 2, 3) Traceback (most recent call last): SyntaxError: cannot assign to __debug__ -# >>> [a, b, c + 1] = [1, 2, 3] -# Traceback (most recent call last): -# SyntaxError: cannot assign to operator +>>> [a, b, c + 1] = [1, 2, 3] +Traceback (most recent call last): +SyntaxError: cannot assign to operator + +>>> [a, b[1], c + 1] = [1, 2, 3] +Traceback (most recent call last): +SyntaxError: cannot assign to operator + +>>> [a, b.c.d, c + 1] = [1, 2, 3] +Traceback (most recent call last): +SyntaxError: cannot assign to operator >>> a if 1 else b = 1 Traceback (most recent call last): SyntaxError: cannot assign to conditional expression +>>> a, b += 1, 2 +Traceback (most recent call last): +SyntaxError: 'tuple' is an illegal expression for augmented assignment + +>>> (a, b) += 1, 2 +Traceback (most recent call last): +SyntaxError: 'tuple' is an illegal expression for augmented assignment + +>>> [a, b] += 1, 2 +Traceback (most recent call last): +SyntaxError: 'list' is an illegal expression for augmented assignment + From compiler_complex_args(): >>> def f(None=1): @@ -334,16 +353,16 @@ More set_context(): >>> (x for x in x) += 1 Traceback (most recent call last): -SyntaxError: cannot assign to generator expression +SyntaxError: 'generator expression' is an illegal expression for augmented assignment >>> None += 1 Traceback (most recent call last): -SyntaxError: cannot assign to None +SyntaxError: 'None' is an illegal expression for augmented assignment >>> __debug__ += 1 Traceback (most recent call last): SyntaxError: cannot assign to __debug__ >>> f() += 1 Traceback (most recent call last): -SyntaxError: cannot assign to function call +SyntaxError: 'function call' is an illegal expression for augmented assignment Test continue in finally in weird combinations. @@ -676,6 +695,7 @@ class SyntaxTestCase(unittest.TestCase): def test_assign_call(self): self._check_error("f() = 1", "assign") + @unittest.skipIf(support.use_old_parser(), "The old parser cannot generate these error messages") def test_assign_del(self): self._check_error("del (,)", "invalid syntax") self._check_error("del 1", "delete literal") |