Issue #19700: set __spec__ appropriately in runpy

Note that __spec__.name is not currently guaranteed to be in
sys.modules when the code is running, only __name__ is.

The "running module is in sys.modules" invariant will be
expanded to also cover __spec__.name in a subsequent patch.

1 files changed, 144 insertions, 35 deletions

diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py
index f19c4ab7eb..786b813b32 100644
--- a/Lib/test/test_runpy.py
+++ b/Lib/test/test_runpy.py
@@ -5,7 +5,7 @@ import os.path
 import sys
 import re
 import tempfile
-import importlib
+import importlib, importlib.machinery, importlib.util
 import py_compile
 from test.support import (
     forget, make_legacy_pyc, run_unittest, unload, verbose, no_tracing,
@@ -47,7 +47,7 @@ implicit_namespace = {
     "__cached__": None,
     "__package__": None,
     "__doc__": None,
-#    "__spec__": None,  # XXX Uncomment.
+    "__spec__": None
 }
 example_namespace =  {
     "sys": sys,
@@ -57,7 +57,7 @@ example_namespace =  {
     "run_name_in_sys_modules": False,
     "module_in_sys_modules": False,
     "nested": dict(implicit_namespace,
-                   x=1, __name__="<run>", __loader__=None, __spec__=None),
+                   x=1, __name__="<run>", __loader__=None),
 }
 example_namespace.update(implicit_namespace)
 
@@ -68,11 +68,19 @@ class CodeExecutionMixin:
     # testing occurs at those upper layers as well, not just at the utility
     # layer
 
+    # Figuring out the loader details in advance is hard to do, so we skip
+    # checking the full details of loader and loader_state
+    CHECKED_SPEC_ATTRIBUTES = ["name", "parent", "origin", "cached",
+                               "has_location", "submodule_search_locations"]
+
     def assertNamespaceMatches(self, result_ns, expected_ns):
         """Check two namespaces match.
 
            Ignores any unspecified interpreter created names
         """
+        # Avoid side effects
+        result_ns = result_ns.copy()
+        expected_ns = expected_ns.copy()
         # Impls are permitted to add extra names, so filter them out
         for k in list(result_ns):
             if k.startswith("__") and k.endswith("__"):
@@ -80,7 +88,25 @@ class CodeExecutionMixin:
                     result_ns.pop(k)
                 if k not in expected_ns["nested"]:
                     result_ns["nested"].pop(k)
-        # Don't use direct dict comparison - the diffs are too hard to debug
+        # Spec equality includes the loader, so we take the spec out of the
+        # result namespace and check that separately
+        result_spec = result_ns.pop("__spec__")
+        expected_spec = expected_ns.pop("__spec__")
+        if expected_spec is None:
+            self.assertIsNone(result_spec)
+        else:
+            # If an expected loader is set, we just check we got the right
+            # type, rather than checking for full equality
+            if expected_spec.loader is not None:
+                self.assertEqual(type(result_spec.loader),
+                                 type(expected_spec.loader))
+            for attr in self.CHECKED_SPEC_ATTRIBUTES:
+                k = "__spec__." + attr
+                actual = (k, getattr(result_spec, attr))
+                expected = (k, getattr(expected_spec, attr))
+                self.assertEqual(actual, expected)
+        # For the rest, we still don't use direct dict comparison on the
+        # namespace, as the diffs are too hard to debug if anything breaks
         self.assertEqual(set(result_ns), set(expected_ns))
         for k in result_ns:
             actual = (k, result_ns[k])
@@ -131,12 +157,16 @@ class ExecutionLayerTestCase(unittest.TestCase, CodeExecutionMixin):
         mod_fname = "Some other nonsense"
         mod_loader = "Now you're just being silly"
         mod_package = '' # Treat as a top level module
+        mod_spec = importlib.machinery.ModuleSpec(mod_name,
+                                                  origin=mod_fname,
+                                                  loader=mod_loader)
         expected_ns = example_namespace.copy()
         expected_ns.update({
             "__name__": mod_name,
             "__file__": mod_fname,
             "__loader__": mod_loader,
             "__package__": mod_package,
+            "__spec__": mod_spec,
             "run_argv0": mod_fname,
             "run_name_in_sys_modules": True,
             "module_in_sys_modules": True,
@@ -145,9 +175,7 @@ class ExecutionLayerTestCase(unittest.TestCase, CodeExecutionMixin):
             return _run_module_code(example_source,
                                     init_globals,
                                     mod_name,
-                                    mod_fname,
-                                    mod_loader,
-                                    mod_package)
+                                    mod_spec)
         self.check_code_execution(create_ns, expected_ns)
 
 # TODO: Use self.addCleanup to get rid of a lot of try-finally blocks
@@ -177,31 +205,43 @@ class RunModuleTestCase(unittest.TestCase, CodeExecutionMixin):
     def test_library_module(self):
         self.assertEqual(run_module("runpy")["__name__"], "runpy")
 
-    def _add_pkg_dir(self, pkg_dir):
+    def _add_pkg_dir(self, pkg_dir, namespace=False):
         os.mkdir(pkg_dir)
+        if namespace:
+            return None
         pkg_fname = os.path.join(pkg_dir, "__init__.py")
         create_empty_file(pkg_fname)
         return pkg_fname
 
-    def _make_pkg(self, source, depth, mod_base="runpy_test"):
+    def _make_pkg(self, source, depth, mod_base="runpy_test",
+                     *, namespace=False, parent_namespaces=False):
+        # Enforce a couple of internal sanity checks on test cases
+        if (namespace or parent_namespaces) and not depth:
+            raise RuntimeError("Can't mark top level module as a "
+                               "namespace package")
         pkg_name = "__runpy_pkg__"
         test_fname = mod_base+os.extsep+"py"
         pkg_dir = sub_dir = os.path.realpath(tempfile.mkdtemp())
         if verbose > 1: print("  Package tree in:", sub_dir)
         sys.path.insert(0, pkg_dir)
         if verbose > 1: print("  Updated sys.path:", sys.path[0])
-        for i in range(depth):
-            sub_dir = os.path.join(sub_dir, pkg_name)
-            pkg_fname = self._add_pkg_dir(sub_dir)
-            if verbose > 1: print("  Next level in:", sub_dir)
-            if verbose > 1: print("  Created:", pkg_fname)
+        if depth:
+            namespace_flags = [parent_namespaces] * depth
+            namespace_flags[-1] = namespace
+            for namespace_flag in namespace_flags:
+                sub_dir = os.path.join(sub_dir, pkg_name)
+                pkg_fname = self._add_pkg_dir(sub_dir, namespace_flag)
+                if verbose > 1: print("  Next level in:", sub_dir)
+                if verbose > 1: print("  Created:", pkg_fname)
         mod_fname = os.path.join(sub_dir, test_fname)
         mod_file = open(mod_fname, "w")
         mod_file.write(source)
         mod_file.close()
         if verbose > 1: print("  Created:", mod_fname)
         mod_name = (pkg_name+".")*depth + mod_base
-        return pkg_dir, mod_fname, mod_name
+        mod_spec = importlib.util.spec_from_file_location(mod_name,
+                                                          mod_fname)
+        return pkg_dir, mod_fname, mod_name, mod_spec
 
     def _del_pkg(self, top, depth, mod_name):
         for entry in list(sys.modules):
@@ -231,20 +271,29 @@ class RunModuleTestCase(unittest.TestCase, CodeExecutionMixin):
     def _fix_ns_for_legacy_pyc(self, ns, alter_sys):
         char_to_add = "c" if __debug__ else "o"
         ns["__file__"] += char_to_add
+        ns["__cached__"] = ns["__file__"]
+        spec = ns["__spec__"]
+        new_spec = importlib.util.spec_from_file_location(spec.name,
+                                                          ns["__file__"])
+        ns["__spec__"] = new_spec
         if alter_sys:
             ns["run_argv0"] += char_to_add
 
 
-    def _check_module(self, depth, alter_sys=False):
-        pkg_dir, mod_fname, mod_name = (
-               self._make_pkg(example_source, depth))
+    def _check_module(self, depth, alter_sys=False,
+                         *, namespace=False, parent_namespaces=False):
+        pkg_dir, mod_fname, mod_name, mod_spec = (
+               self._make_pkg(example_source, depth,
+                              namespace=namespace,
+                              parent_namespaces=parent_namespaces))
         forget(mod_name)
         expected_ns = example_namespace.copy()
         expected_ns.update({
             "__name__": mod_name,
             "__file__": mod_fname,
+            "__cached__": mod_spec.cached,
             "__package__": mod_name.rpartition(".")[0],
-#            "__spec__": None,  # XXX Needs to be set.
+            "__spec__": mod_spec,
         })
         if alter_sys:
             expected_ns.update({
@@ -271,17 +320,21 @@ class RunModuleTestCase(unittest.TestCase, CodeExecutionMixin):
             self._del_pkg(pkg_dir, depth, mod_name)
         if verbose > 1: print("Module executed successfully")
 
-    def _check_package(self, depth, alter_sys=False):
-        pkg_dir, mod_fname, mod_name = (
-               self._make_pkg(example_source, depth, "__main__"))
+    def _check_package(self, depth, alter_sys=False,
+                          *, namespace=False, parent_namespaces=False):
+        pkg_dir, mod_fname, mod_name, mod_spec = (
+               self._make_pkg(example_source, depth, "__main__",
+                              namespace=namespace,
+                              parent_namespaces=parent_namespaces))
         pkg_name = mod_name.rpartition(".")[0]
         forget(mod_name)
         expected_ns = example_namespace.copy()
         expected_ns.update({
             "__name__": mod_name,
             "__file__": mod_fname,
+            "__cached__": importlib.util.cache_from_source(mod_fname),
             "__package__": pkg_name,
-#            "__spec__": None,  # XXX Needs to be set.
+            "__spec__": mod_spec,
         })
         if alter_sys:
             expected_ns.update({
@@ -337,7 +390,7 @@ from __future__ import absolute_import
 from . import sibling
 from ..uncle.cousin import nephew
 """
-        pkg_dir, mod_fname, mod_name = (
+        pkg_dir, mod_fname, mod_name, mod_spec = (
                self._make_pkg(contents, depth))
         if run_name is None:
             expected_name = mod_name
@@ -376,11 +429,31 @@ from ..uncle.cousin import nephew
             if verbose > 1: print("Testing package depth:", depth)
             self._check_module(depth)
 
+    def test_run_module_in_namespace_package(self):
+        for depth in range(1, 4):
+            if verbose > 1: print("Testing package depth:", depth)
+            self._check_module(depth, namespace=True, parent_namespaces=True)
+
     def test_run_package(self):
         for depth in range(1, 4):
             if verbose > 1: print("Testing package depth:", depth)
             self._check_package(depth)
 
+    def test_run_package_in_namespace_package(self):
+        for depth in range(1, 4):
+            if verbose > 1: print("Testing package depth:", depth)
+            self._check_package(depth, parent_namespaces=True)
+
+    def test_run_namespace_package(self):
+        for depth in range(1, 4):
+            if verbose > 1: print("Testing package depth:", depth)
+            self._check_package(depth, namespace=True)
+
+    def test_run_namespace_package_in_namespace_package(self):
+        for depth in range(1, 4):
+            if verbose > 1: print("Testing package depth:", depth)
+            self._check_package(depth, namespace=True, parent_namespaces=True)
+
     def test_run_module_alter_sys(self):
         for depth in range(4):
             if verbose > 1: print("Testing package depth:", depth)
@@ -404,14 +477,16 @@ from ..uncle.cousin import nephew
     def test_run_name(self):
         depth = 1
         run_name = "And now for something completely different"
-        pkg_dir, mod_fname, mod_name = (
+        pkg_dir, mod_fname, mod_name, mod_spec = (
                self._make_pkg(example_source, depth))
         forget(mod_name)
         expected_ns = example_namespace.copy()
         expected_ns.update({
             "__name__": run_name,
             "__file__": mod_fname,
+            "__cached__": importlib.util.cache_from_source(mod_fname),
             "__package__": mod_name.rpartition(".")[0],
+            "__spec__": mod_spec,
         })
         def create_ns(init_globals):
             return run_module(mod_name, init_globals, run_name)
@@ -440,7 +515,7 @@ from ..uncle.cousin import nephew
         pkg_name = ".".join([base_name] * max_depth)
         expected_packages.add(pkg_name)
         expected_modules.add(pkg_name + ".runpy_test")
-        pkg_dir, mod_fname, mod_name = (
+        pkg_dir, mod_fname, mod_name, mod_spec = (
                self._make_pkg("", max_depth))
         self.addCleanup(self._del_pkg, pkg_dir, max_depth, mod_name)
         for depth in range(2, max_depth+1):
@@ -458,21 +533,39 @@ from ..uncle.cousin import nephew
 class RunPathTestCase(unittest.TestCase, CodeExecutionMixin):
     """Unit tests for runpy.run_path"""
 
-    def _make_test_script(self, script_dir, script_basename, source=None):
+    def _make_test_script(self, script_dir, script_basename,
+                          source=None, omit_suffix=False):
         if source is None:
             source = example_source
-        return make_script(script_dir, script_basename, source)
+        return make_script(script_dir, script_basename,
+                           source, omit_suffix)
 
     def _check_script(self, script_name, expected_name, expected_file,
-                            expected_argv0):
+                            expected_argv0, mod_name=None,
+                            expect_spec=True, check_loader=True):
         # First check is without run_name
         def create_ns(init_globals):
             return run_path(script_name, init_globals)
         expected_ns = example_namespace.copy()
+        if mod_name is None:
+            spec_name = expected_name
+        else:
+            spec_name = mod_name
+        if expect_spec:
+            mod_spec = importlib.util.spec_from_file_location(spec_name,
+                                                              expected_file)
+            mod_cached = mod_spec.cached
+            if not check_loader:
+                mod_spec.loader = None
+        else:
+            mod_spec = mod_cached = None
+
         expected_ns.update({
             "__name__": expected_name,
             "__file__": expected_file,
+            "__cached__": mod_cached,
             "__package__": "",
+            "__spec__": mod_spec,
             "run_argv0": expected_argv0,
             "run_name_in_sys_modules": True,
             "module_in_sys_modules": True,
@@ -482,6 +575,12 @@ class RunPathTestCase(unittest.TestCase, CodeExecutionMixin):
         run_name = "prove.issue15230.is.fixed"
         def create_ns(init_globals):
             return run_path(script_name, init_globals, run_name)
+        if expect_spec and mod_name is None:
+            mod_spec = importlib.util.spec_from_file_location(run_name,
+                                                              expected_file)
+            if not check_loader:
+                mod_spec.loader = None
+            expected_ns["__spec__"] = mod_spec
         expected_ns["__name__"] = run_name
         expected_ns["__package__"] = run_name.rpartition(".")[0]
         self.check_code_execution(create_ns, expected_ns)
@@ -495,7 +594,15 @@ class RunPathTestCase(unittest.TestCase, CodeExecutionMixin):
             mod_name = 'script'
             script_name = self._make_test_script(script_dir, mod_name)
             self._check_script(script_name, "<run_path>", script_name,
-                               script_name)
+                               script_name, expect_spec=False)
+
+    def test_basic_script_no_suffix(self):
+        with temp_dir() as script_dir:
+            mod_name = 'script'
+            script_name = self._make_test_script(script_dir, mod_name,
+                                                 omit_suffix=True)
+            self._check_script(script_name, "<run_path>", script_name,
+                               script_name, expect_spec=False)
 
     def test_script_compiled(self):
         with temp_dir() as script_dir:
@@ -504,14 +611,14 @@ class RunPathTestCase(unittest.TestCase, CodeExecutionMixin):
             compiled_name = py_compile.compile(script_name, doraise=True)
             os.remove(script_name)
             self._check_script(compiled_name, "<run_path>", compiled_name,
-                               compiled_name)
+                               compiled_name, expect_spec=False)
 
     def test_directory(self):
         with temp_dir() as script_dir:
             mod_name = '__main__'
             script_name = self._make_test_script(script_dir, mod_name)
             self._check_script(script_dir, "<run_path>", script_name,
-                               script_dir)
+                               script_dir, mod_name=mod_name)
 
     def test_directory_compiled(self):
         with temp_dir() as script_dir:
@@ -522,7 +629,7 @@ class RunPathTestCase(unittest.TestCase, CodeExecutionMixin):
             if not sys.dont_write_bytecode:
                 legacy_pyc = make_legacy_pyc(script_name)
                 self._check_script(script_dir, "<run_path>", legacy_pyc,
-                                   script_dir)
+                                   script_dir, mod_name=mod_name)
 
     def test_directory_error(self):
         with temp_dir() as script_dir:
@@ -536,7 +643,8 @@ class RunPathTestCase(unittest.TestCase, CodeExecutionMixin):
             mod_name = '__main__'
             script_name = self._make_test_script(script_dir, mod_name)
             zip_name, fname = make_zip_script(script_dir, 'test_zip', script_name)
-            self._check_script(zip_name, "<run_path>", fname, zip_name)
+            self._check_script(zip_name, "<run_path>", fname, zip_name,
+                               mod_name=mod_name, check_loader=False)
 
     def test_zipfile_compiled(self):
         with temp_dir() as script_dir:
@@ -545,7 +653,8 @@ class RunPathTestCase(unittest.TestCase, CodeExecutionMixin):
             compiled_name = py_compile.compile(script_name, doraise=True)
             zip_name, fname = make_zip_script(script_dir, 'test_zip',
                                               compiled_name)
-            self._check_script(zip_name, "<run_path>", fname, zip_name)
+            self._check_script(zip_name, "<run_path>", fname, zip_name,
+                               mod_name=mod_name, check_loader=False)
 
     def test_zipfile_error(self):
         with temp_dir() as script_dir: