bpo-31404: Revert "remove modules from Py_InterpreterState (#1638)" (#3565)

PR #1638, for bpo-28411, causes problems in some (very) edge cases. Until that gets sorted out, we're reverting the merge. PR #3506, a fix on top of #1638, is also getting reverted.

1 files changed, 37 insertions, 168 deletions

diff --git a/Python/import.c b/Python/import.c
index 6b2634c349..2aacf65935 100644
--- a/Python/import.c
+++ b/Python/import.c
@@ -290,115 +290,10 @@ _PyImport_Fini(void)
 PyObject *
 PyImport_GetModuleDict(void)
 {
-    PyObject *sysdict = PyThreadState_GET()->interp->sysdict;
-    if (sysdict == NULL) {
-        Py_FatalError("PyImport_GetModuleDict: no sys module!");
-    }
-
-    _Py_IDENTIFIER(modules);
-    PyObject *modules = _PyDict_GetItemId(sysdict, &PyId_modules);
-    if (modules == NULL) {
-        Py_FatalError("lost sys.modules");
-    }
-    return modules;
-}
-
-/* In some corner cases it is important to be sure that the import
-   machinery has been initialized (or not cleaned up yet).  For
-   example, see issue #4236 and PyModule_Create2(). */
-
-int
-_PyImport_IsInitialized(PyInterpreterState *interp)
-{
-    if (interp->sysdict == NULL)
-        return 0;
-    _Py_IDENTIFIER(modules);
-    PyObject *modules = _PyDict_GetItemId(interp->sysdict, &PyId_modules);
-    if (modules == NULL)
-        return 0;
-    return 1;
-}
-
-PyObject *
-_PyImport_GetModule(PyObject *name)
-{
-    PyObject *modules = PyImport_GetModuleDict();
-    if (PyDict_CheckExact(modules)) {
-        return PyDict_GetItem(modules, name);
-    }
-
-    PyObject *mod = PyObject_GetItem(modules, name);
-    // For backward-comaptibility we copy the behavior of PyDict_GetItem().
-    if (PyErr_Occurred()) {
-        PyErr_Clear();
-    }
-    Py_XDECREF(mod);
-    return mod;
-}
-
-PyObject *
-_PyImport_GetModuleWithError(PyObject *name)
-{
-    PyObject *modules = PyImport_GetModuleDict();
-    if (PyDict_CheckExact(modules)) {
-        return PyDict_GetItemWithError(modules, name);
-    }
-
-    PyObject *mod = PyObject_GetItem(modules, name);
-    // For backward-comaptibility we copy the behavior
-    // of PyDict_GetItemWithError().
-    if (PyErr_ExceptionMatches(PyExc_KeyError)) {
-        PyErr_Clear();
-    }
-    return mod;
-}
-
-PyObject *
-_PyImport_GetModuleId(struct _Py_Identifier *nameid)
-{
-    PyObject *name = _PyUnicode_FromId(nameid); /* borrowed */
-    if (name == NULL) {
-        return NULL;
-    }
-    return _PyImport_GetModule(name);
-}
-
-int
-_PyImport_SetModule(PyObject *name, PyObject *m)
-{
-    PyObject *modules = PyImport_GetModuleDict();
-    return PyObject_SetItem(modules, name, m);
-}
-
-int
-_PyImport_SetModuleString(const char *name, PyObject *m)
-{
-    PyObject *modules = PyImport_GetModuleDict();
-    return PyMapping_SetItemString(modules, name, m);
-}
-
-PyObject *
-PyImport_GetModule(PyObject *name)
-{
-    PyObject *m;
-    PyObject *modules = PyImport_GetModuleDict();
-    if (modules == NULL) {
-        PyErr_SetString(PyExc_RuntimeError, "unable to get sys.modules");
-        return NULL;
-    }
-    Py_INCREF(modules);
-    if (PyDict_CheckExact(modules)) {
-        m = PyDict_GetItemWithError(modules, name);  /* borrowed */
-        Py_XINCREF(m);
-    }
-    else {
-        m = PyObject_GetItem(modules, name);
-        if (PyErr_ExceptionMatches(PyExc_KeyError)) {
-            PyErr_Clear();
-        }
-    }
-    Py_DECREF(modules);
-    return m;
+    PyInterpreterState *interp = PyThreadState_GET()->interp;
+    if (interp->modules == NULL)
+        Py_FatalError("PyImport_GetModuleDict: no module dictionary!");
+    return interp->modules;
 }
 
 
@@ -428,7 +323,7 @@ PyImport_Cleanup(void)
     Py_ssize_t pos;
     PyObject *key, *value, *dict;
     PyInterpreterState *interp = PyThreadState_GET()->interp;
-    PyObject *modules = PyImport_GetModuleDict();
+    PyObject *modules = interp->modules;
     PyObject *weaklist = NULL;
     const char * const *p;
 
@@ -490,7 +385,7 @@ PyImport_Cleanup(void)
             if (Py_VerboseFlag && PyUnicode_Check(key))
                 PySys_FormatStderr("# cleanup[2] removing %U\n", key);
             STORE_MODULE_WEAKREF(key, value);
-            PyObject_SetItem(modules, key, Py_None);
+            PyDict_SetItem(modules, key, Py_None);
         }
     }
 
@@ -557,6 +452,7 @@ PyImport_Cleanup(void)
     /* Clear and delete the modules directory.  Actual modules will
        still be there only if imported during the execution of some
        destructor. */
+    interp->modules = NULL;
     Py_DECREF(modules);
 
     /* Once more */
@@ -615,9 +511,9 @@ PyImport_GetMagicTag(void)
 
 int
 _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name,
-                                 PyObject *filename, PyObject *modules)
+                               PyObject *filename)
 {
-    PyObject *dict, *key;
+    PyObject *modules, *dict, *key;
     struct PyModuleDef *def;
     int res;
     if (extensions == NULL) {
@@ -634,10 +530,11 @@ _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name,
         PyErr_BadInternalCall();
         return -1;
     }
-    if (PyObject_SetItem(modules, name, mod) < 0)
+    modules = PyImport_GetModuleDict();
+    if (PyDict_SetItem(modules, name, mod) < 0)
         return -1;
     if (_PyState_AddModule(mod, def) < 0) {
-        PyMapping_DelItem(modules, name);
+        PyDict_DelItem(modules, name);
         return -1;
     }
     if (def->m_size == -1) {
@@ -665,14 +562,14 @@ _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name,
 }
 
 int
-_PyImport_FixupBuiltin(PyObject *mod, const char *name, PyObject *modules)
+_PyImport_FixupBuiltin(PyObject *mod, const char *name)
 {
     int res;
     PyObject *nameobj;
     nameobj = PyUnicode_InternFromString(name);
     if (nameobj == NULL)
         return -1;
-    res = _PyImport_FixupExtensionObject(mod, nameobj, nameobj, modules);
+    res = _PyImport_FixupExtensionObject(mod, nameobj, nameobj);
     Py_DECREF(nameobj);
     return res;
 }
@@ -680,14 +577,6 @@ _PyImport_FixupBuiltin(PyObject *mod, const char *name, PyObject *modules)
 PyObject *
 _PyImport_FindExtensionObject(PyObject *name, PyObject *filename)
 {
-    PyObject *modules = PyImport_GetModuleDict();
-    return _PyImport_FindExtensionObjectEx(name, filename, modules);
-}
-
-PyObject *
-_PyImport_FindExtensionObjectEx(PyObject *name, PyObject *filename,
-                                PyObject *modules)
-{
     PyObject *mod, *mdict, *key;
     PyModuleDef* def;
     if (extensions == NULL)
@@ -703,7 +592,7 @@ _PyImport_FindExtensionObjectEx(PyObject *name, PyObject *filename,
         /* Module does not support repeated initialization */
         if (def->m_base.m_copy == NULL)
             return NULL;
-        mod = _PyImport_AddModuleObject(name, modules);
+        mod = PyImport_AddModuleObject(name);
         if (mod == NULL)
             return NULL;
         mdict = PyModule_GetDict(mod);
@@ -718,14 +607,14 @@ _PyImport_FindExtensionObjectEx(PyObject *name, PyObject *filename,
         mod = def->m_base.m_init();
         if (mod == NULL)
             return NULL;
-        if (PyObject_SetItem(modules, name, mod) == -1) {
+        if (PyDict_SetItem(PyImport_GetModuleDict(), name, mod) == -1) {
             Py_DECREF(mod);
             return NULL;
         }
         Py_DECREF(mod);
     }
     if (_PyState_AddModule(mod, def) < 0) {
-        PyMapping_DelItem(modules, name);
+        PyDict_DelItem(PyImport_GetModuleDict(), name);
         Py_DECREF(mod);
         return NULL;
     }
@@ -737,13 +626,13 @@ _PyImport_FindExtensionObjectEx(PyObject *name, PyObject *filename,
 }
 
 PyObject *
-_PyImport_FindBuiltin(const char *name, PyObject *modules)
+_PyImport_FindBuiltin(const char *name)
 {
     PyObject *res, *nameobj;
     nameobj = PyUnicode_InternFromString(name);
     if (nameobj == NULL)
         return NULL;
-    res = _PyImport_FindExtensionObjectEx(nameobj, nameobj, modules);
+    res = _PyImport_FindExtensionObject(nameobj, nameobj);
     Py_DECREF(nameobj);
     return res;
 }
@@ -758,34 +647,19 @@ PyObject *
 PyImport_AddModuleObject(PyObject *name)
 {
     PyObject *modules = PyImport_GetModuleDict();
-    return _PyImport_AddModuleObject(name, modules);
-}
-
-PyObject *
-_PyImport_AddModuleObject(PyObject *name, PyObject *modules)
-{
     PyObject *m;
-    if (PyDict_CheckExact(modules)) {
-        m = PyDict_GetItemWithError(modules, name);
-    }
-    else {
-        m = PyObject_GetItem(modules, name);
-        // For backward-comaptibility we copy the behavior
-        // of PyDict_GetItemWithError().
-        if (PyErr_ExceptionMatches(PyExc_KeyError)) {
-            PyErr_Clear();
-        }
+
+    if ((m = PyDict_GetItemWithError(modules, name)) != NULL &&
+        PyModule_Check(m)) {
+        return m;
     }
     if (PyErr_Occurred()) {
         return NULL;
     }
-    if (m != NULL && PyModule_Check(m)) {
-        return m;
-    }
     m = PyModule_NewObject(name);
     if (m == NULL)
         return NULL;
-    if (PyObject_SetItem(modules, name, m) != 0) {
+    if (PyDict_SetItem(modules, name, m) != 0) {
         Py_DECREF(m);
         return NULL;
     }
@@ -812,13 +686,11 @@ static void
 remove_module(PyObject *name)
 {
     PyObject *modules = PyImport_GetModuleDict();
-    if (PyMapping_DelItem(modules, name) < 0) {
-        if (!PyMapping_HasKey(modules, name)) {
-            return;
-        }
+    if (PyDict_GetItem(modules, name) == NULL)
+        return;
+    if (PyDict_DelItem(modules, name) < 0)
         Py_FatalError("import:  deleting existing key in"
                       "sys.modules failed");
-    }
 }
 
 
@@ -927,6 +799,7 @@ module_dict_for_exec(PyObject *name)
 static PyObject *
 exec_code_in_module(PyObject *name, PyObject *module_dict, PyObject *code_object)
 {
+    PyObject *modules = PyImport_GetModuleDict();
     PyObject *v, *m;
 
     v = PyEval_EvalCode(code_object, module_dict, module_dict);
@@ -936,8 +809,7 @@ exec_code_in_module(PyObject *name, PyObject *module_dict, PyObject *code_object
     }
     Py_DECREF(v);
 
-    m = _PyImport_GetModule(name);
-    if (m == NULL) {
+    if ((m = PyDict_GetItem(modules, name)) == NULL) {
         PyErr_Format(PyExc_ImportError,
                      "Loaded module %R not found in sys.modules",
                      name);
@@ -1170,7 +1042,6 @@ _imp_create_builtin(PyObject *module, PyObject *spec)
         return NULL;
     }
 
-    PyObject *modules = NULL;
     for (p = PyImport_Inittab; p->name != NULL; p++) {
         PyModuleDef *def;
         if (_PyUnicode_EqualToASCIIString(name, p->name)) {
@@ -1196,11 +1067,7 @@ _imp_create_builtin(PyObject *module, PyObject *spec)
                     return NULL;
                 }
                 def->m_base.m_init = p->initfunc;
-                if (modules == NULL) {
-                    modules = PyImport_GetModuleDict();
-                }
-                if (_PyImport_FixupExtensionObject(mod, name, name,
-                                                   modules) < 0) {
+                if (_PyImport_FixupExtensionObject(mod, name, name) < 0) {
                     Py_DECREF(name);
                     return NULL;
                 }
@@ -1644,7 +1511,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals,
         Py_INCREF(abs_name);
     }
 
-    mod = _PyImport_GetModule(abs_name);
+    mod = PyDict_GetItem(interp->modules, abs_name);
     if (mod != NULL && mod != Py_None) {
         _Py_IDENTIFIER(__spec__);
         _Py_IDENTIFIER(_initializing);
@@ -1731,7 +1598,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals,
                     goto error;
                 }
 
-                final_mod = _PyImport_GetModule(to_return);
+                final_mod = PyDict_GetItem(interp->modules, to_return);
                 Py_DECREF(to_return);
                 if (final_mod == NULL) {
                     PyErr_Format(PyExc_KeyError,
@@ -1784,10 +1651,10 @@ PyImport_ImportModuleLevel(const char *name, PyObject *globals, PyObject *locals
 PyObject *
 PyImport_ReloadModule(PyObject *m)
 {
-    _Py_IDENTIFIER(imp);
     _Py_IDENTIFIER(reload);
     PyObject *reloaded_module = NULL;
-    PyObject *imp = _PyImport_GetModuleId(&PyId_imp);
+    PyObject *modules = PyImport_GetModuleDict();
+    PyObject *imp = PyDict_GetItemString(modules, "imp");
     if (imp == NULL) {
         imp = PyImport_ImportModule("imp");
         if (imp == NULL) {
@@ -1822,6 +1689,7 @@ PyImport_Import(PyObject *module_name)
     PyObject *globals = NULL;
     PyObject *import = NULL;
     PyObject *builtins = NULL;
+    PyObject *modules = NULL;
     PyObject *r = NULL;
 
     /* Initialize constant string objects */
@@ -1876,7 +1744,8 @@ PyImport_Import(PyObject *module_name)
         goto err;
     Py_DECREF(r);
 
-    r = _PyImport_GetModule(module_name);
+    modules = PyImport_GetModuleDict();
+    r = PyDict_GetItemWithError(modules, module_name);
     if (r != NULL) {
         Py_INCREF(r);
     }