Issue #9630: Redecode filenames when setting the filesystem encoding

Redecode the filenames of:

 - all modules: __file__ and __path__ attributes
 - all code objects: co_filename attribute
 - sys.path
 - sys.meta_path
 - sys.executable
 - sys.path_importer_cache (keys)

Keep weak references to all code objects until initfsencoding() is called, to
be able to redecode co_filename attribute of all code objects.

3 files changed, 28 insertions, 1 deletions

diff --git a/Objects/codeobject.c b/Objects/codeobject.c
index da5c09ac8d..470bf56150 100644
--- a/Objects/codeobject.c
+++ b/Objects/codeobject.c
@@ -5,6 +5,8 @@
 #define NAME_CHARS \
     "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz"
 
+PyObject *_Py_code_object_list = NULL;
+
 /* all_name_chars(s): true iff all chars in s are valid NAME_CHARS */
 
 static int
@@ -109,8 +111,23 @@ PyCode_New(int argcount, int kwonlyargcount,
         co->co_lnotab = lnotab;
         co->co_zombieframe = NULL;
         co->co_weakreflist = NULL;
+
+        if (_Py_code_object_list != NULL) {
+            int err;
+            PyObject *ref = PyWeakref_NewRef((PyObject*)co, NULL);
+            if (ref == NULL)
+                goto error;
+            err = PyList_Append(_Py_code_object_list, ref);
+            Py_DECREF(ref);
+            if (err)
+                goto error;
+        }
     }
     return co;
+
+error:
+    Py_DECREF(co);
+    return NULL;
 }
 
 PyCodeObject *
diff --git a/Objects/object.c b/Objects/object.c
index ff3363f432..e322e53103 100644
--- a/Objects/object.c
+++ b/Objects/object.c
@@ -1604,6 +1604,10 @@ _Py_ReadyTypes(void)
     if (PyType_Ready(&PyCode_Type) < 0)
         Py_FatalError("Can't initialize code type");
 
+    _Py_code_object_list = PyList_New(0);
+    if (_Py_code_object_list == NULL)
+        Py_FatalError("Can't initialize code type");
+
     if (PyType_Ready(&PyFrame_Type) < 0)
         Py_FatalError("Can't initialize frame type");
 
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index 527e219896..8ceb3f32b8 100644
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -1510,10 +1510,14 @@ PyObject *PyUnicode_EncodeFSDefault(PyObject *unicode)
         return PyUnicode_AsEncodedString(unicode,
                                          Py_FileSystemDefaultEncoding,
                                          "surrogateescape");
-    } else
+    }
+    else {
+        /* if you change the default encoding, update also
+           PyUnicode_DecodeFSDefaultAndSize() and redecode_filenames() */
         return PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(unicode),
                                     PyUnicode_GET_SIZE(unicode),
                                     "surrogateescape");
+    }
 }
 
 PyObject *PyUnicode_AsEncodedString(PyObject *unicode,
@@ -1680,6 +1684,8 @@ PyUnicode_DecodeFSDefaultAndSize(const char *s, Py_ssize_t size)
                                 "surrogateescape");
     }
     else {
+        /* if you change the default encoding, update also
+           PyUnicode_EncodeFSDefault() and redecode_filenames() */
         return PyUnicode_DecodeUTF8(s, size, "surrogateescape");
     }
 }