From af1d64d9f7a7cf673279725fdbaf4adcca51d41f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 4 Nov 2020 17:34:34 +0100 Subject: bpo-42260: Main init modify sys.flags in-place (GH-23150) When Py_Initialize() is called twice, the second call now updates more sys attributes for the configuration, rather than only sys.argv. * Rename _PySys_InitMain() to _PySys_UpdateConfig(). * _PySys_UpdateConfig() now modifies sys.flags in-place, instead of creating a new flags object. * Remove old commented sys.flags flags (unbuffered and skip_first). * Add private _PySys_GetObject() function. * When Py_Initialize(), Py_InitializeFromConfig() and --- Python/sysmodule.c | 113 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 67 insertions(+), 46 deletions(-) (limited to 'Python/sysmodule.c') diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 945e639ca5..60b2494651 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -84,17 +84,24 @@ _PySys_GetObjectId(_Py_Identifier *key) return sys_get_object_id(tstate, key); } +static PyObject * +_PySys_GetObject(PyThreadState *tstate, const char *name) +{ + PyObject *sysdict = tstate->interp->sysdict; + if (sysdict == NULL) { + return NULL; + } + return _PyDict_GetItemStringWithError(sysdict, name); +} + PyObject * PySys_GetObject(const char *name) { PyThreadState *tstate = _PyThreadState_GET(); - PyObject *sd = tstate->interp->sysdict; - if (sd == NULL) { - return NULL; - } + PyObject *exc_type, *exc_value, *exc_tb; _PyErr_Fetch(tstate, &exc_type, &exc_value, &exc_tb); - PyObject *value = _PyDict_GetItemStringWithError(sd, name); + PyObject *value = _PySys_GetObject(tstate, name); /* XXX Suppress a new exception if it was raised and restore * the old one. */ _PyErr_Restore(tstate, exc_type, exc_value, exc_tb); @@ -2464,8 +2471,6 @@ static PyStructSequence_Field flags_fields[] = { {"no_site", "-S"}, {"ignore_environment", "-E"}, {"verbose", "-v"}, - /* {"unbuffered", "-u"}, */ - /* {"skip_first", "-x"}, */ {"bytes_warning", "-b"}, {"quiet", "-q"}, {"hash_randomization", "-R"}, @@ -2482,21 +2487,27 @@ static PyStructSequence_Desc flags_desc = { 15 }; -static PyObject* -make_flags(PyThreadState *tstate) +static int +set_flags_from_config(PyObject *flags, PyThreadState *tstate) { PyInterpreterState *interp = tstate->interp; const PyPreConfig *preconfig = &interp->runtime->preconfig; const PyConfig *config = _PyInterpreterState_GetConfig(interp); - PyObject *seq = PyStructSequence_New(&FlagsType); - if (seq == NULL) { - return NULL; - } - - int pos = 0; -#define SetFlag(flag) \ - PyStructSequence_SET_ITEM(seq, pos++, PyLong_FromLong(flag)) + // _PySys_UpdateConfig() modifies sys.flags in-place: + // Py_XDECREF() is needed in this case. + Py_ssize_t pos = 0; +#define SetFlagObj(expr) \ + do { \ + PyObject *value = (expr); \ + if (value == NULL) { \ + return -1; \ + } \ + Py_XDECREF(PyStructSequence_GET_ITEM(flags, pos)); \ + PyStructSequence_SET_ITEM(flags, pos, value); \ + pos++; \ + } while (0) +#define SetFlag(expr) SetFlagObj(PyLong_FromLong(expr)) SetFlag(config->parser_debug); SetFlag(config->inspect); @@ -2507,23 +2518,34 @@ make_flags(PyThreadState *tstate) SetFlag(!config->site_import); SetFlag(!config->use_environment); SetFlag(config->verbose); - /* SetFlag(saw_unbuffered_flag); */ - /* SetFlag(skipfirstline); */ SetFlag(config->bytes_warning); SetFlag(config->quiet); SetFlag(config->use_hash_seed == 0 || config->hash_seed != 0); SetFlag(config->isolated); - PyStructSequence_SET_ITEM(seq, pos++, PyBool_FromLong(config->dev_mode)); + SetFlagObj(PyBool_FromLong(config->dev_mode)); SetFlag(preconfig->utf8_mode); +#undef SetFlagObj #undef SetFlag + return 0; +} - if (_PyErr_Occurred(tstate)) { - Py_DECREF(seq); + +static PyObject* +make_flags(PyThreadState *tstate) +{ + PyObject *flags = PyStructSequence_New(&FlagsType); + if (flags == NULL) { return NULL; } - return seq; + + if (set_flags_from_config(flags, tstate) < 0) { + Py_DECREF(flags); + return NULL; + } + return flags; } + PyDoc_STRVAR(version_info__doc__, "sys.version_info\n\ \n\ @@ -2767,14 +2789,23 @@ _PySys_InitCore(PyThreadState *tstate, PyObject *sysdict) /* implementation */ SET_SYS("implementation", make_impl_info(version_info)); - /* flags */ + // sys.flags: updated in-place later by _PySys_UpdateConfig() if (FlagsType.tp_name == 0) { if (PyStructSequence_InitType2(&FlagsType, &flags_desc) < 0) { goto type_init_failed; } } - /* Set flags to their default values (updated by _PySys_InitMain()) */ SET_SYS("flags", make_flags(tstate)); + /* prevent user from creating new instances */ + FlagsType.tp_init = NULL; + FlagsType.tp_new = NULL; + res = PyDict_DelItemString(FlagsType.tp_dict, "__new__"); + if (res < 0) { + if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { + goto err_occurred; + } + _PyErr_Clear(tstate); + } #if defined(MS_WINDOWS) /* getwindowsversion */ @@ -2876,8 +2907,10 @@ sys_create_xoptions_dict(const PyConfig *config) } +// Update sys attributes for a new PyConfig configuration. +// This function also adds attributes that _PySys_InitCore() didn't add. int -_PySys_InitMain(PyThreadState *tstate) +_PySys_UpdateConfig(PyThreadState *tstate) { PyObject *sysdict = tstate->interp->sysdict; const PyConfig *config = _PyInterpreterState_GetConfig(tstate->interp); @@ -2914,28 +2947,16 @@ _PySys_InitMain(PyThreadState *tstate) #undef COPY_LIST #undef SET_SYS_FROM_WSTR - - /* Set flags to their final values */ - SET_SYS("flags", make_flags(tstate)); - /* prevent user from creating new instances */ - FlagsType.tp_init = NULL; - FlagsType.tp_new = NULL; - res = PyDict_DelItemString(FlagsType.tp_dict, "__new__"); - if (res < 0) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { - return res; - } - _PyErr_Clear(tstate); + // sys.flags + PyObject *flags = _PySys_GetObject(tstate, "flags"); // borrowed ref + if (flags == NULL) { + return -1; } - - SET_SYS("dont_write_bytecode", PyBool_FromLong(!config->write_bytecode)); - - if (get_warnoptions(tstate) == NULL) { + if (set_flags_from_config(flags, tstate) < 0) { return -1; } - if (get_xoptions(tstate) == NULL) - return -1; + SET_SYS("dont_write_bytecode", PyBool_FromLong(!config->write_bytecode)); if (_PyErr_Occurred(tstate)) { goto err_occurred; @@ -2977,8 +2998,8 @@ error: } -/* Create sys module without all attributes: _PySys_InitMain() should be called - later to add remaining attributes. */ +/* Create sys module without all attributes. + _PySys_UpdateConfig() should be called later to add remaining attributes. */ PyStatus _PySys_Create(PyThreadState *tstate, PyObject **sysmod_p) { -- cgit v1.2.1 From f3cb81431574453aac3b6dcadb3120331e6a8f1c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 5 Nov 2020 18:12:33 +0100 Subject: bpo-42260: Add _PyConfig_FromDict() (GH-23167) * Rename config_as_dict() to _PyConfig_AsDict(). * Add 'module_search_paths_set' to _PyConfig_AsDict(). * Add _PyConfig_FromDict(). * Add get_config() and set_config() to _testinternalcapi. * Add config_check_consistency(). --- Python/sysmodule.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'Python/sysmodule.c') diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 60b2494651..ae4f0eeb2e 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2922,7 +2922,9 @@ _PySys_UpdateConfig(PyThreadState *tstate) #define SET_SYS_FROM_WSTR(KEY, VALUE) \ SET_SYS(KEY, PyUnicode_FromWideChar(VALUE, -1)); - COPY_LIST("path", config->module_search_paths); + if (config->module_search_paths_set) { + COPY_LIST("path", config->module_search_paths); + } SET_SYS_FROM_WSTR("executable", config->executable); SET_SYS_FROM_WSTR("_base_executable", config->base_executable); -- cgit v1.2.1 From 9e1b828265e6bfb58f1e0299bd78d8ff6347a2ba Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 10 Nov 2020 13:21:52 +0100 Subject: bpo-42260: Compute the path config in the main init (GH-23211) The path configuration is now computed in the "main" initialization. The core initialization no longer computes it. * Add _PyConfig_Read() function to read the configuration without computing the path configuration. * pyinit_core() no longer computes the path configuration: it is now computed by init_interp_main(). * The path configuration output members of PyConfig are now optional: * executable * base_executable * prefix * base_prefix * exec_prefix * base_exec_prefix * _PySys_UpdateConfig() now skips NULL strings in PyConfig. * _testembed: Rename test_set_config() to test_init_set_config() for consistency with other tests. --- Python/sysmodule.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) (limited to 'Python/sysmodule.c') diff --git a/Python/sysmodule.c b/Python/sysmodule.c index ae4f0eeb2e..61741f7432 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2922,17 +2922,22 @@ _PySys_UpdateConfig(PyThreadState *tstate) #define SET_SYS_FROM_WSTR(KEY, VALUE) \ SET_SYS(KEY, PyUnicode_FromWideChar(VALUE, -1)); +#define COPY_WSTR(SYS_ATTR, WSTR) \ + if (WSTR != NULL) { \ + SET_SYS_FROM_WSTR(SYS_ATTR, WSTR); \ + } + if (config->module_search_paths_set) { COPY_LIST("path", config->module_search_paths); } - SET_SYS_FROM_WSTR("executable", config->executable); - SET_SYS_FROM_WSTR("_base_executable", config->base_executable); - SET_SYS_FROM_WSTR("prefix", config->prefix); - SET_SYS_FROM_WSTR("base_prefix", config->base_prefix); - SET_SYS_FROM_WSTR("exec_prefix", config->exec_prefix); - SET_SYS_FROM_WSTR("base_exec_prefix", config->base_exec_prefix); - SET_SYS_FROM_WSTR("platlibdir", config->platlibdir); + COPY_WSTR("executable", config->executable); + COPY_WSTR("_base_executable", config->base_executable); + COPY_WSTR("prefix", config->prefix); + COPY_WSTR("base_prefix", config->base_prefix); + COPY_WSTR("exec_prefix", config->exec_prefix); + COPY_WSTR("base_exec_prefix", config->base_exec_prefix); + COPY_WSTR("platlibdir", config->platlibdir); if (config->pycache_prefix != NULL) { SET_SYS_FROM_WSTR("pycache_prefix", config->pycache_prefix); @@ -2946,8 +2951,9 @@ _PySys_UpdateConfig(PyThreadState *tstate) SET_SYS("_xoptions", sys_create_xoptions_dict(config)); -#undef COPY_LIST #undef SET_SYS_FROM_WSTR +#undef COPY_LIST +#undef COPY_WSTR // sys.flags PyObject *flags = _PySys_GetObject(tstate, "flags"); // borrowed ref -- cgit v1.2.1 From ef75a625cdf8377d687a04948b4db9bc1917bf19 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 12 Nov 2020 15:14:13 +0100 Subject: bpo-42260: Initialize time and warnings earlier at startup (GH-23249) * Call _PyTime_Init() and _PyWarnings_InitState() earlier during the Python initialization. * Inline _PyImportHooks_Init() into _PySys_InitCore(). * The _warnings initialization function no longer call _PyWarnings_InitState() to prevent resetting filters_version to 0. * _PyWarnings_InitState() now returns an int and no longer clear the state in case of error (it's done anyway at Python exit). * Rework init_importlib(), fix refleaks on errors. --- Python/sysmodule.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'Python/sysmodule.c') diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 61741f7432..f05b33a9aa 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2841,6 +2841,11 @@ _PySys_InitCore(PyThreadState *tstate, PyObject *sysdict) } } + /* adding sys.path_hooks and sys.path_importer_cache */ + SET_SYS("meta_path", PyList_New(0)); + SET_SYS("path_importer_cache", PyDict_New()); + SET_SYS("path_hooks", PyList_New(0)); + if (_PyErr_Occurred(tstate)) { goto err_occurred; } -- cgit v1.2.1 From 4e7a69bdb63a104587759d7784124492dcdd496e Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 2 Dec 2020 13:30:55 +0000 Subject: bpo-42500: Fix recursion in or after except (GH-23568) * Use counter, rather boolean state when handling soft overflows. --- Python/sysmodule.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'Python/sysmodule.c') diff --git a/Python/sysmodule.c b/Python/sysmodule.c index f05b33a9aa..b80d37df42 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1181,7 +1181,6 @@ static PyObject * sys_setrecursionlimit_impl(PyObject *module, int new_limit) /*[clinic end generated code: output=35e1c64754800ace input=b0f7a23393924af3]*/ { - int mark; PyThreadState *tstate = _PyThreadState_GET(); if (new_limit < 1) { @@ -1199,8 +1198,7 @@ sys_setrecursionlimit_impl(PyObject *module, int new_limit) Reject too low new limit if the current recursion depth is higher than the new low-water mark. Otherwise it may not be possible anymore to reset the overflowed flag to 0. */ - mark = _Py_RecursionLimitLowerWaterMark(new_limit); - if (tstate->recursion_depth >= mark) { + if (tstate->recursion_depth >= new_limit) { _PyErr_Format(tstate, PyExc_RecursionError, "cannot set the recursion limit to %i at " "the recursion depth %i: the limit is too low", -- cgit v1.2.1 From 46b5c6be29f6470a20dd0dbd34e794debcee7c04 Mon Sep 17 00:00:00 2001 From: Joannah Nanjekye <33177550+nanjekyejoannah@users.noreply.github.com> Date: Tue, 22 Dec 2020 18:31:46 -0400 Subject: Fix typos in sysmodule (GH-23883) --- Python/sysmodule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Python/sysmodule.c') diff --git a/Python/sysmodule.c b/Python/sysmodule.c index b80d37df42..720532eade 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1907,12 +1907,12 @@ sys__debugmallocstats_impl(PyObject *module) } #ifdef Py_TRACE_REFS -/* Defined in objects.c because it uses static globals if that file */ +/* Defined in objects.c because it uses static globals in that file */ extern PyObject *_Py_GetObjects(PyObject *, PyObject *); #endif #ifdef DYNAMIC_EXECUTION_PROFILE -/* Defined in ceval.c because it uses static globals if that file */ +/* Defined in ceval.c because it uses static globals in that file */ extern PyObject *_Py_GetDXProfile(PyObject *, PyObject *); #endif -- cgit v1.2.1