diff options
36 files changed, 605 insertions, 514 deletions
diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index ae0c9fb55a..6342ee3bb0 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -211,19 +211,18 @@ are always available. They are listed here in alphabetical order. @classmethod def f(cls, arg1, arg2, ...): ... - The ``@classmethod`` form is a function :term:`decorator` -- see the description - of function definitions in :ref:`function` for details. + The ``@classmethod`` form is a function :term:`decorator` -- see + :ref:`function` for details. - It can be called either on the class (such as ``C.f()``) or on an instance (such + A class method can be called either on the class (such as ``C.f()``) or on an instance (such as ``C().f()``). The instance is ignored except for its class. If a class method is called for a derived class, the derived class object is passed as the implied first argument. Class methods are different than C++ or Java static methods. If you want those, - see :func:`staticmethod` in this section. + see :func:`staticmethod`. - For more information on class methods, consult the documentation on the standard - type hierarchy in :ref:`types`. + For more information on class methods, see :ref:`types`. .. function:: compile(source, filename, mode, flags=0, dont_inherit=False, optimize=-1) @@ -1452,11 +1451,11 @@ are always available. They are listed here in alphabetical order. @staticmethod def f(arg1, arg2, ...): ... - The ``@staticmethod`` form is a function :term:`decorator` -- see the - description of function definitions in :ref:`function` for details. + The ``@staticmethod`` form is a function :term:`decorator` -- see + :ref:`function` for details. - It can be called either on the class (such as ``C.f()``) or on an instance (such - as ``C().f()``). The instance is ignored except for its class. + A static method can be called either on the class (such as ``C.f()``) or on an instance (such + as ``C().f()``). Static methods in Python are similar to those found in Java or C++. Also see :func:`classmethod` for a variant that is useful for creating alternate class @@ -1471,8 +1470,7 @@ are always available. They are listed here in alphabetical order. class C: builtin_open = staticmethod(open) - For more information on static methods, consult the documentation on the - standard type hierarchy in :ref:`types`. + For more information on static methods, see :ref:`types`. .. index:: diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index 8ef9e2eed5..11e137bf10 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -356,8 +356,8 @@ Shell and Output windows also have the following. Go to file/line Same as in Debug menu. -The Shell window also has an output squeezing facility explained in the -the *Python Shell window* subsection below. +The Shell window also has an output squeezing facility explained in the *Python +Shell window* subsection below. Squeeze If the cursor is over an output line, squeeze all the output between @@ -716,17 +716,15 @@ In contrast, some system text windows only keep the last n lines of output. A Windows console, for instance, keeps a user-settable 1 to 9999 lines, with 300 the default. -A Tk Text widget, and hence IDLE's Shell, displays characters (codepoints) -in the the BMP (Basic Multilingual Plane) subset of Unicode. -Which characters are displayed with a proper glyph and which with a -replacement box depends on the operating system and installed fonts. -Tab characters cause the following text to begin after -the next tab stop. (They occur every 8 'characters'). -Newline characters cause following text to appear on a new line. -Other control characters are ignored or displayed as a space, box, or -something else, depending on the operating system and font. -(Moving the text cursor through such output with arrow keys may exhibit -some surprising spacing behavior.) +A Tk Text widget, and hence IDLE's Shell, displays characters (codepoints) in +the BMP (Basic Multilingual Plane) subset of Unicode. Which characters are +displayed with a proper glyph and which with a replacement box depends on the +operating system and installed fonts. Tab characters cause the following text +to begin after the next tab stop. (They occur every 8 'characters'). Newline +characters cause following text to appear on a new line. Other control +characters are ignored or displayed as a space, box, or something else, +depending on the operating system and font. (Moving the text cursor through +such output with arrow keys may exhibit some surprising spacing behavior.) .. code-block:: none diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index 5adde29f50..08555c3a35 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -1215,6 +1215,10 @@ functions. closing all handlers. This should be called at application exit and no further use of the logging system should be made after this call. + When the logging module is imported, it registers this function as an exit + handler (see :mod:`atexit`), so normally there's no need to do that + manually. + .. function:: setLoggerClass(klass) diff --git a/Doc/library/queue.rst b/Doc/library/queue.rst index f99f6ffb05..2eeab5e262 100644 --- a/Doc/library/queue.rst +++ b/Doc/library/queue.rst @@ -150,6 +150,11 @@ provide the public methods described below. Otherwise (*block* is false), return an item if one is immediately available, else raise the :exc:`Empty` exception (*timeout* is ignored in that case). + Prior to 3.0 on POSIX systems, and for all versions on Windows, if + *block* is true and *timeout* is ``None``, this operation goes into + an uninterruptible wait on an underlying lock. This means that no exceptions + can occur, and in particular a SIGINT will not trigger a :exc:`KeyboardInterrupt`. + .. method:: Queue.get_nowait() diff --git a/Doc/library/wsgiref.rst b/Doc/library/wsgiref.rst index ffca1fc816..b85ec53c8a 100644 --- a/Doc/library/wsgiref.rst +++ b/Doc/library/wsgiref.rst @@ -781,3 +781,35 @@ This is a working "Hello World" WSGI application:: # Serve until process is killed httpd.serve_forever() + + +Example of a small wsgiref-based web server:: + + # Takes a path to serve from and an optional port number (defaults to 8000), + # then tries to serve files. Mime types are guessed from the file names, 404 + # errors are raised if the file is not found. + import sys + import os + import mimetypes + from wsgiref import simple_server, util + + def app(environ, respond): + fn = os.path.join(path, environ['PATH_INFO'][1:]) + if '.' not in fn.split(os.path.sep)[-1]: + fn = os.path.join(fn, 'index.html') + type = mimetypes.guess_type(fn)[0] + + if os.path.exists(fn): + respond('200 OK', [('Content-Type', type)]) + return util.FileWrapper(open(fn, "rb")) + else: + respond('404 Not Found', [('Content-Type', 'text/plain')]) + return [b'not found'] + + path = sys.argv[1] + port = int(sys.argv[2]) if len(sys.argv) > 2 else 8000 + with simple_server.make_server('', port, app) as httpd: + print("Serving {} on port {}, control-C to stop".format(path, port)) + + # Serve until process is killed + httpd.serve_forever() diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index 3855d3604e..6ab7991d8d 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -174,6 +174,20 @@ gettext Added :func:`~gettext.pgettext` and its variants. (Contributed by Franz Glasner, Éric Araujo, and Cheryl Sabella in :issue:`2504`.) +inspect +------- + +The :func:`inspect.getdoc` function can now find docstrings for ``__slots__`` +if that attribute is a :class:`dict` where the values are docstrings. +This provides documentation options similar to what we already have +for :func:`property`, :func:`classmethod`, and :func:`staticmethod`:: + + class AudioClip: + __slots__ = {'bit_rate': 'expressed in kilohertz to one decimal place', + 'duration': 'in seconds, rounded up to an integer'} + def __init__(self, bit_rate, duration): + self.bit_rate = round(bit_rate / 1000.0, 1) + self.duration = ceil(duration) gc -- diff --git a/Include/cpython/coreconfig.h b/Include/cpython/coreconfig.h index bb086cbd12..827a19a145 100644 --- a/Include/cpython/coreconfig.h +++ b/Include/cpython/coreconfig.h @@ -123,7 +123,9 @@ typedef struct { /* --- _PyCoreConfig ---------------------------------------------- */ typedef struct { - _PyPreConfig preconfig; + int isolated; + int use_environment; + int dev_mode; /* Install signal handlers? Yes by default. */ int install_signal_handlers; @@ -375,7 +377,9 @@ typedef struct { #define _PyCoreConfig_INIT \ (_PyCoreConfig){ \ _PyCoreConfig_WINDOWS_INIT \ - .preconfig = _PyPreConfig_INIT, \ + .isolated = -1, \ + .use_environment = -1, \ + .dev_mode = -1, \ .install_signal_handlers = 1, \ .use_hash_seed = -1, \ .faulthandler = -1, \ @@ -400,8 +404,7 @@ typedef struct { /* --- Function used for testing ---------------------------------- */ -PyAPI_FUNC(PyObject *) _Py_GetGlobalVariablesAsDict(void); -PyAPI_FUNC(PyObject *) _PyCoreConfig_AsDict(const _PyCoreConfig *config); +PyAPI_FUNC(PyObject*) _Py_GetConfigsAsDict(void); #ifdef __cplusplus } diff --git a/Include/cpython/pylifecycle.h b/Include/cpython/pylifecycle.h index a226de8446..e32e54cba0 100644 --- a/Include/cpython/pylifecycle.h +++ b/Include/cpython/pylifecycle.h @@ -16,7 +16,7 @@ PyAPI_FUNC(int) Py_SetStandardStreamEncoding(const char *encoding, PyAPI_FUNC(_PyInitError) _Py_PreInitialize(void); PyAPI_FUNC(_PyInitError) _Py_PreInitializeFromPreConfig( - _PyPreConfig *preconfig); + const _PyPreConfig *preconfig); PyAPI_FUNC(_PyInitError) _Py_PreInitializeFromConfig( const _PyCoreConfig *coreconfig); @@ -33,9 +33,6 @@ PyAPI_FUNC(void) _PyMainInterpreterConfig_Clear(_PyMainInterpreterConfig *); PyAPI_FUNC(int) _PyMainInterpreterConfig_Copy( _PyMainInterpreterConfig *config, const _PyMainInterpreterConfig *config2); -/* Used by _testcapi.get_main_config() */ -PyAPI_FUNC(PyObject*) _PyMainInterpreterConfig_AsDict( - const _PyMainInterpreterConfig *config); PyAPI_FUNC(_PyInitError) _Py_InitializeMainInterpreter( PyInterpreterState *interp, diff --git a/Include/internal/pycore_coreconfig.h b/Include/internal/pycore_coreconfig.h index d44172e0dd..d79f590d68 100644 --- a/Include/internal/pycore_coreconfig.h +++ b/Include/internal/pycore_coreconfig.h @@ -16,12 +16,14 @@ typedef struct { _PyWstrList xoptions; /* "-X value" option */ int use_environment; /* -E option */ int isolated; /* -I option */ + int dev_mode; /* -X dev and PYTHONDEVMODE */ } _PyPreCmdline; #define _PyPreCmdline_INIT \ (_PyPreCmdline){ \ .use_environment = -1, \ - .isolated = -1} + .isolated = -1, \ + .dev_mode = -1} /* Note: _PyPreCmdline_INIT sets other fields to 0/NULL */ PyAPI_FUNC(void) _PyPreCmdline_Clear(_PyPreCmdline *cmdline); @@ -76,22 +78,23 @@ PyAPI_FUNC(int) _Py_str_to_int( PyAPI_FUNC(const wchar_t*) _Py_get_xoption( const _PyWstrList *xoptions, const wchar_t *name); +PyAPI_FUNC(const char*) _Py_GetEnv( + int use_environment, + const char *name); PyAPI_FUNC(void) _PyPreConfig_Clear(_PyPreConfig *config); PyAPI_FUNC(int) _PyPreConfig_Copy(_PyPreConfig *config, const _PyPreConfig *config2); PyAPI_FUNC(void) _PyPreConfig_GetGlobalConfig(_PyPreConfig *config); PyAPI_FUNC(void) _PyPreConfig_SetGlobalConfig(const _PyPreConfig *config); -PyAPI_FUNC(const char*) _PyPreConfig_GetEnv(const _PyPreConfig *config, - const char *name); -PyAPI_FUNC(void) _Py_get_env_flag(_PyPreConfig *config, +PyAPI_FUNC(void) _Py_get_env_flag( + int use_environment, int *flag, const char *name); PyAPI_FUNC(_PyInitError) _PyPreConfig_Read(_PyPreConfig *config, const _PyArgv *args, const _PyCoreConfig *coreconfig); -PyAPI_FUNC(int) _PyPreConfig_AsDict(const _PyPreConfig *config, - PyObject *dict); +PyAPI_FUNC(PyObject*) _PyPreConfig_AsDict(const _PyPreConfig *config); PyAPI_FUNC(_PyInitError) _PyPreConfig_ReadFromArgv(_PyPreConfig *config, const _PyArgv *args); PyAPI_FUNC(_PyInitError) _PyPreConfig_Write(_PyPreConfig *config); @@ -108,18 +111,15 @@ PyAPI_FUNC(_PyInitError) _PyCoreConfig_SetPathConfig( const _PyCoreConfig *config); PyAPI_FUNC(void) _PyCoreConfig_GetGlobalConfig(_PyCoreConfig *config); PyAPI_FUNC(void) _PyCoreConfig_SetGlobalConfig(const _PyCoreConfig *config); -PyAPI_FUNC(const char*) _PyCoreConfig_GetEnv( - const _PyCoreConfig *config, - const char *name); -PyAPI_FUNC(int) _PyCoreConfig_GetEnvDup( - const _PyCoreConfig *config, - wchar_t **dest, - wchar_t *wname, - char *name); PyAPI_FUNC(_PyInitError) _PyCoreConfig_Read(_PyCoreConfig *config); PyAPI_FUNC(_PyInitError) _PyCoreConfig_ReadFromArgv(_PyCoreConfig *config, const _PyArgv *args); -PyAPI_FUNC(_PyInitError) _PyCoreConfig_Write(const _PyCoreConfig *config); +PyAPI_FUNC(void) _PyCoreConfig_Write(const _PyCoreConfig *config); + +/* --- _PyMainInterpreterConfig ----------------------------------- */ + +PyAPI_FUNC(PyObject*) _PyMainInterpreterConfig_AsDict( + const _PyMainInterpreterConfig *config); #ifdef __cplusplus } diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index 9514b1c46a..3214d6b06b 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -77,6 +77,9 @@ extern void _PyGILState_Fini(void); PyAPI_FUNC(void) _PyGC_DumpShutdownStats(void); +PyAPI_FUNC(_PyInitError) _Py_PreInitializeInPlace( + _PyPreConfig *config); + #ifdef __cplusplus } #endif diff --git a/Lib/inspect.py b/Lib/inspect.py index b8a142232b..8c398bd353 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -582,9 +582,12 @@ def _finddoc(obj): cls = obj.__objclass__ if getattr(cls, name) is not obj: return None + if ismemberdescriptor(obj): + slots = getattr(cls, '__slots__', None) + if isinstance(slots, dict) and name in slots: + return slots[name] else: return None - for base in cls.__mro__: try: doc = getattr(base, name).__doc__ diff --git a/Lib/keyword.py b/Lib/keyword.py index 150c2bc46d..ddcbb25d3d 100755..100644 --- a/Lib/keyword.py +++ b/Lib/keyword.py @@ -1,98 +1,55 @@ -#! /usr/bin/env python3 - -"""Keywords (from "graminit.c") +"""Keywords (from "Grammar/Grammar") This file is automatically generated; please don't muck it up! To update the symbols in this file, 'cd' to the top directory of -the python source tree after building the interpreter and run: +the python source tree and run: + + python3 -m Parser.pgen.keywordgen Grammar/Grammar \ + Grammar/Tokens \ + Lib/keyword.py - ./python Lib/keyword.py +Alternatively, you can run 'make regen-keyword'. """ __all__ = ["iskeyword", "kwlist"] kwlist = [ -#--start keywords-- - 'False', - 'None', - 'True', - 'and', - 'as', - 'assert', - 'break', - 'class', - 'continue', - 'def', - 'del', - 'elif', - 'else', - 'except', - 'finally', - 'for', - 'from', - 'global', - 'if', - 'import', - 'in', - 'is', - 'lambda', - 'nonlocal', - 'not', - 'or', - 'pass', - 'raise', - 'return', - 'try', - 'while', - 'with', - 'yield', -#--end keywords-- - ] - -kwlist.append('async') -kwlist.append('await') -kwlist.sort() + 'False', + 'None', + 'True', + 'and', + 'as', + 'assert', + 'async', + 'await', + 'break', + 'class', + 'continue', + 'def', + 'del', + 'elif', + 'else', + 'except', + 'finally', + 'for', + 'from', + 'global', + 'if', + 'import', + 'in', + 'is', + 'lambda', + 'nonlocal', + 'not', + 'or', + 'pass', + 'raise', + 'return', + 'try', + 'while', + 'with', + 'yield' +] iskeyword = frozenset(kwlist).__contains__ - -def main(): - import sys, re - - args = sys.argv[1:] - iptfile = args and args[0] or "Python/graminit.c" - if len(args) > 1: optfile = args[1] - else: optfile = "Lib/keyword.py" - - # load the output skeleton from the target, taking care to preserve its - # newline convention. - with open(optfile, newline='') as fp: - format = fp.readlines() - nl = format[0][len(format[0].strip()):] if format else '\n' - - # scan the source file for keywords - with open(iptfile) as fp: - strprog = re.compile('"([^"]+)"') - lines = [] - for line in fp: - if '{1, "' in line: - match = strprog.search(line) - if match: - lines.append(" '" + match.group(1) + "'," + nl) - lines.sort() - - # insert the lines of keywords into the skeleton - try: - start = format.index("#--start keywords--" + nl) + 1 - end = format.index("#--end keywords--" + nl) - format[start:end] = lines - except ValueError: - sys.stderr.write("target does not contain format markers\n") - sys.exit(1) - - # write the output file - with open(optfile, 'w', newline='') as fp: - fp.writelines(format) - -if __name__ == "__main__": - main() diff --git a/Lib/statistics.py b/Lib/statistics.py index e5a62463f0..bd8a6f9638 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -709,7 +709,8 @@ class NormalDist: # https://en.wikipedia.org/wiki/Normal_distribution # https://en.wikipedia.org/wiki/Variance#Properties - __slots__ = ('mu', 'sigma') + __slots__ = {'mu': 'Arithmetic mean of a normal distribution', + 'sigma': 'Standard deviation of a normal distribution'} def __init__(self, mu=0.0, sigma=1.0): 'NormalDist where mu is the mean and sigma is the standard deviation.' diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index 07f3cb3bea..79f7e82e00 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -598,18 +598,15 @@ def collect_get_config(info_add): # Dump global configuration variables, _PyCoreConfig # and _PyMainInterpreterConfig try: - from _testcapi import get_global_config, get_core_config, get_main_config + from _testcapi import get_configs except ImportError: return - for prefix, get_config_func in ( - ('global_config', get_global_config), - ('core_config', get_core_config), - ('main_config', get_main_config), - ): - config = get_config_func() + all_configs = get_configs() + for config_type in sorted(all_configs): + config = all_configs[config_type] for key in sorted(config): - info_add('%s[%s]' % (prefix, key), repr(config[key])) + info_add('%s[%s]' % (config_type, key), repr(config[key])) def collect_subprocess(info_add): diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 374346e3cc..ff3cfb1ea4 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -268,13 +268,26 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): ) # Mark config which should be get by get_default_config() GET_DEFAULT_CONFIG = object() + DEFAULT_PRE_CONFIG = { + 'allocator': None, + 'coerce_c_locale': 0, + 'coerce_c_locale_warn': 0, + 'utf8_mode': 0, + } + COPY_PRE_CONFIG = [ + 'dev_mode', + 'isolated', + 'use_environment', + ] + DEFAULT_CORE_CONFIG = { - 'install_signal_handlers': 1, + 'isolated': 0, 'use_environment': 1, + 'dev_mode': 0, + + 'install_signal_handlers': 1, 'use_hash_seed': 0, 'hash_seed': 0, - 'allocator': None, - 'dev_mode': 0, 'faulthandler': 0, 'tracemalloc': 0, 'import_time': 0, @@ -286,10 +299,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'filesystem_encoding': GET_DEFAULT_CONFIG, 'filesystem_errors': GET_DEFAULT_CONFIG, - 'utf8_mode': 0, - 'coerce_c_locale': 0, - 'coerce_c_locale_warn': 0, - 'pycache_prefix': None, 'program_name': './_testembed', 'argv': [""], @@ -306,7 +315,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'exec_prefix': GET_DEFAULT_CONFIG, 'base_exec_prefix': GET_DEFAULT_CONFIG, - 'isolated': 0, 'site_import': 1, 'bytes_warning': 0, 'inspect': 0, @@ -332,8 +340,10 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): '_frozen': 0, } if MS_WINDOWS: - DEFAULT_CORE_CONFIG.update({ + DEFAULT_PRE_CONFIG.update({ 'legacy_windows_fs_encoding': 0, + }) + DEFAULT_CORE_CONFIG.update({ 'legacy_windows_stdio': 0, }) @@ -359,6 +369,9 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'Py_HashRandomizationFlag': 1, '_Py_HasFileSystemDefaultEncodeErrors': 0, } + COPY_GLOBAL_PRE_CONFIG = [ + ('Py_UTF8Mode', 'utf8_mode'), + ] COPY_GLOBAL_CONFIG = [ # Copy core config to global config for expected values # True means that the core config value is inverted (0 => 1 and 1 => 0) @@ -376,13 +389,14 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): ('Py_NoUserSiteDirectory', 'user_site_directory', True), ('Py_OptimizeFlag', 'optimization_level'), ('Py_QuietFlag', 'quiet'), - ('Py_UTF8Mode', 'utf8_mode'), ('Py_UnbufferedStdioFlag', 'buffered_stdio', True), ('Py_VerboseFlag', 'verbose'), ] if MS_WINDOWS: - COPY_GLOBAL_CONFIG.extend(( + COPY_GLOBAL_PRE_CONFIG.extend(( ('Py_LegacyWindowsFSEncodingFlag', 'legacy_windows_fs_encoding'), + )) + COPY_GLOBAL_CONFIG.extend(( ('Py_LegacyWindowsStdioFlag', 'legacy_windows_stdio'), )) @@ -453,6 +467,11 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): expected[key] = config[key] return expected + def check_pre_config(self, config, expected): + pre_config = dict(config['pre_config']) + core_config = dict(config['core_config']) + self.assertEqual(pre_config, expected) + def check_core_config(self, config, expected): core_config = dict(config['core_config']) for key in self.UNTESTED_CORE_CONFIG: @@ -460,6 +479,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): self.assertEqual(core_config, expected) def check_global_config(self, config): + pre_config = config['pre_config'] core_config = config['core_config'] expected = dict(self.DEFAULT_GLOBAL_CONFIG) @@ -470,10 +490,17 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): else: global_key, core_key = item expected[global_key] = core_config[core_key] + for item in self.COPY_GLOBAL_PRE_CONFIG: + if len(item) == 3: + global_key, core_key, opposite = item + expected[global_key] = 0 if pre_config[core_key] else 1 + else: + global_key, core_key = item + expected[global_key] = pre_config[core_key] self.assertEqual(config['global_config'], expected) - def check_config(self, testname, expected): + def check_config(self, testname, expected_config, expected_preconfig): env = dict(os.environ) # Remove PYTHON* environment variables to get deterministic environment for key in list(env): @@ -488,15 +515,24 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): # Ignore err config = json.loads(out) - expected = self.get_expected_config(expected, env) - self.check_core_config(config, expected) + expected_preconfig = dict(self.DEFAULT_PRE_CONFIG, **expected_preconfig) + expected_config = self.get_expected_config(expected_config, env) + for key in self.COPY_PRE_CONFIG: + if key not in expected_preconfig: + expected_preconfig[key] = expected_config[key] + + self.check_core_config(config, expected_config) + self.check_pre_config(config, expected_preconfig) self.check_main_config(config) self.check_global_config(config) def test_init_default_config(self): - self.check_config("init_default_config", {}) + self.check_config("init_default_config", {}, {}) def test_init_global_config(self): + preconfig = { + 'utf8_mode': 1, + } config = { 'program_name': './globalvar', 'site_import': 0, @@ -509,7 +545,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'quiet': 1, 'buffered_stdio': 0, - 'utf8_mode': 1, 'stdio_encoding': 'utf-8', 'stdio_errors': 'surrogateescape', 'filesystem_encoding': 'utf-8', @@ -517,21 +552,23 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'user_site_directory': 0, '_frozen': 1, } - self.check_config("init_global_config", config) + self.check_config("init_global_config", config, preconfig) def test_init_from_config(self): + preconfig = { + 'allocator': 'malloc', + 'utf8_mode': 1, + } config = { 'install_signal_handlers': 0, 'use_hash_seed': 1, 'hash_seed': 123, - 'allocator': 'malloc', 'tracemalloc': 2, 'import_time': 1, 'show_ref_count': 1, 'show_alloc_count': 1, 'malloc_stats': 1, - 'utf8_mode': 1, 'stdio_encoding': 'iso8859-1', 'stdio_errors': 'replace', 'filesystem_encoding': 'utf-8', @@ -559,16 +596,18 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): '_check_hash_pycs_mode': 'always', '_frozen': 1, } - self.check_config("init_from_config", config) + self.check_config("init_from_config", config, preconfig) + INIT_ENV_PRECONFIG = { + 'allocator': 'malloc', + 'utf8_mode': 1, + } INIT_ENV_CONFIG = { 'use_hash_seed': 1, 'hash_seed': 42, - 'allocator': 'malloc', 'tracemalloc': 2, 'import_time': 1, 'malloc_stats': 1, - 'utf8_mode': 1, 'filesystem_encoding': 'utf-8', 'filesystem_errors': UTF8_MODE_ERRORS, 'inspect': 1, @@ -584,35 +623,43 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): } def test_init_env(self): - self.check_config("init_env", self.INIT_ENV_CONFIG) + self.check_config("init_env", self.INIT_ENV_CONFIG, self.INIT_ENV_PRECONFIG) def test_init_env_dev_mode(self): + preconfig = dict(self.INIT_ENV_PRECONFIG, + allocator='debug') config = dict(self.INIT_ENV_CONFIG, - allocator='debug', dev_mode=1) - self.check_config("init_env_dev_mode", config) + self.check_config("init_env_dev_mode", config, preconfig) - def test_init_env_dev_mode(self): + def test_init_env_dev_mode_alloc(self): + preconfig = dict(self.INIT_ENV_PRECONFIG, + allocator='malloc') config = dict(self.INIT_ENV_CONFIG, - allocator='malloc', dev_mode=1) - self.check_config("init_env_dev_mode_alloc", config) + self.check_config("init_env_dev_mode_alloc", config, preconfig) def test_init_dev_mode(self): + preconfig = { + 'allocator': 'debug', + } config = { - 'dev_mode': 1, 'faulthandler': 1, - 'allocator': 'debug', + 'dev_mode': 1, } - self.check_config("init_dev_mode", config) + self.check_config("init_dev_mode", config, preconfig) def test_init_isolated(self): + preconfig = { + 'isolated': 0, + 'use_environment': 1, + } config = { 'isolated': 1, 'use_environment': 0, 'user_site_directory': 0, } - self.check_config("init_isolated", config) + self.check_config("init_isolated", config, preconfig) if __name__ == "__main__": diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index b9072e0137..bc675aa5df 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -375,6 +375,11 @@ class GetSourceBase(unittest.TestCase): self.assertEqual(inspect.getsource(obj), self.sourcerange(top, bottom)) +class SlotUser: + 'Docstrings for __slots__' + __slots__ = {'power': 'measured in kilowatts', + 'distance': 'measured in kilometers'} + class TestRetrievingSourceCode(GetSourceBase): fodderModule = mod @@ -429,6 +434,10 @@ class TestRetrievingSourceCode(GetSourceBase): 'A longer,\n\nindented\n\ndocstring.') self.assertEqual(inspect.getdoc(git.abuse), 'Another\n\ndocstring\n\ncontaining\n\ntabs') + self.assertEqual(inspect.getdoc(SlotUser.power), + 'measured in kilowatts') + self.assertEqual(inspect.getdoc(SlotUser.distance), + 'measured in kilometers') @unittest.skipIf(sys.flags.optimize >= 2, "Docstrings are omitted with -O2 and above") diff --git a/Lib/test/test_keyword.py b/Lib/test/test_keyword.py index af99f52c63..3e2a8b3fb7 100644 --- a/Lib/test/test_keyword.py +++ b/Lib/test/test_keyword.py @@ -1,20 +1,5 @@ import keyword import unittest -from test import support -import filecmp -import os -import sys -import subprocess -import shutil -import textwrap - -KEYWORD_FILE = support.findfile('keyword.py') -GRAMMAR_FILE = os.path.join(os.path.split(__file__)[0], - '..', '..', 'Python', 'graminit.c') -TEST_PY_FILE = 'keyword_test.py' -GRAMMAR_TEST_FILE = 'graminit_test.c' -PY_FILE_WITHOUT_KEYWORDS = 'minimal_keyword.py' -NONEXISTENT_FILE = 'not_here.txt' class Test_iskeyword(unittest.TestCase): @@ -35,103 +20,17 @@ class Test_iskeyword(unittest.TestCase): keyword.kwlist = ['its', 'all', 'eggs', 'beans', 'and', 'a', 'slice'] self.assertFalse(keyword.iskeyword('eggs')) + def test_all_keywords_fail_to_be_used_as_names(self): + for key in keyword.kwlist: + with self.assertRaises(SyntaxError): + exec(f"{key} = 42") -class TestKeywordGeneration(unittest.TestCase): - - def _copy_file_without_generated_keywords(self, source_file, dest_file): - with open(source_file, 'rb') as fp: - lines = fp.readlines() - nl = lines[0][len(lines[0].strip()):] - with open(dest_file, 'wb') as fp: - fp.writelines(lines[:lines.index(b"#--start keywords--" + nl) + 1]) - fp.writelines(lines[lines.index(b"#--end keywords--" + nl):]) - - def _generate_keywords(self, grammar_file, target_keyword_py_file): - proc = subprocess.Popen([sys.executable, - KEYWORD_FILE, - grammar_file, - target_keyword_py_file], stderr=subprocess.PIPE) - stderr = proc.communicate()[1] - return proc.returncode, stderr - - @unittest.skipIf(not os.path.exists(GRAMMAR_FILE), - 'test only works from source build directory') - def test_real_grammar_and_keyword_file(self): - self._copy_file_without_generated_keywords(KEYWORD_FILE, TEST_PY_FILE) - self.addCleanup(support.unlink, TEST_PY_FILE) - self.assertFalse(filecmp.cmp(KEYWORD_FILE, TEST_PY_FILE)) - self.assertEqual((0, b''), self._generate_keywords(GRAMMAR_FILE, - TEST_PY_FILE)) - self.assertTrue(filecmp.cmp(KEYWORD_FILE, TEST_PY_FILE)) - - def test_grammar(self): - self._copy_file_without_generated_keywords(KEYWORD_FILE, TEST_PY_FILE) - self.addCleanup(support.unlink, TEST_PY_FILE) - with open(GRAMMAR_TEST_FILE, 'w') as fp: - # Some of these are probably implementation accidents. - fp.writelines(textwrap.dedent("""\ - {2, 1}, - {11, "encoding_decl", 0, 2, states_79, - "\000\000\040\000\000\000\000\000\000\000\000\000" - "\000\000\000\000\000\000\000\000\000"}, - {1, "jello"}, - {326, 0}, - {1, "turnip"}, - \t{1, "This one is tab indented" - {278, 0}, - {1, "crazy but legal" - "also legal" {1, " - {1, "continue"}, - {1, "lemon"}, - {1, "tomato"}, - {1, "wigii"}, - {1, 'no good'} - {283, 0}, - {1, "too many spaces"}""")) - self.addCleanup(support.unlink, GRAMMAR_TEST_FILE) - self._generate_keywords(GRAMMAR_TEST_FILE, TEST_PY_FILE) - expected = [ - " 'This one is tab indented',", - " 'also legal',", - " 'continue',", - " 'crazy but legal',", - " 'jello',", - " 'lemon',", - " 'tomato',", - " 'turnip',", - " 'wigii',", - ] - with open(TEST_PY_FILE) as fp: - lines = fp.read().splitlines() - start = lines.index("#--start keywords--") + 1 - end = lines.index("#--end keywords--") - actual = lines[start:end] - self.assertEqual(actual, expected) - - def test_empty_grammar_results_in_no_keywords(self): - self._copy_file_without_generated_keywords(KEYWORD_FILE, - PY_FILE_WITHOUT_KEYWORDS) - self.addCleanup(support.unlink, PY_FILE_WITHOUT_KEYWORDS) - shutil.copyfile(KEYWORD_FILE, TEST_PY_FILE) - self.addCleanup(support.unlink, TEST_PY_FILE) - self.assertEqual((0, b''), self._generate_keywords(os.devnull, - TEST_PY_FILE)) - self.assertTrue(filecmp.cmp(TEST_PY_FILE, PY_FILE_WITHOUT_KEYWORDS)) - - def test_keywords_py_without_markers_produces_error(self): - rc, stderr = self._generate_keywords(os.devnull, os.devnull) - self.assertNotEqual(rc, 0) - self.assertRegex(stderr, b'does not contain format markers') - - def test_missing_grammar_file_produces_error(self): - rc, stderr = self._generate_keywords(NONEXISTENT_FILE, KEYWORD_FILE) - self.assertNotEqual(rc, 0) - self.assertRegex(stderr, b'(?ms)' + NONEXISTENT_FILE.encode()) + def test_async_and_await_are_keywords(self): + self.assertIn("async", keyword.kwlist) + self.assertIn("await", keyword.kwlist) - def test_missing_keywords_py_file_produces_error(self): - rc, stderr = self._generate_keywords(os.devnull, NONEXISTENT_FILE) - self.assertNotEqual(rc, 0) - self.assertRegex(stderr, b'(?ms)' + NONEXISTENT_FILE.encode()) + def test_keywords_are_sorted(self): + self.assertListEqual(sorted(keyword.kwlist), keyword.kwlist) if __name__ == "__main__": diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index 485ffe2403..7f7839de46 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -2051,7 +2051,7 @@ class TestNormalDist(unittest.TestCase): nd = statistics.NormalDist(300, 23) with self.assertRaises(TypeError): vars(nd) - self.assertEqual(nd.__slots__, ('mu', 'sigma')) + self.assertEqual(tuple(nd.__slots__), ('mu', 'sigma')) def test_instantiation_and_attributes(self): nd = statistics.NormalDist(500, 17) diff --git a/Makefile.pre.in b/Makefile.pre.in index 8042e8e81a..174b12c5de 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -724,7 +724,7 @@ regen-importlib: Programs/_freeze_importlib # Regenerate all generated files regen-all: regen-opcode regen-opcode-targets regen-typeslots regen-grammar \ - regen-token regen-symbol regen-ast regen-importlib clinic + regen-token regen-keyword regen-symbol regen-ast regen-importlib clinic ############################################################################ # Special rules for object files @@ -843,6 +843,15 @@ regen-token: $(srcdir)/Grammar/Tokens \ $(srcdir)/Lib/token.py +.PHONY: regen-keyword +regen-keyword: + # Regenerate Lib/keyword.py from Grammar/Grammar and Grammar/Tokens + # using Parser/pgen + $(PYTHON_FOR_REGEN) -m Parser.pgen.keywordgen $(srcdir)/Grammar/Grammar \ + $(srcdir)/Grammar/Tokens \ + $(srcdir)/Lib/keyword.py.new + $(UPDATE_FILE) $(srcdir)/Lib/keyword.py $(srcdir)/Lib/keyword.py.new + .PHONY: regen-symbol regen-symbol: $(srcdir)/Include/graminit.h # Regenerate Lib/symbol.py from Include/graminit.h diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-03-20-00-37-24.bpo-12456.fnKoKo.rst b/Misc/NEWS.d/next/Core and Builtins/2019-03-20-00-37-24.bpo-12456.fnKoKo.rst new file mode 100644 index 0000000000..10d6c49627 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2019-03-20-00-37-24.bpo-12456.fnKoKo.rst @@ -0,0 +1,2 @@ +Regenerate :mod:`keyword` from the Grammar and Tokens file using pgen. Patch +by Pablo Galindo. diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-03-25-23-37-26.bpo-36430.sd9xxQ.rst b/Misc/NEWS.d/next/Core and Builtins/2019-03-25-23-37-26.bpo-36430.sd9xxQ.rst new file mode 100644 index 0000000000..a65ee096ef --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2019-03-25-23-37-26.bpo-36430.sd9xxQ.rst @@ -0,0 +1 @@ +Fix a possible reference leak in :func:`itertools.count`. diff --git a/Misc/NEWS.d/next/Documentation/2019-03-23-09-25-12.bpo-36345.L704Zv.rst b/Misc/NEWS.d/next/Documentation/2019-03-23-09-25-12.bpo-36345.L704Zv.rst new file mode 100644 index 0000000000..c6206a74ab --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2019-03-23-09-25-12.bpo-36345.L704Zv.rst @@ -0,0 +1,2 @@ +Using the code of the ``Tools/scripts/serve.py`` script as an example in the +:mod:`wsgiref` documentation. Contributed by Stéphane Wirtel. diff --git a/Misc/NEWS.d/next/Library/2019-03-22-13-47-52.bpo-36326.WCnEI5.rst b/Misc/NEWS.d/next/Library/2019-03-22-13-47-52.bpo-36326.WCnEI5.rst new file mode 100644 index 0000000000..e458a7024d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-03-22-13-47-52.bpo-36326.WCnEI5.rst @@ -0,0 +1,2 @@ +inspect.getdoc() can now find docstrings for member objects when __slots__ +is a dictionary. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 350ef77163..c82ba0cfd0 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -4675,27 +4675,9 @@ decode_locale_ex(PyObject *self, PyObject *args) static PyObject * -get_global_config(PyObject *self, PyObject *Py_UNUSED(args)) +get_configs(PyObject *self, PyObject *Py_UNUSED(args)) { - return _Py_GetGlobalVariablesAsDict(); -} - - -static PyObject * -get_core_config(PyObject *self, PyObject *Py_UNUSED(args)) -{ - PyInterpreterState *interp = _PyInterpreterState_Get(); - const _PyCoreConfig *config = _PyInterpreterState_GetCoreConfig(interp); - return _PyCoreConfig_AsDict(config); -} - - -static PyObject * -get_main_config(PyObject *self, PyObject *Py_UNUSED(args)) -{ - PyInterpreterState *interp = _PyInterpreterState_Get(); - const _PyMainInterpreterConfig *config = _PyInterpreterState_GetMainConfig(interp); - return _PyMainInterpreterConfig_AsDict(config); + return _Py_GetConfigsAsDict(); } @@ -4942,9 +4924,7 @@ static PyMethodDef TestMethods[] = { {"bad_get", (PyCFunction)(void(*)(void))bad_get, METH_FASTCALL}, {"EncodeLocaleEx", encode_locale_ex, METH_VARARGS}, {"DecodeLocaleEx", decode_locale_ex, METH_VARARGS}, - {"get_global_config", get_global_config, METH_NOARGS}, - {"get_core_config", get_core_config, METH_NOARGS}, - {"get_main_config", get_main_config, METH_NOARGS}, + {"get_configs", get_configs, METH_NOARGS}, #ifdef Py_REF_DEBUG {"negative_refcount", negative_refcount, METH_NOARGS}, #endif diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 536f7fa625..103029d251 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -4089,6 +4089,7 @@ itertools_count_impl(PyTypeObject *type, PyObject *long_cnt, lz = (countobject *)type->tp_alloc(type, 0); if (lz == NULL) { Py_XDECREF(long_cnt); + Py_DECREF(long_step); return NULL; } lz->cnt = cnt; diff --git a/Modules/main.c b/Modules/main.c index 197c9b39eb..9fcc76e58a 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -294,7 +294,7 @@ pymain_init_preconfig(const _PyArgv *args) goto done; } - err = _Py_PreInitializeFromPreConfig(&config); + err = _Py_PreInitializeInPlace(&config); done: _PyPreConfig_Clear(&config); @@ -311,11 +311,6 @@ pymain_init_coreconfig(_PyCoreConfig *config, const _PyArgv *args, return err; } - err = _PyCoreConfig_Write(config); - if (_Py_INIT_FAILED(err)) { - return err; - } - return _Py_InitializeCore(interp_p, config); } @@ -483,7 +478,7 @@ pymain_header(const _PyCoreConfig *config) static void pymain_import_readline(const _PyCoreConfig *config) { - if (config->preconfig.isolated) { + if (config->isolated) { return; } if (!config->inspect && RUN_CODE(config)) { @@ -655,7 +650,7 @@ pymain_run_file(_PyCoreConfig *config, PyCompilerFlags *cf) static void pymain_run_startup(_PyCoreConfig *config, PyCompilerFlags *cf) { - const char *startup = _PyCoreConfig_GetEnv(config, "PYTHONSTARTUP"); + const char *startup = _Py_GetEnv(config->use_environment, "PYTHONSTARTUP"); if (startup == NULL) { return; } @@ -735,7 +730,7 @@ pymain_repl(_PyCoreConfig *config, PyCompilerFlags *cf, int *exitcode) { /* Check this environment variable at the end, to give programs the opportunity to set it from Python. */ - if (!Py_InspectFlag && _PyCoreConfig_GetEnv(config, "PYTHONINSPECT")) { + if (!Py_InspectFlag && _Py_GetEnv(config->use_environment, "PYTHONINSPECT")) { Py_InspectFlag = 1; config->inspect = 1; } @@ -775,7 +770,7 @@ pymain_run_python(PyInterpreterState *interp, int *exitcode) goto done; } } - else if (!config->preconfig.isolated) { + else if (!config->isolated) { PyObject *path0 = NULL; if (_PyPathConfig_ComputeSysPath0(&config->argv, &path0)) { if (path0 == NULL) { diff --git a/Parser/pgen/keywordgen.py b/Parser/pgen/keywordgen.py new file mode 100644 index 0000000000..eeb3ef739f --- /dev/null +++ b/Parser/pgen/keywordgen.py @@ -0,0 +1,60 @@ +"""Generate Lib/keyword.py from the Grammar and Tokens files using pgen""" + +import argparse + +from .pgen import ParserGenerator + +TEMPLATE = r''' +"""Keywords (from "Grammar/Grammar") + +This file is automatically generated; please don't muck it up! + +To update the symbols in this file, 'cd' to the top directory of +the python source tree and run: + + python3 -m Parser.pgen.keywordgen Grammar/Grammar \ + Grammar/Tokens \ + Lib/keyword.py + +Alternatively, you can run 'make regen-keyword'. +""" + +__all__ = ["iskeyword", "kwlist"] + +kwlist = [ + {keywords} +] + +iskeyword = frozenset(kwlist).__contains__ +'''.lstrip() + +EXTRA_KEYWORDS = ["async", "await"] + + +def main(): + parser = argparse.ArgumentParser(description="Generate the Lib/keywords.py " + "file from the grammar.") + parser.add_argument( + "grammar", type=str, help="The file with the grammar definition in EBNF format" + ) + parser.add_argument( + "tokens", type=str, help="The file with the token definitions" + ) + parser.add_argument( + "keyword_file", + type=argparse.FileType('w'), + help="The path to write the keyword definitions", + ) + args = parser.parse_args() + p = ParserGenerator(args.grammar, args.tokens) + grammar = p.make_grammar() + + with args.keyword_file as thefile: + all_keywords = sorted(list(grammar.keywords) + EXTRA_KEYWORDS) + + keywords = ",\n ".join(map(repr, all_keywords)) + thefile.write(TEMPLATE.format(keywords=keywords)) + + +if __name__ == "__main__": + main() diff --git a/Programs/_freeze_importlib.c b/Programs/_freeze_importlib.c index e6c51a4a1d..0818012d8c 100644 --- a/Programs/_freeze_importlib.c +++ b/Programs/_freeze_importlib.c @@ -77,7 +77,7 @@ main(int argc, char *argv[]) text[text_size] = '\0'; _PyCoreConfig config = _PyCoreConfig_INIT; - config.preconfig.use_environment = 0; + config.use_environment = 0; config.user_site_directory = 0; config.site_import = 0; config.program_name = L"./_freeze_importlib"; diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 7c143f1ef3..70ef45f928 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -301,64 +301,29 @@ static int test_initialize_pymain(void) static int dump_config_impl(void) { - PyObject *config = NULL; - PyObject *dict = NULL; - - config = PyDict_New(); + PyObject *config = _Py_GetConfigsAsDict(); if (config == NULL) { - goto error; - } - - /* global config */ - dict = _Py_GetGlobalVariablesAsDict(); - if (dict == NULL) { - goto error; - } - if (PyDict_SetItemString(config, "global_config", dict) < 0) { - goto error; - } - Py_CLEAR(dict); - - /* core config */ - PyInterpreterState *interp = _PyInterpreterState_Get(); - const _PyCoreConfig *core_config = _PyInterpreterState_GetCoreConfig(interp); - dict = _PyCoreConfig_AsDict(core_config); - if (dict == NULL) { - goto error; + return -1; } - if (PyDict_SetItemString(config, "core_config", dict) < 0) { - goto error; - } - Py_CLEAR(dict); - /* main config */ - const _PyMainInterpreterConfig *main_config = _PyInterpreterState_GetMainConfig(interp); - dict = _PyMainInterpreterConfig_AsDict(main_config); - if (dict == NULL) { - goto error; + PyObject *res; + PyObject *json = PyImport_ImportModule("json"); + if (json) { + res = PyObject_CallMethod(json, "dumps", "O", config); + Py_DECREF(json); } - if (PyDict_SetItemString(config, "main_config", dict) < 0) { - goto error; + else { + res = NULL; } - Py_CLEAR(dict); - - PyObject *json = PyImport_ImportModule("json"); - PyObject *res = PyObject_CallMethod(json, "dumps", "O", config); - Py_DECREF(json); Py_CLEAR(config); if (res == NULL) { - goto error; + return -1; } PySys_FormatStdout("%S\n", res); Py_DECREF(res); return 0; - -error: - Py_XDECREF(config); - Py_XDECREF(dict); - return -1; } @@ -432,6 +397,22 @@ static int test_init_global_config(void) static int test_init_from_config(void) { + _PyInitError err; + + _PyPreConfig preconfig = _PyPreConfig_INIT; + + putenv("PYTHONMALLOC=malloc_debug"); + preconfig.allocator = "malloc"; + + putenv("PYTHONUTF8=0"); + Py_UTF8Mode = 0; + preconfig.utf8_mode = 1; + + err = _Py_PreInitializeFromPreConfig(&preconfig); + if (_Py_INIT_FAILED(err)) { + _Py_ExitInitError(err); + } + /* Test _Py_InitializeFromConfig() */ _PyCoreConfig config = _PyCoreConfig_INIT; config.install_signal_handlers = 0; @@ -442,9 +423,6 @@ static int test_init_from_config(void) config.use_hash_seed = 1; config.hash_seed = 123; - putenv("PYTHONMALLOC=malloc_debug"); - config.preconfig.allocator = "malloc"; - /* dev_mode=1 is tested in test_init_dev_mode() */ putenv("PYTHONFAULTHANDLER="); @@ -465,10 +443,6 @@ static int test_init_from_config(void) /* FIXME: test coerce_c_locale and coerce_c_locale_warn */ - putenv("PYTHONUTF8=0"); - Py_UTF8Mode = 0; - config.preconfig.utf8_mode = 1; - putenv("PYTHONPYCACHEPREFIX=env_pycache_prefix"); config.pycache_prefix = L"conf_pycache_prefix"; @@ -556,7 +530,7 @@ static int test_init_from_config(void) Py_FrozenFlag = 0; config._frozen = 1; - _PyInitError err = _Py_InitializeFromConfig(&config); + err = _Py_InitializeFromConfig(&config); /* Don't call _PyCoreConfig_Clear() since all strings are static */ if (_Py_INIT_FAILED(err)) { _Py_ExitInitError(err); @@ -642,20 +616,30 @@ static int test_init_env_dev_mode_alloc(void) static int test_init_isolated(void) { + _PyInitError err; + + _PyPreConfig preconfig = _PyPreConfig_INIT; + + /* Set coerce_c_locale and utf8_mode to not depend on the locale */ + preconfig.coerce_c_locale = 0; + preconfig.utf8_mode = 0; + + err = _Py_PreInitializeFromPreConfig(&preconfig); + if (_Py_INIT_FAILED(err)) { + _Py_ExitInitError(err); + } + /* Test _PyCoreConfig.isolated=1 */ _PyCoreConfig config = _PyCoreConfig_INIT; Py_IsolatedFlag = 0; - config.preconfig.isolated = 1; + config.isolated = 1; - /* Set coerce_c_locale and utf8_mode to not depend on the locale */ - config.preconfig.coerce_c_locale = 0; - config.preconfig.utf8_mode = 0; /* Use path starting with "./" avoids a search along the PATH */ config.program_name = L"./_testembed"; test_init_env_dev_mode_putenvs(); - _PyInitError err = _Py_InitializeFromConfig(&config); + err = _Py_InitializeFromConfig(&config); if (_Py_INIT_FAILED(err)) { _Py_ExitInitError(err); } @@ -670,7 +654,7 @@ static int test_init_dev_mode(void) _PyCoreConfig config = _PyCoreConfig_INIT; putenv("PYTHONFAULTHANDLER="); putenv("PYTHONMALLOC="); - config.preconfig.dev_mode = 1; + config.dev_mode = 1; config.program_name = L"./_testembed"; _PyInitError err = _Py_InitializeFromConfig(&config); if (_Py_INIT_FAILED(err)) { diff --git a/Python/ceval.c b/Python/ceval.c index 40320bf357..28e923219d 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -4948,7 +4948,7 @@ import_from(PyObject *v, PyObject *name) } x = PyImport_GetModule(fullmodname); Py_DECREF(fullmodname); - if (x == NULL) { + if (x == NULL && !PyErr_Occurred()) { goto error; } Py_DECREF(pkgname); @@ -4971,7 +4971,7 @@ import_from(PyObject *v, PyObject *name) "cannot import name %R from %R (unknown location)", name, pkgname_or_unknown ); - /* NULL check for errmsg done by PyErr_SetImportError. */ + /* NULL checks for errmsg and pkgname done by PyErr_SetImportError. */ PyErr_SetImportError(errmsg, pkgname, NULL); } else { @@ -4979,7 +4979,7 @@ import_from(PyObject *v, PyObject *name) "cannot import name %R from %R (%S)", name, pkgname_or_unknown, pkgpath ); - /* NULL check for errmsg done by PyErr_SetImportError. */ + /* NULL checks for errmsg and pkgname done by PyErr_SetImportError. */ PyErr_SetImportError(errmsg, pkgname, pkgpath); } diff --git a/Python/coreconfig.c b/Python/coreconfig.c index ba5abb6ca4..2e6eb40298 100644 --- a/Python/coreconfig.c +++ b/Python/coreconfig.c @@ -131,7 +131,7 @@ int Py_LegacyWindowsStdioFlag = 0; /* Uses FileIO instead of WindowsConsoleIO */ #endif -PyObject * +static PyObject * _Py_GetGlobalVariablesAsDict(void) { PyObject *dict, *obj; @@ -469,8 +469,6 @@ Py_GetArgcArgv(int *argc, wchar_t ***argv) void _PyCoreConfig_Clear(_PyCoreConfig *config) { - _PyPreConfig_Clear(&config->preconfig); - #define CLEAR(ATTR) \ do { \ PyMem_RawFree(ATTR); \ @@ -514,10 +512,6 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2) { _PyCoreConfig_Clear(config); - if (_PyPreConfig_Copy(&config->preconfig, &config2->preconfig) < 0) { - return -1; - } - #define COPY_ATTR(ATTR) config->ATTR = config2->ATTR #define COPY_STR_ATTR(ATTR) \ do { \ @@ -544,6 +538,9 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2) } \ } while (0) + COPY_ATTR(isolated); + COPY_ATTR(use_environment); + COPY_ATTR(dev_mode); COPY_ATTR(install_signal_handlers); COPY_ATTR(use_hash_seed); COPY_ATTR(hash_seed); @@ -610,21 +607,21 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2) } -const char* +static const char* _PyCoreConfig_GetEnv(const _PyCoreConfig *config, const char *name) { - return _PyPreConfig_GetEnv(&config->preconfig, name); + return _Py_GetEnv(config->use_environment, name); } -int +static int _PyCoreConfig_GetEnvDup(const _PyCoreConfig *config, wchar_t **dest, wchar_t *wname, char *name) { - assert(config->preconfig.use_environment >= 0); + assert(config->use_environment >= 0); - if (!config->preconfig.use_environment) { + if (!config->use_environment) { *dest = NULL; return 0; } @@ -668,8 +665,6 @@ _PyCoreConfig_GetEnvDup(const _PyCoreConfig *config, void _PyCoreConfig_GetGlobalConfig(_PyCoreConfig *config) { - _PyPreConfig_GetGlobalConfig(&config->preconfig); - #define COPY_FLAG(ATTR, VALUE) \ if (config->ATTR == -1) { \ config->ATTR = VALUE; \ @@ -679,6 +674,8 @@ _PyCoreConfig_GetGlobalConfig(_PyCoreConfig *config) config->ATTR = !(VALUE); \ } + COPY_FLAG(isolated, Py_IsolatedFlag); + COPY_NOT_FLAG(use_environment, Py_IgnoreEnvironmentFlag); COPY_FLAG(bytes_warning, Py_BytesWarningFlag); COPY_FLAG(inspect, Py_InspectFlag); COPY_FLAG(interactive, Py_InteractiveFlag); @@ -714,6 +711,8 @@ _PyCoreConfig_SetGlobalConfig(const _PyCoreConfig *config) VAR = !config->ATTR; \ } + COPY_FLAG(isolated, Py_IsolatedFlag); + COPY_NOT_FLAG(use_environment, Py_IgnoreEnvironmentFlag); COPY_FLAG(bytes_warning, Py_BytesWarningFlag); COPY_FLAG(inspect, Py_InspectFlag); COPY_FLAG(interactive, Py_InteractiveFlag); @@ -924,34 +923,34 @@ config_wstr_to_int(const wchar_t *wstr, int *result) static _PyInitError config_read_env_vars(_PyCoreConfig *config) { - _PyPreConfig *preconfig = &config->preconfig; + int use_env = config->use_environment; /* Get environment variables */ - _Py_get_env_flag(preconfig, &config->parser_debug, "PYTHONDEBUG"); - _Py_get_env_flag(preconfig, &config->verbose, "PYTHONVERBOSE"); - _Py_get_env_flag(preconfig, &config->optimization_level, "PYTHONOPTIMIZE"); - _Py_get_env_flag(preconfig, &config->inspect, "PYTHONINSPECT"); + _Py_get_env_flag(use_env, &config->parser_debug, "PYTHONDEBUG"); + _Py_get_env_flag(use_env, &config->verbose, "PYTHONVERBOSE"); + _Py_get_env_flag(use_env, &config->optimization_level, "PYTHONOPTIMIZE"); + _Py_get_env_flag(use_env, &config->inspect, "PYTHONINSPECT"); int dont_write_bytecode = 0; - _Py_get_env_flag(preconfig, &dont_write_bytecode, "PYTHONDONTWRITEBYTECODE"); + _Py_get_env_flag(use_env, &dont_write_bytecode, "PYTHONDONTWRITEBYTECODE"); if (dont_write_bytecode) { config->write_bytecode = 0; } int no_user_site_directory = 0; - _Py_get_env_flag(preconfig, &no_user_site_directory, "PYTHONNOUSERSITE"); + _Py_get_env_flag(use_env, &no_user_site_directory, "PYTHONNOUSERSITE"); if (no_user_site_directory) { config->user_site_directory = 0; } int unbuffered_stdio = 0; - _Py_get_env_flag(preconfig, &unbuffered_stdio, "PYTHONUNBUFFERED"); + _Py_get_env_flag(use_env, &unbuffered_stdio, "PYTHONUNBUFFERED"); if (unbuffered_stdio) { config->buffered_stdio = 0; } #ifdef MS_WINDOWS - _Py_get_env_flag(preconfig, &config->legacy_windows_stdio, + _Py_get_env_flag(use_env, &config->legacy_windows_stdio, "PYTHONLEGACYWINDOWSSTDIO"); #endif @@ -1149,7 +1148,8 @@ get_locale_encoding(char **locale_encoding) static _PyInitError -config_init_stdio_encoding(_PyCoreConfig *config) +config_init_stdio_encoding(_PyCoreConfig *config, + const _PyPreConfig *preconfig) { /* If Py_SetStandardStreamEncoding() have been called, use these parameters. */ @@ -1219,7 +1219,7 @@ config_init_stdio_encoding(_PyCoreConfig *config) } /* UTF-8 Mode uses UTF-8/surrogateescape */ - if (config->preconfig.utf8_mode) { + if (preconfig->utf8_mode) { if (config->stdio_encoding == NULL) { config->stdio_encoding = _PyMem_RawStrdup("utf-8"); if (config->stdio_encoding == NULL) { @@ -1254,10 +1254,10 @@ config_init_stdio_encoding(_PyCoreConfig *config) static _PyInitError -config_init_fs_encoding(_PyCoreConfig *config) +config_init_fs_encoding(_PyCoreConfig *config, const _PyPreConfig *preconfig) { #ifdef MS_WINDOWS - if (config->preconfig.legacy_windows_fs_encoding) { + if (preconfig->legacy_windows_fs_encoding) { /* Legacy Windows filesystem encoding: mbcs/replace */ if (config->filesystem_encoding == NULL) { config->filesystem_encoding = _PyMem_RawStrdup("mbcs"); @@ -1292,7 +1292,7 @@ config_init_fs_encoding(_PyCoreConfig *config) } #else if (config->filesystem_encoding == NULL) { - if (config->preconfig.utf8_mode) { + if (preconfig->utf8_mode) { /* UTF-8 Mode use: utf-8/surrogateescape */ config->filesystem_encoding = _PyMem_RawStrdup("utf-8"); /* errors defaults to surrogateescape above */ @@ -1341,12 +1341,8 @@ config_read_impl(_PyCoreConfig *config, _PyPreCmdline *cmdline) { _PyInitError err; - err = _Py_PreInitializeFromConfig(config); - if (_Py_INIT_FAILED(err)) { - return err; - } - - _PyPreCmdline_GetPreConfig(cmdline, &_PyRuntime.preconfig); + const _PyPreConfig *preconfig = &_PyRuntime.preconfig; + _PyPreCmdline_GetPreConfig(cmdline, preconfig); _PyPreCmdline_GetCoreConfig(cmdline, config); err = _PyPreCmdline_Read(cmdline); @@ -1360,19 +1356,16 @@ config_read_impl(_PyCoreConfig *config, _PyPreCmdline *cmdline) return _Py_INIT_NO_MEMORY(); } - if (_PyPreConfig_Copy(&config->preconfig, &_PyRuntime.preconfig) < 0) { - return _Py_INIT_NO_MEMORY(); - } - _PyCoreConfig_GetGlobalConfig(config); - assert(config->preconfig.use_environment >= 0); + assert(config->use_environment >= 0); - if (config->preconfig.isolated > 0) { + if (config->isolated > 0) { + config->use_environment = 0; config->user_site_directory = 0; } - if (config->preconfig.use_environment) { + if (config->use_environment) { err = config_read_env_vars(config); if (_Py_INIT_FAILED(err)) { return err; @@ -1421,7 +1414,7 @@ config_read_impl(_PyCoreConfig *config, _PyPreCmdline *cmdline) } /* default values */ - if (config->preconfig.dev_mode) { + if (config->dev_mode) { if (config->faulthandler < 0) { config->faulthandler = 1; } @@ -1438,13 +1431,13 @@ config_read_impl(_PyCoreConfig *config, _PyPreCmdline *cmdline) } if (config->filesystem_encoding == NULL || config->filesystem_errors == NULL) { - err = config_init_fs_encoding(config); + err = config_init_fs_encoding(config, preconfig); if (_Py_INIT_FAILED(err)) { return err; } } - err = config_init_stdio_encoding(config); + err = config_init_stdio_encoding(config, preconfig); if (_Py_INIT_FAILED(err)) { return err; } @@ -1456,7 +1449,7 @@ config_read_impl(_PyCoreConfig *config, _PyPreCmdline *cmdline) } } - assert(config->preconfig.use_environment >= 0); + assert(config->use_environment >= 0); assert(config->filesystem_encoding != NULL); assert(config->filesystem_errors != NULL); assert(config->stdio_encoding != NULL); @@ -1544,26 +1537,15 @@ config_init_stdio(const _PyCoreConfig *config) - set Py_xxx global configuration variables - initialize C standard streams (stdin, stdout, stderr) */ -_PyInitError +void _PyCoreConfig_Write(const _PyCoreConfig *config) { _PyCoreConfig_SetGlobalConfig(config); config_init_stdio(config); - - /* Write the new pre-configuration into _PyRuntime */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - int res = _PyPreConfig_Copy(&_PyRuntime.preconfig, &config->preconfig); - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - if (res < 0) { - return _Py_INIT_NO_MEMORY(); - } - - return _Py_INIT_OK(); } -PyObject * +static PyObject * _PyCoreConfig_AsDict(const _PyCoreConfig *config) { PyObject *dict; @@ -1573,11 +1555,6 @@ _PyCoreConfig_AsDict(const _PyCoreConfig *config) return NULL; } - if (_PyPreConfig_AsDict(&config->preconfig, dict) < 0) { - Py_DECREF(dict); - return NULL; - } - #define SET_ITEM(KEY, EXPR) \ do { \ PyObject *obj = (EXPR); \ @@ -1609,6 +1586,9 @@ _PyCoreConfig_AsDict(const _PyCoreConfig *config) #define SET_ITEM_WSTRLIST(LIST) \ SET_ITEM(#LIST, _PyWstrList_AsList(&config->LIST)) + SET_ITEM_INT(isolated); + SET_ITEM_INT(use_environment); + SET_ITEM_INT(dev_mode); SET_ITEM_INT(install_signal_handlers); SET_ITEM_INT(use_hash_seed); SET_ITEM_UINT(hash_seed); @@ -1950,7 +1930,7 @@ config_init_warnoptions(_PyCoreConfig *config, const _PyCmdline *cmdline) * the lowest precedence entries first so that later entries override them. */ - if (config->preconfig.dev_mode) { + if (config->dev_mode) { if (_PyWstrList_Append(&config->warnoptions, L"default")) { return _Py_INIT_NO_MEMORY(); } @@ -2106,7 +2086,7 @@ config_from_cmdline(_PyCoreConfig *config, _PyCmdline *cmdline) return err; } - if (config->preconfig.use_environment) { + if (config->use_environment) { err = cmdline_init_env_warnoptions(cmdline, config); if (_Py_INIT_FAILED(err)) { return err; @@ -2158,3 +2138,67 @@ done: cmdline_clear(&cmdline); return err; } + + +PyObject* +_Py_GetConfigsAsDict(void) +{ + PyObject *config = NULL; + PyObject *dict = NULL; + + config = PyDict_New(); + if (config == NULL) { + goto error; + } + + /* global config */ + dict = _Py_GetGlobalVariablesAsDict(); + if (dict == NULL) { + goto error; + } + if (PyDict_SetItemString(config, "global_config", dict) < 0) { + goto error; + } + Py_CLEAR(dict); + + /* pre config */ + PyInterpreterState *interp = _PyInterpreterState_Get(); + const _PyPreConfig *pre_config = &_PyRuntime.preconfig; + dict = _PyPreConfig_AsDict(pre_config); + if (dict == NULL) { + goto error; + } + if (PyDict_SetItemString(config, "pre_config", dict) < 0) { + goto error; + } + Py_CLEAR(dict); + + /* core config */ + const _PyCoreConfig *core_config = _PyInterpreterState_GetCoreConfig(interp); + dict = _PyCoreConfig_AsDict(core_config); + if (dict == NULL) { + goto error; + } + if (PyDict_SetItemString(config, "core_config", dict) < 0) { + goto error; + } + Py_CLEAR(dict); + + /* main config */ + const _PyMainInterpreterConfig *main_config = _PyInterpreterState_GetMainConfig(interp); + dict = _PyMainInterpreterConfig_AsDict(main_config); + if (dict == NULL) { + goto error; + } + if (PyDict_SetItemString(config, "main_config", dict) < 0) { + goto error; + } + Py_CLEAR(dict); + + return config; + +error: + Py_XDECREF(config); + Py_XDECREF(dict); + return NULL; +} diff --git a/Python/import.c b/Python/import.c index bf3a99414f..c00c3aa640 100644 --- a/Python/import.c +++ b/Python/import.c @@ -966,11 +966,10 @@ exec_code_in_module(PyObject *name, PyObject *module_dict, PyObject *code_object Py_DECREF(v); m = PyImport_GetModule(name); - if (m == NULL) { + if (m == NULL && !PyErr_Occurred()) { PyErr_Format(PyExc_ImportError, "Loaded module %R not found in sys.modules", name); - return NULL; } return m; @@ -1735,6 +1734,10 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, } mod = PyImport_GetModule(abs_name); + if (mod == NULL && PyErr_Occurred()) { + goto error; + } + if (mod != NULL && mod != Py_None) { _Py_IDENTIFIER(__spec__); _Py_IDENTIFIER(_lock_unlock_module); @@ -1810,9 +1813,11 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, final_mod = PyImport_GetModule(to_return); Py_DECREF(to_return); if (final_mod == NULL) { - PyErr_Format(PyExc_KeyError, - "%R not in sys.modules as expected", - to_return); + if (!PyErr_Occurred()) { + PyErr_Format(PyExc_KeyError, + "%R not in sys.modules as expected", + to_return); + } goto error; } } @@ -1875,6 +1880,10 @@ PyImport_ReloadModule(PyObject *m) PyObject *reloaded_module = NULL; PyObject *imp = _PyImport_GetModuleId(&PyId_imp); if (imp == NULL) { + if (PyErr_Occurred()) { + return NULL; + } + imp = PyImport_ImportModule("imp"); if (imp == NULL) { return NULL; diff --git a/Python/pathconfig.c b/Python/pathconfig.c index f0b13fd1b0..7fea7c3667 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -331,7 +331,7 @@ _PyCoreConfig_CalculatePathConfig(_PyCoreConfig *config) #endif if (path_config.isolated != -1) { - config->preconfig.isolated = path_config.isolated; + config->isolated = path_config.isolated; } if (path_config.site_import != -1) { config->site_import = path_config.site_import; diff --git a/Python/preconfig.c b/Python/preconfig.c index ac87a7a3c7..d336352d93 100644 --- a/Python/preconfig.c +++ b/Python/preconfig.c @@ -143,6 +143,23 @@ _PyPreCmdline_GetPreConfig(_PyPreCmdline *cmdline, const _PyPreConfig *config) COPY_ATTR(use_environment); COPY_ATTR(isolated); + COPY_ATTR(dev_mode); + +#undef COPY_ATTR +} + + +void +_PyPreCmdline_SetPreConfig(const _PyPreCmdline *cmdline, _PyPreConfig *config) +{ +#define COPY_ATTR(ATTR) \ + if (cmdline->ATTR != -1) { \ + config->ATTR = cmdline->ATTR; \ + } + + COPY_ATTR(use_environment); + COPY_ATTR(isolated); + COPY_ATTR(dev_mode); #undef COPY_ATTR } @@ -152,12 +169,13 @@ void _PyPreCmdline_GetCoreConfig(_PyPreCmdline *cmdline, const _PyCoreConfig *config) { #define COPY_ATTR(ATTR) \ - if (config->preconfig.ATTR != -1) { \ - cmdline->ATTR = config->preconfig.ATTR; \ + if (config->ATTR != -1) { \ + cmdline->ATTR = config->ATTR; \ } COPY_ATTR(use_environment); COPY_ATTR(isolated); + COPY_ATTR(dev_mode); #undef COPY_ATTR } @@ -167,12 +185,13 @@ void _PyPreCmdline_SetCoreConfig(const _PyPreCmdline *cmdline, _PyCoreConfig *config) { #define COPY_ATTR(ATTR) \ - if (config->preconfig.ATTR == -1 && cmdline->ATTR != -1) { \ - config->preconfig.ATTR = cmdline->ATTR; \ + if (config->ATTR == -1 && cmdline->ATTR != -1) { \ + config->ATTR = cmdline->ATTR; \ } COPY_ATTR(use_environment); COPY_ATTR(isolated); + COPY_ATTR(dev_mode); #undef COPY_ATTR } @@ -206,13 +225,13 @@ _PyPreConfig_Copy(_PyPreConfig *config, const _PyPreConfig *config2) COPY_ATTR(isolated); COPY_ATTR(use_environment); + COPY_ATTR(dev_mode); COPY_ATTR(coerce_c_locale); COPY_ATTR(coerce_c_locale_warn); #ifdef MS_WINDOWS COPY_ATTR(legacy_windows_fs_encoding); #endif COPY_ATTR(utf8_mode); - COPY_ATTR(dev_mode); COPY_STR_ATTR(allocator); #undef COPY_ATTR @@ -270,11 +289,11 @@ _PyPreConfig_SetGlobalConfig(const _PyPreConfig *config) const char* -_PyPreConfig_GetEnv(const _PyPreConfig *config, const char *name) +_Py_GetEnv(int use_environment, const char *name) { - assert(config->use_environment >= 0); + assert(use_environment >= 0); - if (!config->use_environment) { + if (!use_environment) { return NULL; } @@ -288,6 +307,13 @@ _PyPreConfig_GetEnv(const _PyPreConfig *config, const char *name) } +static const char* +_PyPreConfig_GetEnv(const _PyPreConfig *config, const char *name) +{ + return _Py_GetEnv(config->use_environment, name); +} + + int _Py_str_to_int(const char *str, int *result) { @@ -307,9 +333,9 @@ _Py_str_to_int(const char *str, int *result) void -_Py_get_env_flag(_PyPreConfig *config, int *flag, const char *name) +_Py_get_env_flag(int use_environment, int *flag, const char *name) { - const char *var = _PyPreConfig_GetEnv(config, name); + const char *var = _Py_GetEnv(use_environment, name); if (!var) { return; } @@ -434,8 +460,9 @@ preconfig_read(_PyPreConfig *config, _PyPreCmdline *cmdline) /* legacy_windows_fs_encoding, utf8_mode, coerce_c_locale */ if (config->use_environment) { #ifdef MS_WINDOWS - _Py_get_env_flag(config, &config->legacy_windows_fs_encoding, - "PYTHONLEGACYWINDOWSFSENCODING"); + _Py_get_env_flag(config->use_environment, + &config->legacy_windows_fs_encoding, + "PYTHONLEGACYWINDOWSFSENCODING"); #endif const char *env = _PyPreConfig_GetEnv(config, "PYTHONCOERCECLOCALE"); @@ -559,24 +586,16 @@ get_ctype_locale(char **locale_p) } -void -_PyPreCmdline_SetPreConfig(const _PyPreCmdline *cmdline, _PyPreConfig *config) +PyObject* +_PyPreConfig_AsDict(const _PyPreConfig *config) { -#define COPY_ATTR(ATTR) \ - if (cmdline->ATTR != -1) { \ - config->ATTR = cmdline->ATTR; \ - } - - COPY_ATTR(use_environment); - COPY_ATTR(isolated); - -#undef COPY_ATTR -} + PyObject *dict; + dict = PyDict_New(); + if (dict == NULL) { + return NULL; + } -int -_PyPreConfig_AsDict(const _PyPreConfig *config, PyObject *dict) -{ #define SET_ITEM(KEY, EXPR) \ do { \ PyObject *obj = (EXPR); \ @@ -608,10 +627,11 @@ _PyPreConfig_AsDict(const _PyPreConfig *config, PyObject *dict) #endif SET_ITEM_INT(dev_mode); SET_ITEM_STR(allocator); - return 0; + return dict; fail: - return -1; + Py_DECREF(dict); + return NULL; #undef FROM_STRING #undef SET_ITEM @@ -696,7 +716,7 @@ _PyPreConfig_Read(_PyPreConfig *config, const _PyArgv *args, if (coreconfig) { _PyPreCmdline_GetCoreConfig(&cmdline, coreconfig); if (config->dev_mode == -1) { - config->dev_mode = coreconfig->preconfig.dev_mode; + config->dev_mode = coreconfig->dev_mode; } } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 66cadc99c7..b12fa820e9 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -286,9 +286,10 @@ static const char *_C_LOCALE_WARNING = "locales is recommended.\n"; static void -_emit_stderr_warning_for_legacy_locale(const _PyCoreConfig *core_config) +_emit_stderr_warning_for_legacy_locale(void) { - if (core_config->preconfig.coerce_c_locale_warn && _Py_LegacyLocaleDetected()) { + const _PyPreConfig *preconfig = &_PyRuntime.preconfig; + if (preconfig->coerce_c_locale_warn && _Py_LegacyLocaleDetected()) { PySys_FormatStderr("%s", _C_LOCALE_WARNING); } } @@ -675,6 +676,8 @@ _Py_InitializeCore_impl(PyInterpreterState **interp_p, { PyInterpreterState *interp; + _PyCoreConfig_Write(core_config); + _PyInitError err = pycore_init_runtime(core_config); if (_Py_INIT_FAILED(err)) { return err; @@ -720,54 +723,64 @@ pyinit_preinit(_PyPreConfig *config, const _PyCoreConfig *coreconfig) { _PyInitError err; + _PyPreConfig local_config = _PyPreConfig_INIT; + if (!config) { + config = &local_config; + } err = _PyRuntime_Initialize(); if (_Py_INIT_FAILED(err)) { - return err; + goto done; } if (_PyRuntime.pre_initialized) { /* If it's already configured: ignored the new configuration */ - return _Py_INIT_OK(); - } - - if (!src_config && coreconfig) { - src_config = &coreconfig->preconfig; + err = _Py_INIT_OK(); + goto done; } if (src_config) { if (_PyPreConfig_Copy(config, src_config) < 0) { - return _Py_INIT_ERR("failed to copy pre config"); + err = _Py_INIT_ERR("failed to copy pre config"); + goto done; } } err = _PyPreConfig_Read(config, NULL, coreconfig); if (_Py_INIT_FAILED(err)) { - return err; + goto done; } err = _PyPreConfig_Write(config); if (_Py_INIT_FAILED(err)) { - return err; + goto done; } _PyRuntime.pre_initialized = 1; - return _Py_INIT_OK(); + err = _Py_INIT_OK(); + +done: + _PyPreConfig_Clear(&local_config); + return err; } _PyInitError _Py_PreInitialize(void) { - _PyPreConfig config = _PyPreConfig_INIT; - _PyInitError err = pyinit_preinit(&config, NULL, NULL); - _PyPreConfig_Clear(&config); - return err; + return pyinit_preinit(NULL, NULL, NULL); +} + + +_PyInitError +_Py_PreInitializeFromPreConfig(const _PyPreConfig *src_config) +{ + return pyinit_preinit(NULL, src_config, NULL); } _PyInitError -_Py_PreInitializeFromPreConfig(_PyPreConfig *config) +_Py_PreInitializeInPlace(_PyPreConfig *config) { return pyinit_preinit(config, NULL, NULL); } @@ -776,10 +789,7 @@ _Py_PreInitializeFromPreConfig(_PyPreConfig *config) _PyInitError _Py_PreInitializeFromConfig(const _PyCoreConfig *coreconfig) { - _PyPreConfig config = _PyPreConfig_INIT; - _PyInitError err = pyinit_preinit(&config, NULL, coreconfig); - _PyPreConfig_Clear(&config); - return err; + return pyinit_preinit(NULL, NULL, coreconfig); } @@ -964,7 +974,7 @@ _Py_InitializeMainInterpreter(PyInterpreterState *interp, } #ifndef MS_WINDOWS - _emit_stderr_warning_for_legacy_locale(core_config); + _emit_stderr_warning_for_legacy_locale(); #endif return _Py_INIT_OK(); @@ -2157,8 +2167,10 @@ wait_for_thread_shutdown(void) PyObject *result; PyObject *threading = _PyImport_GetModuleId(&PyId_threading); if (threading == NULL) { - /* threading not imported */ - PyErr_Clear(); + if (PyErr_Occurred()) { + PyErr_WriteUnraisable(NULL); + } + /* else: threading not imported */ return; } result = _PyObject_CallMethodId(threading, &PyId__shutdown, NULL); diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 4351a7fb37..12ec7d5918 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -283,7 +283,9 @@ sys_displayhook(PyObject *module, PyObject *o) builtins = _PyImport_GetModuleId(&PyId_builtins); if (builtins == NULL) { - PyErr_SetString(PyExc_RuntimeError, "lost builtins module"); + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_RuntimeError, "lost builtins module"); + } return NULL; } Py_DECREF(builtins); @@ -2156,6 +2158,7 @@ make_flags(void) { int pos = 0; PyObject *seq; + const _PyPreConfig *preconfig = &_PyRuntime.preconfig; const _PyCoreConfig *config = &_PyInterpreterState_GET_UNSAFE()->core_config; seq = PyStructSequence_New(&FlagsType); @@ -2172,16 +2175,16 @@ make_flags(void) SetFlag(!config->write_bytecode); SetFlag(!config->user_site_directory); SetFlag(!config->site_import); - SetFlag(!config->preconfig.use_environment); + SetFlag(!config->use_environment); SetFlag(config->verbose); /* SetFlag(saw_unbuffered_flag); */ /* SetFlag(skipfirstline); */ SetFlag(config->bytes_warning); SetFlag(config->quiet); SetFlag(config->use_hash_seed == 0 || config->hash_seed != 0); - SetFlag(config->preconfig.isolated); - PyStructSequence_SET_ITEM(seq, pos++, PyBool_FromLong(config->preconfig.dev_mode)); - SetFlag(config->preconfig.utf8_mode); + SetFlag(config->isolated); + PyStructSequence_SET_ITEM(seq, pos++, PyBool_FromLong(config->dev_mode)); + SetFlag(preconfig->utf8_mode); #undef SetFlag if (PyErr_Occurred()) { |