diff options
Diffstat (limited to 'Python/pythonrun.c')
| -rw-r--r-- | Python/pythonrun.c | 1482 |
1 files changed, 1094 insertions, 388 deletions
diff --git a/Python/pythonrun.c b/Python/pythonrun.c index a2e1e74589..dd32017574 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -11,13 +11,10 @@ #include "parsetok.h" #include "errcode.h" #include "code.h" -#include "compile.h" #include "symtable.h" -#include "pyarena.h" #include "ast.h" -#include "eval.h" #include "marshal.h" -#include "abstract.h" +#include "osdefs.h" #ifdef HAVE_SIGNAL_H #include <signal.h> @@ -35,6 +32,7 @@ #ifdef MS_WINDOWS #undef BYTE #include "windows.h" +#define PATH_MAX MAXPATHLEN #endif #ifndef Py_REF_DEBUG @@ -49,24 +47,32 @@ extern "C" { #endif -extern char *Py_GetPath(void); +extern wchar_t *Py_GetPath(void); extern grammar _PyParser_Grammar; /* From graminit.c */ /* Forward */ -static void initmain(void); +static void initmain(PyInterpreterState *interp); +static int initfsencoding(PyInterpreterState *interp); static void initsite(void); +static int initstdio(void); +static void flush_io(void); static PyObject *run_mod(mod_ty, const char *, PyObject *, PyObject *, PyCompilerFlags *, PyArena *); static PyObject *run_pyc_file(FILE *, const char *, PyObject *, PyObject *, PyCompilerFlags *); static void err_input(perrdetail *); +static void err_free(perrdetail *); static void initsigs(void); +static void call_py_exitfuncs(void); static void wait_for_thread_shutdown(void); -static void call_sys_exitfunc(void); static void call_ll_exitfuncs(void); -extern void _PyUnicode_Init(void); +extern int _PyUnicode_Init(void); extern void _PyUnicode_Fini(void); +extern int _PyLong_Init(void); +extern void PyLong_Fini(void); +extern int _PyFaulthandler_Init(void); +extern void _PyFaulthandler_Fini(void); #ifdef WITH_THREAD extern void _PyGILState_Init(PyInterpreterState *, PyThreadState *); @@ -75,6 +81,7 @@ extern void _PyGILState_Fini(void); int Py_DebugFlag; /* Needed by parser.c */ int Py_VerboseFlag; /* Needed by import.c */ +int Py_QuietFlag; /* Needed by sysmodule.c */ int Py_InteractiveFlag; /* Needed by Py_FdIsInteractive() below */ int Py_InspectFlag; /* Needed to determine whether to exit at SystemExit */ int Py_NoSiteFlag; /* Suppress 'import site' */ @@ -82,15 +89,13 @@ int Py_BytesWarningFlag; /* Warn on str(bytes) and str(buffer) */ int Py_DontWriteBytecodeFlag; /* Suppress writing bytecode files (*.py[co]) */ int Py_UseClassExceptionsFlag = 1; /* Needed by bltinmodule.c: deprecated */ int Py_FrozenFlag; /* Needed by getpath.c */ -int Py_UnicodeFlag = 0; /* Needed by compile.c */ int Py_IgnoreEnvironmentFlag; /* e.g. PYTHONPATH, PYTHONHOME */ -/* _XXX Py_QnewFlag should go away in 2.3. It's true iff -Qnew is passed, - on the command line, and is used in 2.2 by ceval.c to make all "/" divisions - true divisions (which they will be in 2.3). */ -int _Py_QnewFlag = 0; int Py_NoUserSiteDirectory = 0; /* for -s and site.py */ +int Py_UnbufferedStdioFlag = 0; /* Unbuffered binary std{in,out,err} */ int Py_HashRandomizationFlag = 0; /* for -R and PYTHONHASHSEED */ +PyThreadState *_Py_Finalizing = NULL; + /* PyModule_GetWarningsModule is no longer necessary as of 2.6 since _warnings is builtin. This API should not be used. */ PyObject * @@ -132,32 +137,130 @@ add_flag(int flag, const char *envs) return flag; } +static char* +get_codec_name(const char *encoding) +{ + char *name_utf8, *name_str; + PyObject *codec, *name = NULL; + _Py_IDENTIFIER(name); + + codec = _PyCodec_Lookup(encoding); + if (!codec) + goto error; + + name = _PyObject_GetAttrId(codec, &PyId_name); + Py_CLEAR(codec); + if (!name) + goto error; + + name_utf8 = _PyUnicode_AsString(name); + if (name_utf8 == NULL) + goto error; + name_str = strdup(name_utf8); + Py_DECREF(name); + if (name_str == NULL) { + PyErr_NoMemory(); + return NULL; + } + return name_str; + +error: + Py_XDECREF(codec); + Py_XDECREF(name); + return NULL; +} + +static char* +get_locale_encoding(void) +{ +#ifdef MS_WINDOWS + char codepage[100]; + PyOS_snprintf(codepage, sizeof(codepage), "cp%d", GetACP()); + return get_codec_name(codepage); +#elif defined(HAVE_LANGINFO_H) && defined(CODESET) + char* codeset = nl_langinfo(CODESET); + if (!codeset || codeset[0] == '\0') { + PyErr_SetString(PyExc_ValueError, "CODESET is not set or empty"); + return NULL; + } + return get_codec_name(codeset); +#else + PyErr_SetNone(PyExc_NotImplementedError); + return NULL; +#endif +} + +static void +import_init(PyInterpreterState *interp, PyObject *sysmod) +{ + PyObject *importlib; + PyObject *impmod; + PyObject *sys_modules; + PyObject *value; + + /* Import _importlib through its frozen version, _frozen_importlib. */ + if (PyImport_ImportFrozenModule("_frozen_importlib") <= 0) { + Py_FatalError("Py_Initialize: can't import _frozen_importlib"); + } + else if (Py_VerboseFlag) { + PySys_FormatStderr("import _frozen_importlib # frozen\n"); + } + importlib = PyImport_AddModule("_frozen_importlib"); + if (importlib == NULL) { + Py_FatalError("Py_Initialize: couldn't get _frozen_importlib from " + "sys.modules"); + } + interp->importlib = importlib; + Py_INCREF(interp->importlib); + + /* Install _importlib as __import__ */ + impmod = PyInit_imp(); + if (impmod == NULL) { + Py_FatalError("Py_Initialize: can't import imp"); + } + else if (Py_VerboseFlag) { + PySys_FormatStderr("import imp # builtin\n"); + } + sys_modules = PyImport_GetModuleDict(); + if (Py_VerboseFlag) { + PySys_FormatStderr("import sys # builtin\n"); + } + if (PyDict_SetItemString(sys_modules, "_imp", impmod) < 0) { + Py_FatalError("Py_Initialize: can't save _imp to sys.modules"); + } + + value = PyObject_CallMethod(importlib, "_install", "OO", sysmod, impmod); + if (value == NULL) { + PyErr_Print(); + Py_FatalError("Py_Initialize: importlib install failed"); + } + Py_DECREF(value); + Py_DECREF(impmod); + + _PyImportZip_Init(); +} + + void -Py_InitializeEx(int install_sigs) +_Py_InitializeEx_Private(int install_sigs, int install_importlib) { PyInterpreterState *interp; PyThreadState *tstate; - PyObject *bimod, *sysmod; + PyObject *bimod, *sysmod, *pstderr; char *p; - char *icodeset = NULL; /* On Windows, input codeset may theoretically - differ from output codeset. */ - char *codeset = NULL; - char *errors = NULL; - int free_codeset = 0; - int overridden = 0; - PyObject *sys_stream, *sys_isatty; -#if defined(Py_USING_UNICODE) && defined(HAVE_LANGINFO_H) && defined(CODESET) - char *saved_locale, *loc_codeset; -#endif -#ifdef MS_WINDOWS - char ibuf[128]; - char buf[128]; -#endif extern void _Py_ReadyTypes(void); if (initialized) return; initialized = 1; + _Py_Finalizing = NULL; + +#if defined(HAVE_LANGINFO_H) && defined(HAVE_SETLOCALE) + /* Set up the LC_CTYPE locale, so we can obtain + the locale's charset without having to switch + locales. */ + setlocale(LC_CTYPE, ""); +#endif if ((p = Py_GETENV("PYTHONDEBUG")) && *p != '\0') Py_DebugFlag = add_flag(Py_DebugFlag, p); @@ -183,14 +286,23 @@ Py_InitializeEx(int install_sigs) Py_FatalError("Py_Initialize: can't make first thread"); (void) PyThreadState_Swap(tstate); +#ifdef WITH_THREAD + /* We can't call _PyEval_FiniThreads() in Py_Finalize because + destroying the GIL might fail when it is being referenced from + another running thread (see issue #9901). + Instead we destroy the previously created GIL here, which ensures + that we can call Py_Initialize / Py_Finalize multiple times. */ + _PyEval_FiniThreads(); + + /* Auto-thread-state API */ + _PyGILState_Init(interp, tstate); +#endif /* WITH_THREAD */ + _Py_ReadyTypes(); if (!_PyFrame_Init()) Py_FatalError("Py_Initialize: can't init frames"); - if (!_PyInt_Init()) - Py_FatalError("Py_Initialize: can't init ints"); - if (!_PyLong_Init()) Py_FatalError("Py_Initialize: can't init longs"); @@ -202,23 +314,23 @@ Py_InitializeEx(int install_sigs) interp->modules = PyDict_New(); if (interp->modules == NULL) Py_FatalError("Py_Initialize: can't make modules dictionary"); - interp->modules_reloading = PyDict_New(); - if (interp->modules_reloading == NULL) - Py_FatalError("Py_Initialize: can't make modules_reloading dictionary"); -#ifdef Py_USING_UNICODE /* Init Unicode implementation; relies on the codec registry */ - _PyUnicode_Init(); -#endif + if (_PyUnicode_Init() < 0) + Py_FatalError("Py_Initialize: can't initialize unicode"); bimod = _PyBuiltin_Init(); if (bimod == NULL) - Py_FatalError("Py_Initialize: can't initialize __builtin__"); + Py_FatalError("Py_Initialize: can't initialize builtins modules"); + _PyImport_FixupBuiltin(bimod, "builtins"); interp->builtins = PyModule_GetDict(bimod); if (interp->builtins == NULL) Py_FatalError("Py_Initialize: can't initialize builtins dict"); Py_INCREF(interp->builtins); + /* initialize builtin exceptions */ + _PyExc_Init(bimod); + sysmod = _PySys_Init(); if (sysmod == NULL) Py_FatalError("Py_Initialize: can't initialize sys"); @@ -226,149 +338,67 @@ Py_InitializeEx(int install_sigs) if (interp->sysdict == NULL) Py_FatalError("Py_Initialize: can't initialize sys dict"); Py_INCREF(interp->sysdict); - _PyImport_FixupExtension("sys", "sys"); + _PyImport_FixupBuiltin(sysmod, "sys"); PySys_SetPath(Py_GetPath()); PyDict_SetItemString(interp->sysdict, "modules", interp->modules); + /* Set up a preliminary stderr printer until we have enough + infrastructure for the io module in place. */ + pstderr = PyFile_NewStdPrinter(fileno(stderr)); + if (pstderr == NULL) + Py_FatalError("Py_Initialize: can't set preliminary stderr"); + PySys_SetObject("stderr", pstderr); + PySys_SetObject("__stderr__", pstderr); + Py_DECREF(pstderr); + _PyImport_Init(); - /* initialize builtin exceptions */ - _PyExc_Init(); - _PyImport_FixupExtension("exceptions", "exceptions"); + _PyImportHooks_Init(); - /* phase 2 of builtins */ - _PyImport_FixupExtension("__builtin__", "__builtin__"); + /* Initialize _warnings. */ + _PyWarnings_Init(); - _PyImportHooks_Init(); + if (!install_importlib) + return; + + import_init(interp, sysmod); + + /* initialize the faulthandler module */ + if (_PyFaulthandler_Init()) + Py_FatalError("Py_Initialize: can't initialize faulthandler"); + + _PyTime_Init(); + + if (initfsencoding(interp) < 0) + Py_FatalError("Py_Initialize: unable to load the file system codec"); if (install_sigs) initsigs(); /* Signal handling stuff, including initintr() */ + initmain(interp); /* Module __main__ */ + if (initstdio() < 0) + Py_FatalError( + "Py_Initialize: can't initialize sys standard streams"); + /* Initialize warnings. */ - _PyWarnings_Init(); if (PySys_HasWarnOptions()) { PyObject *warnings_module = PyImport_ImportModule("warnings"); - if (!warnings_module) - PyErr_Clear(); + if (warnings_module == NULL) { + fprintf(stderr, "'import warnings' failed; traceback:\n"); + PyErr_Print(); + } Py_XDECREF(warnings_module); } - initmain(); /* Module __main__ */ - - /* auto-thread-state API, if available */ -#ifdef WITH_THREAD - _PyGILState_Init(interp, tstate); -#endif /* WITH_THREAD */ - if (!Py_NoSiteFlag) initsite(); /* Module site */ +} - if ((p = Py_GETENV("PYTHONIOENCODING")) && *p != '\0') { - p = icodeset = codeset = strdup(p); - free_codeset = 1; - errors = strchr(p, ':'); - if (errors) { - *errors = '\0'; - errors++; - } - overridden = 1; - } - -#if defined(Py_USING_UNICODE) && defined(HAVE_LANGINFO_H) && defined(CODESET) - /* On Unix, set the file system encoding according to the - user's preference, if the CODESET names a well-known - Python codec, and Py_FileSystemDefaultEncoding isn't - initialized by other means. Also set the encoding of - stdin and stdout if these are terminals, unless overridden. */ - - if (!overridden || !Py_FileSystemDefaultEncoding) { - saved_locale = strdup(setlocale(LC_CTYPE, NULL)); - setlocale(LC_CTYPE, ""); - loc_codeset = nl_langinfo(CODESET); - if (loc_codeset && *loc_codeset) { - PyObject *enc = PyCodec_Encoder(loc_codeset); - if (enc) { - loc_codeset = strdup(loc_codeset); - Py_DECREF(enc); - } else { - if (PyErr_ExceptionMatches(PyExc_LookupError)) { - PyErr_Clear(); - loc_codeset = NULL; - } else { - PyErr_Print(); - exit(1); - } - } - } else - loc_codeset = NULL; - setlocale(LC_CTYPE, saved_locale); - free(saved_locale); - - if (!overridden) { - codeset = icodeset = loc_codeset; - free_codeset = 1; - } - - /* Initialize Py_FileSystemDefaultEncoding from - locale even if PYTHONIOENCODING is set. */ - if (!Py_FileSystemDefaultEncoding) { - Py_FileSystemDefaultEncoding = loc_codeset; - if (!overridden) - free_codeset = 0; - } - } -#endif - -#ifdef MS_WINDOWS - if (!overridden) { - icodeset = ibuf; - codeset = buf; - sprintf(ibuf, "cp%d", GetConsoleCP()); - sprintf(buf, "cp%d", GetConsoleOutputCP()); - } -#endif - - if (codeset) { - sys_stream = PySys_GetObject("stdin"); - sys_isatty = PyObject_CallMethod(sys_stream, "isatty", ""); - if (!sys_isatty) - PyErr_Clear(); - if ((overridden || - (sys_isatty && PyObject_IsTrue(sys_isatty))) && - PyFile_Check(sys_stream)) { - if (!PyFile_SetEncodingAndErrors(sys_stream, icodeset, errors)) - Py_FatalError("Cannot set codeset of stdin"); - } - Py_XDECREF(sys_isatty); - - sys_stream = PySys_GetObject("stdout"); - sys_isatty = PyObject_CallMethod(sys_stream, "isatty", ""); - if (!sys_isatty) - PyErr_Clear(); - if ((overridden || - (sys_isatty && PyObject_IsTrue(sys_isatty))) && - PyFile_Check(sys_stream)) { - if (!PyFile_SetEncodingAndErrors(sys_stream, codeset, errors)) - Py_FatalError("Cannot set codeset of stdout"); - } - Py_XDECREF(sys_isatty); - - sys_stream = PySys_GetObject("stderr"); - sys_isatty = PyObject_CallMethod(sys_stream, "isatty", ""); - if (!sys_isatty) - PyErr_Clear(); - if((overridden || - (sys_isatty && PyObject_IsTrue(sys_isatty))) && - PyFile_Check(sys_stream)) { - if (!PyFile_SetEncodingAndErrors(sys_stream, codeset, errors)) - Py_FatalError("Cannot set codeset of stderr"); - } - Py_XDECREF(sys_isatty); - - if (free_codeset) - free(codeset); - } +void +Py_InitializeEx(int install_sigs) +{ + _Py_InitializeEx_Private(install_sigs, 1); } void @@ -382,6 +412,49 @@ Py_Initialize(void) extern void dump_counts(FILE*); #endif +/* Flush stdout and stderr */ + +static int +file_is_closed(PyObject *fobj) +{ + int r; + PyObject *tmp = PyObject_GetAttrString(fobj, "closed"); + if (tmp == NULL) { + PyErr_Clear(); + return 0; + } + r = PyObject_IsTrue(tmp); + Py_DECREF(tmp); + if (r < 0) + PyErr_Clear(); + return r > 0; +} + +static void +flush_std_files(void) +{ + PyObject *fout = PySys_GetObject("stdout"); + PyObject *ferr = PySys_GetObject("stderr"); + PyObject *tmp; + _Py_IDENTIFIER(flush); + + if (fout != NULL && fout != Py_None && !file_is_closed(fout)) { + tmp = _PyObject_CallMethodId(fout, &PyId_flush, ""); + if (tmp == NULL) + PyErr_WriteUnraisable(fout); + else + Py_DECREF(tmp); + } + + if (ferr != NULL && ferr != Py_None && !file_is_closed(ferr)) { + tmp = _PyObject_CallMethodId(ferr, &PyId_flush, ""); + if (tmp == NULL) + PyErr_Clear(); + else + Py_DECREF(tmp); + } +} + /* Undo the effect of Py_Initialize(). Beware: if multiple interpreter and/or thread states exist, these @@ -416,13 +489,20 @@ Py_Finalize(void) * threads created thru it, so this also protects pending imports in * the threads created via Threading. */ - call_sys_exitfunc(); - initialized = 0; + call_py_exitfuncs(); /* Get current thread state and interpreter pointer */ tstate = PyThreadState_GET(); interp = tstate->interp; + /* Remaining threads (e.g. daemon threads) will automatically exit + after taking the GIL (in PyEval_RestoreThread()). */ + _Py_Finalizing = tstate; + initialized = 0; + + /* Flush stdout+stderr */ + flush_std_files(); + /* Disable signal handling */ PyOS_FiniInterrupts(); @@ -449,12 +529,18 @@ Py_Finalize(void) while (PyGC_Collect() > 0) /* nothing */; #endif + /* We run this while most interpreter state is still alive, so that + debug information can be printed out */ + _PyGC_Fini(); /* Destroy all modules */ PyImport_Cleanup(); + /* Flush stdout+stderr (again, in case more was printed) */ + flush_std_files(); + /* Collect final garbage. This disposes of cycles created by - * new-style class definitions, for example. + * class definitions, for example. * XXX This is disabled because it caused too many problems. If * XXX a __del__ or weakref callback triggers here, Python code has * XXX a hard time running, because even the sys module has been @@ -475,6 +561,9 @@ Py_Finalize(void) /* Destroy the database used by _PyImport_{Fixup,Find}Extension */ _PyImport_Fini(); + /* unload faulthandler module */ + _PyFaulthandler_Fini(); + /* Debugging stuff */ #ifdef COUNT_ALLOCS dump_counts(stdout); @@ -519,16 +608,21 @@ Py_Finalize(void) PyTuple_Fini(); PyList_Fini(); PySet_Fini(); - PyString_Fini(); + PyBytes_Fini(); PyByteArray_Fini(); - PyInt_Fini(); + PyLong_Fini(); PyFloat_Fini(); PyDict_Fini(); + PySlice_Fini(); -#ifdef Py_USING_UNICODE /* Cleanup Unicode implementation */ _PyUnicode_Fini(); -#endif + + /* reset file system default encoding */ + if (!Py_HasFileSystemDefaultEncoding && Py_FileSystemDefaultEncoding) { + free((char*)Py_FileSystemDefaultEncoding); + Py_FileSystemDefaultEncoding = NULL; + } /* XXX Still allocated: - various static ad-hoc pointers to interned strings @@ -548,7 +642,7 @@ Py_Finalize(void) #endif /* Py_TRACE_REFS */ #ifdef PYMALLOC_DEBUG if (Py_GETENV("PYTHONMALLOCSTATS")) - _PyObject_DebugMallocStats(); + _PyObject_DebugMallocStats(stderr); #endif call_ll_exitfuncs(); @@ -592,17 +686,22 @@ Py_NewInterpreter(void) /* XXX The following is lax in error checking */ interp->modules = PyDict_New(); - interp->modules_reloading = PyDict_New(); - bimod = _PyImport_FindExtension("__builtin__", "__builtin__"); + bimod = _PyImport_FindBuiltin("builtins"); if (bimod != NULL) { interp->builtins = PyModule_GetDict(bimod); if (interp->builtins == NULL) goto handle_error; Py_INCREF(interp->builtins); } - sysmod = _PyImport_FindExtension("sys", "sys"); + + /* initialize builtin exceptions */ + _PyExc_Init(bimod); + + sysmod = _PyImport_FindBuiltin("sys"); if (bimod != NULL && sysmod != NULL) { + PyObject *pstderr; + interp->sysdict = PyModule_GetDict(sysmod); if (interp->sysdict == NULL) goto handle_error; @@ -610,8 +709,26 @@ Py_NewInterpreter(void) PySys_SetPath(Py_GetPath()); PyDict_SetItemString(interp->sysdict, "modules", interp->modules); + /* Set up a preliminary stderr printer until we have enough + infrastructure for the io module in place. */ + pstderr = PyFile_NewStdPrinter(fileno(stderr)); + if (pstderr == NULL) + Py_FatalError("Py_Initialize: can't set preliminary stderr"); + PySys_SetObject("stderr", pstderr); + PySys_SetObject("__stderr__", pstderr); + Py_DECREF(pstderr); + _PyImportHooks_Init(); - initmain(); + + import_init(interp, sysmod); + + if (initfsencoding(interp) < 0) + goto handle_error; + + if (initstdio() < 0) + Py_FatalError( + "Py_Initialize: can't initialize sys standard streams"); + initmain(interp); if (!Py_NoSiteFlag) initsite(); } @@ -622,7 +739,7 @@ Py_NewInterpreter(void) handle_error: /* Oops, it didn't work. Undo it all. */ - PyErr_Print(); + PyErr_PrintEx(0); PyThreadState_Clear(tstate); PyThreadState_Swap(save_tstate); PyThreadState_Delete(tstate); @@ -661,42 +778,54 @@ Py_EndInterpreter(PyThreadState *tstate) PyInterpreterState_Delete(interp); } -static char *progname = "python"; +#ifdef MS_WINDOWS +static wchar_t *progname = L"python"; +#else +static wchar_t *progname = L"python3"; +#endif void -Py_SetProgramName(char *pn) +Py_SetProgramName(wchar_t *pn) { if (pn && *pn) progname = pn; } -char * +wchar_t * Py_GetProgramName(void) { return progname; } -static char *default_home = NULL; +static wchar_t *default_home = NULL; +static wchar_t env_home[PATH_MAX+1]; void -Py_SetPythonHome(char *home) +Py_SetPythonHome(wchar_t *home) { default_home = home; } -char * +wchar_t * Py_GetPythonHome(void) { - char *home = default_home; - if (home == NULL && !Py_IgnoreEnvironmentFlag) - home = Py_GETENV("PYTHONHOME"); + wchar_t *home = default_home; + if (home == NULL && !Py_IgnoreEnvironmentFlag) { + char* chome = Py_GETENV("PYTHONHOME"); + if (chome) { + size_t r = mbstowcs(env_home, chome, PATH_MAX+1); + if (r != (size_t)-1 && r <= PATH_MAX) + home = env_home; + } + + } return home; } /* Create __main__ module */ static void -initmain(void) +initmain(PyInterpreterState *interp) { PyObject *m, *d; m = PyImport_AddModule("__main__"); @@ -704,12 +833,61 @@ initmain(void) Py_FatalError("can't create __main__ module"); d = PyModule_GetDict(m); if (PyDict_GetItemString(d, "__builtins__") == NULL) { - PyObject *bimod = PyImport_ImportModule("__builtin__"); - if (bimod == NULL || - PyDict_SetItemString(d, "__builtins__", bimod) != 0) - Py_FatalError("can't add __builtins__ to __main__"); - Py_XDECREF(bimod); + PyObject *bimod = PyImport_ImportModule("builtins"); + if (bimod == NULL) { + Py_FatalError("Failed to retrieve builtins module"); + } + if (PyDict_SetItemString(d, "__builtins__", bimod) < 0) { + Py_FatalError("Failed to initialize __main__.__builtins__"); + } + Py_DECREF(bimod); + } + /* Main is a little special - imp.is_builtin("__main__") will return + * False, but BuiltinImporter is still the most appropriate initial + * setting for its __loader__ attribute. A more suitable value will + * be set if __main__ gets further initialized later in the startup + * process. + */ + if (PyDict_GetItemString(d, "__loader__") == NULL) { + PyObject *loader = PyObject_GetAttrString(interp->importlib, + "BuiltinImporter"); + if (loader == NULL) { + Py_FatalError("Failed to retrieve BuiltinImporter"); + } + if (PyDict_SetItemString(d, "__loader__", loader) < 0) { + Py_FatalError("Failed to initialize __main__.__loader__"); + } + Py_DECREF(loader); + } +} + +static int +initfsencoding(PyInterpreterState *interp) +{ + PyObject *codec; + + if (Py_FileSystemDefaultEncoding == NULL) + { + Py_FileSystemDefaultEncoding = get_locale_encoding(); + if (Py_FileSystemDefaultEncoding == NULL) + Py_FatalError("Py_Initialize: Unable to get the locale encoding"); + + Py_HasFileSystemDefaultEncoding = 0; + interp->fscodec_initialized = 1; + return 0; + } + + /* the encoding is mbcs, utf-8 or ascii */ + codec = _PyCodec_Lookup(Py_FileSystemDefaultEncoding); + if (!codec) { + /* Such error can only occurs in critical situations: no more + * memory, import a module of the standard library failed, + * etc. */ + return -1; } + Py_DECREF(codec); + interp->fscodec_initialized = 1; + return 0; } /* Import the site module (not into __main__ though) */ @@ -729,6 +907,250 @@ initsite(void) } } +static PyObject* +create_stdio(PyObject* io, + int fd, int write_mode, char* name, + char* encoding, char* errors) +{ + PyObject *buf = NULL, *stream = NULL, *text = NULL, *raw = NULL, *res; + const char* mode; + const char* newline; + PyObject *line_buffering; + int buffering, isatty; + _Py_IDENTIFIER(open); + _Py_IDENTIFIER(isatty); + _Py_IDENTIFIER(TextIOWrapper); + _Py_IDENTIFIER(name); + _Py_IDENTIFIER(mode); + + /* stdin is always opened in buffered mode, first because it shouldn't + make a difference in common use cases, second because TextIOWrapper + depends on the presence of a read1() method which only exists on + buffered streams. + */ + if (Py_UnbufferedStdioFlag && write_mode) + buffering = 0; + else + buffering = -1; + if (write_mode) + mode = "wb"; + else + mode = "rb"; + buf = _PyObject_CallMethodId(io, &PyId_open, "isiOOOi", + fd, mode, buffering, + Py_None, Py_None, Py_None, 0); + if (buf == NULL) + goto error; + + if (buffering) { + _Py_IDENTIFIER(raw); + raw = _PyObject_GetAttrId(buf, &PyId_raw); + if (raw == NULL) + goto error; + } + else { + raw = buf; + Py_INCREF(raw); + } + + text = PyUnicode_FromString(name); + if (text == NULL || _PyObject_SetAttrId(raw, &PyId_name, text) < 0) + goto error; + res = _PyObject_CallMethodId(raw, &PyId_isatty, ""); + if (res == NULL) + goto error; + isatty = PyObject_IsTrue(res); + Py_DECREF(res); + if (isatty == -1) + goto error; + if (isatty || Py_UnbufferedStdioFlag) + line_buffering = Py_True; + else + line_buffering = Py_False; + + Py_CLEAR(raw); + Py_CLEAR(text); + +#ifdef MS_WINDOWS + /* sys.stdin: enable universal newline mode, translate "\r\n" and "\r" + newlines to "\n". + sys.stdout and sys.stderr: translate "\n" to "\r\n". */ + newline = NULL; +#else + /* sys.stdin: split lines at "\n". + sys.stdout and sys.stderr: don't translate newlines (use "\n"). */ + newline = "\n"; +#endif + + stream = _PyObject_CallMethodId(io, &PyId_TextIOWrapper, "OsssO", + buf, encoding, errors, + newline, line_buffering); + Py_CLEAR(buf); + if (stream == NULL) + goto error; + + if (write_mode) + mode = "w"; + else + mode = "r"; + text = PyUnicode_FromString(mode); + if (!text || _PyObject_SetAttrId(stream, &PyId_mode, text) < 0) + goto error; + Py_CLEAR(text); + return stream; + +error: + Py_XDECREF(buf); + Py_XDECREF(stream); + Py_XDECREF(text); + Py_XDECREF(raw); + return NULL; +} + +static int +is_valid_fd(int fd) +{ + int dummy_fd; + if (fd < 0 || !_PyVerify_fd(fd)) + return 0; + dummy_fd = dup(fd); + if (dummy_fd < 0) + return 0; + close(dummy_fd); + return 1; +} + +/* Initialize sys.stdin, stdout, stderr and builtins.open */ +static int +initstdio(void) +{ + PyObject *iomod = NULL, *wrapper; + PyObject *bimod = NULL; + PyObject *m; + PyObject *std = NULL; + int status = 0, fd; + PyObject * encoding_attr; + char *encoding = NULL, *errors; + + /* Hack to avoid a nasty recursion issue when Python is invoked + in verbose mode: pre-import the Latin-1 and UTF-8 codecs */ + if ((m = PyImport_ImportModule("encodings.utf_8")) == NULL) { + goto error; + } + Py_DECREF(m); + + if (!(m = PyImport_ImportModule("encodings.latin_1"))) { + goto error; + } + Py_DECREF(m); + + if (!(bimod = PyImport_ImportModule("builtins"))) { + goto error; + } + + if (!(iomod = PyImport_ImportModule("io"))) { + goto error; + } + if (!(wrapper = PyObject_GetAttrString(iomod, "OpenWrapper"))) { + goto error; + } + + /* Set builtins.open */ + if (PyObject_SetAttrString(bimod, "open", wrapper) == -1) { + Py_DECREF(wrapper); + goto error; + } + Py_DECREF(wrapper); + + encoding = Py_GETENV("PYTHONIOENCODING"); + errors = NULL; + if (encoding) { + encoding = strdup(encoding); + errors = strchr(encoding, ':'); + if (errors) { + *errors = '\0'; + errors++; + } + } + + /* Set sys.stdin */ + fd = fileno(stdin); + /* Under some conditions stdin, stdout and stderr may not be connected + * and fileno() may point to an invalid file descriptor. For example + * GUI apps don't have valid standard streams by default. + */ + if (!is_valid_fd(fd)) { + std = Py_None; + Py_INCREF(std); + } + else { + std = create_stdio(iomod, fd, 0, "<stdin>", encoding, errors); + if (std == NULL) + goto error; + } /* if (fd < 0) */ + PySys_SetObject("__stdin__", std); + PySys_SetObject("stdin", std); + Py_DECREF(std); + + /* Set sys.stdout */ + fd = fileno(stdout); + if (!is_valid_fd(fd)) { + std = Py_None; + Py_INCREF(std); + } + else { + std = create_stdio(iomod, fd, 1, "<stdout>", encoding, errors); + if (std == NULL) + goto error; + } /* if (fd < 0) */ + PySys_SetObject("__stdout__", std); + PySys_SetObject("stdout", std); + Py_DECREF(std); + +#if 1 /* Disable this if you have trouble debugging bootstrap stuff */ + /* Set sys.stderr, replaces the preliminary stderr */ + fd = fileno(stderr); + if (!is_valid_fd(fd)) { + std = Py_None; + Py_INCREF(std); + } + else { + std = create_stdio(iomod, fd, 1, "<stderr>", encoding, "backslashreplace"); + if (std == NULL) + goto error; + } /* if (fd < 0) */ + + /* Same as hack above, pre-import stderr's codec to avoid recursion + when import.c tries to write to stderr in verbose mode. */ + encoding_attr = PyObject_GetAttrString(std, "encoding"); + if (encoding_attr != NULL) { + const char * encoding; + encoding = _PyUnicode_AsString(encoding_attr); + if (encoding != NULL) { + PyObject *codec_info = _PyCodec_Lookup(encoding); + Py_XDECREF(codec_info); + } + Py_DECREF(encoding_attr); + } + PyErr_Clear(); /* Not a fatal error if codec isn't available */ + + PySys_SetObject("__stderr__", std); + PySys_SetObject("stderr", std); + Py_DECREF(std); +#endif + + if (0) { + error: + status = -1; + } + + if (encoding) + free(encoding); + Py_XDECREF(bimod); + Py_XDECREF(iomod); + return status; +} + /* Parse input from a file and execute it */ int @@ -760,12 +1182,12 @@ PyRun_InteractiveLoopFlags(FILE *fp, const char *filename, PyCompilerFlags *flag } v = PySys_GetObject("ps1"); if (v == NULL) { - PySys_SetObject("ps1", v = PyString_FromString(">>> ")); + PySys_SetObject("ps1", v = PyUnicode_FromString(">>> ")); Py_XDECREF(v); } v = PySys_GetObject("ps2"); if (v == NULL) { - PySys_SetObject("ps2", v = PyString_FromString("... ")); + PySys_SetObject("ps2", v = PyUnicode_FromString("... ")); Py_XDECREF(v); } for (;;) { @@ -780,60 +1202,91 @@ PyRun_InteractiveLoopFlags(FILE *fp, const char *filename, PyCompilerFlags *flag } } -#if 0 /* compute parser flags based on compiler flags */ -#define PARSER_FLAGS(flags) \ - ((flags) ? ((((flags)->cf_flags & PyCF_DONT_IMPLY_DEDENT) ? \ - PyPARSE_DONT_IMPLY_DEDENT : 0)) : 0) -#endif -#if 1 +static int PARSER_FLAGS(PyCompilerFlags *flags) +{ + int parser_flags = 0; + if (!flags) + return 0; + if (flags->cf_flags & PyCF_DONT_IMPLY_DEDENT) + parser_flags |= PyPARSE_DONT_IMPLY_DEDENT; + if (flags->cf_flags & PyCF_IGNORE_COOKIE) + parser_flags |= PyPARSE_IGNORE_COOKIE; + if (flags->cf_flags & CO_FUTURE_BARRY_AS_BDFL) + parser_flags |= PyPARSE_BARRY_AS_BDFL; + return parser_flags; +} + +#if 0 /* Keep an example of flags with future keyword support. */ #define PARSER_FLAGS(flags) \ ((flags) ? ((((flags)->cf_flags & PyCF_DONT_IMPLY_DEDENT) ? \ PyPARSE_DONT_IMPLY_DEDENT : 0) \ - | (((flags)->cf_flags & CO_FUTURE_PRINT_FUNCTION) ? \ - PyPARSE_PRINT_IS_FUNCTION : 0) \ - | (((flags)->cf_flags & CO_FUTURE_UNICODE_LITERALS) ? \ - PyPARSE_UNICODE_LITERALS : 0) \ - ) : 0) + | ((flags)->cf_flags & CO_FUTURE_WITH_STATEMENT ? \ + PyPARSE_WITH_IS_KEYWORD : 0)) : 0) #endif int PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags) { - PyObject *m, *d, *v, *w; + PyObject *m, *d, *v, *w, *oenc = NULL; mod_ty mod; PyArena *arena; - char *ps1 = "", *ps2 = ""; + char *ps1 = "", *ps2 = "", *enc = NULL; int errcode = 0; + _Py_IDENTIFIER(encoding); + if (fp == stdin) { + /* Fetch encoding from sys.stdin */ + v = PySys_GetObject("stdin"); + if (v == NULL || v == Py_None) + return -1; + oenc = _PyObject_GetAttrId(v, &PyId_encoding); + if (!oenc) + return -1; + enc = _PyUnicode_AsString(oenc); + if (enc == NULL) + return -1; + } v = PySys_GetObject("ps1"); if (v != NULL) { v = PyObject_Str(v); if (v == NULL) PyErr_Clear(); - else if (PyString_Check(v)) - ps1 = PyString_AsString(v); + else if (PyUnicode_Check(v)) { + ps1 = _PyUnicode_AsString(v); + if (ps1 == NULL) { + PyErr_Clear(); + ps1 = ""; + } + } } w = PySys_GetObject("ps2"); if (w != NULL) { w = PyObject_Str(w); if (w == NULL) PyErr_Clear(); - else if (PyString_Check(w)) - ps2 = PyString_AsString(w); + else if (PyUnicode_Check(w)) { + ps2 = _PyUnicode_AsString(w); + if (ps2 == NULL) { + PyErr_Clear(); + ps2 = ""; + } + } } arena = PyArena_New(); if (arena == NULL) { Py_XDECREF(v); Py_XDECREF(w); + Py_XDECREF(oenc); return -1; } - mod = PyParser_ASTFromFile(fp, filename, + mod = PyParser_ASTFromFile(fp, filename, enc, Py_single_input, ps1, ps2, flags, &errcode, arena); Py_XDECREF(v); Py_XDECREF(w); + Py_XDECREF(oenc); if (mod == NULL) { PyArena_Free(arena); if (errcode == E_EOF) { @@ -851,13 +1304,12 @@ PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags d = PyModule_GetDict(m); v = run_mod(mod, filename, d, d, flags, arena); PyArena_Free(arena); + flush_io(); if (v == NULL) { PyErr_Print(); return -1; } Py_DECREF(v); - if (Py_FlushLine()) - PyErr_Clear(); return 0; } @@ -902,12 +1354,44 @@ maybe_pyc_file(FILE *fp, const char* filename, const char* ext, int closeit) } int +static set_main_loader(PyObject *d, const char *filename, const char *loader_name) +{ + PyInterpreterState *interp; + PyThreadState *tstate; + PyObject *filename_obj, *loader_type, *loader; + int result = 0; + + filename_obj = PyUnicode_DecodeFSDefault(filename); + if (filename_obj == NULL) + return -1; + /* Get current thread state and interpreter pointer */ + tstate = PyThreadState_GET(); + interp = tstate->interp; + loader_type = PyObject_GetAttrString(interp->importlib, loader_name); + if (loader_type == NULL) { + Py_DECREF(filename_obj); + return -1; + } + loader = PyObject_CallFunction(loader_type, "sN", "__main__", filename_obj); + Py_DECREF(loader_type); + if (loader == NULL) { + return -1; + } + if (PyDict_SetItemString(d, "__loader__", loader) < 0) { + result = -1; + } + Py_DECREF(loader); + return result; +} + +int PyRun_SimpleFileExFlags(FILE *fp, const char *filename, int closeit, PyCompilerFlags *flags) { PyObject *m, *d, *v; const char *ext; - int set_file_name = 0, len, ret = -1; + int set_file_name = 0, ret = -1; + size_t len; m = PyImport_AddModule("__main__"); if (m == NULL) @@ -915,41 +1399,61 @@ PyRun_SimpleFileExFlags(FILE *fp, const char *filename, int closeit, Py_INCREF(m); d = PyModule_GetDict(m); if (PyDict_GetItemString(d, "__file__") == NULL) { - PyObject *f = PyString_FromString(filename); + PyObject *f; + f = PyUnicode_DecodeFSDefault(filename); if (f == NULL) goto done; if (PyDict_SetItemString(d, "__file__", f) < 0) { Py_DECREF(f); goto done; } + if (PyDict_SetItemString(d, "__cached__", Py_None) < 0) { + Py_DECREF(f); + goto done; + } set_file_name = 1; Py_DECREF(f); } len = strlen(filename); ext = filename + len - (len > 4 ? 4 : 0); if (maybe_pyc_file(fp, filename, ext, closeit)) { + FILE *pyc_fp; /* Try to run a pyc file. First, re-open in binary */ if (closeit) fclose(fp); - if ((fp = fopen(filename, "rb")) == NULL) { + if ((pyc_fp = fopen(filename, "rb")) == NULL) { fprintf(stderr, "python: Can't reopen .pyc file\n"); goto done; } /* Turn on optimization if a .pyo file is given */ if (strcmp(ext, ".pyo") == 0) Py_OptimizeFlag = 1; - v = run_pyc_file(fp, filename, d, d, flags); + + if (set_main_loader(d, filename, "SourcelessFileLoader") < 0) { + fprintf(stderr, "python: failed to set __main__.__loader__\n"); + ret = -1; + fclose(pyc_fp); + goto done; + } + v = run_pyc_file(pyc_fp, filename, d, d, flags); + fclose(pyc_fp); } else { + /* When running from stdin, leave __main__.__loader__ alone */ + if (strcmp(filename, "<stdin>") != 0 && + set_main_loader(d, filename, "SourceFileLoader") < 0) { + fprintf(stderr, "python: failed to set __main__.__loader__\n"); + ret = -1; + goto done; + } v = PyRun_FileExFlags(fp, filename, Py_file_input, d, d, closeit, flags); } + flush_io(); if (v == NULL) { PyErr_Print(); goto done; } Py_DECREF(v); - if (Py_FlushLine()) - PyErr_Clear(); ret = 0; done: if (set_file_name && PyDict_DelItemString(d, "__file__")) @@ -972,8 +1476,6 @@ PyRun_SimpleStringFlags(const char *command, PyCompilerFlags *flags) return -1; } Py_DECREF(v); - if (Py_FlushLine()) - PyErr_Clear(); return 0; } @@ -983,20 +1485,20 @@ parse_syntax_error(PyObject *err, PyObject **message, const char **filename, { long hold; PyObject *v; - - /* old style errors */ - if (PyTuple_Check(err)) - return PyArg_ParseTuple(err, "O(ziiz)", message, filename, - lineno, offset, text); + _Py_IDENTIFIER(msg); + _Py_IDENTIFIER(filename); + _Py_IDENTIFIER(lineno); + _Py_IDENTIFIER(offset); + _Py_IDENTIFIER(text); *message = NULL; /* new style errors. `err' is an instance */ - *message = PyObject_GetAttrString(err, "msg"); + *message = _PyObject_GetAttrId(err, &PyId_msg); if (!*message) goto finally; - v = PyObject_GetAttrString(err, "filename"); + v = _PyObject_GetAttrId(err, &PyId_filename); if (!v) goto finally; if (v == Py_None) { @@ -1004,36 +1506,36 @@ parse_syntax_error(PyObject *err, PyObject **message, const char **filename, *filename = NULL; } else { - *filename = PyString_AsString(v); + *filename = _PyUnicode_AsString(v); Py_DECREF(v); if (!*filename) goto finally; } - v = PyObject_GetAttrString(err, "lineno"); + v = _PyObject_GetAttrId(err, &PyId_lineno); if (!v) goto finally; - hold = PyInt_AsLong(v); + hold = PyLong_AsLong(v); Py_DECREF(v); if (hold < 0 && PyErr_Occurred()) goto finally; *lineno = (int)hold; - v = PyObject_GetAttrString(err, "offset"); + v = _PyObject_GetAttrId(err, &PyId_offset); if (!v) goto finally; if (v == Py_None) { *offset = -1; Py_DECREF(v); } else { - hold = PyInt_AsLong(v); + hold = PyLong_AsLong(v); Py_DECREF(v); if (hold < 0 && PyErr_Occurred()) goto finally; *offset = (int)hold; } - v = PyObject_GetAttrString(err, "text"); + v = _PyObject_GetAttrId(err, &PyId_text); if (!v) goto finally; if (v == Py_None) { @@ -1041,7 +1543,7 @@ parse_syntax_error(PyObject *err, PyObject **message, const char **filename, *text = NULL; } else { - *text = PyString_AsString(v); + *text = _PyUnicode_AsString(v); Py_DECREF(v); if (!*text) goto finally; @@ -1085,11 +1587,8 @@ print_error_text(PyObject *f, int offset, const char *text) if (offset == -1) return; PyFile_WriteString(" ", f); - offset--; - while (offset > 0) { + while (--offset > 0) PyFile_WriteString(" ", f); - offset--; - } PyFile_WriteString("^\n", f); } @@ -1105,14 +1604,13 @@ handle_system_exit(void) return; PyErr_Fetch(&exception, &value, &tb); - if (Py_FlushLine()) - PyErr_Clear(); fflush(stdout); if (value == NULL || value == Py_None) goto done; if (PyExceptionInstance_Check(value)) { /* The error code should be in the `code' attribute. */ - PyObject *code = PyObject_GetAttrString(value, "code"); + _Py_IDENTIFIER(code); + PyObject *code = _PyObject_GetAttrId(value, &PyId_code); if (code) { Py_DECREF(value); value = code; @@ -1122,8 +1620,8 @@ handle_system_exit(void) /* If we failed to dig out the 'code' attribute, just let the else clause below print the error. */ } - if (PyInt_Check(value)) - exitcode = (int)PyInt_AsLong(value); + if (PyLong_Check(value)) + exitcode = (int)PyLong_AsLong(value); else { PyObject *sys_stderr = PySys_GetObject("stderr"); if (sys_stderr != NULL && sys_stderr != Py_None) { @@ -1159,6 +1657,11 @@ PyErr_PrintEx(int set_sys_last_vars) if (exception == NULL) return; PyErr_NormalizeException(&exception, &v, &tb); + if (tb == NULL) { + tb = Py_None; + Py_INCREF(tb); + } + PyException_SetTraceback(v, tb); if (exception == NULL) return; /* Now we know v != NULL too */ @@ -1168,9 +1671,8 @@ PyErr_PrintEx(int set_sys_last_vars) PySys_SetObject("last_traceback", tb); } hook = PySys_GetObject("excepthook"); - if (hook && hook != Py_None) { - PyObject *args = PyTuple_Pack(3, - exception, v, tb ? tb : Py_None); + if (hook) { + PyObject *args = PyTuple_Pack(3, exception, v, tb); PyObject *result = PyEval_CallObject(hook, args); if (result == NULL) { PyObject *exception2, *v2, *tb2; @@ -1190,8 +1692,6 @@ PyErr_PrintEx(int set_sys_last_vars) v2 = Py_None; Py_INCREF(v2); } - if (Py_FlushLine()) - PyErr_Clear(); fflush(stdout); PySys_WriteStderr("Error in sys.excepthook:\n"); PyErr_Display(exception2, v2, tb2); @@ -1212,100 +1712,110 @@ PyErr_PrintEx(int set_sys_last_vars) Py_XDECREF(tb); } -void -PyErr_Display(PyObject *exception, PyObject *value, PyObject *tb) +static void +print_exception(PyObject *f, PyObject *value) { int err = 0; - PyObject *f = PySys_GetObject("stderr"); + PyObject *type, *tb; + _Py_IDENTIFIER(print_file_and_line); + + if (!PyExceptionInstance_Check(value)) { + PyFile_WriteString("TypeError: print_exception(): Exception expected for value, ", f); + PyFile_WriteString(Py_TYPE(value)->tp_name, f); + PyFile_WriteString(" found\n", f); + return; + } + Py_INCREF(value); - if (f == NULL || f == Py_None) - fprintf(stderr, "lost sys.stderr\n"); - else { - if (Py_FlushLine()) + fflush(stdout); + type = (PyObject *) Py_TYPE(value); + tb = PyException_GetTraceback(value); + if (tb && tb != Py_None) + err = PyTraceBack_Print(tb, f); + if (err == 0 && + _PyObject_HasAttrId(value, &PyId_print_file_and_line)) + { + PyObject *message; + const char *filename, *text; + int lineno, offset; + if (!parse_syntax_error(value, &message, &filename, + &lineno, &offset, &text)) PyErr_Clear(); - fflush(stdout); - if (tb && tb != Py_None) - err = PyTraceBack_Print(tb, f); - if (err == 0 && - PyObject_HasAttrString(value, "print_file_and_line")) - { - PyObject *message; - const char *filename, *text; - int lineno, offset; - if (!parse_syntax_error(value, &message, &filename, - &lineno, &offset, &text)) - PyErr_Clear(); - else { - char buf[10]; - PyFile_WriteString(" File \"", f); - if (filename == NULL) - PyFile_WriteString("<string>", f); - else - PyFile_WriteString(filename, f); - PyFile_WriteString("\", line ", f); - PyOS_snprintf(buf, sizeof(buf), "%d", lineno); - PyFile_WriteString(buf, f); - PyFile_WriteString("\n", f); - if (text != NULL) - print_error_text(f, offset, text); - Py_DECREF(value); - value = message; - /* Can't be bothered to check all those - PyFile_WriteString() calls */ - if (PyErr_Occurred()) - err = -1; - } + else { + char buf[10]; + PyFile_WriteString(" File \"", f); + if (filename == NULL) + PyFile_WriteString("<string>", f); + else + PyFile_WriteString(filename, f); + PyFile_WriteString("\", line ", f); + PyOS_snprintf(buf, sizeof(buf), "%d", lineno); + PyFile_WriteString(buf, f); + PyFile_WriteString("\n", f); + if (text != NULL) + print_error_text(f, offset, text); + Py_DECREF(value); + value = message; + /* Can't be bothered to check all those + PyFile_WriteString() calls */ + if (PyErr_Occurred()) + err = -1; } - if (err) { - /* Don't do anything else */ + } + if (err) { + /* Don't do anything else */ + } + else { + PyObject* moduleName; + char* className; + _Py_IDENTIFIER(__module__); + assert(PyExceptionClass_Check(type)); + className = PyExceptionClass_Name(type); + if (className != NULL) { + char *dot = strrchr(className, '.'); + if (dot != NULL) + className = dot+1; } - else if (PyExceptionClass_Check(exception)) { - PyObject* moduleName; - char* className = PyExceptionClass_Name(exception); - if (className != NULL) { - char *dot = strrchr(className, '.'); - if (dot != NULL) - className = dot+1; - } - moduleName = PyObject_GetAttrString(exception, "__module__"); - if (moduleName == NULL) - err = PyFile_WriteString("<unknown>", f); - else { - char* modstr = PyString_AsString(moduleName); - if (modstr && strcmp(modstr, "exceptions")) - { - err = PyFile_WriteString(modstr, f); - err += PyFile_WriteString(".", f); - } - Py_DECREF(moduleName); + moduleName = _PyObject_GetAttrId(type, &PyId___module__); + if (moduleName == NULL || !PyUnicode_Check(moduleName)) + { + Py_XDECREF(moduleName); + err = PyFile_WriteString("<unknown>", f); + } + else { + char* modstr = _PyUnicode_AsString(moduleName); + if (modstr && strcmp(modstr, "builtins")) + { + err = PyFile_WriteString(modstr, f); + err += PyFile_WriteString(".", f); } - if (err == 0) { - if (className == NULL) + Py_DECREF(moduleName); + } + if (err == 0) { + if (className == NULL) err = PyFile_WriteString("<unknown>", f); - else + else err = PyFile_WriteString(className, f); - } - } - else - err = PyFile_WriteObject(exception, f, Py_PRINT_RAW); - if (err == 0 && (value != Py_None)) { - PyObject *s = PyObject_Str(value); - /* only print colon if the str() of the - object is not the empty string - */ - if (s == NULL) - err = -1; - else if (!PyString_Check(s) || - PyString_GET_SIZE(s) != 0) - err = PyFile_WriteString(": ", f); - if (err == 0) - err = PyFile_WriteObject(s, f, Py_PRINT_RAW); - Py_XDECREF(s); } - /* try to write a newline in any case */ - err += PyFile_WriteString("\n", f); } + if (err == 0 && (value != Py_None)) { + PyObject *s = PyObject_Str(value); + /* only print colon if the str() of the + object is not the empty string + */ + if (s == NULL) + err = -1; + else if (!PyUnicode_Check(s) || + PyUnicode_GetLength(s) != 0) + err = PyFile_WriteString(": ", f); + if (err == 0) + err = PyFile_WriteObject(s, f, Py_PRINT_RAW); + Py_XDECREF(s); + } + /* try to write a newline in any case */ + err += PyFile_WriteString("\n", f); + Py_XDECREF(tb); Py_DECREF(value); /* If an error happened here, don't show it. XXX This is wrong, but too many callers rely on this behavior. */ @@ -1313,6 +1823,83 @@ PyErr_Display(PyObject *exception, PyObject *value, PyObject *tb) PyErr_Clear(); } +static const char *cause_message = + "\nThe above exception was the direct cause " + "of the following exception:\n\n"; + +static const char *context_message = + "\nDuring handling of the above exception, " + "another exception occurred:\n\n"; + +static void +print_exception_recursive(PyObject *f, PyObject *value, PyObject *seen) +{ + int err = 0, res; + PyObject *cause, *context; + + if (seen != NULL) { + /* Exception chaining */ + if (PySet_Add(seen, value) == -1) + PyErr_Clear(); + else if (PyExceptionInstance_Check(value)) { + cause = PyException_GetCause(value); + context = PyException_GetContext(value); + if (cause) { + res = PySet_Contains(seen, cause); + if (res == -1) + PyErr_Clear(); + if (res == 0) { + print_exception_recursive( + f, cause, seen); + err |= PyFile_WriteString( + cause_message, f); + } + } + else if (context && + !((PyBaseExceptionObject *)value)->suppress_context) { + res = PySet_Contains(seen, context); + if (res == -1) + PyErr_Clear(); + if (res == 0) { + print_exception_recursive( + f, context, seen); + err |= PyFile_WriteString( + context_message, f); + } + } + Py_XDECREF(context); + Py_XDECREF(cause); + } + } + print_exception(f, value); + if (err != 0) + PyErr_Clear(); +} + +void +PyErr_Display(PyObject *exception, PyObject *value, PyObject *tb) +{ + PyObject *seen; + PyObject *f = PySys_GetObject("stderr"); + if (f == Py_None) { + /* pass */ + } + else if (f == NULL) { + _PyObject_Dump(value); + fprintf(stderr, "lost sys.stderr\n"); + } + else { + /* We choose to ignore seen being possibly NULL, and report + at least the main exception (it could be a MemoryError). + */ + seen = PySet_New(NULL); + if (seen == NULL) + PyErr_Clear(); + print_exception_recursive(f, value, seen); + Py_XDECREF(seen); + } +} + PyObject * PyRun_StringFlags(const char *str, int start, PyObject *globals, PyObject *locals, PyCompilerFlags *flags) @@ -1340,7 +1927,7 @@ PyRun_FileExFlags(FILE *fp, const char *filename, int start, PyObject *globals, if (arena == NULL) return NULL; - mod = PyParser_ASTFromFile(fp, filename, start, 0, 0, + mod = PyParser_ASTFromFile(fp, filename, NULL, start, 0, 0, flags, NULL, arena); if (closeit) fclose(fp); @@ -1353,6 +1940,36 @@ PyRun_FileExFlags(FILE *fp, const char *filename, int start, PyObject *globals, return ret; } +static void +flush_io(void) +{ + PyObject *f, *r; + PyObject *type, *value, *traceback; + _Py_IDENTIFIER(flush); + + /* Save the current exception */ + PyErr_Fetch(&type, &value, &traceback); + + f = PySys_GetObject("stderr"); + if (f != NULL) { + r = _PyObject_CallMethodId(f, &PyId_flush, ""); + if (r) + Py_DECREF(r); + else + PyErr_Clear(); + } + f = PySys_GetObject("stdout"); + if (f != NULL) { + r = _PyObject_CallMethodId(f, &PyId_flush, ""); + if (r) + Py_DECREF(r); + else + PyErr_Clear(); + } + + PyErr_Restore(type, value, traceback); +} + static PyObject * run_mod(mod_ty mod, const char *filename, PyObject *globals, PyObject *locals, PyCompilerFlags *flags, PyArena *arena) @@ -1362,7 +1979,7 @@ run_mod(mod_ty mod, const char *filename, PyObject *globals, PyObject *locals, co = PyAST_Compile(mod, filename, flags, arena); if (co == NULL) return NULL; - v = PyEval_EvalCode(co, globals, locals); + v = PyEval_EvalCode((PyObject*)co, globals, locals); Py_DECREF(co); return v; } @@ -1382,9 +1999,10 @@ run_pyc_file(FILE *fp, const char *filename, PyObject *globals, "Bad magic number in .pyc file"); return NULL; } + /* Skip mtime and size */ + (void) PyMarshal_ReadLongFromFile(fp); (void) PyMarshal_ReadLongFromFile(fp); v = PyMarshal_ReadLastObjectFromFile(fp); - fclose(fp); if (v == NULL || !PyCode_Check(v)) { Py_XDECREF(v); PyErr_SetString(PyExc_RuntimeError, @@ -1392,7 +2010,7 @@ run_pyc_file(FILE *fp, const char *filename, PyObject *globals, return NULL; } co = (PyCodeObject *)v; - v = PyEval_EvalCode(co, globals, locals); + v = PyEval_EvalCode((PyObject*)co, globals, locals); if (v && flags) flags->cf_flags |= (co->co_flags & PyCF_MASK); Py_DECREF(co); @@ -1400,8 +2018,8 @@ run_pyc_file(FILE *fp, const char *filename, PyObject *globals, } PyObject * -Py_CompileStringFlags(const char *str, const char *filename, int start, - PyCompilerFlags *flags) +Py_CompileStringExFlags(const char *str, const char *filename, int start, + PyCompilerFlags *flags, int optimize) { PyCodeObject *co; mod_ty mod; @@ -1419,11 +2037,19 @@ Py_CompileStringFlags(const char *str, const char *filename, int start, PyArena_Free(arena); return result; } - co = PyAST_Compile(mod, filename, flags, arena); + co = PyAST_CompileEx(mod, filename, flags, optimize, arena); PyArena_Free(arena); return (PyObject *)co; } +/* For use in Py_LIMITED_API */ +#undef Py_CompileString +PyObject * +PyCompileString(const char *str, const char *filename, int start) +{ + return Py_CompileStringFlags(str, filename, start, NULL); +} + struct symtable * Py_SymtableString(const char *str, const char *filename, int start) { @@ -1435,7 +2061,6 @@ Py_SymtableString(const char *str, const char *filename, int start) return NULL; flags.cf_flags = 0; - mod = PyParser_ASTFromString(str, filename, start, &flags, arena); if (mod == NULL) { PyArena_Free(arena); @@ -1467,16 +2092,18 @@ PyParser_ASTFromString(const char *s, const char *filename, int start, flags->cf_flags |= iflags & PyCF_MASK; mod = PyAST_FromNode(n, flags, filename, arena); PyNode_Free(n); - return mod; } else { err_input(&err); - return NULL; + mod = NULL; } + err_free(&err); + return mod; } mod_ty -PyParser_ASTFromFile(FILE *fp, const char *filename, int start, char *ps1, +PyParser_ASTFromFile(FILE *fp, const char *filename, const char* enc, + int start, char *ps1, char *ps2, PyCompilerFlags *flags, int *errcode, PyArena *arena) { @@ -1485,7 +2112,8 @@ PyParser_ASTFromFile(FILE *fp, const char *filename, int start, char *ps1, perrdetail err; int iflags = PARSER_FLAGS(flags); - node *n = PyParser_ParseFileFlagsEx(fp, filename, &_PyParser_Grammar, + node *n = PyParser_ParseFileFlagsEx(fp, filename, enc, + &_PyParser_Grammar, start, ps1, ps2, &err, &iflags); if (flags == NULL) { localflags.cf_flags = 0; @@ -1495,14 +2123,15 @@ PyParser_ASTFromFile(FILE *fp, const char *filename, int start, char *ps1, flags->cf_flags |= iflags & PyCF_MASK; mod = PyAST_FromNode(n, flags, filename, arena); PyNode_Free(n); - return mod; } else { err_input(&err); if (errcode) *errcode = err.error; - return NULL; + mod = NULL; } + err_free(&err); + return mod; } /* Simplified interface to parsefile -- return node or set exception */ @@ -1511,10 +2140,12 @@ node * PyParser_SimpleParseFileFlags(FILE *fp, const char *filename, int start, int flags) { perrdetail err; - node *n = PyParser_ParseFileFlags(fp, filename, &_PyParser_Grammar, + node *n = PyParser_ParseFileFlags(fp, filename, NULL, + &_PyParser_Grammar, start, NULL, NULL, &err, flags); if (n == NULL) err_input(&err); + err_free(&err); return n; } @@ -1529,6 +2160,7 @@ PyParser_SimpleParseStringFlags(const char *str, int start, int flags) start, &err, flags); if (n == NULL) err_input(&err); + err_free(&err); return n; } @@ -1541,6 +2173,7 @@ PyParser_SimpleParseStringFlagsFilename(const char *str, const char *filename, &_PyParser_Grammar, start, &err, flags); if (n == NULL) err_input(&err); + err_free(&err); return n; } @@ -1554,19 +2187,32 @@ PyParser_SimpleParseStringFilename(const char *str, const char *filename, int st even parser modules. */ void +PyParser_ClearError(perrdetail *err) +{ + err_free(err); +} + +void PyParser_SetError(perrdetail *err) { err_input(err); } +static void +err_free(perrdetail *err) +{ + Py_CLEAR(err->filename); +} + /* Set the error appropriate to the given input error code (see errcode.h) */ static void err_input(perrdetail *err) { - PyObject *v, *w, *errtype; - PyObject* u = NULL; + PyObject *v, *w, *errtype, *errtext; + PyObject *msg_obj = NULL; char *msg = NULL; + errtype = PyExc_SyntaxError; switch (err->error) { case E_ERROR: @@ -1621,14 +2267,9 @@ err_input(perrdetail *err) case E_DECODE: { PyObject *type, *value, *tb; PyErr_Fetch(&type, &value, &tb); - if (value != NULL) { - u = PyObject_Str(value); - if (u != NULL) { - msg = PyString_AsString(u); - } - } - if (msg == NULL) - msg = "unknown decode error"; + msg = "unknown decode error"; + if (value != NULL) + msg_obj = PyObject_Str(value); Py_XDECREF(type); Py_XDECREF(value); Py_XDECREF(tb); @@ -1637,21 +2278,41 @@ err_input(perrdetail *err) case E_LINECONT: msg = "unexpected character after line continuation character"; break; + + case E_IDENTIFIER: + msg = "invalid character in identifier"; + break; + case E_BADSINGLE: + msg = "multiple statements found while compiling a single statement"; + break; default: fprintf(stderr, "error=%d\n", err->error); msg = "unknown parsing error"; break; } - v = Py_BuildValue("(ziiz)", err->filename, - err->lineno, err->offset, err->text); - w = NULL; - if (v != NULL) - w = Py_BuildValue("(sO)", msg, v); - Py_XDECREF(u); + /* err->text may not be UTF-8 in case of decoding errors. + Explicitly convert to an object. */ + if (!err->text) { + errtext = Py_None; + Py_INCREF(Py_None); + } else { + errtext = PyUnicode_DecodeUTF8(err->text, strlen(err->text), + "replace"); + } + v = Py_BuildValue("(OiiN)", err->filename, + err->lineno, err->offset, errtext); + if (v != NULL) { + if (msg_obj) + w = Py_BuildValue("(OO)", msg_obj, v); + else + w = Py_BuildValue("(sO)", msg, v); + } else + w = NULL; Py_XDECREF(v); PyErr_SetObject(errtype, w); Py_XDECREF(w); cleanup: + Py_XDECREF(msg_obj); if (err->text != NULL) { PyObject_FREE(err->text); err->text = NULL; @@ -1663,8 +2324,23 @@ cleanup: void Py_FatalError(const char *msg) { + const int fd = fileno(stderr); + PyThreadState *tstate; + fprintf(stderr, "Fatal Python error: %s\n", msg); fflush(stderr); /* it helps in Windows debug build */ + if (PyErr_Occurred()) { + PyErr_PrintEx(0); + } + else { + tstate = _Py_atomic_load_relaxed(&_PyThreadState_Current); + if (tstate != NULL) { + fputc('\n', stderr); + fflush(stderr); + _Py_DumpTracebackThreads(fd, tstate->interp, tstate); + } + _PyFaulthandler_Fini(); + } #ifdef MS_WINDOWS { @@ -1695,6 +2371,23 @@ Py_FatalError(const char *msg) #include "pythread.h" #endif +static void (*pyexitfunc)(void) = NULL; +/* For the atexit module. */ +void _Py_PyAtExit(void (*func)(void)) +{ + pyexitfunc = func; +} + +static void +call_py_exitfuncs(void) +{ + if (pyexitfunc == NULL) + return; + + (*pyexitfunc)(); + PyErr_Clear(); +} + /* Wait until threading._shutdown completes, provided the threading module was imported in the first place. The shutdown routine will wait until all non-daemon @@ -1703,6 +2396,7 @@ static void wait_for_thread_shutdown(void) { #ifdef WITH_THREAD + _Py_IDENTIFIER(_shutdown); PyObject *result; PyThreadState *tstate = PyThreadState_GET(); PyObject *threading = PyMapping_GetItemString(tstate->interp->modules, @@ -1712,11 +2406,13 @@ wait_for_thread_shutdown(void) PyErr_Clear(); return; } - result = PyObject_CallMethod(threading, "_shutdown", ""); - if (result == NULL) + result = _PyObject_CallMethodId(threading, &PyId__shutdown, ""); + if (result == NULL) { PyErr_WriteUnraisable(threading); - else + } + else { Py_DECREF(result); + } Py_DECREF(threading); #endif } @@ -1734,29 +2430,6 @@ int Py_AtExit(void (*func)(void)) } static void -call_sys_exitfunc(void) -{ - PyObject *exitfunc = PySys_GetObject("exitfunc"); - - if (exitfunc) { - PyObject *res; - Py_INCREF(exitfunc); - PySys_SetObject("exitfunc", (PyObject *)NULL); - res = PyEval_CallObject(exitfunc, (PyObject *)NULL); - if (res == NULL) { - if (!PyErr_ExceptionMatches(PyExc_SystemExit)) { - PySys_WriteStderr("Error in sys.exitfunc:\n"); - } - PyErr_Print(); - } - Py_DECREF(exitfunc); - } - - if (Py_FlushLine()) - PyErr_Clear(); -} - -static void call_ll_exitfuncs(void) { while (nexitfuncs > 0) @@ -1790,6 +2463,27 @@ initsigs(void) } +/* Restore signals that the interpreter has called SIG_IGN on to SIG_DFL. + * + * All of the code in this function must only use async-signal-safe functions, + * listed at `man 7 signal` or + * http://www.opengroup.org/onlinepubs/009695399/functions/xsh_chap02_04.html. + */ +void +_Py_RestoreSignals(void) +{ +#ifdef SIGPIPE + PyOS_setsig(SIGPIPE, SIG_DFL); +#endif +#ifdef SIGXFZ + PyOS_setsig(SIGXFZ, SIG_DFL); +#endif +#ifdef SIGXFSZ + PyOS_setsig(SIGXFSZ, SIG_DFL); +#endif +} + + /* * The file descriptor fd is considered ``interactive'' if either * a) isatty(fd) is TRUE, or @@ -1883,6 +2577,11 @@ PyOS_getsig(int sig) #endif } +/* + * All of the code in this function must only use async-signal-safe functions, + * listed at `man 7 signal` or + * http://www.opengroup.org/onlinepubs/009695399/functions/xsh_chap02_04.html. + */ PyOS_sighandler_t PyOS_setsig(int sig, PyOS_sighandler_t handler) { @@ -2000,7 +2699,15 @@ PyRun_SimpleString(const char *s) PyAPI_FUNC(PyObject *) Py_CompileString(const char *str, const char *p, int s) { - return Py_CompileStringFlags(str, p, s, NULL); + return Py_CompileStringExFlags(str, p, s, NULL, -1); +} + +#undef Py_CompileStringFlags +PyAPI_FUNC(PyObject *) +Py_CompileStringFlags(const char *str, const char *p, int s, + PyCompilerFlags *flags) +{ + return Py_CompileStringExFlags(str, p, s, flags, -1); } #undef PyRun_InteractiveOne @@ -2020,4 +2727,3 @@ PyRun_InteractiveLoop(FILE *f, const char *p) #ifdef __cplusplus } #endif - |