diff options
Diffstat (limited to 'Python')
45 files changed, 7025 insertions, 2709 deletions
diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 45fe444e26..dcfde3c349 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -2,7 +2,7 @@ /* - __version__ . + __version__ 82160. This module must be committed separately after each AST grammar change; The __version__ number is set to the revision number of the commit @@ -188,11 +188,26 @@ static char *Dict_fields[]={ "keys", "values", }; +static PyTypeObject *Set_type; +static char *Set_fields[]={ + "elts", +}; static PyTypeObject *ListComp_type; static char *ListComp_fields[]={ "elt", "generators", }; +static PyTypeObject *SetComp_type; +static char *SetComp_fields[]={ + "elt", + "generators", +}; +static PyTypeObject *DictComp_type; +static char *DictComp_fields[]={ + "key", + "value", + "generators", +}; static PyTypeObject *GeneratorExp_type; static char *GeneratorExp_fields[]={ "elt", @@ -512,8 +527,9 @@ static int add_attributes(PyTypeObject* type, char**attrs, int num_fields) { int i, result; PyObject *s, *l = PyTuple_New(num_fields); - if (!l) return 0; - for(i = 0; i < num_fields; i++) { + if (!l) + return 0; + for (i = 0; i < num_fields; i++) { s = PyString_FromString(attrs[i]); if (!s) { Py_DECREF(l); @@ -578,8 +594,25 @@ static int obj2ast_object(PyObject* obj, PyObject** out, PyArena* arena) return 0; } -#define obj2ast_identifier obj2ast_object -#define obj2ast_string obj2ast_object +static int obj2ast_identifier(PyObject* obj, PyObject** out, PyArena* arena) +{ + if (!PyString_CheckExact(obj) && obj != Py_None) { + PyErr_Format(PyExc_TypeError, + "AST identifier must be of type str"); + return 1; + } + return obj2ast_object(obj, out, arena); +} + +static int obj2ast_string(PyObject* obj, PyObject** out, PyArena* arena) +{ + if (!PyString_CheckExact(obj) && !PyUnicode_CheckExact(obj)) { + PyErr_SetString(PyExc_TypeError, + "AST string must be of type str or unicode"); + return 1; + } + return obj2ast_object(obj, out, arena); +} static int obj2ast_int(PyObject* obj, int* out, PyArena* arena) { @@ -718,8 +751,14 @@ static int init_types(void) if (!IfExp_type) return 0; Dict_type = make_type("Dict", expr_type, Dict_fields, 2); if (!Dict_type) return 0; + Set_type = make_type("Set", expr_type, Set_fields, 1); + if (!Set_type) return 0; ListComp_type = make_type("ListComp", expr_type, ListComp_fields, 2); if (!ListComp_type) return 0; + SetComp_type = make_type("SetComp", expr_type, SetComp_fields, 2); + if (!SetComp_type) return 0; + DictComp_type = make_type("DictComp", expr_type, DictComp_fields, 3); + if (!DictComp_type) return 0; GeneratorExp_type = make_type("GeneratorExp", expr_type, GeneratorExp_fields, 2); if (!GeneratorExp_type) return 0; @@ -1330,11 +1369,6 @@ ImportFrom(identifier module, asdl_seq * names, int level, int lineno, int col_offset, PyArena *arena) { stmt_ty p; - if (!module) { - PyErr_SetString(PyExc_ValueError, - "field module is required for ImportFrom"); - return NULL; - } p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p)); if (!p) return NULL; @@ -1595,6 +1629,20 @@ Dict(asdl_seq * keys, asdl_seq * values, int lineno, int col_offset, PyArena } expr_ty +Set(asdl_seq * elts, int lineno, int col_offset, PyArena *arena) +{ + expr_ty p; + p = (expr_ty)PyArena_Malloc(arena, sizeof(*p)); + if (!p) + return NULL; + p->kind = Set_kind; + p->v.Set.elts = elts; + p->lineno = lineno; + p->col_offset = col_offset; + return p; +} + +expr_ty ListComp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset, PyArena *arena) { @@ -1616,6 +1664,54 @@ ListComp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset, } expr_ty +SetComp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset, PyArena + *arena) +{ + expr_ty p; + if (!elt) { + PyErr_SetString(PyExc_ValueError, + "field elt is required for SetComp"); + return NULL; + } + p = (expr_ty)PyArena_Malloc(arena, sizeof(*p)); + if (!p) + return NULL; + p->kind = SetComp_kind; + p->v.SetComp.elt = elt; + p->v.SetComp.generators = generators; + p->lineno = lineno; + p->col_offset = col_offset; + return p; +} + +expr_ty +DictComp(expr_ty key, expr_ty value, asdl_seq * generators, int lineno, int + col_offset, PyArena *arena) +{ + expr_ty p; + if (!key) { + PyErr_SetString(PyExc_ValueError, + "field key is required for DictComp"); + return NULL; + } + if (!value) { + PyErr_SetString(PyExc_ValueError, + "field value is required for DictComp"); + return NULL; + } + p = (expr_ty)PyArena_Malloc(arena, sizeof(*p)); + if (!p) + return NULL; + p->kind = DictComp_kind; + p->v.DictComp.key = key; + p->v.DictComp.value = value; + p->v.DictComp.generators = generators; + p->lineno = lineno; + p->col_offset = col_offset; + return p; +} + +expr_ty GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset, PyArena *arena) { @@ -2571,6 +2667,15 @@ ast2obj_expr(void* _o) goto failed; Py_DECREF(value); break; + case Set_kind: + result = PyType_GenericNew(Set_type, NULL, NULL); + if (!result) goto failed; + value = ast2obj_list(o->v.Set.elts, ast2obj_expr); + if (!value) goto failed; + if (PyObject_SetAttrString(result, "elts", value) == -1) + goto failed; + Py_DECREF(value); + break; case ListComp_kind: result = PyType_GenericNew(ListComp_type, NULL, NULL); if (!result) goto failed; @@ -2586,6 +2691,41 @@ ast2obj_expr(void* _o) goto failed; Py_DECREF(value); break; + case SetComp_kind: + result = PyType_GenericNew(SetComp_type, NULL, NULL); + if (!result) goto failed; + value = ast2obj_expr(o->v.SetComp.elt); + if (!value) goto failed; + if (PyObject_SetAttrString(result, "elt", value) == -1) + goto failed; + Py_DECREF(value); + value = ast2obj_list(o->v.SetComp.generators, + ast2obj_comprehension); + if (!value) goto failed; + if (PyObject_SetAttrString(result, "generators", value) == -1) + goto failed; + Py_DECREF(value); + break; + case DictComp_kind: + result = PyType_GenericNew(DictComp_type, NULL, NULL); + if (!result) goto failed; + value = ast2obj_expr(o->v.DictComp.key); + if (!value) goto failed; + if (PyObject_SetAttrString(result, "key", value) == -1) + goto failed; + Py_DECREF(value); + value = ast2obj_expr(o->v.DictComp.value); + if (!value) goto failed; + if (PyObject_SetAttrString(result, "value", value) == -1) + goto failed; + Py_DECREF(value); + value = ast2obj_list(o->v.DictComp.generators, + ast2obj_comprehension); + if (!value) goto failed; + if (PyObject_SetAttrString(result, "generators", value) == -1) + goto failed; + Py_DECREF(value); + break; case GeneratorExp_kind: result = PyType_GenericNew(GeneratorExp_type, NULL, NULL); if (!result) goto failed; @@ -4359,8 +4499,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) Py_XDECREF(tmp); tmp = NULL; } else { - PyErr_SetString(PyExc_TypeError, "required field \"module\" missing from ImportFrom"); - return 1; + module = NULL; } if (PyObject_HasAttrString(obj, "names")) { int res; @@ -4866,6 +5005,42 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) if (*out == NULL) goto failed; return 0; } + isinstance = PyObject_IsInstance(obj, (PyObject*)Set_type); + if (isinstance == -1) { + return 1; + } + if (isinstance) { + asdl_seq* elts; + + if (PyObject_HasAttrString(obj, "elts")) { + int res; + Py_ssize_t len; + Py_ssize_t i; + tmp = PyObject_GetAttrString(obj, "elts"); + if (tmp == NULL) goto failed; + if (!PyList_Check(tmp)) { + PyErr_Format(PyExc_TypeError, "Set field \"elts\" must be a list, not a %.200s", tmp->ob_type->tp_name); + goto failed; + } + len = PyList_GET_SIZE(tmp); + elts = asdl_seq_new(len, arena); + if (elts == NULL) goto failed; + for (i = 0; i < len; i++) { + expr_ty value; + res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(elts, i, value); + } + Py_XDECREF(tmp); + tmp = NULL; + } else { + PyErr_SetString(PyExc_TypeError, "required field \"elts\" missing from Set"); + return 1; + } + *out = Set(elts, lineno, col_offset, arena); + if (*out == NULL) goto failed; + return 0; + } isinstance = PyObject_IsInstance(obj, (PyObject*)ListComp_type); if (isinstance == -1) { return 1; @@ -4915,6 +5090,118 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) if (*out == NULL) goto failed; return 0; } + isinstance = PyObject_IsInstance(obj, (PyObject*)SetComp_type); + if (isinstance == -1) { + return 1; + } + if (isinstance) { + expr_ty elt; + asdl_seq* generators; + + if (PyObject_HasAttrString(obj, "elt")) { + int res; + tmp = PyObject_GetAttrString(obj, "elt"); + if (tmp == NULL) goto failed; + res = obj2ast_expr(tmp, &elt, arena); + if (res != 0) goto failed; + Py_XDECREF(tmp); + tmp = NULL; + } else { + PyErr_SetString(PyExc_TypeError, "required field \"elt\" missing from SetComp"); + return 1; + } + if (PyObject_HasAttrString(obj, "generators")) { + int res; + Py_ssize_t len; + Py_ssize_t i; + tmp = PyObject_GetAttrString(obj, "generators"); + if (tmp == NULL) goto failed; + if (!PyList_Check(tmp)) { + PyErr_Format(PyExc_TypeError, "SetComp field \"generators\" must be a list, not a %.200s", tmp->ob_type->tp_name); + goto failed; + } + len = PyList_GET_SIZE(tmp); + generators = asdl_seq_new(len, arena); + if (generators == NULL) goto failed; + for (i = 0; i < len; i++) { + comprehension_ty value; + res = obj2ast_comprehension(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(generators, i, value); + } + Py_XDECREF(tmp); + tmp = NULL; + } else { + PyErr_SetString(PyExc_TypeError, "required field \"generators\" missing from SetComp"); + return 1; + } + *out = SetComp(elt, generators, lineno, col_offset, arena); + if (*out == NULL) goto failed; + return 0; + } + isinstance = PyObject_IsInstance(obj, (PyObject*)DictComp_type); + if (isinstance == -1) { + return 1; + } + if (isinstance) { + expr_ty key; + expr_ty value; + asdl_seq* generators; + + if (PyObject_HasAttrString(obj, "key")) { + int res; + tmp = PyObject_GetAttrString(obj, "key"); + if (tmp == NULL) goto failed; + res = obj2ast_expr(tmp, &key, arena); + if (res != 0) goto failed; + Py_XDECREF(tmp); + tmp = NULL; + } else { + PyErr_SetString(PyExc_TypeError, "required field \"key\" missing from DictComp"); + return 1; + } + if (PyObject_HasAttrString(obj, "value")) { + int res; + tmp = PyObject_GetAttrString(obj, "value"); + if (tmp == NULL) goto failed; + res = obj2ast_expr(tmp, &value, arena); + if (res != 0) goto failed; + Py_XDECREF(tmp); + tmp = NULL; + } else { + PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from DictComp"); + return 1; + } + if (PyObject_HasAttrString(obj, "generators")) { + int res; + Py_ssize_t len; + Py_ssize_t i; + tmp = PyObject_GetAttrString(obj, "generators"); + if (tmp == NULL) goto failed; + if (!PyList_Check(tmp)) { + PyErr_Format(PyExc_TypeError, "DictComp field \"generators\" must be a list, not a %.200s", tmp->ob_type->tp_name); + goto failed; + } + len = PyList_GET_SIZE(tmp); + generators = asdl_seq_new(len, arena); + if (generators == NULL) goto failed; + for (i = 0; i < len; i++) { + comprehension_ty value; + res = obj2ast_comprehension(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(generators, i, value); + } + Py_XDECREF(tmp); + tmp = NULL; + } else { + PyErr_SetString(PyExc_TypeError, "required field \"generators\" missing from DictComp"); + return 1; + } + *out = DictComp(key, value, generators, lineno, col_offset, + arena); + if (*out == NULL) goto failed; + return 0; + } isinstance = PyObject_IsInstance(obj, (PyObject*)GeneratorExp_type); if (isinstance == -1) { return 1; @@ -6300,7 +6587,7 @@ init_ast(void) if (PyDict_SetItemString(d, "AST", (PyObject*)&AST_type) < 0) return; if (PyModule_AddIntConstant(m, "PyCF_ONLY_AST", PyCF_ONLY_AST) < 0) return; - if (PyModule_AddStringConstant(m, "__version__", "") < 0) + if (PyModule_AddStringConstant(m, "__version__", "82160") < 0) return; if (PyDict_SetItemString(d, "mod", (PyObject*)mod_type) < 0) return; if (PyDict_SetItemString(d, "Module", (PyObject*)Module_type) < 0) @@ -6357,8 +6644,13 @@ init_ast(void) return; if (PyDict_SetItemString(d, "IfExp", (PyObject*)IfExp_type) < 0) return; if (PyDict_SetItemString(d, "Dict", (PyObject*)Dict_type) < 0) return; + if (PyDict_SetItemString(d, "Set", (PyObject*)Set_type) < 0) return; if (PyDict_SetItemString(d, "ListComp", (PyObject*)ListComp_type) < 0) return; + if (PyDict_SetItemString(d, "SetComp", (PyObject*)SetComp_type) < 0) + return; + if (PyDict_SetItemString(d, "DictComp", (PyObject*)DictComp_type) < 0) + return; if (PyDict_SetItemString(d, "GeneratorExp", (PyObject*)GeneratorExp_type) < 0) return; if (PyDict_SetItemString(d, "Yield", (PyObject*)Yield_type) < 0) return; diff --git a/Python/_warnings.c b/Python/_warnings.c index 3313c59f40..445ff6ba9e 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -1,9 +1,7 @@ #include "Python.h" -#include "code.h" /* For DeprecationWarning about adding 'line'. */ #include "frameobject.h" #define MODULE_NAME "_warnings" -#define DEFAULT_ACTION_NAME "default_action" PyDoc_STRVAR(warnings__doc__, MODULE_NAME " provides basic warning filtering support.\n" @@ -13,6 +11,7 @@ MODULE_NAME " provides basic warning filtering support.\n" get_warnings_attr() will reset these variables accordingly. */ static PyObject *_filters; /* List */ static PyObject *_once_registry; /* Dict */ +static PyObject *_default_action; /* String */ static int @@ -22,10 +21,10 @@ check_matched(PyObject *obj, PyObject *arg) int rc; if (obj == Py_None) - return 1; + return 1; result = PyObject_CallMethod(obj, "match", "O", arg); if (result == NULL) - return -1; + return -1; rc = PyObject_IsTrue(result); Py_DECREF(result); @@ -45,19 +44,19 @@ get_warnings_attr(const char *attr) int result; if (warnings_str == NULL) { - warnings_str = PyString_InternFromString("warnings"); - if (warnings_str == NULL) - return NULL; + warnings_str = PyString_InternFromString("warnings"); + if (warnings_str == NULL) + return NULL; } all_modules = PyImport_GetModuleDict(); result = PyDict_Contains(all_modules, warnings_str); if (result == -1 || result == 0) - return NULL; + return NULL; warnings_module = PyDict_GetItem(all_modules, warnings_str); if (!PyObject_HasAttrString(warnings_module, attr)) - return NULL; + return NULL; return PyObject_GetAttrString(warnings_module, attr); } @@ -69,9 +68,9 @@ get_once_registry(void) registry = get_warnings_attr("onceregistry"); if (registry == NULL) { - if (PyErr_Occurred()) - return NULL; - return _once_registry; + if (PyErr_Occurred()) + return NULL; + return _once_registry; } Py_DECREF(_once_registry); _once_registry = registry; @@ -79,97 +78,111 @@ get_once_registry(void) } +static PyObject * +get_default_action(void) +{ + PyObject *default_action; + + default_action = get_warnings_attr("defaultaction"); + if (default_action == NULL) { + if (PyErr_Occurred()) { + return NULL; + } + return _default_action; + } + + Py_DECREF(_default_action); + _default_action = default_action; + return default_action; +} + + /* The item is a borrowed reference. */ static const char * get_filter(PyObject *category, PyObject *text, Py_ssize_t lineno, PyObject *module, PyObject **item) { - PyObject *action, *m, *d; + PyObject *action; Py_ssize_t i; PyObject *warnings_filters; warnings_filters = get_warnings_attr("filters"); if (warnings_filters == NULL) { - if (PyErr_Occurred()) - return NULL; + if (PyErr_Occurred()) + return NULL; } else { - Py_DECREF(_filters); - _filters = warnings_filters; + Py_DECREF(_filters); + _filters = warnings_filters; } if (!PyList_Check(_filters)) { - PyErr_SetString(PyExc_ValueError, - MODULE_NAME ".filters must be a list"); - return NULL; + PyErr_SetString(PyExc_ValueError, + MODULE_NAME ".filters must be a list"); + return NULL; } /* _filters could change while we are iterating over it. */ for (i = 0; i < PyList_GET_SIZE(_filters); i++) { - PyObject *tmp_item, *action, *msg, *cat, *mod, *ln_obj; - Py_ssize_t ln; - int is_subclass, good_msg, good_mod; - - tmp_item = *item = PyList_GET_ITEM(_filters, i); - if (PyTuple_Size(tmp_item) != 5) { - PyErr_Format(PyExc_ValueError, - MODULE_NAME ".filters item %zd isn't a 5-tuple", i); - return NULL; - } + PyObject *tmp_item, *action, *msg, *cat, *mod, *ln_obj; + Py_ssize_t ln; + int is_subclass, good_msg, good_mod; + + tmp_item = *item = PyList_GET_ITEM(_filters, i); + if (PyTuple_Size(tmp_item) != 5) { + PyErr_Format(PyExc_ValueError, + MODULE_NAME ".filters item %zd isn't a 5-tuple", i); + return NULL; + } - /* Python code: action, msg, cat, mod, ln = item */ - action = PyTuple_GET_ITEM(tmp_item, 0); - msg = PyTuple_GET_ITEM(tmp_item, 1); - cat = PyTuple_GET_ITEM(tmp_item, 2); - mod = PyTuple_GET_ITEM(tmp_item, 3); - ln_obj = PyTuple_GET_ITEM(tmp_item, 4); + /* Python code: action, msg, cat, mod, ln = item */ + action = PyTuple_GET_ITEM(tmp_item, 0); + msg = PyTuple_GET_ITEM(tmp_item, 1); + cat = PyTuple_GET_ITEM(tmp_item, 2); + mod = PyTuple_GET_ITEM(tmp_item, 3); + ln_obj = PyTuple_GET_ITEM(tmp_item, 4); - good_msg = check_matched(msg, text); - good_mod = check_matched(mod, module); - is_subclass = PyObject_IsSubclass(category, cat); - ln = PyInt_AsSsize_t(ln_obj); - if (good_msg == -1 || good_mod == -1 || is_subclass == -1 || - (ln == -1 && PyErr_Occurred())) - return NULL; + good_msg = check_matched(msg, text); + good_mod = check_matched(mod, module); + is_subclass = PyObject_IsSubclass(category, cat); + ln = PyInt_AsSsize_t(ln_obj); + if (good_msg == -1 || good_mod == -1 || is_subclass == -1 || + (ln == -1 && PyErr_Occurred())) + return NULL; - if (good_msg && is_subclass && good_mod && (ln == 0 || lineno == ln)) - return PyString_AsString(action); + if (good_msg && is_subclass && good_mod && (ln == 0 || lineno == ln)) + return PyString_AsString(action); } - m = PyImport_ImportModule(MODULE_NAME); - if (m == NULL) - return NULL; - d = PyModule_GetDict(m); - Py_DECREF(m); - if (d == NULL) - return NULL; - action = PyDict_GetItemString(d, DEFAULT_ACTION_NAME); - if (action != NULL) - return PyString_AsString(action); + action = get_default_action(); + if (action != NULL) { + return PyString_AsString(action); + } PyErr_SetString(PyExc_ValueError, - MODULE_NAME "." DEFAULT_ACTION_NAME " not found"); + MODULE_NAME ".defaultaction not found"); return NULL; } + static int already_warned(PyObject *registry, PyObject *key, int should_set) { PyObject *already_warned; if (key == NULL) - return -1; + return -1; already_warned = PyDict_GetItem(registry, key); if (already_warned != NULL) { - int rc = PyObject_IsTrue(already_warned); - if (rc != 0) - return rc; + int rc = PyObject_IsTrue(already_warned); + if (rc != 0) + return rc; } /* This warning wasn't found in the registry, set it. */ if (should_set) - return PyDict_SetItem(registry, key, Py_True); + return PyDict_SetItem(registry, key, Py_True); return 0; } @@ -183,23 +196,23 @@ normalize_module(PyObject *filename) int rc = PyObject_IsTrue(filename); if (rc == -1) - return NULL; + return NULL; else if (rc == 0) - return PyString_FromString("<unknown>"); + return PyString_FromString("<unknown>"); mod_str = PyString_AsString(filename); if (mod_str == NULL) return NULL; len = PyString_Size(filename); if (len < 0) - return NULL; + return NULL; if (len >= 3 && - strncmp(mod_str + (len - 3), ".py", 3) == 0) { - module = PyString_FromStringAndSize(mod_str, len-3); + strncmp(mod_str + (len - 3), ".py", 3) == 0) { + module = PyString_FromStringAndSize(mod_str, len-3); } else { - module = filename; - Py_INCREF(module); + module = filename; + Py_INCREF(module); } return module; } @@ -212,13 +225,13 @@ update_registry(PyObject *registry, PyObject *text, PyObject *category, int rc; if (add_zero) { - zero = PyInt_FromLong(0); - if (zero == NULL) - return -1; - altkey = PyTuple_Pack(3, text, category, zero); + zero = PyInt_FromLong(0); + if (zero == NULL) + return -1; + altkey = PyTuple_Pack(3, text, category, zero); } else - altkey = PyTuple_Pack(2, text, category); + altkey = PyTuple_Pack(2, text, category); rc = already_warned(registry, altkey, 1); Py_XDECREF(zero); @@ -242,9 +255,9 @@ show_warning(PyObject *filename, int lineno, PyObject *text, PyObject f_stderr = PySys_GetObject("stderr"); if (f_stderr == NULL) { - fprintf(stderr, "lost sys.stderr\n"); - Py_DECREF(name); - return; + fprintf(stderr, "lost sys.stderr\n"); + Py_DECREF(name); + return; } /* Print "filename:lineno: category: text\n" */ @@ -258,17 +271,17 @@ show_warning(PyObject *filename, int lineno, PyObject *text, PyObject /* Print " source_line\n" */ if (sourceline) { - char *source_line_str = PyString_AS_STRING(sourceline); - while (*source_line_str == ' ' || *source_line_str == '\t' || - *source_line_str == '\014') - source_line_str++; + char *source_line_str = PyString_AS_STRING(sourceline); + while (*source_line_str == ' ' || *source_line_str == '\t' || + *source_line_str == '\014') + source_line_str++; - PyFile_WriteString(source_line_str, f_stderr); - PyFile_WriteString("\n", f_stderr); + PyFile_WriteString(source_line_str, f_stderr); + PyFile_WriteString("\n", f_stderr); } else - _Py_DisplaySourceLine(f_stderr, PyString_AS_STRING(filename), - lineno, 2); + _Py_DisplaySourceLine(f_stderr, PyString_AS_STRING(filename), + lineno, 2); PyErr_Clear(); } @@ -283,164 +296,133 @@ warn_explicit(PyObject *category, PyObject *message, int rc; if (registry && !PyDict_Check(registry) && (registry != Py_None)) { - PyErr_SetString(PyExc_TypeError, "'registry' must be a dict"); - return NULL; + PyErr_SetString(PyExc_TypeError, "'registry' must be a dict"); + return NULL; } /* Normalize module. */ if (module == NULL) { - module = normalize_module(filename); - if (module == NULL) - return NULL; + module = normalize_module(filename); + if (module == NULL) + return NULL; } else - Py_INCREF(module); + Py_INCREF(module); /* Normalize message. */ Py_INCREF(message); /* DECREF'ed in cleanup. */ rc = PyObject_IsInstance(message, PyExc_Warning); if (rc == -1) { - goto cleanup; + goto cleanup; } if (rc == 1) { - text = PyObject_Str(message); - if (text == NULL) - goto cleanup; - category = (PyObject*)message->ob_type; + text = PyObject_Str(message); + if (text == NULL) + goto cleanup; + category = (PyObject*)message->ob_type; } else { - text = message; - message = PyObject_CallFunction(category, "O", message); - if (message == NULL) - goto cleanup; + text = message; + message = PyObject_CallFunction(category, "O", message); + if (message == NULL) + goto cleanup; } lineno_obj = PyInt_FromLong(lineno); if (lineno_obj == NULL) - goto cleanup; + goto cleanup; /* Create key. */ key = PyTuple_Pack(3, text, category, lineno_obj); if (key == NULL) - goto cleanup; + goto cleanup; if ((registry != NULL) && (registry != Py_None)) { - rc = already_warned(registry, key, 0); - if (rc == -1) - goto cleanup; - else if (rc == 1) - goto return_none; - /* Else this warning hasn't been generated before. */ + rc = already_warned(registry, key, 0); + if (rc == -1) + goto cleanup; + else if (rc == 1) + goto return_none; + /* Else this warning hasn't been generated before. */ } action = get_filter(category, text, lineno, module, &item); if (action == NULL) - goto cleanup; + goto cleanup; if (strcmp(action, "error") == 0) { - PyErr_SetObject(category, message); - goto cleanup; + PyErr_SetObject(category, message); + goto cleanup; } /* Store in the registry that we've been here, *except* when the action is "always". */ rc = 0; if (strcmp(action, "always") != 0) { - if (registry != NULL && registry != Py_None && - PyDict_SetItem(registry, key, Py_True) < 0) - goto cleanup; - else if (strcmp(action, "ignore") == 0) - goto return_none; - else if (strcmp(action, "once") == 0) { - if (registry == NULL || registry == Py_None) { - registry = get_once_registry(); - if (registry == NULL) + if (registry != NULL && registry != Py_None && + PyDict_SetItem(registry, key, Py_True) < 0) goto cleanup; + else if (strcmp(action, "ignore") == 0) + goto return_none; + else if (strcmp(action, "once") == 0) { + if (registry == NULL || registry == Py_None) { + registry = get_once_registry(); + if (registry == NULL) + goto cleanup; + } + /* _once_registry[(text, category)] = 1 */ + rc = update_registry(registry, text, category, 0); } - /* _once_registry[(text, category)] = 1 */ - rc = update_registry(registry, text, category, 0); - } - else if (strcmp(action, "module") == 0) { - /* registry[(text, category, 0)] = 1 */ - if (registry != NULL && registry != Py_None) - rc = update_registry(registry, text, category, 0); - } - else if (strcmp(action, "default") != 0) { - PyObject *to_str = PyObject_Str(item); - const char *err_str = "???"; - - if (to_str != NULL) - err_str = PyString_AS_STRING(to_str); - PyErr_Format(PyExc_RuntimeError, - "Unrecognized action (%s) in warnings.filters:\n %s", - action, err_str); - Py_XDECREF(to_str); - goto cleanup; - } - } - - if (rc == 1) /* Already warned for this module. */ - goto return_none; - if (rc == 0) { - PyObject *show_fxn = get_warnings_attr("showwarning"); - if (show_fxn == NULL) { - if (PyErr_Occurred()) - goto cleanup; - show_warning(filename, lineno, text, category, sourceline); - } - else { - const char *msg = "functions overriding warnings.showwarning() " - "must support the 'line' argument"; - const char *text_char = PyString_AS_STRING(text); - - if (strcmp(msg, text_char) == 0) { - /* Prevent infinite recursion by using built-in implementation - of showwarning(). */ - show_warning(filename, lineno, text, category, sourceline); + else if (strcmp(action, "module") == 0) { + /* registry[(text, category, 0)] = 1 */ + if (registry != NULL && registry != Py_None) + rc = update_registry(registry, text, category, 0); } - else { - PyObject *check_fxn; - PyObject *defaults; - PyObject *res; - - if (PyMethod_Check(show_fxn)) - check_fxn = PyMethod_Function(show_fxn); - else if (PyFunction_Check(show_fxn)) - check_fxn = show_fxn; - else { - PyErr_SetString(PyExc_TypeError, - "warnings.showwarning() must be set to a " - "function or method"); - Py_DECREF(show_fxn); + else if (strcmp(action, "default") != 0) { + PyObject *to_str = PyObject_Str(item); + const char *err_str = "???"; + + if (to_str != NULL) + err_str = PyString_AS_STRING(to_str); + PyErr_Format(PyExc_RuntimeError, + "Unrecognized action (%s) in warnings.filters:\n %s", + action, err_str); + Py_XDECREF(to_str); goto cleanup; } + } - defaults = PyFunction_GetDefaults(check_fxn); - /* A proper implementation of warnings.showwarning() should - have at least two default arguments. */ - if ((defaults == NULL) || (PyTuple_Size(defaults) < 2)) { - PyCodeObject *code = (PyCodeObject *) - PyFunction_GetCode(check_fxn); - if (!(code->co_flags & CO_VARARGS)) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, msg, 1) < - 0) { - Py_DECREF(show_fxn); + if (rc == 1) /* Already warned for this module. */ + goto return_none; + if (rc == 0) { + PyObject *show_fxn = get_warnings_attr("showwarning"); + if (show_fxn == NULL) { + if (PyErr_Occurred()) goto cleanup; - } - } + show_warning(filename, lineno, text, category, sourceline); } - res = PyObject_CallFunctionObjArgs(show_fxn, message, category, - filename, lineno_obj, - NULL); - Py_DECREF(show_fxn); - Py_XDECREF(res); - if (res == NULL) - goto cleanup; + else { + PyObject *res; + + if (!PyMethod_Check(show_fxn) && !PyFunction_Check(show_fxn)) { + PyErr_SetString(PyExc_TypeError, + "warnings.showwarning() must be set to a " + "function or method"); + Py_DECREF(show_fxn); + goto cleanup; + } + + res = PyObject_CallFunctionObjArgs(show_fxn, message, category, + filename, lineno_obj, + NULL); + Py_DECREF(show_fxn); + Py_XDECREF(res); + if (res == NULL) + goto cleanup; } } - } else /* if (rc == -1) */ - goto cleanup; + goto cleanup; return_none: result = Py_None; @@ -466,15 +448,15 @@ setup_context(Py_ssize_t stack_level, PyObject **filename, int *lineno, /* Setup globals and lineno. */ PyFrameObject *f = PyThreadState_GET()->frame; while (--stack_level > 0 && f != NULL) - f = f->f_back; + f = f->f_back; if (f == NULL) { - globals = PyThreadState_Get()->interp->sysdict; - *lineno = 1; + globals = PyThreadState_Get()->interp->sysdict; + *lineno = 1; } else { - globals = f->f_globals; - *lineno = PyCode_Addr2Line(f->f_code, f->f_lasti); + globals = f->f_globals; + *lineno = PyFrame_GetLineNumber(f); } *module = NULL; @@ -484,85 +466,86 @@ setup_context(Py_ssize_t stack_level, PyObject **filename, int *lineno, assert(PyDict_Check(globals)); *registry = PyDict_GetItemString(globals, "__warningregistry__"); if (*registry == NULL) { - int rc; + int rc; - *registry = PyDict_New(); - if (*registry == NULL) - return 0; + *registry = PyDict_New(); + if (*registry == NULL) + return 0; - rc = PyDict_SetItemString(globals, "__warningregistry__", *registry); - if (rc < 0) - goto handle_error; + rc = PyDict_SetItemString(globals, "__warningregistry__", *registry); + if (rc < 0) + goto handle_error; } else - Py_INCREF(*registry); + Py_INCREF(*registry); /* Setup module. */ *module = PyDict_GetItemString(globals, "__name__"); if (*module == NULL) { - *module = PyString_FromString("<string>"); - if (*module == NULL) - goto handle_error; + *module = PyString_FromString("<string>"); + if (*module == NULL) + goto handle_error; } else - Py_INCREF(*module); + Py_INCREF(*module); /* Setup filename. */ *filename = PyDict_GetItemString(globals, "__file__"); - if (*filename != NULL) { - Py_ssize_t len = PyString_Size(*filename); - const char *file_str = PyString_AsString(*filename); - if (file_str == NULL || (len < 0 && PyErr_Occurred())) - goto handle_error; - - /* if filename.lower().endswith((".pyc", ".pyo")): */ - if (len >= 4 && - file_str[len-4] == '.' && - tolower(file_str[len-3]) == 'p' && - tolower(file_str[len-2]) == 'y' && - (tolower(file_str[len-1]) == 'c' || - tolower(file_str[len-1]) == 'o')) - { - *filename = PyString_FromStringAndSize(file_str, len-1); - if (*filename == NULL) + if (*filename != NULL && PyString_Check(*filename)) { + Py_ssize_t len = PyString_Size(*filename); + const char *file_str = PyString_AsString(*filename); + if (file_str == NULL || (len < 0 && PyErr_Occurred())) goto handle_error; + + /* if filename.lower().endswith((".pyc", ".pyo")): */ + if (len >= 4 && + file_str[len-4] == '.' && + tolower(file_str[len-3]) == 'p' && + tolower(file_str[len-2]) == 'y' && + (tolower(file_str[len-1]) == 'c' || + tolower(file_str[len-1]) == 'o')) + { + *filename = PyString_FromStringAndSize(file_str, len-1); + if (*filename == NULL) + goto handle_error; } else - Py_INCREF(*filename); + Py_INCREF(*filename); } else { - const char *module_str = PyString_AsString(*module); - if (module_str && strcmp(module_str, "__main__") == 0) { - PyObject *argv = PySys_GetObject("argv"); - if (argv != NULL && PyList_Size(argv) > 0) { - int is_true; - *filename = PyList_GetItem(argv, 0); - Py_INCREF(*filename); - /* If sys.argv[0] is false, then use '__main__'. */ - is_true = PyObject_IsTrue(*filename); - if (is_true < 0) { - Py_DECREF(*filename); - goto handle_error; - } - else if (!is_true) { - Py_DECREF(*filename); - *filename = PyString_FromString("__main__"); - if (*filename == NULL) - goto handle_error; - } + const char *module_str = PyString_AsString(*module); + *filename = NULL; + if (module_str && strcmp(module_str, "__main__") == 0) { + PyObject *argv = PySys_GetObject("argv"); + if (argv != NULL && PyList_Size(argv) > 0) { + int is_true; + *filename = PyList_GetItem(argv, 0); + Py_INCREF(*filename); + /* If sys.argv[0] is false, then use '__main__'. */ + is_true = PyObject_IsTrue(*filename); + if (is_true < 0) { + Py_DECREF(*filename); + goto handle_error; + } + else if (!is_true) { + Py_DECREF(*filename); + *filename = PyString_FromString("__main__"); + if (*filename == NULL) + goto handle_error; + } + } + else { + /* embedded interpreters don't have sys.argv, see bug #839151 */ + *filename = PyString_FromString("__main__"); + if (*filename == NULL) + goto handle_error; + } } - else { - /* embedded interpreters don't have sys.argv, see bug #839151 */ - *filename = PyString_FromString("__main__"); - if (*filename == NULL) - goto handle_error; + if (*filename == NULL) { + *filename = *module; + Py_INCREF(*filename); } } - if (*filename == NULL) { - *filename = *module; - Py_INCREF(*filename); - } - } return 1; @@ -582,21 +565,21 @@ get_category(PyObject *message, PyObject *category) /* Get category. */ rc = PyObject_IsInstance(message, PyExc_Warning); if (rc == -1) - return NULL; + return NULL; if (rc == 1) - category = (PyObject*)message->ob_type; + category = (PyObject*)message->ob_type; else if (category == NULL) - category = PyExc_UserWarning; + category = PyExc_UserWarning; /* Validate category. */ rc = PyObject_IsSubclass(category, PyExc_Warning); if (rc == -1) - return NULL; + return NULL; if (rc == 0) { - PyErr_SetString(PyExc_ValueError, - "category is not a subclass of Warning"); - return NULL; + PyErr_SetString(PyExc_ValueError, + "category is not a subclass of Warning"); + return NULL; } return category; @@ -609,7 +592,7 @@ do_warn(PyObject *message, PyObject *category, Py_ssize_t stack_level) int lineno; if (!setup_context(stack_level, &filename, &lineno, &module, ®istry)) - return NULL; + return NULL; res = warn_explicit(category, message, filename, lineno, module, registry, NULL); @@ -628,11 +611,11 @@ warnings_warn(PyObject *self, PyObject *args, PyObject *kwds) if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|On:warn", kw_list, &message, &category, &stack_level)) - return NULL; + return NULL; category = get_category(message, category); if (category == NULL) - return NULL; + return NULL; return do_warn(message, category, stack_level); } @@ -652,68 +635,68 @@ warnings_warn_explicit(PyObject *self, PyObject *args, PyObject *kwds) if (!PyArg_ParseTupleAndKeywords(args, kwds, "OOOi|OOO:warn_explicit", kwd_list, &message, &category, &filename, &lineno, &module, ®istry, &module_globals)) - return NULL; - - if (module_globals) { - static PyObject *get_source_name = NULL; - static PyObject *splitlines_name = NULL; - PyObject *loader; - PyObject *module_name; - PyObject *source; - PyObject *source_list; - PyObject *source_line; - PyObject *returned; - - if (get_source_name == NULL) { - get_source_name = PyString_InternFromString("get_source"); - if (!get_source_name) return NULL; - } - if (splitlines_name == NULL) { - splitlines_name = PyString_InternFromString("splitlines"); - if (!splitlines_name) - return NULL; - } - /* Check/get the requisite pieces needed for the loader. */ - loader = PyDict_GetItemString(module_globals, "__loader__"); - module_name = PyDict_GetItemString(module_globals, "__name__"); - - if (loader == NULL || module_name == NULL) - goto standard_call; + if (module_globals) { + static PyObject *get_source_name = NULL; + static PyObject *splitlines_name = NULL; + PyObject *loader; + PyObject *module_name; + PyObject *source; + PyObject *source_list; + PyObject *source_line; + PyObject *returned; + + if (get_source_name == NULL) { + get_source_name = PyString_InternFromString("get_source"); + if (!get_source_name) + return NULL; + } + if (splitlines_name == NULL) { + splitlines_name = PyString_InternFromString("splitlines"); + if (!splitlines_name) + return NULL; + } - /* Make sure the loader implements the optional get_source() method. */ - if (!PyObject_HasAttrString(loader, "get_source")) - goto standard_call; - /* Call get_source() to get the source code. */ - source = PyObject_CallMethodObjArgs(loader, get_source_name, - module_name, NULL); - if (!source) - return NULL; - else if (source == Py_None) { - Py_DECREF(Py_None); - goto standard_call; - } + /* Check/get the requisite pieces needed for the loader. */ + loader = PyDict_GetItemString(module_globals, "__loader__"); + module_name = PyDict_GetItemString(module_globals, "__name__"); + + if (loader == NULL || module_name == NULL) + goto standard_call; + + /* Make sure the loader implements the optional get_source() method. */ + if (!PyObject_HasAttrString(loader, "get_source")) + goto standard_call; + /* Call get_source() to get the source code. */ + source = PyObject_CallMethodObjArgs(loader, get_source_name, + module_name, NULL); + if (!source) + return NULL; + else if (source == Py_None) { + Py_DECREF(Py_None); + goto standard_call; + } - /* Split the source into lines. */ - source_list = PyObject_CallMethodObjArgs(source, splitlines_name, - NULL); - Py_DECREF(source); - if (!source_list) - return NULL; + /* Split the source into lines. */ + source_list = PyObject_CallMethodObjArgs(source, splitlines_name, + NULL); + Py_DECREF(source); + if (!source_list) + return NULL; + + /* Get the source line. */ + source_line = PyList_GetItem(source_list, lineno-1); + if (!source_line) { + Py_DECREF(source_list); + return NULL; + } - /* Get the source line. */ - source_line = PyList_GetItem(source_list, lineno-1); - if (!source_line) { + /* Handle the warning. */ + returned = warn_explicit(category, message, filename, lineno, module, + registry, source_line); Py_DECREF(source_list); - return NULL; - } - - /* Handle the warning. */ - returned = warn_explicit(category, message, filename, lineno, module, - registry, source_line); - Py_DECREF(source_list); - return returned; + return returned; } standard_call: @@ -729,21 +712,21 @@ PyErr_WarnEx(PyObject *category, const char *text, Py_ssize_t stack_level) PyObject *res; PyObject *message = PyString_FromString(text); if (message == NULL) - return -1; + return -1; if (category == NULL) - category = PyExc_RuntimeWarning; + category = PyExc_RuntimeWarning; res = do_warn(message, category, stack_level); Py_DECREF(message); if (res == NULL) - return -1; + return -1; Py_DECREF(res); return 0; } -/* PyErr_Warn is only for backwards compatability and will be removed. +/* PyErr_Warn is only for backwards compatibility and will be removed. Use PyErr_WarnEx instead. */ #undef PyErr_Warn @@ -767,19 +750,19 @@ PyErr_WarnExplicit(PyObject *category, const char *text, int ret = -1; if (message == NULL || filename == NULL) - goto exit; - if (module_str != NULL) { - module = PyString_FromString(module_str); - if (module == NULL) goto exit; + if (module_str != NULL) { + module = PyString_FromString(module_str); + if (module == NULL) + goto exit; } if (category == NULL) - category = PyExc_RuntimeWarning; + category = PyExc_RuntimeWarning; res = warn_explicit(category, message, filename, lineno, module, registry, NULL); if (res == NULL) - goto exit; + goto exit; Py_DECREF(res); ret = 0; @@ -799,12 +782,12 @@ PyDoc_STRVAR(warn_explicit_doc, static PyMethodDef warnings_functions[] = { {"warn", (PyCFunction)warnings_warn, METH_VARARGS | METH_KEYWORDS, - warn_doc}, + warn_doc}, {"warn_explicit", (PyCFunction)warnings_warn_explicit, - METH_VARARGS | METH_KEYWORDS, warn_explicit_doc}, + METH_VARARGS | METH_KEYWORDS, warn_explicit_doc}, /* XXX(brett.cannon): add showwarning? */ /* XXX(brett.cannon): Reasonable to add formatwarning? */ - {NULL, NULL} /* sentinel */ + {NULL, NULL} /* sentinel */ }; @@ -818,37 +801,37 @@ create_filter(PyObject *category, const char *action) PyObject *lineno, *result; if (!strcmp(action, "ignore")) { - if (ignore_str == NULL) { - ignore_str = PyString_InternFromString("ignore"); - if (ignore_str == NULL) - return NULL; - } - action_obj = ignore_str; + if (ignore_str == NULL) { + ignore_str = PyString_InternFromString("ignore"); + if (ignore_str == NULL) + return NULL; + } + action_obj = ignore_str; } else if (!strcmp(action, "error")) { - if (error_str == NULL) { - error_str = PyString_InternFromString("error"); - if (error_str == NULL) - return NULL; - } - action_obj = error_str; + if (error_str == NULL) { + error_str = PyString_InternFromString("error"); + if (error_str == NULL) + return NULL; + } + action_obj = error_str; } else if (!strcmp(action, "default")) { - if (default_str == NULL) { - default_str = PyString_InternFromString("default"); - if (default_str == NULL) - return NULL; - } - action_obj = default_str; + if (default_str == NULL) { + default_str = PyString_InternFromString("default"); + if (default_str == NULL) + return NULL; + } + action_obj = default_str; } else { - Py_FatalError("unknown action"); + Py_FatalError("unknown action"); } /* This assumes the line number is zero for now. */ lineno = PyInt_FromLong(0); if (lineno == NULL) - return NULL; + return NULL; result = PyTuple_Pack(5, action_obj, Py_None, category, Py_None, lineno); Py_DECREF(lineno); return result; @@ -857,28 +840,39 @@ create_filter(PyObject *category, const char *action) static PyObject * init_filters(void) { - PyObject *filters = PyList_New(3); + /* Don't silence DeprecationWarning if -3 or -Q were used. */ + PyObject *filters = PyList_New(Py_Py3kWarningFlag || + Py_DivisionWarningFlag ? 3 : 4); + unsigned int pos = 0; /* Post-incremented in each use. */ + unsigned int x; const char *bytes_action; + if (filters == NULL) - return NULL; + return NULL; - PyList_SET_ITEM(filters, 0, + /* If guard changes, make sure to update 'filters' initialization above. */ + if (!Py_Py3kWarningFlag && !Py_DivisionWarningFlag) { + PyList_SET_ITEM(filters, pos++, + create_filter(PyExc_DeprecationWarning, "ignore")); + } + PyList_SET_ITEM(filters, pos++, create_filter(PyExc_PendingDeprecationWarning, "ignore")); - PyList_SET_ITEM(filters, 1, create_filter(PyExc_ImportWarning, "ignore")); + PyList_SET_ITEM(filters, pos++, + create_filter(PyExc_ImportWarning, "ignore")); if (Py_BytesWarningFlag > 1) - bytes_action = "error"; + bytes_action = "error"; else if (Py_BytesWarningFlag) - bytes_action = "default"; + bytes_action = "default"; else - bytes_action = "ignore"; - PyList_SET_ITEM(filters, 2, create_filter(PyExc_BytesWarning, + bytes_action = "ignore"; + PyList_SET_ITEM(filters, pos++, create_filter(PyExc_BytesWarning, bytes_action)); - if (PyList_GET_ITEM(filters, 0) == NULL || - PyList_GET_ITEM(filters, 1) == NULL || - PyList_GET_ITEM(filters, 2) == NULL) { - Py_DECREF(filters); - return NULL; + for (x = 0; x < pos; x += 1) { + if (PyList_GET_ITEM(filters, x) == NULL) { + Py_DECREF(filters); + return NULL; + } } return filters; @@ -888,29 +882,29 @@ init_filters(void) PyMODINIT_FUNC _PyWarnings_Init(void) { - PyObject *m, *default_action; + PyObject *m; m = Py_InitModule3(MODULE_NAME, warnings_functions, warnings__doc__); if (m == NULL) - return; + return; _filters = init_filters(); if (_filters == NULL) - return; + return; Py_INCREF(_filters); if (PyModule_AddObject(m, "filters", _filters) < 0) - return; + return; _once_registry = PyDict_New(); if (_once_registry == NULL) - return; + return; Py_INCREF(_once_registry); if (PyModule_AddObject(m, "once_registry", _once_registry) < 0) - return; + return; - default_action = PyString_InternFromString("default"); - if (default_action == NULL) - return; - if (PyModule_AddObject(m, DEFAULT_ACTION_NAME, default_action) < 0) - return; + _default_action = PyString_FromString("default"); + if (_default_action == NULL) + return; + if (PyModule_AddObject(m, "default_action", _default_action) < 0) + return; } diff --git a/Python/ast.c b/Python/ast.c index 574a3d817c..88e47450b6 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -31,7 +31,7 @@ static asdl_seq *ast_for_exprlist(struct compiling *, const node *, expr_context_ty); static expr_ty ast_for_testlist(struct compiling *, const node *); static stmt_ty ast_for_classdef(struct compiling *, const node *, asdl_seq *); -static expr_ty ast_for_testlist_gexp(struct compiling *, const node *); +static expr_ty ast_for_testlist_comp(struct compiling *, const node *); /* Note different signature for ast_for_call */ static expr_ty ast_for_call(struct compiling *, const node *, expr_ty); @@ -44,6 +44,9 @@ static PyObject *parsestrplus(struct compiling *, const node *n); #define LINENO(n) ((n)->n_lineno) #endif +#define COMP_GENEXP 0 +#define COMP_SETCOMP 1 + static identifier new_identifier(const char* n, PyArena *arena) { PyObject* id = PyString_InternFromString(n); @@ -131,10 +134,17 @@ static int forbidden_check(struct compiling *c, const node *n, const char *x) { if (!strcmp(x, "None")) - return ast_error(n, "assignment to None"); - if (Py_Py3kWarningFlag && !(strcmp(x, "True") && strcmp(x, "False")) && - !ast_warn(c, n, "assignment to True or False is forbidden in 3.x")) - return 0; + return ast_error(n, "cannot assign to None"); + if (!strcmp(x, "__debug__")) + return ast_error(n, "cannot assign to __debug__"); + if (Py_Py3kWarningFlag) { + if (!(strcmp(x, "True") && strcmp(x, "False")) && + !ast_warn(c, n, "assignment to True or False is forbidden in 3.x")) + return 0; + if (!strcmp(x, "nonlocal") && + !ast_warn(c, n, "nonlocal is a keyword in 3.x")) + return 0; + } return 1; } @@ -190,7 +200,7 @@ num_stmts(const node *n) default: { char buf[128]; - sprintf(buf, "Non-statement found: %d %d\n", + sprintf(buf, "Non-statement found: %d %d", TYPE(n), NCH(n)); Py_FatalError(buf); } @@ -261,7 +271,7 @@ PyAST_FromNode(const node *n, PyCompilerFlags *flags, const char *filename, case eval_input: { expr_ty testlist_ast; - /* XXX Why not gen_for here? */ + /* XXX Why not comp_for here? */ testlist_ast = ast_for_testlist(&c, CHILD(n, 0)); if (!testlist_ast) goto error; @@ -395,10 +405,13 @@ set_context(struct compiling *c, expr_ty e, expr_context_ty ctx, const node *n) s = e->v.List.elts; break; case Tuple_kind: - if (asdl_seq_LEN(e->v.Tuple.elts) == 0) - return ast_error(n, "can't assign to ()"); - e->v.Tuple.ctx = ctx; - s = e->v.Tuple.elts; + if (asdl_seq_LEN(e->v.Tuple.elts)) { + e->v.Tuple.ctx = ctx; + s = e->v.Tuple.elts; + } + else { + expr_name = "()"; + } break; case Lambda_kind: expr_name = "lambda"; @@ -420,7 +433,14 @@ set_context(struct compiling *c, expr_ty e, expr_context_ty ctx, const node *n) case ListComp_kind: expr_name = "list comprehension"; break; + case SetComp_kind: + expr_name = "set comprehension"; + break; + case DictComp_kind: + expr_name = "dict comprehension"; + break; case Dict_kind: + case Set_kind: case Num_kind: case Str_kind: expr_name = "literal"; @@ -435,8 +455,8 @@ set_context(struct compiling *c, expr_ty e, expr_context_ty ctx, const node *n) expr_name = "conditional expression"; break; default: - PyErr_Format(PyExc_SystemError, - "unexpected expression in assignment %d (line %d)", + PyErr_Format(PyExc_SystemError, + "unexpected expression in assignment %d (line %d)", e->kind, e->lineno); return 0; } @@ -451,7 +471,7 @@ set_context(struct compiling *c, expr_ty e, expr_context_ty ctx, const node *n) } /* If the LHS is a list or tuple, we need to set the assignment - context for all the contained elements. + context for all the contained elements. */ if (s) { int i; @@ -563,7 +583,7 @@ seq_for_testlist(struct compiling *c, const node *n) int i; assert(TYPE(n) == testlist || TYPE(n) == listmaker || - TYPE(n) == testlist_gexp || + TYPE(n) == testlist_comp || TYPE(n) == testlist_safe || TYPE(n) == testlist1); @@ -668,10 +688,10 @@ ast_for_arguments(struct compiling *c, const node *n) } args = (n_args ? asdl_seq_new(n_args, c->c_arena) : NULL); if (!args && n_args) - return NULL; /* Don't need to goto error; no objects allocated */ + return NULL; defaults = (n_defaults ? asdl_seq_new(n_defaults, c->c_arena) : NULL); if (!defaults && n_defaults) - return NULL; /* Don't need to goto error; no objects allocated */ + return NULL; /* fpdef: NAME | '(' fplist ')' fplist: fpdef (',' fpdef)* [','] @@ -691,7 +711,7 @@ ast_for_arguments(struct compiling *c, const node *n) if (i + 1 < NCH(n) && TYPE(CHILD(n, i + 1)) == EQUAL) { expr_ty expression = ast_for_expr(c, CHILD(n, i + 2)); if (!expression) - goto error; + return NULL; assert(defaults != NULL); asdl_seq_SET(defaults, j++, expression); i += 2; @@ -702,11 +722,11 @@ ast_for_arguments(struct compiling *c, const node *n) def f((x, (y))): pass will just incur the tuple unpacking warning. */ if (parenthesized && !complex_args) { ast_error(n, "parenthesized arg with default"); - goto error; + return NULL; } - ast_error(n, + ast_error(n, "non-default argument follows default argument"); - goto error; + return NULL; } if (NCH(ch) == 3) { ch = CHILD(ch, 1); @@ -715,11 +735,11 @@ ast_for_arguments(struct compiling *c, const node *n) /* We have complex arguments, setup for unpacking. */ if (Py_Py3kWarningFlag && !ast_warn(c, ch, "tuple parameter unpacking has been removed in 3.x")) - goto error; + return NULL; complex_args = 1; asdl_seq_SET(args, k++, compiler_complex_args(c, ch)); if (!asdl_seq_GET(args, k-1)) - goto error; + return NULL; } else { /* def foo((x)): setup for checking NAME below. */ /* Loop because there can be many parens and tuple @@ -734,55 +754,50 @@ ast_for_arguments(struct compiling *c, const node *n) PyObject *id; expr_ty name; if (!forbidden_check(c, n, STR(CHILD(ch, 0)))) - goto error; + return NULL; id = NEW_IDENTIFIER(CHILD(ch, 0)); if (!id) - goto error; + return NULL; name = Name(id, Param, LINENO(ch), ch->n_col_offset, c->c_arena); if (!name) - goto error; + return NULL; asdl_seq_SET(args, k++, name); - + } i += 2; /* the name and the comma */ if (parenthesized && Py_Py3kWarningFlag && !ast_warn(c, ch, "parenthesized argument names " "are invalid in 3.x")) - goto error; + return NULL; break; } case STAR: if (!forbidden_check(c, CHILD(n, i+1), STR(CHILD(n, i+1)))) - goto error; + return NULL; vararg = NEW_IDENTIFIER(CHILD(n, i+1)); if (!vararg) - goto error; + return NULL; i += 3; break; case DOUBLESTAR: if (!forbidden_check(c, CHILD(n, i+1), STR(CHILD(n, i+1)))) - goto error; + return NULL; kwarg = NEW_IDENTIFIER(CHILD(n, i+1)); if (!kwarg) - goto error; + return NULL; i += 3; break; default: PyErr_Format(PyExc_SystemError, "unexpected node in varargslist: %d @ %d", TYPE(ch), i); - goto error; + return NULL; } } return arguments(args, vararg, kwarg, defaults, c->c_arena); - - error: - Py_XDECREF(vararg); - Py_XDECREF(kwarg); - return NULL; } static expr_ty @@ -823,15 +838,15 @@ ast_for_decorator(struct compiling *c, const node *n) /* decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE */ expr_ty d = NULL; expr_ty name_expr; - + REQ(n, decorator); REQ(CHILD(n, 0), AT); REQ(RCHILD(n, -1), NEWLINE); - + name_expr = ast_for_dotted_name(c, CHILD(n, 1)); if (!name_expr) return NULL; - + if (NCH(n) == 3) { /* No arguments */ d = name_expr; name_expr = NULL; @@ -859,17 +874,17 @@ ast_for_decorators(struct compiling *c, const node *n) asdl_seq* decorator_seq; expr_ty d; int i; - + REQ(n, decorators); decorator_seq = asdl_seq_new(NCH(n), c->c_arena); if (!decorator_seq) return NULL; - + for (i = 0; i < NCH(n); i++) { d = ast_for_decorator(c, CHILD(n, i)); - if (!d) - return NULL; - asdl_seq_SET(decorator_seq, i, d); + if (!d) + return NULL; + asdl_seq_SET(decorator_seq, i, d); } return decorator_seq; } @@ -961,7 +976,7 @@ ast_for_lambdef(struct compiling *c, const node *n) static expr_ty ast_for_ifexpr(struct compiling *c, const node *n) { - /* test: or_test 'if' or_test 'else' test */ + /* test: or_test 'if' or_test 'else' test */ expr_ty expression, body, orelse; assert(NCH(n) == 5); @@ -1052,7 +1067,7 @@ ast_for_listcomp(struct compiling *c, const node *n) list_if: 'if' test [list_iter] testlist_safe: test [(',' test)+ [',']] */ - expr_ty elt; + expr_ty elt, first; asdl_seq *listcomps; int i, n_fors; node *ch; @@ -1078,9 +1093,9 @@ ast_for_listcomp(struct compiling *c, const node *n) asdl_seq *t; expr_ty expression; node *for_ch; - + REQ(ch, list_for); - + for_ch = CHILD(ch, 1); t = ast_for_exprlist(c, for_ch, Store); if (!t) @@ -1088,15 +1103,15 @@ ast_for_listcomp(struct compiling *c, const node *n) expression = ast_for_testlist(c, CHILD(ch, 3)); if (!expression) return NULL; - + /* Check the # of children rather than the length of t, since [x for x, in ... ] has 1 element in t, but still requires a Tuple. */ + first = (expr_ty)asdl_seq_GET(t, 0); if (NCH(for_ch) == 1) - lc = comprehension((expr_ty)asdl_seq_GET(t, 0), expression, NULL, - c->c_arena); + lc = comprehension(first, expression, NULL, c->c_arena); else - lc = comprehension(Tuple(t, Store, LINENO(ch), ch->n_col_offset, + lc = comprehension(Tuple(t, Store, first->lineno, first->col_offset, c->c_arena), expression, NULL, c->c_arena); if (!lc) @@ -1120,11 +1135,11 @@ ast_for_listcomp(struct compiling *c, const node *n) REQ(ch, list_iter); ch = CHILD(ch, 0); REQ(ch, list_if); - + list_for_expr = ast_for_expr(c, CHILD(ch, 1)); if (!list_for_expr) return NULL; - + asdl_seq_SET(ifs, j, list_for_expr); if (NCH(ch) == 3) ch = CHILD(ch, 2); @@ -1140,60 +1155,60 @@ ast_for_listcomp(struct compiling *c, const node *n) return ListComp(elt, listcomps, LINENO(n), n->n_col_offset, c->c_arena); } -/* Count the number of 'for' loops in a generator expression. +/* + Count the number of 'for' loops in a comprehension. - Helper for ast_for_genexp(). + Helper for ast_for_comprehension(). */ static int -count_gen_fors(struct compiling *c, const node *n) +count_comp_fors(struct compiling *c, const node *n) { int n_fors = 0; - node *ch = CHILD(n, 1); - count_gen_for: + count_comp_for: n_fors++; - REQ(ch, gen_for); - if (NCH(ch) == 5) - ch = CHILD(ch, 4); + REQ(n, comp_for); + if (NCH(n) == 5) + n = CHILD(n, 4); else return n_fors; - count_gen_iter: - REQ(ch, gen_iter); - ch = CHILD(ch, 0); - if (TYPE(ch) == gen_for) - goto count_gen_for; - else if (TYPE(ch) == gen_if) { - if (NCH(ch) == 3) { - ch = CHILD(ch, 2); - goto count_gen_iter; + count_comp_iter: + REQ(n, comp_iter); + n = CHILD(n, 0); + if (TYPE(n) == comp_for) + goto count_comp_for; + else if (TYPE(n) == comp_if) { + if (NCH(n) == 3) { + n = CHILD(n, 2); + goto count_comp_iter; } else return n_fors; } - + /* Should never be reached */ PyErr_SetString(PyExc_SystemError, - "logic error in count_gen_fors"); + "logic error in count_comp_fors"); return -1; } -/* Count the number of 'if' statements in a generator expression. +/* Count the number of 'if' statements in a comprehension. - Helper for ast_for_genexp(). + Helper for ast_for_comprehension(). */ static int -count_gen_ifs(struct compiling *c, const node *n) +count_comp_ifs(struct compiling *c, const node *n) { int n_ifs = 0; while (1) { - REQ(n, gen_iter); - if (TYPE(CHILD(n, 0)) == gen_for) + REQ(n, comp_iter); + if (TYPE(CHILD(n, 0)) == comp_for) return n_ifs; n = CHILD(n, 0); - REQ(n, gen_if); + REQ(n, comp_if); n_ifs++; if (NCH(n) == 2) return n_ifs; @@ -1201,68 +1216,54 @@ count_gen_ifs(struct compiling *c, const node *n) } } -/* TODO(jhylton): Combine with list comprehension code? */ -static expr_ty -ast_for_genexp(struct compiling *c, const node *n) +static asdl_seq * +ast_for_comprehension(struct compiling *c, const node *n) { - /* testlist_gexp: test ( gen_for | (',' test)* [','] ) - argument: [test '='] test [gen_for] # Really [keyword '='] test */ - expr_ty elt; - asdl_seq *genexps; int i, n_fors; - node *ch; - - assert(TYPE(n) == (testlist_gexp) || TYPE(n) == (argument)); - assert(NCH(n) > 1); - - elt = ast_for_expr(c, CHILD(n, 0)); - if (!elt) - return NULL; - - n_fors = count_gen_fors(c, n); + asdl_seq *comps; + + n_fors = count_comp_fors(c, n); if (n_fors == -1) return NULL; - genexps = asdl_seq_new(n_fors, c->c_arena); - if (!genexps) + comps = asdl_seq_new(n_fors, c->c_arena); + if (!comps) return NULL; - ch = CHILD(n, 1); for (i = 0; i < n_fors; i++) { - comprehension_ty ge; + comprehension_ty comp; asdl_seq *t; - expr_ty expression; + expr_ty expression, first; node *for_ch; - - REQ(ch, gen_for); - - for_ch = CHILD(ch, 1); + + REQ(n, comp_for); + + for_ch = CHILD(n, 1); t = ast_for_exprlist(c, for_ch, Store); if (!t) return NULL; - expression = ast_for_expr(c, CHILD(ch, 3)); + expression = ast_for_expr(c, CHILD(n, 3)); if (!expression) return NULL; /* Check the # of children rather than the length of t, since (x for x, in ...) has 1 element in t, but still requires a Tuple. */ + first = (expr_ty)asdl_seq_GET(t, 0); if (NCH(for_ch) == 1) - ge = comprehension((expr_ty)asdl_seq_GET(t, 0), expression, - NULL, c->c_arena); + comp = comprehension(first, expression, NULL, c->c_arena); else - ge = comprehension(Tuple(t, Store, LINENO(ch), ch->n_col_offset, + comp = comprehension(Tuple(t, Store, first->lineno, first->col_offset, c->c_arena), expression, NULL, c->c_arena); - - if (!ge) + if (!comp) return NULL; - if (NCH(ch) == 5) { + if (NCH(n) == 5) { int j, n_ifs; asdl_seq *ifs; - - ch = CHILD(ch, 4); - n_ifs = count_gen_ifs(c, ch); + + n = CHILD(n, 4); + n_ifs = count_comp_ifs(c, n); if (n_ifs == -1) return NULL; @@ -1271,36 +1272,98 @@ ast_for_genexp(struct compiling *c, const node *n) return NULL; for (j = 0; j < n_ifs; j++) { - REQ(ch, gen_iter); - ch = CHILD(ch, 0); - REQ(ch, gen_if); - - expression = ast_for_expr(c, CHILD(ch, 1)); + REQ(n, comp_iter); + n = CHILD(n, 0); + REQ(n, comp_if); + + expression = ast_for_expr(c, CHILD(n, 1)); if (!expression) return NULL; asdl_seq_SET(ifs, j, expression); - if (NCH(ch) == 3) - ch = CHILD(ch, 2); + if (NCH(n) == 3) + n = CHILD(n, 2); } - /* on exit, must guarantee that ch is a gen_for */ - if (TYPE(ch) == gen_iter) - ch = CHILD(ch, 0); - ge->ifs = ifs; + /* on exit, must guarantee that n is a comp_for */ + if (TYPE(n) == comp_iter) + n = CHILD(n, 0); + comp->ifs = ifs; } - asdl_seq_SET(genexps, i, ge); + asdl_seq_SET(comps, i, comp); } - - return GeneratorExp(elt, genexps, LINENO(n), n->n_col_offset, c->c_arena); + return comps; +} + +static expr_ty +ast_for_itercomp(struct compiling *c, const node *n, int type) +{ + expr_ty elt; + asdl_seq *comps; + + assert(NCH(n) > 1); + + elt = ast_for_expr(c, CHILD(n, 0)); + if (!elt) + return NULL; + + comps = ast_for_comprehension(c, CHILD(n, 1)); + if (!comps) + return NULL; + + if (type == COMP_GENEXP) + return GeneratorExp(elt, comps, LINENO(n), n->n_col_offset, c->c_arena); + else if (type == COMP_SETCOMP) + return SetComp(elt, comps, LINENO(n), n->n_col_offset, c->c_arena); + else + /* Should never happen */ + return NULL; +} + +static expr_ty +ast_for_dictcomp(struct compiling *c, const node *n) +{ + expr_ty key, value; + asdl_seq *comps; + + assert(NCH(n) > 3); + REQ(CHILD(n, 1), COLON); + + key = ast_for_expr(c, CHILD(n, 0)); + if (!key) + return NULL; + + value = ast_for_expr(c, CHILD(n, 2)); + if (!value) + return NULL; + + comps = ast_for_comprehension(c, CHILD(n, 3)); + if (!comps) + return NULL; + + return DictComp(key, value, comps, LINENO(n), n->n_col_offset, c->c_arena); +} + +static expr_ty +ast_for_genexp(struct compiling *c, const node *n) +{ + assert(TYPE(n) == (testlist_comp) || TYPE(n) == (argument)); + return ast_for_itercomp(c, n, COMP_GENEXP); +} + +static expr_ty +ast_for_setcomp(struct compiling *c, const node *n) +{ + assert(TYPE(n) == (dictorsetmaker)); + return ast_for_itercomp(c, n, COMP_SETCOMP); } static expr_ty ast_for_atom(struct compiling *c, const node *n) { - /* atom: '(' [yield_expr|testlist_gexp] ')' | '[' [listmaker] ']' + /* atom: '(' [yield_expr|testlist_comp] ')' | '[' [listmaker] ']' | '{' [dictmaker] '}' | '`' testlist '`' | NAME | NUMBER | STRING+ */ node *ch = CHILD(n, 0); - + switch (TYPE(ch)) { case NAME: { /* All names start in Load context, but may later be @@ -1348,23 +1411,20 @@ ast_for_atom(struct compiling *c, const node *n) } case LPAR: /* some parenthesized expressions */ ch = CHILD(n, 1); - + if (TYPE(ch) == RPAR) return Tuple(NULL, Load, LINENO(n), n->n_col_offset, c->c_arena); - + if (TYPE(ch) == yield_expr) return ast_for_expr(c, ch); - - if ((NCH(ch) > 1) && (TYPE(CHILD(ch, 1)) == gen_for)) - return ast_for_genexp(c, ch); - - return ast_for_testlist_gexp(c, ch); + + return ast_for_testlist_comp(c, ch); case LSQB: /* list (or list comprehension) */ ch = CHILD(n, 1); - + if (TYPE(ch) == RSQB) return List(NULL, Load, LINENO(n), n->n_col_offset, c->c_arena); - + REQ(ch, listmaker); if (NCH(ch) == 1 || TYPE(CHILD(ch, 1)) == COMMA) { asdl_seq *elts = seq_for_testlist(c, ch); @@ -1376,36 +1436,65 @@ ast_for_atom(struct compiling *c, const node *n) else return ast_for_listcomp(c, ch); case LBRACE: { - /* dictmaker: test ':' test (',' test ':' test)* [','] */ + /* dictorsetmaker: + * (test ':' test (comp_for | (',' test ':' test)* [','])) | + * (test (comp_for | (',' test)* [','])) + */ int i, size; asdl_seq *keys, *values; - + ch = CHILD(n, 1); - size = (NCH(ch) + 1) / 4; /* +1 in case no trailing comma */ - keys = asdl_seq_new(size, c->c_arena); - if (!keys) - return NULL; - - values = asdl_seq_new(size, c->c_arena); - if (!values) - return NULL; - - for (i = 0; i < NCH(ch); i += 4) { - expr_ty expression; - - expression = ast_for_expr(c, CHILD(ch, i)); - if (!expression) + if (TYPE(ch) == RBRACE) { + /* it's an empty dict */ + return Dict(NULL, NULL, LINENO(n), n->n_col_offset, c->c_arena); + } else if (NCH(ch) == 1 || TYPE(CHILD(ch, 1)) == COMMA) { + /* it's a simple set */ + asdl_seq *elts; + size = (NCH(ch) + 1) / 2; /* +1 in case no trailing comma */ + elts = asdl_seq_new(size, c->c_arena); + if (!elts) + return NULL; + for (i = 0; i < NCH(ch); i += 2) { + expr_ty expression; + expression = ast_for_expr(c, CHILD(ch, i)); + if (!expression) + return NULL; + asdl_seq_SET(elts, i / 2, expression); + } + return Set(elts, LINENO(n), n->n_col_offset, c->c_arena); + } else if (TYPE(CHILD(ch, 1)) == comp_for) { + /* it's a set comprehension */ + return ast_for_setcomp(c, ch); + } else if (NCH(ch) > 3 && TYPE(CHILD(ch, 3)) == comp_for) { + return ast_for_dictcomp(c, ch); + } else { + /* it's a dict */ + size = (NCH(ch) + 1) / 4; /* +1 in case no trailing comma */ + keys = asdl_seq_new(size, c->c_arena); + if (!keys) return NULL; - asdl_seq_SET(keys, i / 4, expression); - - expression = ast_for_expr(c, CHILD(ch, i + 2)); - if (!expression) + values = asdl_seq_new(size, c->c_arena); + if (!values) return NULL; - asdl_seq_SET(values, i / 4, expression); + for (i = 0; i < NCH(ch); i += 4) { + expr_ty expression; + + expression = ast_for_expr(c, CHILD(ch, i)); + if (!expression) + return NULL; + + asdl_seq_SET(keys, i / 4, expression); + + expression = ast_for_expr(c, CHILD(ch, i + 2)); + if (!expression) + return NULL; + + asdl_seq_SET(values, i / 4, expression); + } + return Dict(keys, values, LINENO(n), n->n_col_offset, c->c_arena); } - return Dict(keys, values, LINENO(n), n->n_col_offset, c->c_arena); } case BACKQUOTE: { /* repr */ expr_ty expression; @@ -1443,10 +1532,10 @@ ast_for_slice(struct compiling *c, const node *n) if (NCH(n) == 1 && TYPE(ch) == test) { /* 'step' variable hold no significance in terms of being used over other vars */ - step = ast_for_expr(c, ch); + step = ast_for_expr(c, ch); if (!step) return NULL; - + return Index(step, c->c_arena); } @@ -1480,10 +1569,17 @@ ast_for_slice(struct compiling *c, const node *n) ch = CHILD(n, NCH(n) - 1); if (TYPE(ch) == sliceop) { if (NCH(ch) == 1) { - /* No expression, so step is None */ + /* + This is an extended slice (ie "x[::]") with no expression in the + step field. We set this literally to "None" in order to + disambiguate it from x[:]. (The interpreter might have to call + __getslice__ for x[:], but it must call __getitem__ for x[::].) + */ + identifier none = new_identifier("None", c->c_arena); + if (!none) + return NULL; ch = CHILD(ch, 0); - step = Name(new_identifier("None", c->c_arena), Load, - LINENO(ch), ch->n_col_offset, c->c_arena); + step = Name(none, Load, LINENO(ch), ch->n_col_offset, c->c_arena); if (!step) return NULL; } else { @@ -1503,7 +1599,7 @@ static expr_ty ast_for_binop(struct compiling *c, const node *n) { /* Must account for a sequence of expressions. - How should A op B op C by represented? + How should A op B op C by represented? BinOp(BinOp(A, op, B), op, C). */ @@ -1541,10 +1637,10 @@ ast_for_binop(struct compiling *c, const node *n) if (!tmp) return NULL; - tmp_result = BinOp(result, newoperator, tmp, + tmp_result = BinOp(result, newoperator, tmp, LINENO(next_oper), next_oper->n_col_offset, c->c_arena); - if (!tmp_result) + if (!tmp_result) return NULL; result = tmp_result; } @@ -1554,7 +1650,7 @@ ast_for_binop(struct compiling *c, const node *n) static expr_ty ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr) { - /* trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME + /* trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME subscriptlist: subscript (',' subscript)* [','] subscript: '.' '.' '.' | test | [test] ':' [test] [sliceop] */ @@ -1585,7 +1681,7 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr) c->c_arena); } else { - /* The grammar is ambiguous here. The ambiguity is resolved + /* The grammar is ambiguous here. The ambiguity is resolved by treating the sequence as a tuple literal if there are no slice features. */ @@ -1722,7 +1818,7 @@ ast_for_expr(struct compiling *c, const node *n) { /* handle the full range of simple expressions test: or_test ['if' or_test 'else' test] | lambdef - or_test: and_test ('or' and_test)* + or_test: and_test ('or' and_test)* and_test: not_test ('and' not_test)* not_test: 'not' not_test | comparison comparison: expr (comp_op expr)* @@ -1739,7 +1835,7 @@ ast_for_expr(struct compiling *c, const node *n) to explicitly allow: [ x for x in lambda: 0, lambda: 1 ] (which would be ambiguous without these extra rules) - + old_test: or_test | old_lambdef old_lambdef: 'lambda' [vararglist] ':' old_test @@ -1819,7 +1915,7 @@ ast_for_expr(struct compiling *c, const node *n) if (!expression) { return NULL; } - + asdl_seq_SET(ops, i / 2, newoperator); asdl_seq_SET(cmps, i / 2, expression); } @@ -1827,7 +1923,7 @@ ast_for_expr(struct compiling *c, const node *n) if (!expression) { return NULL; } - + return Compare(expression, ops, cmps, LINENO(n), n->n_col_offset, c->c_arena); } @@ -1879,7 +1975,7 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func) /* arglist: (argument ',')* (argument [',']| '*' test [',' '**' test] | '**' test) - argument: [test '='] test [gen_for] # Really [keyword '='] test + argument: [test '='] test [comp_for] # Really [keyword '='] test */ int i, nargs, nkeywords, ngens; @@ -1897,7 +1993,7 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func) if (TYPE(ch) == argument) { if (NCH(ch) == 1) nargs++; - else if (TYPE(CHILD(ch, 1)) == gen_for) + else if (TYPE(CHILD(ch, 1)) == comp_for) ngens++; else nkeywords++; @@ -1941,8 +2037,8 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func) if (!e) return NULL; asdl_seq_SET(args, nargs++, e); - } - else if (TYPE(CHILD(ch, 1)) == gen_for) { + } + else if (TYPE(CHILD(ch, 1)) == comp_for) { e = ast_for_genexp(c, ch); if (!e) return NULL; @@ -1954,7 +2050,7 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func) int k; char *tmp; - /* CHILD(ch, 0) is test, but must be an identifier? */ + /* CHILD(ch, 0) is test, but must be an identifier? */ e = ast_for_expr(c, CHILD(ch, 0)); if (!e) return NULL; @@ -2012,14 +2108,14 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func) static expr_ty ast_for_testlist(struct compiling *c, const node* n) { - /* testlist_gexp: test (',' test)* [','] */ + /* testlist_comp: test (',' test)* [','] */ /* testlist: test (',' test)* [','] */ /* testlist_safe: test (',' test)+ [','] */ /* testlist1: test (',' test)* */ assert(NCH(n) > 0); - if (TYPE(n) == testlist_gexp) { + if (TYPE(n) == testlist_comp) { if (NCH(n) > 1) - assert(TYPE(CHILD(n, 1)) != gen_for); + assert(TYPE(CHILD(n, 1)) != comp_for); } else { assert(TYPE(n) == testlist || @@ -2037,12 +2133,12 @@ ast_for_testlist(struct compiling *c, const node* n) } static expr_ty -ast_for_testlist_gexp(struct compiling *c, const node* n) +ast_for_testlist_comp(struct compiling *c, const node* n) { - /* testlist_gexp: test ( gen_for | (',' test)* [','] ) */ - /* argument: test [ gen_for ] */ - assert(TYPE(n) == testlist_gexp || TYPE(n) == argument); - if (NCH(n) > 1 && TYPE(CHILD(n, 1)) == gen_for) + /* testlist_comp: test ( comp_for | (',' test)* [','] ) */ + /* argument: test [ comp_for ] */ + assert(TYPE(n) == testlist_comp || TYPE(n) == argument); + if (NCH(n) > 1 && TYPE(CHILD(n, 1)) == comp_for) return ast_for_genexp(c, n); return ast_for_testlist(c, n); } @@ -2073,7 +2169,7 @@ static stmt_ty ast_for_expr_stmt(struct compiling *c, const node *n) { REQ(n, expr_stmt); - /* expr_stmt: testlist (augassign (yield_expr|testlist) + /* expr_stmt: testlist (augassign (yield_expr|testlist) | ('=' (yield_expr|testlist))*) testlist: test (',' test)* [','] augassign: '+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' @@ -2096,33 +2192,21 @@ ast_for_expr_stmt(struct compiling *c, const node *n) expr1 = ast_for_testlist(c, ch); if (!expr1) return NULL; - /* TODO(nas): Remove duplicated error checks (set_context does it) */ + if(!set_context(c, expr1, Store, ch)) + return NULL; + /* set_context checks that most expressions are not the left side. + Augmented assignments can only have a name, a subscript, or an + attribute on the left, though, so we have to explicitly check for + those. */ switch (expr1->kind) { - case GeneratorExp_kind: - ast_error(ch, "augmented assignment to generator " - "expression not possible"); - return NULL; - case Yield_kind: - ast_error(ch, "augmented assignment to yield " - "expression not possible"); - return NULL; - case Name_kind: { - const char *var_name = PyBytes_AS_STRING(expr1->v.Name.id); - if ((var_name[0] == 'N' || var_name[0] == 'T' || var_name[0] == 'F') && - !forbidden_check(c, ch, var_name)) - return NULL; - break; - } + case Name_kind: case Attribute_kind: case Subscript_kind: break; default: - ast_error(ch, "illegal expression for augmented " - "assignment"); + ast_error(ch, "illegal expression for augmented assignment"); return NULL; } - if(!set_context(c, expr1, Store, ch)) - return NULL; ch = CHILD(n, 2); if (TYPE(ch) == testlist) @@ -2158,11 +2242,10 @@ ast_for_expr_stmt(struct compiling *c, const node *n) return NULL; } e = ast_for_testlist(c, ch); - - /* set context to assign */ - if (!e) + if (!e) return NULL; + /* set context to assign */ if (!set_context(c, e, Store, CHILD(n, i))) return NULL; @@ -2187,9 +2270,9 @@ ast_for_print_stmt(struct compiling *c, const node *n) | '>>' test [ (',' test)+ [','] ] ) */ expr_ty dest = NULL, expression; - asdl_seq *seq; + asdl_seq *seq = NULL; bool nl; - int i, j, start = 1; + int i, j, values_count, start = 1; REQ(n, print_stmt); if (NCH(n) >= 2 && TYPE(CHILD(n, 1)) == RIGHTSHIFT) { @@ -2198,14 +2281,17 @@ ast_for_print_stmt(struct compiling *c, const node *n) return NULL; start = 4; } - seq = asdl_seq_new((NCH(n) + 1 - start) / 2, c->c_arena); - if (!seq) - return NULL; - for (i = start, j = 0; i < NCH(n); i += 2, ++j) { - expression = ast_for_expr(c, CHILD(n, i)); - if (!expression) + values_count = (NCH(n) + 1 - start) / 2; + if (values_count) { + seq = asdl_seq_new(values_count, c->c_arena); + if (!seq) return NULL; - asdl_seq_SET(seq, j, expression); + for (i = start, j = 0; i < NCH(n); i += 2, ++j) { + expression = ast_for_expr(c, CHILD(n, i)); + if (!expression) + return NULL; + asdl_seq_SET(seq, j, expression); + } } nl = (TYPE(CHILD(n, NCH(n) - 1)) == COMMA) ? false : true; return Print(dest, seq, nl, LINENO(n), n->n_col_offset, c->c_arena); @@ -2238,7 +2324,7 @@ static stmt_ty ast_for_del_stmt(struct compiling *c, const node *n) { asdl_seq *expr_list; - + /* del_stmt: 'del' exprlist */ REQ(n, del_stmt); @@ -2322,7 +2408,7 @@ ast_for_flow_stmt(struct compiling *c, const node *n) expr3 = ast_for_expr(c, CHILD(ch, 5)); if (!expr3) return NULL; - + return Raise(expr1, expr2, expr3, LINENO(n), n->n_col_offset, c->c_arena); } @@ -2337,7 +2423,7 @@ ast_for_flow_stmt(struct compiling *c, const node *n) } static alias_ty -alias_for_import_name(struct compiling *c, const node *n) +alias_for_import_name(struct compiling *c, const node *n, int store) { /* import_as_name: NAME ['as' NAME] @@ -2348,28 +2434,40 @@ alias_for_import_name(struct compiling *c, const node *n) loop: switch (TYPE(n)) { - case import_as_name: + case import_as_name: { + node *name_node = CHILD(n, 0); str = NULL; if (NCH(n) == 3) { - str = NEW_IDENTIFIER(CHILD(n, 2)); + node *str_node = CHILD(n, 2); + if (store && !forbidden_check(c, str_node, STR(str_node))) + return NULL; + str = NEW_IDENTIFIER(str_node); if (!str) return NULL; } - name = NEW_IDENTIFIER(CHILD(n, 0)); + else { + if (!forbidden_check(c, name_node, STR(name_node))) + return NULL; + } + name = NEW_IDENTIFIER(name_node); if (!name) return NULL; return alias(name, str, c->c_arena); + } case dotted_as_name: if (NCH(n) == 1) { n = CHILD(n, 0); goto loop; } else { - alias_ty a = alias_for_import_name(c, CHILD(n, 0)); + node *asname_node = CHILD(n, 2); + alias_ty a = alias_for_import_name(c, CHILD(n, 0), 0); if (!a) return NULL; assert(!a->asname); - a->asname = NEW_IDENTIFIER(CHILD(n, 2)); + if (!forbidden_check(c, asname_node, STR(asname_node))) + return NULL; + a->asname = NEW_IDENTIFIER(asname_node); if (!a->asname) return NULL; return a; @@ -2377,7 +2475,10 @@ alias_for_import_name(struct compiling *c, const node *n) break; case dotted_name: if (NCH(n) == 1) { - name = NEW_IDENTIFIER(CHILD(n, 0)); + node *name_node = CHILD(n, 0); + if (store && !forbidden_check(c, name_node, STR(name_node))) + return NULL; + name = NEW_IDENTIFIER(name_node); if (!name) return NULL; return alias(name, NULL, c->c_arena); @@ -2451,7 +2552,7 @@ ast_for_import_stmt(struct compiling *c, const node *n) if (!aliases) return NULL; for (i = 0; i < NCH(n); i += 2) { - alias_ty import_alias = alias_for_import_name(c, CHILD(n, i)); + alias_ty import_alias = alias_for_import_name(c, CHILD(n, i), 1); if (!import_alias) return NULL; asdl_seq_SET(aliases, i / 2, import_alias); @@ -2462,13 +2563,15 @@ ast_for_import_stmt(struct compiling *c, const node *n) int n_children; int idx, ndots = 0; alias_ty mod = NULL; - identifier modname; - + identifier modname = NULL; + /* Count the number of dots (for relative imports) and check for the optional module name */ for (idx = 1; idx < NCH(n); idx++) { if (TYPE(CHILD(n, idx)) == dotted_name) { - mod = alias_for_import_name(c, CHILD(n, idx)); + mod = alias_for_import_name(c, CHILD(n, idx), 0); + if (!mod) + return NULL; idx++; break; } else if (TYPE(CHILD(n, idx)) != DOT) { @@ -2509,14 +2612,14 @@ ast_for_import_stmt(struct compiling *c, const node *n) /* handle "from ... import *" special b/c there's no children */ if (TYPE(n) == STAR) { - alias_ty import_alias = alias_for_import_name(c, n); + alias_ty import_alias = alias_for_import_name(c, n, 1); if (!import_alias) return NULL; asdl_seq_SET(aliases, 0, import_alias); } else { for (i = 0; i < NCH(n); i += 2) { - alias_ty import_alias = alias_for_import_name(c, CHILD(n, i)); + alias_ty import_alias = alias_for_import_name(c, CHILD(n, i), 1); if (!import_alias) return NULL; asdl_seq_SET(aliases, i / 2, import_alias); @@ -2524,8 +2627,6 @@ ast_for_import_stmt(struct compiling *c, const node *n) } if (mod != NULL) modname = mod->name; - else - modname = new_identifier("", c->c_arena); return ImportFrom(modname, aliases, ndots, lineno, col_offset, c->c_arena); } @@ -2609,7 +2710,7 @@ ast_for_assert_stmt(struct compiling *c, const node *n) expr2 = ast_for_expr(c, CHILD(n, 3)); if (!expr2) return NULL; - + return Assert(expr1, expr2, LINENO(n), n->n_col_offset, c->c_arena); } PyErr_Format(PyExc_SystemError, @@ -2636,7 +2737,7 @@ ast_for_suite(struct compiling *c, const node *n) if (TYPE(CHILD(n, 0)) == simple_stmt) { n = CHILD(n, 0); /* simple_stmt always ends with a NEWLINE, - and may have a trailing SEMI + and may have a trailing SEMI */ end = NCH(n) - 1; if (TYPE(CHILD(n, end - 1)) == SEMI) @@ -2701,10 +2802,10 @@ ast_for_if_stmt(struct compiling *c, const node *n) expression = ast_for_expr(c, CHILD(n, 1)); if (!expression) return NULL; - suite_seq = ast_for_suite(c, CHILD(n, 3)); + suite_seq = ast_for_suite(c, CHILD(n, 3)); if (!suite_seq) return NULL; - + return If(expression, suite_seq, NULL, LINENO(n), n->n_col_offset, c->c_arena); } @@ -2762,8 +2863,8 @@ ast_for_if_stmt(struct compiling *c, const node *n) if (!suite_seq2) return NULL; - asdl_seq_SET(orelse, 0, - If(expression, suite_seq, suite_seq2, + asdl_seq_SET(orelse, 0, + If(expression, suite_seq, suite_seq2, LINENO(CHILD(n, NCH(n) - 6)), CHILD(n, NCH(n) - 6)->n_col_offset, c->c_arena)); @@ -2784,7 +2885,7 @@ ast_for_if_stmt(struct compiling *c, const node *n) return NULL; asdl_seq_SET(newobj, 0, - If(expression, suite_seq, orelse, + If(expression, suite_seq, orelse, LINENO(CHILD(n, off)), CHILD(n, off)->n_col_offset, c->c_arena)); orelse = newobj; @@ -2852,7 +2953,7 @@ ast_for_for_stmt(struct compiling *c, const node *n) { asdl_seq *_target, *seq = NULL, *suite_seq; expr_ty expression; - expr_ty target; + expr_ty target, first; const node *node_target; /* for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] */ REQ(n, for_stmt); @@ -2869,10 +2970,11 @@ ast_for_for_stmt(struct compiling *c, const node *n) return NULL; /* Check the # of children rather than the length of _target, since for x, in ... has 1 element in _target, but still requires a Tuple. */ + first = (expr_ty)asdl_seq_GET(_target, 0); if (NCH(node_target) == 1) - target = (expr_ty)asdl_seq_GET(_target, 0); + target = first; else - target = Tuple(_target, Store, LINENO(n), n->n_col_offset, c->c_arena); + target = Tuple(_target, Store, first->lineno, first->col_offset, c->c_arena); expression = ast_for_testlist(c, CHILD(n, 3)); if (!expression) @@ -2982,7 +3084,7 @@ ast_for_try_stmt(struct compiling *c, const node *n) ast_error(n, "malformed 'try' statement"); return NULL; } - + if (n_except > 0) { int i; stmt_ty except_st; @@ -3017,27 +3119,18 @@ ast_for_try_stmt(struct compiling *c, const node *n) return TryFinally(body, finally, LINENO(n), n->n_col_offset, c->c_arena); } -static expr_ty -ast_for_with_var(struct compiling *c, const node *n) -{ - REQ(n, with_var); - return ast_for_expr(c, CHILD(n, 1)); -} - -/* with_stmt: 'with' test [ with_var ] ':' suite */ +/* with_item: test ['as' expr] */ static stmt_ty -ast_for_with_stmt(struct compiling *c, const node *n) +ast_for_with_item(struct compiling *c, const node *n, asdl_seq *content) { expr_ty context_expr, optional_vars = NULL; - int suite_index = 3; /* skip 'with', test, and ':' */ - asdl_seq *suite_seq; - assert(TYPE(n) == with_stmt); - context_expr = ast_for_expr(c, CHILD(n, 1)); + REQ(n, with_item); + context_expr = ast_for_expr(c, CHILD(n, 0)); if (!context_expr) return NULL; - if (TYPE(CHILD(n, 2)) == with_var) { - optional_vars = ast_for_with_var(c, CHILD(n, 2)); + if (NCH(n) == 3) { + optional_vars = ast_for_expr(c, CHILD(n, 2)); if (!optional_vars) { return NULL; @@ -3045,15 +3138,45 @@ ast_for_with_stmt(struct compiling *c, const node *n) if (!set_context(c, optional_vars, Store, n)) { return NULL; } - suite_index = 4; } - suite_seq = ast_for_suite(c, CHILD(n, suite_index)); - if (!suite_seq) { + return With(context_expr, optional_vars, content, LINENO(n), + n->n_col_offset, c->c_arena); +} + +/* with_stmt: 'with' with_item (',' with_item)* ':' suite */ +static stmt_ty +ast_for_with_stmt(struct compiling *c, const node *n) +{ + int i; + stmt_ty ret; + asdl_seq *inner; + + REQ(n, with_stmt); + + /* process the with items inside-out */ + i = NCH(n) - 1; + /* the suite of the innermost with item is the suite of the with stmt */ + inner = ast_for_suite(c, CHILD(n, i)); + if (!inner) return NULL; + + for (;;) { + i -= 2; + ret = ast_for_with_item(c, CHILD(n, i), inner); + if (!ret) + return NULL; + /* was this the last item? */ + if (i == 1) + break; + /* if not, wrap the result so far in a new sequence */ + inner = asdl_seq_new(1, c->c_arena); + if (!inner) + return NULL; + asdl_seq_SET(inner, 0, ret); } - return With(context_expr, optional_vars, suite_seq, LINENO(n), - n->n_col_offset, c->c_arena); + + return ret; } static stmt_ty @@ -3062,7 +3185,7 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq) /* classdef: 'class' NAME ['(' testlist ')'] ':' suite */ PyObject *classname; asdl_seq *bases, *s; - + REQ(n, classdef); if (!forbidden_check(c, n, STR(CHILD(n, 1)))) @@ -3117,7 +3240,6 @@ ast_for_stmt(struct compiling *c, const node *n) n = CHILD(n, 0); } if (TYPE(n) == small_stmt) { - REQ(n, small_stmt); n = CHILD(n, 0); /* small_stmt: expr_stmt | print_stmt | del_stmt | pass_stmt | flow_stmt | import_stmt | global_stmt | exec_stmt @@ -3210,17 +3332,17 @@ parsenumber(struct compiling *c, const char *s) #ifndef WITHOUT_COMPLEX if (imflag) { complex.real = 0.; - PyFPE_START_PROTECT("atof", return 0) - complex.imag = PyOS_ascii_atof(s); - PyFPE_END_PROTECT(complex) + complex.imag = PyOS_string_to_double(s, (char **)&end, NULL); + if (complex.imag == -1.0 && PyErr_Occurred()) + return NULL; return PyComplex_FromCComplex(complex); } else #endif { - PyFPE_START_PROTECT("atof", return 0) - dx = PyOS_ascii_atof(s); - PyFPE_END_PROTECT(dx) + dx = PyOS_string_to_double(s, NULL, NULL); + if (dx == -1.0 && PyErr_Occurred()) + return NULL; return PyFloat_FromDouble(dx); } } @@ -3251,17 +3373,12 @@ decode_utf8(struct compiling *c, const char **sPtr, const char *end, char* encod static PyObject * decode_unicode(struct compiling *c, const char *s, size_t len, int rawmode, const char *encoding) { - PyObject *v, *u; + PyObject *v; + PyObject *u = NULL; char *buf; char *p; const char *end; - if (encoding == NULL) { - buf = (char *)s; - u = NULL; - } else if (strcmp(encoding, "iso-8859-1") == 0) { - buf = (char *)s; - u = NULL; - } else { + if (encoding != NULL && strcmp(encoding, "iso-8859-1")) { /* check for integer overflow */ if (len > PY_SIZE_MAX / 6) return NULL; @@ -3351,7 +3468,7 @@ parsestr(struct compiling *c, const char *s) s++; len = strlen(s); if (len > INT_MAX) { - PyErr_SetString(PyExc_OverflowError, + PyErr_SetString(PyExc_OverflowError, "string to parse is too long"); return NULL; } diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 57d9df8e84..c25588a45e 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -8,6 +8,7 @@ #include "eval.h" #include <ctype.h> +#include <float.h> /* for DBL_MANT_DIG and friends */ #ifdef RISCOS #include "unixstuff.h" @@ -223,9 +224,6 @@ Return the binary representation of an integer or long integer."); static PyObject * builtin_callable(PyObject *self, PyObject *v) { - if (PyErr_WarnPy3k("callable() not supported in 3.x; " - "use isinstance(x, collections.Callable)", 1) < 0) - return NULL; return PyBool_FromLong((long)PyCallable_Check(v)); } @@ -603,7 +601,7 @@ builtin_divmod(PyObject *self, PyObject *args) } PyDoc_STRVAR(divmod_doc, -"divmod(x, y) -> (div, mod)\n\ +"divmod(x, y) -> (quotient, remainder)\n\ \n\ Return the tuple ((x-x%y)/y, x%y). Invariant: div*y + mod == x."); @@ -1489,7 +1487,7 @@ PyDoc_STRVAR(open_doc, "open(name[, mode[, buffering]]) -> file object\n\ \n\ Open a file using the file() type, returns a file object. This is the\n\ -preferred way to open a file."); +preferred way to open a file. See file.__doc__ for further information."); static PyObject * @@ -1796,7 +1794,7 @@ handle_range_longs(PyObject *self, PyObject *args) PyObject *curnum = NULL; PyObject *v = NULL; long bign; - int i, n; + Py_ssize_t i, n; int cmp_result; PyObject *zero = PyLong_FromLong(0); @@ -1864,7 +1862,7 @@ handle_range_longs(PyObject *self, PyObject *args) Py_DECREF(neg_step); } - n = (int)bign; + n = (Py_ssize_t)bign; if (bign < 0 || (long)n != bign) { PyErr_SetString(PyExc_OverflowError, "range() result has too many items"); @@ -1944,7 +1942,7 @@ builtin_range(PyObject *self, PyObject *args) { long ilow = 0, ihigh = 0, istep = 1; long bign; - int i, n; + Py_ssize_t i, n; PyObject *v; @@ -1973,7 +1971,7 @@ builtin_range(PyObject *self, PyObject *args) bign = get_len_of_range(ilow, ihigh, istep); else bign = get_len_of_range(ihigh, ilow, -istep); - n = (int)bign; + n = (Py_ssize_t)bign; if (bign < 0 || (long)n != bign) { PyErr_SetString(PyExc_OverflowError, "range() result has too many items"); @@ -2151,36 +2149,47 @@ For most object types, eval(repr(object)) == object."); static PyObject * builtin_round(PyObject *self, PyObject *args, PyObject *kwds) { - double number, abs_number, abs_result; - double f; - int ndigits = 0; - int i; + double x; + PyObject *o_ndigits = NULL; + Py_ssize_t ndigits; static char *kwlist[] = {"number", "ndigits", 0}; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "d|i:round", - kwlist, &number, &ndigits)) + if (!PyArg_ParseTupleAndKeywords(args, kwds, "d|O:round", + kwlist, &x, &o_ndigits)) return NULL; - f = 1.0; - i = abs(ndigits); - while (--i >= 0) - f = f*10.0; - if (ndigits < 0) - number /= f; - else - number *= f; - /* round `number` to nearest integer, rounding halves away from zero */ - abs_number = fabs(number); - abs_result = floor(abs_number); - if (abs_number - abs_result >= 0.5) - abs_result += 1.0; - number = copysign(abs_result, number); + if (o_ndigits == NULL) { + /* second argument defaults to 0 */ + ndigits = 0; + } + else { + /* interpret 2nd argument as a Py_ssize_t; clip on overflow */ + ndigits = PyNumber_AsSsize_t(o_ndigits, NULL); + if (ndigits == -1 && PyErr_Occurred()) + return NULL; + } - if (ndigits < 0) - number *= f; + /* nans, infinities and zeros round to themselves */ + if (!Py_IS_FINITE(x) || x == 0.0) + return PyFloat_FromDouble(x); + + /* Deal with extreme values for ndigits. For ndigits > NDIGITS_MAX, x + always rounds to itself. For ndigits < NDIGITS_MIN, x always + rounds to +-0.0. Here 0.30103 is an upper bound for log10(2). */ +#define NDIGITS_MAX ((int)((DBL_MANT_DIG-DBL_MIN_EXP) * 0.30103)) +#define NDIGITS_MIN (-(int)((DBL_MAX_EXP + 1) * 0.30103)) + if (ndigits > NDIGITS_MAX) + /* return x */ + return PyFloat_FromDouble(x); + else if (ndigits < NDIGITS_MIN) + /* return 0.0, but with sign of x */ + return PyFloat_FromDouble(0.0*x); else - number /= f; - return PyFloat_FromDouble(number); + /* finite x, and ndigits is not unreasonably large */ + /* _Py_double_round is defined in floatobject.c */ + return _Py_double_round(x, (int)ndigits); +#undef NDIGITS_MAX +#undef NDIGITS_MIN } PyDoc_STRVAR(round_doc, @@ -2388,6 +2397,15 @@ builtin_sum(PyObject *self, PyObject *args) } break; } + /* It's tempting to use PyNumber_InPlaceAdd instead of + PyNumber_Add here, to avoid quadratic running time + when doing 'sum(list_of_lists, [])'. However, this + would produce a change in behaviour: a snippet like + + empty = [] + sum([[x] for x in range(10)], empty) + + would change the value of empty. */ temp = PyNumber_Add(result, item); Py_DECREF(result); Py_DECREF(item); @@ -2477,8 +2495,10 @@ builtin_zip(PyObject *self, PyObject *args) len = -1; /* unknown */ for (i = 0; i < itemsize; ++i) { PyObject *item = PyTuple_GET_ITEM(args, i); - Py_ssize_t thislen = _PyObject_LengthHint(item, -1); + Py_ssize_t thislen = _PyObject_LengthHint(item, -2); if (thislen < 0) { + if (thislen == -1) + return NULL; len = -1; break; } @@ -2664,7 +2684,7 @@ _PyBuiltin_Init(void) SETBUILTIN("True", Py_True); SETBUILTIN("basestring", &PyBaseString_Type); SETBUILTIN("bool", &PyBool_Type); - /* SETBUILTIN("memoryview", &PyMemoryView_Type); */ + SETBUILTIN("memoryview", &PyMemoryView_Type); SETBUILTIN("bytearray", &PyByteArray_Type); SETBUILTIN("bytes", &PyString_Type); SETBUILTIN("buffer", &PyBuffer_Type); diff --git a/Python/ceval.c b/Python/ceval.c index 1e44b1cc9c..06ada97274 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -27,10 +27,11 @@ typedef unsigned long long uint64; -#if defined(__ppc__) /* <- Don't know if this is the correct symbol; this - section should work for GCC on any PowerPC - platform, irrespective of OS. - POWER? Who knows :-) */ +/* PowerPC support. + "__ppc__" appears to be the preprocessor definition to detect on OS X, whereas + "__powerpc__" appears to be the correct one for Linux with GCC +*/ +#if defined(__ppc__) || defined (__powerpc__) #define READ_TIMESTAMP(var) ppc_getcounter(&var) @@ -146,6 +147,7 @@ static void format_exc_check_arg(PyObject *, char *, PyObject *); static PyObject * string_concatenate(PyObject *, PyObject *, PyFrameObject *, unsigned char *); static PyObject * kwd_as_string(PyObject *); +static PyObject * special_lookup(PyObject *, char *, PyObject **); #define NAME_ERROR_MSG \ "name '%.200s' is not defined" @@ -232,6 +234,7 @@ PyEval_GetCallStats(PyObject *self) #include "pythread.h" static PyThread_type_lock interpreter_lock = 0; /* This is the GIL */ +static PyThread_type_lock pending_lock = 0; /* for pending calls */ static long main_thread = 0; int @@ -303,6 +306,7 @@ PyEval_ReInitThreads(void) adding a new function to each thread_*.h. Instead, just create a new lock and waste a little bit of memory */ interpreter_lock = PyThread_allocate_lock(); + pending_lock = PyThread_allocate_lock(); PyThread_acquire_lock(interpreter_lock, 1); main_thread = PyThread_get_thread_ident(); @@ -375,19 +379,145 @@ PyEval_RestoreThread(PyThreadState *tstate) #ifdef WITH_THREAD Any thread can schedule pending calls, but only the main thread will execute them. + There is no facility to schedule calls to a particular thread, but + that should be easy to change, should that ever be required. In + that case, the static variables here should go into the python + threadstate. #endif +*/ + +#ifdef WITH_THREAD + +/* The WITH_THREAD implementation is thread-safe. It allows + scheduling to be made from any thread, and even from an executing + callback. + */ + +#define NPENDINGCALLS 32 +static struct { + int (*func)(void *); + void *arg; +} pendingcalls[NPENDINGCALLS]; +static int pendingfirst = 0; +static int pendinglast = 0; +static volatile int pendingcalls_to_do = 1; /* trigger initialization of lock */ +static char pendingbusy = 0; + +int +Py_AddPendingCall(int (*func)(void *), void *arg) +{ + int i, j, result=0; + PyThread_type_lock lock = pending_lock; + + /* try a few times for the lock. Since this mechanism is used + * for signal handling (on the main thread), there is a (slim) + * chance that a signal is delivered on the same thread while we + * hold the lock during the Py_MakePendingCalls() function. + * This avoids a deadlock in that case. + * Note that signals can be delivered on any thread. In particular, + * on Windows, a SIGINT is delivered on a system-created worker + * thread. + * We also check for lock being NULL, in the unlikely case that + * this function is called before any bytecode evaluation takes place. + */ + if (lock != NULL) { + for (i = 0; i<100; i++) { + if (PyThread_acquire_lock(lock, NOWAIT_LOCK)) + break; + } + if (i == 100) + return -1; + } + + i = pendinglast; + j = (i + 1) % NPENDINGCALLS; + if (j == pendingfirst) { + result = -1; /* Queue full */ + } else { + pendingcalls[i].func = func; + pendingcalls[i].arg = arg; + pendinglast = j; + } + /* signal main loop */ + _Py_Ticker = 0; + pendingcalls_to_do = 1; + if (lock != NULL) + PyThread_release_lock(lock); + return result; +} + +int +Py_MakePendingCalls(void) +{ + int i; + int r = 0; + + if (!pending_lock) { + /* initial allocation of the lock */ + pending_lock = PyThread_allocate_lock(); + if (pending_lock == NULL) + return -1; + } + + /* only service pending calls on main thread */ + if (main_thread && PyThread_get_thread_ident() != main_thread) + return 0; + /* don't perform recursive pending calls */ + if (pendingbusy) + return 0; + pendingbusy = 1; + /* perform a bounded number of calls, in case of recursion */ + for (i=0; i<NPENDINGCALLS; i++) { + int j; + int (*func)(void *); + void *arg = NULL; + + /* pop one item off the queue while holding the lock */ + PyThread_acquire_lock(pending_lock, WAIT_LOCK); + j = pendingfirst; + if (j == pendinglast) { + func = NULL; /* Queue empty */ + } else { + func = pendingcalls[j].func; + arg = pendingcalls[j].arg; + pendingfirst = (j + 1) % NPENDINGCALLS; + } + pendingcalls_to_do = pendingfirst != pendinglast; + PyThread_release_lock(pending_lock); + /* having released the lock, perform the callback */ + if (func == NULL) + break; + r = func(arg); + if (r) + break; + } + pendingbusy = 0; + return r; +} + +#else /* if ! defined WITH_THREAD */ + +/* + WARNING! ASYNCHRONOUSLY EXECUTING CODE! + This code is used for signal handling in python that isn't built + with WITH_THREAD. + Don't use this implementation when Py_AddPendingCalls() can happen + on a different thread! - XXX WARNING! ASYNCHRONOUSLY EXECUTING CODE! There are two possible race conditions: - (1) nested asynchronous registry calls; - (2) registry calls made while pending calls are being processed. - While (1) is very unlikely, (2) is a real possibility. + (1) nested asynchronous calls to Py_AddPendingCall() + (2) AddPendingCall() calls made while pending calls are being processed. + + (1) is very unlikely because typically signal delivery + is blocked during signal handling. So it should be impossible. + (2) is a real possibility. The current code is safe against (2), but not against (1). The safety against (2) is derived from the fact that only one - thread (the main thread) ever takes things out of the queue. - - XXX Darn! With the advent of thread state, we should have an array - of pending calls per thread in the thread state! Later... + thread is present, interrupted by signals, and that the critical + section is protected with the "busy" variable. On Windows, which + delivers SIGINT on a system thread, this does not hold and therefore + Windows really shouldn't use this version. + The two threads could theoretically wiggle around the "busy" variable. */ #define NPENDINGCALLS 32 @@ -397,7 +527,7 @@ static struct { } pendingcalls[NPENDINGCALLS]; static volatile int pendingfirst = 0; static volatile int pendinglast = 0; -static volatile int things_to_do = 0; +static volatile int pendingcalls_to_do = 0; int Py_AddPendingCall(int (*func)(void *), void *arg) @@ -405,8 +535,6 @@ Py_AddPendingCall(int (*func)(void *), void *arg) static volatile int busy = 0; int i, j; /* XXX Begin critical section */ - /* XXX If you want this to be safe against nested - XXX asynchronous calls, you'll have to work harder! */ if (busy) return -1; busy = 1; @@ -421,7 +549,7 @@ Py_AddPendingCall(int (*func)(void *), void *arg) pendinglast = j; _Py_Ticker = 0; - things_to_do = 1; /* Signal main loop */ + pendingcalls_to_do = 1; /* Signal main loop */ busy = 0; /* XXX End critical section */ return 0; @@ -431,14 +559,10 @@ int Py_MakePendingCalls(void) { static int busy = 0; -#ifdef WITH_THREAD - if (main_thread && PyThread_get_thread_ident() != main_thread) - return 0; -#endif if (busy) return 0; busy = 1; - things_to_do = 0; + pendingcalls_to_do = 0; for (;;) { int i; int (*func)(void *); @@ -451,7 +575,7 @@ Py_MakePendingCalls(void) pendingfirst = (i + 1) % NPENDINGCALLS; if (func(arg) < 0) { busy = 0; - things_to_do = 1; /* We're not done yet */ + pendingcalls_to_do = 1; /* We're not done yet */ return -1; } } @@ -459,6 +583,8 @@ Py_MakePendingCalls(void) return 0; } +#endif /* WITH_THREAD */ + /* The interpreter's recursion limit */ @@ -533,7 +659,7 @@ static int _Py_TracingPossible = 0; /* for manipulating the thread switch and periodic "stuff" - used to be per thread, now just a pair o' globals */ int _Py_CheckInterval = 100; -volatile int _Py_Ticker = 100; +volatile int _Py_Ticker = 0; /* so that we hit a "tick" first thing */ PyObject * PyEval_EvalCode(PyCodeObject *co, PyObject *globals, PyObject *locals) @@ -658,8 +784,8 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) /* OpCode prediction macros Some opcodes tend to come in pairs thus making it possible to predict the second code when the first is run. For example, - COMPARE_OP is often followed by JUMP_IF_FALSE or JUMP_IF_TRUE. And, - those opcodes are often followed by a POP_TOP. + GET_ITER is often followed by FOR_ITER. And FOR_ITER is often + followed by STORE_FAST or UNPACK_SEQUENCE. Verifying the prediction costs a single high-speed test of a register variable against a constant. If the pairing was good, then the @@ -696,10 +822,12 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) #define SECOND() (stack_pointer[-2]) #define THIRD() (stack_pointer[-3]) #define FOURTH() (stack_pointer[-4]) +#define PEEK(n) (stack_pointer[-(n)]) #define SET_TOP(v) (stack_pointer[-1] = (v)) #define SET_SECOND(v) (stack_pointer[-2] = (v)) #define SET_THIRD(v) (stack_pointer[-3] = (v)) #define SET_FOURTH(v) (stack_pointer[-4] = (v)) +#define SET_VALUE(n, v) (stack_pointer[-(n)] = (v)) #define BASIC_STACKADJ(n) (stack_pointer += n) #define BASIC_PUSH(v) (*stack_pointer++ = (v)) #define BASIC_POP() (*--stack_pointer) @@ -851,7 +979,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) /* Do periodic things. Doing this every time through the loop would add too much overhead, so we do it only every Nth instruction. We also do it if - ``things_to_do'' is set, i.e. when an asynchronous + ``pendingcalls_to_do'' is set, i.e. when an asynchronous event needs attention (e.g. a signal handler or async I/O handler); see Py_AddPendingCall() and Py_MakePendingCalls() above. */ @@ -859,7 +987,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) if (--_Py_Ticker < 0) { if (*next_instr == SETUP_FINALLY) { /* Make the last opcode before - a try: finally: block uninterruptable. */ + a try: finally: block uninterruptible. */ goto fast_next_opcode; } _Py_Ticker = _Py_CheckInterval; @@ -867,12 +995,12 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) #ifdef WITH_TSC ticked = 1; #endif - if (things_to_do) { + if (pendingcalls_to_do) { if (Py_MakePendingCalls() < 0) { why = WHY_EXCEPTION; goto on_error; } - if (things_to_do) + if (pendingcalls_to_do) /* MakePendingCalls() didn't succeed. Force early re-execution of this "periodic" code, possibly after @@ -1004,7 +1132,6 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) SETLOCAL(oparg, v); goto fast_next_opcode; - PREDICTED(POP_TOP); case POP_TOP: v = POP(); Py_DECREF(v); @@ -1177,7 +1304,10 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) case BINARY_MODULO: w = POP(); v = TOP(); - x = PyNumber_Remainder(v, w); + if (PyString_CheckExact(v)) + x = PyString_Format(v, w); + else + x = PyNumber_Remainder(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); @@ -1317,9 +1447,19 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) case LIST_APPEND: w = POP(); - v = POP(); + v = PEEK(oparg); err = PyList_Append(v, w); - Py_DECREF(v); + Py_DECREF(w); + if (err == 0) { + PREDICT(JUMP_ABSOLUTE); + continue; + } + break; + + case SET_ADD: + w = POP(); + v = stack_pointer[-oparg]; + err = PySet_Add(v, w); Py_DECREF(w); if (err == 0) { PREDICT(JUMP_ABSOLUTE); @@ -1848,7 +1988,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) } } else if (unpack_iterable(v, oparg, stack_pointer + oparg)) { - stack_pointer += oparg; + STACKADJ(oparg); } else { /* unpack_iterable() raised an exception */ why = WHY_EXCEPTION; @@ -2058,6 +2198,25 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) } break; + case BUILD_SET: + x = PySet_New(NULL); + if (x != NULL) { + for (; --oparg >= 0;) { + w = POP(); + if (err == 0) + err = PySet_Add(x, w); + Py_DECREF(w); + } + if (err != 0) { + Py_DECREF(x); + break; + } + PUSH(x); + continue; + } + break; + + case BUILD_MAP: x = _PyDict_NewPresized((Py_ssize_t)oparg); PUSH(x); @@ -2076,6 +2235,21 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) if (err == 0) continue; break; + case MAP_ADD: + w = TOP(); /* key */ + u = SECOND(); /* value */ + STACKADJ(-2); + v = stack_pointer[-oparg]; /* dict */ + assert (PyDict_CheckExact(v)); + err = PyDict_SetItem(v, w, u); /* v[w] = u */ + Py_DECREF(u); + Py_DECREF(w); + if (err == 0) { + PREDICT(JUMP_ABSOLUTE); + continue; + } + break; + case LOAD_ATTR: w = GETITEM(names, oparg); v = TOP(); @@ -2116,8 +2290,8 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) Py_DECREF(w); SET_TOP(x); if (x == NULL) break; - PREDICT(JUMP_IF_FALSE); - PREDICT(JUMP_IF_TRUE); + PREDICT(POP_JUMP_IF_FALSE); + PREDICT(POP_JUMP_IF_TRUE); continue; case IMPORT_NAME: @@ -2194,41 +2368,45 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) JUMPBY(oparg); goto fast_next_opcode; - PREDICTED_WITH_ARG(JUMP_IF_FALSE); - case JUMP_IF_FALSE: - w = TOP(); + PREDICTED_WITH_ARG(POP_JUMP_IF_FALSE); + case POP_JUMP_IF_FALSE: + w = POP(); if (w == Py_True) { - PREDICT(POP_TOP); + Py_DECREF(w); goto fast_next_opcode; } if (w == Py_False) { - JUMPBY(oparg); + Py_DECREF(w); + JUMPTO(oparg); goto fast_next_opcode; } err = PyObject_IsTrue(w); + Py_DECREF(w); if (err > 0) err = 0; else if (err == 0) - JUMPBY(oparg); + JUMPTO(oparg); else break; continue; - PREDICTED_WITH_ARG(JUMP_IF_TRUE); - case JUMP_IF_TRUE: - w = TOP(); + PREDICTED_WITH_ARG(POP_JUMP_IF_TRUE); + case POP_JUMP_IF_TRUE: + w = POP(); if (w == Py_False) { - PREDICT(POP_TOP); + Py_DECREF(w); goto fast_next_opcode; } if (w == Py_True) { - JUMPBY(oparg); + Py_DECREF(w); + JUMPTO(oparg); goto fast_next_opcode; } err = PyObject_IsTrue(w); + Py_DECREF(w); if (err > 0) { err = 0; - JUMPBY(oparg); + JUMPTO(oparg); } else if (err == 0) ; @@ -2236,6 +2414,53 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) break; continue; + case JUMP_IF_FALSE_OR_POP: + w = TOP(); + if (w == Py_True) { + STACKADJ(-1); + Py_DECREF(w); + goto fast_next_opcode; + } + if (w == Py_False) { + JUMPTO(oparg); + goto fast_next_opcode; + } + err = PyObject_IsTrue(w); + if (err > 0) { + STACKADJ(-1); + Py_DECREF(w); + err = 0; + } + else if (err == 0) + JUMPTO(oparg); + else + break; + continue; + + case JUMP_IF_TRUE_OR_POP: + w = TOP(); + if (w == Py_False) { + STACKADJ(-1); + Py_DECREF(w); + goto fast_next_opcode; + } + if (w == Py_True) { + JUMPTO(oparg); + goto fast_next_opcode; + } + err = PyObject_IsTrue(w); + if (err > 0) { + err = 0; + JUMPTO(oparg); + } + else if (err == 0) { + STACKADJ(-1); + Py_DECREF(w); + } + else + break; + continue; + PREDICTED_WITH_ARG(JUMP_ABSOLUTE); case JUMP_ABSOLUTE: JUMPTO(oparg); @@ -2313,6 +2538,35 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) STACK_LEVEL()); continue; + case SETUP_WITH: + { + static PyObject *exit, *enter; + w = TOP(); + x = special_lookup(w, "__exit__", &exit); + if (!x) + break; + SET_TOP(x); + u = special_lookup(w, "__enter__", &enter); + Py_DECREF(w); + if (!u) { + x = NULL; + break; + } + x = PyObject_CallFunctionObjArgs(u, NULL); + Py_DECREF(u); + if (!x) + break; + /* Setup a finally block (SETUP_WITH as a block is + equivalent to SETUP_FINALLY except it normalizes + the exception) before pushing the result of + __enter__ on the stack. */ + PyFrame_BlockSetup(f, SETUP_WITH, INSTR_OFFSET() + oparg, + STACK_LEVEL()); + + PUSH(x); + continue; + } + case WITH_CLEANUP: { /* At the top of the stack are 1-3 values indicating @@ -2444,7 +2698,6 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) Py_DECREF(*pfunc); *pfunc = self; na++; - n++; } else Py_INCREF(func); sp = stack_pointer; @@ -2544,7 +2797,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) default: fprintf(stderr, "XXX lineno: %d, opcode: %d\n", - PyCode_Addr2Line(f->f_code, f->f_lasti), + PyFrame_GetLineNumber(f), opcode); PyErr_SetString(PyExc_SystemError, "unknown opcode"); why = WHY_EXCEPTION; @@ -2622,20 +2875,20 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) fast_block_end: while (why != WHY_NOT && f->f_iblock > 0) { - PyTryBlock *b = PyFrame_BlockPop(f); + /* Peek at the current block. */ + PyTryBlock *b = &f->f_blockstack[f->f_iblock - 1]; assert(why != WHY_YIELD); if (b->b_type == SETUP_LOOP && why == WHY_CONTINUE) { - /* For a continue inside a try block, - don't pop the block for the loop. */ - PyFrame_BlockSetup(f, b->b_type, b->b_handler, - b->b_level); why = WHY_NOT; JUMPTO(PyInt_AS_LONG(retval)); Py_DECREF(retval); break; } + /* Now we have to pop the block. */ + f->f_iblock--; + while (STACK_LEVEL() > b->b_level) { v = POP(); Py_XDECREF(v); @@ -2647,7 +2900,8 @@ fast_block_end: } if (b->b_type == SETUP_FINALLY || (b->b_type == SETUP_EXCEPT && - why == WHY_EXCEPTION)) { + why == WHY_EXCEPTION) || + b->b_type == SETUP_WITH) { if (why == WHY_EXCEPTION) { PyObject *exc, *val, *tb; PyErr_Fetch(&exc, &val, &tb); @@ -2660,7 +2914,8 @@ fast_block_end: so a program can emulate the Python main loop. Don't do this for 'finally'. */ - if (b->b_type == SETUP_EXCEPT) { + if (b->b_type == SETUP_EXCEPT || + b->b_type == SETUP_WITH) { PyErr_NormalizeException( &exc, &val, &tb); set_exc_info(tstate, @@ -2800,13 +3055,12 @@ PyEval_EvalCodeEx(PyCodeObject *co, PyObject *globals, PyObject *locals, if (!(co->co_flags & CO_VARARGS)) { PyErr_Format(PyExc_TypeError, "%.200s() takes %s %d " - "%sargument%s (%d given)", + "argument%s (%d given)", PyString_AsString(co->co_name), defcount ? "at most" : "exactly", co->co_argcount, - kwcount ? "non-keyword " : "", co->co_argcount == 1 ? "" : "s", - argcount); + argcount + kwcount); goto fail; } n = co->co_argcount; @@ -2832,8 +3086,11 @@ PyEval_EvalCodeEx(PyCodeObject *co, PyObject *globals, PyObject *locals, PyObject *keyword = kws[2*i]; PyObject *value = kws[2*i + 1]; int j; - if (keyword == NULL || !(PyString_Check(keyword) || - PyUnicode_Check(keyword))) { + if (keyword == NULL || !(PyString_Check(keyword) +#ifdef Py_USING_UNICODE + || PyUnicode_Check(keyword) +#endif + )) { PyErr_Format(PyExc_TypeError, "%.200s() keywords must be strings", PyString_AsString(co->co_name)); @@ -2841,7 +3098,7 @@ PyEval_EvalCodeEx(PyCodeObject *co, PyObject *globals, PyObject *locals, } /* Speed hack: do raw pointer compares. As names are normally interned this should almost always hit. */ - co_varnames = PySequence_Fast_ITEMS(co->co_varnames); + co_varnames = ((PyTupleObject *)(co->co_varnames))->ob_item; for (j = 0; j < co->co_argcount; j++) { PyObject *nm = co_varnames[j]; if (nm == keyword) @@ -2857,26 +3114,21 @@ PyEval_EvalCodeEx(PyCodeObject *co, PyObject *globals, PyObject *locals, else if (cmp < 0) goto fail; } - /* Check errors from Compare */ - if (PyErr_Occurred()) - goto fail; - if (j >= co->co_argcount) { - if (kwdict == NULL) { - PyObject *kwd_str = kwd_as_string(keyword); - if (kwd_str) { - PyErr_Format(PyExc_TypeError, - "%.200s() got an unexpected " - "keyword argument '%.400s'", - PyString_AsString(co->co_name), - PyString_AsString(kwd_str)); - Py_DECREF(kwd_str); - } - goto fail; + if (kwdict == NULL) { + PyObject *kwd_str = kwd_as_string(keyword); + if (kwd_str) { + PyErr_Format(PyExc_TypeError, + "%.200s() got an unexpected " + "keyword argument '%.400s'", + PyString_AsString(co->co_name), + PyString_AsString(kwd_str)); + Py_DECREF(kwd_str); } - PyDict_SetItem(kwdict, keyword, value); - continue; + goto fail; } -kw_found: + PyDict_SetItem(kwdict, keyword, value); + continue; + kw_found: if (GETLOCAL(j) != NULL) { PyObject *kwd_str = kwd_as_string(keyword); if (kwd_str) { @@ -2897,15 +3149,18 @@ kw_found: int m = co->co_argcount - defcount; for (i = argcount; i < m; i++) { if (GETLOCAL(i) == NULL) { + int j, given = 0; + for (j = 0; j < co->co_argcount; j++) + if (GETLOCAL(j)) + given++; PyErr_Format(PyExc_TypeError, "%.200s() takes %s %d " - "%sargument%s (%d given)", + "argument%s (%d given)", PyString_AsString(co->co_name), ((co->co_flags & CO_VARARGS) || defcount) ? "at least" : "exactly", - m, kwcount ? "non-keyword " : "", - m == 1 ? "" : "s", i); + m, m == 1 ? "" : "s", given); goto fail; } } @@ -2922,14 +3177,12 @@ kw_found: } } } - else { - if (argcount > 0 || kwcount > 0) { - PyErr_Format(PyExc_TypeError, - "%.200s() takes no arguments (%d given)", - PyString_AsString(co->co_name), - argcount + kwcount); - goto fail; - } + else if (argcount > 0 || kwcount > 0) { + PyErr_Format(PyExc_TypeError, + "%.200s() takes no arguments (%d given)", + PyString_AsString(co->co_name), + argcount + kwcount); + goto fail; } /* Allocate and initialize storage for cell vars, and copy free vars into frame. This isn't too efficient right now. */ @@ -3015,13 +3268,37 @@ fail: /* Jump here from prelude on failure */ static PyObject * +special_lookup(PyObject *o, char *meth, PyObject **cache) +{ + PyObject *res; + if (PyInstance_Check(o)) { + if (!*cache) + return PyObject_GetAttrString(o, meth); + else + return PyObject_GetAttr(o, *cache); + } + res = _PyObject_LookupSpecial(o, meth, cache); + if (res == NULL && !PyErr_Occurred()) { + PyErr_SetObject(PyExc_AttributeError, *cache); + return NULL; + } + return res; +} + + +static PyObject * kwd_as_string(PyObject *kwd) { +#ifdef Py_USING_UNICODE if (PyString_Check(kwd)) { +#else + assert(PyString_Check(kwd)); +#endif Py_INCREF(kwd); return kwd; +#ifdef Py_USING_UNICODE } - else - return _PyUnicode_AsDefaultEncodedString(kwd, "replace"); + return _PyUnicode_AsDefaultEncodedString(kwd, "replace"); +#endif } @@ -3238,9 +3515,17 @@ do_raise(PyObject *type, PyObject *value, PyObject *tb) Py_DECREF(tmp); } - if (PyExceptionClass_Check(type)) + if (PyExceptionClass_Check(type)) { PyErr_NormalizeException(&type, &value, &tb); - + if (!PyExceptionInstance_Check(value)) { + PyErr_Format(PyExc_TypeError, + "calling %s() should have returned an instance of " + "BaseException, not '%s'", + ((PyTypeObject *)type)->tp_name, + Py_TYPE(value)->tp_name); + goto raise_error; + } + } else if (PyExceptionInstance_Check(type)) { /* Raising an instance. The value should be a dummy. */ if (value != Py_None) { @@ -3429,33 +3714,30 @@ _PyEval_CallTracing(PyObject *func, PyObject *args) return result; } +/* See Objects/lnotab_notes.txt for a description of how tracing works. */ static int maybe_call_line_trace(Py_tracefunc func, PyObject *obj, PyFrameObject *frame, int *instr_lb, int *instr_ub, int *instr_prev) { int result = 0; + int line = frame->f_lineno; /* If the last instruction executed isn't in the current - instruction window, reset the window. If the last - instruction happens to fall at the start of a line or if it - represents a jump backwards, call the trace function. + instruction window, reset the window. */ - if ((frame->f_lasti < *instr_lb || frame->f_lasti >= *instr_ub)) { - int line; + if (frame->f_lasti < *instr_lb || frame->f_lasti >= *instr_ub) { PyAddrPair bounds; - - line = PyCode_CheckLineNumber(frame->f_code, frame->f_lasti, - &bounds); - if (line >= 0) { - frame->f_lineno = line; - result = call_trace(func, obj, frame, - PyTrace_LINE, Py_None); - } + line = _PyCode_CheckLineNumber(frame->f_code, frame->f_lasti, + &bounds); *instr_lb = bounds.ap_lower; *instr_ub = bounds.ap_upper; } - else if (frame->f_lasti <= *instr_prev) { + /* If the last instruction falls at the start of a line or if + it represents a jump backwards, update the frame's line + number and call the trace function. */ + if (frame->f_lasti == *instr_lb || frame->f_lasti < *instr_prev) { + frame->f_lineno = line; result = call_trace(func, obj, frame, PyTrace_LINE, Py_None); } *instr_prev = frame->f_lasti; @@ -3578,18 +3860,7 @@ Py_FlushLine(void) /* External interface to call any callable object. - The arg must be a tuple or NULL. */ - -#undef PyEval_CallObject -/* for backward compatibility: export this interface */ - -PyObject * -PyEval_CallObject(PyObject *func, PyObject *arg) -{ - return PyEval_CallObjectWithKeywords(func, arg, (PyObject *)NULL); -} -#define PyEval_CallObject(func,arg) \ - PyEval_CallObjectWithKeywords(func, arg, (PyObject *)NULL) + The arg must be a tuple or NULL. The kw must be a dict or NULL. */ PyObject * PyEval_CallObjectWithKeywords(PyObject *func, PyObject *arg, PyObject *kw) @@ -4417,7 +4688,9 @@ exec_statement(PyFrameObject *f, PyObject *prog, PyObject *globals, else if (locals == Py_None) locals = globals; if (!PyString_Check(prog) && +#ifdef Py_USING_UNICODE !PyUnicode_Check(prog) && +#endif !PyCode_Check(prog) && !PyFile_Check(prog)) { PyErr_SetString(PyExc_TypeError, diff --git a/Python/codecs.c b/Python/codecs.c index 57bd0f41ff..7334eb3e36 100644 --- a/Python/codecs.c +++ b/Python/codecs.c @@ -70,7 +70,7 @@ PyObject *normalizestring(const char *string) if (ch == ' ') ch = '-'; else - ch = tolower(Py_CHARMASK(ch)); + ch = Py_TOLOWER(Py_CHARMASK(ch)); p[i] = ch; } return v; diff --git a/Python/compile.c b/Python/compile.c index 6650504ec0..119c60f9b2 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -39,6 +39,10 @@ int Py_OptimizeFlag = 0; #define DEFAULT_CODE_SIZE 128 #define DEFAULT_LNOTAB_SIZE 16 +#define COMP_GENEXP 0 +#define COMP_SETCOMP 1 +#define COMP_DICTCOMP 2 + struct instr { unsigned i_jabs : 1; unsigned i_jrel : 1; @@ -111,7 +115,6 @@ struct compiler_unit { members, you can reach all early allocated blocks. */ basicblock *u_blocks; basicblock *u_curblock; /* pointer to current block */ - int u_tmpname; /* temporary variables for list comps */ int u_nfblocks; struct fblockinfo u_fblock[CO_MAXBLOCKS]; @@ -140,7 +143,6 @@ struct compiler { struct compiler_unit *u; /* compiler state for current block */ PyObject *c_stack; /* Python list holding compiler_unit ptrs */ - char *c_encoding; /* source encoding (a borrowed reference) */ PyArena *c_arena; /* pointer to memory allocation arena */ }; @@ -179,6 +181,8 @@ static int compiler_with(struct compiler *, stmt_ty); static PyCodeObject *assemble(struct compiler *, int addNone); static PyObject *__doc__; +#define COMPILER_CAPSULE_NAME_COMPILER_UNIT "compile.c compiler unit" + PyObject * _Py_Mangle(PyObject *privateobj, PyObject *ident) { @@ -282,9 +286,6 @@ PyAST_Compile(mod_ty mod, const char *filename, PyCompilerFlags *flags, goto finally; } - /* XXX initialize to NULL for now, need to handle */ - c.c_encoding = NULL; - co = compiler_mod(&c, mod); finally: @@ -472,7 +473,6 @@ compiler_enter_scope(struct compiler *c, identifier name, void *key, } u->u_blocks = NULL; - u->u_tmpname = 0; u->u_nfblocks = 0; u->u_firstlineno = lineno; u->u_lineno = 0; @@ -492,13 +492,13 @@ compiler_enter_scope(struct compiler *c, identifier name, void *key, /* Push the old compiler_unit on the stack. */ if (c->u) { - PyObject *wrapper = PyCObject_FromVoidPtr(c->u, NULL); - if (!wrapper || PyList_Append(c->c_stack, wrapper) < 0) { - Py_XDECREF(wrapper); + PyObject *capsule = PyCapsule_New(c->u, COMPILER_CAPSULE_NAME_COMPILER_UNIT, NULL); + if (!capsule || PyList_Append(c->c_stack, capsule) < 0) { + Py_XDECREF(capsule); compiler_unit_free(u); return 0; } - Py_DECREF(wrapper); + Py_DECREF(capsule); u->u_private = c->u->u_private; Py_XINCREF(u->u_private); } @@ -515,15 +515,15 @@ static void compiler_exit_scope(struct compiler *c) { int n; - PyObject *wrapper; + PyObject *capsule; c->c_nestlevel--; compiler_unit_free(c->u); /* Restore c->u to the parent unit. */ n = PyList_GET_SIZE(c->c_stack) - 1; if (n >= 0) { - wrapper = PyList_GET_ITEM(c->c_stack, n); - c->u = (struct compiler_unit *)PyCObject_AsVoidPtr(wrapper); + capsule = PyList_GET_ITEM(c->c_stack, n); + c->u = (struct compiler_unit *)PyCapsule_GetPointer(capsule, COMPILER_CAPSULE_NAME_COMPILER_UNIT); assert(c->u); /* we are deleting from a list so this really shouldn't fail */ if (PySequence_DelItem(c->c_stack, n) < 0) @@ -535,18 +535,6 @@ compiler_exit_scope(struct compiler *c) } -/* Allocate a new "anonymous" local variable. - Used by list comprehensions and with statements. -*/ - -static PyObject * -compiler_new_tmpname(struct compiler *c) -{ - char tmpname[256]; - PyOS_snprintf(tmpname, sizeof(tmpname), "_[%d]", ++c->u->u_tmpname); - return PyString_FromString(tmpname); -} - /* Allocate a new block and return a pointer to it. Returns NULL on error. */ @@ -692,7 +680,11 @@ opcode_stack_effect(int opcode, int oparg) case UNARY_INVERT: return 0; + case SET_ADD: case LIST_APPEND: + return -1; + + case MAP_ADD: return -2; case BINARY_POWER: @@ -710,13 +702,13 @@ opcode_stack_effect(int opcode, int oparg) return -1; case SLICE+0: - return 1; - case SLICE+1: return 0; + case SLICE+1: + return -1; case SLICE+2: - return 0; - case SLICE+3: return -1; + case SLICE+3: + return -2; case STORE_SLICE+0: return -2; @@ -778,6 +770,8 @@ opcode_stack_effect(int opcode, int oparg) return -1; case BREAK_LOOP: return 0; + case SETUP_WITH: + return 4; case WITH_CLEANUP: return -1; /* XXX Sometimes more */ case LOAD_LOCALS: @@ -794,7 +788,8 @@ opcode_stack_effect(int opcode, int oparg) case POP_BLOCK: return 0; case END_FINALLY: - return -1; /* or -2 or -3 if exception occurred */ + return -3; /* or -1 or -2 if no exception occurred or + return/break/continue */ case BUILD_CLASS: return -2; @@ -805,7 +800,7 @@ opcode_stack_effect(int opcode, int oparg) case UNPACK_SEQUENCE: return oparg-1; case FOR_ITER: - return 1; + return 1; /* or -1, at end of iterator */ case STORE_ATTR: return -2; @@ -823,6 +818,7 @@ opcode_stack_effect(int opcode, int oparg) return 1; case BUILD_TUPLE: case BUILD_LIST: + case BUILD_SET: return 1-oparg; case BUILD_MAP: return 1; @@ -831,26 +827,29 @@ opcode_stack_effect(int opcode, int oparg) case COMPARE_OP: return -1; case IMPORT_NAME: - return 0; + return -1; case IMPORT_FROM: return 1; case JUMP_FORWARD: - case JUMP_IF_FALSE: - case JUMP_IF_TRUE: + case JUMP_IF_TRUE_OR_POP: /* -1 if jump not taken */ + case JUMP_IF_FALSE_OR_POP: /* "" */ case JUMP_ABSOLUTE: return 0; + case POP_JUMP_IF_FALSE: + case POP_JUMP_IF_TRUE: + return -1; + case LOAD_GLOBAL: return 1; case CONTINUE_LOOP: return 0; case SETUP_LOOP: - return 0; case SETUP_EXCEPT: case SETUP_FINALLY: - return 3; /* actually pushed by an exception */ + return 0; case LOAD_FAST: return 1; @@ -879,7 +878,7 @@ opcode_stack_effect(int opcode, int oparg) return -1; case MAKE_CLOSURE: - return -oparg; + return -oparg-1; case LOAD_CLOSURE: return 1; case LOAD_DEREF: @@ -1246,7 +1245,7 @@ get_ref_type(struct compiler *c, PyObject *name) char buf[350]; PyOS_snprintf(buf, sizeof(buf), "unknown scope for %.100s in %.100s(%s) in %s\n" - "symbols: %s\nlocals: %s\nglobals: %s\n", + "symbols: %s\nlocals: %s\nglobals: %s", PyString_AS_STRING(name), PyString_AS_STRING(c->u->u_name), PyObject_REPR(c->u->u_ste->ste_id), @@ -1491,12 +1490,10 @@ compiler_ifexp(struct compiler *c, expr_ty e) if (next == NULL) return 0; VISIT(c, expr, e->v.IfExp.test); - ADDOP_JREL(c, JUMP_IF_FALSE, next); - ADDOP(c, POP_TOP); + ADDOP_JABS(c, POP_JUMP_IF_FALSE, next); VISIT(c, expr, e->v.IfExp.body); ADDOP_JREL(c, JUMP_FORWARD, end); compiler_use_next_block(c, next); - ADDOP(c, POP_TOP); VISIT(c, expr, e->v.IfExp.orelse); compiler_use_next_block(c, end); return 1; @@ -1524,9 +1521,19 @@ compiler_lambda(struct compiler *c, expr_ty e) /* unpack nested arguments */ compiler_arguments(c, args); + /* Make None the first constant, so the lambda can't have a + docstring. */ + if (compiler_add_o(c, c->u->u_consts, Py_None) < 0) + return 0; + c->u->u_argcount = asdl_seq_LEN(args->args); VISIT_IN_SCOPE(c, expr, e->v.Lambda.body); - ADDOP_IN_SCOPE(c, RETURN_VALUE); + if (c->u->u_ste->ste_generator) { + ADDOP_IN_SCOPE(c, POP_TOP); + } + else { + ADDOP_IN_SCOPE(c, RETURN_VALUE); + } co = assemble(c, 1); compiler_exit_scope(c); if (co == NULL) @@ -1584,9 +1591,6 @@ compiler_if(struct compiler *c, stmt_ty s) end = compiler_new_block(c); if (end == NULL) return 0; - next = compiler_new_block(c); - if (next == NULL) - return 0; constant = expr_constant(s->v.If.test); /* constant = 0: "if 0" @@ -1598,15 +1602,21 @@ compiler_if(struct compiler *c, stmt_ty s) } else if (constant == 1) { VISIT_SEQ(c, stmt, s->v.If.body); } else { + if (s->v.If.orelse) { + next = compiler_new_block(c); + if (next == NULL) + return 0; + } + else + next = end; VISIT(c, expr, s->v.If.test); - ADDOP_JREL(c, JUMP_IF_FALSE, next); - ADDOP(c, POP_TOP); + ADDOP_JABS(c, POP_JUMP_IF_FALSE, next); VISIT_SEQ(c, stmt, s->v.If.body); ADDOP_JREL(c, JUMP_FORWARD, end); - compiler_use_next_block(c, next); - ADDOP(c, POP_TOP); - if (s->v.If.orelse) + if (s->v.If.orelse) { + compiler_use_next_block(c, next); VISIT_SEQ(c, stmt, s->v.If.orelse); + } } compiler_use_next_block(c, end); return 1; @@ -1628,9 +1638,6 @@ compiler_for(struct compiler *c, stmt_ty s) VISIT(c, expr, s->v.For.iter); ADDOP(c, GET_ITER); compiler_use_next_block(c, start); - /* for expressions must be traced on each iteration, - so we need to set an extra line number. */ - c->u->u_lineno_set = false; ADDOP_JREL(c, FOR_ITER, cleanup); VISIT(c, expr, s->v.For.target); VISIT_SEQ(c, stmt, s->v.For.body); @@ -1676,12 +1683,8 @@ compiler_while(struct compiler *c, stmt_ty s) if (!compiler_push_fblock(c, LOOP, loop)) return 0; if (constant == -1) { - /* while expressions must be traced on each iteration, - so we need to set an extra line number. */ - c->u->u_lineno_set = false; VISIT(c, expr, s->v.While.test); - ADDOP_JREL(c, JUMP_IF_FALSE, anchor); - ADDOP(c, POP_TOP); + ADDOP_JABS(c, POP_JUMP_IF_FALSE, anchor); } VISIT_SEQ(c, stmt, s->v.While.body); ADDOP_JABS(c, JUMP_ABSOLUTE, loop); @@ -1692,7 +1695,6 @@ compiler_while(struct compiler *c, stmt_ty s) if (constant == -1) { compiler_use_next_block(c, anchor); - ADDOP(c, POP_TOP); ADDOP(c, POP_BLOCK); } compiler_pop_fblock(c, LOOP, loop); @@ -1813,20 +1815,17 @@ compiler_try_finally(struct compiler *c, stmt_ty s) [tb, val, exc] L1: DUP ) [tb, val, exc, exc] <evaluate E1> ) [tb, val, exc, exc, E1] COMPARE_OP EXC_MATCH ) only if E1 - [tb, val, exc, 1-or-0] JUMP_IF_FALSE L2 ) - [tb, val, exc, 1] POP ) + [tb, val, exc, 1-or-0] POP_JUMP_IF_FALSE L2 ) [tb, val, exc] POP [tb, val] <assign to V1> (or POP if no V1) [tb] POP [] <code for S1> JUMP_FORWARD L0 - [tb, val, exc, 0] L2: POP - [tb, val, exc] DUP + [tb, val, exc] L2: DUP .............................etc....................... - [tb, val, exc, 0] Ln+1: POP - [tb, val, exc] END_FINALLY # re-raise exception + [tb, val, exc] Ln+1: END_FINALLY # re-raise exception [] L0: <next statement> @@ -1868,8 +1867,7 @@ compiler_try_except(struct compiler *c, stmt_ty s) ADDOP(c, DUP_TOP); VISIT(c, expr, handler->v.ExceptHandler.type); ADDOP_I(c, COMPARE_OP, PyCmp_EXC_MATCH); - ADDOP_JREL(c, JUMP_IF_FALSE, except); - ADDOP(c, POP_TOP); + ADDOP_JABS(c, POP_JUMP_IF_FALSE, except); } ADDOP(c, POP_TOP); if (handler->v.ExceptHandler.name) { @@ -1882,8 +1880,6 @@ compiler_try_except(struct compiler *c, stmt_ty s) VISIT_SEQ(c, stmt, handler->v.ExceptHandler.body); ADDOP_JREL(c, JUMP_FORWARD, end); compiler_use_next_block(c, except); - if (handler->v.ExceptHandler.type) - ADDOP(c, POP_TOP); } ADDOP(c, END_FINALLY); compiler_use_next_block(c, orelse); @@ -1982,6 +1978,13 @@ compiler_from_import(struct compiler *c, stmt_ty s) PyObject *names = PyTuple_New(n); PyObject *level; + static PyObject *empty_string; + + if (!empty_string) { + empty_string = PyString_FromString(""); + if (!empty_string) + return 0; + } if (!names) return 0; @@ -2004,23 +2007,24 @@ compiler_from_import(struct compiler *c, stmt_ty s) PyTuple_SET_ITEM(names, i, alias->name); } - if (s->lineno > c->c_future->ff_lineno) { - if (!strcmp(PyString_AS_STRING(s->v.ImportFrom.module), - "__future__")) { - Py_DECREF(level); - Py_DECREF(names); - return compiler_error(c, - "from __future__ imports must occur " - "at the beginning of the file"); - - } + if (s->lineno > c->c_future->ff_lineno && s->v.ImportFrom.module && + !strcmp(PyString_AS_STRING(s->v.ImportFrom.module), "__future__")) { + Py_DECREF(level); + Py_DECREF(names); + return compiler_error(c, "from __future__ imports must occur " + "at the beginning of the file"); } ADDOP_O(c, LOAD_CONST, level, consts); Py_DECREF(level); ADDOP_O(c, LOAD_CONST, names, consts); Py_DECREF(names); - ADDOP_NAME(c, IMPORT_NAME, s->v.ImportFrom.module, names); + if (s->v.ImportFrom.module) { + ADDOP_NAME(c, IMPORT_NAME, s->v.ImportFrom.module, names); + } + else { + ADDOP_NAME(c, IMPORT_NAME, empty_string, names); + } for (i = 0; i < n; i++) { alias_ty alias = (alias_ty)asdl_seq_GET(s->v.ImportFrom.names, i); identifier store_name; @@ -2071,18 +2075,14 @@ compiler_assert(struct compiler *c, stmt_ty s) end = compiler_new_block(c); if (end == NULL) return 0; - ADDOP_JREL(c, JUMP_IF_TRUE, end); - ADDOP(c, POP_TOP); + ADDOP_JABS(c, POP_JUMP_IF_TRUE, end); ADDOP_O(c, LOAD_GLOBAL, assertion_error, names); if (s->v.Assert.msg) { VISIT(c, expr, s->v.Assert.msg); - ADDOP_I(c, RAISE_VARARGS, 2); - } - else { - ADDOP_I(c, RAISE_VARARGS, 1); + ADDOP_I(c, CALL_FUNCTION, 1); } + ADDOP_I(c, RAISE_VARARGS, 1); compiler_use_next_block(c, end); - ADDOP(c, POP_TOP); return 1; } @@ -2336,12 +2336,6 @@ compiler_nameop(struct compiler *c, identifier name, expr_context_ty ctx) PyObject *mangled; /* XXX AugStore isn't used anywhere! */ - /* First check for assignment to __debug__. Param? */ - if ((ctx == Store || ctx == AugStore || ctx == Del) - && !strcmp(PyString_AS_STRING(name), "__debug__")) { - return compiler_error(c, "can not assign to __debug__"); - } - mangled = _Py_Mangle(c->u->u_private, name); if (!mangled) return 0; @@ -2466,9 +2460,9 @@ compiler_boolop(struct compiler *c, expr_ty e) assert(e->kind == BoolOp_kind); if (e->v.BoolOp.op == And) - jumpi = JUMP_IF_FALSE; + jumpi = JUMP_IF_FALSE_OR_POP; else - jumpi = JUMP_IF_TRUE; + jumpi = JUMP_IF_TRUE_OR_POP; end = compiler_new_block(c); if (end == NULL) return 0; @@ -2477,8 +2471,7 @@ compiler_boolop(struct compiler *c, expr_ty e) assert(n >= 0); for (i = 0; i < n; ++i) { VISIT(c, expr, (expr_ty)asdl_seq_GET(s, i)); - ADDOP_JREL(c, jumpi, end); - ADDOP(c, POP_TOP) + ADDOP_JABS(c, jumpi, end); } VISIT(c, expr, (expr_ty)asdl_seq_GET(s, n)); compiler_use_next_block(c, end); @@ -2536,9 +2529,8 @@ compiler_compare(struct compiler *c, expr_ty e) ADDOP_I(c, COMPARE_OP, cmpop((cmpop_ty)(asdl_seq_GET( e->v.Compare.ops, i - 1)))); - ADDOP_JREL(c, JUMP_IF_FALSE, cleanup); + ADDOP_JABS(c, JUMP_IF_FALSE_OR_POP, cleanup); NEXT_BLOCK(c); - ADDOP(c, POP_TOP); if (i < (n - 1)) VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Compare.comparators, i)); @@ -2597,9 +2589,8 @@ compiler_call(struct compiler *c, expr_ty e) } static int -compiler_listcomp_generator(struct compiler *c, PyObject *tmpname, - asdl_seq *generators, int gen_index, - expr_ty elt) +compiler_listcomp_generator(struct compiler *c, asdl_seq *generators, + int gen_index, expr_ty elt) { /* generate code for the iterator, then each of the ifs, and then write to the element */ @@ -2630,37 +2621,24 @@ compiler_listcomp_generator(struct compiler *c, PyObject *tmpname, for (i = 0; i < n; i++) { expr_ty e = (expr_ty)asdl_seq_GET(l->ifs, i); VISIT(c, expr, e); - ADDOP_JREL(c, JUMP_IF_FALSE, if_cleanup); + ADDOP_JABS(c, POP_JUMP_IF_FALSE, if_cleanup); NEXT_BLOCK(c); - ADDOP(c, POP_TOP); } if (++gen_index < asdl_seq_LEN(generators)) - if (!compiler_listcomp_generator(c, tmpname, - generators, gen_index, elt)) + if (!compiler_listcomp_generator(c, generators, gen_index, elt)) return 0; /* only append after the last for generator */ if (gen_index >= asdl_seq_LEN(generators)) { - if (!compiler_nameop(c, tmpname, Load)) - return 0; VISIT(c, expr, elt); - ADDOP(c, LIST_APPEND); + ADDOP_I(c, LIST_APPEND, gen_index+1); compiler_use_next_block(c, skip); } - for (i = 0; i < n; i++) { - ADDOP_I(c, JUMP_FORWARD, 1); - if (i == 0) - compiler_use_next_block(c, if_cleanup); - ADDOP(c, POP_TOP); - } + compiler_use_next_block(c, if_cleanup); ADDOP_JABS(c, JUMP_ABSOLUTE, start); compiler_use_next_block(c, anchor); - /* delete the temporary list name added to locals */ - if (gen_index == 1) - if (!compiler_nameop(c, tmpname, Del)) - return 0; return 1; } @@ -2668,49 +2646,49 @@ compiler_listcomp_generator(struct compiler *c, PyObject *tmpname, static int compiler_listcomp(struct compiler *c, expr_ty e) { - identifier tmp; - int rc = 0; - asdl_seq *generators = e->v.ListComp.generators; - assert(e->kind == ListComp_kind); - tmp = compiler_new_tmpname(c); - if (!tmp) - return 0; ADDOP_I(c, BUILD_LIST, 0); - ADDOP(c, DUP_TOP); - if (compiler_nameop(c, tmp, Store)) - rc = compiler_listcomp_generator(c, tmp, generators, 0, - e->v.ListComp.elt); - Py_DECREF(tmp); - return rc; -} + return compiler_listcomp_generator(c, e->v.ListComp.generators, 0, + e->v.ListComp.elt); +} + +/* Dict and set comprehensions and generator expressions work by creating a + nested function to perform the actual iteration. This means that the + iteration variables don't leak into the current scope. + The defined function is called immediately following its definition, with the + result of that call being the result of the expression. + The LC/SC version returns the populated container, while the GE version is + flagged in symtable.c as a generator, so it returns the generator object + when the function is called. + This code *knows* that the loop cannot contain break, continue, or return, + so it cheats and skips the SETUP_LOOP/POP_BLOCK steps used in normal loops. + + Possible cleanups: + - iterate over the generator sequence instead of using recursion +*/ static int -compiler_genexp_generator(struct compiler *c, - asdl_seq *generators, int gen_index, - expr_ty elt) +compiler_comprehension_generator(struct compiler *c, + asdl_seq *generators, int gen_index, + expr_ty elt, expr_ty val, int type) { /* generate code for the iterator, then each of the ifs, and then write to the element */ - comprehension_ty ge; - basicblock *start, *anchor, *skip, *if_cleanup, *end; + comprehension_ty gen; + basicblock *start, *anchor, *skip, *if_cleanup; int i, n; start = compiler_new_block(c); skip = compiler_new_block(c); if_cleanup = compiler_new_block(c); anchor = compiler_new_block(c); - end = compiler_new_block(c); if (start == NULL || skip == NULL || if_cleanup == NULL || - anchor == NULL || end == NULL) + anchor == NULL) return 0; - ge = (comprehension_ty)asdl_seq_GET(generators, gen_index); - ADDOP_JREL(c, SETUP_LOOP, end); - if (!compiler_push_fblock(c, LOOP, start)) - return 0; + gen = (comprehension_ty)asdl_seq_GET(generators, gen_index); if (gen_index == 0) { /* Receive outermost iter as an implicit argument */ @@ -2719,84 +2697,164 @@ compiler_genexp_generator(struct compiler *c, } else { /* Sub-iter - calculate on the fly */ - VISIT(c, expr, ge->iter); + VISIT(c, expr, gen->iter); ADDOP(c, GET_ITER); } compiler_use_next_block(c, start); ADDOP_JREL(c, FOR_ITER, anchor); NEXT_BLOCK(c); - VISIT(c, expr, ge->target); + VISIT(c, expr, gen->target); /* XXX this needs to be cleaned up...a lot! */ - n = asdl_seq_LEN(ge->ifs); + n = asdl_seq_LEN(gen->ifs); for (i = 0; i < n; i++) { - expr_ty e = (expr_ty)asdl_seq_GET(ge->ifs, i); + expr_ty e = (expr_ty)asdl_seq_GET(gen->ifs, i); VISIT(c, expr, e); - ADDOP_JREL(c, JUMP_IF_FALSE, if_cleanup); + ADDOP_JABS(c, POP_JUMP_IF_FALSE, if_cleanup); NEXT_BLOCK(c); - ADDOP(c, POP_TOP); } if (++gen_index < asdl_seq_LEN(generators)) - if (!compiler_genexp_generator(c, generators, gen_index, elt)) - return 0; + if (!compiler_comprehension_generator(c, + generators, gen_index, + elt, val, type)) + return 0; - /* only append after the last 'for' generator */ + /* only append after the last for generator */ if (gen_index >= asdl_seq_LEN(generators)) { - VISIT(c, expr, elt); - ADDOP(c, YIELD_VALUE); - ADDOP(c, POP_TOP); + /* comprehension specific code */ + switch (type) { + case COMP_GENEXP: + VISIT(c, expr, elt); + ADDOP(c, YIELD_VALUE); + ADDOP(c, POP_TOP); + break; + case COMP_SETCOMP: + VISIT(c, expr, elt); + ADDOP_I(c, SET_ADD, gen_index + 1); + break; + case COMP_DICTCOMP: + /* With 'd[k] = v', v is evaluated before k, so we do + the same. */ + VISIT(c, expr, val); + VISIT(c, expr, elt); + ADDOP_I(c, MAP_ADD, gen_index + 1); + break; + default: + return 0; + } compiler_use_next_block(c, skip); } - for (i = 0; i < n; i++) { - ADDOP_I(c, JUMP_FORWARD, 1); - if (i == 0) - compiler_use_next_block(c, if_cleanup); - - ADDOP(c, POP_TOP); - } + compiler_use_next_block(c, if_cleanup); ADDOP_JABS(c, JUMP_ABSOLUTE, start); compiler_use_next_block(c, anchor); - ADDOP(c, POP_BLOCK); - compiler_pop_fblock(c, LOOP, start); - compiler_use_next_block(c, end); return 1; } static int -compiler_genexp(struct compiler *c, expr_ty e) +compiler_comprehension(struct compiler *c, expr_ty e, int type, identifier name, + asdl_seq *generators, expr_ty elt, expr_ty val) { - static identifier name; - PyCodeObject *co; - expr_ty outermost_iter = ((comprehension_ty) - (asdl_seq_GET(e->v.GeneratorExp.generators, - 0)))->iter; + PyCodeObject *co = NULL; + expr_ty outermost_iter; - if (!name) { - name = PyString_FromString("<genexpr>"); - if (!name) - return 0; - } + outermost_iter = ((comprehension_ty) + asdl_seq_GET(generators, 0))->iter; if (!compiler_enter_scope(c, name, (void *)e, e->lineno)) - return 0; - compiler_genexp_generator(c, e->v.GeneratorExp.generators, 0, - e->v.GeneratorExp.elt); + goto error; + + if (type != COMP_GENEXP) { + int op; + switch (type) { + case COMP_SETCOMP: + op = BUILD_SET; + break; + case COMP_DICTCOMP: + op = BUILD_MAP; + break; + default: + PyErr_Format(PyExc_SystemError, + "unknown comprehension type %d", type); + goto error_in_scope; + } + + ADDOP_I(c, op, 0); + } + + if (!compiler_comprehension_generator(c, generators, 0, elt, + val, type)) + goto error_in_scope; + + if (type != COMP_GENEXP) { + ADDOP(c, RETURN_VALUE); + } + co = assemble(c, 1); compiler_exit_scope(c); if (co == NULL) - return 0; + goto error; - compiler_make_closure(c, co, 0); + if (!compiler_make_closure(c, co, 0)) + goto error; Py_DECREF(co); VISIT(c, expr, outermost_iter); ADDOP(c, GET_ITER); ADDOP_I(c, CALL_FUNCTION, 1); - return 1; +error_in_scope: + compiler_exit_scope(c); +error: + Py_XDECREF(co); + return 0; +} + +static int +compiler_genexp(struct compiler *c, expr_ty e) +{ + static identifier name; + if (!name) { + name = PyString_FromString("<genexpr>"); + if (!name) + return 0; + } + assert(e->kind == GeneratorExp_kind); + return compiler_comprehension(c, e, COMP_GENEXP, name, + e->v.GeneratorExp.generators, + e->v.GeneratorExp.elt, NULL); +} + +static int +compiler_setcomp(struct compiler *c, expr_ty e) +{ + static identifier name; + if (!name) { + name = PyString_FromString("<setcomp>"); + if (!name) + return 0; + } + assert(e->kind == SetComp_kind); + return compiler_comprehension(c, e, COMP_SETCOMP, name, + e->v.SetComp.generators, + e->v.SetComp.elt, NULL); +} + +static int +compiler_dictcomp(struct compiler *c, expr_ty e) +{ + static identifier name; + if (!name) { + name = PyString_FromString("<dictcomp>"); + if (!name) + return 0; + } + assert(e->kind == DictComp_kind); + return compiler_comprehension(c, e, COMP_DICTCOMP, name, + e->v.DictComp.generators, + e->v.DictComp.key, e->v.DictComp.value); } static int @@ -2859,81 +2917,35 @@ expr_constant(expr_ty e) static int compiler_with(struct compiler *c, stmt_ty s) { - static identifier enter_attr, exit_attr; basicblock *block, *finally; - identifier tmpvalue = NULL; assert(s->kind == With_kind); - if (!enter_attr) { - enter_attr = PyString_InternFromString("__enter__"); - if (!enter_attr) - return 0; - } - if (!exit_attr) { - exit_attr = PyString_InternFromString("__exit__"); - if (!exit_attr) - return 0; - } - block = compiler_new_block(c); finally = compiler_new_block(c); if (!block || !finally) return 0; - if (s->v.With.optional_vars) { - /* Create a temporary variable to hold context.__enter__(). - We need to do this rather than preserving it on the stack - because SETUP_FINALLY remembers the stack level. - We need to do the assignment *inside* the try/finally - so that context.__exit__() is called when the assignment - fails. But we need to call context.__enter__() *before* - the try/finally so that if it fails we won't call - context.__exit__(). - */ - tmpvalue = compiler_new_tmpname(c); - if (tmpvalue == NULL) - return 0; - PyArena_AddPyObject(c->c_arena, tmpvalue); - } - /* Evaluate EXPR */ VISIT(c, expr, s->v.With.context_expr); + ADDOP_JREL(c, SETUP_WITH, finally); - /* Squirrel away context.__exit__ by stuffing it under context */ - ADDOP(c, DUP_TOP); - ADDOP_O(c, LOAD_ATTR, exit_attr, names); - ADDOP(c, ROT_TWO); - - /* Call context.__enter__() */ - ADDOP_O(c, LOAD_ATTR, enter_attr, names); - ADDOP_I(c, CALL_FUNCTION, 0); - - if (s->v.With.optional_vars) { - /* Store it in tmpvalue */ - if (!compiler_nameop(c, tmpvalue, Store)) - return 0; - } - else { - /* Discard result from context.__enter__() */ - ADDOP(c, POP_TOP); - } - - /* Start the try block */ - ADDOP_JREL(c, SETUP_FINALLY, finally); - + /* SETUP_WITH pushes a finally block. */ compiler_use_next_block(c, block); + /* Note that the block is actually called SETUP_WITH in ceval.c, but + functions the same as SETUP_FINALLY except that exceptions are + normalized. */ if (!compiler_push_fblock(c, FINALLY_TRY, block)) { return 0; } if (s->v.With.optional_vars) { - /* Bind saved result of context.__enter__() to VAR */ - if (!compiler_nameop(c, tmpvalue, Load) || - !compiler_nameop(c, tmpvalue, Del)) - return 0; VISIT(c, expr, s->v.With.optional_vars); } + else { + /* Discard result from context.__enter__() */ + ADDOP(c, POP_TOP); + } /* BLOCK code */ VISIT_SEQ(c, stmt, s->v.With.body); @@ -2997,8 +3009,17 @@ compiler_visit_expr(struct compiler *c, expr_ty e) ADDOP(c, STORE_MAP); } break; + case Set_kind: + n = asdl_seq_LEN(e->v.Set.elts); + VISIT_SEQ(c, expr, e->v.Set.elts); + ADDOP_I(c, BUILD_SET, n); + break; case ListComp_kind: return compiler_listcomp(c, e); + case SetComp_kind: + return compiler_setcomp(c, e); + case DictComp_kind: + return compiler_dictcomp(c, e); case GeneratorExp_kind: return compiler_genexp(c, e); case Yield_kind: @@ -3423,7 +3444,7 @@ dfs(struct compiler *c, basicblock *b, struct assembler *a) static int stackdepth_walk(struct compiler *c, basicblock *b, int depth, int maxdepth) { - int i; + int i, target_depth; struct instr *instr; if (b->b_seen || b->b_startdepth >= depth) return maxdepth; @@ -3436,8 +3457,17 @@ stackdepth_walk(struct compiler *c, basicblock *b, int depth, int maxdepth) maxdepth = depth; assert(depth >= 0); /* invalid code or bug in stackdepth() */ if (instr->i_jrel || instr->i_jabs) { + target_depth = depth; + if (instr->i_opcode == FOR_ITER) { + target_depth = depth-2; + } else if (instr->i_opcode == SETUP_FINALLY || + instr->i_opcode == SETUP_EXCEPT) { + target_depth = depth+3; + if (target_depth > maxdepth) + maxdepth = target_depth; + } maxdepth = stackdepth_walk(c, instr->i_target, - depth, maxdepth); + target_depth, maxdepth); if (instr->i_opcode == JUMP_ABSOLUTE || instr->i_opcode == JUMP_FORWARD) { goto out; /* remaining code is dead */ @@ -3525,51 +3555,9 @@ blocksize(basicblock *b) return size; } -/* All about a_lnotab. - -c_lnotab is an array of unsigned bytes disguised as a Python string. -It is used to map bytecode offsets to source code line #s (when needed -for tracebacks). - -The array is conceptually a list of - (bytecode offset increment, line number increment) -pairs. The details are important and delicate, best illustrated by example: - - byte code offset source code line number - 0 1 - 6 2 - 50 7 - 350 307 - 361 308 - -The first trick is that these numbers aren't stored, only the increments -from one row to the next (this doesn't really work, but it's a start): - - 0, 1, 6, 1, 44, 5, 300, 300, 11, 1 - -The second trick is that an unsigned byte can't hold negative values, or -values larger than 255, so (a) there's a deep assumption that byte code -offsets and their corresponding line #s both increase monotonically, and (b) -if at least one column jumps by more than 255 from one row to the next, more -than one pair is written to the table. In case #b, there's no way to know -from looking at the table later how many were written. That's the delicate -part. A user of c_lnotab desiring to find the source line number -corresponding to a bytecode address A should do something like this - - lineno = addr = 0 - for addr_incr, line_incr in c_lnotab: - addr += addr_incr - if addr > A: - return lineno - lineno += line_incr - -In order for this to work, when the addr field increments by more than 255, -the line # increment in each pair generated must be 0 until the remaining addr -increment is < 256. So, in the example above, assemble_lnotab (it used -to be called com_set_lineno) should not (as was actually done until 2.2) -expand 300, 300 to 255, 255, 45, 45, - but to 255, 0, 45, 255, 0, 45. -*/ +/* Appends a pair to the end of the line number table, a_lnotab, representing + the instruction's bytecode offset and line number. See + Objects/lnotab_notes.txt for the description of the line number table. */ static int assemble_lnotab(struct assembler *a, struct instr *i) @@ -3711,49 +3699,47 @@ static void assemble_jump_offsets(struct assembler *a, struct compiler *c) { basicblock *b; - int bsize, totsize, extended_arg_count, last_extended_arg_count = 0; + int bsize, totsize, extended_arg_count = 0, last_extended_arg_count; int i; /* Compute the size of each block and fixup jump args. Replace block pointer with position in bytecode. */ -start: - totsize = 0; - for (i = a->a_nblocks - 1; i >= 0; i--) { - b = a->a_postorder[i]; - bsize = blocksize(b); - b->b_offset = totsize; - totsize += bsize; - } - extended_arg_count = 0; - for (b = c->u->u_blocks; b != NULL; b = b->b_list) { - bsize = b->b_offset; - for (i = 0; i < b->b_iused; i++) { - struct instr *instr = &b->b_instr[i]; - /* Relative jumps are computed relative to - the instruction pointer after fetching - the jump instruction. - */ - bsize += instrsize(instr); - if (instr->i_jabs) - instr->i_oparg = instr->i_target->b_offset; - else if (instr->i_jrel) { - int delta = instr->i_target->b_offset - bsize; - instr->i_oparg = delta; + do { + totsize = 0; + for (i = a->a_nblocks - 1; i >= 0; i--) { + b = a->a_postorder[i]; + bsize = blocksize(b); + b->b_offset = totsize; + totsize += bsize; + } + last_extended_arg_count = extended_arg_count; + extended_arg_count = 0; + for (b = c->u->u_blocks; b != NULL; b = b->b_list) { + bsize = b->b_offset; + for (i = 0; i < b->b_iused; i++) { + struct instr *instr = &b->b_instr[i]; + /* Relative jumps are computed relative to + the instruction pointer after fetching + the jump instruction. + */ + bsize += instrsize(instr); + if (instr->i_jabs) + instr->i_oparg = instr->i_target->b_offset; + else if (instr->i_jrel) { + int delta = instr->i_target->b_offset - bsize; + instr->i_oparg = delta; + } + else + continue; + if (instr->i_oparg > 0xffff) + extended_arg_count++; } - else - continue; - if (instr->i_oparg > 0xffff) - extended_arg_count++; } - } /* XXX: This is an awful hack that could hurt performance, but on the bright side it should work until we come up with a better solution. - In the meantime, should the goto be dropped in favor - of a loop? - The issue is that in the first loop blocksize() is called which calls instrsize() which requires i_oparg be set appropriately. There is a bootstrap problem because @@ -3764,10 +3750,7 @@ start: ones in jump instructions. So this should converge fairly quickly. */ - if (last_extended_arg_count != extended_arg_count) { - last_extended_arg_count = extended_arg_count; - goto start; - } + } while (last_extended_arg_count != extended_arg_count); } static PyObject * @@ -3781,6 +3764,8 @@ dict_keys_inorder(PyObject *dict, int offset) return NULL; while (PyDict_Next(dict, &pos, &k, &v)) { i = PyInt_AS_LONG(v); + /* The keys of the dictionary are tuples. (see compiler_add_o) + The object we want is always first, though. */ k = PyTuple_GET_ITEM(k, 0); Py_INCREF(k); assert((i - offset) < size); @@ -3804,13 +3789,11 @@ compute_code_flags(struct compiler *c) flags |= CO_NESTED; if (ste->ste_generator) flags |= CO_GENERATOR; + if (ste->ste_varargs) + flags |= CO_VARARGS; + if (ste->ste_varkeywords) + flags |= CO_VARKEYWORDS; } - if (ste->ste_varargs) - flags |= CO_VARARGS; - if (ste->ste_varkeywords) - flags |= CO_VARKEYWORDS; - if (ste->ste_generator) - flags |= CO_GENERATOR; /* (Only) inherit compilerflags in PyCF_MASK */ flags |= (c->c_flags->cf_flags & PyCF_MASK); diff --git a/Python/dtoa.c b/Python/dtoa.c new file mode 100644 index 0000000000..44dc01f1d5 --- /dev/null +++ b/Python/dtoa.c @@ -0,0 +1,2918 @@ +/**************************************************************** + * + * The author of this software is David M. Gay. + * + * Copyright (c) 1991, 2000, 2001 by Lucent Technologies. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose without fee is hereby granted, provided that this entire notice + * is included in all copies of any software which is or includes a copy + * or modification of this software and in all copies of the supporting + * documentation for such software. + * + * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTY. IN PARTICULAR, NEITHER THE AUTHOR NOR LUCENT MAKES ANY + * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE MERCHANTABILITY + * OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. + * + ***************************************************************/ + +/**************************************************************** + * This is dtoa.c by David M. Gay, downloaded from + * http://www.netlib.org/fp/dtoa.c on April 15, 2009 and modified for + * inclusion into the Python core by Mark E. T. Dickinson and Eric V. Smith. + * + * Please remember to check http://www.netlib.org/fp regularly (and especially + * before any Python release) for bugfixes and updates. + * + * The major modifications from Gay's original code are as follows: + * + * 0. The original code has been specialized to Python's needs by removing + * many of the #ifdef'd sections. In particular, code to support VAX and + * IBM floating-point formats, hex NaNs, hex floats, locale-aware + * treatment of the decimal point, and setting of the inexact flag have + * been removed. + * + * 1. We use PyMem_Malloc and PyMem_Free in place of malloc and free. + * + * 2. The public functions strtod, dtoa and freedtoa all now have + * a _Py_dg_ prefix. + * + * 3. Instead of assuming that PyMem_Malloc always succeeds, we thread + * PyMem_Malloc failures through the code. The functions + * + * Balloc, multadd, s2b, i2b, mult, pow5mult, lshift, diff, d2b + * + * of return type *Bigint all return NULL to indicate a malloc failure. + * Similarly, rv_alloc and nrv_alloc (return type char *) return NULL on + * failure. bigcomp now has return type int (it used to be void) and + * returns -1 on failure and 0 otherwise. _Py_dg_dtoa returns NULL + * on failure. _Py_dg_strtod indicates failure due to malloc failure + * by returning -1.0, setting errno=ENOMEM and *se to s00. + * + * 4. The static variable dtoa_result has been removed. Callers of + * _Py_dg_dtoa are expected to call _Py_dg_freedtoa to free + * the memory allocated by _Py_dg_dtoa. + * + * 5. The code has been reformatted to better fit with Python's + * C style guide (PEP 7). + * + * 6. A bug in the memory allocation has been fixed: to avoid FREEing memory + * that hasn't been MALLOC'ed, private_mem should only be used when k <= + * Kmax. + * + * 7. _Py_dg_strtod has been modified so that it doesn't accept strings with + * leading whitespace. + * + ***************************************************************/ + +/* Please send bug reports for the original dtoa.c code to David M. Gay (dmg + * at acm dot org, with " at " changed at "@" and " dot " changed to "."). + * Please report bugs for this modified version using the Python issue tracker + * (http://bugs.python.org). */ + +/* On a machine with IEEE extended-precision registers, it is + * necessary to specify double-precision (53-bit) rounding precision + * before invoking strtod or dtoa. If the machine uses (the equivalent + * of) Intel 80x87 arithmetic, the call + * _control87(PC_53, MCW_PC); + * does this with many compilers. Whether this or another call is + * appropriate depends on the compiler; for this to work, it may be + * necessary to #include "float.h" or another system-dependent header + * file. + */ + +/* strtod for IEEE-, VAX-, and IBM-arithmetic machines. + * + * This strtod returns a nearest machine number to the input decimal + * string (or sets errno to ERANGE). With IEEE arithmetic, ties are + * broken by the IEEE round-even rule. Otherwise ties are broken by + * biased rounding (add half and chop). + * + * Inspired loosely by William D. Clinger's paper "How to Read Floating + * Point Numbers Accurately" [Proc. ACM SIGPLAN '90, pp. 92-101]. + * + * Modifications: + * + * 1. We only require IEEE, IBM, or VAX double-precision + * arithmetic (not IEEE double-extended). + * 2. We get by with floating-point arithmetic in a case that + * Clinger missed -- when we're computing d * 10^n + * for a small integer d and the integer n is not too + * much larger than 22 (the maximum integer k for which + * we can represent 10^k exactly), we may be able to + * compute (d*10^k) * 10^(e-k) with just one roundoff. + * 3. Rather than a bit-at-a-time adjustment of the binary + * result in the hard case, we use floating-point + * arithmetic to determine the adjustment to within + * one bit; only in really hard cases do we need to + * compute a second residual. + * 4. Because of 3., we don't need a large table of powers of 10 + * for ten-to-e (just some small tables, e.g. of 10^k + * for 0 <= k <= 22). + */ + +/* Linking of Python's #defines to Gay's #defines starts here. */ + +#include "Python.h" + +/* if PY_NO_SHORT_FLOAT_REPR is defined, then don't even try to compile + the following code */ +#ifndef PY_NO_SHORT_FLOAT_REPR + +#include "float.h" + +#define MALLOC PyMem_Malloc +#define FREE PyMem_Free + +/* This code should also work for ARM mixed-endian format on little-endian + machines, where doubles have byte order 45670123 (in increasing address + order, 0 being the least significant byte). */ +#ifdef DOUBLE_IS_LITTLE_ENDIAN_IEEE754 +# define IEEE_8087 +#endif +#if defined(DOUBLE_IS_BIG_ENDIAN_IEEE754) || \ + defined(DOUBLE_IS_ARM_MIXED_ENDIAN_IEEE754) +# define IEEE_MC68k +#endif +#if defined(IEEE_8087) + defined(IEEE_MC68k) != 1 +#error "Exactly one of IEEE_8087 or IEEE_MC68k should be defined." +#endif + +/* The code below assumes that the endianness of integers matches the + endianness of the two 32-bit words of a double. Check this. */ +#if defined(WORDS_BIGENDIAN) && (defined(DOUBLE_IS_LITTLE_ENDIAN_IEEE754) || \ + defined(DOUBLE_IS_ARM_MIXED_ENDIAN_IEEE754)) +#error "doubles and ints have incompatible endianness" +#endif + +#if !defined(WORDS_BIGENDIAN) && defined(DOUBLE_IS_BIG_ENDIAN_IEEE754) +#error "doubles and ints have incompatible endianness" +#endif + + +#if defined(HAVE_UINT32_T) && defined(HAVE_INT32_T) +typedef PY_UINT32_T ULong; +typedef PY_INT32_T Long; +#else +#error "Failed to find an exact-width 32-bit integer type" +#endif + +#if defined(HAVE_UINT64_T) +#define ULLong PY_UINT64_T +#else +#undef ULLong +#endif + +#undef DEBUG +#ifdef Py_DEBUG +#define DEBUG +#endif + +/* End Python #define linking */ + +#ifdef DEBUG +#define Bug(x) {fprintf(stderr, "%s\n", x); exit(1);} +#endif + +#ifndef PRIVATE_MEM +#define PRIVATE_MEM 2304 +#endif +#define PRIVATE_mem ((PRIVATE_MEM+sizeof(double)-1)/sizeof(double)) +static double private_mem[PRIVATE_mem], *pmem_next = private_mem; + +#ifdef __cplusplus +extern "C" { +#endif + +typedef union { double d; ULong L[2]; } U; + +#ifdef IEEE_8087 +#define word0(x) (x)->L[1] +#define word1(x) (x)->L[0] +#else +#define word0(x) (x)->L[0] +#define word1(x) (x)->L[1] +#endif +#define dval(x) (x)->d + +#ifndef STRTOD_DIGLIM +#define STRTOD_DIGLIM 40 +#endif + +/* maximum permitted exponent value for strtod; exponents larger than + MAX_ABS_EXP in absolute value get truncated to +-MAX_ABS_EXP. MAX_ABS_EXP + should fit into an int. */ +#ifndef MAX_ABS_EXP +#define MAX_ABS_EXP 19999U +#endif + +/* The following definition of Storeinc is appropriate for MIPS processors. + * An alternative that might be better on some machines is + * #define Storeinc(a,b,c) (*a++ = b << 16 | c & 0xffff) + */ +#if defined(IEEE_8087) +#define Storeinc(a,b,c) (((unsigned short *)a)[1] = (unsigned short)b, \ + ((unsigned short *)a)[0] = (unsigned short)c, a++) +#else +#define Storeinc(a,b,c) (((unsigned short *)a)[0] = (unsigned short)b, \ + ((unsigned short *)a)[1] = (unsigned short)c, a++) +#endif + +/* #define P DBL_MANT_DIG */ +/* Ten_pmax = floor(P*log(2)/log(5)) */ +/* Bletch = (highest power of 2 < DBL_MAX_10_EXP) / 16 */ +/* Quick_max = floor((P-1)*log(FLT_RADIX)/log(10) - 1) */ +/* Int_max = floor(P*log(FLT_RADIX)/log(10) - 1) */ + +#define Exp_shift 20 +#define Exp_shift1 20 +#define Exp_msk1 0x100000 +#define Exp_msk11 0x100000 +#define Exp_mask 0x7ff00000 +#define P 53 +#define Nbits 53 +#define Bias 1023 +#define Emax 1023 +#define Emin (-1022) +#define Etiny (-1074) /* smallest denormal is 2**Etiny */ +#define Exp_1 0x3ff00000 +#define Exp_11 0x3ff00000 +#define Ebits 11 +#define Frac_mask 0xfffff +#define Frac_mask1 0xfffff +#define Ten_pmax 22 +#define Bletch 0x10 +#define Bndry_mask 0xfffff +#define Bndry_mask1 0xfffff +#define Sign_bit 0x80000000 +#define Log2P 1 +#define Tiny0 0 +#define Tiny1 1 +#define Quick_max 14 +#define Int_max 14 + +#ifndef Flt_Rounds +#ifdef FLT_ROUNDS +#define Flt_Rounds FLT_ROUNDS +#else +#define Flt_Rounds 1 +#endif +#endif /*Flt_Rounds*/ + +#define Rounding Flt_Rounds + +#define Big0 (Frac_mask1 | Exp_msk1*(DBL_MAX_EXP+Bias-1)) +#define Big1 0xffffffff + +/* struct BCinfo is used to pass information from _Py_dg_strtod to bigcomp */ + +typedef struct BCinfo BCinfo; +struct +BCinfo { + int e0, nd, nd0, scale; +}; + +#define FFFFFFFF 0xffffffffUL + +#define Kmax 7 + +/* struct Bigint is used to represent arbitrary-precision integers. These + integers are stored in sign-magnitude format, with the magnitude stored as + an array of base 2**32 digits. Bigints are always normalized: if x is a + Bigint then x->wds >= 1, and either x->wds == 1 or x[wds-1] is nonzero. + + The Bigint fields are as follows: + + - next is a header used by Balloc and Bfree to keep track of lists + of freed Bigints; it's also used for the linked list of + powers of 5 of the form 5**2**i used by pow5mult. + - k indicates which pool this Bigint was allocated from + - maxwds is the maximum number of words space was allocated for + (usually maxwds == 2**k) + - sign is 1 for negative Bigints, 0 for positive. The sign is unused + (ignored on inputs, set to 0 on outputs) in almost all operations + involving Bigints: a notable exception is the diff function, which + ignores signs on inputs but sets the sign of the output correctly. + - wds is the actual number of significant words + - x contains the vector of words (digits) for this Bigint, from least + significant (x[0]) to most significant (x[wds-1]). +*/ + +struct +Bigint { + struct Bigint *next; + int k, maxwds, sign, wds; + ULong x[1]; +}; + +typedef struct Bigint Bigint; + +#ifndef Py_USING_MEMORY_DEBUGGER + +/* Memory management: memory is allocated from, and returned to, Kmax+1 pools + of memory, where pool k (0 <= k <= Kmax) is for Bigints b with b->maxwds == + 1 << k. These pools are maintained as linked lists, with freelist[k] + pointing to the head of the list for pool k. + + On allocation, if there's no free slot in the appropriate pool, MALLOC is + called to get more memory. This memory is not returned to the system until + Python quits. There's also a private memory pool that's allocated from + in preference to using MALLOC. + + For Bigints with more than (1 << Kmax) digits (which implies at least 1233 + decimal digits), memory is directly allocated using MALLOC, and freed using + FREE. + + XXX: it would be easy to bypass this memory-management system and + translate each call to Balloc into a call to PyMem_Malloc, and each + Bfree to PyMem_Free. Investigate whether this has any significant + performance on impact. */ + +static Bigint *freelist[Kmax+1]; + +/* Allocate space for a Bigint with up to 1<<k digits */ + +static Bigint * +Balloc(int k) +{ + int x; + Bigint *rv; + unsigned int len; + + if (k <= Kmax && (rv = freelist[k])) + freelist[k] = rv->next; + else { + x = 1 << k; + len = (sizeof(Bigint) + (x-1)*sizeof(ULong) + sizeof(double) - 1) + /sizeof(double); + if (k <= Kmax && pmem_next - private_mem + len <= PRIVATE_mem) { + rv = (Bigint*)pmem_next; + pmem_next += len; + } + else { + rv = (Bigint*)MALLOC(len*sizeof(double)); + if (rv == NULL) + return NULL; + } + rv->k = k; + rv->maxwds = x; + } + rv->sign = rv->wds = 0; + return rv; +} + +/* Free a Bigint allocated with Balloc */ + +static void +Bfree(Bigint *v) +{ + if (v) { + if (v->k > Kmax) + FREE((void*)v); + else { + v->next = freelist[v->k]; + freelist[v->k] = v; + } + } +} + +#else + +/* Alternative versions of Balloc and Bfree that use PyMem_Malloc and + PyMem_Free directly in place of the custom memory allocation scheme above. + These are provided for the benefit of memory debugging tools like + Valgrind. */ + +/* Allocate space for a Bigint with up to 1<<k digits */ + +static Bigint * +Balloc(int k) +{ + int x; + Bigint *rv; + unsigned int len; + + x = 1 << k; + len = (sizeof(Bigint) + (x-1)*sizeof(ULong) + sizeof(double) - 1) + /sizeof(double); + + rv = (Bigint*)MALLOC(len*sizeof(double)); + if (rv == NULL) + return NULL; + + rv->k = k; + rv->maxwds = x; + rv->sign = rv->wds = 0; + return rv; +} + +/* Free a Bigint allocated with Balloc */ + +static void +Bfree(Bigint *v) +{ + if (v) { + FREE((void*)v); + } +} + +#endif /* Py_USING_MEMORY_DEBUGGER */ + +#define Bcopy(x,y) memcpy((char *)&x->sign, (char *)&y->sign, \ + y->wds*sizeof(Long) + 2*sizeof(int)) + +/* Multiply a Bigint b by m and add a. Either modifies b in place and returns + a pointer to the modified b, or Bfrees b and returns a pointer to a copy. + On failure, return NULL. In this case, b will have been already freed. */ + +static Bigint * +multadd(Bigint *b, int m, int a) /* multiply by m and add a */ +{ + int i, wds; +#ifdef ULLong + ULong *x; + ULLong carry, y; +#else + ULong carry, *x, y; + ULong xi, z; +#endif + Bigint *b1; + + wds = b->wds; + x = b->x; + i = 0; + carry = a; + do { +#ifdef ULLong + y = *x * (ULLong)m + carry; + carry = y >> 32; + *x++ = (ULong)(y & FFFFFFFF); +#else + xi = *x; + y = (xi & 0xffff) * m + carry; + z = (xi >> 16) * m + (y >> 16); + carry = z >> 16; + *x++ = (z << 16) + (y & 0xffff); +#endif + } + while(++i < wds); + if (carry) { + if (wds >= b->maxwds) { + b1 = Balloc(b->k+1); + if (b1 == NULL){ + Bfree(b); + return NULL; + } + Bcopy(b1, b); + Bfree(b); + b = b1; + } + b->x[wds++] = (ULong)carry; + b->wds = wds; + } + return b; +} + +/* convert a string s containing nd decimal digits (possibly containing a + decimal separator at position nd0, which is ignored) to a Bigint. This + function carries on where the parsing code in _Py_dg_strtod leaves off: on + entry, y9 contains the result of converting the first 9 digits. Returns + NULL on failure. */ + +static Bigint * +s2b(const char *s, int nd0, int nd, ULong y9) +{ + Bigint *b; + int i, k; + Long x, y; + + x = (nd + 8) / 9; + for(k = 0, y = 1; x > y; y <<= 1, k++) ; + b = Balloc(k); + if (b == NULL) + return NULL; + b->x[0] = y9; + b->wds = 1; + + if (nd <= 9) + return b; + + s += 9; + for (i = 9; i < nd0; i++) { + b = multadd(b, 10, *s++ - '0'); + if (b == NULL) + return NULL; + } + s++; + for(; i < nd; i++) { + b = multadd(b, 10, *s++ - '0'); + if (b == NULL) + return NULL; + } + return b; +} + +/* count leading 0 bits in the 32-bit integer x. */ + +static int +hi0bits(ULong x) +{ + int k = 0; + + if (!(x & 0xffff0000)) { + k = 16; + x <<= 16; + } + if (!(x & 0xff000000)) { + k += 8; + x <<= 8; + } + if (!(x & 0xf0000000)) { + k += 4; + x <<= 4; + } + if (!(x & 0xc0000000)) { + k += 2; + x <<= 2; + } + if (!(x & 0x80000000)) { + k++; + if (!(x & 0x40000000)) + return 32; + } + return k; +} + +/* count trailing 0 bits in the 32-bit integer y, and shift y right by that + number of bits. */ + +static int +lo0bits(ULong *y) +{ + int k; + ULong x = *y; + + if (x & 7) { + if (x & 1) + return 0; + if (x & 2) { + *y = x >> 1; + return 1; + } + *y = x >> 2; + return 2; + } + k = 0; + if (!(x & 0xffff)) { + k = 16; + x >>= 16; + } + if (!(x & 0xff)) { + k += 8; + x >>= 8; + } + if (!(x & 0xf)) { + k += 4; + x >>= 4; + } + if (!(x & 0x3)) { + k += 2; + x >>= 2; + } + if (!(x & 1)) { + k++; + x >>= 1; + if (!x) + return 32; + } + *y = x; + return k; +} + +/* convert a small nonnegative integer to a Bigint */ + +static Bigint * +i2b(int i) +{ + Bigint *b; + + b = Balloc(1); + if (b == NULL) + return NULL; + b->x[0] = i; + b->wds = 1; + return b; +} + +/* multiply two Bigints. Returns a new Bigint, or NULL on failure. Ignores + the signs of a and b. */ + +static Bigint * +mult(Bigint *a, Bigint *b) +{ + Bigint *c; + int k, wa, wb, wc; + ULong *x, *xa, *xae, *xb, *xbe, *xc, *xc0; + ULong y; +#ifdef ULLong + ULLong carry, z; +#else + ULong carry, z; + ULong z2; +#endif + + if ((!a->x[0] && a->wds == 1) || (!b->x[0] && b->wds == 1)) { + c = Balloc(0); + if (c == NULL) + return NULL; + c->wds = 1; + c->x[0] = 0; + return c; + } + + if (a->wds < b->wds) { + c = a; + a = b; + b = c; + } + k = a->k; + wa = a->wds; + wb = b->wds; + wc = wa + wb; + if (wc > a->maxwds) + k++; + c = Balloc(k); + if (c == NULL) + return NULL; + for(x = c->x, xa = x + wc; x < xa; x++) + *x = 0; + xa = a->x; + xae = xa + wa; + xb = b->x; + xbe = xb + wb; + xc0 = c->x; +#ifdef ULLong + for(; xb < xbe; xc0++) { + if ((y = *xb++)) { + x = xa; + xc = xc0; + carry = 0; + do { + z = *x++ * (ULLong)y + *xc + carry; + carry = z >> 32; + *xc++ = (ULong)(z & FFFFFFFF); + } + while(x < xae); + *xc = (ULong)carry; + } + } +#else + for(; xb < xbe; xb++, xc0++) { + if (y = *xb & 0xffff) { + x = xa; + xc = xc0; + carry = 0; + do { + z = (*x & 0xffff) * y + (*xc & 0xffff) + carry; + carry = z >> 16; + z2 = (*x++ >> 16) * y + (*xc >> 16) + carry; + carry = z2 >> 16; + Storeinc(xc, z2, z); + } + while(x < xae); + *xc = carry; + } + if (y = *xb >> 16) { + x = xa; + xc = xc0; + carry = 0; + z2 = *xc; + do { + z = (*x & 0xffff) * y + (*xc >> 16) + carry; + carry = z >> 16; + Storeinc(xc, z, z2); + z2 = (*x++ >> 16) * y + (*xc & 0xffff) + carry; + carry = z2 >> 16; + } + while(x < xae); + *xc = z2; + } + } +#endif + for(xc0 = c->x, xc = xc0 + wc; wc > 0 && !*--xc; --wc) ; + c->wds = wc; + return c; +} + +#ifndef Py_USING_MEMORY_DEBUGGER + +/* p5s is a linked list of powers of 5 of the form 5**(2**i), i >= 2 */ + +static Bigint *p5s; + +/* multiply the Bigint b by 5**k. Returns a pointer to the result, or NULL on + failure; if the returned pointer is distinct from b then the original + Bigint b will have been Bfree'd. Ignores the sign of b. */ + +static Bigint * +pow5mult(Bigint *b, int k) +{ + Bigint *b1, *p5, *p51; + int i; + static int p05[3] = { 5, 25, 125 }; + + if ((i = k & 3)) { + b = multadd(b, p05[i-1], 0); + if (b == NULL) + return NULL; + } + + if (!(k >>= 2)) + return b; + p5 = p5s; + if (!p5) { + /* first time */ + p5 = i2b(625); + if (p5 == NULL) { + Bfree(b); + return NULL; + } + p5s = p5; + p5->next = 0; + } + for(;;) { + if (k & 1) { + b1 = mult(b, p5); + Bfree(b); + b = b1; + if (b == NULL) + return NULL; + } + if (!(k >>= 1)) + break; + p51 = p5->next; + if (!p51) { + p51 = mult(p5,p5); + if (p51 == NULL) { + Bfree(b); + return NULL; + } + p51->next = 0; + p5->next = p51; + } + p5 = p51; + } + return b; +} + +#else + +/* Version of pow5mult that doesn't cache powers of 5. Provided for + the benefit of memory debugging tools like Valgrind. */ + +static Bigint * +pow5mult(Bigint *b, int k) +{ + Bigint *b1, *p5, *p51; + int i; + static int p05[3] = { 5, 25, 125 }; + + if ((i = k & 3)) { + b = multadd(b, p05[i-1], 0); + if (b == NULL) + return NULL; + } + + if (!(k >>= 2)) + return b; + p5 = i2b(625); + if (p5 == NULL) { + Bfree(b); + return NULL; + } + + for(;;) { + if (k & 1) { + b1 = mult(b, p5); + Bfree(b); + b = b1; + if (b == NULL) { + Bfree(p5); + return NULL; + } + } + if (!(k >>= 1)) + break; + p51 = mult(p5, p5); + Bfree(p5); + p5 = p51; + if (p5 == NULL) { + Bfree(b); + return NULL; + } + } + Bfree(p5); + return b; +} + +#endif /* Py_USING_MEMORY_DEBUGGER */ + +/* shift a Bigint b left by k bits. Return a pointer to the shifted result, + or NULL on failure. If the returned pointer is distinct from b then the + original b will have been Bfree'd. Ignores the sign of b. */ + +static Bigint * +lshift(Bigint *b, int k) +{ + int i, k1, n, n1; + Bigint *b1; + ULong *x, *x1, *xe, z; + + if (!k || (!b->x[0] && b->wds == 1)) + return b; + + n = k >> 5; + k1 = b->k; + n1 = n + b->wds + 1; + for(i = b->maxwds; n1 > i; i <<= 1) + k1++; + b1 = Balloc(k1); + if (b1 == NULL) { + Bfree(b); + return NULL; + } + x1 = b1->x; + for(i = 0; i < n; i++) + *x1++ = 0; + x = b->x; + xe = x + b->wds; + if (k &= 0x1f) { + k1 = 32 - k; + z = 0; + do { + *x1++ = *x << k | z; + z = *x++ >> k1; + } + while(x < xe); + if ((*x1 = z)) + ++n1; + } + else do + *x1++ = *x++; + while(x < xe); + b1->wds = n1 - 1; + Bfree(b); + return b1; +} + +/* Do a three-way compare of a and b, returning -1 if a < b, 0 if a == b and + 1 if a > b. Ignores signs of a and b. */ + +static int +cmp(Bigint *a, Bigint *b) +{ + ULong *xa, *xa0, *xb, *xb0; + int i, j; + + i = a->wds; + j = b->wds; +#ifdef DEBUG + if (i > 1 && !a->x[i-1]) + Bug("cmp called with a->x[a->wds-1] == 0"); + if (j > 1 && !b->x[j-1]) + Bug("cmp called with b->x[b->wds-1] == 0"); +#endif + if (i -= j) + return i; + xa0 = a->x; + xa = xa0 + j; + xb0 = b->x; + xb = xb0 + j; + for(;;) { + if (*--xa != *--xb) + return *xa < *xb ? -1 : 1; + if (xa <= xa0) + break; + } + return 0; +} + +/* Take the difference of Bigints a and b, returning a new Bigint. Returns + NULL on failure. The signs of a and b are ignored, but the sign of the + result is set appropriately. */ + +static Bigint * +diff(Bigint *a, Bigint *b) +{ + Bigint *c; + int i, wa, wb; + ULong *xa, *xae, *xb, *xbe, *xc; +#ifdef ULLong + ULLong borrow, y; +#else + ULong borrow, y; + ULong z; +#endif + + i = cmp(a,b); + if (!i) { + c = Balloc(0); + if (c == NULL) + return NULL; + c->wds = 1; + c->x[0] = 0; + return c; + } + if (i < 0) { + c = a; + a = b; + b = c; + i = 1; + } + else + i = 0; + c = Balloc(a->k); + if (c == NULL) + return NULL; + c->sign = i; + wa = a->wds; + xa = a->x; + xae = xa + wa; + wb = b->wds; + xb = b->x; + xbe = xb + wb; + xc = c->x; + borrow = 0; +#ifdef ULLong + do { + y = (ULLong)*xa++ - *xb++ - borrow; + borrow = y >> 32 & (ULong)1; + *xc++ = (ULong)(y & FFFFFFFF); + } + while(xb < xbe); + while(xa < xae) { + y = *xa++ - borrow; + borrow = y >> 32 & (ULong)1; + *xc++ = (ULong)(y & FFFFFFFF); + } +#else + do { + y = (*xa & 0xffff) - (*xb & 0xffff) - borrow; + borrow = (y & 0x10000) >> 16; + z = (*xa++ >> 16) - (*xb++ >> 16) - borrow; + borrow = (z & 0x10000) >> 16; + Storeinc(xc, z, y); + } + while(xb < xbe); + while(xa < xae) { + y = (*xa & 0xffff) - borrow; + borrow = (y & 0x10000) >> 16; + z = (*xa++ >> 16) - borrow; + borrow = (z & 0x10000) >> 16; + Storeinc(xc, z, y); + } +#endif + while(!*--xc) + wa--; + c->wds = wa; + return c; +} + +/* Given a positive normal double x, return the difference between x and the + next double up. Doesn't give correct results for subnormals. */ + +static double +ulp(U *x) +{ + Long L; + U u; + + L = (word0(x) & Exp_mask) - (P-1)*Exp_msk1; + word0(&u) = L; + word1(&u) = 0; + return dval(&u); +} + +/* Convert a Bigint to a double plus an exponent */ + +static double +b2d(Bigint *a, int *e) +{ + ULong *xa, *xa0, w, y, z; + int k; + U d; + + xa0 = a->x; + xa = xa0 + a->wds; + y = *--xa; +#ifdef DEBUG + if (!y) Bug("zero y in b2d"); +#endif + k = hi0bits(y); + *e = 32 - k; + if (k < Ebits) { + word0(&d) = Exp_1 | y >> (Ebits - k); + w = xa > xa0 ? *--xa : 0; + word1(&d) = y << ((32-Ebits) + k) | w >> (Ebits - k); + goto ret_d; + } + z = xa > xa0 ? *--xa : 0; + if (k -= Ebits) { + word0(&d) = Exp_1 | y << k | z >> (32 - k); + y = xa > xa0 ? *--xa : 0; + word1(&d) = z << k | y >> (32 - k); + } + else { + word0(&d) = Exp_1 | y; + word1(&d) = z; + } + ret_d: + return dval(&d); +} + +/* Convert a scaled double to a Bigint plus an exponent. Similar to d2b, + except that it accepts the scale parameter used in _Py_dg_strtod (which + should be either 0 or 2*P), and the normalization for the return value is + different (see below). On input, d should be finite and nonnegative, and d + / 2**scale should be exactly representable as an IEEE 754 double. + + Returns a Bigint b and an integer e such that + + dval(d) / 2**scale = b * 2**e. + + Unlike d2b, b is not necessarily odd: b and e are normalized so + that either 2**(P-1) <= b < 2**P and e >= Etiny, or b < 2**P + and e == Etiny. This applies equally to an input of 0.0: in that + case the return values are b = 0 and e = Etiny. + + The above normalization ensures that for all possible inputs d, + 2**e gives ulp(d/2**scale). + + Returns NULL on failure. +*/ + +static Bigint * +sd2b(U *d, int scale, int *e) +{ + Bigint *b; + + b = Balloc(1); + if (b == NULL) + return NULL; + + /* First construct b and e assuming that scale == 0. */ + b->wds = 2; + b->x[0] = word1(d); + b->x[1] = word0(d) & Frac_mask; + *e = Etiny - 1 + (int)((word0(d) & Exp_mask) >> Exp_shift); + if (*e < Etiny) + *e = Etiny; + else + b->x[1] |= Exp_msk1; + + /* Now adjust for scale, provided that b != 0. */ + if (scale && (b->x[0] || b->x[1])) { + *e -= scale; + if (*e < Etiny) { + scale = Etiny - *e; + *e = Etiny; + /* We can't shift more than P-1 bits without shifting out a 1. */ + assert(0 < scale && scale <= P - 1); + if (scale >= 32) { + /* The bits shifted out should all be zero. */ + assert(b->x[0] == 0); + b->x[0] = b->x[1]; + b->x[1] = 0; + scale -= 32; + } + if (scale) { + /* The bits shifted out should all be zero. */ + assert(b->x[0] << (32 - scale) == 0); + b->x[0] = (b->x[0] >> scale) | (b->x[1] << (32 - scale)); + b->x[1] >>= scale; + } + } + } + /* Ensure b is normalized. */ + if (!b->x[1]) + b->wds = 1; + + return b; +} + +/* Convert a double to a Bigint plus an exponent. Return NULL on failure. + + Given a finite nonzero double d, return an odd Bigint b and exponent *e + such that fabs(d) = b * 2**e. On return, *bbits gives the number of + significant bits of b; that is, 2**(*bbits-1) <= b < 2**(*bbits). + + If d is zero, then b == 0, *e == -1010, *bbits = 0. + */ + +static Bigint * +d2b(U *d, int *e, int *bits) +{ + Bigint *b; + int de, k; + ULong *x, y, z; + int i; + + b = Balloc(1); + if (b == NULL) + return NULL; + x = b->x; + + z = word0(d) & Frac_mask; + word0(d) &= 0x7fffffff; /* clear sign bit, which we ignore */ + if ((de = (int)(word0(d) >> Exp_shift))) + z |= Exp_msk1; + if ((y = word1(d))) { + if ((k = lo0bits(&y))) { + x[0] = y | z << (32 - k); + z >>= k; + } + else + x[0] = y; + i = + b->wds = (x[1] = z) ? 2 : 1; + } + else { + k = lo0bits(&z); + x[0] = z; + i = + b->wds = 1; + k += 32; + } + if (de) { + *e = de - Bias - (P-1) + k; + *bits = P - k; + } + else { + *e = de - Bias - (P-1) + 1 + k; + *bits = 32*i - hi0bits(x[i-1]); + } + return b; +} + +/* Compute the ratio of two Bigints, as a double. The result may have an + error of up to 2.5 ulps. */ + +static double +ratio(Bigint *a, Bigint *b) +{ + U da, db; + int k, ka, kb; + + dval(&da) = b2d(a, &ka); + dval(&db) = b2d(b, &kb); + k = ka - kb + 32*(a->wds - b->wds); + if (k > 0) + word0(&da) += k*Exp_msk1; + else { + k = -k; + word0(&db) += k*Exp_msk1; + } + return dval(&da) / dval(&db); +} + +static const double +tens[] = { + 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, + 1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, + 1e20, 1e21, 1e22 +}; + +static const double +bigtens[] = { 1e16, 1e32, 1e64, 1e128, 1e256 }; +static const double tinytens[] = { 1e-16, 1e-32, 1e-64, 1e-128, + 9007199254740992.*9007199254740992.e-256 + /* = 2^106 * 1e-256 */ +}; +/* The factor of 2^53 in tinytens[4] helps us avoid setting the underflow */ +/* flag unnecessarily. It leads to a song and dance at the end of strtod. */ +#define Scale_Bit 0x10 +#define n_bigtens 5 + +#define ULbits 32 +#define kshift 5 +#define kmask 31 + + +static int +dshift(Bigint *b, int p2) +{ + int rv = hi0bits(b->x[b->wds-1]) - 4; + if (p2 > 0) + rv -= p2; + return rv & kmask; +} + +/* special case of Bigint division. The quotient is always in the range 0 <= + quotient < 10, and on entry the divisor S is normalized so that its top 4 + bits (28--31) are zero and bit 27 is set. */ + +static int +quorem(Bigint *b, Bigint *S) +{ + int n; + ULong *bx, *bxe, q, *sx, *sxe; +#ifdef ULLong + ULLong borrow, carry, y, ys; +#else + ULong borrow, carry, y, ys; + ULong si, z, zs; +#endif + + n = S->wds; +#ifdef DEBUG + /*debug*/ if (b->wds > n) + /*debug*/ Bug("oversize b in quorem"); +#endif + if (b->wds < n) + return 0; + sx = S->x; + sxe = sx + --n; + bx = b->x; + bxe = bx + n; + q = *bxe / (*sxe + 1); /* ensure q <= true quotient */ +#ifdef DEBUG + /*debug*/ if (q > 9) + /*debug*/ Bug("oversized quotient in quorem"); +#endif + if (q) { + borrow = 0; + carry = 0; + do { +#ifdef ULLong + ys = *sx++ * (ULLong)q + carry; + carry = ys >> 32; + y = *bx - (ys & FFFFFFFF) - borrow; + borrow = y >> 32 & (ULong)1; + *bx++ = (ULong)(y & FFFFFFFF); +#else + si = *sx++; + ys = (si & 0xffff) * q + carry; + zs = (si >> 16) * q + (ys >> 16); + carry = zs >> 16; + y = (*bx & 0xffff) - (ys & 0xffff) - borrow; + borrow = (y & 0x10000) >> 16; + z = (*bx >> 16) - (zs & 0xffff) - borrow; + borrow = (z & 0x10000) >> 16; + Storeinc(bx, z, y); +#endif + } + while(sx <= sxe); + if (!*bxe) { + bx = b->x; + while(--bxe > bx && !*bxe) + --n; + b->wds = n; + } + } + if (cmp(b, S) >= 0) { + q++; + borrow = 0; + carry = 0; + bx = b->x; + sx = S->x; + do { +#ifdef ULLong + ys = *sx++ + carry; + carry = ys >> 32; + y = *bx - (ys & FFFFFFFF) - borrow; + borrow = y >> 32 & (ULong)1; + *bx++ = (ULong)(y & FFFFFFFF); +#else + si = *sx++; + ys = (si & 0xffff) + carry; + zs = (si >> 16) + (ys >> 16); + carry = zs >> 16; + y = (*bx & 0xffff) - (ys & 0xffff) - borrow; + borrow = (y & 0x10000) >> 16; + z = (*bx >> 16) - (zs & 0xffff) - borrow; + borrow = (z & 0x10000) >> 16; + Storeinc(bx, z, y); +#endif + } + while(sx <= sxe); + bx = b->x; + bxe = bx + n; + if (!*bxe) { + while(--bxe > bx && !*bxe) + --n; + b->wds = n; + } + } + return q; +} + +/* sulp(x) is a version of ulp(x) that takes bc.scale into account. + + Assuming that x is finite and nonnegative (positive zero is fine + here) and x / 2^bc.scale is exactly representable as a double, + sulp(x) is equivalent to 2^bc.scale * ulp(x / 2^bc.scale). */ + +static double +sulp(U *x, BCinfo *bc) +{ + U u; + + if (bc->scale && 2*P + 1 > (int)((word0(x) & Exp_mask) >> Exp_shift)) { + /* rv/2^bc->scale is subnormal */ + word0(&u) = (P+2)*Exp_msk1; + word1(&u) = 0; + return u.d; + } + else { + assert(word0(x) || word1(x)); /* x != 0.0 */ + return ulp(x); + } +} + +/* The bigcomp function handles some hard cases for strtod, for inputs + with more than STRTOD_DIGLIM digits. It's called once an initial + estimate for the double corresponding to the input string has + already been obtained by the code in _Py_dg_strtod. + + The bigcomp function is only called after _Py_dg_strtod has found a + double value rv such that either rv or rv + 1ulp represents the + correctly rounded value corresponding to the original string. It + determines which of these two values is the correct one by + computing the decimal digits of rv + 0.5ulp and comparing them with + the corresponding digits of s0. + + In the following, write dv for the absolute value of the number represented + by the input string. + + Inputs: + + s0 points to the first significant digit of the input string. + + rv is a (possibly scaled) estimate for the closest double value to the + value represented by the original input to _Py_dg_strtod. If + bc->scale is nonzero, then rv/2^(bc->scale) is the approximation to + the input value. + + bc is a struct containing information gathered during the parsing and + estimation steps of _Py_dg_strtod. Description of fields follows: + + bc->e0 gives the exponent of the input value, such that dv = (integer + given by the bd->nd digits of s0) * 10**e0 + + bc->nd gives the total number of significant digits of s0. It will + be at least 1. + + bc->nd0 gives the number of significant digits of s0 before the + decimal separator. If there's no decimal separator, bc->nd0 == + bc->nd. + + bc->scale is the value used to scale rv to avoid doing arithmetic with + subnormal values. It's either 0 or 2*P (=106). + + Outputs: + + On successful exit, rv/2^(bc->scale) is the closest double to dv. + + Returns 0 on success, -1 on failure (e.g., due to a failed malloc call). */ + +static int +bigcomp(U *rv, const char *s0, BCinfo *bc) +{ + Bigint *b, *d; + int b2, d2, dd, i, nd, nd0, odd, p2, p5; + + nd = bc->nd; + nd0 = bc->nd0; + p5 = nd + bc->e0; + b = sd2b(rv, bc->scale, &p2); + if (b == NULL) + return -1; + + /* record whether the lsb of rv/2^(bc->scale) is odd: in the exact halfway + case, this is used for round to even. */ + odd = b->x[0] & 1; + + /* left shift b by 1 bit and or a 1 into the least significant bit; + this gives us b * 2**p2 = rv/2^(bc->scale) + 0.5 ulp. */ + b = lshift(b, 1); + if (b == NULL) + return -1; + b->x[0] |= 1; + p2--; + + p2 -= p5; + d = i2b(1); + if (d == NULL) { + Bfree(b); + return -1; + } + /* Arrange for convenient computation of quotients: + * shift left if necessary so divisor has 4 leading 0 bits. + */ + if (p5 > 0) { + d = pow5mult(d, p5); + if (d == NULL) { + Bfree(b); + return -1; + } + } + else if (p5 < 0) { + b = pow5mult(b, -p5); + if (b == NULL) { + Bfree(d); + return -1; + } + } + if (p2 > 0) { + b2 = p2; + d2 = 0; + } + else { + b2 = 0; + d2 = -p2; + } + i = dshift(d, d2); + if ((b2 += i) > 0) { + b = lshift(b, b2); + if (b == NULL) { + Bfree(d); + return -1; + } + } + if ((d2 += i) > 0) { + d = lshift(d, d2); + if (d == NULL) { + Bfree(b); + return -1; + } + } + + /* Compare s0 with b/d: set dd to -1, 0, or 1 according as s0 < b/d, s0 == + * b/d, or s0 > b/d. Here the digits of s0 are thought of as representing + * a number in the range [0.1, 1). */ + if (cmp(b, d) >= 0) + /* b/d >= 1 */ + dd = -1; + else { + i = 0; + for(;;) { + b = multadd(b, 10, 0); + if (b == NULL) { + Bfree(d); + return -1; + } + dd = s0[i < nd0 ? i : i+1] - '0' - quorem(b, d); + i++; + + if (dd) + break; + if (!b->x[0] && b->wds == 1) { + /* b/d == 0 */ + dd = i < nd; + break; + } + if (!(i < nd)) { + /* b/d != 0, but digits of s0 exhausted */ + dd = -1; + break; + } + } + } + Bfree(b); + Bfree(d); + if (dd > 0 || (dd == 0 && odd)) + dval(rv) += sulp(rv, bc); + return 0; +} + +double +_Py_dg_strtod(const char *s00, char **se) +{ + int bb2, bb5, bbe, bd2, bd5, bs2, c, dsign, e, e1, error; + int esign, i, j, k, lz, nd, nd0, odd, sign; + const char *s, *s0, *s1; + double aadj, aadj1; + U aadj2, adj, rv, rv0; + ULong y, z, abs_exp; + Long L; + BCinfo bc; + Bigint *bb, *bb1, *bd, *bd0, *bs, *delta; + + dval(&rv) = 0.; + + /* Start parsing. */ + c = *(s = s00); + + /* Parse optional sign, if present. */ + sign = 0; + switch (c) { + case '-': + sign = 1; + /* no break */ + case '+': + c = *++s; + } + + /* Skip leading zeros: lz is true iff there were leading zeros. */ + s1 = s; + while (c == '0') + c = *++s; + lz = s != s1; + + /* Point s0 at the first nonzero digit (if any). nd0 will be the position + of the point relative to s0. nd will be the total number of digits + ignoring leading zeros. */ + s0 = s1 = s; + while ('0' <= c && c <= '9') + c = *++s; + nd0 = nd = s - s1; + + /* Parse decimal point and following digits. */ + if (c == '.') { + c = *++s; + if (!nd) { + s1 = s; + while (c == '0') + c = *++s; + lz = lz || s != s1; + nd0 -= s - s1; + s0 = s; + } + s1 = s; + while ('0' <= c && c <= '9') + c = *++s; + nd += s - s1; + } + + /* Now lz is true if and only if there were leading zero digits, and nd + gives the total number of digits ignoring leading zeros. A valid input + must have at least one digit. */ + if (!nd && !lz) { + if (se) + *se = (char *)s00; + goto parse_error; + } + + /* Parse exponent. */ + e = 0; + if (c == 'e' || c == 'E') { + s00 = s; + c = *++s; + + /* Exponent sign. */ + esign = 0; + switch (c) { + case '-': + esign = 1; + /* no break */ + case '+': + c = *++s; + } + + /* Skip zeros. lz is true iff there are leading zeros. */ + s1 = s; + while (c == '0') + c = *++s; + lz = s != s1; + + /* Get absolute value of the exponent. */ + s1 = s; + abs_exp = 0; + while ('0' <= c && c <= '9') { + abs_exp = 10*abs_exp + (c - '0'); + c = *++s; + } + + /* abs_exp will be correct modulo 2**32. But 10**9 < 2**32, so if + there are at most 9 significant exponent digits then overflow is + impossible. */ + if (s - s1 > 9 || abs_exp > MAX_ABS_EXP) + e = (int)MAX_ABS_EXP; + else + e = (int)abs_exp; + if (esign) + e = -e; + + /* A valid exponent must have at least one digit. */ + if (s == s1 && !lz) + s = s00; + } + + /* Adjust exponent to take into account position of the point. */ + e -= nd - nd0; + if (nd0 <= 0) + nd0 = nd; + + /* Finished parsing. Set se to indicate how far we parsed */ + if (se) + *se = (char *)s; + + /* If all digits were zero, exit with return value +-0.0. Otherwise, + strip trailing zeros: scan back until we hit a nonzero digit. */ + if (!nd) + goto ret; + for (i = nd; i > 0; ) { + --i; + if (s0[i < nd0 ? i : i+1] != '0') { + ++i; + break; + } + } + e += nd - i; + nd = i; + if (nd0 > nd) + nd0 = nd; + + /* Summary of parsing results. After parsing, and dealing with zero + * inputs, we have values s0, nd0, nd, e, sign, where: + * + * - s0 points to the first significant digit of the input string + * + * - nd is the total number of significant digits (here, and + * below, 'significant digits' means the set of digits of the + * significand of the input that remain after ignoring leading + * and trailing zeros). + * + * - nd0 indicates the position of the decimal point, if present; it + * satisfies 1 <= nd0 <= nd. The nd significant digits are in + * s0[0:nd0] and s0[nd0+1:nd+1] using the usual Python half-open slice + * notation. (If nd0 < nd, then s0[nd0] contains a '.' character; if + * nd0 == nd, then s0[nd0] could be any non-digit character.) + * + * - e is the adjusted exponent: the absolute value of the number + * represented by the original input string is n * 10**e, where + * n is the integer represented by the concatenation of + * s0[0:nd0] and s0[nd0+1:nd+1] + * + * - sign gives the sign of the input: 1 for negative, 0 for positive + * + * - the first and last significant digits are nonzero + */ + + /* put first DBL_DIG+1 digits into integer y and z. + * + * - y contains the value represented by the first min(9, nd) + * significant digits + * + * - if nd > 9, z contains the value represented by significant digits + * with indices in [9, min(16, nd)). So y * 10**(min(16, nd) - 9) + z + * gives the value represented by the first min(16, nd) sig. digits. + */ + + bc.e0 = e1 = e; + y = z = 0; + for (i = 0; i < nd; i++) { + if (i < 9) + y = 10*y + s0[i < nd0 ? i : i+1] - '0'; + else if (i < DBL_DIG+1) + z = 10*z + s0[i < nd0 ? i : i+1] - '0'; + else + break; + } + + k = nd < DBL_DIG + 1 ? nd : DBL_DIG + 1; + dval(&rv) = y; + if (k > 9) { + dval(&rv) = tens[k - 9] * dval(&rv) + z; + } + bd0 = 0; + if (nd <= DBL_DIG + && Flt_Rounds == 1 + ) { + if (!e) + goto ret; + if (e > 0) { + if (e <= Ten_pmax) { + dval(&rv) *= tens[e]; + goto ret; + } + i = DBL_DIG - nd; + if (e <= Ten_pmax + i) { + /* A fancier test would sometimes let us do + * this for larger i values. + */ + e -= i; + dval(&rv) *= tens[i]; + dval(&rv) *= tens[e]; + goto ret; + } + } + else if (e >= -Ten_pmax) { + dval(&rv) /= tens[-e]; + goto ret; + } + } + e1 += nd - k; + + bc.scale = 0; + + /* Get starting approximation = rv * 10**e1 */ + + if (e1 > 0) { + if ((i = e1 & 15)) + dval(&rv) *= tens[i]; + if (e1 &= ~15) { + if (e1 > DBL_MAX_10_EXP) + goto ovfl; + e1 >>= 4; + for(j = 0; e1 > 1; j++, e1 >>= 1) + if (e1 & 1) + dval(&rv) *= bigtens[j]; + /* The last multiplication could overflow. */ + word0(&rv) -= P*Exp_msk1; + dval(&rv) *= bigtens[j]; + if ((z = word0(&rv) & Exp_mask) + > Exp_msk1*(DBL_MAX_EXP+Bias-P)) + goto ovfl; + if (z > Exp_msk1*(DBL_MAX_EXP+Bias-1-P)) { + /* set to largest number */ + /* (Can't trust DBL_MAX) */ + word0(&rv) = Big0; + word1(&rv) = Big1; + } + else + word0(&rv) += P*Exp_msk1; + } + } + else if (e1 < 0) { + /* The input decimal value lies in [10**e1, 10**(e1+16)). + + If e1 <= -512, underflow immediately. + If e1 <= -256, set bc.scale to 2*P. + + So for input value < 1e-256, bc.scale is always set; + for input value >= 1e-240, bc.scale is never set. + For input values in [1e-256, 1e-240), bc.scale may or may + not be set. */ + + e1 = -e1; + if ((i = e1 & 15)) + dval(&rv) /= tens[i]; + if (e1 >>= 4) { + if (e1 >= 1 << n_bigtens) + goto undfl; + if (e1 & Scale_Bit) + bc.scale = 2*P; + for(j = 0; e1 > 0; j++, e1 >>= 1) + if (e1 & 1) + dval(&rv) *= tinytens[j]; + if (bc.scale && (j = 2*P + 1 - ((word0(&rv) & Exp_mask) + >> Exp_shift)) > 0) { + /* scaled rv is denormal; clear j low bits */ + if (j >= 32) { + word1(&rv) = 0; + if (j >= 53) + word0(&rv) = (P+2)*Exp_msk1; + else + word0(&rv) &= 0xffffffff << (j-32); + } + else + word1(&rv) &= 0xffffffff << j; + } + if (!dval(&rv)) + goto undfl; + } + } + + /* Now the hard part -- adjusting rv to the correct value.*/ + + /* Put digits into bd: true value = bd * 10^e */ + + bc.nd = nd; + bc.nd0 = nd0; /* Only needed if nd > STRTOD_DIGLIM, but done here */ + /* to silence an erroneous warning about bc.nd0 */ + /* possibly not being initialized. */ + if (nd > STRTOD_DIGLIM) { + /* ASSERT(STRTOD_DIGLIM >= 18); 18 == one more than the */ + /* minimum number of decimal digits to distinguish double values */ + /* in IEEE arithmetic. */ + + /* Truncate input to 18 significant digits, then discard any trailing + zeros on the result by updating nd, nd0, e and y suitably. (There's + no need to update z; it's not reused beyond this point.) */ + for (i = 18; i > 0; ) { + /* scan back until we hit a nonzero digit. significant digit 'i' + is s0[i] if i < nd0, s0[i+1] if i >= nd0. */ + --i; + if (s0[i < nd0 ? i : i+1] != '0') { + ++i; + break; + } + } + e += nd - i; + nd = i; + if (nd0 > nd) + nd0 = nd; + if (nd < 9) { /* must recompute y */ + y = 0; + for(i = 0; i < nd0; ++i) + y = 10*y + s0[i] - '0'; + for(; i < nd; ++i) + y = 10*y + s0[i+1] - '0'; + } + } + bd0 = s2b(s0, nd0, nd, y); + if (bd0 == NULL) + goto failed_malloc; + + /* Notation for the comments below. Write: + + - dv for the absolute value of the number represented by the original + decimal input string. + + - if we've truncated dv, write tdv for the truncated value. + Otherwise, set tdv == dv. + + - srv for the quantity rv/2^bc.scale; so srv is the current binary + approximation to tdv (and dv). It should be exactly representable + in an IEEE 754 double. + */ + + for(;;) { + + /* This is the main correction loop for _Py_dg_strtod. + + We've got a decimal value tdv, and a floating-point approximation + srv=rv/2^bc.scale to tdv. The aim is to determine whether srv is + close enough (i.e., within 0.5 ulps) to tdv, and to compute a new + approximation if not. + + To determine whether srv is close enough to tdv, compute integers + bd, bb and bs proportional to tdv, srv and 0.5 ulp(srv) + respectively, and then use integer arithmetic to determine whether + |tdv - srv| is less than, equal to, or greater than 0.5 ulp(srv). + */ + + bd = Balloc(bd0->k); + if (bd == NULL) { + Bfree(bd0); + goto failed_malloc; + } + Bcopy(bd, bd0); + bb = sd2b(&rv, bc.scale, &bbe); /* srv = bb * 2^bbe */ + if (bb == NULL) { + Bfree(bd); + Bfree(bd0); + goto failed_malloc; + } + /* Record whether lsb of bb is odd, in case we need this + for the round-to-even step later. */ + odd = bb->x[0] & 1; + + /* tdv = bd * 10**e; srv = bb * 2**bbe */ + bs = i2b(1); + if (bs == NULL) { + Bfree(bb); + Bfree(bd); + Bfree(bd0); + goto failed_malloc; + } + + if (e >= 0) { + bb2 = bb5 = 0; + bd2 = bd5 = e; + } + else { + bb2 = bb5 = -e; + bd2 = bd5 = 0; + } + if (bbe >= 0) + bb2 += bbe; + else + bd2 -= bbe; + bs2 = bb2; + bb2++; + bd2++; + + /* At this stage bd5 - bb5 == e == bd2 - bb2 + bbe, bb2 - bs2 == 1, + and bs == 1, so: + + tdv == bd * 10**e = bd * 2**(bbe - bb2 + bd2) * 5**(bd5 - bb5) + srv == bb * 2**bbe = bb * 2**(bbe - bb2 + bb2) + 0.5 ulp(srv) == 2**(bbe-1) = bs * 2**(bbe - bb2 + bs2) + + It follows that: + + M * tdv = bd * 2**bd2 * 5**bd5 + M * srv = bb * 2**bb2 * 5**bb5 + M * 0.5 ulp(srv) = bs * 2**bs2 * 5**bb5 + + for some constant M. (Actually, M == 2**(bb2 - bbe) * 5**bb5, but + this fact is not needed below.) + */ + + /* Remove factor of 2**i, where i = min(bb2, bd2, bs2). */ + i = bb2 < bd2 ? bb2 : bd2; + if (i > bs2) + i = bs2; + if (i > 0) { + bb2 -= i; + bd2 -= i; + bs2 -= i; + } + + /* Scale bb, bd, bs by the appropriate powers of 2 and 5. */ + if (bb5 > 0) { + bs = pow5mult(bs, bb5); + if (bs == NULL) { + Bfree(bb); + Bfree(bd); + Bfree(bd0); + goto failed_malloc; + } + bb1 = mult(bs, bb); + Bfree(bb); + bb = bb1; + if (bb == NULL) { + Bfree(bs); + Bfree(bd); + Bfree(bd0); + goto failed_malloc; + } + } + if (bb2 > 0) { + bb = lshift(bb, bb2); + if (bb == NULL) { + Bfree(bs); + Bfree(bd); + Bfree(bd0); + goto failed_malloc; + } + } + if (bd5 > 0) { + bd = pow5mult(bd, bd5); + if (bd == NULL) { + Bfree(bb); + Bfree(bs); + Bfree(bd0); + goto failed_malloc; + } + } + if (bd2 > 0) { + bd = lshift(bd, bd2); + if (bd == NULL) { + Bfree(bb); + Bfree(bs); + Bfree(bd0); + goto failed_malloc; + } + } + if (bs2 > 0) { + bs = lshift(bs, bs2); + if (bs == NULL) { + Bfree(bb); + Bfree(bd); + Bfree(bd0); + goto failed_malloc; + } + } + + /* Now bd, bb and bs are scaled versions of tdv, srv and 0.5 ulp(srv), + respectively. Compute the difference |tdv - srv|, and compare + with 0.5 ulp(srv). */ + + delta = diff(bb, bd); + if (delta == NULL) { + Bfree(bb); + Bfree(bs); + Bfree(bd); + Bfree(bd0); + goto failed_malloc; + } + dsign = delta->sign; + delta->sign = 0; + i = cmp(delta, bs); + if (bc.nd > nd && i <= 0) { + if (dsign) + break; /* Must use bigcomp(). */ + + /* Here rv overestimates the truncated decimal value by at most + 0.5 ulp(rv). Hence rv either overestimates the true decimal + value by <= 0.5 ulp(rv), or underestimates it by some small + amount (< 0.1 ulp(rv)); either way, rv is within 0.5 ulps of + the true decimal value, so it's possible to exit. + + Exception: if scaled rv is a normal exact power of 2, but not + DBL_MIN, then rv - 0.5 ulp(rv) takes us all the way down to the + next double, so the correctly rounded result is either rv - 0.5 + ulp(rv) or rv; in this case, use bigcomp to distinguish. */ + + if (!word1(&rv) && !(word0(&rv) & Bndry_mask)) { + /* rv can't be 0, since it's an overestimate for some + nonzero value. So rv is a normal power of 2. */ + j = (int)(word0(&rv) & Exp_mask) >> Exp_shift; + /* rv / 2^bc.scale = 2^(j - 1023 - bc.scale); use bigcomp if + rv / 2^bc.scale >= 2^-1021. */ + if (j - bc.scale >= 2) { + dval(&rv) -= 0.5 * sulp(&rv, &bc); + break; /* Use bigcomp. */ + } + } + + { + bc.nd = nd; + i = -1; /* Discarded digits make delta smaller. */ + } + } + + if (i < 0) { + /* Error is less than half an ulp -- check for + * special case of mantissa a power of two. + */ + if (dsign || word1(&rv) || word0(&rv) & Bndry_mask + || (word0(&rv) & Exp_mask) <= (2*P+1)*Exp_msk1 + ) { + break; + } + if (!delta->x[0] && delta->wds <= 1) { + /* exact result */ + break; + } + delta = lshift(delta,Log2P); + if (delta == NULL) { + Bfree(bb); + Bfree(bs); + Bfree(bd); + Bfree(bd0); + goto failed_malloc; + } + if (cmp(delta, bs) > 0) + goto drop_down; + break; + } + if (i == 0) { + /* exactly half-way between */ + if (dsign) { + if ((word0(&rv) & Bndry_mask1) == Bndry_mask1 + && word1(&rv) == ( + (bc.scale && + (y = word0(&rv) & Exp_mask) <= 2*P*Exp_msk1) ? + (0xffffffff & (0xffffffff << (2*P+1-(y>>Exp_shift)))) : + 0xffffffff)) { + /*boundary case -- increment exponent*/ + word0(&rv) = (word0(&rv) & Exp_mask) + + Exp_msk1 + ; + word1(&rv) = 0; + dsign = 0; + break; + } + } + else if (!(word0(&rv) & Bndry_mask) && !word1(&rv)) { + drop_down: + /* boundary case -- decrement exponent */ + if (bc.scale) { + L = word0(&rv) & Exp_mask; + if (L <= (2*P+1)*Exp_msk1) { + if (L > (P+2)*Exp_msk1) + /* round even ==> */ + /* accept rv */ + break; + /* rv = smallest denormal */ + if (bc.nd > nd) + break; + goto undfl; + } + } + L = (word0(&rv) & Exp_mask) - Exp_msk1; + word0(&rv) = L | Bndry_mask1; + word1(&rv) = 0xffffffff; + break; + } + if (!odd) + break; + if (dsign) + dval(&rv) += sulp(&rv, &bc); + else { + dval(&rv) -= sulp(&rv, &bc); + if (!dval(&rv)) { + if (bc.nd >nd) + break; + goto undfl; + } + } + dsign = 1 - dsign; + break; + } + if ((aadj = ratio(delta, bs)) <= 2.) { + if (dsign) + aadj = aadj1 = 1.; + else if (word1(&rv) || word0(&rv) & Bndry_mask) { + if (word1(&rv) == Tiny1 && !word0(&rv)) { + if (bc.nd >nd) + break; + goto undfl; + } + aadj = 1.; + aadj1 = -1.; + } + else { + /* special case -- power of FLT_RADIX to be */ + /* rounded down... */ + + if (aadj < 2./FLT_RADIX) + aadj = 1./FLT_RADIX; + else + aadj *= 0.5; + aadj1 = -aadj; + } + } + else { + aadj *= 0.5; + aadj1 = dsign ? aadj : -aadj; + if (Flt_Rounds == 0) + aadj1 += 0.5; + } + y = word0(&rv) & Exp_mask; + + /* Check for overflow */ + + if (y == Exp_msk1*(DBL_MAX_EXP+Bias-1)) { + dval(&rv0) = dval(&rv); + word0(&rv) -= P*Exp_msk1; + adj.d = aadj1 * ulp(&rv); + dval(&rv) += adj.d; + if ((word0(&rv) & Exp_mask) >= + Exp_msk1*(DBL_MAX_EXP+Bias-P)) { + if (word0(&rv0) == Big0 && word1(&rv0) == Big1) { + Bfree(bb); + Bfree(bd); + Bfree(bs); + Bfree(bd0); + Bfree(delta); + goto ovfl; + } + word0(&rv) = Big0; + word1(&rv) = Big1; + goto cont; + } + else + word0(&rv) += P*Exp_msk1; + } + else { + if (bc.scale && y <= 2*P*Exp_msk1) { + if (aadj <= 0x7fffffff) { + if ((z = (ULong)aadj) <= 0) + z = 1; + aadj = z; + aadj1 = dsign ? aadj : -aadj; + } + dval(&aadj2) = aadj1; + word0(&aadj2) += (2*P+1)*Exp_msk1 - y; + aadj1 = dval(&aadj2); + } + adj.d = aadj1 * ulp(&rv); + dval(&rv) += adj.d; + } + z = word0(&rv) & Exp_mask; + if (bc.nd == nd) { + if (!bc.scale) + if (y == z) { + /* Can we stop now? */ + L = (Long)aadj; + aadj -= L; + /* The tolerances below are conservative. */ + if (dsign || word1(&rv) || word0(&rv) & Bndry_mask) { + if (aadj < .4999999 || aadj > .5000001) + break; + } + else if (aadj < .4999999/FLT_RADIX) + break; + } + } + cont: + Bfree(bb); + Bfree(bd); + Bfree(bs); + Bfree(delta); + } + Bfree(bb); + Bfree(bd); + Bfree(bs); + Bfree(bd0); + Bfree(delta); + if (bc.nd > nd) { + error = bigcomp(&rv, s0, &bc); + if (error) + goto failed_malloc; + } + + if (bc.scale) { + word0(&rv0) = Exp_1 - 2*P*Exp_msk1; + word1(&rv0) = 0; + dval(&rv) *= dval(&rv0); + } + + ret: + return sign ? -dval(&rv) : dval(&rv); + + parse_error: + return 0.0; + + failed_malloc: + errno = ENOMEM; + return -1.0; + + undfl: + return sign ? -0.0 : 0.0; + + ovfl: + errno = ERANGE; + /* Can't trust HUGE_VAL */ + word0(&rv) = Exp_mask; + word1(&rv) = 0; + return sign ? -dval(&rv) : dval(&rv); + +} + +static char * +rv_alloc(int i) +{ + int j, k, *r; + + j = sizeof(ULong); + for(k = 0; + sizeof(Bigint) - sizeof(ULong) - sizeof(int) + j <= (unsigned)i; + j <<= 1) + k++; + r = (int*)Balloc(k); + if (r == NULL) + return NULL; + *r = k; + return (char *)(r+1); +} + +static char * +nrv_alloc(char *s, char **rve, int n) +{ + char *rv, *t; + + rv = rv_alloc(n); + if (rv == NULL) + return NULL; + t = rv; + while((*t = *s++)) t++; + if (rve) + *rve = t; + return rv; +} + +/* freedtoa(s) must be used to free values s returned by dtoa + * when MULTIPLE_THREADS is #defined. It should be used in all cases, + * but for consistency with earlier versions of dtoa, it is optional + * when MULTIPLE_THREADS is not defined. + */ + +void +_Py_dg_freedtoa(char *s) +{ + Bigint *b = (Bigint *)((int *)s - 1); + b->maxwds = 1 << (b->k = *(int*)b); + Bfree(b); +} + +/* dtoa for IEEE arithmetic (dmg): convert double to ASCII string. + * + * Inspired by "How to Print Floating-Point Numbers Accurately" by + * Guy L. Steele, Jr. and Jon L. White [Proc. ACM SIGPLAN '90, pp. 112-126]. + * + * Modifications: + * 1. Rather than iterating, we use a simple numeric overestimate + * to determine k = floor(log10(d)). We scale relevant + * quantities using O(log2(k)) rather than O(k) multiplications. + * 2. For some modes > 2 (corresponding to ecvt and fcvt), we don't + * try to generate digits strictly left to right. Instead, we + * compute with fewer bits and propagate the carry if necessary + * when rounding the final digit up. This is often faster. + * 3. Under the assumption that input will be rounded nearest, + * mode 0 renders 1e23 as 1e23 rather than 9.999999999999999e22. + * That is, we allow equality in stopping tests when the + * round-nearest rule will give the same floating-point value + * as would satisfaction of the stopping test with strict + * inequality. + * 4. We remove common factors of powers of 2 from relevant + * quantities. + * 5. When converting floating-point integers less than 1e16, + * we use floating-point arithmetic rather than resorting + * to multiple-precision integers. + * 6. When asked to produce fewer than 15 digits, we first try + * to get by with floating-point arithmetic; we resort to + * multiple-precision integer arithmetic only if we cannot + * guarantee that the floating-point calculation has given + * the correctly rounded result. For k requested digits and + * "uniformly" distributed input, the probability is + * something like 10^(k-15) that we must resort to the Long + * calculation. + */ + +/* Additional notes (METD): (1) returns NULL on failure. (2) to avoid memory + leakage, a successful call to _Py_dg_dtoa should always be matched by a + call to _Py_dg_freedtoa. */ + +char * +_Py_dg_dtoa(double dd, int mode, int ndigits, + int *decpt, int *sign, char **rve) +{ + /* Arguments ndigits, decpt, sign are similar to those + of ecvt and fcvt; trailing zeros are suppressed from + the returned string. If not null, *rve is set to point + to the end of the return value. If d is +-Infinity or NaN, + then *decpt is set to 9999. + + mode: + 0 ==> shortest string that yields d when read in + and rounded to nearest. + 1 ==> like 0, but with Steele & White stopping rule; + e.g. with IEEE P754 arithmetic , mode 0 gives + 1e23 whereas mode 1 gives 9.999999999999999e22. + 2 ==> max(1,ndigits) significant digits. This gives a + return value similar to that of ecvt, except + that trailing zeros are suppressed. + 3 ==> through ndigits past the decimal point. This + gives a return value similar to that from fcvt, + except that trailing zeros are suppressed, and + ndigits can be negative. + 4,5 ==> similar to 2 and 3, respectively, but (in + round-nearest mode) with the tests of mode 0 to + possibly return a shorter string that rounds to d. + With IEEE arithmetic and compilation with + -DHonor_FLT_ROUNDS, modes 4 and 5 behave the same + as modes 2 and 3 when FLT_ROUNDS != 1. + 6-9 ==> Debugging modes similar to mode - 4: don't try + fast floating-point estimate (if applicable). + + Values of mode other than 0-9 are treated as mode 0. + + Sufficient space is allocated to the return value + to hold the suppressed trailing zeros. + */ + + int bbits, b2, b5, be, dig, i, ieps, ilim, ilim0, ilim1, + j, j1, k, k0, k_check, leftright, m2, m5, s2, s5, + spec_case, try_quick; + Long L; + int denorm; + ULong x; + Bigint *b, *b1, *delta, *mlo, *mhi, *S; + U d2, eps, u; + double ds; + char *s, *s0; + + /* set pointers to NULL, to silence gcc compiler warnings and make + cleanup easier on error */ + mlo = mhi = S = 0; + s0 = 0; + + u.d = dd; + if (word0(&u) & Sign_bit) { + /* set sign for everything, including 0's and NaNs */ + *sign = 1; + word0(&u) &= ~Sign_bit; /* clear sign bit */ + } + else + *sign = 0; + + /* quick return for Infinities, NaNs and zeros */ + if ((word0(&u) & Exp_mask) == Exp_mask) + { + /* Infinity or NaN */ + *decpt = 9999; + if (!word1(&u) && !(word0(&u) & 0xfffff)) + return nrv_alloc("Infinity", rve, 8); + return nrv_alloc("NaN", rve, 3); + } + if (!dval(&u)) { + *decpt = 1; + return nrv_alloc("0", rve, 1); + } + + /* compute k = floor(log10(d)). The computation may leave k + one too large, but should never leave k too small. */ + b = d2b(&u, &be, &bbits); + if (b == NULL) + goto failed_malloc; + if ((i = (int)(word0(&u) >> Exp_shift1 & (Exp_mask>>Exp_shift1)))) { + dval(&d2) = dval(&u); + word0(&d2) &= Frac_mask1; + word0(&d2) |= Exp_11; + + /* log(x) ~=~ log(1.5) + (x-1.5)/1.5 + * log10(x) = log(x) / log(10) + * ~=~ log(1.5)/log(10) + (x-1.5)/(1.5*log(10)) + * log10(d) = (i-Bias)*log(2)/log(10) + log10(d2) + * + * This suggests computing an approximation k to log10(d) by + * + * k = (i - Bias)*0.301029995663981 + * + ( (d2-1.5)*0.289529654602168 + 0.176091259055681 ); + * + * We want k to be too large rather than too small. + * The error in the first-order Taylor series approximation + * is in our favor, so we just round up the constant enough + * to compensate for any error in the multiplication of + * (i - Bias) by 0.301029995663981; since |i - Bias| <= 1077, + * and 1077 * 0.30103 * 2^-52 ~=~ 7.2e-14, + * adding 1e-13 to the constant term more than suffices. + * Hence we adjust the constant term to 0.1760912590558. + * (We could get a more accurate k by invoking log10, + * but this is probably not worthwhile.) + */ + + i -= Bias; + denorm = 0; + } + else { + /* d is denormalized */ + + i = bbits + be + (Bias + (P-1) - 1); + x = i > 32 ? word0(&u) << (64 - i) | word1(&u) >> (i - 32) + : word1(&u) << (32 - i); + dval(&d2) = x; + word0(&d2) -= 31*Exp_msk1; /* adjust exponent */ + i -= (Bias + (P-1) - 1) + 1; + denorm = 1; + } + ds = (dval(&d2)-1.5)*0.289529654602168 + 0.1760912590558 + + i*0.301029995663981; + k = (int)ds; + if (ds < 0. && ds != k) + k--; /* want k = floor(ds) */ + k_check = 1; + if (k >= 0 && k <= Ten_pmax) { + if (dval(&u) < tens[k]) + k--; + k_check = 0; + } + j = bbits - i - 1; + if (j >= 0) { + b2 = 0; + s2 = j; + } + else { + b2 = -j; + s2 = 0; + } + if (k >= 0) { + b5 = 0; + s5 = k; + s2 += k; + } + else { + b2 -= k; + b5 = -k; + s5 = 0; + } + if (mode < 0 || mode > 9) + mode = 0; + + try_quick = 1; + + if (mode > 5) { + mode -= 4; + try_quick = 0; + } + leftright = 1; + ilim = ilim1 = -1; /* Values for cases 0 and 1; done here to */ + /* silence erroneous "gcc -Wall" warning. */ + switch(mode) { + case 0: + case 1: + i = 18; + ndigits = 0; + break; + case 2: + leftright = 0; + /* no break */ + case 4: + if (ndigits <= 0) + ndigits = 1; + ilim = ilim1 = i = ndigits; + break; + case 3: + leftright = 0; + /* no break */ + case 5: + i = ndigits + k + 1; + ilim = i; + ilim1 = i - 1; + if (i <= 0) + i = 1; + } + s0 = rv_alloc(i); + if (s0 == NULL) + goto failed_malloc; + s = s0; + + + if (ilim >= 0 && ilim <= Quick_max && try_quick) { + + /* Try to get by with floating-point arithmetic. */ + + i = 0; + dval(&d2) = dval(&u); + k0 = k; + ilim0 = ilim; + ieps = 2; /* conservative */ + if (k > 0) { + ds = tens[k&0xf]; + j = k >> 4; + if (j & Bletch) { + /* prevent overflows */ + j &= Bletch - 1; + dval(&u) /= bigtens[n_bigtens-1]; + ieps++; + } + for(; j; j >>= 1, i++) + if (j & 1) { + ieps++; + ds *= bigtens[i]; + } + dval(&u) /= ds; + } + else if ((j1 = -k)) { + dval(&u) *= tens[j1 & 0xf]; + for(j = j1 >> 4; j; j >>= 1, i++) + if (j & 1) { + ieps++; + dval(&u) *= bigtens[i]; + } + } + if (k_check && dval(&u) < 1. && ilim > 0) { + if (ilim1 <= 0) + goto fast_failed; + ilim = ilim1; + k--; + dval(&u) *= 10.; + ieps++; + } + dval(&eps) = ieps*dval(&u) + 7.; + word0(&eps) -= (P-1)*Exp_msk1; + if (ilim == 0) { + S = mhi = 0; + dval(&u) -= 5.; + if (dval(&u) > dval(&eps)) + goto one_digit; + if (dval(&u) < -dval(&eps)) + goto no_digits; + goto fast_failed; + } + if (leftright) { + /* Use Steele & White method of only + * generating digits needed. + */ + dval(&eps) = 0.5/tens[ilim-1] - dval(&eps); + for(i = 0;;) { + L = (Long)dval(&u); + dval(&u) -= L; + *s++ = '0' + (int)L; + if (dval(&u) < dval(&eps)) + goto ret1; + if (1. - dval(&u) < dval(&eps)) + goto bump_up; + if (++i >= ilim) + break; + dval(&eps) *= 10.; + dval(&u) *= 10.; + } + } + else { + /* Generate ilim digits, then fix them up. */ + dval(&eps) *= tens[ilim-1]; + for(i = 1;; i++, dval(&u) *= 10.) { + L = (Long)(dval(&u)); + if (!(dval(&u) -= L)) + ilim = i; + *s++ = '0' + (int)L; + if (i == ilim) { + if (dval(&u) > 0.5 + dval(&eps)) + goto bump_up; + else if (dval(&u) < 0.5 - dval(&eps)) { + while(*--s == '0'); + s++; + goto ret1; + } + break; + } + } + } + fast_failed: + s = s0; + dval(&u) = dval(&d2); + k = k0; + ilim = ilim0; + } + + /* Do we have a "small" integer? */ + + if (be >= 0 && k <= Int_max) { + /* Yes. */ + ds = tens[k]; + if (ndigits < 0 && ilim <= 0) { + S = mhi = 0; + if (ilim < 0 || dval(&u) <= 5*ds) + goto no_digits; + goto one_digit; + } + for(i = 1;; i++, dval(&u) *= 10.) { + L = (Long)(dval(&u) / ds); + dval(&u) -= L*ds; + *s++ = '0' + (int)L; + if (!dval(&u)) { + break; + } + if (i == ilim) { + dval(&u) += dval(&u); + if (dval(&u) > ds || (dval(&u) == ds && L & 1)) { + bump_up: + while(*--s == '9') + if (s == s0) { + k++; + *s = '0'; + break; + } + ++*s++; + } + break; + } + } + goto ret1; + } + + m2 = b2; + m5 = b5; + if (leftright) { + i = + denorm ? be + (Bias + (P-1) - 1 + 1) : + 1 + P - bbits; + b2 += i; + s2 += i; + mhi = i2b(1); + if (mhi == NULL) + goto failed_malloc; + } + if (m2 > 0 && s2 > 0) { + i = m2 < s2 ? m2 : s2; + b2 -= i; + m2 -= i; + s2 -= i; + } + if (b5 > 0) { + if (leftright) { + if (m5 > 0) { + mhi = pow5mult(mhi, m5); + if (mhi == NULL) + goto failed_malloc; + b1 = mult(mhi, b); + Bfree(b); + b = b1; + if (b == NULL) + goto failed_malloc; + } + if ((j = b5 - m5)) { + b = pow5mult(b, j); + if (b == NULL) + goto failed_malloc; + } + } + else { + b = pow5mult(b, b5); + if (b == NULL) + goto failed_malloc; + } + } + S = i2b(1); + if (S == NULL) + goto failed_malloc; + if (s5 > 0) { + S = pow5mult(S, s5); + if (S == NULL) + goto failed_malloc; + } + + /* Check for special case that d is a normalized power of 2. */ + + spec_case = 0; + if ((mode < 2 || leftright) + ) { + if (!word1(&u) && !(word0(&u) & Bndry_mask) + && word0(&u) & (Exp_mask & ~Exp_msk1) + ) { + /* The special case */ + b2 += Log2P; + s2 += Log2P; + spec_case = 1; + } + } + + /* Arrange for convenient computation of quotients: + * shift left if necessary so divisor has 4 leading 0 bits. + * + * Perhaps we should just compute leading 28 bits of S once + * and for all and pass them and a shift to quorem, so it + * can do shifts and ors to compute the numerator for q. + */ +#define iInc 28 + i = dshift(S, s2); + b2 += i; + m2 += i; + s2 += i; + if (b2 > 0) { + b = lshift(b, b2); + if (b == NULL) + goto failed_malloc; + } + if (s2 > 0) { + S = lshift(S, s2); + if (S == NULL) + goto failed_malloc; + } + if (k_check) { + if (cmp(b,S) < 0) { + k--; + b = multadd(b, 10, 0); /* we botched the k estimate */ + if (b == NULL) + goto failed_malloc; + if (leftright) { + mhi = multadd(mhi, 10, 0); + if (mhi == NULL) + goto failed_malloc; + } + ilim = ilim1; + } + } + if (ilim <= 0 && (mode == 3 || mode == 5)) { + if (ilim < 0) { + /* no digits, fcvt style */ + no_digits: + k = -1 - ndigits; + goto ret; + } + else { + S = multadd(S, 5, 0); + if (S == NULL) + goto failed_malloc; + if (cmp(b, S) <= 0) + goto no_digits; + } + one_digit: + *s++ = '1'; + k++; + goto ret; + } + if (leftright) { + if (m2 > 0) { + mhi = lshift(mhi, m2); + if (mhi == NULL) + goto failed_malloc; + } + + /* Compute mlo -- check for special case + * that d is a normalized power of 2. + */ + + mlo = mhi; + if (spec_case) { + mhi = Balloc(mhi->k); + if (mhi == NULL) + goto failed_malloc; + Bcopy(mhi, mlo); + mhi = lshift(mhi, Log2P); + if (mhi == NULL) + goto failed_malloc; + } + + for(i = 1;;i++) { + dig = quorem(b,S) + '0'; + /* Do we yet have the shortest decimal string + * that will round to d? + */ + j = cmp(b, mlo); + delta = diff(S, mhi); + if (delta == NULL) + goto failed_malloc; + j1 = delta->sign ? 1 : cmp(b, delta); + Bfree(delta); + if (j1 == 0 && mode != 1 && !(word1(&u) & 1) + ) { + if (dig == '9') + goto round_9_up; + if (j > 0) + dig++; + *s++ = dig; + goto ret; + } + if (j < 0 || (j == 0 && mode != 1 + && !(word1(&u) & 1) + )) { + if (!b->x[0] && b->wds <= 1) { + goto accept_dig; + } + if (j1 > 0) { + b = lshift(b, 1); + if (b == NULL) + goto failed_malloc; + j1 = cmp(b, S); + if ((j1 > 0 || (j1 == 0 && dig & 1)) + && dig++ == '9') + goto round_9_up; + } + accept_dig: + *s++ = dig; + goto ret; + } + if (j1 > 0) { + if (dig == '9') { /* possible if i == 1 */ + round_9_up: + *s++ = '9'; + goto roundoff; + } + *s++ = dig + 1; + goto ret; + } + *s++ = dig; + if (i == ilim) + break; + b = multadd(b, 10, 0); + if (b == NULL) + goto failed_malloc; + if (mlo == mhi) { + mlo = mhi = multadd(mhi, 10, 0); + if (mlo == NULL) + goto failed_malloc; + } + else { + mlo = multadd(mlo, 10, 0); + if (mlo == NULL) + goto failed_malloc; + mhi = multadd(mhi, 10, 0); + if (mhi == NULL) + goto failed_malloc; + } + } + } + else + for(i = 1;; i++) { + *s++ = dig = quorem(b,S) + '0'; + if (!b->x[0] && b->wds <= 1) { + goto ret; + } + if (i >= ilim) + break; + b = multadd(b, 10, 0); + if (b == NULL) + goto failed_malloc; + } + + /* Round off last digit */ + + b = lshift(b, 1); + if (b == NULL) + goto failed_malloc; + j = cmp(b, S); + if (j > 0 || (j == 0 && dig & 1)) { + roundoff: + while(*--s == '9') + if (s == s0) { + k++; + *s++ = '1'; + goto ret; + } + ++*s++; + } + else { + while(*--s == '0'); + s++; + } + ret: + Bfree(S); + if (mhi) { + if (mlo && mlo != mhi) + Bfree(mlo); + Bfree(mhi); + } + ret1: + Bfree(b); + *s = 0; + *decpt = k + 1; + if (rve) + *rve = s; + return s0; + failed_malloc: + if (S) + Bfree(S); + if (mlo && mlo != mhi) + Bfree(mlo); + if (mhi) + Bfree(mhi); + if (b) + Bfree(b); + if (s0) + _Py_dg_freedtoa(s0); + return NULL; +} +#ifdef __cplusplus +} +#endif + +#endif /* PY_NO_SHORT_FLOAT_REPR */ diff --git a/Python/dup2.c b/Python/dup2.c index ba7413a0ca..2579afd443 100644 --- a/Python/dup2.c +++ b/Python/dup2.c @@ -12,6 +12,7 @@ */ #include <fcntl.h> +#include <unistd.h> #define BADEXIT -1 diff --git a/Python/dynload_aix.c b/Python/dynload_aix.c index 8d56d7d008..149990d799 100644 --- a/Python/dynload_aix.c +++ b/Python/dynload_aix.c @@ -12,7 +12,7 @@ #ifdef AIX_GENUINE_CPLUSPLUS -#include "/usr/lpp/xlC/include/load.h" +#include <load.h> #define aix_load loadAndInit #else #define aix_load load diff --git a/Python/errors.c b/Python/errors.c index da4b463d5e..64ba05dd6c 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -106,10 +106,18 @@ PyErr_GivenExceptionMatches(PyObject *err, PyObject *exc) err = PyExceptionInstance_Class(err); if (PyExceptionClass_Check(err) && PyExceptionClass_Check(exc)) { - int res = 0; + int res = 0, reclimit; PyObject *exception, *value, *tb; PyErr_Fetch(&exception, &value, &tb); + /* Temporarily bump the recursion limit, so that in the most + common case PyObject_IsSubclass will not raise a recursion + error we have to ignore anyway. Don't do it when the limit + is already insanely high, to avoid overflow */ + reclimit = Py_GetRecursionLimit(); + if (reclimit < (1 << 30)) + Py_SetRecursionLimit(reclimit + 5); res = PyObject_IsSubclass(err, exc); + Py_SetRecursionLimit(reclimit); /* This function must not fail, so print the error here */ if (res == -1) { PyErr_WriteUnraisable(err); @@ -365,7 +373,7 @@ PyErr_SetFromErrnoWithFilenameObject(PyObject *exc, PyObject *filenameObject) PyObject * -PyErr_SetFromErrnoWithFilename(PyObject *exc, char *filename) +PyErr_SetFromErrnoWithFilename(PyObject *exc, const char *filename) { PyObject *name = filename ? PyString_FromString(filename) : NULL; PyObject *result = PyErr_SetFromErrnoWithFilenameObject(exc, name); @@ -373,9 +381,9 @@ PyErr_SetFromErrnoWithFilename(PyObject *exc, char *filename) return result; } -#ifdef Py_WIN_WIDE_FILENAMES +#ifdef MS_WINDOWS PyObject * -PyErr_SetFromErrnoWithUnicodeFilename(PyObject *exc, Py_UNICODE *filename) +PyErr_SetFromErrnoWithUnicodeFilename(PyObject *exc, const Py_UNICODE *filename) { PyObject *name = filename ? PyUnicode_FromUnicode(filename, wcslen(filename)) : @@ -384,7 +392,7 @@ PyErr_SetFromErrnoWithUnicodeFilename(PyObject *exc, Py_UNICODE *filename) Py_XDECREF(name); return result; } -#endif /* Py_WIN_WIDE_FILENAMES */ +#endif /* MS_WINDOWS */ PyObject * PyErr_SetFromErrno(PyObject *exc) @@ -454,7 +462,6 @@ PyObject *PyErr_SetExcFromWindowsErrWithFilename( return ret; } -#ifdef Py_WIN_WIDE_FILENAMES PyObject *PyErr_SetExcFromWindowsErrWithUnicodeFilename( PyObject *exc, int ierr, @@ -469,7 +476,6 @@ PyObject *PyErr_SetExcFromWindowsErrWithUnicodeFilename( Py_XDECREF(name); return ret; } -#endif /* Py_WIN_WIDE_FILENAMES */ PyObject *PyErr_SetExcFromWindowsErr(PyObject *exc, int ierr) { @@ -493,7 +499,6 @@ PyObject *PyErr_SetFromWindowsErrWithFilename( return result; } -#ifdef Py_WIN_WIDE_FILENAMES PyObject *PyErr_SetFromWindowsErrWithUnicodeFilename( int ierr, const Py_UNICODE *filename) @@ -507,7 +512,6 @@ PyObject *PyErr_SetFromWindowsErrWithUnicodeFilename( Py_XDECREF(name); return result; } -#endif /* Py_WIN_WIDE_FILENAMES */ #endif /* MS_WINDOWS */ void @@ -602,6 +606,40 @@ PyErr_NewException(char *name, PyObject *base, PyObject *dict) return result; } + +/* Create an exception with docstring */ +PyObject * +PyErr_NewExceptionWithDoc(char *name, char *doc, PyObject *base, PyObject *dict) +{ + int result; + PyObject *ret = NULL; + PyObject *mydict = NULL; /* points to the dict only if we create it */ + PyObject *docobj; + + if (dict == NULL) { + dict = mydict = PyDict_New(); + if (dict == NULL) { + return NULL; + } + } + + if (doc != NULL) { + docobj = PyString_FromString(doc); + if (docobj == NULL) + goto failure; + result = PyDict_SetItemString(dict, "__doc__", docobj); + Py_DECREF(docobj); + if (result < 0) + goto failure; + } + + ret = PyErr_NewException(name, base, dict); + failure: + Py_XDECREF(mydict); + return ret; +} + + /* Call when an exception has occurred but there is no way for Python to handle it. Examples: exception in __del__ or during GC. */ void diff --git a/Python/formatter_string.c b/Python/formatter_string.c index f33ad70586..06a2ef5903 100644 --- a/Python/formatter_string.c +++ b/Python/formatter_string.c @@ -1,14 +1,17 @@ /***********************************************************************/ /* Implements the string (as opposed to unicode) version of the built-in formatters for string, int, float. That is, the versions - of int.__float__, etc., that take and return string objects */ + of int.__format__, etc., that take and return string objects */ #include "Python.h" #include "../Objects/stringlib/stringdefs.h" -#define FORMAT_STRING _PyBytes_FormatAdvanced -#define FORMAT_LONG _PyLong_FormatAdvanced -#define FORMAT_INT _PyInt_FormatAdvanced -#define FORMAT_FLOAT _PyFloat_FormatAdvanced +#define FORMAT_STRING _PyBytes_FormatAdvanced +#define FORMAT_LONG _PyLong_FormatAdvanced +#define FORMAT_INT _PyInt_FormatAdvanced +#define FORMAT_FLOAT _PyFloat_FormatAdvanced +#ifndef WITHOUT_COMPLEX +#define FORMAT_COMPLEX _PyComplex_FormatAdvanced +#endif #include "../Objects/stringlib/formatter.h" diff --git a/Python/formatter_unicode.c b/Python/formatter_unicode.c index 4f2e53fe92..6e3685d16e 100644 --- a/Python/formatter_unicode.c +++ b/Python/formatter_unicode.c @@ -2,12 +2,17 @@ built-in formatter for unicode. That is, unicode.__format__(). */ #include "Python.h" + +#ifdef Py_USING_UNICODE + #include "../Objects/stringlib/unicodedefs.h" #define FORMAT_STRING _PyUnicode_FormatAdvanced -/* don't define FORMAT_LONG and FORMAT_FLOAT, since we can live - with only the string versions of those. The builtin format() - will convert them to unicode. */ +/* don't define FORMAT_LONG, FORMAT_FLOAT, and FORMAT_COMPLEX, since + we can live with only the string versions of those. The builtin + format() will convert them to unicode. */ #include "../Objects/stringlib/formatter.h" + +#endif diff --git a/Python/future.c b/Python/future.c index 9158df45c0..0e68845981 100644 --- a/Python/future.c +++ b/Python/future.c @@ -8,6 +8,8 @@ #include "symtable.h" #define UNDEFINED_FUTURE_FEATURE "future feature %.100s is not defined" +#define ERR_LATE_FUTURE \ +"from __future__ imports must occur at the beginning of the file" static int future_check_features(PyFutureFeatures *ff, stmt_ty s, const char *filename) @@ -57,13 +59,6 @@ future_parse(PyFutureFeatures *ff, mod_ty mod, const char *filename) { int i, found_docstring = 0, done = 0, prev_line = 0; - static PyObject *future; - if (!future) { - future = PyString_InternFromString("__future__"); - if (!future) - return 0; - } - if (!(mod->kind == Module_kind || mod->kind == Interactive_kind)) return 1; @@ -90,7 +85,9 @@ future_parse(PyFutureFeatures *ff, mod_ty mod, const char *filename) */ if (s->kind == ImportFrom_kind) { - if (s->v.ImportFrom.module == future) { + identifier modname = s->v.ImportFrom.module; + if (modname && PyString_GET_SIZE(modname) == 10 && + !strcmp(PyString_AS_STRING(modname), "__future__")) { if (done) { PyErr_SetString(PyExc_SyntaxError, ERR_LATE_FUTURE); diff --git a/Python/getargs.c b/Python/getargs.c index b39e07ee8f..a6cebbc75d 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -139,22 +139,33 @@ _PyArg_VaParse_SizeT(PyObject *args, char *format, va_list va) /* Handle cleanup of allocated memory in case of exception */ +#define GETARGS_CAPSULE_NAME_CLEANUP_PTR "getargs.cleanup_ptr" +#define GETARGS_CAPSULE_NAME_CLEANUP_BUFFER "getargs.cleanup_buffer" + static void -cleanup_ptr(void *ptr) +cleanup_ptr(PyObject *self) { - PyMem_FREE(ptr); + void *ptr = PyCapsule_GetPointer(self, GETARGS_CAPSULE_NAME_CLEANUP_PTR); + if (ptr) { + PyMem_FREE(ptr); + } } static void -cleanup_buffer(void *ptr) +cleanup_buffer(PyObject *self) { - PyBuffer_Release((Py_buffer *) ptr); + Py_buffer *ptr = (Py_buffer *)PyCapsule_GetPointer(self, GETARGS_CAPSULE_NAME_CLEANUP_BUFFER); + if (ptr) { + PyBuffer_Release(ptr); + } } static int -addcleanup(void *ptr, PyObject **freelist, void (*destr)(void *)) +addcleanup(void *ptr, PyObject **freelist, PyCapsule_Destructor destr) { PyObject *cobj; + const char *name; + if (!*freelist) { *freelist = PyList_New(0); if (!*freelist) { @@ -162,7 +173,15 @@ addcleanup(void *ptr, PyObject **freelist, void (*destr)(void *)) return -1; } } - cobj = PyCObject_FromVoidPtr(ptr, destr); + + if (destr == cleanup_ptr) { + name = GETARGS_CAPSULE_NAME_CLEANUP_PTR; + } else if (destr == cleanup_buffer) { + name = GETARGS_CAPSULE_NAME_CLEANUP_BUFFER; + } else { + return -1; + } + cobj = PyCapsule_New(ptr, name, destr); if (!cobj) { destr(ptr); return -1; @@ -183,8 +202,7 @@ cleanreturn(int retval, PyObject *freelist) don't get called. */ Py_ssize_t len = PyList_GET_SIZE(freelist), i; for (i = 0; i < len; i++) - ((PyCObject *) PyList_GET_ITEM(freelist, i)) - ->destructor = NULL; + PyCapsule_SetDestructor(PyList_GET_ITEM(freelist, i), NULL); } Py_XDECREF(freelist); return retval; @@ -328,7 +346,7 @@ vgetargs1(PyObject *args, const char *format, va_list *p_va, int flags) flags, levels, msgbuf, sizeof(msgbuf), &freelist); if (msg) { - seterror(i+1, msg, levels, fname, message); + seterror(i+1, msg, levels, fname, msg); return cleanreturn(0, freelist); } } @@ -526,7 +544,7 @@ converterr(const char *expected, PyObject *arg, char *msgbuf, size_t bufsize) /* explicitly check for float arguments when integers are expected. For now * signal a warning. Returns true if an exception was raised. */ static int -float_argument_error(PyObject *arg) +float_argument_warning(PyObject *arg) { if (PyFloat_Check(arg) && PyErr_Warn(PyExc_DeprecationWarning, @@ -536,6 +554,20 @@ float_argument_error(PyObject *arg) return 0; } +/* explicitly check for float arguments when integers are expected. Raises + TypeError and returns true for float arguments. */ +static int +float_argument_error(PyObject *arg) +{ + if (PyFloat_Check(arg)) { + PyErr_SetString(PyExc_TypeError, + "integer argument expected, got float"); + return 1; + } + else + return 0; +} + /* Convert a non-tuple argument. Return NULL if conversion went OK, or a string with a message describing the failure. The message is formatted as "must be <desired type>, not <actual type>". @@ -553,7 +585,17 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, #define FETCH_SIZE int *q=NULL;Py_ssize_t *q2=NULL;\ if (flags & FLAG_SIZE_T) q2=va_arg(*p_va, Py_ssize_t*); \ else q=va_arg(*p_va, int*); -#define STORE_SIZE(s) if (flags & FLAG_SIZE_T) *q2=s; else *q=s; +#define STORE_SIZE(s) \ + if (flags & FLAG_SIZE_T) \ + *q2=s; \ + else { \ + if (INT_MAX < s) { \ + PyErr_SetString(PyExc_OverflowError, \ + "size does not fit in an int"); \ + return converterr("", arg, msgbuf, bufsize); \ + } \ + *q=s; \ + } #define BUFFER_LEN ((flags & FLAG_SIZE_T) ? *q2:*q) const char *format = *p_format; @@ -719,7 +761,10 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, #ifdef HAVE_LONG_LONG case 'L': {/* PY_LONG_LONG */ PY_LONG_LONG *p = va_arg( *p_va, PY_LONG_LONG * ); - PY_LONG_LONG ival = PyLong_AsLongLong( arg ); + PY_LONG_LONG ival; + if (float_argument_warning(arg)) + return converterr("long<L>", arg, msgbuf, bufsize); + ival = PyLong_AsLongLong(arg); if (ival == (PY_LONG_LONG)-1 && PyErr_Occurred() ) { return converterr("long<L>", arg, msgbuf, bufsize); } else { @@ -949,10 +994,11 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, if (*format == '#') { FETCH_SIZE; assert(0); /* XXX redundant with if-case */ - if (arg == Py_None) - *q = 0; - else - *q = PyString_Size(arg); + if (arg == Py_None) { + STORE_SIZE(0); + } else { + STORE_SIZE(PyString_Size(arg)); + } format++; } else if (*p != NULL && @@ -1364,7 +1410,7 @@ getbuffer(PyObject *arg, Py_buffer *view, char **errmsg) *errmsg = "convertible to a buffer"; return count; } - PyBuffer_FillInfo(view, NULL, buf, count, 1, 0); + PyBuffer_FillInfo(view, arg, buf, count, 1, 0); return 0; } @@ -1732,16 +1778,6 @@ skipitem(const char **p_format, va_list *p_va, int flags) (void) va_arg(*p_va, PyTypeObject*); (void) va_arg(*p_va, PyObject **); } -#if 0 -/* I don't know what this is for */ - else if (*format == '?') { - inquiry pred = va_arg(*p_va, inquiry); - format++; - if ((*pred)(arg)) { - (void) va_arg(*p_va, PyObject **); - } - } -#endif else if (*format == '&') { typedef int (*converter)(PyObject *, void *); (void) va_arg(*p_va, converter); diff --git a/Python/getcwd.c b/Python/getcwd.c index 4bedbd1f75..44dc9e69ab 100644 --- a/Python/getcwd.c +++ b/Python/getcwd.c @@ -59,14 +59,13 @@ getcwd(char *buf, int size) { FILE *fp; char *p; - int sts; if (size <= 0) { errno = EINVAL; return NULL; } if ((fp = popen(PWD_CMD, "r")) == NULL) return NULL; - if (fgets(buf, size, fp) == NULL || (sts = pclose(fp)) != 0) { + if (fgets(buf, size, fp) == NULL || pclose(fp) != 0) { errno = EACCES; /* Most likely error */ return NULL; } diff --git a/Python/getmtime.c b/Python/getmtime.c deleted file mode 100644 index 54edb531df..0000000000 --- a/Python/getmtime.c +++ /dev/null @@ -1,26 +0,0 @@ - -/* Subroutine to get the last modification time of a file */ - -/* (A separate file because this may be OS dependent) */ - -#include "Python.h" -#include "pyconfig.h" - -#ifdef __cplusplus -extern "C" { -#endif - -time_t -PyOS_GetLastModificationTime(char *path, FILE *fp) -{ - struct stat st; - if (fstat(fileno(fp), &st) != 0) - return -1; - else - return st.st_mtime; -} - -#ifdef __cplusplus -} -#endif - diff --git a/Python/graminit.c b/Python/graminit.c index c8cf35a74a..5db6bb5e53 100644 --- a/Python/graminit.c +++ b/Python/graminit.c @@ -901,42 +901,43 @@ static arc arcs_41_0[1] = { {100, 1}, }; static arc arcs_41_1[1] = { - {28, 2}, + {101, 2}, }; static arc arcs_41_2[2] = { - {101, 3}, - {23, 4}, + {29, 1}, + {23, 3}, }; static arc arcs_41_3[1] = { - {23, 4}, + {24, 4}, }; static arc arcs_41_4[1] = { - {24, 5}, -}; -static arc arcs_41_5[1] = { - {0, 5}, + {0, 4}, }; -static state states_41[6] = { +static state states_41[5] = { {1, arcs_41_0}, {1, arcs_41_1}, {2, arcs_41_2}, {1, arcs_41_3}, {1, arcs_41_4}, - {1, arcs_41_5}, }; static arc arcs_42_0[1] = { - {80, 1}, + {28, 1}, }; -static arc arcs_42_1[1] = { - {84, 2}, +static arc arcs_42_1[2] = { + {80, 2}, + {0, 1}, }; static arc arcs_42_2[1] = { - {0, 2}, + {84, 3}, }; -static state states_42[3] = { +static arc arcs_42_3[1] = { + {0, 3}, +}; +static state states_42[4] = { {1, arcs_42_0}, - {1, arcs_42_1}, + {2, arcs_42_1}, {1, arcs_42_2}, + {1, arcs_42_3}, }; static arc arcs_43_0[1] = { {102, 1}, @@ -1525,26 +1526,57 @@ static state states_71[3] = { static arc arcs_72_0[1] = { {28, 1}, }; -static arc arcs_72_1[1] = { +static arc arcs_72_1[4] = { {23, 2}, + {157, 3}, + {29, 4}, + {0, 1}, }; static arc arcs_72_2[1] = { - {28, 3}, + {28, 5}, }; -static arc arcs_72_3[2] = { - {29, 4}, +static arc arcs_72_3[1] = { {0, 3}, }; static arc arcs_72_4[2] = { - {28, 1}, + {28, 6}, {0, 4}, }; -static state states_72[5] = { +static arc arcs_72_5[3] = { + {157, 3}, + {29, 7}, + {0, 5}, +}; +static arc arcs_72_6[2] = { + {29, 4}, + {0, 6}, +}; +static arc arcs_72_7[2] = { + {28, 8}, + {0, 7}, +}; +static arc arcs_72_8[1] = { + {23, 9}, +}; +static arc arcs_72_9[1] = { + {28, 10}, +}; +static arc arcs_72_10[2] = { + {29, 7}, + {0, 10}, +}; +static state states_72[11] = { {1, arcs_72_0}, - {1, arcs_72_1}, + {4, arcs_72_1}, {1, arcs_72_2}, - {2, arcs_72_3}, + {1, arcs_72_3}, {2, arcs_72_4}, + {3, arcs_72_5}, + {2, arcs_72_6}, + {2, arcs_72_7}, + {1, arcs_72_8}, + {1, arcs_72_9}, + {2, arcs_72_10}, }; static arc arcs_73_0[1] = { {161, 1}, @@ -1877,10 +1909,10 @@ static dfa dfas[85] = { "\000\000\000\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000\000"}, {296, "try_stmt", 0, 13, states_40, "\000\000\000\000\000\000\000\000\000\000\000\000\002\000\000\000\000\000\000\000\000\000"}, - {297, "with_stmt", 0, 6, states_41, + {297, "with_stmt", 0, 5, states_41, "\000\000\000\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000"}, - {298, "with_var", 0, 3, states_42, - "\000\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000\000\000\000"}, + {298, "with_item", 0, 4, states_42, + "\000\040\040\000\000\000\000\000\000\000\000\000\000\040\010\000\200\041\044\015\000\000"}, {299, "except_clause", 0, 5, states_43, "\000\000\000\000\000\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000\000"}, {300, "suite", 0, 5, states_44, @@ -1923,7 +1955,7 @@ static dfa dfas[85] = { "\000\040\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\044\015\000\000"}, {319, "listmaker", 0, 5, states_63, "\000\040\040\000\000\000\000\000\000\000\000\000\000\040\010\000\200\041\044\015\000\000"}, - {320, "testlist_gexp", 0, 5, states_64, + {320, "testlist_comp", 0, 5, states_64, "\000\040\040\000\000\000\000\000\000\000\000\000\000\040\010\000\200\041\044\015\000\000"}, {321, "lambdef", 0, 5, states_65, "\000\000\000\000\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000"}, @@ -1939,7 +1971,7 @@ static dfa dfas[85] = { "\000\040\040\000\000\000\000\000\000\000\000\000\000\000\000\000\200\041\044\015\000\000"}, {327, "testlist", 0, 3, states_71, "\000\040\040\000\000\000\000\000\000\000\000\000\000\040\010\000\200\041\044\015\000\000"}, - {328, "dictmaker", 0, 5, states_72, + {328, "dictorsetmaker", 0, 11, states_72, "\000\040\040\000\000\000\000\000\000\000\000\000\000\040\010\000\200\041\044\015\000\000"}, {329, "classdef", 0, 8, states_73, "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\002\000"}, @@ -1953,11 +1985,11 @@ static dfa dfas[85] = { "\000\000\000\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000\000"}, {334, "list_if", 0, 4, states_78, "\000\000\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000"}, - {335, "gen_iter", 0, 2, states_79, + {335, "comp_iter", 0, 2, states_79, "\000\000\000\000\000\000\000\000\000\000\000\020\001\000\000\000\000\000\000\000\000\000"}, - {336, "gen_for", 0, 6, states_80, + {336, "comp_for", 0, 6, states_80, "\000\000\000\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000\000"}, - {337, "gen_if", 0, 4, states_81, + {337, "comp_if", 0, 4, states_81, "\000\000\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000"}, {338, "testlist1", 0, 2, states_82, "\000\040\040\000\000\000\000\000\000\000\000\000\000\040\010\000\200\041\044\015\000\000"}, diff --git a/Python/import.c b/Python/import.c index 07f572000f..4d8a610c3d 100644 --- a/Python/import.c +++ b/Python/import.c @@ -27,8 +27,6 @@ extern "C" { typedef unsigned short mode_t; #endif -extern time_t PyOS_GetLastModificationTime(char *, FILE *); - /* In getmtime.c */ /* Magic word to reject .pyc files generated by other Python versions. It should change for each incompatible change to the bytecode. @@ -73,9 +71,15 @@ extern time_t PyOS_GetLastModificationTime(char *, FILE *); Python 2.5c2: 62131 (fix wrong code: for x, in ... in listcomp/genexp) Python 2.6a0: 62151 (peephole optimizations and STORE_MAP opcode) Python 2.6a1: 62161 (WITH_CLEANUP optimization) + Python 2.7a0: 62171 (optimize list comprehensions/change LIST_APPEND) + Python 2.7a0: 62181 (optimize conditional branches: + introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE) + Python 2.7a0 62191 (introduce SETUP_WITH) + Python 2.7a0 62201 (introduce BUILD_SET) + Python 2.7a0 62211 (introduce MAP_ADD and SET_ADD) . */ -#define MAGIC (62161 | ((long)'\r'<<16) | ((long)'\n'<<24)) +#define MAGIC (62211 | ((long)'\r'<<16) | ((long)'\n'<<24)) /* Magic word as global; note that _PyImport_Init() can change the value of this global to accommodate for alterations of how the @@ -110,6 +114,34 @@ static const struct filedescr _PyImport_StandardFiletab[] = { }; #endif +#ifdef MS_WINDOWS +int isdir(char *path) { + DWORD rv; + /* see issue1293 and issue3677: + * stat() on Windows doesn't recognise paths like + * "e:\\shared\\" and "\\\\whiterab-c2znlh\\shared" as dirs. + * Also reference issue6727: + * stat() on Windows is broken and doesn't resolve symlinks properly. + */ + rv = GetFileAttributesA(path); + return rv != INVALID_FILE_ATTRIBUTES && rv & FILE_ATTRIBUTE_DIRECTORY; +} +#else +#ifdef HAVE_STAT +int isdir(char *path) { + struct stat statbuf; + return stat(path, &statbuf) == 0 && S_ISDIR(statbuf.st_mode); +} +#else +#ifdef RISCOS +/* with RISCOS, isdir is in unixstuff */ +#else +int isdir(char *path) { + return 0; +} +#endif /* RISCOS */ +#endif /* HAVE_STAT */ +#endif /* MS_WINDOWS */ /* Initialize things */ @@ -619,7 +651,7 @@ PyImport_AddModule(const char *name) /* Remove name from sys.modules, if it's there. */ static void -_RemoveModule(const char *name) +remove_module(const char *name) { PyObject *modules = PyImport_GetModuleDict(); if (PyDict_GetItemString(modules, name) == NULL) @@ -691,7 +723,7 @@ PyImport_ExecCodeModuleEx(char *name, PyObject *co, char *pathname) return m; error: - _RemoveModule(name); + remove_module(name); return NULL; } @@ -901,9 +933,9 @@ write_compiled_module(PyCodeObject *co, char *cpathname, struct stat *srcstat) (void) unlink(cpathname); return; } - /* Now write the true mtime */ + /* Now write the true mtime (as a 32-bit field) */ fseek(fp, 4L, 0); - assert(mtime < LONG_MAX); + assert(mtime <= 0xFFFFFFFF); PyMarshal_WriteLongToFile((long)mtime, fp, Py_MARSHAL_VERSION); fflush(fp); fclose(fp); @@ -964,7 +996,7 @@ load_source_module(char *name, char *pathname, FILE *fp) { struct stat st; FILE *fpc; - char buf[MAXPATHLEN+1]; + char *buf; char *cpathname; PyCodeObject *co; PyObject *m; @@ -975,17 +1007,18 @@ load_source_module(char *name, char *pathname, FILE *fp) pathname); return NULL; } -#if SIZEOF_TIME_T > 4 - /* Python's .pyc timestamp handling presumes that the timestamp fits - in 4 bytes. This will be fine until sometime in the year 2038, - when a 4-byte signed time_t will overflow. - */ - if (st.st_mtime >> 32) { - PyErr_SetString(PyExc_OverflowError, - "modification time overflows a 4 byte field"); - return NULL; + if (sizeof st.st_mtime > 4) { + /* Python's .pyc timestamp handling presumes that the timestamp fits + in 4 bytes. Since the code only does an equality comparison, + ordering is not important and we can safely ignore the higher bits + (collisions are extremely unlikely). + */ + st.st_mtime &= 0xFFFFFFFF; + } + buf = PyMem_MALLOC(MAXPATHLEN+1); + if (buf == NULL) { + return PyErr_NoMemory(); } -#endif cpathname = make_compiled_pathname(pathname, buf, (size_t)MAXPATHLEN + 1); if (cpathname != NULL && @@ -993,9 +1026,9 @@ load_source_module(char *name, char *pathname, FILE *fp) co = read_compiled_module(cpathname, fpc); fclose(fpc); if (co == NULL) - return NULL; + goto error_exit; if (update_compiled_module(co, pathname) < 0) - return NULL; + goto error_exit; if (Py_VerboseFlag) PySys_WriteStderr("import %s # precompiled from %s\n", name, cpathname); @@ -1004,7 +1037,7 @@ load_source_module(char *name, char *pathname, FILE *fp) else { co = parse_source_module(pathname, fp); if (co == NULL) - return NULL; + goto error_exit; if (Py_VerboseFlag) PySys_WriteStderr("import %s # from %s\n", name, pathname); @@ -1017,7 +1050,12 @@ load_source_module(char *name, char *pathname, FILE *fp) m = PyImport_ExecCodeModuleEx(name, (PyObject *)co, pathname); Py_DECREF(co); + PyMem_FREE(buf); return m; + +error_exit: + PyMem_FREE(buf); + return NULL; } @@ -1037,7 +1075,7 @@ load_package(char *name, char *pathname) PyObject *file = NULL; PyObject *path = NULL; int err; - char buf[MAXPATHLEN+1]; + char *buf = NULL; FILE *fp = NULL; struct filedescr *fdp; @@ -1059,8 +1097,13 @@ load_package(char *name, char *pathname) err = PyDict_SetItemString(d, "__path__", path); if (err != 0) goto error; + buf = PyMem_MALLOC(MAXPATHLEN+1); + if (buf == NULL) { + PyErr_NoMemory(); + goto error; + } buf[0] = '\0'; - fdp = find_module(name, "__init__", path, buf, sizeof(buf), &fp, NULL); + fdp = find_module(name, "__init__", path, buf, MAXPATHLEN+1, &fp, NULL); if (fdp == NULL) { if (PyErr_ExceptionMatches(PyExc_ImportError)) { PyErr_Clear(); @@ -1078,6 +1121,8 @@ load_package(char *name, char *pathname) error: m = NULL; cleanup: + if (buf) + PyMem_FREE(buf); Py_XDECREF(path); Py_XDECREF(file); return m; @@ -1203,13 +1248,10 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, char *filemode; FILE *fp = NULL; PyObject *path_hooks, *path_importer_cache; -#ifndef RISCOS - struct stat statbuf; -#endif static struct filedescr fd_frozen = {"", "", PY_FROZEN}; static struct filedescr fd_builtin = {"", "", C_BUILTIN}; static struct filedescr fd_package = {"", "", PKG_DIRECTORY}; - char name[MAXPATHLEN+1]; + char *name; #if defined(PYOS_OS2) size_t saved_len; size_t saved_namelen; @@ -1223,6 +1265,10 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, "module name is too long"); return NULL; } + name = PyMem_MALLOC(MAXPATHLEN+1); + if (name == NULL) { + return PyErr_NoMemory(); + } strcpy(name, subname); /* sys.meta_path import hook */ @@ -1231,10 +1277,10 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, meta_path = PySys_GetObject("meta_path"); if (meta_path == NULL || !PyList_Check(meta_path)) { - PyErr_SetString(PyExc_ImportError, + PyErr_SetString(PyExc_RuntimeError, "sys.meta_path must be a list of " "import hooks"); - return NULL; + goto error_exit; } Py_INCREF(meta_path); /* zap guard */ npath = PyList_Size(meta_path); @@ -1247,12 +1293,13 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, path : Py_None); if (loader == NULL) { Py_DECREF(meta_path); - return NULL; /* true error */ + goto error_exit; /* true error */ } if (loader != Py_None) { /* a loader was found */ *p_loader = loader; Py_DECREF(meta_path); + PyMem_FREE(name); return &importhookdescr; } Py_DECREF(loader); @@ -1266,7 +1313,7 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, if (PyString_Size(path) + 1 + strlen(name) >= (size_t)buflen) { PyErr_SetString(PyExc_ImportError, "full frozen module name too long"); - return NULL; + goto error_exit; } strcpy(buf, PyString_AsString(path)); strcat(buf, "."); @@ -1274,19 +1321,22 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, strcpy(name, buf); if (find_frozen(name) != NULL) { strcpy(buf, name); + PyMem_FREE(name); return &fd_frozen; } PyErr_Format(PyExc_ImportError, "No frozen submodule named %.200s", name); - return NULL; + goto error_exit; } if (path == NULL) { if (is_builtin(name)) { strcpy(buf, name); + PyMem_FREE(name); return &fd_builtin; } if ((find_frozen(name)) != NULL) { strcpy(buf, name); + PyMem_FREE(name); return &fd_frozen; } @@ -1294,30 +1344,31 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, fp = PyWin_FindRegisteredModule(name, &fdp, buf, buflen); if (fp != NULL) { *p_fp = fp; + PyMem_FREE(name); return fdp; } #endif path = PySys_GetObject("path"); } if (path == NULL || !PyList_Check(path)) { - PyErr_SetString(PyExc_ImportError, + PyErr_SetString(PyExc_RuntimeError, "sys.path must be a list of directory names"); - return NULL; + goto error_exit; } path_hooks = PySys_GetObject("path_hooks"); if (path_hooks == NULL || !PyList_Check(path_hooks)) { - PyErr_SetString(PyExc_ImportError, + PyErr_SetString(PyExc_RuntimeError, "sys.path_hooks must be a list of " "import hooks"); - return NULL; + goto error_exit; } path_importer_cache = PySys_GetObject("path_importer_cache"); if (path_importer_cache == NULL || !PyDict_Check(path_importer_cache)) { - PyErr_SetString(PyExc_ImportError, + PyErr_SetString(PyExc_RuntimeError, "sys.path_importer_cache must be a dict"); - return NULL; + goto error_exit; } npath = PyList_Size(path); @@ -1326,13 +1377,13 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, PyObject *copy = NULL; PyObject *v = PyList_GetItem(path, i); if (!v) - return NULL; + goto error_exit; #ifdef Py_USING_UNICODE if (PyUnicode_Check(v)) { copy = PyUnicode_Encode(PyUnicode_AS_UNICODE(v), PyUnicode_GET_SIZE(v), Py_FileSystemDefaultEncoding, NULL); if (copy == NULL) - return NULL; + goto error_exit; v = copy; } else @@ -1358,7 +1409,7 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, path_hooks, v); if (importer == NULL) { Py_XDECREF(copy); - return NULL; + goto error_exit; } /* Note: importer is a borrowed reference */ if (importer != Py_None) { @@ -1368,10 +1419,11 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, "s", fullname); Py_XDECREF(copy); if (loader == NULL) - return NULL; /* error */ + goto error_exit; /* error */ if (loader != Py_None) { /* a loader was found */ *p_loader = loader; + PyMem_FREE(name); return &importhookdescr; } Py_DECREF(loader); @@ -1391,12 +1443,11 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, /* Check for package import (buf holds a directory name, and there's an __init__ module in that directory */ -#ifdef HAVE_STAT - if (stat(buf, &statbuf) == 0 && /* it exists */ - S_ISDIR(statbuf.st_mode) && /* it's a directory */ + if (isdir(buf) && /* it's an existing directory */ case_ok(buf, len, namelen, name)) { /* case matches */ if (find_init_module(buf)) { /* and has __init__.py */ Py_XDECREF(copy); + PyMem_FREE(name); return &fd_package; } else { @@ -1407,32 +1458,10 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, if (PyErr_Warn(PyExc_ImportWarning, warnstr)) { Py_XDECREF(copy); - return NULL; + goto error_exit; } } } -#else - /* XXX How are you going to test for directories? */ -#ifdef RISCOS - if (isdir(buf) && - case_ok(buf, len, namelen, name)) { - if (find_init_module(buf)) { - Py_XDECREF(copy); - return &fd_package; - } - else { - char warnstr[MAXPATHLEN+80]; - sprintf(warnstr, "Not importing directory " - "'%.*s': missing __init__.py", - MAXPATHLEN, buf); - if (PyErr_Warn(PyExc_ImportWarning, - warnstr)) { - Py_XDECREF(copy); - return NULL; - } - } -#endif -#endif #if defined(PYOS_OS2) /* take a snapshot of the module spec for restoration * after the 8 character DLL hackery @@ -1504,10 +1533,15 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, if (fp == NULL) { PyErr_Format(PyExc_ImportError, "No module named %.200s", name); - return NULL; + goto error_exit; } *p_fp = fp; + PyMem_FREE(name); return fdp; + +error_exit: + PyMem_FREE(name); + return NULL; } /* Helpers for main.c @@ -1797,7 +1831,7 @@ static int init_builtin(char *); /* Forward */ its module object WITH INCREMENTED REFERENCE COUNT */ static PyObject * -load_module(char *name, FILE *fp, char *buf, int type, PyObject *loader) +load_module(char *name, FILE *fp, char *pathname, int type, PyObject *loader) { PyObject *modules; PyObject *m; @@ -1818,27 +1852,27 @@ load_module(char *name, FILE *fp, char *buf, int type, PyObject *loader) switch (type) { case PY_SOURCE: - m = load_source_module(name, buf, fp); + m = load_source_module(name, pathname, fp); break; case PY_COMPILED: - m = load_compiled_module(name, buf, fp); + m = load_compiled_module(name, pathname, fp); break; #ifdef HAVE_DYNAMIC_LOADING case C_EXTENSION: - m = _PyImport_LoadDynamicModule(name, buf, fp); + m = _PyImport_LoadDynamicModule(name, pathname, fp); break; #endif case PKG_DIRECTORY: - m = load_package(name, buf); + m = load_package(name, pathname); break; case C_BUILTIN: case PY_FROZEN: - if (buf != NULL && buf[0] != '\0') - name = buf; + if (pathname != NULL && pathname[0] != '\0') + name = pathname; if (type == C_BUILTIN) err = init_builtin(name); else @@ -2060,7 +2094,9 @@ PyImport_ImportModuleNoBlock(const char *name) { PyObject *result; PyObject *modules; +#ifdef WITH_THREAD long me; +#endif /* Try to get the module from sys.modules[name] */ modules = PyImport_GetModuleDict(); @@ -2112,7 +2148,7 @@ static PyObject * import_module_level(char *name, PyObject *globals, PyObject *locals, PyObject *fromlist, int level) { - char buf[MAXPATHLEN+1]; + char *buf; Py_ssize_t buflen = 0; PyObject *parent, *head, *next, *tail; @@ -2126,14 +2162,18 @@ import_module_level(char *name, PyObject *globals, PyObject *locals, return NULL; } + buf = PyMem_MALLOC(MAXPATHLEN+1); + if (buf == NULL) { + return PyErr_NoMemory(); + } parent = get_parent(globals, buf, &buflen, level); if (parent == NULL) - return NULL; + goto error_exit; head = load_next(parent, level < 0 ? Py_None : parent, &name, buf, &buflen); if (head == NULL) - return NULL; + goto error_exit; tail = head; Py_INCREF(tail); @@ -2142,7 +2182,7 @@ import_module_level(char *name, PyObject *globals, PyObject *locals, Py_DECREF(tail); if (next == NULL) { Py_DECREF(head); - return NULL; + goto error_exit; } tail = next; } @@ -2154,7 +2194,7 @@ import_module_level(char *name, PyObject *globals, PyObject *locals, Py_DECREF(head); PyErr_SetString(PyExc_ValueError, "Empty module name"); - return NULL; + goto error_exit; } if (fromlist != NULL) { @@ -2164,16 +2204,22 @@ import_module_level(char *name, PyObject *globals, PyObject *locals, if (fromlist == NULL) { Py_DECREF(tail); + PyMem_FREE(buf); return head; } Py_DECREF(head); if (!ensure_fromlist(tail, fromlist, buf, buflen, 0)) { Py_DECREF(tail); - return NULL; + goto error_exit; } + PyMem_FREE(buf); return tail; + +error_exit: + PyMem_FREE(buf); + return NULL; } PyObject * @@ -2563,7 +2609,7 @@ import_submodule(PyObject *mod, char *subname, char *fullname) } else { PyObject *path, *loader = NULL; - char buf[MAXPATHLEN+1]; + char *buf; struct filedescr *fdp; FILE *fp = NULL; @@ -2578,11 +2624,16 @@ import_submodule(PyObject *mod, char *subname, char *fullname) } } + buf = PyMem_MALLOC(MAXPATHLEN+1); + if (buf == NULL) { + return PyErr_NoMemory(); + } buf[0] = '\0'; fdp = find_module(fullname, subname, path, buf, MAXPATHLEN+1, &fp, &loader); Py_XDECREF(path); if (fdp == NULL) { + PyMem_FREE(buf); if (!PyErr_ExceptionMatches(PyExc_ImportError)) return NULL; PyErr_Clear(); @@ -2597,6 +2648,7 @@ import_submodule(PyObject *mod, char *subname, char *fullname) Py_XDECREF(m); m = NULL; } + PyMem_FREE(buf); } return m; @@ -2614,7 +2666,7 @@ PyImport_ReloadModule(PyObject *m) PyObject *modules = PyImport_GetModuleDict(); PyObject *path = NULL, *loader = NULL, *existing_m = NULL; char *name, *subname; - char buf[MAXPATHLEN+1]; + char *buf; struct filedescr *fdp; FILE *fp = NULL; PyObject *newm; @@ -2674,6 +2726,11 @@ PyImport_ReloadModule(PyObject *m) if (path == NULL) PyErr_Clear(); } + buf = PyMem_MALLOC(MAXPATHLEN+1); + if (buf == NULL) { + Py_XDECREF(path); + return PyErr_NoMemory(); + } buf[0] = '\0'; fdp = find_module(name, subname, path, buf, MAXPATHLEN+1, &fp, &loader); Py_XDECREF(path); @@ -2681,6 +2738,7 @@ PyImport_ReloadModule(PyObject *m) if (fdp == NULL) { Py_XDECREF(loader); imp_modules_reloading_clear(); + PyMem_FREE(buf); return NULL; } @@ -2698,6 +2756,7 @@ PyImport_ReloadModule(PyObject *m) PyDict_SetItemString(modules, name, m); } imp_modules_reloading_clear(); + PyMem_FREE(buf); return newm; } @@ -2828,19 +2887,25 @@ call_find_module(char *name, PyObject *path) extern int fclose(FILE *); PyObject *fob, *ret; struct filedescr *fdp; - char pathname[MAXPATHLEN+1]; + char *pathname; FILE *fp = NULL; + pathname = PyMem_MALLOC(MAXPATHLEN+1); + if (pathname == NULL) { + return PyErr_NoMemory(); + } pathname[0] = '\0'; if (path == Py_None) path = NULL; fdp = find_module(NULL, name, path, pathname, MAXPATHLEN+1, &fp, NULL); - if (fdp == NULL) + if (fdp == NULL) { + PyMem_FREE(pathname); return NULL; + } if (fp != NULL) { fob = PyFile_FromFile(fp, pathname, fdp->mode, fclose); if (fob == NULL) { - fclose(fp); + PyMem_FREE(pathname); return NULL; } } @@ -2851,6 +2916,7 @@ call_find_module(char *name, PyObject *path) ret = Py_BuildValue("Os(ssi)", fob, pathname, fdp->suffix, fdp->mode, fdp->type); Py_DECREF(fob); + PyMem_FREE(pathname); return ret; } @@ -3198,49 +3264,11 @@ NullImporter_init(NullImporter *self, PyObject *args, PyObject *kwds) PyErr_SetString(PyExc_ImportError, "empty pathname"); return -1; } else { -#ifndef RISCOS -#ifndef MS_WINDOWS - struct stat statbuf; - int rv; - - rv = stat(path, &statbuf); - if (rv == 0) { - /* it exists */ - if (S_ISDIR(statbuf.st_mode)) { - /* it's a directory */ - PyErr_SetString(PyExc_ImportError, - "existing directory"); - return -1; - } - } -#else /* MS_WINDOWS */ - DWORD rv; - /* see issue1293 and issue3677: - * stat() on Windows doesn't recognise paths like - * "e:\\shared\\" and "\\\\whiterab-c2znlh\\shared" as dirs. - */ - rv = GetFileAttributesA(path); - if (rv != INVALID_FILE_ATTRIBUTES) { - /* it exists */ - if (rv & FILE_ATTRIBUTE_DIRECTORY) { - /* it's a directory */ - PyErr_SetString(PyExc_ImportError, - "existing directory"); - return -1; - } - } -#endif -#else /* RISCOS */ - if (object_exists(path)) { - /* it exists */ - if (isdir(path)) { - /* it's a directory */ - PyErr_SetString(PyExc_ImportError, - "existing directory"); - return -1; - } + if(isdir(path)) { + PyErr_SetString(PyExc_ImportError, + "existing directory"); + return -1; } -#endif } return 0; } @@ -3375,13 +3403,13 @@ PyImport_ExtendInittab(struct _inittab *newtab) /* Shorthand to add a single entry given a name and a function */ int -PyImport_AppendInittab(char *name, void (*initfunc)(void)) +PyImport_AppendInittab(const char *name, void (*initfunc)(void)) { struct _inittab newtab[2]; memset(newtab, '\0', sizeof newtab); - newtab[0].name = name; + newtab[0].name = (char *)name; newtab[0].initfunc = initfunc; return PyImport_ExtendInittab(newtab); diff --git a/Python/marshal.c b/Python/marshal.c index bcc8636009..faca027d4c 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -11,6 +11,8 @@ #include "code.h" #include "marshal.h" +#define ABS(x) ((x) < 0 ? -(x) : (x)) + /* High water mark to determine when the marshalled object is dangerously deep * and risks coring the interpreter. When the object stack gets this deep, * raise an exception instead of continuing. @@ -42,9 +44,14 @@ #define TYPE_SET '<' #define TYPE_FROZENSET '>' +#define WFERR_OK 0 +#define WFERR_UNMARSHALLABLE 1 +#define WFERR_NESTEDTOODEEP 2 +#define WFERR_NOMEMORY 3 + typedef struct { FILE *fp; - int error; + int error; /* see WFERR_* values */ int depth; /* If fp == NULL, the following are valid: */ PyObject *str; @@ -119,6 +126,56 @@ w_long64(long x, WFILE *p) } #endif +/* We assume that Python longs are stored internally in base some power of + 2**15; for the sake of portability we'll always read and write them in base + exactly 2**15. */ + +#define PyLong_MARSHAL_SHIFT 15 +#define PyLong_MARSHAL_BASE ((short)1 << PyLong_MARSHAL_SHIFT) +#define PyLong_MARSHAL_MASK (PyLong_MARSHAL_BASE - 1) +#if PyLong_SHIFT % PyLong_MARSHAL_SHIFT != 0 +#error "PyLong_SHIFT must be a multiple of PyLong_MARSHAL_SHIFT" +#endif +#define PyLong_MARSHAL_RATIO (PyLong_SHIFT / PyLong_MARSHAL_SHIFT) + +static void +w_PyLong(const PyLongObject *ob, WFILE *p) +{ + Py_ssize_t i, j, n, l; + digit d; + + w_byte(TYPE_LONG, p); + if (Py_SIZE(ob) == 0) { + w_long((long)0, p); + return; + } + + /* set l to number of base PyLong_MARSHAL_BASE digits */ + n = ABS(Py_SIZE(ob)); + l = (n-1) * PyLong_MARSHAL_RATIO; + d = ob->ob_digit[n-1]; + assert(d != 0); /* a PyLong is always normalized */ + do { + d >>= PyLong_MARSHAL_SHIFT; + l++; + } while (d != 0); + w_long((long)(Py_SIZE(ob) > 0 ? l : -l), p); + + for (i=0; i < n-1; i++) { + d = ob->ob_digit[i]; + for (j=0; j < PyLong_MARSHAL_RATIO; j++) { + w_short(d & PyLong_MARSHAL_MASK, p); + d >>= PyLong_MARSHAL_SHIFT; + } + assert (d == 0); + } + d = ob->ob_digit[n-1]; + do { + w_short(d & PyLong_MARSHAL_MASK, p); + d >>= PyLong_MARSHAL_SHIFT; + } while (d != 0); +} + static void w_object(PyObject *v, WFILE *p) { @@ -127,7 +184,7 @@ w_object(PyObject *v, WFILE *p) p->depth++; if (p->depth > MAX_MARSHAL_STACK_DEPTH) { - p->error = 2; + p->error = WFERR_NESTEDTOODEEP; } else if (v == NULL) { w_byte(TYPE_NULL, p); @@ -164,32 +221,31 @@ w_object(PyObject *v, WFILE *p) } else if (PyLong_CheckExact(v)) { PyLongObject *ob = (PyLongObject *)v; - w_byte(TYPE_LONG, p); - n = ob->ob_size; - w_long((long)n, p); - if (n < 0) - n = -n; - for (i = 0; i < n; i++) - w_short(ob->ob_digit[i], p); + w_PyLong(ob, p); } else if (PyFloat_CheckExact(v)) { if (p->version > 1) { unsigned char buf[8]; if (_PyFloat_Pack8(PyFloat_AsDouble(v), buf, 1) < 0) { - p->error = 1; + p->error = WFERR_UNMARSHALLABLE; return; } w_byte(TYPE_BINARY_FLOAT, p); w_string((char*)buf, 8, p); } else { - char buf[256]; /* Plenty to format any double */ - PyFloat_AsReprString(buf, (PyFloatObject *)v); + char *buf = PyOS_double_to_string(PyFloat_AS_DOUBLE(v), + 'g', 17, 0, NULL); + if (!buf) { + p->error = WFERR_NOMEMORY; + return; + } n = strlen(buf); w_byte(TYPE_FLOAT, p); w_byte((int)n, p); w_string(buf, (int)n, p); + PyMem_Free(buf); } } #ifndef WITHOUT_COMPLEX @@ -198,44 +254,41 @@ w_object(PyObject *v, WFILE *p) unsigned char buf[8]; if (_PyFloat_Pack8(PyComplex_RealAsDouble(v), buf, 1) < 0) { - p->error = 1; + p->error = WFERR_UNMARSHALLABLE; return; } w_byte(TYPE_BINARY_COMPLEX, p); w_string((char*)buf, 8, p); if (_PyFloat_Pack8(PyComplex_ImagAsDouble(v), buf, 1) < 0) { - p->error = 1; + p->error = WFERR_UNMARSHALLABLE; return; } w_string((char*)buf, 8, p); } else { - char buf[256]; /* Plenty to format any double */ - PyFloatObject *temp; + char *buf; w_byte(TYPE_COMPLEX, p); - temp = (PyFloatObject*)PyFloat_FromDouble( - PyComplex_RealAsDouble(v)); - if (!temp) { - p->error = 1; + buf = PyOS_double_to_string(PyComplex_RealAsDouble(v), + 'g', 17, 0, NULL); + if (!buf) { + p->error = WFERR_NOMEMORY; return; } - PyFloat_AsReprString(buf, temp); - Py_DECREF(temp); n = strlen(buf); w_byte((int)n, p); w_string(buf, (int)n, p); - temp = (PyFloatObject*)PyFloat_FromDouble( - PyComplex_ImagAsDouble(v)); - if (!temp) { - p->error = 1; + PyMem_Free(buf); + buf = PyOS_double_to_string(PyComplex_ImagAsDouble(v), + 'g', 17, 0, NULL); + if (!buf) { + p->error = WFERR_NOMEMORY; return; } - PyFloat_AsReprString(buf, temp); - Py_DECREF(temp); n = strlen(buf); w_byte((int)n, p); w_string(buf, (int)n, p); + PyMem_Free(buf); } } #endif @@ -256,7 +309,7 @@ w_object(PyObject *v, WFILE *p) Py_XDECREF(o); if (!ok) { p->depth--; - p->error = 1; + p->error = WFERR_UNMARSHALLABLE; return; } w_byte(TYPE_INTERNED, p); @@ -269,7 +322,7 @@ w_object(PyObject *v, WFILE *p) if (n > INT_MAX) { /* huge strings are not supported */ p->depth--; - p->error = 1; + p->error = WFERR_UNMARSHALLABLE; return; } w_long((long)n, p); @@ -281,14 +334,14 @@ w_object(PyObject *v, WFILE *p) utf8 = PyUnicode_AsUTF8String(v); if (utf8 == NULL) { p->depth--; - p->error = 1; + p->error = WFERR_UNMARSHALLABLE; return; } w_byte(TYPE_UNICODE, p); n = PyString_GET_SIZE(utf8); if (n > INT_MAX) { p->depth--; - p->error = 1; + p->error = WFERR_UNMARSHALLABLE; return; } w_long((long)n, p); @@ -334,14 +387,14 @@ w_object(PyObject *v, WFILE *p) n = PyObject_Size(v); if (n == -1) { p->depth--; - p->error = 1; + p->error = WFERR_UNMARSHALLABLE; return; } w_long((long)n, p); it = PyObject_GetIter(v); if (it == NULL) { p->depth--; - p->error = 1; + p->error = WFERR_UNMARSHALLABLE; return; } while ((value = PyIter_Next(it)) != NULL) { @@ -351,7 +404,7 @@ w_object(PyObject *v, WFILE *p) Py_DECREF(it); if (PyErr_Occurred()) { p->depth--; - p->error = 1; + p->error = WFERR_UNMARSHALLABLE; return; } } @@ -381,7 +434,7 @@ w_object(PyObject *v, WFILE *p) n = (*pb->bf_getreadbuffer)(v, 0, (void **)&s); if (n > INT_MAX) { p->depth--; - p->error = 1; + p->error = WFERR_UNMARSHALLABLE; return; } w_long((long)n, p); @@ -389,7 +442,7 @@ w_object(PyObject *v, WFILE *p) } else { w_byte(TYPE_UNKNOWN, p); - p->error = 1; + p->error = WFERR_UNMARSHALLABLE; } exit: p->depth--; @@ -401,7 +454,7 @@ PyMarshal_WriteLongToFile(long x, FILE *fp, int version) { WFILE wf; wf.fp = fp; - wf.error = 0; + wf.error = WFERR_OK; wf.depth = 0; wf.strings = NULL; wf.version = version; @@ -413,7 +466,7 @@ PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp, int version) { WFILE wf; wf.fp = fp; - wf.error = 0; + wf.error = WFERR_OK; wf.depth = 0; wf.strings = (version > 0) ? PyDict_New() : NULL; wf.version = version; @@ -507,6 +560,66 @@ r_long64(RFILE *p) } static PyObject * +r_PyLong(RFILE *p) +{ + PyLongObject *ob; + int size, i, j, md, shorts_in_top_digit; + long n; + digit d; + + n = r_long(p); + if (n == 0) + return (PyObject *)_PyLong_New(0); + if (n < -INT_MAX || n > INT_MAX) { + PyErr_SetString(PyExc_ValueError, + "bad marshal data (long size out of range)"); + return NULL; + } + + size = 1 + (ABS(n) - 1) / PyLong_MARSHAL_RATIO; + shorts_in_top_digit = 1 + (ABS(n) - 1) % PyLong_MARSHAL_RATIO; + ob = _PyLong_New(size); + if (ob == NULL) + return NULL; + Py_SIZE(ob) = n > 0 ? size : -size; + + for (i = 0; i < size-1; i++) { + d = 0; + for (j=0; j < PyLong_MARSHAL_RATIO; j++) { + md = r_short(p); + if (md < 0 || md > PyLong_MARSHAL_BASE) + goto bad_digit; + d += (digit)md << j*PyLong_MARSHAL_SHIFT; + } + ob->ob_digit[i] = d; + } + d = 0; + for (j=0; j < shorts_in_top_digit; j++) { + md = r_short(p); + if (md < 0 || md > PyLong_MARSHAL_BASE) + goto bad_digit; + /* topmost marshal digit should be nonzero */ + if (md == 0 && j == shorts_in_top_digit - 1) { + Py_DECREF(ob); + PyErr_SetString(PyExc_ValueError, + "bad marshal data (unnormalized long data)"); + return NULL; + } + d += (digit)md << j*PyLong_MARSHAL_SHIFT; + } + /* top digit should be nonzero, else the resulting PyLong won't be + normalized */ + ob->ob_digit[size-1] = d; + return (PyObject *)ob; + bad_digit: + Py_DECREF(ob); + PyErr_SetString(PyExc_ValueError, + "bad marshal data (digit out of range in long)"); + return NULL; +} + + +static PyObject * r_object(RFILE *p) { /* NULL is a valid return value, it does not necessarily means that @@ -570,39 +683,8 @@ r_object(RFILE *p) break; case TYPE_LONG: - { - int size; - PyLongObject *ob; - n = r_long(p); - if (n < -INT_MAX || n > INT_MAX) { - PyErr_SetString(PyExc_ValueError, - "bad marshal data"); - retval = NULL; - break; - } - size = n<0 ? -n : n; - ob = _PyLong_New(size); - if (ob == NULL) { - retval = NULL; - break; - } - ob->ob_size = n; - for (i = 0; i < size; i++) { - int digit = r_short(p); - if (digit < 0 || - (digit == 0 && i == size-1)) { - Py_DECREF(ob); - PyErr_SetString(PyExc_ValueError, - "bad marshal data"); - ob = NULL; - break; - } - if (ob != NULL) - ob->ob_digit[i] = digit; - } - retval = (PyObject *)ob; - break; - } + retval = r_PyLong(p); + break; case TYPE_FLOAT: { @@ -616,10 +698,11 @@ r_object(RFILE *p) break; } buf[n] = '\0'; - retval = NULL; - PyFPE_START_PROTECT("atof", break) - dx = PyOS_ascii_atof(buf); - PyFPE_END_PROTECT(dx) + dx = PyOS_string_to_double(buf, NULL, NULL); + if (dx == -1.0 && PyErr_Occurred()) { + retval = NULL; + break; + } retval = PyFloat_FromDouble(dx); break; } @@ -656,10 +739,11 @@ r_object(RFILE *p) break; } buf[n] = '\0'; - retval = NULL; - PyFPE_START_PROTECT("atof", break;) - c.real = PyOS_ascii_atof(buf); - PyFPE_END_PROTECT(c) + c.real = PyOS_string_to_double(buf, NULL, NULL); + if (c.real == -1.0 && PyErr_Occurred()) { + retval = NULL; + break; + } n = r_byte(p); if (n == EOF || r_string(buf, (int)n, p) != n) { PyErr_SetString(PyExc_EOFError, @@ -668,9 +752,11 @@ r_object(RFILE *p) break; } buf[n] = '\0'; - PyFPE_START_PROTECT("atof", break) - c.imag = PyOS_ascii_atof(buf); - PyFPE_END_PROTECT(c) + c.imag = PyOS_string_to_double(buf, NULL, NULL); + if (c.imag == -1.0 && PyErr_Occurred()) { + retval = NULL; + break; + } retval = PyComplex_FromCComplex(c); break; } @@ -710,7 +796,7 @@ r_object(RFILE *p) case TYPE_STRING: n = r_long(p); if (n < 0 || n > INT_MAX) { - PyErr_SetString(PyExc_ValueError, "bad marshal data"); + PyErr_SetString(PyExc_ValueError, "bad marshal data (string size out of range)"); retval = NULL; break; } @@ -739,7 +825,7 @@ r_object(RFILE *p) case TYPE_STRINGREF: n = r_long(p); if (n < 0 || n >= PyList_GET_SIZE(p->strings)) { - PyErr_SetString(PyExc_ValueError, "bad marshal data"); + PyErr_SetString(PyExc_ValueError, "bad marshal data (string ref out of range)"); retval = NULL; break; } @@ -755,7 +841,7 @@ r_object(RFILE *p) n = r_long(p); if (n < 0 || n > INT_MAX) { - PyErr_SetString(PyExc_ValueError, "bad marshal data"); + PyErr_SetString(PyExc_ValueError, "bad marshal data (unicode size out of range)"); retval = NULL; break; } @@ -781,7 +867,7 @@ r_object(RFILE *p) case TYPE_TUPLE: n = r_long(p); if (n < 0 || n > INT_MAX) { - PyErr_SetString(PyExc_ValueError, "bad marshal data"); + PyErr_SetString(PyExc_ValueError, "bad marshal data (tuple size out of range)"); retval = NULL; break; } @@ -795,7 +881,7 @@ r_object(RFILE *p) if ( v2 == NULL ) { if (!PyErr_Occurred()) PyErr_SetString(PyExc_TypeError, - "NULL object in marshal data"); + "NULL object in marshal data for tuple"); Py_DECREF(v); v = NULL; break; @@ -808,7 +894,7 @@ r_object(RFILE *p) case TYPE_LIST: n = r_long(p); if (n < 0 || n > INT_MAX) { - PyErr_SetString(PyExc_ValueError, "bad marshal data"); + PyErr_SetString(PyExc_ValueError, "bad marshal data (list size out of range)"); retval = NULL; break; } @@ -822,7 +908,7 @@ r_object(RFILE *p) if ( v2 == NULL ) { if (!PyErr_Occurred()) PyErr_SetString(PyExc_TypeError, - "NULL object in marshal data"); + "NULL object in marshal data for list"); Py_DECREF(v); v = NULL; break; @@ -860,7 +946,7 @@ r_object(RFILE *p) case TYPE_FROZENSET: n = r_long(p); if (n < 0 || n > INT_MAX) { - PyErr_SetString(PyExc_ValueError, "bad marshal data"); + PyErr_SetString(PyExc_ValueError, "bad marshal data (set size out of range)"); retval = NULL; break; } @@ -874,7 +960,7 @@ r_object(RFILE *p) if ( v2 == NULL ) { if (!PyErr_Occurred()) PyErr_SetString(PyExc_TypeError, - "NULL object in marshal data"); + "NULL object in marshal data for set"); Py_DECREF(v); v = NULL; break; @@ -974,7 +1060,7 @@ r_object(RFILE *p) default: /* Bogus data got written, which isn't ideal. This will let you keep working and recover. */ - PyErr_SetString(PyExc_ValueError, "bad marshal data"); + PyErr_SetString(PyExc_ValueError, "bad marshal data (unknown type code)"); retval = NULL; break; @@ -993,7 +1079,7 @@ read_object(RFILE *p) } v = r_object(p); if (v == NULL && !PyErr_Occurred()) - PyErr_SetString(PyExc_TypeError, "NULL object in marshal data"); + PyErr_SetString(PyExc_TypeError, "NULL object in marshal data for object"); return v; } @@ -1040,23 +1126,13 @@ getfilesize(FILE *fp) PyObject * PyMarshal_ReadLastObjectFromFile(FILE *fp) { -/* 75% of 2.1's .pyc files can exploit SMALL_FILE_LIMIT. - * REASONABLE_FILE_LIMIT is by defn something big enough for Tkinter.pyc. - */ -#define SMALL_FILE_LIMIT (1L << 14) +/* REASONABLE_FILE_LIMIT is by defn something big enough for Tkinter.pyc. */ #define REASONABLE_FILE_LIMIT (1L << 18) #ifdef HAVE_FSTAT off_t filesize; -#endif -#ifdef HAVE_FSTAT filesize = getfilesize(fp); - if (filesize > 0) { - char buf[SMALL_FILE_LIMIT]; - char* pBuf = NULL; - if (filesize <= SMALL_FILE_LIMIT) - pBuf = buf; - else if (filesize <= REASONABLE_FILE_LIMIT) - pBuf = (char *)PyMem_MALLOC(filesize); + if (filesize > 0 && filesize <= REASONABLE_FILE_LIMIT) { + char* pBuf = (char *)PyMem_MALLOC(filesize); if (pBuf != NULL) { PyObject* v; size_t n; @@ -1064,8 +1140,7 @@ PyMarshal_ReadLastObjectFromFile(FILE *fp) is smaller than REASONABLE_FILE_LIMIT */ n = fread(pBuf, 1, (int)filesize, fp); v = PyMarshal_ReadObjectFromString(pBuf, n); - if (pBuf != buf) - PyMem_FREE(pBuf); + PyMem_FREE(pBuf); return v; } @@ -1076,7 +1151,6 @@ PyMarshal_ReadLastObjectFromFile(FILE *fp) */ return PyMarshal_ReadObjectFromFile(fp); -#undef SMALL_FILE_LIMIT #undef REASONABLE_FILE_LIMIT } @@ -1109,6 +1183,24 @@ PyMarshal_ReadObjectFromString(char *str, Py_ssize_t len) return result; } +static void +set_error(int error) +{ + switch (error) { + case WFERR_NOMEMORY: + PyErr_NoMemory(); + break; + case WFERR_UNMARSHALLABLE: + PyErr_SetString(PyExc_ValueError, "unmarshallable object"); + break; + case WFERR_NESTEDTOODEEP: + default: + PyErr_SetString(PyExc_ValueError, + "object too deeply nested to marshal"); + break; + } +} + PyObject * PyMarshal_WriteObjectToString(PyObject *x, int version) { @@ -1119,7 +1211,7 @@ PyMarshal_WriteObjectToString(PyObject *x, int version) return NULL; wf.ptr = PyString_AS_STRING((PyStringObject *)wf.str); wf.end = wf.ptr + PyString_Size(wf.str); - wf.error = 0; + wf.error = WFERR_OK; wf.depth = 0; wf.version = version; wf.strings = (version > 0) ? PyDict_New() : NULL; @@ -1133,13 +1225,12 @@ PyMarshal_WriteObjectToString(PyObject *x, int version) "too much marshall data for a string"); return NULL; } - _PyString_Resize(&wf.str, (Py_ssize_t)(wf.ptr - base)); + if (_PyString_Resize(&wf.str, (Py_ssize_t)(wf.ptr - base))) + return NULL; } - if (wf.error) { + if (wf.error != WFERR_OK) { Py_XDECREF(wf.str); - PyErr_SetString(PyExc_ValueError, - (wf.error==1)?"unmarshallable object" - :"object too deeply nested to marshal"); + set_error(wf.error); return NULL; } return wf.str; @@ -1164,16 +1255,14 @@ marshal_dump(PyObject *self, PyObject *args) wf.fp = PyFile_AsFile(f); wf.str = NULL; wf.ptr = wf.end = NULL; - wf.error = 0; + wf.error = WFERR_OK; wf.depth = 0; wf.strings = (version > 0) ? PyDict_New() : 0; wf.version = version; w_object(x, &wf); Py_XDECREF(wf.strings); - if (wf.error) { - PyErr_SetString(PyExc_ValueError, - (wf.error==1)?"unmarshallable object" - :"object too deeply nested to marshal"); + if (wf.error != WFERR_OK) { + set_error(wf.error); return NULL; } Py_INCREF(Py_None); diff --git a/Python/modsupport.c b/Python/modsupport.c index 6ee48f3f1e..8bdec8b792 100644 --- a/Python/modsupport.c +++ b/Python/modsupport.c @@ -34,8 +34,9 @@ Py_InitModule4(const char *name, PyMethodDef *methods, const char *doc, { PyObject *m, *d, *v, *n; PyMethodDef *ml; - if (!Py_IsInitialized()) - Py_FatalError("Interpreter not initialized (version mismatch?)"); + PyInterpreterState *interp = PyThreadState_Get()->interp; + if (interp->modules == NULL) + Py_FatalError("Python import machinery not initialized"); if (module_api_version != PYTHON_API_VERSION) { char message[512]; PyOS_snprintf(message, sizeof(message), diff --git a/Python/peephole.c b/Python/peephole.c index 95ce9d04d6..433fe27d0e 100644 --- a/Python/peephole.c +++ b/Python/peephole.c @@ -13,7 +13,12 @@ #define GETARG(arr, i) ((int)((arr[i+2]<<8) + arr[i+1])) #define UNCONDITIONAL_JUMP(op) (op==JUMP_ABSOLUTE || op==JUMP_FORWARD) -#define ABSOLUTE_JUMP(op) (op==JUMP_ABSOLUTE || op==CONTINUE_LOOP) +#define CONDITIONAL_JUMP(op) (op==POP_JUMP_IF_FALSE || op==POP_JUMP_IF_TRUE \ + || op==JUMP_IF_FALSE_OR_POP || op==JUMP_IF_TRUE_OR_POP) +#define ABSOLUTE_JUMP(op) (op==JUMP_ABSOLUTE || op==CONTINUE_LOOP \ + || op==POP_JUMP_IF_FALSE || op==POP_JUMP_IF_TRUE \ + || op==JUMP_IF_FALSE_OR_POP || op==JUMP_IF_TRUE_OR_POP) +#define JUMPS_ON_TRUE(op) (op==POP_JUMP_IF_TRUE || op==JUMP_IF_TRUE_OR_POP) #define GETJUMPTGT(arr, i) (GETARG(arr,i) + (ABSOLUTE_JUMP(arr[i]) ? 0 : i+3)) #define SETARG(arr, i, val) arr[i+2] = val>>8; arr[i+1] = val & 255 #define CODESIZE(op) (HAS_ARG(op) ? 3 : 1) @@ -124,6 +129,24 @@ fold_binops_on_constants(unsigned char *codestr, PyObject *consts) break; case BINARY_SUBSCR: newconst = PyObject_GetItem(v, w); + /* #5057: if v is unicode, there might be differences between + wide and narrow builds in cases like u'\U00012345'[0]. + Wide builds will return a non-BMP char, whereas narrow builds + will return a surrogate. In both the cases skip the + optimization in order to produce compatible pycs. + */ + if (newconst != NULL && + PyUnicode_Check(v) && PyUnicode_Check(newconst)) { + Py_UNICODE ch = PyUnicode_AS_UNICODE(newconst)[0]; +#ifdef Py_UNICODE_WIDE + if (ch > 0xFFFF) { +#else + if (ch >= 0xD800 && ch <= 0xDFFF) { +#endif + Py_DECREF(newconst); + return 0; + } + } break; case BINARY_LSHIFT: newconst = PyNumber_Lshift(v, w); @@ -245,13 +268,16 @@ markblocks(unsigned char *code, Py_ssize_t len) switch (opcode) { case FOR_ITER: case JUMP_FORWARD: - case JUMP_IF_FALSE: - case JUMP_IF_TRUE: + case JUMP_IF_FALSE_OR_POP: + case JUMP_IF_TRUE_OR_POP: + case POP_JUMP_IF_FALSE: + case POP_JUMP_IF_TRUE: case JUMP_ABSOLUTE: case CONTINUE_LOOP: case SETUP_LOOP: case SETUP_EXCEPT: case SETUP_FINALLY: + case SETUP_WITH: j = GETJUMPTGT(code, i); blocks[j] = 1; break; @@ -338,29 +364,24 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, assert(PyList_Check(consts)); for (i=0 ; i<codelen ; i += CODESIZE(codestr[i])) { + reoptimize_current: opcode = codestr[i]; lastlc = cumlc; cumlc = 0; switch (opcode) { - - /* Replace UNARY_NOT JUMP_IF_FALSE POP_TOP with - with JUMP_IF_TRUE POP_TOP */ + /* Replace UNARY_NOT POP_JUMP_IF_FALSE + with POP_JUMP_IF_TRUE */ case UNARY_NOT: - if (codestr[i+1] != JUMP_IF_FALSE || - codestr[i+4] != POP_TOP || - !ISBASICBLOCK(blocks,i,5)) - continue; - tgt = GETJUMPTGT(codestr, (i+1)); - if (codestr[tgt] != POP_TOP) + if (codestr[i+1] != POP_JUMP_IF_FALSE + || !ISBASICBLOCK(blocks,i,4)) continue; - j = GETARG(codestr, i+1) + 1; - codestr[i] = JUMP_IF_TRUE; + j = GETARG(codestr, i+1); + codestr[i] = POP_JUMP_IF_TRUE; SETARG(codestr, i, j); - codestr[i+3] = POP_TOP; - codestr[i+4] = NOP; - break; + codestr[i+3] = NOP; + goto reoptimize_current; /* not a is b --> a is not b not a in b --> a not in b @@ -400,16 +421,16 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, break; /* Skip over LOAD_CONST trueconst - JUMP_IF_FALSE xx POP_TOP */ + POP_JUMP_IF_FALSE xx. This improves + "while 1" performance. */ case LOAD_CONST: cumlc = lastlc + 1; j = GETARG(codestr, i); - if (codestr[i+3] != JUMP_IF_FALSE || - codestr[i+6] != POP_TOP || - !ISBASICBLOCK(blocks,i,7) || + if (codestr[i+3] != POP_JUMP_IF_FALSE || + !ISBASICBLOCK(blocks,i,6) || !PyObject_IsTrue(PyList_GET_ITEM(consts, j))) continue; - memset(codestr+i, NOP, 7); + memset(codestr+i, NOP, 6); cumlc = 0; break; @@ -498,27 +519,49 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, "if a or b:" "a and b or c" "(a and b) and c" - x:JUMP_IF_FALSE y y:JUMP_IF_FALSE z --> x:JUMP_IF_FALSE z - x:JUMP_IF_FALSE y y:JUMP_IF_TRUE z --> x:JUMP_IF_FALSE y+3 + x:JUMP_IF_FALSE_OR_POP y y:JUMP_IF_FALSE_OR_POP z + --> x:JUMP_IF_FALSE_OR_POP z + x:JUMP_IF_FALSE_OR_POP y y:JUMP_IF_TRUE_OR_POP z + --> x:POP_JUMP_IF_FALSE y+3 where y+3 is the instruction following the second test. */ - case JUMP_IF_FALSE: - case JUMP_IF_TRUE: + case JUMP_IF_FALSE_OR_POP: + case JUMP_IF_TRUE_OR_POP: tgt = GETJUMPTGT(codestr, i); j = codestr[tgt]; - if (j == JUMP_IF_FALSE || j == JUMP_IF_TRUE) { - if (j == opcode) { - tgttgt = GETJUMPTGT(codestr, tgt) - i - 3; + if (CONDITIONAL_JUMP(j)) { + /* NOTE: all possible jumps here are + absolute! */ + if (JUMPS_ON_TRUE(j) == JUMPS_ON_TRUE(opcode)) { + /* The second jump will be + taken iff the first is. */ + tgttgt = GETJUMPTGT(codestr, tgt); + /* The current opcode inherits + its target's stack behaviour */ + codestr[i] = j; SETARG(codestr, i, tgttgt); + goto reoptimize_current; } else { - tgt -= i; - SETARG(codestr, i, tgt); + /* The second jump is not taken + if the first is (so jump past + it), and all conditional + jumps pop their argument when + they're not taken (so change + the first jump to pop its + argument when it's taken). */ + if (JUMPS_ON_TRUE(opcode)) + codestr[i] = POP_JUMP_IF_TRUE; + else + codestr[i] = POP_JUMP_IF_FALSE; + SETARG(codestr, i, (tgt + 3)); + goto reoptimize_current; } - break; } /* Intentional fallthrough */ /* Replace jumps to unconditional jumps */ + case POP_JUMP_IF_FALSE: + case POP_JUMP_IF_TRUE: case FOR_ITER: case JUMP_FORWARD: case JUMP_ABSOLUTE: @@ -526,6 +569,7 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, case SETUP_LOOP: case SETUP_EXCEPT: case SETUP_FINALLY: + case SETUP_WITH: tgt = GETJUMPTGT(codestr, i); /* Replace JUMP_* to a RETURN into just a RETURN */ if (UNCONDITIONAL_JUMP(opcode) && @@ -591,17 +635,20 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, case JUMP_ABSOLUTE: case CONTINUE_LOOP: + case POP_JUMP_IF_FALSE: + case POP_JUMP_IF_TRUE: + case JUMP_IF_FALSE_OR_POP: + case JUMP_IF_TRUE_OR_POP: j = addrmap[GETARG(codestr, i)]; SETARG(codestr, i, j); break; case FOR_ITER: case JUMP_FORWARD: - case JUMP_IF_FALSE: - case JUMP_IF_TRUE: case SETUP_LOOP: case SETUP_EXCEPT: case SETUP_FINALLY: + case SETUP_WITH: j = addrmap[GETARG(codestr, i) + i + 3] - addrmap[i] - 3; SETARG(codestr, i, j); break; diff --git a/Python/pyctype.c b/Python/pyctype.c new file mode 100644 index 0000000000..da117d58fd --- /dev/null +++ b/Python/pyctype.c @@ -0,0 +1,214 @@ +#include "Python.h" + +/* Our own locale-independent ctype.h-like macros */ + +const unsigned int _Py_ctype_table[256] = { + 0, /* 0x0 '\x00' */ + 0, /* 0x1 '\x01' */ + 0, /* 0x2 '\x02' */ + 0, /* 0x3 '\x03' */ + 0, /* 0x4 '\x04' */ + 0, /* 0x5 '\x05' */ + 0, /* 0x6 '\x06' */ + 0, /* 0x7 '\x07' */ + 0, /* 0x8 '\x08' */ + PY_CTF_SPACE, /* 0x9 '\t' */ + PY_CTF_SPACE, /* 0xa '\n' */ + PY_CTF_SPACE, /* 0xb '\v' */ + PY_CTF_SPACE, /* 0xc '\f' */ + PY_CTF_SPACE, /* 0xd '\r' */ + 0, /* 0xe '\x0e' */ + 0, /* 0xf '\x0f' */ + 0, /* 0x10 '\x10' */ + 0, /* 0x11 '\x11' */ + 0, /* 0x12 '\x12' */ + 0, /* 0x13 '\x13' */ + 0, /* 0x14 '\x14' */ + 0, /* 0x15 '\x15' */ + 0, /* 0x16 '\x16' */ + 0, /* 0x17 '\x17' */ + 0, /* 0x18 '\x18' */ + 0, /* 0x19 '\x19' */ + 0, /* 0x1a '\x1a' */ + 0, /* 0x1b '\x1b' */ + 0, /* 0x1c '\x1c' */ + 0, /* 0x1d '\x1d' */ + 0, /* 0x1e '\x1e' */ + 0, /* 0x1f '\x1f' */ + PY_CTF_SPACE, /* 0x20 ' ' */ + 0, /* 0x21 '!' */ + 0, /* 0x22 '"' */ + 0, /* 0x23 '#' */ + 0, /* 0x24 '$' */ + 0, /* 0x25 '%' */ + 0, /* 0x26 '&' */ + 0, /* 0x27 "'" */ + 0, /* 0x28 '(' */ + 0, /* 0x29 ')' */ + 0, /* 0x2a '*' */ + 0, /* 0x2b '+' */ + 0, /* 0x2c ',' */ + 0, /* 0x2d '-' */ + 0, /* 0x2e '.' */ + 0, /* 0x2f '/' */ + PY_CTF_DIGIT|PY_CTF_XDIGIT, /* 0x30 '0' */ + PY_CTF_DIGIT|PY_CTF_XDIGIT, /* 0x31 '1' */ + PY_CTF_DIGIT|PY_CTF_XDIGIT, /* 0x32 '2' */ + PY_CTF_DIGIT|PY_CTF_XDIGIT, /* 0x33 '3' */ + PY_CTF_DIGIT|PY_CTF_XDIGIT, /* 0x34 '4' */ + PY_CTF_DIGIT|PY_CTF_XDIGIT, /* 0x35 '5' */ + PY_CTF_DIGIT|PY_CTF_XDIGIT, /* 0x36 '6' */ + PY_CTF_DIGIT|PY_CTF_XDIGIT, /* 0x37 '7' */ + PY_CTF_DIGIT|PY_CTF_XDIGIT, /* 0x38 '8' */ + PY_CTF_DIGIT|PY_CTF_XDIGIT, /* 0x39 '9' */ + 0, /* 0x3a ':' */ + 0, /* 0x3b ';' */ + 0, /* 0x3c '<' */ + 0, /* 0x3d '=' */ + 0, /* 0x3e '>' */ + 0, /* 0x3f '?' */ + 0, /* 0x40 '@' */ + PY_CTF_UPPER|PY_CTF_XDIGIT, /* 0x41 'A' */ + PY_CTF_UPPER|PY_CTF_XDIGIT, /* 0x42 'B' */ + PY_CTF_UPPER|PY_CTF_XDIGIT, /* 0x43 'C' */ + PY_CTF_UPPER|PY_CTF_XDIGIT, /* 0x44 'D' */ + PY_CTF_UPPER|PY_CTF_XDIGIT, /* 0x45 'E' */ + PY_CTF_UPPER|PY_CTF_XDIGIT, /* 0x46 'F' */ + PY_CTF_UPPER, /* 0x47 'G' */ + PY_CTF_UPPER, /* 0x48 'H' */ + PY_CTF_UPPER, /* 0x49 'I' */ + PY_CTF_UPPER, /* 0x4a 'J' */ + PY_CTF_UPPER, /* 0x4b 'K' */ + PY_CTF_UPPER, /* 0x4c 'L' */ + PY_CTF_UPPER, /* 0x4d 'M' */ + PY_CTF_UPPER, /* 0x4e 'N' */ + PY_CTF_UPPER, /* 0x4f 'O' */ + PY_CTF_UPPER, /* 0x50 'P' */ + PY_CTF_UPPER, /* 0x51 'Q' */ + PY_CTF_UPPER, /* 0x52 'R' */ + PY_CTF_UPPER, /* 0x53 'S' */ + PY_CTF_UPPER, /* 0x54 'T' */ + PY_CTF_UPPER, /* 0x55 'U' */ + PY_CTF_UPPER, /* 0x56 'V' */ + PY_CTF_UPPER, /* 0x57 'W' */ + PY_CTF_UPPER, /* 0x58 'X' */ + PY_CTF_UPPER, /* 0x59 'Y' */ + PY_CTF_UPPER, /* 0x5a 'Z' */ + 0, /* 0x5b '[' */ + 0, /* 0x5c '\\' */ + 0, /* 0x5d ']' */ + 0, /* 0x5e '^' */ + 0, /* 0x5f '_' */ + 0, /* 0x60 '`' */ + PY_CTF_LOWER|PY_CTF_XDIGIT, /* 0x61 'a' */ + PY_CTF_LOWER|PY_CTF_XDIGIT, /* 0x62 'b' */ + PY_CTF_LOWER|PY_CTF_XDIGIT, /* 0x63 'c' */ + PY_CTF_LOWER|PY_CTF_XDIGIT, /* 0x64 'd' */ + PY_CTF_LOWER|PY_CTF_XDIGIT, /* 0x65 'e' */ + PY_CTF_LOWER|PY_CTF_XDIGIT, /* 0x66 'f' */ + PY_CTF_LOWER, /* 0x67 'g' */ + PY_CTF_LOWER, /* 0x68 'h' */ + PY_CTF_LOWER, /* 0x69 'i' */ + PY_CTF_LOWER, /* 0x6a 'j' */ + PY_CTF_LOWER, /* 0x6b 'k' */ + PY_CTF_LOWER, /* 0x6c 'l' */ + PY_CTF_LOWER, /* 0x6d 'm' */ + PY_CTF_LOWER, /* 0x6e 'n' */ + PY_CTF_LOWER, /* 0x6f 'o' */ + PY_CTF_LOWER, /* 0x70 'p' */ + PY_CTF_LOWER, /* 0x71 'q' */ + PY_CTF_LOWER, /* 0x72 'r' */ + PY_CTF_LOWER, /* 0x73 's' */ + PY_CTF_LOWER, /* 0x74 't' */ + PY_CTF_LOWER, /* 0x75 'u' */ + PY_CTF_LOWER, /* 0x76 'v' */ + PY_CTF_LOWER, /* 0x77 'w' */ + PY_CTF_LOWER, /* 0x78 'x' */ + PY_CTF_LOWER, /* 0x79 'y' */ + PY_CTF_LOWER, /* 0x7a 'z' */ + 0, /* 0x7b '{' */ + 0, /* 0x7c '|' */ + 0, /* 0x7d '}' */ + 0, /* 0x7e '~' */ + 0, /* 0x7f '\x7f' */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; + + +const unsigned char _Py_ctype_tolower[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, + 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, +}; + +const unsigned char _Py_ctype_toupper[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, + 0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, +}; + diff --git a/Python/pymath.c b/Python/pymath.c index 1ba2e63620..827a773a6a 100644 --- a/Python/pymath.c +++ b/Python/pymath.c @@ -13,6 +13,24 @@ double _Py_force_double(double x) } #endif +#ifdef HAVE_GCC_ASM_FOR_X87 + +/* inline assembly for getting and setting the 387 FPU control word on + gcc/x86 */ + +unsigned short _Py_get_387controlword(void) { + unsigned short cw; + __asm__ __volatile__ ("fnstcw %0" : "=m" (cw)); + return cw; +} + +void _Py_set_387controlword(unsigned short cw) { + __asm__ __volatile__ ("fldcw %0" : : "m" (cw)); +} + +#endif + + #ifndef HAVE_HYPOT double hypot(double x, double y) { @@ -47,201 +65,15 @@ copysign(double x, double y) } #endif /* HAVE_COPYSIGN */ -#ifndef HAVE_LOG1P -#include <float.h> - -double -log1p(double x) -{ - /* For x small, we use the following approach. Let y be the nearest - float to 1+x, then - - 1+x = y * (1 - (y-1-x)/y) - - so log(1+x) = log(y) + log(1-(y-1-x)/y). Since (y-1-x)/y is tiny, - the second term is well approximated by (y-1-x)/y. If abs(x) >= - DBL_EPSILON/2 or the rounding-mode is some form of round-to-nearest - then y-1-x will be exactly representable, and is computed exactly - by (y-1)-x. - - If abs(x) < DBL_EPSILON/2 and the rounding mode is not known to be - round-to-nearest then this method is slightly dangerous: 1+x could - be rounded up to 1+DBL_EPSILON instead of down to 1, and in that - case y-1-x will not be exactly representable any more and the - result can be off by many ulps. But this is easily fixed: for a - floating-point number |x| < DBL_EPSILON/2., the closest - floating-point number to log(1+x) is exactly x. - */ - - double y; - if (fabs(x) < DBL_EPSILON/2.) { - return x; - } else if (-0.5 <= x && x <= 1.) { - /* WARNING: it's possible than an overeager compiler - will incorrectly optimize the following two lines - to the equivalent of "return log(1.+x)". If this - happens, then results from log1p will be inaccurate - for small x. */ - y = 1.+x; - return log(y)-((y-1.)-x)/y; - } else { - /* NaNs and infinities should end up here */ - return log(1.+x); - } -} -#endif /* HAVE_LOG1P */ - -/* - * ==================================================== - * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. - * - * Developed at SunPro, a Sun Microsystems, Inc. business. - * Permission to use, copy, modify, and distribute this - * software is freely granted, provided that this notice - * is preserved. - * ==================================================== - */ - -static const double ln2 = 6.93147180559945286227E-01; -static const double two_pow_m28 = 3.7252902984619141E-09; /* 2**-28 */ -static const double two_pow_p28 = 268435456.0; /* 2**28 */ -static const double zero = 0.0; - -/* asinh(x) - * Method : - * Based on - * asinh(x) = sign(x) * log [ |x| + sqrt(x*x+1) ] - * we have - * asinh(x) := x if 1+x*x=1, - * := sign(x)*(log(x)+ln2)) for large |x|, else - * := sign(x)*log(2|x|+1/(|x|+sqrt(x*x+1))) if|x|>2, else - * := sign(x)*log1p(|x| + x^2/(1 + sqrt(1+x^2))) - */ - -#ifndef HAVE_ASINH -double -asinh(double x) -{ - double w; - double absx = fabs(x); - - if (Py_IS_NAN(x) || Py_IS_INFINITY(x)) { - return x+x; - } - if (absx < two_pow_m28) { /* |x| < 2**-28 */ - return x; /* return x inexact except 0 */ - } - if (absx > two_pow_p28) { /* |x| > 2**28 */ - w = log(absx)+ln2; - } - else if (absx > 2.0) { /* 2 < |x| < 2**28 */ - w = log(2.0*absx + 1.0 / (sqrt(x*x + 1.0) + absx)); - } - else { /* 2**-28 <= |x| < 2= */ - double t = x*x; - w = log1p(absx + t / (1.0 + sqrt(1.0 + t))); - } - return copysign(w, x); - -} -#endif /* HAVE_ASINH */ - -/* acosh(x) - * Method : - * Based on - * acosh(x) = log [ x + sqrt(x*x-1) ] - * we have - * acosh(x) := log(x)+ln2, if x is large; else - * acosh(x) := log(2x-1/(sqrt(x*x-1)+x)) if x>2; else - * acosh(x) := log1p(t+sqrt(2.0*t+t*t)); where t=x-1. - * - * Special cases: - * acosh(x) is NaN with signal if x<1. - * acosh(NaN) is NaN without signal. - */ - -#ifndef HAVE_ACOSH +#ifndef HAVE_ROUND double -acosh(double x) +round(double x) { - if (Py_IS_NAN(x)) { - return x+x; - } - if (x < 1.) { /* x < 1; return a signaling NaN */ - errno = EDOM; -#ifdef Py_NAN - return Py_NAN; -#else - return (x-x)/(x-x); -#endif - } - else if (x >= two_pow_p28) { /* x > 2**28 */ - if (Py_IS_INFINITY(x)) { - return x+x; - } else { - return log(x)+ln2; /* acosh(huge)=log(2x) */ - } - } - else if (x == 1.) { - return 0.0; /* acosh(1) = 0 */ - } - else if (x > 2.) { /* 2 < x < 2**28 */ - double t = x*x; - return log(2.0*x - 1.0 / (x + sqrt(t - 1.0))); - } - else { /* 1 < x <= 2 */ - double t = x - 1.0; - return log1p(t + sqrt(2.0*t + t*t)); - } -} -#endif /* HAVE_ACOSH */ - -/* atanh(x) - * Method : - * 1.Reduced x to positive by atanh(-x) = -atanh(x) - * 2.For x>=0.5 - * 1 2x x - * atanh(x) = --- * log(1 + -------) = 0.5 * log1p(2 * --------) - * 2 1 - x 1 - x - * - * For x<0.5 - * atanh(x) = 0.5*log1p(2x+2x*x/(1-x)) - * - * Special cases: - * atanh(x) is NaN if |x| >= 1 with signal; - * atanh(NaN) is that NaN with no signal; - * - */ - -#ifndef HAVE_ATANH -double -atanh(double x) -{ - double absx; - double t; - - if (Py_IS_NAN(x)) { - return x+x; - } + double absx, y; absx = fabs(x); - if (absx >= 1.) { /* |x| >= 1 */ - errno = EDOM; -#ifdef Py_NAN - return Py_NAN; -#else - return x/zero; -#endif - } - if (absx < two_pow_m28) { /* |x| < 2**-28 */ - return x; - } - if (absx < 0.5) { /* |x| < 0.5 */ - t = absx+absx; - t = 0.5 * log1p(t + t*absx / (1.0 - absx)); - } - else { /* 0.5 <= |x| <= 1.0 */ - t = 0.5 * log1p((absx + absx) / (1.0 - absx)); - } - return copysign(t, x); + y = floor(absx); + if (absx - y >= 0.5) + y += 1.0; + return copysign(y, x); } -#endif /* HAVE_ATANH */ +#endif /* HAVE_ROUND */ diff --git a/Python/pystate.c b/Python/pystate.c index 5afc01a722..ddb7d42589 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -298,7 +298,7 @@ PyThreadState_Delete(PyThreadState *tstate) Py_FatalError("PyThreadState_Delete: tstate is still current"); tstate_delete_common(tstate); #ifdef WITH_THREAD - if (autoTLSkey && PyThread_get_key_value(autoTLSkey) == tstate) + if (autoInterpreterState && PyThread_get_key_value(autoTLSkey) == tstate) PyThread_delete_key_value(autoTLSkey); #endif /* WITH_THREAD */ } @@ -314,7 +314,7 @@ PyThreadState_DeleteCurrent() "PyThreadState_DeleteCurrent: no current tstate"); _PyThreadState_Current = NULL; tstate_delete_common(tstate); - if (autoTLSkey && PyThread_get_key_value(autoTLSkey) == tstate) + if (autoInterpreterState && PyThread_get_key_value(autoTLSkey) == tstate) PyThread_delete_key_value(autoTLSkey); PyEval_ReleaseLock(); } @@ -463,7 +463,7 @@ _PyThread_CurrentFrames(void) /* for i in all interpreters: * for t in all of i's thread states: * if t's frame isn't NULL, map t's id to its frame - * Because these lists can mutute even when the GIL is held, we + * Because these lists can mutate even when the GIL is held, we * need to grab head_mutex for the duration. */ HEAD_LOCK(); @@ -534,7 +534,6 @@ void _PyGILState_Fini(void) { PyThread_delete_key(autoTLSkey); - autoTLSkey = 0; autoInterpreterState = NULL; } @@ -546,10 +545,10 @@ _PyGILState_Fini(void) static void _PyGILState_NoteThreadState(PyThreadState* tstate) { - /* If autoTLSkey is 0, this must be the very first threadstate created - in Py_Initialize(). Don't do anything for now (we'll be back here - when _PyGILState_Init is called). */ - if (!autoTLSkey) + /* If autoTLSkey isn't initialized, this must be the very first + threadstate created in Py_Initialize(). Don't do anything for now + (we'll be back here when _PyGILState_Init is called). */ + if (!autoInterpreterState) return; /* Stick the thread state for this thread in thread local storage. @@ -577,7 +576,7 @@ _PyGILState_NoteThreadState(PyThreadState* tstate) PyThreadState * PyGILState_GetThisThreadState(void) { - if (autoInterpreterState == NULL || autoTLSkey == 0) + if (autoInterpreterState == NULL) return NULL; return (PyThreadState *)PyThread_get_key_value(autoTLSkey); } diff --git a/Python/pystrtod.c b/Python/pystrtod.c index 9004cf4641..859af435f9 100644 --- a/Python/pystrtod.c +++ b/Python/pystrtod.c @@ -3,11 +3,59 @@ #include <Python.h> #include <locale.h> -/* ascii character tests (as opposed to locale tests) */ -#define ISSPACE(c) ((c) == ' ' || (c) == '\f' || (c) == '\n' || \ - (c) == '\r' || (c) == '\t' || (c) == '\v') -#define ISDIGIT(c) ((c) >= '0' && (c) <= '9') +/* Case-insensitive string match used for nan and inf detection; t should be + lower-case. Returns 1 for a successful match, 0 otherwise. */ +static int +case_insensitive_match(const char *s, const char *t) +{ + while(*t && Py_TOLOWER(*s) == *t) { + s++; + t++; + } + return *t ? 0 : 1; +} + +/* _Py_parse_inf_or_nan: Attempt to parse a string of the form "nan", "inf" or + "infinity", with an optional leading sign of "+" or "-". On success, + return the NaN or Infinity as a double and set *endptr to point just beyond + the successfully parsed portion of the string. On failure, return -1.0 and + set *endptr to point to the start of the string. */ + +double +_Py_parse_inf_or_nan(const char *p, char **endptr) +{ + double retval; + const char *s; + int negate = 0; + + s = p; + if (*s == '-') { + negate = 1; + s++; + } + else if (*s == '+') { + s++; + } + if (case_insensitive_match(s, "inf")) { + s += 3; + if (case_insensitive_match(s, "inity")) + s += 5; + retval = negate ? -Py_HUGE_VAL : Py_HUGE_VAL; + } +#ifdef Py_NAN + else if (case_insensitive_match(s, "nan")) { + s += 3; + retval = negate ? -Py_NAN : Py_NAN; + } +#endif + else { + s = p; + retval = -1.0; + } + *endptr = (char *)s; + return retval; +} /** * PyOS_ascii_strtod: @@ -37,8 +85,44 @@ * * Return value: the #gdouble value. **/ + +#ifndef PY_NO_SHORT_FLOAT_REPR + double -PyOS_ascii_strtod(const char *nptr, char **endptr) +_PyOS_ascii_strtod(const char *nptr, char **endptr) +{ + double result; + _Py_SET_53BIT_PRECISION_HEADER; + + assert(nptr != NULL); + /* Set errno to zero, so that we can distinguish zero results + and underflows */ + errno = 0; + + _Py_SET_53BIT_PRECISION_START; + result = _Py_dg_strtod(nptr, endptr); + _Py_SET_53BIT_PRECISION_END; + + if (*endptr == nptr) + /* string might represent an inf or nan */ + result = _Py_parse_inf_or_nan(nptr, endptr); + + return result; + +} + +#else + +/* + Use system strtod; since strtod is locale aware, we may + have to first fix the decimal separator. + + Note that unlike _Py_dg_strtod, the system strtod may not always give + correctly rounded results. +*/ + +double +_PyOS_ascii_strtod(const char *nptr, char **endptr) { char *fail_pos; double val = -1.0; @@ -62,85 +146,79 @@ PyOS_ascii_strtod(const char *nptr, char **endptr) decimal_point_pos = NULL; - /* We process any leading whitespace and the optional sign manually, - then pass the remainder to the system strtod. This ensures that - the result of an underflow has the correct sign. (bug #1725) */ + /* Parse infinities and nans */ + val = _Py_parse_inf_or_nan(nptr, endptr); + if (*endptr != nptr) + return val; - p = nptr; - /* Skip leading space */ - while (ISSPACE(*p)) - p++; + /* Set errno to zero, so that we can distinguish zero results + and underflows */ + errno = 0; + /* We process the optional sign manually, then pass the remainder to + the system strtod. This ensures that the result of an underflow + has the correct sign. (bug #1725) */ + p = nptr; /* Process leading sign, if present */ if (*p == '-') { negate = 1; p++; - } else if (*p == '+') { + } + else if (*p == '+') { p++; } - /* What's left should begin with a digit, a decimal point, or one of - the letters i, I, n, N. It should not begin with 0x or 0X */ - if ((!ISDIGIT(*p) && - *p != '.' && *p != 'i' && *p != 'I' && *p != 'n' && *p != 'N') - || - (*p == '0' && (p[1] == 'x' || p[1] == 'X'))) - { - if (endptr) - *endptr = (char*)nptr; - errno = EINVAL; - return val; - } - digits_pos = p; + /* Some platform strtods accept hex floats; Python shouldn't (at the + moment), so we check explicitly for strings starting with '0x'. */ + if (*p == '0' && (*(p+1) == 'x' || *(p+1) == 'X')) + goto invalid_string; + + /* Check that what's left begins with a digit or decimal point */ + if (!Py_ISDIGIT(*p) && *p != '.') + goto invalid_string; + digits_pos = p; if (decimal_point[0] != '.' || decimal_point[1] != 0) { - while (ISDIGIT(*p)) + /* Look for a '.' in the input; if present, it'll need to be + swapped for the current locale's decimal point before we + call strtod. On the other hand, if we find the current + locale's decimal point then the input is invalid. */ + while (Py_ISDIGIT(*p)) p++; if (*p == '.') { decimal_point_pos = p++; - while (ISDIGIT(*p)) + /* locate end of number */ + while (Py_ISDIGIT(*p)) p++; if (*p == 'e' || *p == 'E') p++; if (*p == '+' || *p == '-') p++; - while (ISDIGIT(*p)) + while (Py_ISDIGIT(*p)) p++; end = p; } else if (strncmp(p, decimal_point, decimal_point_len) == 0) - { /* Python bug #1417699 */ - if (endptr) - *endptr = (char*)nptr; - errno = EINVAL; - return val; - } + goto invalid_string; /* For the other cases, we need not convert the decimal point */ } - /* Set errno to zero, so that we can distinguish zero results - and underflows */ - errno = 0; - - if (decimal_point_pos) - { + if (decimal_point_pos) { char *copy, *c; - - /* We need to convert the '.' to the locale specific decimal - point */ + /* Create a copy of the input, with the '.' converted to the + locale-specific decimal point */ copy = (char *)PyMem_MALLOC(end - digits_pos + 1 + decimal_point_len); if (copy == NULL) { - if (endptr) - *endptr = (char *)nptr; + *endptr = (char *)nptr; errno = ENOMEM; return val; } @@ -176,15 +254,119 @@ PyOS_ascii_strtod(const char *nptr, char **endptr) } if (fail_pos == digits_pos) - fail_pos = (char *)nptr; + goto invalid_string; if (negate && fail_pos != nptr) val = -val; + *endptr = fail_pos; + + return val; + + invalid_string: + *endptr = (char*)nptr; + errno = EINVAL; + return -1.0; +} + +#endif + +/* PyOS_ascii_strtod is DEPRECATED in Python 2.7 and 3.1 */ +double +PyOS_ascii_strtod(const char *nptr, char **endptr) +{ + char *fail_pos; + const char *p; + double x; + + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "PyOS_ascii_strtod and PyOS_ascii_atof are " + "deprecated. Use PyOS_string_to_double " + "instead.", 1) < 0) + return -1.0; + + /* _PyOS_ascii_strtod already does everything that we want, + except that it doesn't parse leading whitespace */ + p = nptr; + while (Py_ISSPACE(*p)) + p++; + x = _PyOS_ascii_strtod(p, &fail_pos); + if (fail_pos == p) + fail_pos = (char *)nptr; if (endptr) - *endptr = fail_pos; + *endptr = (char *)fail_pos; + return x; +} - return val; +/* PyOS_ascii_strtod is DEPRECATED in Python 2.7 and 3.1 */ + +double +PyOS_ascii_atof(const char *nptr) +{ + return PyOS_ascii_strtod(nptr, NULL); +} + +/* PyOS_string_to_double is the recommended replacement for the deprecated + PyOS_ascii_strtod and PyOS_ascii_atof functions. It converts a + null-terminated byte string s (interpreted as a string of ASCII characters) + to a float. The string should not have leading or trailing whitespace (in + contrast, PyOS_ascii_strtod allows leading whitespace but not trailing + whitespace). The conversion is independent of the current locale. + + If endptr is NULL, try to convert the whole string. Raise ValueError and + return -1.0 if the string is not a valid representation of a floating-point + number. + + If endptr is non-NULL, try to convert as much of the string as possible. + If no initial segment of the string is the valid representation of a + floating-point number then *endptr is set to point to the beginning of the + string, -1.0 is returned and again ValueError is raised. + + On overflow (e.g., when trying to convert '1e500' on an IEEE 754 machine), + if overflow_exception is NULL then +-Py_HUGE_VAL is returned, and no Python + exception is raised. Otherwise, overflow_exception should point to a + a Python exception, this exception will be raised, -1.0 will be returned, + and *endptr will point just past the end of the converted value. + + If any other failure occurs (for example lack of memory), -1.0 is returned + and the appropriate Python exception will have been set. +*/ + +double +PyOS_string_to_double(const char *s, + char **endptr, + PyObject *overflow_exception) +{ + double x, result=-1.0; + char *fail_pos; + + errno = 0; + PyFPE_START_PROTECT("PyOS_string_to_double", return -1.0) + x = _PyOS_ascii_strtod(s, &fail_pos); + PyFPE_END_PROTECT(x) + + if (errno == ENOMEM) { + PyErr_NoMemory(); + fail_pos = (char *)s; + } + else if (!endptr && (fail_pos == s || *fail_pos != '\0')) + PyErr_Format(PyExc_ValueError, + "could not convert string to float: " + "%.200s", s); + else if (fail_pos == s) + PyErr_Format(PyExc_ValueError, + "could not convert string to float: " + "%.200s", s); + else if (errno == ERANGE && fabs(x) >= 1.0 && overflow_exception) + PyErr_Format(overflow_exception, + "value too large to convert to float: " + "%.200s", s); + else + result = x; + + if (endptr != NULL) + *endptr = fail_pos; + return result; } /* Given a string that may have a decimal point in the current @@ -201,7 +383,7 @@ change_decimal_from_locale_to_dot(char* buffer) if (*buffer == '+' || *buffer == '-') buffer++; - while (isdigit(Py_CHARMASK(*buffer))) + while (Py_ISDIGIT(*buffer)) buffer++; if (strncmp(buffer, decimal_point, decimal_point_len) == 0) { *buffer = '.'; @@ -244,7 +426,7 @@ ensure_minimum_exponent_length(char* buffer, size_t buf_size) /* Find the end of the exponent, keeping track of leading zeros. */ - while (*p && isdigit(Py_CHARMASK(*p))) { + while (*p && Py_ISDIGIT(*p)) { if (in_leading_zeros && *p == '0') ++leading_zero_cnt; if (*p != '0') @@ -292,13 +474,61 @@ ensure_minimum_exponent_length(char* buffer, size_t buf_size) } } -/* Ensure that buffer has a decimal point in it. The decimal point - will not be in the current locale, it will always be '.' */ +/* Remove trailing zeros after the decimal point from a numeric string; also + remove the decimal point if all digits following it are zero. The numeric + string must end in '\0', and should not have any leading or trailing + whitespace. Assumes that the decimal point is '.'. */ Py_LOCAL_INLINE(void) -ensure_decimal_point(char* buffer, size_t buf_size) +remove_trailing_zeros(char *buffer) { - int insert_count = 0; - char* chars_to_insert; + char *old_fraction_end, *new_fraction_end, *end, *p; + + p = buffer; + if (*p == '-' || *p == '+') + /* Skip leading sign, if present */ + ++p; + while (Py_ISDIGIT(*p)) + ++p; + + /* if there's no decimal point there's nothing to do */ + if (*p++ != '.') + return; + + /* scan any digits after the point */ + while (Py_ISDIGIT(*p)) + ++p; + old_fraction_end = p; + + /* scan up to ending '\0' */ + while (*p != '\0') + p++; + /* +1 to make sure that we move the null byte as well */ + end = p+1; + + /* scan back from fraction_end, looking for removable zeros */ + p = old_fraction_end; + while (*(p-1) == '0') + --p; + /* and remove point if we've got that far */ + if (*(p-1) == '.') + --p; + new_fraction_end = p; + + memmove(new_fraction_end, old_fraction_end, end-old_fraction_end); +} + +/* Ensure that buffer has a decimal point in it. The decimal point will not + be in the current locale, it will always be '.'. Don't add a decimal point + if an exponent is present. Also, convert to exponential notation where + adding a '.0' would produce too many significant digits (see issue 5864). + + Returns a pointer to the fixed buffer, or NULL on failure. +*/ +Py_LOCAL_INLINE(char *) +ensure_decimal_point(char* buffer, size_t buf_size, int precision) +{ + int digit_count, insert_count = 0, convert_to_exp = 0; + char *chars_to_insert, *digits_start; /* search for the first non-digit character */ char *p = buffer; @@ -306,25 +536,44 @@ ensure_decimal_point(char* buffer, size_t buf_size) /* Skip leading sign, if present. I think this could only ever be '-', but it can't hurt to check for both. */ ++p; - while (*p && isdigit(Py_CHARMASK(*p))) + digits_start = p; + while (*p && Py_ISDIGIT(*p)) ++p; + digit_count = Py_SAFE_DOWNCAST(p - digits_start, Py_ssize_t, int); if (*p == '.') { - if (isdigit(Py_CHARMASK(*(p+1)))) { + if (Py_ISDIGIT(*(p+1))) { /* Nothing to do, we already have a decimal point and a digit after it */ } else { /* We have a decimal point, but no following digit. Insert a zero after the decimal. */ + /* can't ever get here via PyOS_double_to_string */ + assert(precision == -1); ++p; chars_to_insert = "0"; insert_count = 1; } } - else { - chars_to_insert = ".0"; - insert_count = 2; + else if (!(*p == 'e' || *p == 'E')) { + /* Don't add ".0" if we have an exponent. */ + if (digit_count == precision) { + /* issue 5864: don't add a trailing .0 in the case + where the '%g'-formatted result already has as many + significant digits as were requested. Switch to + exponential notation instead. */ + convert_to_exp = 1; + /* no exponent, no point, and we shouldn't land here + for infs and nans, so we must be at the end of the + string. */ + assert(*p == '\0'); + } + else { + assert(precision == -1 || digit_count < precision); + chars_to_insert = ".0"; + insert_count = 2; + } } if (insert_count) { size_t buf_len = strlen(buffer); @@ -339,37 +588,30 @@ ensure_decimal_point(char* buffer, size_t buf_size) memcpy(p, chars_to_insert, insert_count); } } -} - -/* Add the locale specific grouping characters to buffer. Note - that any decimal point (if it's present) in buffer is already - locale-specific. Return 0 on error, else 1. */ -Py_LOCAL_INLINE(int) -add_thousands_grouping(char* buffer, size_t buf_size) -{ - Py_ssize_t len = strlen(buffer); - struct lconv *locale_data = localeconv(); - const char *decimal_point = locale_data->decimal_point; - - /* Find the decimal point, if any. We're only concerned - about the characters to the left of the decimal when - adding grouping. */ - char *p = strstr(buffer, decimal_point); - if (!p) { - /* No decimal, use the entire string. */ - - /* If any exponent, adjust p. */ - p = strpbrk(buffer, "eE"); - if (!p) - /* No exponent and no decimal. Use the entire - string. */ - p = buffer + len; + if (convert_to_exp) { + int written; + size_t buf_avail; + p = digits_start; + /* insert decimal point */ + assert(digit_count >= 1); + memmove(p+2, p+1, digit_count); /* safe, but overwrites nul */ + p[1] = '.'; + p += digit_count+1; + assert(p <= buf_size+buffer); + buf_avail = buf_size+buffer-p; + if (buf_avail == 0) + return NULL; + /* Add exponent. It's okay to use lower case 'e': we only + arrive here as a result of using the empty format code or + repr/str builtins and those never want an upper case 'E' */ + written = PyOS_snprintf(p, buf_avail, "e%+.02d", digit_count-1); + if (!(0 <= written && + written < Py_SAFE_DOWNCAST(buf_avail, size_t, int))) + /* output truncated, or something else bad happened */ + return NULL; + remove_trailing_zeros(buffer); } - /* At this point, p points just past the right-most character we - want to format. We need to add the grouping string for the - characters between buffer and p. */ - return _PyString_InsertThousandsGrouping(buffer, len, p-buffer, - buf_size, NULL, 1); + return buffer; } /* see FORMATBUFLEN in unicodeobject.c */ @@ -386,28 +628,24 @@ add_thousands_grouping(char* buffer, size_t buf_size) * Converts a #gdouble to a string, using the '.' as * decimal point. To format the number you pass in * a printf()-style format string. Allowed conversion - * specifiers are 'e', 'E', 'f', 'F', 'g', 'G', and 'n'. + * specifiers are 'e', 'E', 'f', 'F', 'g', 'G', and 'Z'. * - * 'n' is the same as 'g', except it uses the current locale. * 'Z' is the same as 'g', except it always has a decimal and * at least one digit after the decimal. * * Return value: The pointer to the buffer with the converted string. + * On failure returns NULL but does not set any Python exception. **/ char * -PyOS_ascii_formatd(char *buffer, +_PyOS_ascii_formatd(char *buffer, size_t buf_size, const char *format, - double d) + double d, + int precision) { char format_char; size_t format_len = strlen(format); - /* For type 'n', we need to make a copy of the format string, because - we're going to modify 'n' -> 'g', and format is const char*, so we - can't modify it directly. FLOAT_FORMATBUFLEN should be longer than - we ever need this to be. There's an upcoming check to ensure it's - big enough. */ /* Issue 2264: code 'Z' requires copying the format. 'Z' is 'g', but also with at least one character past the decimal. */ char tmp_format[FLOAT_FORMATBUFLEN]; @@ -433,12 +671,12 @@ PyOS_ascii_formatd(char *buffer, if (!(format_char == 'e' || format_char == 'E' || format_char == 'f' || format_char == 'F' || format_char == 'g' || format_char == 'G' || - format_char == 'n' || format_char == 'Z')) + format_char == 'Z')) return NULL; - /* Map 'n' or 'Z' format_char to 'g', by copying the format string and + /* Map 'Z' format_char to 'g', by copying the format string and replacing the final char with a 'g' */ - if (format_char == 'n' || format_char == 'Z') { + if (format_char == 'Z') { if (format_len + 1 >= sizeof(tmp_format)) { /* The format won't fit in our copy. Error out. In practice, this will never happen and will be @@ -457,11 +695,8 @@ PyOS_ascii_formatd(char *buffer, /* Do various fixups on the return string */ /* Get the current locale, and find the decimal point string. - Convert that string back to a dot. Do not do this if using the - 'n' (number) format code, since we want to keep the localized - decimal point in that case. */ - if (format_char != 'n') - change_decimal_from_locale_to_dot(buffer); + Convert that string back to a dot. */ + change_decimal_from_locale_to_dot(buffer); /* If an exponent exists, ensure that the exponent is at least MIN_EXPONENT_DIGITS digits, providing the buffer is large enough @@ -471,20 +706,544 @@ PyOS_ascii_formatd(char *buffer, ensure_minimum_exponent_length(buffer, buf_size); /* If format_char is 'Z', make sure we have at least one character - after the decimal point (and make sure we have a decimal point). */ + after the decimal point (and make sure we have a decimal point); + also switch to exponential notation in some edge cases where the + extra character would produce more significant digits that we + really want. */ if (format_char == 'Z') - ensure_decimal_point(buffer, buf_size); + buffer = ensure_decimal_point(buffer, buf_size, precision); + + return buffer; +} + +char * +PyOS_ascii_formatd(char *buffer, + size_t buf_size, + const char *format, + double d) +{ + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "PyOS_ascii_formatd is deprecated, " + "use PyOS_double_to_string instead", 1) < 0) + return NULL; + + return _PyOS_ascii_formatd(buffer, buf_size, format, d, -1); +} + +#ifdef PY_NO_SHORT_FLOAT_REPR - /* If format_char is 'n', add the thousands grouping. */ - if (format_char == 'n') - if (!add_thousands_grouping(buffer, buf_size)) +/* The fallback code to use if _Py_dg_dtoa is not available. */ + +PyAPI_FUNC(char *) PyOS_double_to_string(double val, + char format_code, + int precision, + int flags, + int *type) +{ + char format[32]; + Py_ssize_t bufsize; + char *buf; + int t, exp; + int upper = 0; + + /* Validate format_code, and map upper and lower case */ + switch (format_code) { + case 'e': /* exponent */ + case 'f': /* fixed */ + case 'g': /* general */ + break; + case 'E': + upper = 1; + format_code = 'e'; + break; + case 'F': + upper = 1; + format_code = 'f'; + break; + case 'G': + upper = 1; + format_code = 'g'; + break; + case 'r': /* repr format */ + /* Supplied precision is unused, must be 0. */ + if (precision != 0) { + PyErr_BadInternalCall(); return NULL; + } + /* The repr() precision (17 significant decimal digits) is the + minimal number that is guaranteed to have enough precision + so that if the number is read back in the exact same binary + value is recreated. This is true for IEEE floating point + by design, and also happens to work for all other modern + hardware. */ + precision = 17; + format_code = 'g'; + break; + default: + PyErr_BadInternalCall(); + return NULL; + } - return buffer; + /* Here's a quick-and-dirty calculation to figure out how big a buffer + we need. In general, for a finite float we need: + + 1 byte for each digit of the decimal significand, and + + 1 for a possible sign + 1 for a possible decimal point + 2 for a possible [eE][+-] + 1 for each digit of the exponent; if we allow 19 digits + total then we're safe up to exponents of 2**63. + 1 for the trailing nul byte + + This gives a total of 24 + the number of digits in the significand, + and the number of digits in the significand is: + + for 'g' format: at most precision, except possibly + when precision == 0, when it's 1. + for 'e' format: precision+1 + for 'f' format: precision digits after the point, at least 1 + before. To figure out how many digits appear before the point + we have to examine the size of the number. If fabs(val) < 1.0 + then there will be only one digit before the point. If + fabs(val) >= 1.0, then there are at most + + 1+floor(log10(ceiling(fabs(val)))) + + digits before the point (where the 'ceiling' allows for the + possibility that the rounding rounds the integer part of val + up). A safe upper bound for the above quantity is + 1+floor(exp/3), where exp is the unique integer such that 0.5 + <= fabs(val)/2**exp < 1.0. This exp can be obtained from + frexp. + + So we allow room for precision+1 digits for all formats, plus an + extra floor(exp/3) digits for 'f' format. + + */ + + if (Py_IS_NAN(val) || Py_IS_INFINITY(val)) + /* 3 for 'inf'/'nan', 1 for sign, 1 for '\0' */ + bufsize = 5; + else { + bufsize = 25 + precision; + if (format_code == 'f' && fabs(val) >= 1.0) { + frexp(val, &exp); + bufsize += exp/3; + } + } + + buf = PyMem_Malloc(bufsize); + if (buf == NULL) { + PyErr_NoMemory(); + return NULL; + } + + /* Handle nan and inf. */ + if (Py_IS_NAN(val)) { + strcpy(buf, "nan"); + t = Py_DTST_NAN; + } else if (Py_IS_INFINITY(val)) { + if (copysign(1., val) == 1.) + strcpy(buf, "inf"); + else + strcpy(buf, "-inf"); + t = Py_DTST_INFINITE; + } else { + t = Py_DTST_FINITE; + if (flags & Py_DTSF_ADD_DOT_0) + format_code = 'Z'; + + PyOS_snprintf(format, sizeof(format), "%%%s.%i%c", + (flags & Py_DTSF_ALT ? "#" : ""), precision, + format_code); + _PyOS_ascii_formatd(buf, bufsize, format, val, precision); + } + + /* Add sign when requested. It's convenient (esp. when formatting + complex numbers) to include a sign even for inf and nan. */ + if (flags & Py_DTSF_SIGN && buf[0] != '-') { + size_t len = strlen(buf); + /* the bufsize calculations above should ensure that we've got + space to add a sign */ + assert((size_t)bufsize >= len+2); + memmove(buf+1, buf, len+1); + buf[0] = '+'; + } + if (upper) { + /* Convert to upper case. */ + char *p1; + for (p1 = buf; *p1; p1++) + *p1 = Py_TOUPPER(*p1); + } + + if (type) + *type = t; + return buf; } -double -PyOS_ascii_atof(const char *nptr) +#else + +/* _Py_dg_dtoa is available. */ + +/* I'm using a lookup table here so that I don't have to invent a non-locale + specific way to convert to uppercase */ +#define OFS_INF 0 +#define OFS_NAN 1 +#define OFS_E 2 + +/* The lengths of these are known to the code below, so don't change them */ +static char *lc_float_strings[] = { + "inf", + "nan", + "e", +}; +static char *uc_float_strings[] = { + "INF", + "NAN", + "E", +}; + + +/* Convert a double d to a string, and return a PyMem_Malloc'd block of + memory contain the resulting string. + + Arguments: + d is the double to be converted + format_code is one of 'e', 'f', 'g', 'r'. 'e', 'f' and 'g' + correspond to '%e', '%f' and '%g'; 'r' corresponds to repr. + mode is one of '0', '2' or '3', and is completely determined by + format_code: 'e' and 'g' use mode 2; 'f' mode 3, 'r' mode 0. + precision is the desired precision + always_add_sign is nonzero if a '+' sign should be included for positive + numbers + add_dot_0_if_integer is nonzero if integers in non-exponential form + should have ".0" added. Only applies to format codes 'r' and 'g'. + use_alt_formatting is nonzero if alternative formatting should be + used. Only applies to format codes 'e', 'f' and 'g'. For code 'g', + at most one of use_alt_formatting and add_dot_0_if_integer should + be nonzero. + type, if non-NULL, will be set to one of these constants to identify + the type of the 'd' argument: + Py_DTST_FINITE + Py_DTST_INFINITE + Py_DTST_NAN + + Returns a PyMem_Malloc'd block of memory containing the resulting string, + or NULL on error. If NULL is returned, the Python error has been set. + */ + +static char * +format_float_short(double d, char format_code, + int mode, Py_ssize_t precision, + int always_add_sign, int add_dot_0_if_integer, + int use_alt_formatting, char **float_strings, int *type) { - return PyOS_ascii_strtod(nptr, NULL); + char *buf = NULL; + char *p = NULL; + Py_ssize_t bufsize = 0; + char *digits, *digits_end; + int decpt_as_int, sign, exp_len, exp = 0, use_exp = 0; + Py_ssize_t decpt, digits_len, vdigits_start, vdigits_end; + _Py_SET_53BIT_PRECISION_HEADER; + + /* _Py_dg_dtoa returns a digit string (no decimal point or exponent). + Must be matched by a call to _Py_dg_freedtoa. */ + _Py_SET_53BIT_PRECISION_START; + digits = _Py_dg_dtoa(d, mode, precision, &decpt_as_int, &sign, + &digits_end); + _Py_SET_53BIT_PRECISION_END; + + decpt = (Py_ssize_t)decpt_as_int; + if (digits == NULL) { + /* The only failure mode is no memory. */ + PyErr_NoMemory(); + goto exit; + } + assert(digits_end != NULL && digits_end >= digits); + digits_len = digits_end - digits; + + if (digits_len && !Py_ISDIGIT(digits[0])) { + /* Infinities and nans here; adapt Gay's output, + so convert Infinity to inf and NaN to nan, and + ignore sign of nan. Then return. */ + + /* ignore the actual sign of a nan */ + if (digits[0] == 'n' || digits[0] == 'N') + sign = 0; + + /* We only need 5 bytes to hold the result "+inf\0" . */ + bufsize = 5; /* Used later in an assert. */ + buf = (char *)PyMem_Malloc(bufsize); + if (buf == NULL) { + PyErr_NoMemory(); + goto exit; + } + p = buf; + + if (sign == 1) { + *p++ = '-'; + } + else if (always_add_sign) { + *p++ = '+'; + } + if (digits[0] == 'i' || digits[0] == 'I') { + strncpy(p, float_strings[OFS_INF], 3); + p += 3; + + if (type) + *type = Py_DTST_INFINITE; + } + else if (digits[0] == 'n' || digits[0] == 'N') { + strncpy(p, float_strings[OFS_NAN], 3); + p += 3; + + if (type) + *type = Py_DTST_NAN; + } + else { + /* shouldn't get here: Gay's code should always return + something starting with a digit, an 'I', or 'N' */ + strncpy(p, "ERR", 3); + p += 3; + assert(0); + } + goto exit; + } + + /* The result must be finite (not inf or nan). */ + if (type) + *type = Py_DTST_FINITE; + + + /* We got digits back, format them. We may need to pad 'digits' + either on the left or right (or both) with extra zeros, so in + general the resulting string has the form + + [<sign>]<zeros><digits><zeros>[<exponent>] + + where either of the <zeros> pieces could be empty, and there's a + decimal point that could appear either in <digits> or in the + leading or trailing <zeros>. + + Imagine an infinite 'virtual' string vdigits, consisting of the + string 'digits' (starting at index 0) padded on both the left and + right with infinite strings of zeros. We want to output a slice + + vdigits[vdigits_start : vdigits_end] + + of this virtual string. Thus if vdigits_start < 0 then we'll end + up producing some leading zeros; if vdigits_end > digits_len there + will be trailing zeros in the output. The next section of code + determines whether to use an exponent or not, figures out the + position 'decpt' of the decimal point, and computes 'vdigits_start' + and 'vdigits_end'. */ + vdigits_end = digits_len; + switch (format_code) { + case 'e': + use_exp = 1; + vdigits_end = precision; + break; + case 'f': + vdigits_end = decpt + precision; + break; + case 'g': + if (decpt <= -4 || decpt > + (add_dot_0_if_integer ? precision-1 : precision)) + use_exp = 1; + if (use_alt_formatting) + vdigits_end = precision; + break; + case 'r': + /* convert to exponential format at 1e16. We used to convert + at 1e17, but that gives odd-looking results for some values + when a 16-digit 'shortest' repr is padded with bogus zeros. + For example, repr(2e16+8) would give 20000000000000010.0; + the true value is 20000000000000008.0. */ + if (decpt <= -4 || decpt > 16) + use_exp = 1; + break; + default: + PyErr_BadInternalCall(); + goto exit; + } + + /* if using an exponent, reset decimal point position to 1 and adjust + exponent accordingly.*/ + if (use_exp) { + exp = decpt - 1; + decpt = 1; + } + /* ensure vdigits_start < decpt <= vdigits_end, or vdigits_start < + decpt < vdigits_end if add_dot_0_if_integer and no exponent */ + vdigits_start = decpt <= 0 ? decpt-1 : 0; + if (!use_exp && add_dot_0_if_integer) + vdigits_end = vdigits_end > decpt ? vdigits_end : decpt + 1; + else + vdigits_end = vdigits_end > decpt ? vdigits_end : decpt; + + /* double check inequalities */ + assert(vdigits_start <= 0 && + 0 <= digits_len && + digits_len <= vdigits_end); + /* decimal point should be in (vdigits_start, vdigits_end] */ + assert(vdigits_start < decpt && decpt <= vdigits_end); + + /* Compute an upper bound how much memory we need. This might be a few + chars too long, but no big deal. */ + bufsize = + /* sign, decimal point and trailing 0 byte */ + 3 + + + /* total digit count (including zero padding on both sides) */ + (vdigits_end - vdigits_start) + + + /* exponent "e+100", max 3 numerical digits */ + (use_exp ? 5 : 0); + + /* Now allocate the memory and initialize p to point to the start of + it. */ + buf = (char *)PyMem_Malloc(bufsize); + if (buf == NULL) { + PyErr_NoMemory(); + goto exit; + } + p = buf; + + /* Add a negative sign if negative, and a plus sign if non-negative + and always_add_sign is true. */ + if (sign == 1) + *p++ = '-'; + else if (always_add_sign) + *p++ = '+'; + + /* note that exactly one of the three 'if' conditions is true, + so we include exactly one decimal point */ + /* Zero padding on left of digit string */ + if (decpt <= 0) { + memset(p, '0', decpt-vdigits_start); + p += decpt - vdigits_start; + *p++ = '.'; + memset(p, '0', 0-decpt); + p += 0-decpt; + } + else { + memset(p, '0', 0-vdigits_start); + p += 0 - vdigits_start; + } + + /* Digits, with included decimal point */ + if (0 < decpt && decpt <= digits_len) { + strncpy(p, digits, decpt-0); + p += decpt-0; + *p++ = '.'; + strncpy(p, digits+decpt, digits_len-decpt); + p += digits_len-decpt; + } + else { + strncpy(p, digits, digits_len); + p += digits_len; + } + + /* And zeros on the right */ + if (digits_len < decpt) { + memset(p, '0', decpt-digits_len); + p += decpt-digits_len; + *p++ = '.'; + memset(p, '0', vdigits_end-decpt); + p += vdigits_end-decpt; + } + else { + memset(p, '0', vdigits_end-digits_len); + p += vdigits_end-digits_len; + } + + /* Delete a trailing decimal pt unless using alternative formatting. */ + if (p[-1] == '.' && !use_alt_formatting) + p--; + + /* Now that we've done zero padding, add an exponent if needed. */ + if (use_exp) { + *p++ = float_strings[OFS_E][0]; + exp_len = sprintf(p, "%+.02d", exp); + p += exp_len; + } + exit: + if (buf) { + *p = '\0'; + /* It's too late if this fails, as we've already stepped on + memory that isn't ours. But it's an okay debugging test. */ + assert(p-buf < bufsize); + } + if (digits) + _Py_dg_freedtoa(digits); + + return buf; +} + + +PyAPI_FUNC(char *) PyOS_double_to_string(double val, + char format_code, + int precision, + int flags, + int *type) +{ + char **float_strings = lc_float_strings; + int mode; + + /* Validate format_code, and map upper and lower case. Compute the + mode and make any adjustments as needed. */ + switch (format_code) { + /* exponent */ + case 'E': + float_strings = uc_float_strings; + format_code = 'e'; + /* Fall through. */ + case 'e': + mode = 2; + precision++; + break; + + /* fixed */ + case 'F': + float_strings = uc_float_strings; + format_code = 'f'; + /* Fall through. */ + case 'f': + mode = 3; + break; + + /* general */ + case 'G': + float_strings = uc_float_strings; + format_code = 'g'; + /* Fall through. */ + case 'g': + mode = 2; + /* precision 0 makes no sense for 'g' format; interpret as 1 */ + if (precision == 0) + precision = 1; + break; + + /* repr format */ + case 'r': + mode = 0; + /* Supplied precision is unused, must be 0. */ + if (precision != 0) { + PyErr_BadInternalCall(); + return NULL; + } + break; + + default: + PyErr_BadInternalCall(); + return NULL; + } + + return format_float_short(val, format_code, mode, precision, + flags & Py_DTSF_SIGN, + flags & Py_DTSF_ADD_DOT_0, + flags & Py_DTSF_ALT, + float_strings, type); } +#endif /* ifdef PY_NO_SHORT_FLOAT_REPR */ diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 2f8318bc5f..afb4c51017 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -76,7 +76,7 @@ extern void _PyGILState_Fini(void); int Py_DebugFlag; /* Needed by parser.c */ int Py_VerboseFlag; /* Needed by import.c */ int Py_InteractiveFlag; /* Needed by Py_FdIsInteractive() below */ -int Py_InspectFlag; /* Needed to determine whether to exit at SystemError */ +int Py_InspectFlag; /* Needed to determine whether to exit at SystemExit */ int Py_NoSiteFlag; /* Suppress 'import site' */ int Py_BytesWarningFlag; /* Warn on str(bytes) and str(buffer) */ int Py_DontWriteBytecodeFlag; /* Suppress writing bytecode files (*.py[co]) */ @@ -191,6 +191,9 @@ Py_InitializeEx(int install_sigs) if (!_PyInt_Init()) Py_FatalError("Py_Initialize: can't init ints"); + if (!_PyLong_Init()) + Py_FatalError("Py_Initialize: can't init longs"); + if (!PyByteArray_Init()) Py_FatalError("Py_Initialize: can't init bytearray"); @@ -705,7 +708,7 @@ initmain(void) if (bimod == NULL || PyDict_SetItemString(d, "__builtins__", bimod) != 0) Py_FatalError("can't add __builtins__ to __main__"); - Py_DECREF(bimod); + Py_XDECREF(bimod); } } @@ -714,20 +717,12 @@ initmain(void) static void initsite(void) { - PyObject *m, *f; + PyObject *m; m = PyImport_ImportModule("site"); if (m == NULL) { - f = PySys_GetObject("stderr"); - if (Py_VerboseFlag) { - PyFile_WriteString( - "'import site' failed; traceback:\n", f); - PyErr_Print(); - } - else { - PyFile_WriteString( - "'import site' failed; use -v for traceback\n", f); - PyErr_Clear(); - } + PyErr_Print(); + Py_Finalize(); + exit(1); } else { Py_DECREF(m); @@ -994,55 +989,67 @@ parse_syntax_error(PyObject *err, PyObject **message, const char **filename, return PyArg_ParseTuple(err, "O(ziiz)", message, filename, lineno, offset, text); - /* new style errors. `err' is an instance */ + *message = NULL; - if (! (v = PyObject_GetAttrString(err, "msg"))) + /* new style errors. `err' is an instance */ + *message = PyObject_GetAttrString(err, "msg"); + if (!*message) goto finally; - *message = v; - if (!(v = PyObject_GetAttrString(err, "filename"))) + v = PyObject_GetAttrString(err, "filename"); + if (!v) goto finally; - if (v == Py_None) + if (v == Py_None) { + Py_DECREF(v); *filename = NULL; - else if (! (*filename = PyString_AsString(v))) - goto finally; + } + else { + *filename = PyString_AsString(v); + Py_DECREF(v); + if (!*filename) + goto finally; + } - Py_DECREF(v); - if (!(v = PyObject_GetAttrString(err, "lineno"))) + v = PyObject_GetAttrString(err, "lineno"); + if (!v) goto finally; hold = PyInt_AsLong(v); Py_DECREF(v); - v = NULL; if (hold < 0 && PyErr_Occurred()) goto finally; *lineno = (int)hold; - if (!(v = PyObject_GetAttrString(err, "offset"))) + v = PyObject_GetAttrString(err, "offset"); + if (!v) goto finally; if (v == Py_None) { *offset = -1; Py_DECREF(v); - v = NULL; } else { hold = PyInt_AsLong(v); Py_DECREF(v); - v = NULL; if (hold < 0 && PyErr_Occurred()) goto finally; *offset = (int)hold; } - if (!(v = PyObject_GetAttrString(err, "text"))) + v = PyObject_GetAttrString(err, "text"); + if (!v) goto finally; - if (v == Py_None) + if (v == Py_None) { + Py_DECREF(v); *text = NULL; - else if (! (*text = PyString_AsString(v))) - goto finally; - Py_DECREF(v); + } + else { + *text = PyString_AsString(v); + Py_DECREF(v); + if (!*text) + goto finally; + } return 1; finally: - Py_XDECREF(v); + Py_XDECREF(*message); return 0; } @@ -1057,7 +1064,7 @@ print_error_text(PyObject *f, int offset, const char *text) { char *nl; if (offset >= 0) { - if (offset > 0 && offset == (int)strlen(text)) + if (offset > 0 && offset == strlen(text) && text[offset - 1] == '\n') offset--; for (;;) { nl = strchr(text, '\n'); @@ -1161,7 +1168,7 @@ PyErr_PrintEx(int set_sys_last_vars) PySys_SetObject("last_traceback", tb); } hook = PySys_GetObject("excepthook"); - if (hook) { + if (hook && hook != Py_None) { PyObject *args = PyTuple_Pack(3, exception, v, tb ? tb : Py_None); PyObject *result = PyEval_CallObject(hook, args); @@ -1211,7 +1218,7 @@ PyErr_Display(PyObject *exception, PyObject *value, PyObject *tb) int err = 0; PyObject *f = PySys_GetObject("stderr"); Py_INCREF(value); - if (f == NULL) + if (f == NULL || f == Py_None) fprintf(stderr, "lost sys.stderr\n"); else { if (Py_FlushLine()) diff --git a/Python/strtod.c b/Python/strtod.c index 6798efa927..ee558982d5 100644 --- a/Python/strtod.c +++ b/Python/strtod.c @@ -1,3 +1,6 @@ +#include <stdio.h> +#include <string.h> + #include "pyconfig.h" /* comp.sources.misc strtod(), as posted in comp.lang.tcl, @@ -77,7 +80,7 @@ double strtod(char *str, char **ptr) dp = buffer; *dp++ = '0'; *dp++ = '.'; buforg = dp, buflim = buffer+48; - for (save = sp; c = *sp; sp++) + for (save = sp; (c = *sp); sp++) if (c == '.') { if (dotseen) break; dotseen++; diff --git a/Python/structmember.c b/Python/structmember.c index 4cef42fdc5..e347b51aac 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -257,12 +257,13 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) } case T_UINT:{ unsigned long ulong_val = PyLong_AsUnsignedLong(v); - if ((ulong_val == (unsigned int)-1) && PyErr_Occurred()) { + if ((ulong_val == (unsigned long)-1) && PyErr_Occurred()) { /* XXX: For compatibility, accept negative int values as well. */ PyErr_Clear(); ulong_val = PyLong_AsLong(v); - if ((ulong_val == (unsigned int)-1) && PyErr_Occurred()) + if ((ulong_val == (unsigned long)-1) && + PyErr_Occurred()) return -1; *(unsigned int *)addr = (unsigned int)ulong_val; WARN("Writing negative value into unsigned field"); @@ -286,7 +287,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) as well. */ PyErr_Clear(); *(unsigned long*)addr = PyLong_AsLong(v); - if ((*(unsigned long*)addr == (unsigned int)-1) + if ((*(unsigned long*)addr == (unsigned long)-1) && PyErr_Occurred()) return -1; WARN("Writing negative value into unsigned field"); diff --git a/Python/symtable.c b/Python/symtable.c index f30ba0f761..51b59f0310 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -32,7 +32,6 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block, goto fail; ste->ste_table = st; ste->ste_id = k; - ste->ste_tmpname = 0; ste->ste_name = name; Py_INCREF(name); @@ -168,6 +167,8 @@ static int symtable_exit_block(struct symtable *st, void *ast); static int symtable_visit_stmt(struct symtable *st, stmt_ty s); static int symtable_visit_expr(struct symtable *st, expr_ty s); static int symtable_visit_genexp(struct symtable *st, expr_ty s); +static int symtable_visit_setcomp(struct symtable *st, expr_ty e); +static int symtable_visit_dictcomp(struct symtable *st, expr_ty e); static int symtable_visit_arguments(struct symtable *st, arguments_ty); static int symtable_visit_excepthandler(struct symtable *st, excepthandler_ty); static int symtable_visit_alias(struct symtable *st, alias_ty); @@ -179,7 +180,8 @@ static int symtable_visit_params_nested(struct symtable *st, asdl_seq *args); static int symtable_implicit_arg(struct symtable *st, int pos); -static identifier top = NULL, lambda = NULL, genexpr = NULL; +static identifier top = NULL, lambda = NULL, genexpr = NULL, setcomp = NULL, + dictcomp = NULL; #define GET_IDENTIFIER(VAR) \ ((VAR) ? (VAR) : ((VAR) = PyString_InternFromString(# VAR))) @@ -204,7 +206,6 @@ symtable_new(void) if ((st->st_symbols = PyDict_New()) == NULL) goto fail; st->st_cur = NULL; - st->st_tmpname = 0; st->st_private = NULL; return st; fail: @@ -495,7 +496,7 @@ check_unoptimized(const PySTEntryObject* ste) { case OPT_IMPORT_STAR: PyOS_snprintf(buf, sizeof(buf), "import * is not allowed in function '%.100s' " - "because it is %s", + "because it %s", PyString_AS_STRING(ste->ste_name), trailer); break; case OPT_BARE_EXEC: @@ -702,7 +703,8 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, ste->ste_child_free = 1; } - PyDict_Update(newfree, allfree); + if (PyDict_Update(newfree, allfree) < 0) + goto error; if (ste->ste_type == FunctionBlock && !analyze_cells(scope, newfree)) goto error; if (!update_symbols(ste->ste_symbols, scope, bound, newfree, @@ -846,7 +848,7 @@ symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block, st->st_cur = ste_new(st, name, block, ast, lineno); if (st->st_cur == NULL) return 0; - if (name == GET_IDENTIFIER(top)) + if (block == ModuleBlock) st->st_global = st->st_cur->ste_symbols; if (prev) { if (PyList_Append(prev->ste_children, @@ -994,23 +996,6 @@ error: } static int -symtable_new_tmpname(struct symtable *st) -{ - char tmpname[256]; - identifier tmp; - - PyOS_snprintf(tmpname, sizeof(tmpname), "_[%d]", - ++st->st_cur->ste_tmpname); - tmp = PyString_InternFromString(tmpname); - if (!tmp) - return 0; - if (!symtable_add_def(st, tmp, DEF_LOCAL)) - return 0; - Py_DECREF(tmp); - return 1; -} - -static int symtable_visit_stmt(struct symtable *st, stmt_ty s) { switch (s->kind) { @@ -1183,12 +1168,8 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) /* nothing to do here */ break; case With_kind: - if (!symtable_new_tmpname(st)) - return 0; VISIT(st, expr, s->v.With.context_expr); if (s->v.With.optional_vars) { - if (!symtable_new_tmpname(st)) - return 0; VISIT(st, expr, s->v.With.optional_vars); } VISIT_SEQ(st, stmt, s->v.With.body); @@ -1212,8 +1193,7 @@ symtable_visit_expr(struct symtable *st, expr_ty e) VISIT(st, expr, e->v.UnaryOp.operand); break; case Lambda_kind: { - if (!GET_IDENTIFIER(lambda) || - !symtable_add_def(st, lambda, DEF_LOCAL)) + if (!GET_IDENTIFIER(lambda)) return 0; if (e->v.Lambda.args->defaults) VISIT_SEQ(st, expr, e->v.Lambda.args->defaults); @@ -1235,9 +1215,10 @@ symtable_visit_expr(struct symtable *st, expr_ty e) VISIT_SEQ(st, expr, e->v.Dict.keys); VISIT_SEQ(st, expr, e->v.Dict.values); break; + case Set_kind: + VISIT_SEQ(st, expr, e->v.Set.elts); + break; case ListComp_kind: - if (!symtable_new_tmpname(st)) - return 0; VISIT(st, expr, e->v.ListComp.elt); VISIT_SEQ(st, comprehension, e->v.ListComp.generators); break; @@ -1245,6 +1226,14 @@ symtable_visit_expr(struct symtable *st, expr_ty e) if (!symtable_visit_genexp(st, e)) return 0; break; + case SetComp_kind: + if (!symtable_visit_setcomp(st, e)) + return 0; + break; + case DictComp_kind: + if (!symtable_visit_dictcomp(st, e)) + return 0; + break; case Yield_kind: if (e->v.Yield.value) VISIT(st, expr, e->v.Yield.value); @@ -1407,7 +1396,7 @@ static int symtable_visit_alias(struct symtable *st, alias_ty a) { /* Compute store_name, the name actually bound by the import - operation. It is diferent than a->name when a->name is a + operation. It is different than a->name when a->name is a dotted package name (e.g. spam.eggs) */ PyObject *store_name; @@ -1486,27 +1475,80 @@ symtable_visit_slice(struct symtable *st, slice_ty s) } static int -symtable_visit_genexp(struct symtable *st, expr_ty e) +symtable_new_tmpname(struct symtable *st) { + char tmpname[256]; + identifier tmp; + + PyOS_snprintf(tmpname, sizeof(tmpname), "_[%d]", + ++st->st_cur->ste_tmpname); + tmp = PyString_InternFromString(tmpname); + if (!tmp) + return 0; + if (!symtable_add_def(st, tmp, DEF_LOCAL)) + return 0; + Py_DECREF(tmp); + return 1; +} + +static int +symtable_handle_comprehension(struct symtable *st, expr_ty e, + identifier scope_name, asdl_seq *generators, + expr_ty elt, expr_ty value) +{ + int is_generator = (e->kind == GeneratorExp_kind); + int needs_tmp = !is_generator; comprehension_ty outermost = ((comprehension_ty) - (asdl_seq_GET(e->v.GeneratorExp.generators, 0))); + asdl_seq_GET(generators, 0)); /* Outermost iterator is evaluated in current scope */ VISIT(st, expr, outermost->iter); - /* Create generator scope for the rest */ - if (!GET_IDENTIFIER(genexpr) || - !symtable_enter_block(st, genexpr, FunctionBlock, (void *)e, e->lineno)) { + /* Create comprehension scope for the rest */ + if (!scope_name || + !symtable_enter_block(st, scope_name, FunctionBlock, (void *)e, 0)) { return 0; } - st->st_cur->ste_generator = 1; + st->st_cur->ste_generator = is_generator; /* Outermost iter is received as an argument */ if (!symtable_implicit_arg(st, 0)) { symtable_exit_block(st, (void *)e); return 0; } + /* Allocate temporary name if needed */ + if (needs_tmp && !symtable_new_tmpname(st)) { + symtable_exit_block(st, (void *)e); + return 0; + } VISIT_IN_BLOCK(st, expr, outermost->target, (void*)e); VISIT_SEQ_IN_BLOCK(st, expr, outermost->ifs, (void*)e); VISIT_SEQ_TAIL_IN_BLOCK(st, comprehension, - e->v.GeneratorExp.generators, 1, (void*)e); - VISIT_IN_BLOCK(st, expr, e->v.GeneratorExp.elt, (void*)e); + generators, 1, (void*)e); + if (value) + VISIT_IN_BLOCK(st, expr, value, (void*)e); + VISIT_IN_BLOCK(st, expr, elt, (void*)e); return symtable_exit_block(st, (void *)e); } + +static int +symtable_visit_genexp(struct symtable *st, expr_ty e) +{ + return symtable_handle_comprehension(st, e, GET_IDENTIFIER(genexpr), + e->v.GeneratorExp.generators, + e->v.GeneratorExp.elt, NULL); +} + +static int +symtable_visit_setcomp(struct symtable *st, expr_ty e) +{ + return symtable_handle_comprehension(st, e, GET_IDENTIFIER(setcomp), + e->v.SetComp.generators, + e->v.SetComp.elt, NULL); +} + +static int +symtable_visit_dictcomp(struct symtable *st, expr_ty e) +{ + return symtable_handle_comprehension(st, e, GET_IDENTIFIER(dictcomp), + e->v.DictComp.generators, + e->v.DictComp.key, + e->v.DictComp.value); +} diff --git a/Python/sysmodule.c b/Python/sysmodule.c index e73e5c283e..814eccb157 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -126,7 +126,7 @@ sys_displayhook(PyObject *self, PyObject *o) PyDoc_STRVAR(displayhook_doc, "displayhook(object) -> None\n" "\n" -"Print an object to sys.stdout and also save it in __builtin__.\n" +"Print an object to sys.stdout and also save it in __builtin__._\n" ); static PyObject * @@ -466,6 +466,7 @@ sys_setcheckinterval(PyObject *self, PyObject *args) { if (!PyArg_ParseTuple(args, "i:setcheckinterval", &_Py_CheckInterval)) return NULL; + _Py_Ticker = _Py_CheckInterval; Py_INCREF(Py_None); return Py_None; } @@ -557,26 +558,65 @@ recursion from causing an overflow of the C stack and crashing Python." PyDoc_STRVAR(getwindowsversion_doc, "getwindowsversion()\n\ \n\ -Return information about the running version of Windows.\n\ -The result is a tuple of (major, minor, build, platform, text)\n\ -All elements are numbers, except text which is a string.\n\ -Platform may be 0 for win32s, 1 for Windows 9x/ME, 2 for Windows NT/2000/XP\n\ -" +Return information about the running version of Windows as a named tuple.\n\ +The members are named: major, minor, build, platform, service_pack,\n\ +service_pack_major, service_pack_minor, suite_mask, and product_type. For\n\ +backward compatibility, only the first 5 items are available by indexing.\n\ +All elements are numbers, except service_pack which is a string. Platform\n\ +may be 0 for win32s, 1 for Windows 9x/ME, 2 for Windows NT/2000/XP/Vista/7,\n\ +3 for Windows CE. Product_type may be 1 for a workstation, 2 for a domain\n\ +controller, 3 for a server." ); +static PyTypeObject WindowsVersionType = {0, 0, 0, 0, 0, 0}; + +static PyStructSequence_Field windows_version_fields[] = { + {"major", "Major version number"}, + {"minor", "Minor version number"}, + {"build", "Build number"}, + {"platform", "Operating system platform"}, + {"service_pack", "Latest Service Pack installed on the system"}, + {"service_pack_major", "Service Pack major version number"}, + {"service_pack_minor", "Service Pack minor version number"}, + {"suite_mask", "Bit mask identifying available product suites"}, + {"product_type", "System product type"}, + {0} +}; + +static PyStructSequence_Desc windows_version_desc = { + "sys.getwindowsversion", /* name */ + getwindowsversion_doc, /* doc */ + windows_version_fields, /* fields */ + 5 /* For backward compatibility, + only the first 5 items are accessible + via indexing, the rest are name only */ +}; + static PyObject * sys_getwindowsversion(PyObject *self) { - OSVERSIONINFO ver; + PyObject *version; + int pos = 0; + OSVERSIONINFOEX ver; ver.dwOSVersionInfoSize = sizeof(ver); - if (!GetVersionEx(&ver)) + if (!GetVersionEx((OSVERSIONINFO*) &ver)) return PyErr_SetFromWindowsErr(0); - return Py_BuildValue("HHHHs", - ver.dwMajorVersion, - ver.dwMinorVersion, - ver.dwBuildNumber, - ver.dwPlatformId, - ver.szCSDVersion); + + version = PyStructSequence_New(&WindowsVersionType); + if (version == NULL) + return NULL; + + PyStructSequence_SET_ITEM(version, pos++, PyInt_FromLong(ver.dwMajorVersion)); + PyStructSequence_SET_ITEM(version, pos++, PyInt_FromLong(ver.dwMinorVersion)); + PyStructSequence_SET_ITEM(version, pos++, PyInt_FromLong(ver.dwBuildNumber)); + PyStructSequence_SET_ITEM(version, pos++, PyInt_FromLong(ver.dwPlatformId)); + PyStructSequence_SET_ITEM(version, pos++, PyString_FromString(ver.szCSDVersion)); + PyStructSequence_SET_ITEM(version, pos++, PyInt_FromLong(ver.wServicePackMajor)); + PyStructSequence_SET_ITEM(version, pos++, PyInt_FromLong(ver.wServicePackMinor)); + PyStructSequence_SET_ITEM(version, pos++, PyInt_FromLong(ver.wSuiteMask)); + PyStructSequence_SET_ITEM(version, pos++, PyInt_FromLong(ver.wProductType)); + + return version; } #endif /* MS_WINDOWS */ @@ -599,12 +639,14 @@ sys_setdlopenflags(PyObject *self, PyObject *args) PyDoc_STRVAR(setdlopenflags_doc, "setdlopenflags(n) -> None\n\ \n\ -Set the flags that will be used for dlopen() calls. Among other\n\ -things, this will enable a lazy resolving of symbols when importing\n\ -a module, if called as sys.setdlopenflags(0)\n\ -To share symbols across extension modules, call as\n\ -sys.setdlopenflags(dl.RTLD_NOW|dl.RTLD_GLOBAL)" -); +Set the flags used by the interpreter for dlopen calls, such as when the\n\ +interpreter loads extension modules. Among other things, this will enable\n\ +a lazy resolving of symbols when importing a module, if called as\n\ +sys.setdlopenflags(0). To share symbols across extension modules, call as\n\ +sys.setdlopenflags(ctypes.RTLD_GLOBAL). Symbolic names for the flag modules\n\ +can be either found in the ctypes module, or in the DLFCN module. If DLFCN\n\ +is not available, it can be generated from /usr/include/dlfcn.h using the\n\ +h2py script."); static PyObject * sys_getdlopenflags(PyObject *self, PyObject *args) @@ -618,10 +660,10 @@ sys_getdlopenflags(PyObject *self, PyObject *args) PyDoc_STRVAR(getdlopenflags_doc, "getdlopenflags() -> int\n\ \n\ -Return the current value of the flags that are used for dlopen()\n\ -calls. The flag constants are defined in the dl module." -); -#endif +Return the current value of the flags that are used for dlopen calls.\n\ +The flag constants are defined in the ctypes and DLFCN modules."); + +#endif /* HAVE_DLOPEN */ #ifdef USE_MALLOPT /* Link with -lmalloc (or -lmpc) on an SGI */ @@ -643,7 +685,7 @@ static PyObject * sys_getsizeof(PyObject *self, PyObject *args, PyObject *kwds) { PyObject *res = NULL; - static PyObject *str__sizeof__, *gc_head_size = NULL; + static PyObject *str__sizeof__ = NULL, *gc_head_size = NULL; static char *kwlist[] = {"object", "default", 0}; PyObject *o, *dflt = NULL; @@ -651,13 +693,6 @@ sys_getsizeof(PyObject *self, PyObject *args, PyObject *kwds) kwlist, &o, &dflt)) return NULL; - /* Initialize static variable needed by _PyType_Lookup */ - if (str__sizeof__ == NULL) { - str__sizeof__ = PyString_InternFromString("__sizeof__"); - if (str__sizeof__ == NULL) - return NULL; - } - /* Initialize static variable for GC head size */ if (gc_head_size == NULL) { gc_head_size = PyInt_FromSsize_t(sizeof(PyGC_Head)); @@ -674,14 +709,18 @@ sys_getsizeof(PyObject *self, PyObject *args, PyObject *kwds) res = PyInt_FromSsize_t(PyInstance_Type.tp_basicsize); /* all other objects */ else { - PyObject *method = _PyType_Lookup(Py_TYPE(o), - str__sizeof__); - if (method == NULL) - PyErr_Format(PyExc_TypeError, - "Type %.100s doesn't define __sizeof__", - Py_TYPE(o)->tp_name); - else - res = PyObject_CallFunctionObjArgs(method, o, NULL); + PyObject *method = _PyObject_LookupSpecial(o, "__sizeof__", + &str__sizeof__); + if (method == NULL) { + if (!PyErr_Occurred()) + PyErr_Format(PyExc_TypeError, + "Type %.100s doesn't define __sizeof__", + Py_TYPE(o)->tp_name); + } + else { + res = PyObject_CallFunctionObjArgs(method, NULL); + Py_DECREF(method); + } } /* Has a default value been given? */ @@ -1043,18 +1082,21 @@ PyDoc_STR( "\n\ Static objects:\n\ \n\ +float_info -- a dict with information about the float inplementation.\n\ +long_info -- a struct sequence with information about the long implementation.\n\ maxint -- the largest supported integer (the smallest is -maxint-1)\n\ maxsize -- the largest supported length of containers.\n\ maxunicode -- the largest supported character\n\ builtin_module_names -- tuple of module names built into this interpreter\n\ version -- the version of this interpreter as a string\n\ -version_info -- version information as a tuple\n\ +version_info -- version information as a named tuple\n\ hexversion -- version information encoded as a single integer\n\ copyright -- copyright notice pertaining to this interpreter\n\ platform -- platform identifier\n\ -executable -- pathname of this Python interpreter\n\ +executable -- absolute path of the executable binary of the Python interpreter\n\ prefix -- prefix used to find the Python library\n\ exec_prefix -- prefix used to find the machine-specific Python library\n\ +float_repr_style -- string indicating the style of repr() output for floats\n\ " ) #ifdef MS_WINDOWS @@ -1102,8 +1144,6 @@ _check_and_flush (FILE *stream) } /* Subversion branch and revision management */ -static const char _patchlevel_revision[] = PY_PATCHLEVEL_REVISION; -static const char headurl[] = "$HeadURL$"; static int svn_initialized; static char patchlevel_revision[50]; /* Just the number */ static char branch[50]; @@ -1113,69 +1153,14 @@ static const char *svn_revision; static void svnversion_init(void) { - const char *python, *br_start, *br_end, *br_end2, *svnversion; - Py_ssize_t len; - int istag; - if (svn_initialized) return; - - python = strstr(headurl, "/python/"); - if (!python) { - /* XXX quick hack to get bzr working */ - *patchlevel_revision = '\0'; - strcpy(branch, ""); - strcpy(shortbranch, "unknown"); - svn_revision = ""; - return; - /* Py_FatalError("subversion keywords missing"); */ - } - - br_start = python + 8; - br_end = strchr(br_start, '/'); - assert(br_end); - - /* Works even for trunk, - as we are in trunk/Python/sysmodule.c */ - br_end2 = strchr(br_end+1, '/'); - - istag = strncmp(br_start, "tags", 4) == 0; - if (strncmp(br_start, "trunk", 5) == 0) { - strcpy(branch, "trunk"); - strcpy(shortbranch, "trunk"); - - } - else if (istag || strncmp(br_start, "branches", 8) == 0) { - len = br_end2 - br_start; - strncpy(branch, br_start, len); - branch[len] = '\0'; - - len = br_end2 - (br_end + 1); - strncpy(shortbranch, br_end + 1, len); - shortbranch[len] = '\0'; - } - else { - Py_FatalError("bad HeadURL"); - return; - } - - - svnversion = _Py_svnversion(); - if (strcmp(svnversion, "Unversioned directory") != 0 && strcmp(svnversion, "exported") != 0) - svn_revision = svnversion; - else if (istag) { - len = strlen(_patchlevel_revision); - assert(len >= 13); - assert(len < (sizeof(patchlevel_revision) + 13)); - strncpy(patchlevel_revision, _patchlevel_revision + 11, - len - 13); - patchlevel_revision[len - 13] = '\0'; - svn_revision = patchlevel_revision; - } - else - svn_revision = ""; - svn_initialized = 1; + *patchlevel_revision = '\0'; + strcpy(branch, ""); + strcpy(shortbranch, "unknown"); + svn_revision = ""; + return; } /* Return svnversion output if available. @@ -1281,6 +1266,75 @@ make_flags(void) return seq; } +PyDoc_STRVAR(version_info__doc__, +"sys.version_info\n\ +\n\ +Version information as a named tuple."); + +static PyTypeObject VersionInfoType = {0, 0, 0, 0, 0, 0}; + +static PyStructSequence_Field version_info_fields[] = { + {"major", "Major release number"}, + {"minor", "Minor release number"}, + {"micro", "Patch release number"}, + {"releaselevel", "'alpha', 'beta', 'candidate', or 'release'"}, + {"serial", "Serial release number"}, + {0} +}; + +static PyStructSequence_Desc version_info_desc = { + "sys.version_info", /* name */ + version_info__doc__, /* doc */ + version_info_fields, /* fields */ + 5 +}; + +static PyObject * +make_version_info(void) +{ + PyObject *version_info; + char *s; + int pos = 0; + + version_info = PyStructSequence_New(&VersionInfoType); + if (version_info == NULL) { + return NULL; + } + + /* + * These release level checks are mutually exclusive and cover + * the field, so don't get too fancy with the pre-processor! + */ +#if PY_RELEASE_LEVEL == PY_RELEASE_LEVEL_ALPHA + s = "alpha"; +#elif PY_RELEASE_LEVEL == PY_RELEASE_LEVEL_BETA + s = "beta"; +#elif PY_RELEASE_LEVEL == PY_RELEASE_LEVEL_GAMMA + s = "candidate"; +#elif PY_RELEASE_LEVEL == PY_RELEASE_LEVEL_FINAL + s = "final"; +#endif + +#define SetIntItem(flag) \ + PyStructSequence_SET_ITEM(version_info, pos++, PyInt_FromLong(flag)) +#define SetStrItem(flag) \ + PyStructSequence_SET_ITEM(version_info, pos++, PyString_FromString(flag)) + + SetIntItem(PY_MAJOR_VERSION); + SetIntItem(PY_MINOR_VERSION); + SetIntItem(PY_MICRO_VERSION); + SetStrItem(s); + SetIntItem(PY_RELEASE_SERIAL); +#undef SetIntItem +#undef SetStrItem + + if (PyErr_Occurred()) { + Py_CLEAR(version_info); + return NULL; + } + return version_info; +} + PyObject * _PySys_Init(void) { @@ -1354,27 +1408,11 @@ _PySys_Init(void) SET_SYS_FROM_STRING("subversion", Py_BuildValue("(ssz)", "CPython", branch, svn_revision)); + SET_SYS_FROM_STRING("_mercurial", + Py_BuildValue("(szz)", "CPython", _Py_hgidentifier(), + _Py_hgversion())); SET_SYS_FROM_STRING("dont_write_bytecode", PyBool_FromLong(Py_DontWriteBytecodeFlag)); - /* - * These release level checks are mutually exclusive and cover - * the field, so don't get too fancy with the pre-processor! - */ -#if PY_RELEASE_LEVEL == PY_RELEASE_LEVEL_ALPHA - s = "alpha"; -#elif PY_RELEASE_LEVEL == PY_RELEASE_LEVEL_BETA - s = "beta"; -#elif PY_RELEASE_LEVEL == PY_RELEASE_LEVEL_GAMMA - s = "candidate"; -#elif PY_RELEASE_LEVEL == PY_RELEASE_LEVEL_FINAL - s = "final"; -#endif - - SET_SYS_FROM_STRING("version_info", - Py_BuildValue("iiisi", PY_MAJOR_VERSION, - PY_MINOR_VERSION, - PY_MICRO_VERSION, s, - PY_RELEASE_SERIAL)); SET_SYS_FROM_STRING("api_version", PyInt_FromLong(PYTHON_API_VERSION)); SET_SYS_FROM_STRING("copyright", @@ -1395,6 +1433,8 @@ _PySys_Init(void) PyBool_FromLong(Py_Py3kWarningFlag)); SET_SYS_FROM_STRING("float_info", PyFloat_GetInfo()); + SET_SYS_FROM_STRING("long_info", + PyLong_GetInfo()); #ifdef Py_USING_UNICODE SET_SYS_FROM_STRING("maxunicode", PyInt_FromLong(PyUnicode_GetMax())); @@ -1431,6 +1471,15 @@ _PySys_Init(void) PyDict_SetItemString(sysdict, "warnoptions", warnoptions); } + /* version_info */ + if (VersionInfoType.tp_name == 0) + PyStructSequence_InitType(&VersionInfoType, &version_info_desc); + SET_SYS_FROM_STRING("version_info", make_version_info()); + /* prevent user from creating new instances */ + VersionInfoType.tp_init = NULL; + VersionInfoType.tp_new = NULL; + + /* flags */ if (FlagsType.tp_name == 0) PyStructSequence_InitType(&FlagsType, &flags_desc); SET_SYS_FROM_STRING("flags", make_flags()); @@ -1438,6 +1487,25 @@ _PySys_Init(void) FlagsType.tp_init = NULL; FlagsType.tp_new = NULL; + +#if defined(MS_WINDOWS) + /* getwindowsversion */ + if (WindowsVersionType.tp_name == 0) + PyStructSequence_InitType(&WindowsVersionType, &windows_version_desc); + /* prevent user from creating new instances */ + WindowsVersionType.tp_init = NULL; + WindowsVersionType.tp_new = NULL; +#endif + + /* float repr style: 0.03 (short) vs 0.029999999999999999 (legacy) */ +#ifndef PY_NO_SHORT_FLOAT_REPR + SET_SYS_FROM_STRING("float_repr_style", + PyString_FromString("short")); +#else + SET_SYS_FROM_STRING("float_repr_style", + PyString_FromString("legacy")); +#endif + #undef SET_SYS_FROM_STRING if (PyErr_Occurred()) return NULL; diff --git a/Python/thread.c b/Python/thread.c index 097e72d789..dd333e8f94 100644 --- a/Python/thread.c +++ b/Python/thread.c @@ -24,7 +24,7 @@ #include <stdlib.h> #ifdef __sgi -#ifndef HAVE_PTHREAD_H /* XXX Need to check in configure.in */ +#ifndef HAVE_PTHREAD_H /* XXX Need to check in configure.ac */ #undef _POSIX_THREADS #endif #endif @@ -46,7 +46,7 @@ #endif /* Check if we're running on HP-UX and _SC_THREADS is defined. If so, then - enough of the Posix threads package is implimented to support python + enough of the Posix threads package is implemented to support python threads. This is valid for HP-UX 11.23 running on an ia64 system. If needed, add @@ -137,10 +137,6 @@ static size_t _pythread_stacksize = 0; #include "thread_beos.h" #endif -#ifdef WINCE_THREADS -#include "thread_wince.h" -#endif - #ifdef PLAN9_THREADS #include "thread_plan9.h" #endif diff --git a/Python/thread_atheos.h b/Python/thread_atheos.h index c390b57d4a..230594f144 100644 --- a/Python/thread_atheos.h +++ b/Python/thread_atheos.h @@ -128,17 +128,14 @@ long PyThread_get_thread_ident(void) } -static void do_PyThread_exit_thread(int no_cleanup) +void PyThread_exit_thread(void) { dprintf(("PyThread_exit_thread called\n")); /* Thread-safe way to read a variable without a mutex: */ if (atomic_add(&thread_count, 0) == 0) { /* No threads around, so exit main(). */ - if (no_cleanup) - _exit(0); - else - exit(0); + exit(0); } else { /* We're a thread */ exit_thread(0); @@ -146,44 +143,6 @@ static void do_PyThread_exit_thread(int no_cleanup) } -void PyThread_exit_thread(void) -{ - do_PyThread_exit_thread(0); -} - - -void PyThread__exit_thread(void) -{ - do_PyThread_exit_thread(1); -} - - -#ifndef NO_EXIT_PROG -static void do_PyThread_exit_prog(int status, int no_cleanup) -{ - dprintf(("PyThread_exit_prog(%d) called\n", status)); - - /* No need to do anything, the threads get torn down if main()exits. */ - if (no_cleanup) - _exit(status); - else - exit(status); -} - - -void PyThread_exit_prog(int status) -{ - do_PyThread_exit_prog(status, 0); -} - - -void PyThread__exit_prog(int status) -{ - do_PyThread_exit_prog(status, 1); -} -#endif /* NO_EXIT_PROG */ - - /* * Lock support. * diff --git a/Python/thread_beos.h b/Python/thread_beos.h index ede44df341..65dc470bcd 100644 --- a/Python/thread_beos.h +++ b/Python/thread_beos.h @@ -144,7 +144,7 @@ long PyThread_get_thread_ident( void ) return ( tid != B_NAME_NOT_FOUND ? tid : -1 ); } -static void do_PyThread_exit_thread( int no_cleanup ) +void PyThread_exit_thread( void ) { int32 threads; @@ -155,52 +155,13 @@ static void do_PyThread_exit_thread( int no_cleanup ) if( threads == 0 ) { /* No threads around, so exit main(). */ - if( no_cleanup ) { - _exit(0); - } else { - exit(0); - } + exit(0); } else { /* Oh, we're a thread, let's try to exit gracefully... */ exit_thread( B_NO_ERROR ); } } -void PyThread_exit_thread( void ) -{ - do_PyThread_exit_thread(0); -} - -void PyThread__exit_thread( void ) -{ - do_PyThread_exit_thread(1); -} - -#ifndef NO_EXIT_PROG -static void do_PyThread_exit_prog( int status, int no_cleanup ) -{ - dprintf(("PyThread_exit_prog(%d) called\n", status)); - - /* No need to do anything, the threads get torn down if main() exits. */ - - if (no_cleanup) { - _exit(status); - } else { - exit(status); - } -} - -void PyThread_exit_prog( int status ) -{ - do_PyThread_exit_prog(status, 0); -} - -void PyThread__exit_prog( int status ) -{ - do_PyThread_exit_prog(status, 1); -} -#endif /* NO_EXIT_PROG */ - /* ---------------------------------------------------------------------- * Lock support. */ diff --git a/Python/thread_cthread.h b/Python/thread_cthread.h index 8cb504459f..1b3e3904cd 100644 --- a/Python/thread_cthread.h +++ b/Python/thread_cthread.h @@ -50,58 +50,14 @@ PyThread_get_thread_ident(void) return (long) cthread_self(); } -static void -do_PyThread_exit_thread(int no_cleanup) -{ - dprintf(("PyThread_exit_thread called\n")); - if (!initialized) - if (no_cleanup) - _exit(0); - else - exit(0); - cthread_exit(0); -} - void PyThread_exit_thread(void) { - do_PyThread_exit_thread(0); -} - -void -PyThread__exit_thread(void) -{ - do_PyThread_exit_thread(1); -} - -#ifndef NO_EXIT_PROG -static -void do_PyThread_exit_prog(int status, int no_cleanup) -{ - dprintf(("PyThread_exit_prog(%d) called\n", status)); + dprintf(("PyThread_exit_thread called\n")); if (!initialized) - if (no_cleanup) - _exit(status); - else - exit(status); - if (no_cleanup) - _exit(status); - else - exit(status); -} - -void -PyThread_exit_prog(int status) -{ - do_PyThread_exit_prog(status, 0); -} - -void -PyThread__exit_prog(int status) -{ - do_PyThread_exit_prog(status, 1); + exit(0); + cthread_exit(0); } -#endif /* NO_EXIT_PROG */ /* * Lock support. diff --git a/Python/thread_foobar.h b/Python/thread_foobar.h index c2dffa66c1..d2b78c5cae 100644 --- a/Python/thread_foobar.h +++ b/Python/thread_foobar.h @@ -29,53 +29,13 @@ PyThread_get_thread_ident(void) PyThread_init_thread(); } -static -void do_PyThread_exit_thread(int no_cleanup) -{ - dprintf(("PyThread_exit_thread called\n")); - if (!initialized) - if (no_cleanup) - _exit(0); - else - exit(0); -} - void PyThread_exit_thread(void) { - do_PyThread_exit_thread(0); -} - -void -PyThread__exit_thread(void) -{ - do_PyThread_exit_thread(1); -} - -#ifndef NO_EXIT_PROG -static -void do_PyThread_exit_prog(int status, int no_cleanup) -{ - dprintf(("PyThread_exit_prog(%d) called\n", status)); + dprintf(("PyThread_exit_thread called\n")); if (!initialized) - if (no_cleanup) - _exit(status); - else - exit(status); -} - -void -PyThread_exit_prog(int status) -{ - do_PyThread_exit_prog(status, 0); -} - -void -PyThread__exit_prog(int status) -{ - do_PyThread_exit_prog(status, 1); + exit(0); } -#endif /* NO_EXIT_PROG */ /* * Lock support. diff --git a/Python/thread_lwp.h b/Python/thread_lwp.h index 7519cd454e..ba7b37ad7e 100644 --- a/Python/thread_lwp.h +++ b/Python/thread_lwp.h @@ -47,50 +47,14 @@ long PyThread_get_thread_ident(void) return tid.thread_id; } -static void do_PyThread_exit_thread(int no_cleanup) +void PyThread_exit_thread(void) { dprintf(("PyThread_exit_thread called\n")); if (!initialized) - if (no_cleanup) - _exit(0); - else - exit(0); + exit(0); lwp_destroy(SELF); } -void PyThread_exit_thread(void) -{ - do_PyThread_exit_thread(0); -} - -void PyThread__exit_thread(void) -{ - do_PyThread_exit_thread(1); -} - -#ifndef NO_EXIT_PROG -static void do_PyThread_exit_prog(int status, int no_cleanup) -{ - dprintf(("PyThread_exit_prog(%d) called\n", status)); - if (!initialized) - if (no_cleanup) - _exit(status); - else - exit(status); - pod_exit(status); -} - -void PyThread_exit_prog(int status) -{ - do_PyThread_exit_prog(status, 0); -} - -void PyThread__exit_prog(int status) -{ - do_PyThread_exit_prog(status, 1); -} -#endif /* NO_EXIT_PROG */ - /* * Lock support. */ diff --git a/Python/thread_nt.h b/Python/thread_nt.h index 7512c73062..9796a34141 100644 --- a/Python/thread_nt.h +++ b/Python/thread_nt.h @@ -104,20 +104,21 @@ PyThread__init_thread(void) typedef struct { void (*func)(void*); void *arg; - long id; - HANDLE done; } callobj; -static int +/* thunker to call adapt between the function type used by the system's +thread start function and the internally used one. */ +#if defined(MS_WINCE) +static DWORD WINAPI +#else +static unsigned __stdcall +#endif bootstrap(void *call) { callobj *obj = (callobj*)call; - /* copy callobj since other thread might free it before we're done */ void (*func)(void*) = obj->func; void *arg = obj->arg; - - obj->id = PyThread_get_thread_ident(); - ReleaseSemaphore(obj->done, 1, NULL); + HeapFree(GetProcessHeap(), 0, obj); func(arg); return 0; } @@ -125,42 +126,55 @@ bootstrap(void *call) long PyThread_start_new_thread(void (*func)(void *), void *arg) { - Py_uintptr_t rv; - callobj obj; + HANDLE hThread; + unsigned threadID; + callobj *obj; dprintf(("%ld: PyThread_start_new_thread called\n", PyThread_get_thread_ident())); if (!initialized) PyThread_init_thread(); - obj.id = -1; /* guilty until proved innocent */ - obj.func = func; - obj.arg = arg; - obj.done = CreateSemaphore(NULL, 0, 1, NULL); - if (obj.done == NULL) + obj = (callobj*)HeapAlloc(GetProcessHeap(), 0, sizeof(*obj)); + if (!obj) return -1; - - rv = _beginthread(bootstrap, + obj->func = func; + obj->arg = arg; +#if defined(MS_WINCE) + hThread = CreateThread(NULL, + Py_SAFE_DOWNCAST(_pythread_stacksize, Py_ssize_t, SIZE_T), + bootstrap, obj, 0, &threadID); +#else + hThread = (HANDLE)_beginthreadex(0, Py_SAFE_DOWNCAST(_pythread_stacksize, - Py_ssize_t, int), - &obj); - if (rv == (Py_uintptr_t)-1) { + Py_ssize_t, unsigned int), + bootstrap, obj, + 0, &threadID); +#endif + if (hThread == 0) { +#if defined(MS_WINCE) + /* Save error in variable, to prevent PyThread_get_thread_ident + from clobbering it. */ + unsigned e = GetLastError(); + dprintf(("%ld: PyThread_start_new_thread failed, win32 error code %u\n", + PyThread_get_thread_ident(), e)); +#else /* I've seen errno == EAGAIN here, which means "there are * too many threads". */ - dprintf(("%ld: PyThread_start_new_thread failed: %p errno %d\n", - PyThread_get_thread_ident(), (void*)rv, errno)); - obj.id = -1; + int e = errno; + dprintf(("%ld: PyThread_start_new_thread failed, errno %d\n", + PyThread_get_thread_ident(), e)); +#endif + threadID = (unsigned)-1; + HeapFree(GetProcessHeap(), 0, obj); } else { dprintf(("%ld: PyThread_start_new_thread succeeded: %p\n", - PyThread_get_thread_ident(), (void*)rv)); - /* wait for thread to initialize, so we can get its id */ - WaitForSingleObject(obj.done, INFINITE); - assert(obj.id != -1); + PyThread_get_thread_ident(), (void*)hThread)); + CloseHandle(hThread); } - CloseHandle((HANDLE)obj.done); - return obj.id; + return (long) threadID; } /* @@ -176,54 +190,18 @@ PyThread_get_thread_ident(void) return GetCurrentThreadId(); } -static void -do_PyThread_exit_thread(int no_cleanup) -{ - dprintf(("%ld: PyThread_exit_thread called\n", PyThread_get_thread_ident())); - if (!initialized) - if (no_cleanup) - _exit(0); - else - exit(0); - _endthread(); -} - void PyThread_exit_thread(void) { - do_PyThread_exit_thread(0); -} - -void -PyThread__exit_thread(void) -{ - do_PyThread_exit_thread(1); -} - -#ifndef NO_EXIT_PROG -static void -do_PyThread_exit_prog(int status, int no_cleanup) -{ - dprintf(("PyThread_exit_prog(%d) called\n", status)); + dprintf(("%ld: PyThread_exit_thread called\n", PyThread_get_thread_ident())); if (!initialized) - if (no_cleanup) - _exit(status); - else - exit(status); -} - -void -PyThread_exit_prog(int status) -{ - do_PyThread_exit_prog(status, 0); -} - -void -PyThread__exit_prog(int status) -{ - do_PyThread_exit_prog(status, 1); + exit(0); +#if defined(MS_WINCE) + ExitThread(0); +#else + _endthreadex(0); +#endif } -#endif /* NO_EXIT_PROG */ /* * Lock support. It has too be implemented as semaphores. @@ -309,3 +287,73 @@ _pythread_nt_set_stacksize(size_t size) } #define THREAD_SET_STACKSIZE(x) _pythread_nt_set_stacksize(x) + + +/* use native Windows TLS functions */ +#define Py_HAVE_NATIVE_TLS + +#ifdef Py_HAVE_NATIVE_TLS +int +PyThread_create_key(void) +{ + return (int) TlsAlloc(); +} + +void +PyThread_delete_key(int key) +{ + TlsFree(key); +} + +/* We must be careful to emulate the strange semantics implemented in thread.c, + * where the value is only set if it hasn't been set before. + */ +int +PyThread_set_key_value(int key, void *value) +{ + BOOL ok; + void *oldvalue; + + assert(value != NULL); + oldvalue = TlsGetValue(key); + if (oldvalue != NULL) + /* ignore value if already set */ + return 0; + ok = TlsSetValue(key, value); + if (!ok) + return -1; + return 0; +} + +void * +PyThread_get_key_value(int key) +{ + /* because TLS is used in the Py_END_ALLOW_THREAD macro, + * it is necessary to preserve the windows error state, because + * it is assumed to be preserved across the call to the macro. + * Ideally, the macro should be fixed, but it is simpler to + * do it here. + */ + DWORD error = GetLastError(); + void *result = TlsGetValue(key); + SetLastError(error); + return result; +} + +void +PyThread_delete_key_value(int key) +{ + /* NULL is used as "key missing", and it is also the default + * given by TlsGetValue() if nothing has been set yet. + */ + TlsSetValue(key, NULL); +} + +/* reinitialization of TLS is not necessary after fork when using + * the native TLS functions. And forking isn't supported on Windows either. + */ +void +PyThread_ReInitTLS(void) +{} + +#endif diff --git a/Python/thread_os2.h b/Python/thread_os2.h index 28284bbc06..1b264b5ad5 100644 --- a/Python/thread_os2.h +++ b/Python/thread_os2.h @@ -68,56 +68,16 @@ PyThread_get_thread_ident(void) #endif } -static void -do_PyThread_exit_thread(int no_cleanup) +void +PyThread_exit_thread(void) { dprintf(("%ld: PyThread_exit_thread called\n", PyThread_get_thread_ident())); if (!initialized) - if (no_cleanup) - _exit(0); - else - exit(0); + exit(0); _endthread(); } -void -PyThread_exit_thread(void) -{ - do_PyThread_exit_thread(0); -} - -void -PyThread__exit_thread(void) -{ - do_PyThread_exit_thread(1); -} - -#ifndef NO_EXIT_PROG -static void -do_PyThread_exit_prog(int status, int no_cleanup) -{ - dprintf(("PyThread_exit_prog(%d) called\n", status)); - if (!initialized) - if (no_cleanup) - _exit(status); - else - exit(status); -} - -void -PyThread_exit_prog(int status) -{ - do_PyThread_exit_prog(status, 0); -} - -void -PyThread__exit_prog(int status) -{ - do_PyThread_exit_prog(status, 1); -} -#endif /* NO_EXIT_PROG */ - /* * Lock support. This is implemented with an event semaphore and critical * sections to make it behave more like a posix mutex than its OS/2 diff --git a/Python/thread_pth.h b/Python/thread_pth.h index 1f1694a4d4..82a00e72ba 100644 --- a/Python/thread_pth.h +++ b/Python/thread_pth.h @@ -74,49 +74,14 @@ long PyThread_get_thread_ident(void) return (long) *(long *) &threadid; } -static void do_PyThread_exit_thread(int no_cleanup) +void PyThread_exit_thread(void) { dprintf(("PyThread_exit_thread called\n")); if (!initialized) { - if (no_cleanup) - _exit(0); - else - exit(0); + exit(0); } } -void PyThread_exit_thread(void) -{ - do_PyThread_exit_thread(0); -} - -void PyThread__exit_thread(void) -{ - do_PyThread_exit_thread(1); -} - -#ifndef NO_EXIT_PROG -static void do_PyThread_exit_prog(int status, int no_cleanup) -{ - dprintf(("PyThread_exit_prog(%d) called\n", status)); - if (!initialized) - if (no_cleanup) - _exit(status); - else - exit(status); -} - -void PyThread_exit_prog(int status) -{ - do_PyThread_exit_prog(status, 0); -} - -void PyThread__exit_prog(int status) -{ - do_PyThread_exit_prog(status, 1); -} -#endif /* NO_EXIT_PROG */ - /* * Lock support. */ diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index 32fd2d0ef7..44e2552a8a 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -18,6 +18,18 @@ #ifndef THREAD_STACK_SIZE #define THREAD_STACK_SIZE 0 /* use default stack size */ #endif + +#if (defined(__APPLE__) || defined(__FreeBSD__)) && defined(THREAD_STACK_SIZE) && THREAD_STACK_SIZE == 0 + /* The default stack size for new threads on OSX is small enough that + * we'll get hard crashes instead of 'maximum recursion depth exceeded' + * exceptions. + * + * The default stack size below is the minimal stack size where a + * simple recursive function doesn't cause a hard crash. + */ +#undef THREAD_STACK_SIZE +#define THREAD_STACK_SIZE 0x400000 +#endif /* for safety, ensure a viable minimum stacksize */ #define THREAD_STACK_MIN 0x8000 /* 32kB */ #else /* !_POSIX_THREAD_ATTR_STACKSIZE */ @@ -225,55 +237,15 @@ PyThread_get_thread_ident(void) #endif } -static void -do_PyThread_exit_thread(int no_cleanup) +void +PyThread_exit_thread(void) { dprintf(("PyThread_exit_thread called\n")); if (!initialized) { - if (no_cleanup) - _exit(0); - else - exit(0); + exit(0); } } -void -PyThread_exit_thread(void) -{ - do_PyThread_exit_thread(0); -} - -void -PyThread__exit_thread(void) -{ - do_PyThread_exit_thread(1); -} - -#ifndef NO_EXIT_PROG -static void -do_PyThread_exit_prog(int status, int no_cleanup) -{ - dprintf(("PyThread_exit_prog(%d) called\n", status)); - if (!initialized) - if (no_cleanup) - _exit(status); - else - exit(status); -} - -void -PyThread_exit_prog(int status) -{ - do_PyThread_exit_prog(status, 0); -} - -void -PyThread__exit_prog(int status) -{ - do_PyThread_exit_prog(status, 1); -} -#endif /* NO_EXIT_PROG */ - #ifdef USE_SEMAPHORES /* diff --git a/Python/thread_sgi.h b/Python/thread_sgi.h index 679d113583..771ab2cc60 100644 --- a/Python/thread_sgi.h +++ b/Python/thread_sgi.h @@ -17,9 +17,6 @@ static ulock_t wait_lock; /* lock used to wait for other threads */ static int waiting_for_threads; /* protected by count_lock */ static int nthreads; /* protected by count_lock */ static int exit_status; -#ifndef NO_EXIT_PROG -static int do_exit; /* indicates that the program is to exit */ -#endif static int exiting; /* we're already exiting (for maybe_exit) */ static pid_t my_pid; /* PID of main thread */ static struct pidlist { @@ -27,53 +24,11 @@ static struct pidlist { pid_t child; } pidlist[MAXPROC]; /* PIDs of other threads; protected by count_lock */ static int maxpidindex; /* # of PIDs in pidlist */ - -#ifndef NO_EXIT_PROG -/* - * This routine is called as a signal handler when another thread - * exits. When that happens, we must see whether we have to exit as - * well (because of an PyThread_exit_prog()) or whether we should continue on. - */ -static void exit_sig(void) -{ - d2printf(("exit_sig called\n")); - if (exiting && getpid() == my_pid) { - d2printf(("already exiting\n")); - return; - } - if (do_exit) { - d2printf(("exiting in exit_sig\n")); -#ifdef Py_DEBUG - if ((thread_debug & 8) == 0) - thread_debug &= ~1; /* don't produce debug messages */ -#endif - PyThread_exit_thread(); - } -} - -/* - * This routine is called when a process calls exit(). If that wasn't - * done from the library, we do as if an PyThread_exit_prog() was intended. - */ -static void maybe_exit(void) -{ - dprintf(("maybe_exit called\n")); - if (exiting) { - dprintf(("already exiting\n")); - return; - } - PyThread_exit_prog(0); -} -#endif /* NO_EXIT_PROG */ - /* * Initialization. */ static void PyThread__init_thread(void) { -#ifndef NO_EXIT_PROG - struct sigaction s; -#endif /* NO_EXIT_PROG */ #ifdef USE_DL long addr, size; #endif /* USE_DL */ @@ -93,16 +48,6 @@ static void PyThread__init_thread(void) if (usconfig(CONF_INITUSERS, 16) < 0) perror("usconfig - CONF_INITUSERS"); my_pid = getpid(); /* so that we know which is the main thread */ -#ifndef NO_EXIT_PROG - atexit(maybe_exit); - s.sa_handler = exit_sig; - sigemptyset(&s.sa_mask); - /*sigaddset(&s.sa_mask, SIGUSR1);*/ - s.sa_flags = 0; - sigaction(SIGUSR1, &s, 0); - if (prctl(PR_SETEXITSIG, SIGUSR1) < 0) - perror("prctl - PR_SETEXITSIG"); -#endif /* NO_EXIT_PROG */ if (usconfig(CONF_ARENATYPE, US_SHAREDONLY) < 0) perror("usconfig - CONF_ARENATYPE"); usconfig(CONF_LOCKTYPE, US_DEBUG); /* XXX */ @@ -227,46 +172,24 @@ long PyThread_get_thread_ident(void) return getpid(); } -static void do_PyThread_exit_thread(int no_cleanup) +void PyThread_exit_thread(void) { dprintf(("PyThread_exit_thread called\n")); if (!initialized) - if (no_cleanup) - _exit(0); - else - exit(0); + exit(0); if (ussetlock(count_lock) < 0) perror("ussetlock (count_lock)"); nthreads--; if (getpid() == my_pid) { /* main thread; wait for other threads to exit */ exiting = 1; -#ifndef NO_EXIT_PROG - if (do_exit) { - int i; - - /* notify other threads */ - clean_threads(); - if (nthreads >= 0) { - dprintf(("kill other threads\n")); - for (i = 0; i < maxpidindex; i++) - if (pidlist[i].child > 0) - (void) kill(pidlist[i].child, - SIGKILL); - _exit(exit_status); - } - } -#endif /* NO_EXIT_PROG */ waiting_for_threads = 1; if (ussetlock(wait_lock) < 0) perror("ussetlock (wait_lock)"); for (;;) { if (nthreads < 0) { dprintf(("really exit (%d)\n", exit_status)); - if (no_cleanup) - _exit(exit_status); - else - exit(exit_status); + exit(exit_status); } if (usunsetlock(count_lock) < 0) perror("usunsetlock (count_lock)"); @@ -283,50 +206,11 @@ static void do_PyThread_exit_thread(int no_cleanup) if (usunsetlock(wait_lock) < 0) perror("usunsetlock (wait_lock)"); } -#ifndef NO_EXIT_PROG - else if (do_exit) - (void) kill(my_pid, SIGUSR1); -#endif /* NO_EXIT_PROG */ if (usunsetlock(count_lock) < 0) perror("usunsetlock (count_lock)"); _exit(0); } -void PyThread_exit_thread(void) -{ - do_PyThread_exit_thread(0); -} - -void PyThread__exit_thread(void) -{ - do_PyThread_exit_thread(1); -} - -#ifndef NO_EXIT_PROG -static void do_PyThread_exit_prog(int status, int no_cleanup) -{ - dprintf(("PyThread_exit_prog(%d) called\n", status)); - if (!initialized) - if (no_cleanup) - _exit(status); - else - exit(status); - do_exit = 1; - exit_status = status; - do_PyThread_exit_thread(no_cleanup); -} - -void PyThread_exit_prog(int status) -{ - do_PyThread_exit_prog(status, 0); -} - -void PyThread__exit_prog(int status) -{ - do_PyThread_exit_prog(status, 1); -} -#endif /* NO_EXIT_PROG */ - /* * Lock support. */ diff --git a/Python/thread_solaris.h b/Python/thread_solaris.h index 56ac8ae40f..1ce1cfcba9 100644 --- a/Python/thread_solaris.h +++ b/Python/thread_solaris.h @@ -64,58 +64,14 @@ PyThread_get_thread_ident(void) return thr_self(); } -static void -do_PyThread_exit_thread(int no_cleanup) -{ - dprintf(("PyThread_exit_thread called\n")); - if (!initialized) - if (no_cleanup) - _exit(0); - else - exit(0); - thr_exit(0); -} - void PyThread_exit_thread(void) { - do_PyThread_exit_thread(0); -} - -void -PyThread__exit_thread(void) -{ - do_PyThread_exit_thread(1); -} - -#ifndef NO_EXIT_PROG -static void -do_PyThread_exit_prog(int status, int no_cleanup) -{ - dprintf(("PyThread_exit_prog(%d) called\n", status)); + dprintf(("PyThread_exit_thread called\n")); if (!initialized) - if (no_cleanup) - _exit(status); - else - exit(status); - if (no_cleanup) - _exit(status); - else - exit(status); -} - -void -PyThread_exit_prog(int status) -{ - do_PyThread_exit_prog(status, 0); -} - -void -PyThread__exit_prog(int status) -{ - do_PyThread_exit_prog(status, 1); + exit(0); + thr_exit(0); } -#endif /* NO_EXIT_PROG */ /* * Lock support. diff --git a/Python/thread_wince.h b/Python/thread_wince.h index 33cf5a4fad..51ddc02f9a 100644 --- a/Python/thread_wince.h +++ b/Python/thread_wince.h @@ -53,48 +53,13 @@ long PyThread_get_thread_ident(void) return GetCurrentThreadId(); } -static void do_PyThread_exit_thread(int no_cleanup) -{ - dprintf(("%ld: do_PyThread_exit_thread called\n", PyThread_get_thread_ident())); - if (!initialized) - if (no_cleanup) - exit(0); /* XXX - was _exit()!! */ - else - exit(0); - _endthread(); -} - void PyThread_exit_thread(void) { - do_PyThread_exit_thread(0); -} - -void PyThread__exit_thread(void) -{ - do_PyThread_exit_thread(1); -} - -#ifndef NO_EXIT_PROG -static void do_PyThread_exit_prog(int status, int no_cleanup) -{ - dprintf(("PyThread_exit_prog(%d) called\n", status)); + dprintf(("%ld: PyThread_exit_thread called\n", PyThread_get_thread_ident())); if (!initialized) - if (no_cleanup) - _exit(status); - else - exit(status); -} - -void PyThread_exit_prog(int status) -{ - do_PyThread_exit_prog(status, 0); -} - -void PyThread__exit_prog(int status) -{ - do_PyThread_exit_prog(status, 1); + exit(0); + _endthread(); } -#endif /* NO_EXIT_PROG */ /* * Lock support. It has to be implemented using Mutexes, as diff --git a/Python/traceback.c b/Python/traceback.c index a60a183d19..adfd66c0de 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -96,8 +96,7 @@ newtracebackobject(PyTracebackObject *next, PyFrameObject *frame) Py_XINCREF(frame); tb->tb_frame = frame; tb->tb_lasti = frame->f_lasti; - tb->tb_lineno = PyCode_Addr2Line(frame->f_code, - frame->f_lasti); + tb->tb_lineno = PyFrame_GetLineNumber(frame); PyObject_GC_Track(tb); } return tb; @@ -162,7 +161,6 @@ _Py_DisplaySourceLine(PyObject *f, const char *filename, int lineno, int indent) strcpy(namebuf+len, tail); xfp = fopen(namebuf, "r" PY_STDIOTEXTMODE); if (xfp != NULL) { - filename = namebuf; break; } } |