diff options
Diffstat (limited to 'Python')
55 files changed, 11734 insertions, 10631 deletions
diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 6cf99ec5ad..a276b6cf21 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -2,7 +2,7 @@ /* - __version__ 82160. + __version__ 82163. This module must be committed separately after each AST grammar change; The __version__ number is set to the revision number of the commit @@ -43,11 +43,15 @@ static char *FunctionDef_fields[]={ "args", "body", "decorator_list", + "returns", }; static PyTypeObject *ClassDef_type; static char *ClassDef_fields[]={ "name", "bases", + "keywords", + "starargs", + "kwargs", "body", "decorator_list", }; @@ -70,12 +74,6 @@ static char *AugAssign_fields[]={ "op", "value", }; -static PyTypeObject *Print_type; -static char *Print_fields[]={ - "dest", - "values", - "nl", -}; static PyTypeObject *For_type; static char *For_fields[]={ "target", @@ -103,9 +101,8 @@ static char *With_fields[]={ }; static PyTypeObject *Raise_type; static char *Raise_fields[]={ - "type", - "inst", - "tback", + "exc", + "cause", }; static PyTypeObject *TryExcept_type; static char *TryExcept_fields[]={ @@ -133,16 +130,14 @@ static char *ImportFrom_fields[]={ "names", "level", }; -static PyTypeObject *Exec_type; -static char *Exec_fields[]={ - "body", - "globals", - "locals", -}; static PyTypeObject *Global_type; static char *Global_fields[]={ "names", }; +static PyTypeObject *Nonlocal_type; +static char *Nonlocal_fields[]={ + "names", +}; static PyTypeObject *Expr_type; static char *Expr_fields[]={ "value", @@ -231,10 +226,6 @@ static char *Call_fields[]={ "starargs", "kwargs", }; -static PyTypeObject *Repr_type; -static char *Repr_fields[]={ - "value", -}; static PyTypeObject *Num_type; static char *Num_fields[]={ "n", @@ -243,6 +234,11 @@ static PyTypeObject *Str_type; static char *Str_fields[]={ "s", }; +static PyTypeObject *Bytes_type; +static char *Bytes_fields[]={ + "s", +}; +static PyTypeObject *Ellipsis_type; static PyTypeObject *Attribute_type; static char *Attribute_fields[]={ "value", @@ -255,6 +251,11 @@ static char *Subscript_fields[]={ "slice", "ctx", }; +static PyTypeObject *Starred_type; +static char *Starred_fields[]={ + "value", + "ctx", +}; static PyTypeObject *Name_type; static char *Name_fields[]={ "id", @@ -282,7 +283,6 @@ static PyTypeObject *AugStore_type; static PyTypeObject *Param_type; static PyTypeObject *slice_type; static PyObject* ast2obj_slice(void*); -static PyTypeObject *Ellipsis_type; static PyTypeObject *Slice_type; static char *Slice_fields[]={ "lower", @@ -367,8 +367,18 @@ static PyObject* ast2obj_arguments(void*); static char *arguments_fields[]={ "args", "vararg", + "varargannotation", + "kwonlyargs", "kwarg", + "kwargannotation", "defaults", + "kw_defaults", +}; +static PyTypeObject *arg_type; +static PyObject* ast2obj_arg(void*); +static char *arg_fields[]={ + "arg", + "annotation", }; static PyTypeObject *keyword_type; static PyObject* ast2obj_keyword(void*); @@ -469,7 +479,7 @@ static PyTypeObject AST_type = { 0, /* tp_print */ 0, /* tp_getattr */ 0, /* tp_setattr */ - 0, /* tp_compare */ + 0, /* tp_reserved */ 0, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ @@ -510,7 +520,7 @@ static PyTypeObject* make_type(char *type, PyTypeObject* base, char**fields, int fnames = PyTuple_New(num_fields); if (!fnames) return NULL; for (i = 0; i < num_fields; i++) { - PyObject *field = PyString_FromString(fields[i]); + PyObject *field = PyUnicode_FromString(fields[i]); if (!field) { Py_DECREF(fnames); return NULL; @@ -530,7 +540,7 @@ static int add_attributes(PyTypeObject* type, char**attrs, int num_fields) if (!l) return 0; for (i = 0; i < num_fields; i++) { - s = PyString_FromString(attrs[i]); + s = PyUnicode_FromString(attrs[i]); if (!s) { Py_DECREF(l); return 0; @@ -571,14 +581,10 @@ static PyObject* ast2obj_object(void *o) } #define ast2obj_identifier ast2obj_object #define ast2obj_string ast2obj_object -static PyObject* ast2obj_bool(bool b) -{ - return PyBool_FromLong(b); -} static PyObject* ast2obj_int(long b) { - return PyInt_FromLong(b); + return PyLong_FromLong(b); } /* Conversion Python -> AST */ @@ -596,9 +602,8 @@ static int obj2ast_object(PyObject* obj, PyObject** out, PyArena* arena) static int obj2ast_identifier(PyObject* obj, PyObject** out, PyArena* arena) { - if (!PyString_CheckExact(obj) && obj != Py_None) { - PyErr_Format(PyExc_TypeError, - "AST identifier must be of type str"); + if (!PyUnicode_CheckExact(obj) && obj != Py_None) { + PyErr_SetString(PyExc_TypeError, "AST identifier must be of type str"); return 1; } return obj2ast_object(obj, out, arena); @@ -606,9 +611,8 @@ static int obj2ast_identifier(PyObject* obj, PyObject** out, PyArena* arena) static int obj2ast_string(PyObject* obj, PyObject** out, PyArena* arena) { - if (!PyString_CheckExact(obj) && !PyUnicode_CheckExact(obj)) { - PyErr_SetString(PyExc_TypeError, - "AST string must be of type str or unicode"); + if (!PyUnicode_CheckExact(obj) && !PyBytes_CheckExact(obj)) { + PyErr_SetString(PyExc_TypeError, "AST string must be of type str"); return 1; } return obj2ast_object(obj, out, arena); @@ -617,12 +621,8 @@ static int obj2ast_string(PyObject* obj, PyObject** out, PyArena* arena) static int obj2ast_int(PyObject* obj, int* out, PyArena* arena) { int i; - if (!PyInt_Check(obj) && !PyLong_Check(obj)) { - PyObject *s = PyObject_Repr(obj); - if (s == NULL) return 1; - PyErr_Format(PyExc_ValueError, "invalid integer value: %.400s", - PyString_AS_STRING(s)); - Py_DECREF(s); + if (!PyLong_Check(obj)) { + PyErr_Format(PyExc_ValueError, "invalid integer value: %R", obj); return 1; } @@ -633,21 +633,6 @@ static int obj2ast_int(PyObject* obj, int* out, PyArena* arena) return 0; } -static int obj2ast_bool(PyObject* obj, bool* out, PyArena* arena) -{ - if (!PyBool_Check(obj)) { - PyObject *s = PyObject_Repr(obj); - if (s == NULL) return 1; - PyErr_Format(PyExc_ValueError, "invalid boolean value: %.400s", - PyString_AS_STRING(s)); - Py_DECREF(s); - return 1; - } - - *out = (obj == Py_True); - return 0; -} - static int add_ast_fields(void) { PyObject *empty_tuple, *d; @@ -688,9 +673,9 @@ static int init_types(void) if (!stmt_type) return 0; if (!add_attributes(stmt_type, stmt_attributes, 2)) return 0; FunctionDef_type = make_type("FunctionDef", stmt_type, - FunctionDef_fields, 4); + FunctionDef_fields, 5); if (!FunctionDef_type) return 0; - ClassDef_type = make_type("ClassDef", stmt_type, ClassDef_fields, 4); + ClassDef_type = make_type("ClassDef", stmt_type, ClassDef_fields, 7); if (!ClassDef_type) return 0; Return_type = make_type("Return", stmt_type, Return_fields, 1); if (!Return_type) return 0; @@ -700,8 +685,6 @@ static int init_types(void) if (!Assign_type) return 0; AugAssign_type = make_type("AugAssign", stmt_type, AugAssign_fields, 3); if (!AugAssign_type) return 0; - Print_type = make_type("Print", stmt_type, Print_fields, 3); - if (!Print_type) return 0; For_type = make_type("For", stmt_type, For_fields, 4); if (!For_type) return 0; While_type = make_type("While", stmt_type, While_fields, 3); @@ -710,7 +693,7 @@ static int init_types(void) if (!If_type) return 0; With_type = make_type("With", stmt_type, With_fields, 3); if (!With_type) return 0; - Raise_type = make_type("Raise", stmt_type, Raise_fields, 3); + Raise_type = make_type("Raise", stmt_type, Raise_fields, 2); if (!Raise_type) return 0; TryExcept_type = make_type("TryExcept", stmt_type, TryExcept_fields, 3); if (!TryExcept_type) return 0; @@ -724,10 +707,10 @@ static int init_types(void) ImportFrom_type = make_type("ImportFrom", stmt_type, ImportFrom_fields, 3); if (!ImportFrom_type) return 0; - Exec_type = make_type("Exec", stmt_type, Exec_fields, 3); - if (!Exec_type) return 0; Global_type = make_type("Global", stmt_type, Global_fields, 1); if (!Global_type) return 0; + Nonlocal_type = make_type("Nonlocal", stmt_type, Nonlocal_fields, 1); + if (!Nonlocal_type) return 0; Expr_type = make_type("Expr", stmt_type, Expr_fields, 1); if (!Expr_type) return 0; Pass_type = make_type("Pass", stmt_type, NULL, 0); @@ -768,16 +751,20 @@ static int init_types(void) if (!Compare_type) return 0; Call_type = make_type("Call", expr_type, Call_fields, 5); if (!Call_type) return 0; - Repr_type = make_type("Repr", expr_type, Repr_fields, 1); - if (!Repr_type) return 0; Num_type = make_type("Num", expr_type, Num_fields, 1); if (!Num_type) return 0; Str_type = make_type("Str", expr_type, Str_fields, 1); if (!Str_type) return 0; + Bytes_type = make_type("Bytes", expr_type, Bytes_fields, 1); + if (!Bytes_type) return 0; + Ellipsis_type = make_type("Ellipsis", expr_type, NULL, 0); + if (!Ellipsis_type) return 0; Attribute_type = make_type("Attribute", expr_type, Attribute_fields, 3); if (!Attribute_type) return 0; Subscript_type = make_type("Subscript", expr_type, Subscript_fields, 3); if (!Subscript_type) return 0; + Starred_type = make_type("Starred", expr_type, Starred_fields, 2); + if (!Starred_type) return 0; Name_type = make_type("Name", expr_type, Name_fields, 2); if (!Name_type) return 0; List_type = make_type("List", expr_type, List_fields, 2); @@ -814,8 +801,6 @@ static int init_types(void) slice_type = make_type("slice", &AST_type, NULL, 0); if (!slice_type) return 0; if (!add_attributes(slice_type, NULL, 0)) return 0; - Ellipsis_type = make_type("Ellipsis", slice_type, NULL, 0); - if (!Ellipsis_type) return 0; Slice_type = make_type("Slice", slice_type, Slice_fields, 3); if (!Slice_type) return 0; ExtSlice_type = make_type("ExtSlice", slice_type, ExtSlice_fields, 1); @@ -956,8 +941,10 @@ static int init_types(void) ExceptHandler_type = make_type("ExceptHandler", excepthandler_type, ExceptHandler_fields, 3); if (!ExceptHandler_type) return 0; - arguments_type = make_type("arguments", &AST_type, arguments_fields, 4); + arguments_type = make_type("arguments", &AST_type, arguments_fields, 8); if (!arguments_type) return 0; + arg_type = make_type("arg", &AST_type, arg_fields, 2); + if (!arg_type) return 0; keyword_type = make_type("keyword", &AST_type, keyword_fields, 2); if (!keyword_type) return 0; alias_type = make_type("alias", &AST_type, alias_fields, 2); @@ -981,6 +968,7 @@ static int obj2ast_comprehension(PyObject* obj, comprehension_ty* out, PyArena* static int obj2ast_excepthandler(PyObject* obj, excepthandler_ty* out, PyArena* arena); static int obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena); +static int obj2ast_arg(PyObject* obj, arg_ty* out, PyArena* arena); static int obj2ast_keyword(PyObject* obj, keyword_ty* out, PyArena* arena); static int obj2ast_alias(PyObject* obj, alias_ty* out, PyArena* arena); @@ -1039,7 +1027,8 @@ Suite(asdl_seq * body, PyArena *arena) stmt_ty FunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq * - decorator_list, int lineno, int col_offset, PyArena *arena) + decorator_list, expr_ty returns, int lineno, int col_offset, + PyArena *arena) { stmt_ty p; if (!name) { @@ -1060,14 +1049,16 @@ FunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq * p->v.FunctionDef.args = args; p->v.FunctionDef.body = body; p->v.FunctionDef.decorator_list = decorator_list; + p->v.FunctionDef.returns = returns; p->lineno = lineno; p->col_offset = col_offset; return p; } stmt_ty -ClassDef(identifier name, asdl_seq * bases, asdl_seq * body, asdl_seq * - decorator_list, int lineno, int col_offset, PyArena *arena) +ClassDef(identifier name, asdl_seq * bases, asdl_seq * keywords, expr_ty + starargs, expr_ty kwargs, asdl_seq * body, asdl_seq * decorator_list, + int lineno, int col_offset, PyArena *arena) { stmt_ty p; if (!name) { @@ -1081,6 +1072,9 @@ ClassDef(identifier name, asdl_seq * bases, asdl_seq * body, asdl_seq * p->kind = ClassDef_kind; p->v.ClassDef.name = name; p->v.ClassDef.bases = bases; + p->v.ClassDef.keywords = keywords; + p->v.ClassDef.starargs = starargs; + p->v.ClassDef.kwargs = kwargs; p->v.ClassDef.body = body; p->v.ClassDef.decorator_list = decorator_list; p->lineno = lineno; @@ -1170,23 +1164,6 @@ AugAssign(expr_ty target, operator_ty op, expr_ty value, int lineno, int } stmt_ty -Print(expr_ty dest, asdl_seq * values, bool nl, int lineno, int col_offset, - PyArena *arena) -{ - stmt_ty p; - p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p)); - if (!p) - return NULL; - p->kind = Print_kind; - p->v.Print.dest = dest; - p->v.Print.values = values; - p->v.Print.nl = nl; - p->lineno = lineno; - p->col_offset = col_offset; - return p; -} - -stmt_ty For(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse, int lineno, int col_offset, PyArena *arena) { @@ -1281,17 +1258,15 @@ With(expr_ty context_expr, expr_ty optional_vars, asdl_seq * body, int lineno, } stmt_ty -Raise(expr_ty type, expr_ty inst, expr_ty tback, int lineno, int col_offset, - PyArena *arena) +Raise(expr_ty exc, expr_ty cause, int lineno, int col_offset, PyArena *arena) { stmt_ty p; p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p)); if (!p) return NULL; p->kind = Raise_kind; - p->v.Raise.type = type; - p->v.Raise.inst = inst; - p->v.Raise.tback = tback; + p->v.Raise.exc = exc; + p->v.Raise.cause = cause; p->lineno = lineno; p->col_offset = col_offset; return p; @@ -1382,36 +1357,28 @@ ImportFrom(identifier module, asdl_seq * names, int level, int lineno, int } stmt_ty -Exec(expr_ty body, expr_ty globals, expr_ty locals, int lineno, int col_offset, - PyArena *arena) +Global(asdl_seq * names, int lineno, int col_offset, PyArena *arena) { stmt_ty p; - if (!body) { - PyErr_SetString(PyExc_ValueError, - "field body is required for Exec"); - return NULL; - } p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p)); if (!p) return NULL; - p->kind = Exec_kind; - p->v.Exec.body = body; - p->v.Exec.globals = globals; - p->v.Exec.locals = locals; + p->kind = Global_kind; + p->v.Global.names = names; p->lineno = lineno; p->col_offset = col_offset; return p; } stmt_ty -Global(asdl_seq * names, int lineno, int col_offset, PyArena *arena) +Nonlocal(asdl_seq * names, int lineno, int col_offset, PyArena *arena) { stmt_ty p; p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p)); if (!p) return NULL; - p->kind = Global_kind; - p->v.Global.names = names; + p->kind = Nonlocal_kind; + p->v.Nonlocal.names = names; p->lineno = lineno; p->col_offset = col_offset; return p; @@ -1793,57 +1760,70 @@ Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, expr_ty starargs, } expr_ty -Repr(expr_ty value, int lineno, int col_offset, PyArena *arena) +Num(object n, int lineno, int col_offset, PyArena *arena) { expr_ty p; - if (!value) { + if (!n) { PyErr_SetString(PyExc_ValueError, - "field value is required for Repr"); + "field n is required for Num"); return NULL; } p = (expr_ty)PyArena_Malloc(arena, sizeof(*p)); if (!p) return NULL; - p->kind = Repr_kind; - p->v.Repr.value = value; + p->kind = Num_kind; + p->v.Num.n = n; p->lineno = lineno; p->col_offset = col_offset; return p; } expr_ty -Num(object n, int lineno, int col_offset, PyArena *arena) +Str(string s, int lineno, int col_offset, PyArena *arena) { expr_ty p; - if (!n) { + if (!s) { PyErr_SetString(PyExc_ValueError, - "field n is required for Num"); + "field s is required for Str"); return NULL; } p = (expr_ty)PyArena_Malloc(arena, sizeof(*p)); if (!p) return NULL; - p->kind = Num_kind; - p->v.Num.n = n; + p->kind = Str_kind; + p->v.Str.s = s; p->lineno = lineno; p->col_offset = col_offset; return p; } expr_ty -Str(string s, int lineno, int col_offset, PyArena *arena) +Bytes(string s, int lineno, int col_offset, PyArena *arena) { expr_ty p; if (!s) { PyErr_SetString(PyExc_ValueError, - "field s is required for Str"); + "field s is required for Bytes"); return NULL; } p = (expr_ty)PyArena_Malloc(arena, sizeof(*p)); if (!p) return NULL; - p->kind = Str_kind; - p->v.Str.s = s; + p->kind = Bytes_kind; + p->v.Bytes.s = s; + p->lineno = lineno; + p->col_offset = col_offset; + return p; +} + +expr_ty +Ellipsis(int lineno, int col_offset, PyArena *arena) +{ + expr_ty p; + p = (expr_ty)PyArena_Malloc(arena, sizeof(*p)); + if (!p) + return NULL; + p->kind = Ellipsis_kind; p->lineno = lineno; p->col_offset = col_offset; return p; @@ -1914,6 +1894,32 @@ Subscript(expr_ty value, slice_ty slice, expr_context_ty ctx, int lineno, int } expr_ty +Starred(expr_ty value, expr_context_ty ctx, int lineno, int col_offset, PyArena + *arena) +{ + expr_ty p; + if (!value) { + PyErr_SetString(PyExc_ValueError, + "field value is required for Starred"); + return NULL; + } + if (!ctx) { + PyErr_SetString(PyExc_ValueError, + "field ctx is required for Starred"); + return NULL; + } + p = (expr_ty)PyArena_Malloc(arena, sizeof(*p)); + if (!p) + return NULL; + p->kind = Starred_kind; + p->v.Starred.value = value; + p->v.Starred.ctx = ctx; + p->lineno = lineno; + p->col_offset = col_offset; + return p; +} + +expr_ty Name(identifier id, expr_context_ty ctx, int lineno, int col_offset, PyArena *arena) { @@ -1982,17 +1988,6 @@ Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset, PyArena } slice_ty -Ellipsis(PyArena *arena) -{ - slice_ty p; - p = (slice_ty)PyArena_Malloc(arena, sizeof(*p)); - if (!p) - return NULL; - p->kind = Ellipsis_kind; - return p; -} - -slice_ty Slice(expr_ty lower, expr_ty upper, expr_ty step, PyArena *arena) { slice_ty p; @@ -2059,7 +2054,7 @@ comprehension(expr_ty target, expr_ty iter, asdl_seq * ifs, PyArena *arena) } excepthandler_ty -ExceptHandler(expr_ty type, expr_ty name, asdl_seq * body, int lineno, int +ExceptHandler(expr_ty type, identifier name, asdl_seq * body, int lineno, int col_offset, PyArena *arena) { excepthandler_ty p; @@ -2076,8 +2071,9 @@ ExceptHandler(expr_ty type, expr_ty name, asdl_seq * body, int lineno, int } arguments_ty -arguments(asdl_seq * args, identifier vararg, identifier kwarg, asdl_seq * - defaults, PyArena *arena) +arguments(asdl_seq * args, identifier vararg, expr_ty varargannotation, + asdl_seq * kwonlyargs, identifier kwarg, expr_ty kwargannotation, + asdl_seq * defaults, asdl_seq * kw_defaults, PyArena *arena) { arguments_ty p; p = (arguments_ty)PyArena_Malloc(arena, sizeof(*p)); @@ -2085,8 +2081,29 @@ arguments(asdl_seq * args, identifier vararg, identifier kwarg, asdl_seq * return NULL; p->args = args; p->vararg = vararg; + p->varargannotation = varargannotation; + p->kwonlyargs = kwonlyargs; p->kwarg = kwarg; + p->kwargannotation = kwargannotation; p->defaults = defaults; + p->kw_defaults = kw_defaults; + return p; +} + +arg_ty +arg(identifier arg, expr_ty annotation, PyArena *arena) +{ + arg_ty p; + if (!arg) { + PyErr_SetString(PyExc_ValueError, + "field arg is required for arg"); + return NULL; + } + p = (arg_ty)PyArena_Malloc(arena, sizeof(*p)); + if (!p) + return NULL; + p->arg = arg; + p->annotation = annotation; return p; } @@ -2221,6 +2238,11 @@ ast2obj_stmt(void* _o) -1) goto failed; Py_DECREF(value); + value = ast2obj_expr(o->v.FunctionDef.returns); + if (!value) goto failed; + if (PyObject_SetAttrString(result, "returns", value) == -1) + goto failed; + Py_DECREF(value); break; case ClassDef_kind: result = PyType_GenericNew(ClassDef_type, NULL, NULL); @@ -2235,6 +2257,21 @@ ast2obj_stmt(void* _o) if (PyObject_SetAttrString(result, "bases", value) == -1) goto failed; Py_DECREF(value); + value = ast2obj_list(o->v.ClassDef.keywords, ast2obj_keyword); + if (!value) goto failed; + if (PyObject_SetAttrString(result, "keywords", value) == -1) + goto failed; + Py_DECREF(value); + value = ast2obj_expr(o->v.ClassDef.starargs); + if (!value) goto failed; + if (PyObject_SetAttrString(result, "starargs", value) == -1) + goto failed; + Py_DECREF(value); + value = ast2obj_expr(o->v.ClassDef.kwargs); + if (!value) goto failed; + if (PyObject_SetAttrString(result, "kwargs", value) == -1) + goto failed; + Py_DECREF(value); value = ast2obj_list(o->v.ClassDef.body, ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttrString(result, "body", value) == -1) @@ -2299,25 +2336,6 @@ ast2obj_stmt(void* _o) goto failed; Py_DECREF(value); break; - case Print_kind: - result = PyType_GenericNew(Print_type, NULL, NULL); - if (!result) goto failed; - value = ast2obj_expr(o->v.Print.dest); - if (!value) goto failed; - if (PyObject_SetAttrString(result, "dest", value) == -1) - goto failed; - Py_DECREF(value); - value = ast2obj_list(o->v.Print.values, ast2obj_expr); - if (!value) goto failed; - if (PyObject_SetAttrString(result, "values", value) == -1) - goto failed; - Py_DECREF(value); - value = ast2obj_bool(o->v.Print.nl); - if (!value) goto failed; - if (PyObject_SetAttrString(result, "nl", value) == -1) - goto failed; - Py_DECREF(value); - break; case For_kind: result = PyType_GenericNew(For_type, NULL, NULL); if (!result) goto failed; @@ -2403,19 +2421,14 @@ ast2obj_stmt(void* _o) case Raise_kind: result = PyType_GenericNew(Raise_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(o->v.Raise.type); + value = ast2obj_expr(o->v.Raise.exc); if (!value) goto failed; - if (PyObject_SetAttrString(result, "type", value) == -1) + if (PyObject_SetAttrString(result, "exc", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(o->v.Raise.inst); + value = ast2obj_expr(o->v.Raise.cause); if (!value) goto failed; - if (PyObject_SetAttrString(result, "inst", value) == -1) - goto failed; - Py_DECREF(value); - value = ast2obj_expr(o->v.Raise.tback); - if (!value) goto failed; - if (PyObject_SetAttrString(result, "tback", value) == -1) + if (PyObject_SetAttrString(result, "cause", value) == -1) goto failed; Py_DECREF(value); break; @@ -2495,29 +2508,19 @@ ast2obj_stmt(void* _o) goto failed; Py_DECREF(value); break; - case Exec_kind: - result = PyType_GenericNew(Exec_type, NULL, NULL); + case Global_kind: + result = PyType_GenericNew(Global_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(o->v.Exec.body); - if (!value) goto failed; - if (PyObject_SetAttrString(result, "body", value) == -1) - goto failed; - Py_DECREF(value); - value = ast2obj_expr(o->v.Exec.globals); - if (!value) goto failed; - if (PyObject_SetAttrString(result, "globals", value) == -1) - goto failed; - Py_DECREF(value); - value = ast2obj_expr(o->v.Exec.locals); + value = ast2obj_list(o->v.Global.names, ast2obj_identifier); if (!value) goto failed; - if (PyObject_SetAttrString(result, "locals", value) == -1) + if (PyObject_SetAttrString(result, "names", value) == -1) goto failed; Py_DECREF(value); break; - case Global_kind: - result = PyType_GenericNew(Global_type, NULL, NULL); + case Nonlocal_kind: + result = PyType_GenericNew(Nonlocal_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(o->v.Global.names, ast2obj_identifier); + value = ast2obj_list(o->v.Nonlocal.names, ast2obj_identifier); if (!value) goto failed; if (PyObject_SetAttrString(result, "names", value) == -1) goto failed; @@ -2804,15 +2807,6 @@ ast2obj_expr(void* _o) goto failed; Py_DECREF(value); break; - case Repr_kind: - result = PyType_GenericNew(Repr_type, NULL, NULL); - if (!result) goto failed; - value = ast2obj_expr(o->v.Repr.value); - if (!value) goto failed; - if (PyObject_SetAttrString(result, "value", value) == -1) - goto failed; - Py_DECREF(value); - break; case Num_kind: result = PyType_GenericNew(Num_type, NULL, NULL); if (!result) goto failed; @@ -2831,6 +2825,19 @@ ast2obj_expr(void* _o) goto failed; Py_DECREF(value); break; + case Bytes_kind: + result = PyType_GenericNew(Bytes_type, NULL, NULL); + if (!result) goto failed; + value = ast2obj_string(o->v.Bytes.s); + if (!value) goto failed; + if (PyObject_SetAttrString(result, "s", value) == -1) + goto failed; + Py_DECREF(value); + break; + case Ellipsis_kind: + result = PyType_GenericNew(Ellipsis_type, NULL, NULL); + if (!result) goto failed; + break; case Attribute_kind: result = PyType_GenericNew(Attribute_type, NULL, NULL); if (!result) goto failed; @@ -2869,6 +2876,20 @@ ast2obj_expr(void* _o) goto failed; Py_DECREF(value); break; + case Starred_kind: + result = PyType_GenericNew(Starred_type, NULL, NULL); + if (!result) goto failed; + value = ast2obj_expr(o->v.Starred.value); + if (!value) goto failed; + if (PyObject_SetAttrString(result, "value", value) == -1) + goto failed; + Py_DECREF(value); + value = ast2obj_expr_context(o->v.Starred.ctx); + if (!value) goto failed; + if (PyObject_SetAttrString(result, "ctx", value) == -1) + goto failed; + Py_DECREF(value); + break; case Name_kind: result = PyType_GenericNew(Name_type, NULL, NULL); if (!result) goto failed; @@ -2967,10 +2988,6 @@ ast2obj_slice(void* _o) } switch (o->kind) { - case Ellipsis_kind: - result = PyType_GenericNew(Ellipsis_type, NULL, NULL); - if (!result) goto failed; - break; case Slice_kind: result = PyType_GenericNew(Slice_type, NULL, NULL); if (!result) goto failed; @@ -3189,7 +3206,7 @@ ast2obj_excepthandler(void* _o) if (PyObject_SetAttrString(result, "type", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(o->v.ExceptHandler.name); + value = ast2obj_identifier(o->v.ExceptHandler.name); if (!value) goto failed; if (PyObject_SetAttrString(result, "name", value) == -1) goto failed; @@ -3230,7 +3247,7 @@ ast2obj_arguments(void* _o) result = PyType_GenericNew(arguments_type, NULL, NULL); if (!result) return NULL; - value = ast2obj_list(o->args, ast2obj_expr); + value = ast2obj_list(o->args, ast2obj_arg); if (!value) goto failed; if (PyObject_SetAttrString(result, "args", value) == -1) goto failed; @@ -3240,16 +3257,65 @@ ast2obj_arguments(void* _o) if (PyObject_SetAttrString(result, "vararg", value) == -1) goto failed; Py_DECREF(value); + value = ast2obj_expr(o->varargannotation); + if (!value) goto failed; + if (PyObject_SetAttrString(result, "varargannotation", value) == -1) + goto failed; + Py_DECREF(value); + value = ast2obj_list(o->kwonlyargs, ast2obj_arg); + if (!value) goto failed; + if (PyObject_SetAttrString(result, "kwonlyargs", value) == -1) + goto failed; + Py_DECREF(value); value = ast2obj_identifier(o->kwarg); if (!value) goto failed; if (PyObject_SetAttrString(result, "kwarg", value) == -1) goto failed; Py_DECREF(value); + value = ast2obj_expr(o->kwargannotation); + if (!value) goto failed; + if (PyObject_SetAttrString(result, "kwargannotation", value) == -1) + goto failed; + Py_DECREF(value); value = ast2obj_list(o->defaults, ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttrString(result, "defaults", value) == -1) goto failed; Py_DECREF(value); + value = ast2obj_list(o->kw_defaults, ast2obj_expr); + if (!value) goto failed; + if (PyObject_SetAttrString(result, "kw_defaults", value) == -1) + goto failed; + Py_DECREF(value); + return result; +failed: + Py_XDECREF(value); + Py_XDECREF(result); + return NULL; +} + +PyObject* +ast2obj_arg(void* _o) +{ + arg_ty o = (arg_ty)_o; + PyObject *result = NULL, *value = NULL; + if (!o) { + Py_INCREF(Py_None); + return Py_None; + } + + result = PyType_GenericNew(arg_type, NULL, NULL); + if (!result) return NULL; + value = ast2obj_identifier(o->arg); + if (!value) goto failed; + if (PyObject_SetAttrString(result, "arg", value) == -1) + goto failed; + Py_DECREF(value); + value = ast2obj_expr(o->annotation); + if (!value) goto failed; + if (PyObject_SetAttrString(result, "annotation", value) == -1) + goto failed; + Py_DECREF(value); return result; failed: Py_XDECREF(value); @@ -3319,9 +3385,9 @@ failed: int obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena) { - PyObject* tmp = NULL; int isinstance; + PyObject *tmp = NULL; if (obj == Py_None) { *out = NULL; @@ -3459,10 +3525,8 @@ obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena) return 0; } - tmp = PyObject_Repr(obj); - if (tmp == NULL) goto failed; - PyErr_Format(PyExc_TypeError, "expected some sort of mod, but got %.400s", PyString_AS_STRING(tmp)); -failed: + PyErr_Format(PyExc_TypeError, "expected some sort of mod, but got %R", obj); + failed: Py_XDECREF(tmp); return 1; } @@ -3470,9 +3534,9 @@ failed: int obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) { - PyObject* tmp = NULL; int isinstance; + PyObject *tmp = NULL; int lineno; int col_offset; @@ -3513,6 +3577,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) arguments_ty args; asdl_seq* body; asdl_seq* decorator_list; + expr_ty returns; if (PyObject_HasAttrString(obj, "name")) { int res; @@ -3588,8 +3653,19 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"decorator_list\" missing from FunctionDef"); return 1; } - *out = FunctionDef(name, args, body, decorator_list, lineno, - col_offset, arena); + if (PyObject_HasAttrString(obj, "returns")) { + int res; + tmp = PyObject_GetAttrString(obj, "returns"); + if (tmp == NULL) goto failed; + res = obj2ast_expr(tmp, &returns, arena); + if (res != 0) goto failed; + Py_XDECREF(tmp); + tmp = NULL; + } else { + returns = NULL; + } + *out = FunctionDef(name, args, body, decorator_list, returns, + lineno, col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -3600,6 +3676,9 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) if (isinstance) { identifier name; asdl_seq* bases; + asdl_seq* keywords; + expr_ty starargs; + expr_ty kwargs; asdl_seq* body; asdl_seq* decorator_list; @@ -3640,6 +3719,53 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"bases\" missing from ClassDef"); return 1; } + if (PyObject_HasAttrString(obj, "keywords")) { + int res; + Py_ssize_t len; + Py_ssize_t i; + tmp = PyObject_GetAttrString(obj, "keywords"); + if (tmp == NULL) goto failed; + if (!PyList_Check(tmp)) { + PyErr_Format(PyExc_TypeError, "ClassDef field \"keywords\" must be a list, not a %.200s", tmp->ob_type->tp_name); + goto failed; + } + len = PyList_GET_SIZE(tmp); + keywords = asdl_seq_new(len, arena); + if (keywords == NULL) goto failed; + for (i = 0; i < len; i++) { + keyword_ty value; + res = obj2ast_keyword(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(keywords, i, value); + } + Py_XDECREF(tmp); + tmp = NULL; + } else { + PyErr_SetString(PyExc_TypeError, "required field \"keywords\" missing from ClassDef"); + return 1; + } + if (PyObject_HasAttrString(obj, "starargs")) { + int res; + tmp = PyObject_GetAttrString(obj, "starargs"); + if (tmp == NULL) goto failed; + res = obj2ast_expr(tmp, &starargs, arena); + if (res != 0) goto failed; + Py_XDECREF(tmp); + tmp = NULL; + } else { + starargs = NULL; + } + if (PyObject_HasAttrString(obj, "kwargs")) { + int res; + tmp = PyObject_GetAttrString(obj, "kwargs"); + if (tmp == NULL) goto failed; + res = obj2ast_expr(tmp, &kwargs, arena); + if (res != 0) goto failed; + Py_XDECREF(tmp); + tmp = NULL; + } else { + kwargs = NULL; + } if (PyObject_HasAttrString(obj, "body")) { int res; Py_ssize_t len; @@ -3690,8 +3816,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"decorator_list\" missing from ClassDef"); return 1; } - *out = ClassDef(name, bases, body, decorator_list, lineno, - col_offset, arena); + *out = ClassDef(name, bases, keywords, starargs, kwargs, body, + decorator_list, lineno, col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -3851,67 +3977,6 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) if (*out == NULL) goto failed; return 0; } - isinstance = PyObject_IsInstance(obj, (PyObject*)Print_type); - if (isinstance == -1) { - return 1; - } - if (isinstance) { - expr_ty dest; - asdl_seq* values; - bool nl; - - if (PyObject_HasAttrString(obj, "dest")) { - int res; - tmp = PyObject_GetAttrString(obj, "dest"); - if (tmp == NULL) goto failed; - res = obj2ast_expr(tmp, &dest, arena); - if (res != 0) goto failed; - Py_XDECREF(tmp); - tmp = NULL; - } else { - dest = NULL; - } - if (PyObject_HasAttrString(obj, "values")) { - int res; - Py_ssize_t len; - Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "values"); - if (tmp == NULL) goto failed; - if (!PyList_Check(tmp)) { - PyErr_Format(PyExc_TypeError, "Print field \"values\" must be a list, not a %.200s", tmp->ob_type->tp_name); - goto failed; - } - len = PyList_GET_SIZE(tmp); - values = asdl_seq_new(len, arena); - if (values == NULL) goto failed; - for (i = 0; i < len; i++) { - expr_ty value; - res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(values, i, value); - } - Py_XDECREF(tmp); - tmp = NULL; - } else { - PyErr_SetString(PyExc_TypeError, "required field \"values\" missing from Print"); - return 1; - } - if (PyObject_HasAttrString(obj, "nl")) { - int res; - tmp = PyObject_GetAttrString(obj, "nl"); - if (tmp == NULL) goto failed; - res = obj2ast_bool(tmp, &nl, arena); - if (res != 0) goto failed; - Py_XDECREF(tmp); - tmp = NULL; - } else { - PyErr_SetString(PyExc_TypeError, "required field \"nl\" missing from Print"); - return 1; - } - *out = Print(dest, values, nl, lineno, col_offset, arena); - if (*out == NULL) goto failed; - return 0; - } isinstance = PyObject_IsInstance(obj, (PyObject*)For_type); if (isinstance == -1) { return 1; @@ -4218,44 +4283,32 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) return 1; } if (isinstance) { - expr_ty type; - expr_ty inst; - expr_ty tback; + expr_ty exc; + expr_ty cause; - if (PyObject_HasAttrString(obj, "type")) { - int res; - tmp = PyObject_GetAttrString(obj, "type"); - if (tmp == NULL) goto failed; - res = obj2ast_expr(tmp, &type, arena); - if (res != 0) goto failed; - Py_XDECREF(tmp); - tmp = NULL; - } else { - type = NULL; - } - if (PyObject_HasAttrString(obj, "inst")) { + if (PyObject_HasAttrString(obj, "exc")) { int res; - tmp = PyObject_GetAttrString(obj, "inst"); + tmp = PyObject_GetAttrString(obj, "exc"); if (tmp == NULL) goto failed; - res = obj2ast_expr(tmp, &inst, arena); + res = obj2ast_expr(tmp, &exc, arena); if (res != 0) goto failed; Py_XDECREF(tmp); tmp = NULL; } else { - inst = NULL; + exc = NULL; } - if (PyObject_HasAttrString(obj, "tback")) { + if (PyObject_HasAttrString(obj, "cause")) { int res; - tmp = PyObject_GetAttrString(obj, "tback"); + tmp = PyObject_GetAttrString(obj, "cause"); if (tmp == NULL) goto failed; - res = obj2ast_expr(tmp, &tback, arena); + res = obj2ast_expr(tmp, &cause, arena); if (res != 0) goto failed; Py_XDECREF(tmp); tmp = NULL; } else { - tback = NULL; + cause = NULL; } - *out = Raise(type, inst, tback, lineno, col_offset, arena); + *out = Raise(exc, cause, lineno, col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -4542,54 +4595,43 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) if (*out == NULL) goto failed; return 0; } - isinstance = PyObject_IsInstance(obj, (PyObject*)Exec_type); + isinstance = PyObject_IsInstance(obj, (PyObject*)Global_type); if (isinstance == -1) { return 1; } if (isinstance) { - expr_ty body; - expr_ty globals; - expr_ty locals; + asdl_seq* names; - if (PyObject_HasAttrString(obj, "body")) { + if (PyObject_HasAttrString(obj, "names")) { int res; - tmp = PyObject_GetAttrString(obj, "body"); + Py_ssize_t len; + Py_ssize_t i; + tmp = PyObject_GetAttrString(obj, "names"); if (tmp == NULL) goto failed; - res = obj2ast_expr(tmp, &body, arena); - if (res != 0) goto failed; + if (!PyList_Check(tmp)) { + PyErr_Format(PyExc_TypeError, "Global field \"names\" must be a list, not a %.200s", tmp->ob_type->tp_name); + goto failed; + } + len = PyList_GET_SIZE(tmp); + names = asdl_seq_new(len, arena); + if (names == NULL) goto failed; + for (i = 0; i < len; i++) { + identifier value; + res = obj2ast_identifier(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(names, i, value); + } Py_XDECREF(tmp); tmp = NULL; } else { - PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from Exec"); + PyErr_SetString(PyExc_TypeError, "required field \"names\" missing from Global"); return 1; } - if (PyObject_HasAttrString(obj, "globals")) { - int res; - tmp = PyObject_GetAttrString(obj, "globals"); - if (tmp == NULL) goto failed; - res = obj2ast_expr(tmp, &globals, arena); - if (res != 0) goto failed; - Py_XDECREF(tmp); - tmp = NULL; - } else { - globals = NULL; - } - if (PyObject_HasAttrString(obj, "locals")) { - int res; - tmp = PyObject_GetAttrString(obj, "locals"); - if (tmp == NULL) goto failed; - res = obj2ast_expr(tmp, &locals, arena); - if (res != 0) goto failed; - Py_XDECREF(tmp); - tmp = NULL; - } else { - locals = NULL; - } - *out = Exec(body, globals, locals, lineno, col_offset, arena); + *out = Global(names, lineno, col_offset, arena); if (*out == NULL) goto failed; return 0; } - isinstance = PyObject_IsInstance(obj, (PyObject*)Global_type); + isinstance = PyObject_IsInstance(obj, (PyObject*)Nonlocal_type); if (isinstance == -1) { return 1; } @@ -4603,7 +4645,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) tmp = PyObject_GetAttrString(obj, "names"); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { - PyErr_Format(PyExc_TypeError, "Global field \"names\" must be a list, not a %.200s", tmp->ob_type->tp_name); + PyErr_Format(PyExc_TypeError, "Nonlocal field \"names\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; } len = PyList_GET_SIZE(tmp); @@ -4618,10 +4660,10 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) Py_XDECREF(tmp); tmp = NULL; } else { - PyErr_SetString(PyExc_TypeError, "required field \"names\" missing from Global"); + PyErr_SetString(PyExc_TypeError, "required field \"names\" missing from Nonlocal"); return 1; } - *out = Global(names, lineno, col_offset, arena); + *out = Nonlocal(names, lineno, col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -4679,10 +4721,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) return 0; } - tmp = PyObject_Repr(obj); - if (tmp == NULL) goto failed; - PyErr_Format(PyExc_TypeError, "expected some sort of stmt, but got %.400s", PyString_AS_STRING(tmp)); -failed: + PyErr_Format(PyExc_TypeError, "expected some sort of stmt, but got %R", obj); + failed: Py_XDECREF(tmp); return 1; } @@ -4690,9 +4730,9 @@ failed: int obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) { - PyObject* tmp = NULL; int isinstance; + PyObject *tmp = NULL; int lineno; int col_offset; @@ -5449,53 +5489,53 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) if (*out == NULL) goto failed; return 0; } - isinstance = PyObject_IsInstance(obj, (PyObject*)Repr_type); + isinstance = PyObject_IsInstance(obj, (PyObject*)Num_type); if (isinstance == -1) { return 1; } if (isinstance) { - expr_ty value; + object n; - if (PyObject_HasAttrString(obj, "value")) { + if (PyObject_HasAttrString(obj, "n")) { int res; - tmp = PyObject_GetAttrString(obj, "value"); + tmp = PyObject_GetAttrString(obj, "n"); if (tmp == NULL) goto failed; - res = obj2ast_expr(tmp, &value, arena); + res = obj2ast_object(tmp, &n, arena); if (res != 0) goto failed; Py_XDECREF(tmp); tmp = NULL; } else { - PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Repr"); + PyErr_SetString(PyExc_TypeError, "required field \"n\" missing from Num"); return 1; } - *out = Repr(value, lineno, col_offset, arena); + *out = Num(n, lineno, col_offset, arena); if (*out == NULL) goto failed; return 0; } - isinstance = PyObject_IsInstance(obj, (PyObject*)Num_type); + isinstance = PyObject_IsInstance(obj, (PyObject*)Str_type); if (isinstance == -1) { return 1; } if (isinstance) { - object n; + string s; - if (PyObject_HasAttrString(obj, "n")) { + if (PyObject_HasAttrString(obj, "s")) { int res; - tmp = PyObject_GetAttrString(obj, "n"); + tmp = PyObject_GetAttrString(obj, "s"); if (tmp == NULL) goto failed; - res = obj2ast_object(tmp, &n, arena); + res = obj2ast_string(tmp, &s, arena); if (res != 0) goto failed; Py_XDECREF(tmp); tmp = NULL; } else { - PyErr_SetString(PyExc_TypeError, "required field \"n\" missing from Num"); + PyErr_SetString(PyExc_TypeError, "required field \"s\" missing from Str"); return 1; } - *out = Num(n, lineno, col_offset, arena); + *out = Str(s, lineno, col_offset, arena); if (*out == NULL) goto failed; return 0; } - isinstance = PyObject_IsInstance(obj, (PyObject*)Str_type); + isinstance = PyObject_IsInstance(obj, (PyObject*)Bytes_type); if (isinstance == -1) { return 1; } @@ -5511,10 +5551,20 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) Py_XDECREF(tmp); tmp = NULL; } else { - PyErr_SetString(PyExc_TypeError, "required field \"s\" missing from Str"); + PyErr_SetString(PyExc_TypeError, "required field \"s\" missing from Bytes"); return 1; } - *out = Str(s, lineno, col_offset, arena); + *out = Bytes(s, lineno, col_offset, arena); + if (*out == NULL) goto failed; + return 0; + } + isinstance = PyObject_IsInstance(obj, (PyObject*)Ellipsis_type); + if (isinstance == -1) { + return 1; + } + if (isinstance) { + + *out = Ellipsis(lineno, col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -5616,6 +5666,42 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) if (*out == NULL) goto failed; return 0; } + isinstance = PyObject_IsInstance(obj, (PyObject*)Starred_type); + if (isinstance == -1) { + return 1; + } + if (isinstance) { + expr_ty value; + expr_context_ty ctx; + + if (PyObject_HasAttrString(obj, "value")) { + int res; + tmp = PyObject_GetAttrString(obj, "value"); + if (tmp == NULL) goto failed; + res = obj2ast_expr(tmp, &value, arena); + if (res != 0) goto failed; + Py_XDECREF(tmp); + tmp = NULL; + } else { + PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Starred"); + return 1; + } + if (PyObject_HasAttrString(obj, "ctx")) { + int res; + tmp = PyObject_GetAttrString(obj, "ctx"); + if (tmp == NULL) goto failed; + res = obj2ast_expr_context(tmp, &ctx, arena); + if (res != 0) goto failed; + Py_XDECREF(tmp); + tmp = NULL; + } else { + PyErr_SetString(PyExc_TypeError, "required field \"ctx\" missing from Starred"); + return 1; + } + *out = Starred(value, ctx, lineno, col_offset, arena); + if (*out == NULL) goto failed; + return 0; + } isinstance = PyObject_IsInstance(obj, (PyObject*)Name_type); if (isinstance == -1) { return 1; @@ -5751,10 +5837,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) return 0; } - tmp = PyObject_Repr(obj); - if (tmp == NULL) goto failed; - PyErr_Format(PyExc_TypeError, "expected some sort of expr, but got %.400s", PyString_AS_STRING(tmp)); -failed: + PyErr_Format(PyExc_TypeError, "expected some sort of expr, but got %R", obj); + failed: Py_XDECREF(tmp); return 1; } @@ -5762,7 +5846,6 @@ failed: int obj2ast_expr_context(PyObject* obj, expr_context_ty* out, PyArena* arena) { - PyObject* tmp = NULL; int isinstance; isinstance = PyObject_IsInstance(obj, (PyObject *)Load_type); @@ -5814,35 +5897,21 @@ obj2ast_expr_context(PyObject* obj, expr_context_ty* out, PyArena* arena) return 0; } - tmp = PyObject_Repr(obj); - if (tmp == NULL) goto failed; - PyErr_Format(PyExc_TypeError, "expected some sort of expr_context, but got %.400s", PyString_AS_STRING(tmp)); -failed: - Py_XDECREF(tmp); + PyErr_Format(PyExc_TypeError, "expected some sort of expr_context, but got %R", obj); return 1; } int obj2ast_slice(PyObject* obj, slice_ty* out, PyArena* arena) { - PyObject* tmp = NULL; int isinstance; + PyObject *tmp = NULL; if (obj == Py_None) { *out = NULL; return 0; } - isinstance = PyObject_IsInstance(obj, (PyObject*)Ellipsis_type); - if (isinstance == -1) { - return 1; - } - if (isinstance) { - - *out = Ellipsis(arena); - if (*out == NULL) goto failed; - return 0; - } isinstance = PyObject_IsInstance(obj, (PyObject*)Slice_type); if (isinstance == -1) { return 1; @@ -5949,10 +6018,8 @@ obj2ast_slice(PyObject* obj, slice_ty* out, PyArena* arena) return 0; } - tmp = PyObject_Repr(obj); - if (tmp == NULL) goto failed; - PyErr_Format(PyExc_TypeError, "expected some sort of slice, but got %.400s", PyString_AS_STRING(tmp)); -failed: + PyErr_Format(PyExc_TypeError, "expected some sort of slice, but got %R", obj); + failed: Py_XDECREF(tmp); return 1; } @@ -5960,7 +6027,6 @@ failed: int obj2ast_boolop(PyObject* obj, boolop_ty* out, PyArena* arena) { - PyObject* tmp = NULL; int isinstance; isinstance = PyObject_IsInstance(obj, (PyObject *)And_type); @@ -5980,18 +6046,13 @@ obj2ast_boolop(PyObject* obj, boolop_ty* out, PyArena* arena) return 0; } - tmp = PyObject_Repr(obj); - if (tmp == NULL) goto failed; - PyErr_Format(PyExc_TypeError, "expected some sort of boolop, but got %.400s", PyString_AS_STRING(tmp)); -failed: - Py_XDECREF(tmp); + PyErr_Format(PyExc_TypeError, "expected some sort of boolop, but got %R", obj); return 1; } int obj2ast_operator(PyObject* obj, operator_ty* out, PyArena* arena) { - PyObject* tmp = NULL; int isinstance; isinstance = PyObject_IsInstance(obj, (PyObject *)Add_type); @@ -6091,18 +6152,13 @@ obj2ast_operator(PyObject* obj, operator_ty* out, PyArena* arena) return 0; } - tmp = PyObject_Repr(obj); - if (tmp == NULL) goto failed; - PyErr_Format(PyExc_TypeError, "expected some sort of operator, but got %.400s", PyString_AS_STRING(tmp)); -failed: - Py_XDECREF(tmp); + PyErr_Format(PyExc_TypeError, "expected some sort of operator, but got %R", obj); return 1; } int obj2ast_unaryop(PyObject* obj, unaryop_ty* out, PyArena* arena) { - PyObject* tmp = NULL; int isinstance; isinstance = PyObject_IsInstance(obj, (PyObject *)Invert_type); @@ -6138,18 +6194,13 @@ obj2ast_unaryop(PyObject* obj, unaryop_ty* out, PyArena* arena) return 0; } - tmp = PyObject_Repr(obj); - if (tmp == NULL) goto failed; - PyErr_Format(PyExc_TypeError, "expected some sort of unaryop, but got %.400s", PyString_AS_STRING(tmp)); -failed: - Py_XDECREF(tmp); + PyErr_Format(PyExc_TypeError, "expected some sort of unaryop, but got %R", obj); return 1; } int obj2ast_cmpop(PyObject* obj, cmpop_ty* out, PyArena* arena) { - PyObject* tmp = NULL; int isinstance; isinstance = PyObject_IsInstance(obj, (PyObject *)Eq_type); @@ -6233,11 +6284,7 @@ obj2ast_cmpop(PyObject* obj, cmpop_ty* out, PyArena* arena) return 0; } - tmp = PyObject_Repr(obj); - if (tmp == NULL) goto failed; - PyErr_Format(PyExc_TypeError, "expected some sort of cmpop, but got %.400s", PyString_AS_STRING(tmp)); -failed: - Py_XDECREF(tmp); + PyErr_Format(PyExc_TypeError, "expected some sort of cmpop, but got %R", obj); return 1; } @@ -6308,9 +6355,9 @@ failed: int obj2ast_excepthandler(PyObject* obj, excepthandler_ty* out, PyArena* arena) { - PyObject* tmp = NULL; int isinstance; + PyObject *tmp = NULL; int lineno; int col_offset; @@ -6348,7 +6395,7 @@ obj2ast_excepthandler(PyObject* obj, excepthandler_ty* out, PyArena* arena) } if (isinstance) { expr_ty type; - expr_ty name; + identifier name; asdl_seq* body; if (PyObject_HasAttrString(obj, "type")) { @@ -6366,7 +6413,7 @@ obj2ast_excepthandler(PyObject* obj, excepthandler_ty* out, PyArena* arena) int res; tmp = PyObject_GetAttrString(obj, "name"); if (tmp == NULL) goto failed; - res = obj2ast_expr(tmp, &name, arena); + res = obj2ast_identifier(tmp, &name, arena); if (res != 0) goto failed; Py_XDECREF(tmp); tmp = NULL; @@ -6404,10 +6451,8 @@ obj2ast_excepthandler(PyObject* obj, excepthandler_ty* out, PyArena* arena) return 0; } - tmp = PyObject_Repr(obj); - if (tmp == NULL) goto failed; - PyErr_Format(PyExc_TypeError, "expected some sort of excepthandler, but got %.400s", PyString_AS_STRING(tmp)); -failed: + PyErr_Format(PyExc_TypeError, "expected some sort of excepthandler, but got %R", obj); + failed: Py_XDECREF(tmp); return 1; } @@ -6418,8 +6463,12 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena) PyObject* tmp = NULL; asdl_seq* args; identifier vararg; + expr_ty varargannotation; + asdl_seq* kwonlyargs; identifier kwarg; + expr_ty kwargannotation; asdl_seq* defaults; + asdl_seq* kw_defaults; if (PyObject_HasAttrString(obj, "args")) { int res; @@ -6435,8 +6484,8 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena) args = asdl_seq_new(len, arena); if (args == NULL) goto failed; for (i = 0; i < len; i++) { - expr_ty value; - res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); + arg_ty value; + res = obj2ast_arg(PyList_GET_ITEM(tmp, i), &value, arena); if (res != 0) goto failed; asdl_seq_SET(args, i, value); } @@ -6457,6 +6506,42 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena) } else { vararg = NULL; } + if (PyObject_HasAttrString(obj, "varargannotation")) { + int res; + tmp = PyObject_GetAttrString(obj, "varargannotation"); + if (tmp == NULL) goto failed; + res = obj2ast_expr(tmp, &varargannotation, arena); + if (res != 0) goto failed; + Py_XDECREF(tmp); + tmp = NULL; + } else { + varargannotation = NULL; + } + if (PyObject_HasAttrString(obj, "kwonlyargs")) { + int res; + Py_ssize_t len; + Py_ssize_t i; + tmp = PyObject_GetAttrString(obj, "kwonlyargs"); + if (tmp == NULL) goto failed; + if (!PyList_Check(tmp)) { + PyErr_Format(PyExc_TypeError, "arguments field \"kwonlyargs\" must be a list, not a %.200s", tmp->ob_type->tp_name); + goto failed; + } + len = PyList_GET_SIZE(tmp); + kwonlyargs = asdl_seq_new(len, arena); + if (kwonlyargs == NULL) goto failed; + for (i = 0; i < len; i++) { + arg_ty value; + res = obj2ast_arg(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(kwonlyargs, i, value); + } + Py_XDECREF(tmp); + tmp = NULL; + } else { + PyErr_SetString(PyExc_TypeError, "required field \"kwonlyargs\" missing from arguments"); + return 1; + } if (PyObject_HasAttrString(obj, "kwarg")) { int res; tmp = PyObject_GetAttrString(obj, "kwarg"); @@ -6468,6 +6553,17 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena) } else { kwarg = NULL; } + if (PyObject_HasAttrString(obj, "kwargannotation")) { + int res; + tmp = PyObject_GetAttrString(obj, "kwargannotation"); + if (tmp == NULL) goto failed; + res = obj2ast_expr(tmp, &kwargannotation, arena); + if (res != 0) goto failed; + Py_XDECREF(tmp); + tmp = NULL; + } else { + kwargannotation = NULL; + } if (PyObject_HasAttrString(obj, "defaults")) { int res; Py_ssize_t len; @@ -6493,7 +6589,70 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"defaults\" missing from arguments"); return 1; } - *out = arguments(args, vararg, kwarg, defaults, arena); + if (PyObject_HasAttrString(obj, "kw_defaults")) { + int res; + Py_ssize_t len; + Py_ssize_t i; + tmp = PyObject_GetAttrString(obj, "kw_defaults"); + if (tmp == NULL) goto failed; + if (!PyList_Check(tmp)) { + PyErr_Format(PyExc_TypeError, "arguments field \"kw_defaults\" must be a list, not a %.200s", tmp->ob_type->tp_name); + goto failed; + } + len = PyList_GET_SIZE(tmp); + kw_defaults = asdl_seq_new(len, arena); + if (kw_defaults == NULL) goto failed; + for (i = 0; i < len; i++) { + expr_ty value; + res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(kw_defaults, i, value); + } + Py_XDECREF(tmp); + tmp = NULL; + } else { + PyErr_SetString(PyExc_TypeError, "required field \"kw_defaults\" missing from arguments"); + return 1; + } + *out = arguments(args, vararg, varargannotation, kwonlyargs, kwarg, + kwargannotation, defaults, kw_defaults, arena); + return 0; +failed: + Py_XDECREF(tmp); + return 1; +} + +int +obj2ast_arg(PyObject* obj, arg_ty* out, PyArena* arena) +{ + PyObject* tmp = NULL; + identifier arg; + expr_ty annotation; + + if (PyObject_HasAttrString(obj, "arg")) { + int res; + tmp = PyObject_GetAttrString(obj, "arg"); + if (tmp == NULL) goto failed; + res = obj2ast_identifier(tmp, &arg, arena); + if (res != 0) goto failed; + Py_XDECREF(tmp); + tmp = NULL; + } else { + PyErr_SetString(PyExc_TypeError, "required field \"arg\" missing from arg"); + return 1; + } + if (PyObject_HasAttrString(obj, "annotation")) { + int res; + tmp = PyObject_GetAttrString(obj, "annotation"); + if (tmp == NULL) goto failed; + res = obj2ast_expr(tmp, &annotation, arena); + if (res != 0) goto failed; + Py_XDECREF(tmp); + tmp = NULL; + } else { + annotation = NULL; + } + *out = arg(arg, annotation, arena); return 0; failed: Py_XDECREF(tmp); @@ -6576,166 +6735,223 @@ failed: } +static struct PyModuleDef _astmodule = { + PyModuleDef_HEAD_INIT, "_ast" +}; PyMODINIT_FUNC -init_ast(void) +PyInit__ast(void) { PyObject *m, *d; - if (!init_types()) return; - m = Py_InitModule3("_ast", NULL, NULL); - if (!m) return; + if (!init_types()) return NULL; + m = PyModule_Create(&_astmodule); + if (!m) return NULL; d = PyModule_GetDict(m); - if (PyDict_SetItemString(d, "AST", (PyObject*)&AST_type) < 0) return; + if (PyDict_SetItemString(d, "AST", (PyObject*)&AST_type) < 0) return + NULL; if (PyModule_AddIntConstant(m, "PyCF_ONLY_AST", PyCF_ONLY_AST) < 0) - return; - if (PyModule_AddStringConstant(m, "__version__", "82160") < 0) - return; - if (PyDict_SetItemString(d, "mod", (PyObject*)mod_type) < 0) return; + return NULL; + if (PyModule_AddStringConstant(m, "__version__", "82163") < 0) + return NULL; + if (PyDict_SetItemString(d, "mod", (PyObject*)mod_type) < 0) return + NULL; if (PyDict_SetItemString(d, "Module", (PyObject*)Module_type) < 0) - return; + return NULL; if (PyDict_SetItemString(d, "Interactive", (PyObject*)Interactive_type) - < 0) return; + < 0) return NULL; if (PyDict_SetItemString(d, "Expression", (PyObject*)Expression_type) < - 0) return; - if (PyDict_SetItemString(d, "Suite", (PyObject*)Suite_type) < 0) return; - if (PyDict_SetItemString(d, "stmt", (PyObject*)stmt_type) < 0) return; + 0) return NULL; + if (PyDict_SetItemString(d, "Suite", (PyObject*)Suite_type) < 0) return + NULL; + if (PyDict_SetItemString(d, "stmt", (PyObject*)stmt_type) < 0) return + NULL; if (PyDict_SetItemString(d, "FunctionDef", (PyObject*)FunctionDef_type) - < 0) return; + < 0) return NULL; if (PyDict_SetItemString(d, "ClassDef", (PyObject*)ClassDef_type) < 0) - return; + return NULL; if (PyDict_SetItemString(d, "Return", (PyObject*)Return_type) < 0) - return; + return NULL; if (PyDict_SetItemString(d, "Delete", (PyObject*)Delete_type) < 0) - return; + return NULL; if (PyDict_SetItemString(d, "Assign", (PyObject*)Assign_type) < 0) - return; + return NULL; if (PyDict_SetItemString(d, "AugAssign", (PyObject*)AugAssign_type) < - 0) return; - if (PyDict_SetItemString(d, "Print", (PyObject*)Print_type) < 0) return; - if (PyDict_SetItemString(d, "For", (PyObject*)For_type) < 0) return; - if (PyDict_SetItemString(d, "While", (PyObject*)While_type) < 0) return; - if (PyDict_SetItemString(d, "If", (PyObject*)If_type) < 0) return; - if (PyDict_SetItemString(d, "With", (PyObject*)With_type) < 0) return; - if (PyDict_SetItemString(d, "Raise", (PyObject*)Raise_type) < 0) return; + 0) return NULL; + if (PyDict_SetItemString(d, "For", (PyObject*)For_type) < 0) return + NULL; + if (PyDict_SetItemString(d, "While", (PyObject*)While_type) < 0) return + NULL; + if (PyDict_SetItemString(d, "If", (PyObject*)If_type) < 0) return NULL; + if (PyDict_SetItemString(d, "With", (PyObject*)With_type) < 0) return + NULL; + if (PyDict_SetItemString(d, "Raise", (PyObject*)Raise_type) < 0) return + NULL; if (PyDict_SetItemString(d, "TryExcept", (PyObject*)TryExcept_type) < - 0) return; + 0) return NULL; if (PyDict_SetItemString(d, "TryFinally", (PyObject*)TryFinally_type) < - 0) return; + 0) return NULL; if (PyDict_SetItemString(d, "Assert", (PyObject*)Assert_type) < 0) - return; + return NULL; if (PyDict_SetItemString(d, "Import", (PyObject*)Import_type) < 0) - return; + return NULL; if (PyDict_SetItemString(d, "ImportFrom", (PyObject*)ImportFrom_type) < - 0) return; - if (PyDict_SetItemString(d, "Exec", (PyObject*)Exec_type) < 0) return; + 0) return NULL; if (PyDict_SetItemString(d, "Global", (PyObject*)Global_type) < 0) - return; - if (PyDict_SetItemString(d, "Expr", (PyObject*)Expr_type) < 0) return; - if (PyDict_SetItemString(d, "Pass", (PyObject*)Pass_type) < 0) return; - if (PyDict_SetItemString(d, "Break", (PyObject*)Break_type) < 0) return; + return NULL; + if (PyDict_SetItemString(d, "Nonlocal", (PyObject*)Nonlocal_type) < 0) + return NULL; + if (PyDict_SetItemString(d, "Expr", (PyObject*)Expr_type) < 0) return + NULL; + if (PyDict_SetItemString(d, "Pass", (PyObject*)Pass_type) < 0) return + NULL; + if (PyDict_SetItemString(d, "Break", (PyObject*)Break_type) < 0) return + NULL; if (PyDict_SetItemString(d, "Continue", (PyObject*)Continue_type) < 0) - return; - if (PyDict_SetItemString(d, "expr", (PyObject*)expr_type) < 0) return; + return NULL; + if (PyDict_SetItemString(d, "expr", (PyObject*)expr_type) < 0) return + NULL; if (PyDict_SetItemString(d, "BoolOp", (PyObject*)BoolOp_type) < 0) - return; - if (PyDict_SetItemString(d, "BinOp", (PyObject*)BinOp_type) < 0) return; + return NULL; + if (PyDict_SetItemString(d, "BinOp", (PyObject*)BinOp_type) < 0) return + NULL; if (PyDict_SetItemString(d, "UnaryOp", (PyObject*)UnaryOp_type) < 0) - return; + return NULL; if (PyDict_SetItemString(d, "Lambda", (PyObject*)Lambda_type) < 0) - return; - if (PyDict_SetItemString(d, "IfExp", (PyObject*)IfExp_type) < 0) return; - if (PyDict_SetItemString(d, "Dict", (PyObject*)Dict_type) < 0) return; - if (PyDict_SetItemString(d, "Set", (PyObject*)Set_type) < 0) return; + return NULL; + if (PyDict_SetItemString(d, "IfExp", (PyObject*)IfExp_type) < 0) return + NULL; + if (PyDict_SetItemString(d, "Dict", (PyObject*)Dict_type) < 0) return + NULL; + if (PyDict_SetItemString(d, "Set", (PyObject*)Set_type) < 0) return + NULL; if (PyDict_SetItemString(d, "ListComp", (PyObject*)ListComp_type) < 0) - return; + return NULL; if (PyDict_SetItemString(d, "SetComp", (PyObject*)SetComp_type) < 0) - return; + return NULL; if (PyDict_SetItemString(d, "DictComp", (PyObject*)DictComp_type) < 0) - return; + return NULL; if (PyDict_SetItemString(d, "GeneratorExp", - (PyObject*)GeneratorExp_type) < 0) return; - if (PyDict_SetItemString(d, "Yield", (PyObject*)Yield_type) < 0) return; + (PyObject*)GeneratorExp_type) < 0) return NULL; + if (PyDict_SetItemString(d, "Yield", (PyObject*)Yield_type) < 0) return + NULL; if (PyDict_SetItemString(d, "Compare", (PyObject*)Compare_type) < 0) - return; - if (PyDict_SetItemString(d, "Call", (PyObject*)Call_type) < 0) return; - if (PyDict_SetItemString(d, "Repr", (PyObject*)Repr_type) < 0) return; - if (PyDict_SetItemString(d, "Num", (PyObject*)Num_type) < 0) return; - if (PyDict_SetItemString(d, "Str", (PyObject*)Str_type) < 0) return; + return NULL; + if (PyDict_SetItemString(d, "Call", (PyObject*)Call_type) < 0) return + NULL; + if (PyDict_SetItemString(d, "Num", (PyObject*)Num_type) < 0) return + NULL; + if (PyDict_SetItemString(d, "Str", (PyObject*)Str_type) < 0) return + NULL; + if (PyDict_SetItemString(d, "Bytes", (PyObject*)Bytes_type) < 0) return + NULL; + if (PyDict_SetItemString(d, "Ellipsis", (PyObject*)Ellipsis_type) < 0) + return NULL; if (PyDict_SetItemString(d, "Attribute", (PyObject*)Attribute_type) < - 0) return; + 0) return NULL; if (PyDict_SetItemString(d, "Subscript", (PyObject*)Subscript_type) < - 0) return; - if (PyDict_SetItemString(d, "Name", (PyObject*)Name_type) < 0) return; - if (PyDict_SetItemString(d, "List", (PyObject*)List_type) < 0) return; - if (PyDict_SetItemString(d, "Tuple", (PyObject*)Tuple_type) < 0) return; + 0) return NULL; + if (PyDict_SetItemString(d, "Starred", (PyObject*)Starred_type) < 0) + return NULL; + if (PyDict_SetItemString(d, "Name", (PyObject*)Name_type) < 0) return + NULL; + if (PyDict_SetItemString(d, "List", (PyObject*)List_type) < 0) return + NULL; + if (PyDict_SetItemString(d, "Tuple", (PyObject*)Tuple_type) < 0) return + NULL; if (PyDict_SetItemString(d, "expr_context", - (PyObject*)expr_context_type) < 0) return; - if (PyDict_SetItemString(d, "Load", (PyObject*)Load_type) < 0) return; - if (PyDict_SetItemString(d, "Store", (PyObject*)Store_type) < 0) return; - if (PyDict_SetItemString(d, "Del", (PyObject*)Del_type) < 0) return; + (PyObject*)expr_context_type) < 0) return NULL; + if (PyDict_SetItemString(d, "Load", (PyObject*)Load_type) < 0) return + NULL; + if (PyDict_SetItemString(d, "Store", (PyObject*)Store_type) < 0) return + NULL; + if (PyDict_SetItemString(d, "Del", (PyObject*)Del_type) < 0) return + NULL; if (PyDict_SetItemString(d, "AugLoad", (PyObject*)AugLoad_type) < 0) - return; + return NULL; if (PyDict_SetItemString(d, "AugStore", (PyObject*)AugStore_type) < 0) - return; - if (PyDict_SetItemString(d, "Param", (PyObject*)Param_type) < 0) return; - if (PyDict_SetItemString(d, "slice", (PyObject*)slice_type) < 0) return; - if (PyDict_SetItemString(d, "Ellipsis", (PyObject*)Ellipsis_type) < 0) - return; - if (PyDict_SetItemString(d, "Slice", (PyObject*)Slice_type) < 0) return; + return NULL; + if (PyDict_SetItemString(d, "Param", (PyObject*)Param_type) < 0) return + NULL; + if (PyDict_SetItemString(d, "slice", (PyObject*)slice_type) < 0) return + NULL; + if (PyDict_SetItemString(d, "Slice", (PyObject*)Slice_type) < 0) return + NULL; if (PyDict_SetItemString(d, "ExtSlice", (PyObject*)ExtSlice_type) < 0) - return; - if (PyDict_SetItemString(d, "Index", (PyObject*)Index_type) < 0) return; + return NULL; + if (PyDict_SetItemString(d, "Index", (PyObject*)Index_type) < 0) return + NULL; if (PyDict_SetItemString(d, "boolop", (PyObject*)boolop_type) < 0) - return; - if (PyDict_SetItemString(d, "And", (PyObject*)And_type) < 0) return; - if (PyDict_SetItemString(d, "Or", (PyObject*)Or_type) < 0) return; + return NULL; + if (PyDict_SetItemString(d, "And", (PyObject*)And_type) < 0) return + NULL; + if (PyDict_SetItemString(d, "Or", (PyObject*)Or_type) < 0) return NULL; if (PyDict_SetItemString(d, "operator", (PyObject*)operator_type) < 0) - return; - if (PyDict_SetItemString(d, "Add", (PyObject*)Add_type) < 0) return; - if (PyDict_SetItemString(d, "Sub", (PyObject*)Sub_type) < 0) return; - if (PyDict_SetItemString(d, "Mult", (PyObject*)Mult_type) < 0) return; - if (PyDict_SetItemString(d, "Div", (PyObject*)Div_type) < 0) return; - if (PyDict_SetItemString(d, "Mod", (PyObject*)Mod_type) < 0) return; - if (PyDict_SetItemString(d, "Pow", (PyObject*)Pow_type) < 0) return; + return NULL; + if (PyDict_SetItemString(d, "Add", (PyObject*)Add_type) < 0) return + NULL; + if (PyDict_SetItemString(d, "Sub", (PyObject*)Sub_type) < 0) return + NULL; + if (PyDict_SetItemString(d, "Mult", (PyObject*)Mult_type) < 0) return + NULL; + if (PyDict_SetItemString(d, "Div", (PyObject*)Div_type) < 0) return + NULL; + if (PyDict_SetItemString(d, "Mod", (PyObject*)Mod_type) < 0) return + NULL; + if (PyDict_SetItemString(d, "Pow", (PyObject*)Pow_type) < 0) return + NULL; if (PyDict_SetItemString(d, "LShift", (PyObject*)LShift_type) < 0) - return; + return NULL; if (PyDict_SetItemString(d, "RShift", (PyObject*)RShift_type) < 0) - return; - if (PyDict_SetItemString(d, "BitOr", (PyObject*)BitOr_type) < 0) return; + return NULL; + if (PyDict_SetItemString(d, "BitOr", (PyObject*)BitOr_type) < 0) return + NULL; if (PyDict_SetItemString(d, "BitXor", (PyObject*)BitXor_type) < 0) - return; + return NULL; if (PyDict_SetItemString(d, "BitAnd", (PyObject*)BitAnd_type) < 0) - return; + return NULL; if (PyDict_SetItemString(d, "FloorDiv", (PyObject*)FloorDiv_type) < 0) - return; + return NULL; if (PyDict_SetItemString(d, "unaryop", (PyObject*)unaryop_type) < 0) - return; + return NULL; if (PyDict_SetItemString(d, "Invert", (PyObject*)Invert_type) < 0) - return; - if (PyDict_SetItemString(d, "Not", (PyObject*)Not_type) < 0) return; - if (PyDict_SetItemString(d, "UAdd", (PyObject*)UAdd_type) < 0) return; - if (PyDict_SetItemString(d, "USub", (PyObject*)USub_type) < 0) return; - if (PyDict_SetItemString(d, "cmpop", (PyObject*)cmpop_type) < 0) return; - if (PyDict_SetItemString(d, "Eq", (PyObject*)Eq_type) < 0) return; - if (PyDict_SetItemString(d, "NotEq", (PyObject*)NotEq_type) < 0) return; - if (PyDict_SetItemString(d, "Lt", (PyObject*)Lt_type) < 0) return; - if (PyDict_SetItemString(d, "LtE", (PyObject*)LtE_type) < 0) return; - if (PyDict_SetItemString(d, "Gt", (PyObject*)Gt_type) < 0) return; - if (PyDict_SetItemString(d, "GtE", (PyObject*)GtE_type) < 0) return; - if (PyDict_SetItemString(d, "Is", (PyObject*)Is_type) < 0) return; - if (PyDict_SetItemString(d, "IsNot", (PyObject*)IsNot_type) < 0) return; - if (PyDict_SetItemString(d, "In", (PyObject*)In_type) < 0) return; - if (PyDict_SetItemString(d, "NotIn", (PyObject*)NotIn_type) < 0) return; + return NULL; + if (PyDict_SetItemString(d, "Not", (PyObject*)Not_type) < 0) return + NULL; + if (PyDict_SetItemString(d, "UAdd", (PyObject*)UAdd_type) < 0) return + NULL; + if (PyDict_SetItemString(d, "USub", (PyObject*)USub_type) < 0) return + NULL; + if (PyDict_SetItemString(d, "cmpop", (PyObject*)cmpop_type) < 0) return + NULL; + if (PyDict_SetItemString(d, "Eq", (PyObject*)Eq_type) < 0) return NULL; + if (PyDict_SetItemString(d, "NotEq", (PyObject*)NotEq_type) < 0) return + NULL; + if (PyDict_SetItemString(d, "Lt", (PyObject*)Lt_type) < 0) return NULL; + if (PyDict_SetItemString(d, "LtE", (PyObject*)LtE_type) < 0) return + NULL; + if (PyDict_SetItemString(d, "Gt", (PyObject*)Gt_type) < 0) return NULL; + if (PyDict_SetItemString(d, "GtE", (PyObject*)GtE_type) < 0) return + NULL; + if (PyDict_SetItemString(d, "Is", (PyObject*)Is_type) < 0) return NULL; + if (PyDict_SetItemString(d, "IsNot", (PyObject*)IsNot_type) < 0) return + NULL; + if (PyDict_SetItemString(d, "In", (PyObject*)In_type) < 0) return NULL; + if (PyDict_SetItemString(d, "NotIn", (PyObject*)NotIn_type) < 0) return + NULL; if (PyDict_SetItemString(d, "comprehension", - (PyObject*)comprehension_type) < 0) return; + (PyObject*)comprehension_type) < 0) return NULL; if (PyDict_SetItemString(d, "excepthandler", - (PyObject*)excepthandler_type) < 0) return; + (PyObject*)excepthandler_type) < 0) return NULL; if (PyDict_SetItemString(d, "ExceptHandler", - (PyObject*)ExceptHandler_type) < 0) return; + (PyObject*)ExceptHandler_type) < 0) return NULL; if (PyDict_SetItemString(d, "arguments", (PyObject*)arguments_type) < - 0) return; + 0) return NULL; + if (PyDict_SetItemString(d, "arg", (PyObject*)arg_type) < 0) return + NULL; if (PyDict_SetItemString(d, "keyword", (PyObject*)keyword_type) < 0) - return; - if (PyDict_SetItemString(d, "alias", (PyObject*)alias_type) < 0) return; + return NULL; + if (PyDict_SetItemString(d, "alias", (PyObject*)alias_type) < 0) return + NULL; + return m; } @@ -6749,18 +6965,10 @@ PyObject* PyAST_mod2obj(mod_ty t) mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode) { mod_ty res; - PyObject *req_type[3]; - char *req_name[3]; + PyObject *req_type[] = {(PyObject*)Module_type, (PyObject*)Expression_type, + (PyObject*)Interactive_type}; + char *req_name[] = {"Module", "Expression", "Interactive"}; int isinstance; - - req_type[0] = (PyObject*)Module_type; - req_type[1] = (PyObject*)Expression_type; - req_type[2] = (PyObject*)Interactive_type; - - req_name[0] = "Module"; - req_name[1] = "Expression"; - req_name[2] = "Interactive"; - assert(0 <= mode && mode <= 2); init_types(); diff --git a/Python/_warnings.c b/Python/_warnings.c index 445ff6ba9e..c12db44d19 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -44,7 +44,7 @@ get_warnings_attr(const char *attr) int result; if (warnings_str == NULL) { - warnings_str = PyString_InternFromString("warnings"); + warnings_str = PyUnicode_InternFromString("warnings"); if (warnings_str == NULL) return NULL; } @@ -116,7 +116,7 @@ get_filter(PyObject *category, PyObject *text, Py_ssize_t lineno, _filters = warnings_filters; } - if (!PyList_Check(_filters)) { + if (_filters == NULL || !PyList_Check(_filters)) { PyErr_SetString(PyExc_ValueError, MODULE_NAME ".filters must be a list"); return NULL; @@ -145,18 +145,18 @@ get_filter(PyObject *category, PyObject *text, Py_ssize_t lineno, good_msg = check_matched(msg, text); good_mod = check_matched(mod, module); is_subclass = PyObject_IsSubclass(category, cat); - ln = PyInt_AsSsize_t(ln_obj); + ln = PyLong_AsSsize_t(ln_obj); if (good_msg == -1 || good_mod == -1 || is_subclass == -1 || (ln == -1 && PyErr_Occurred())) return NULL; if (good_msg && is_subclass && good_mod && (ln == 0 || lineno == ln)) - return PyString_AsString(action); + return _PyUnicode_AsString(action); } action = get_default_action(); if (action != NULL) { - return PyString_AsString(action); + return _PyUnicode_AsString(action); } PyErr_SetString(PyExc_ValueError, @@ -198,17 +198,17 @@ normalize_module(PyObject *filename) if (rc == -1) return NULL; else if (rc == 0) - return PyString_FromString("<unknown>"); + return PyUnicode_FromString("<unknown>"); - mod_str = PyString_AsString(filename); + mod_str = _PyUnicode_AsString(filename); if (mod_str == NULL) - return NULL; - len = PyString_Size(filename); + return NULL; + len = PyUnicode_GetSize(filename); if (len < 0) return NULL; if (len >= 3 && - strncmp(mod_str + (len - 3), ".py", 3) == 0) { - module = PyString_FromStringAndSize(mod_str, len-3); + strncmp(mod_str + (len - 3), ".py", 3) == 0) { + module = PyUnicode_FromStringAndSize(mod_str, len-3); } else { module = filename; @@ -225,7 +225,7 @@ update_registry(PyObject *registry, PyObject *text, PyObject *category, int rc; if (add_zero) { - zero = PyInt_FromLong(0); + zero = PyLong_FromLong(0); if (zero == NULL) return -1; altkey = PyTuple_Pack(3, text, category, zero); @@ -271,7 +271,9 @@ show_warning(PyObject *filename, int lineno, PyObject *text, PyObject /* Print " source_line\n" */ if (sourceline) { - char *source_line_str = PyString_AS_STRING(sourceline); + char *source_line_str = _PyUnicode_AsString(sourceline); + if (source_line_str == NULL) + return; while (*source_line_str == ' ' || *source_line_str == '\t' || *source_line_str == '\014') source_line_str++; @@ -280,8 +282,8 @@ show_warning(PyObject *filename, int lineno, PyObject *text, PyObject PyFile_WriteString("\n", f_stderr); } else - _Py_DisplaySourceLine(f_stderr, PyString_AS_STRING(filename), - lineno, 2); + if (_Py_DisplaySourceLine(f_stderr, filename, lineno, 2) < 0) + return; PyErr_Clear(); } @@ -328,7 +330,7 @@ warn_explicit(PyObject *category, PyObject *message, goto cleanup; } - lineno_obj = PyInt_FromLong(lineno); + lineno_obj = PyLong_FromLong(lineno); if (lineno_obj == NULL) goto cleanup; @@ -382,8 +384,11 @@ warn_explicit(PyObject *category, PyObject *message, PyObject *to_str = PyObject_Str(item); const char *err_str = "???"; - if (to_str != NULL) - err_str = PyString_AS_STRING(to_str); + if (to_str != NULL) { + err_str = _PyUnicode_AsString(to_str); + if (err_str == NULL) + goto cleanup; + } PyErr_Format(PyExc_RuntimeError, "Unrecognized action (%s) in warnings.filters:\n %s", action, err_str); @@ -402,23 +407,23 @@ warn_explicit(PyObject *category, PyObject *message, show_warning(filename, lineno, text, category, sourceline); } else { - PyObject *res; - - if (!PyMethod_Check(show_fxn) && !PyFunction_Check(show_fxn)) { - PyErr_SetString(PyExc_TypeError, - "warnings.showwarning() must be set to a " - "function or method"); - Py_DECREF(show_fxn); - goto cleanup; - } - - res = PyObject_CallFunctionObjArgs(show_fxn, message, category, - filename, lineno_obj, - NULL); - Py_DECREF(show_fxn); - Py_XDECREF(res); - if (res == NULL) - goto cleanup; + PyObject *res; + + if (!PyMethod_Check(show_fxn) && !PyFunction_Check(show_fxn)) { + PyErr_SetString(PyExc_TypeError, + "warnings.showwarning() must be set to a " + "function or method"); + Py_DECREF(show_fxn); + goto cleanup; + } + + res = PyObject_CallFunctionObjArgs(show_fxn, message, category, + filename, lineno_obj, + NULL); + Py_DECREF(show_fxn); + Py_XDECREF(res); + if (res == NULL) + goto cleanup; } } else /* if (rc == -1) */ @@ -482,7 +487,7 @@ setup_context(Py_ssize_t stack_level, PyObject **filename, int *lineno, /* Setup module. */ *module = PyDict_GetItemString(globals, "__name__"); if (*module == NULL) { - *module = PyString_FromString("<string>"); + *module = PyUnicode_FromString("<string>"); if (*module == NULL) goto handle_error; } @@ -491,21 +496,19 @@ setup_context(Py_ssize_t stack_level, PyObject **filename, int *lineno, /* Setup filename. */ *filename = PyDict_GetItemString(globals, "__file__"); - if (*filename != NULL && PyString_Check(*filename)) { - Py_ssize_t len = PyString_Size(*filename); - const char *file_str = PyString_AsString(*filename); - if (file_str == NULL || (len < 0 && PyErr_Occurred())) - goto handle_error; + if (*filename != NULL && PyUnicode_Check(*filename)) { + Py_ssize_t len = PyUnicode_GetSize(*filename); + Py_UNICODE *unicode = PyUnicode_AS_UNICODE(*filename); /* if filename.lower().endswith((".pyc", ".pyo")): */ if (len >= 4 && - file_str[len-4] == '.' && - tolower(file_str[len-3]) == 'p' && - tolower(file_str[len-2]) == 'y' && - (tolower(file_str[len-1]) == 'c' || - tolower(file_str[len-1]) == 'o')) + unicode[len-4] == '.' && + Py_UNICODE_TOLOWER(unicode[len-3]) == 'p' && + Py_UNICODE_TOLOWER(unicode[len-2]) == 'y' && + (Py_UNICODE_TOLOWER(unicode[len-1]) == 'c' || + Py_UNICODE_TOLOWER(unicode[len-1]) == 'o')) { - *filename = PyString_FromStringAndSize(file_str, len-1); + *filename = PyUnicode_FromUnicode(unicode, len-1); if (*filename == NULL) goto handle_error; } @@ -513,9 +516,11 @@ setup_context(Py_ssize_t stack_level, PyObject **filename, int *lineno, Py_INCREF(*filename); } else { - const char *module_str = PyString_AsString(*module); + const char *module_str = _PyUnicode_AsString(*module); *filename = NULL; - if (module_str && strcmp(module_str, "__main__") == 0) { + if (module_str == NULL) + goto handle_error; + if (strcmp(module_str, "__main__") == 0) { PyObject *argv = PySys_GetObject("argv"); if (argv != NULL && PyList_Size(argv) > 0) { int is_true; @@ -529,16 +534,16 @@ setup_context(Py_ssize_t stack_level, PyObject **filename, int *lineno, } else if (!is_true) { Py_DECREF(*filename); - *filename = PyString_FromString("__main__"); + *filename = PyUnicode_FromString("__main__"); if (*filename == NULL) goto handle_error; } } else { /* embedded interpreters don't have sys.argv, see bug #839151 */ - *filename = PyString_FromString("__main__"); - if (*filename == NULL) - goto handle_error; + *filename = PyUnicode_FromString("__main__"); + if (*filename == NULL) + goto handle_error; } } if (*filename == NULL) { @@ -648,12 +653,12 @@ warnings_warn_explicit(PyObject *self, PyObject *args, PyObject *kwds) PyObject *returned; if (get_source_name == NULL) { - get_source_name = PyString_InternFromString("get_source"); + get_source_name = PyUnicode_InternFromString("get_source"); if (!get_source_name) return NULL; } if (splitlines_name == NULL) { - splitlines_name = PyString_InternFromString("splitlines"); + splitlines_name = PyUnicode_InternFromString("splitlines"); if (!splitlines_name) return NULL; } @@ -706,19 +711,17 @@ warnings_warn_explicit(PyObject *self, PyObject *args, PyObject *kwds) /* Function to issue a warning message; may raise an exception. */ -int -PyErr_WarnEx(PyObject *category, const char *text, Py_ssize_t stack_level) + +static int +warn_unicode(PyObject *category, PyObject *message, + Py_ssize_t stack_level) { PyObject *res; - PyObject *message = PyString_FromString(text); - if (message == NULL) - return -1; if (category == NULL) category = PyExc_RuntimeWarning; res = do_warn(message, category, stack_level); - Py_DECREF(message); if (res == NULL) return -1; Py_DECREF(res); @@ -726,6 +729,42 @@ PyErr_WarnEx(PyObject *category, const char *text, Py_ssize_t stack_level) return 0; } +int +PyErr_WarnFormat(PyObject *category, Py_ssize_t stack_level, + const char *format, ...) +{ + int ret; + PyObject *message; + va_list vargs; + +#ifdef HAVE_STDARG_PROTOTYPES + va_start(vargs, format); +#else + va_start(vargs); +#endif + message = PyUnicode_FromFormatV(format, vargs); + if (message != NULL) { + ret = warn_unicode(category, message, stack_level); + Py_DECREF(message); + } + else + ret = -1; + va_end(vargs); + return ret; +} + +int +PyErr_WarnEx(PyObject *category, const char *text, Py_ssize_t stack_level) +{ + int ret; + PyObject *message = PyUnicode_FromString(text); + if (message == NULL) + return -1; + ret = warn_unicode(category, message, stack_level); + Py_DECREF(message); + return ret; +} + /* PyErr_Warn is only for backwards compatibility and will be removed. Use PyErr_WarnEx instead. */ @@ -744,15 +783,15 @@ PyErr_WarnExplicit(PyObject *category, const char *text, const char *module_str, PyObject *registry) { PyObject *res; - PyObject *message = PyString_FromString(text); - PyObject *filename = PyString_FromString(filename_str); + PyObject *message = PyUnicode_FromString(text); + PyObject *filename = PyUnicode_DecodeFSDefault(filename_str); PyObject *module = NULL; int ret = -1; if (message == NULL || filename == NULL) goto exit; if (module_str != NULL) { - module = PyString_FromString(module_str); + module = PyUnicode_FromString(module_str); if (module == NULL) goto exit; } @@ -787,7 +826,7 @@ static PyMethodDef warnings_functions[] = { METH_VARARGS | METH_KEYWORDS, warn_explicit_doc}, /* XXX(brett.cannon): add showwarning? */ /* XXX(brett.cannon): Reasonable to add formatwarning? */ - {NULL, NULL} /* sentinel */ + {NULL, NULL} /* sentinel */ }; @@ -797,12 +836,13 @@ create_filter(PyObject *category, const char *action) static PyObject *ignore_str = NULL; static PyObject *error_str = NULL; static PyObject *default_str = NULL; + static PyObject *always_str = NULL; PyObject *action_obj = NULL; PyObject *lineno, *result; if (!strcmp(action, "ignore")) { if (ignore_str == NULL) { - ignore_str = PyString_InternFromString("ignore"); + ignore_str = PyUnicode_InternFromString("ignore"); if (ignore_str == NULL) return NULL; } @@ -810,7 +850,7 @@ create_filter(PyObject *category, const char *action) } else if (!strcmp(action, "error")) { if (error_str == NULL) { - error_str = PyString_InternFromString("error"); + error_str = PyUnicode_InternFromString("error"); if (error_str == NULL) return NULL; } @@ -818,18 +858,26 @@ create_filter(PyObject *category, const char *action) } else if (!strcmp(action, "default")) { if (default_str == NULL) { - default_str = PyString_InternFromString("default"); + default_str = PyUnicode_InternFromString("default"); if (default_str == NULL) return NULL; } action_obj = default_str; } + else if (!strcmp(action, "always")) { + if (always_str == NULL) { + always_str = PyUnicode_InternFromString("always"); + if (always_str == NULL) + return NULL; + } + action_obj = always_str; + } else { Py_FatalError("unknown action"); } /* This assumes the line number is zero for now. */ - lineno = PyInt_FromLong(0); + lineno = PyLong_FromLong(0); if (lineno == NULL) return NULL; result = PyTuple_Pack(5, action_obj, Py_None, category, Py_None, lineno); @@ -840,21 +888,16 @@ create_filter(PyObject *category, const char *action) static PyObject * init_filters(void) { - /* Don't silence DeprecationWarning if -3 or -Q were used. */ - PyObject *filters = PyList_New(Py_Py3kWarningFlag || - Py_DivisionWarningFlag ? 3 : 4); + PyObject *filters = PyList_New(5); unsigned int pos = 0; /* Post-incremented in each use. */ unsigned int x; - const char *bytes_action; + const char *bytes_action, *resource_action; if (filters == NULL) return NULL; - /* If guard changes, make sure to update 'filters' initialization above. */ - if (!Py_Py3kWarningFlag && !Py_DivisionWarningFlag) { - PyList_SET_ITEM(filters, pos++, - create_filter(PyExc_DeprecationWarning, "ignore")); - } + PyList_SET_ITEM(filters, pos++, + create_filter(PyExc_DeprecationWarning, "ignore")); PyList_SET_ITEM(filters, pos++, create_filter(PyExc_PendingDeprecationWarning, "ignore")); PyList_SET_ITEM(filters, pos++, @@ -867,7 +910,14 @@ init_filters(void) bytes_action = "ignore"; PyList_SET_ITEM(filters, pos++, create_filter(PyExc_BytesWarning, bytes_action)); - + /* resource usage warnings are enabled by default in pydebug mode */ +#ifdef Py_DEBUG + resource_action = "always"; +#else + resource_action = "ignore"; +#endif + PyList_SET_ITEM(filters, pos++, create_filter(PyExc_ResourceWarning, + resource_action)); for (x = 0; x < pos; x += 1) { if (PyList_GET_ITEM(filters, x) == NULL) { Py_DECREF(filters); @@ -878,33 +928,46 @@ init_filters(void) return filters; } +static struct PyModuleDef warningsmodule = { + PyModuleDef_HEAD_INIT, + MODULE_NAME, + warnings__doc__, + 0, + warnings_functions, + NULL, + NULL, + NULL, + NULL +}; + PyMODINIT_FUNC _PyWarnings_Init(void) { PyObject *m; - m = Py_InitModule3(MODULE_NAME, warnings_functions, warnings__doc__); + m = PyModule_Create(&warningsmodule); if (m == NULL) - return; + return NULL; _filters = init_filters(); if (_filters == NULL) - return; + return NULL; Py_INCREF(_filters); if (PyModule_AddObject(m, "filters", _filters) < 0) - return; + return NULL; _once_registry = PyDict_New(); if (_once_registry == NULL) - return; + return NULL; Py_INCREF(_once_registry); - if (PyModule_AddObject(m, "once_registry", _once_registry) < 0) - return; + if (PyModule_AddObject(m, "_onceregistry", _once_registry) < 0) + return NULL; - _default_action = PyString_FromString("default"); + _default_action = PyUnicode_FromString("default"); if (_default_action == NULL) - return; - if (PyModule_AddObject(m, "default_action", _default_action) < 0) - return; + return NULL; + if (PyModule_AddObject(m, "_defaultaction", _default_action) < 0) + return NULL; + return m; } diff --git a/Python/ast.c b/Python/ast.c index fc6f002cde..d2f063bb5b 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -7,7 +7,6 @@ #include "Python-ast.h" #include "grammar.h" #include "node.h" -#include "pyarena.h" #include "ast.h" #include "token.h" #include "parsetok.h" @@ -18,7 +17,6 @@ /* Data structure used internally */ struct compiling { char *c_encoding; /* source encoding */ - int c_future_unicode; /* __future__ unicode literals flag */ PyArena *c_arena; /* arena for allocating memeory */ const char *c_filename; /* filename */ }; @@ -31,27 +29,50 @@ static asdl_seq *ast_for_exprlist(struct compiling *, const node *, expr_context_ty); static expr_ty ast_for_testlist(struct compiling *, const node *); static stmt_ty ast_for_classdef(struct compiling *, const node *, asdl_seq *); -static expr_ty ast_for_testlist_comp(struct compiling *, const node *); /* Note different signature for ast_for_call */ static expr_ty ast_for_call(struct compiling *, const node *, expr_ty); static PyObject *parsenumber(struct compiling *, const char *); -static PyObject *parsestr(struct compiling *, const char *); -static PyObject *parsestrplus(struct compiling *, const node *n); +static PyObject *parsestr(struct compiling *, const node *n, int *bytesmode); +static PyObject *parsestrplus(struct compiling *, const node *n, + int *bytesmode); #ifndef LINENO #define LINENO(n) ((n)->n_lineno) #endif -#define COMP_GENEXP 0 -#define COMP_SETCOMP 1 +#define COMP_GENEXP 0 +#define COMP_LISTCOMP 1 +#define COMP_SETCOMP 2 static identifier -new_identifier(const char* n, PyArena *arena) { - PyObject* id = PyString_InternFromString(n); - if (id != NULL) - PyArena_AddPyObject(arena, id); +new_identifier(const char *n, PyArena *arena) +{ + PyObject *id = PyUnicode_DecodeUTF8(n, strlen(n), NULL); + Py_UNICODE *u; + if (!id) + return NULL; + u = PyUnicode_AS_UNICODE(id); + /* Check whether there are non-ASCII characters in the + identifier; if so, normalize to NFKC. */ + for (; *u; u++) { + if (*u >= 128) { + PyObject *m = PyImport_ImportModuleNoBlock("unicodedata"); + PyObject *id2; + if (!m) + return NULL; + id2 = PyObject_CallMethod(m, "normalize", "sO", "NFKC", id); + Py_DECREF(m); + if (!id2) + return NULL; + Py_DECREF(id); + id = id2; + break; + } + } + PyUnicode_InternInPlace(&id); + PyArena_AddPyObject(arena, id); return id; } @@ -68,10 +89,18 @@ new_identifier(const char* n, PyArena *arena) { static int ast_error(const node *n, const char *errstr) { - PyObject *u = Py_BuildValue("zi", errstr, LINENO(n)); + PyObject *u = Py_BuildValue("zii", errstr, LINENO(n), n->n_col_offset), *save; if (!u) return 0; + /* + * Prevent the error from being chained. PyErr_SetObject will normalize the + * exception in order to chain it. ast_error_finish, however, requires the + * error not to be normalized. + */ + save = PyThreadState_GET()->exc_value; + PyThreadState_GET()->exc_value = NULL; PyErr_SetObject(PyExc_SyntaxError, u); + PyThreadState_GET()->exc_value = save; Py_DECREF(u); return 0; } @@ -79,7 +108,8 @@ ast_error(const node *n, const char *errstr) static void ast_error_finish(const char *filename) { - PyObject *type, *value, *tback, *errstr, *loc, *tmp; + PyObject *type, *value, *tback, *errstr, *offset, *loc, *tmp; + PyObject *filename_obj; long lineno; assert(PyErr_Occurred()); @@ -91,11 +121,16 @@ ast_error_finish(const char *filename) if (!errstr) return; Py_INCREF(errstr); - lineno = PyInt_AsLong(PyTuple_GetItem(value, 1)); + lineno = PyLong_AsLong(PyTuple_GetItem(value, 1)); if (lineno == -1) { Py_DECREF(errstr); return; } + offset = PyTuple_GetItem(value, 2); + if (!offset) { + Py_DECREF(errstr); + return; + } Py_DECREF(value); loc = PyErr_ProgramText(filename, lineno); @@ -103,7 +138,16 @@ ast_error_finish(const char *filename) Py_INCREF(Py_None); loc = Py_None; } - tmp = Py_BuildValue("(zlOO)", filename, lineno, Py_None, loc); + if (filename != NULL) + filename_obj = PyUnicode_DecodeFSDefault(filename); + else { + Py_INCREF(Py_None); + filename_obj = Py_None; + } + if (filename_obj != NULL) + tmp = Py_BuildValue("(NlOO)", filename_obj, lineno, offset, loc); + else + tmp = NULL; Py_DECREF(loc); if (!tmp) { Py_DECREF(errstr); @@ -117,37 +161,6 @@ ast_error_finish(const char *filename) PyErr_Restore(type, value, tback); } -static int -ast_warn(struct compiling *c, const node *n, char *msg) -{ - if (PyErr_WarnExplicit(PyExc_SyntaxWarning, msg, c->c_filename, LINENO(n), - NULL, NULL) < 0) { - /* if -Werr, change it to a SyntaxError */ - if (PyErr_Occurred() && PyErr_ExceptionMatches(PyExc_SyntaxWarning)) - ast_error(n, msg); - return 0; - } - return 1; -} - -static int -forbidden_check(struct compiling *c, const node *n, const char *x) -{ - if (!strcmp(x, "None")) - return ast_error(n, "cannot assign to None"); - if (!strcmp(x, "__debug__")) - return ast_error(n, "cannot assign to __debug__"); - if (Py_Py3kWarningFlag) { - if (!(strcmp(x, "True") && strcmp(x, "False")) && - !ast_warn(c, n, "assignment to True or False is forbidden in 3.x")) - return 0; - if (!strcmp(x, "nonlocal") && - !ast_warn(c, n, "nonlocal is a keyword in 3.x")) - return 0; - } - return 1; -} - /* num_stmts() returns number of contained statements. Use this routine to determine how big a sequence is needed for @@ -225,16 +238,19 @@ PyAST_FromNode(const node *n, PyCompilerFlags *flags, const char *filename, if (flags && flags->cf_flags & PyCF_SOURCE_IS_UTF8) { c.c_encoding = "utf-8"; if (TYPE(n) == encoding_decl) { +#if 0 ast_error(n, "encoding declaration in Unicode string"); goto error; +#endif + n = CHILD(n, 0); } } else if (TYPE(n) == encoding_decl) { c.c_encoding = STR(n); n = CHILD(n, 0); } else { - c.c_encoding = NULL; + /* PEP 3120 */ + c.c_encoding = "utf-8"; } - c.c_future_unicode = flags && flags->cf_flags & CO_FUTURE_UNICODE_LITERALS; c.c_arena = arena; c.c_filename = filename; @@ -359,6 +375,33 @@ get_operator(const node *n) } } +static const char* FORBIDDEN[] = { + "None", + "True", + "False", + NULL, +}; + +static int +forbidden_name(identifier name, const node *n, int full_checks) +{ + assert(PyUnicode_Check(name)); + if (PyUnicode_CompareWithASCIIString(name, "__debug__") == 0) { + ast_error(n, "assignment to keyword"); + return 1; + } + if (full_checks) { + const char **p; + for (p = FORBIDDEN; *p; p++) { + if (PyUnicode_CompareWithASCIIString(name, *p) == 0) { + ast_error(n, "assignment to keyword"); + return 1; + } + } + } + return 0; +} + /* Set the context ctx for expr_ty e, recursively traversing e. Only sets context for expr kinds that "can appear in assignment context" @@ -386,18 +429,23 @@ set_context(struct compiling *c, expr_ty e, expr_context_ty ctx, const node *n) switch (e->kind) { case Attribute_kind: - if (ctx == Store && !forbidden_check(c, n, - PyBytes_AS_STRING(e->v.Attribute.attr))) - return 0; e->v.Attribute.ctx = ctx; + if (ctx == Store && forbidden_name(e->v.Attribute.attr, n, 1)) + return 0; break; case Subscript_kind: e->v.Subscript.ctx = ctx; break; + case Starred_kind: + e->v.Starred.ctx = ctx; + if (!set_context(c, e->v.Starred.value, ctx, n)) + return 0; + break; case Name_kind: - if (ctx == Store && !forbidden_check(c, n, - PyBytes_AS_STRING(e->v.Name.id))) - return 0; + if (ctx == Store) { + if (forbidden_name(e->v.Name.id, n, 1)) + return 0; /* forbidden_name() calls ast_error() */ + } e->v.Name.ctx = ctx; break; case List_kind: @@ -443,14 +491,15 @@ set_context(struct compiling *c, expr_ty e, expr_context_ty ctx, const node *n) case Set_kind: case Num_kind: case Str_kind: + case Bytes_kind: expr_name = "literal"; break; + case Ellipsis_kind: + expr_name = "Ellipsis"; + break; case Compare_kind: expr_name = "comparison"; break; - case Repr_kind: - expr_name = "repr"; - break; case IfExp_kind: expr_name = "conditional expression"; break; @@ -525,7 +574,7 @@ ast_for_augassign(struct compiling *c, const node *n) static cmpop_ty ast_for_comp_op(struct compiling *c, const node *n) { - /* comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is' + /* comp_op: '<'|'>'|'=='|'>='|'<='|'!='|'in'|'not' 'in'|'is' |'is' 'not' */ REQ(n, comp_op); @@ -577,24 +626,23 @@ ast_for_comp_op(struct compiling *c, const node *n) static asdl_seq * seq_for_testlist(struct compiling *c, const node *n) { - /* testlist: test (',' test)* [','] */ + /* testlist: test (',' test)* [','] + testlist_star_expr: test|star_expr (',' test|star_expr)* [','] + */ asdl_seq *seq; expr_ty expression; int i; - assert(TYPE(n) == testlist || - TYPE(n) == listmaker || - TYPE(n) == testlist_comp || - TYPE(n) == testlist_safe || - TYPE(n) == testlist1); + assert(TYPE(n) == testlist || TYPE(n) == testlist_star_expr || TYPE(n) == testlist_comp); seq = asdl_seq_new((NCH(n) + 1) / 2, c->c_arena); if (!seq) return NULL; for (i = 0; i < NCH(n); i += 2) { - assert(TYPE(CHILD(n, i)) == test || TYPE(CHILD(n, i)) == old_test); + const node *ch = CHILD(n, i); + assert(TYPE(ch) == test || TYPE(ch) == test_nocond || TYPE(ch) == star_expr); - expression = ast_for_expr(c, CHILD(n, i)); + expression = ast_for_expr(c, ch); if (!expression) return NULL; @@ -604,97 +652,189 @@ seq_for_testlist(struct compiling *c, const node *n) return seq; } -static expr_ty -compiler_complex_args(struct compiling *c, const node *n) +static arg_ty +ast_for_arg(struct compiling *c, const node *n) { - int i, len = (NCH(n) + 1) / 2; - expr_ty result; - asdl_seq *args = asdl_seq_new(len, c->c_arena); - if (!args) + identifier name; + expr_ty annotation = NULL; + node *ch; + + assert(TYPE(n) == tfpdef || TYPE(n) == vfpdef); + ch = CHILD(n, 0); + name = NEW_IDENTIFIER(ch); + if (!name) + return NULL; + if (forbidden_name(name, ch, 0)) return NULL; - /* fpdef: NAME | '(' fplist ')' - fplist: fpdef (',' fpdef)* [','] - */ - REQ(n, fplist); - for (i = 0; i < len; i++) { - PyObject *arg_id; - const node *fpdef_node = CHILD(n, 2*i); - const node *child; - expr_ty arg; -set_name: - /* fpdef_node is either a NAME or an fplist */ - child = CHILD(fpdef_node, 0); - if (TYPE(child) == NAME) { - if (!forbidden_check(c, n, STR(child))) - return NULL; - arg_id = NEW_IDENTIFIER(child); - if (!arg_id) - return NULL; - arg = Name(arg_id, Store, LINENO(child), child->n_col_offset, - c->c_arena); - } - else { - assert(TYPE(fpdef_node) == fpdef); - /* fpdef_node[0] is not a name, so it must be '(', get CHILD[1] */ - child = CHILD(fpdef_node, 1); - assert(TYPE(child) == fplist); - /* NCH == 1 means we have (x), we need to elide the extra parens */ - if (NCH(child) == 1) { - fpdef_node = CHILD(child, 0); - assert(TYPE(fpdef_node) == fpdef); - goto set_name; - } - arg = compiler_complex_args(c, child); - } - asdl_seq_SET(args, i, arg); + if (NCH(n) == 3 && TYPE(CHILD(n, 1)) == COLON) { + annotation = ast_for_expr(c, CHILD(n, 2)); + if (!annotation) + return NULL; } - result = Tuple(args, Store, LINENO(n), n->n_col_offset, c->c_arena); - if (!set_context(c, result, Store, n)) - return NULL; - return result; + return arg(name, annotation, c->c_arena); } +/* returns -1 if failed to handle keyword only arguments + returns new position to keep processing if successful + (',' tfpdef ['=' test])* + ^^^ + start pointing here + */ +static int +handle_keywordonly_args(struct compiling *c, const node *n, int start, + asdl_seq *kwonlyargs, asdl_seq *kwdefaults) +{ + PyObject *argname; + node *ch; + expr_ty expression, annotation; + arg_ty arg; + int i = start; + int j = 0; /* index for kwdefaults and kwonlyargs */ + + if (kwonlyargs == NULL) { + ast_error(CHILD(n, start), "named arguments must follow bare *"); + return -1; + } + assert(kwdefaults != NULL); + while (i < NCH(n)) { + ch = CHILD(n, i); + switch (TYPE(ch)) { + case vfpdef: + case tfpdef: + if (i + 1 < NCH(n) && TYPE(CHILD(n, i + 1)) == EQUAL) { + expression = ast_for_expr(c, CHILD(n, i + 2)); + if (!expression) + goto error; + asdl_seq_SET(kwdefaults, j, expression); + i += 2; /* '=' and test */ + } + else { /* setting NULL if no default value exists */ + asdl_seq_SET(kwdefaults, j, NULL); + } + if (NCH(ch) == 3) { + /* ch is NAME ':' test */ + annotation = ast_for_expr(c, CHILD(ch, 2)); + if (!annotation) + goto error; + } + else { + annotation = NULL; + } + ch = CHILD(ch, 0); + argname = NEW_IDENTIFIER(ch); + if (!argname) + goto error; + if (forbidden_name(argname, ch, 0)) + goto error; + arg = arg(argname, annotation, c->c_arena); + if (!arg) + goto error; + asdl_seq_SET(kwonlyargs, j++, arg); + i += 2; /* the name and the comma */ + break; + case DOUBLESTAR: + return i; + default: + ast_error(ch, "unexpected node"); + goto error; + } + } + return i; + error: + return -1; +} /* Create AST for argument list. */ static arguments_ty ast_for_arguments(struct compiling *c, const node *n) { - /* parameters: '(' [varargslist] ')' - varargslist: (fpdef ['=' test] ',')* ('*' NAME [',' '**' NAME] - | '**' NAME) | fpdef ['=' test] (',' fpdef ['=' test])* [','] + /* This function handles both typedargslist (function definition) + and varargslist (lambda definition). + + parameters: '(' [typedargslist] ')' + typedargslist: ((tfpdef ['=' test] ',')* + ('*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] + | '**' tfpdef) + | tfpdef ['=' test] (',' tfpdef ['=' test])* [',']) + tfpdef: NAME [':' test] + varargslist: ((vfpdef ['=' test] ',')* + ('*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] + | '**' vfpdef) + | vfpdef ['=' test] (',' vfpdef ['=' test])* [',']) + vfpdef: NAME */ - int i, j, k, n_args = 0, n_defaults = 0, found_default = 0; - asdl_seq *args, *defaults; + int i, j, k, nposargs = 0, nkwonlyargs = 0; + int nposdefaults = 0, found_default = 0; + asdl_seq *posargs, *posdefaults, *kwonlyargs, *kwdefaults; identifier vararg = NULL, kwarg = NULL; + arg_ty arg; + expr_ty varargannotation = NULL, kwargannotation = NULL; node *ch; if (TYPE(n) == parameters) { if (NCH(n) == 2) /* () as argument list */ - return arguments(NULL, NULL, NULL, NULL, c->c_arena); + return arguments(NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, c->c_arena); n = CHILD(n, 1); } - REQ(n, varargslist); + assert(TYPE(n) == typedargslist || TYPE(n) == varargslist); - /* first count the number of normal args & defaults */ + /* First count the number of positional args & defaults. The + variable i is the loop index for this for loop and the next. + The next loop picks up where the first leaves off. + */ for (i = 0; i < NCH(n); i++) { ch = CHILD(n, i); - if (TYPE(ch) == fpdef) - n_args++; - if (TYPE(ch) == EQUAL) - n_defaults++; + if (TYPE(ch) == STAR) { + /* skip star */ + i++; + if (i < NCH(n) && /* skip argument following star */ + (TYPE(CHILD(n, i)) == tfpdef || + TYPE(CHILD(n, i)) == vfpdef)) { + i++; + } + break; + } + if (TYPE(ch) == DOUBLESTAR) break; + if (TYPE(ch) == vfpdef || TYPE(ch) == tfpdef) nposargs++; + if (TYPE(ch) == EQUAL) nposdefaults++; + } + /* count the number of keyword only args & + defaults for keyword only args */ + for ( ; i < NCH(n); ++i) { + ch = CHILD(n, i); + if (TYPE(ch) == DOUBLESTAR) break; + if (TYPE(ch) == tfpdef || TYPE(ch) == vfpdef) nkwonlyargs++; } - args = (n_args ? asdl_seq_new(n_args, c->c_arena) : NULL); - if (!args && n_args) + posargs = (nposargs ? asdl_seq_new(nposargs, c->c_arena) : NULL); + if (!posargs && nposargs) + return NULL; + kwonlyargs = (nkwonlyargs ? + asdl_seq_new(nkwonlyargs, c->c_arena) : NULL); + if (!kwonlyargs && nkwonlyargs) return NULL; - defaults = (n_defaults ? asdl_seq_new(n_defaults, c->c_arena) : NULL); - if (!defaults && n_defaults) + posdefaults = (nposdefaults ? + asdl_seq_new(nposdefaults, c->c_arena) : NULL); + if (!posdefaults && nposdefaults) return NULL; + /* The length of kwonlyargs and kwdefaults are same + since we set NULL as default for keyword only argument w/o default + - we have sequence data structure, but no dictionary */ + kwdefaults = (nkwonlyargs ? + asdl_seq_new(nkwonlyargs, c->c_arena) : NULL); + if (!kwdefaults && nkwonlyargs) + return NULL; + + if (nposargs + nkwonlyargs > 255) { + ast_error(n, "more than 255 arguments"); + return NULL; + } - /* fpdef: NAME | '(' fplist ')' - fplist: fpdef (',' fpdef)* [','] + /* tfpdef: NAME [':' test] + vfpdef: NAME */ i = 0; j = 0; /* index for defaults */ @@ -702,9 +842,8 @@ ast_for_arguments(struct compiling *c, const node *n) while (i < NCH(n)) { ch = CHILD(n, i); switch (TYPE(ch)) { - case fpdef: { - int complex_args = 0, parenthesized = 0; - handle_fpdef: + case tfpdef: + case vfpdef: /* XXX Need to worry about checking if TYPE(CHILD(n, i+1)) is anything other than EQUAL or a comma? */ /* XXX Should NCH(n) check be made a separate check? */ @@ -712,81 +851,74 @@ ast_for_arguments(struct compiling *c, const node *n) expr_ty expression = ast_for_expr(c, CHILD(n, i + 2)); if (!expression) return NULL; - assert(defaults != NULL); - asdl_seq_SET(defaults, j++, expression); + assert(posdefaults != NULL); + asdl_seq_SET(posdefaults, j++, expression); i += 2; found_default = 1; } else if (found_default) { - /* def f((x)=4): pass should raise an error. - def f((x, (y))): pass will just incur the tuple unpacking warning. */ - if (parenthesized && !complex_args) { - ast_error(n, "parenthesized arg with default"); - return NULL; - } ast_error(n, "non-default argument follows default argument"); return NULL; } - if (NCH(ch) == 3) { - ch = CHILD(ch, 1); - /* def foo((x)): is not complex, special case. */ - if (NCH(ch) != 1) { - /* We have complex arguments, setup for unpacking. */ - if (Py_Py3kWarningFlag && !ast_warn(c, ch, - "tuple parameter unpacking has been removed in 3.x")) - return NULL; - complex_args = 1; - asdl_seq_SET(args, k++, compiler_complex_args(c, ch)); - if (!asdl_seq_GET(args, k-1)) - return NULL; - } else { - /* def foo((x)): setup for checking NAME below. */ - /* Loop because there can be many parens and tuple - unpacking mixed in. */ - parenthesized = 1; - ch = CHILD(ch, 0); - assert(TYPE(ch) == fpdef); - goto handle_fpdef; - } - } - if (TYPE(CHILD(ch, 0)) == NAME) { - PyObject *id; - expr_ty name; - if (!forbidden_check(c, n, STR(CHILD(ch, 0)))) - return NULL; - id = NEW_IDENTIFIER(CHILD(ch, 0)); - if (!id) - return NULL; - name = Name(id, Param, LINENO(ch), ch->n_col_offset, - c->c_arena); - if (!name) - return NULL; - asdl_seq_SET(args, k++, name); - - } - i += 2; /* the name and the comma */ - if (parenthesized && Py_Py3kWarningFlag && - !ast_warn(c, ch, "parenthesized argument names " - "are invalid in 3.x")) + arg = ast_for_arg(c, ch); + if (!arg) return NULL; - + asdl_seq_SET(posargs, k++, arg); + i += 2; /* the name and the comma */ break; - } case STAR: - if (!forbidden_check(c, CHILD(n, i+1), STR(CHILD(n, i+1)))) + if (i+1 >= NCH(n)) { + ast_error(CHILD(n, i), + "named arguments must follow bare *"); return NULL; - vararg = NEW_IDENTIFIER(CHILD(n, i+1)); - if (!vararg) - return NULL; - i += 3; + } + ch = CHILD(n, i+1); /* tfpdef or COMMA */ + if (TYPE(ch) == COMMA) { + int res = 0; + i += 2; /* now follows keyword only arguments */ + res = handle_keywordonly_args(c, n, i, + kwonlyargs, kwdefaults); + if (res == -1) return NULL; + i = res; /* res has new position to process */ + } + else { + vararg = NEW_IDENTIFIER(CHILD(ch, 0)); + if (!vararg) + return NULL; + if (forbidden_name(vararg, CHILD(ch, 0), 0)) + return NULL; + if (NCH(ch) > 1) { + /* there is an annotation on the vararg */ + varargannotation = ast_for_expr(c, CHILD(ch, 2)); + if (!varargannotation) + return NULL; + } + i += 3; + if (i < NCH(n) && (TYPE(CHILD(n, i)) == tfpdef + || TYPE(CHILD(n, i)) == vfpdef)) { + int res = 0; + res = handle_keywordonly_args(c, n, i, + kwonlyargs, kwdefaults); + if (res == -1) return NULL; + i = res; /* res has new position to process */ + } + } break; case DOUBLESTAR: - if (!forbidden_check(c, CHILD(n, i+1), STR(CHILD(n, i+1)))) - return NULL; - kwarg = NEW_IDENTIFIER(CHILD(n, i+1)); + ch = CHILD(n, i+1); /* tfpdef */ + assert(TYPE(ch) == tfpdef || TYPE(ch) == vfpdef); + kwarg = NEW_IDENTIFIER(CHILD(ch, 0)); if (!kwarg) return NULL; + if (NCH(ch) > 1) { + /* there is an annotation on the kwarg */ + kwargannotation = ast_for_expr(c, CHILD(ch, 2)); + if (!kwargannotation) + return NULL; + } + if (forbidden_name(kwarg, CHILD(ch, 0), 0)) + return NULL; i += 3; break; default: @@ -796,8 +928,8 @@ ast_for_arguments(struct compiling *c, const node *n) return NULL; } } - - return arguments(args, vararg, kwarg, defaults, c->c_arena); + return arguments(posargs, vararg, varargannotation, kwonlyargs, kwarg, + kwargannotation, posdefaults, kwdefaults, c->c_arena); } static expr_ty @@ -892,10 +1024,11 @@ ast_for_decorators(struct compiling *c, const node *n) static stmt_ty ast_for_funcdef(struct compiling *c, const node *n, asdl_seq *decorator_seq) { - /* funcdef: 'def' NAME parameters ':' suite */ + /* funcdef: 'def' NAME parameters ['->' test] ':' suite */ identifier name; arguments_ty args; asdl_seq *body; + expr_ty returns = NULL; int name_i = 1; REQ(n, funcdef); @@ -903,16 +1036,22 @@ ast_for_funcdef(struct compiling *c, const node *n, asdl_seq *decorator_seq) name = NEW_IDENTIFIER(CHILD(n, name_i)); if (!name) return NULL; - else if (!forbidden_check(c, CHILD(n, name_i), STR(CHILD(n, name_i)))) + if (forbidden_name(name, CHILD(n, name_i), 0)) return NULL; args = ast_for_arguments(c, CHILD(n, name_i + 1)); if (!args) return NULL; + if (TYPE(CHILD(n, name_i+2)) == RARROW) { + returns = ast_for_expr(c, CHILD(n, name_i + 3)); + if (!returns) + return NULL; + name_i += 2; + } body = ast_for_suite(c, CHILD(n, name_i + 3)); if (!body) return NULL; - return FunctionDef(name, args, body, decorator_seq, LINENO(n), + return FunctionDef(name, args, body, decorator_seq, returns, LINENO(n), n->n_col_offset, c->c_arena); } @@ -938,7 +1077,7 @@ ast_for_decorated(struct compiling *c, const node *n) thing = ast_for_classdef(c, CHILD(n, 1), decorator_seq); } /* we count the decorators in when talking about the class' or - function's line number */ + * function's line number */ if (thing) { thing->lineno = LINENO(n); thing->col_offset = n->n_col_offset; @@ -949,12 +1088,14 @@ ast_for_decorated(struct compiling *c, const node *n) static expr_ty ast_for_lambdef(struct compiling *c, const node *n) { - /* lambdef: 'lambda' [varargslist] ':' test */ + /* lambdef: 'lambda' [varargslist] ':' test + lambdef_nocond: 'lambda' [varargslist] ':' test_nocond */ arguments_ty args; expr_ty expression; if (NCH(n) == 3) { - args = arguments(NULL, NULL, NULL, NULL, c->c_arena); + args = arguments(NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, c->c_arena); if (!args) return NULL; expression = ast_for_expr(c, CHILD(n, 2)); @@ -993,168 +1134,6 @@ ast_for_ifexpr(struct compiling *c, const node *n) c->c_arena); } -/* XXX(nnorwitz): the listcomp and genexpr code should be refactored - so there is only a single version. Possibly for loops can also re-use - the code. -*/ - -/* Count the number of 'for' loop in a list comprehension. - - Helper for ast_for_listcomp(). -*/ - -static int -count_list_fors(struct compiling *c, const node *n) -{ - int n_fors = 0; - node *ch = CHILD(n, 1); - - count_list_for: - n_fors++; - REQ(ch, list_for); - if (NCH(ch) == 5) - ch = CHILD(ch, 4); - else - return n_fors; - count_list_iter: - REQ(ch, list_iter); - ch = CHILD(ch, 0); - if (TYPE(ch) == list_for) - goto count_list_for; - else if (TYPE(ch) == list_if) { - if (NCH(ch) == 3) { - ch = CHILD(ch, 2); - goto count_list_iter; - } - else - return n_fors; - } - - /* Should never be reached */ - PyErr_SetString(PyExc_SystemError, "logic error in count_list_fors"); - return -1; -} - -/* Count the number of 'if' statements in a list comprehension. - - Helper for ast_for_listcomp(). -*/ - -static int -count_list_ifs(struct compiling *c, const node *n) -{ - int n_ifs = 0; - - count_list_iter: - REQ(n, list_iter); - if (TYPE(CHILD(n, 0)) == list_for) - return n_ifs; - n = CHILD(n, 0); - REQ(n, list_if); - n_ifs++; - if (NCH(n) == 2) - return n_ifs; - n = CHILD(n, 2); - goto count_list_iter; -} - -static expr_ty -ast_for_listcomp(struct compiling *c, const node *n) -{ - /* listmaker: test ( list_for | (',' test)* [','] ) - list_for: 'for' exprlist 'in' testlist_safe [list_iter] - list_iter: list_for | list_if - list_if: 'if' test [list_iter] - testlist_safe: test [(',' test)+ [',']] - */ - expr_ty elt, first; - asdl_seq *listcomps; - int i, n_fors; - node *ch; - - REQ(n, listmaker); - assert(NCH(n) > 1); - - elt = ast_for_expr(c, CHILD(n, 0)); - if (!elt) - return NULL; - - n_fors = count_list_fors(c, n); - if (n_fors == -1) - return NULL; - - listcomps = asdl_seq_new(n_fors, c->c_arena); - if (!listcomps) - return NULL; - - ch = CHILD(n, 1); - for (i = 0; i < n_fors; i++) { - comprehension_ty lc; - asdl_seq *t; - expr_ty expression; - node *for_ch; - - REQ(ch, list_for); - - for_ch = CHILD(ch, 1); - t = ast_for_exprlist(c, for_ch, Store); - if (!t) - return NULL; - expression = ast_for_testlist(c, CHILD(ch, 3)); - if (!expression) - return NULL; - - /* Check the # of children rather than the length of t, since - [x for x, in ... ] has 1 element in t, but still requires a Tuple. - */ - first = (expr_ty)asdl_seq_GET(t, 0); - if (NCH(for_ch) == 1) - lc = comprehension(first, expression, NULL, c->c_arena); - else - lc = comprehension(Tuple(t, Store, first->lineno, first->col_offset, - c->c_arena), - expression, NULL, c->c_arena); - if (!lc) - return NULL; - - if (NCH(ch) == 5) { - int j, n_ifs; - asdl_seq *ifs; - expr_ty list_for_expr; - - ch = CHILD(ch, 4); - n_ifs = count_list_ifs(c, ch); - if (n_ifs == -1) - return NULL; - - ifs = asdl_seq_new(n_ifs, c->c_arena); - if (!ifs) - return NULL; - - for (j = 0; j < n_ifs; j++) { - REQ(ch, list_iter); - ch = CHILD(ch, 0); - REQ(ch, list_if); - - list_for_expr = ast_for_expr(c, CHILD(ch, 1)); - if (!list_for_expr) - return NULL; - - asdl_seq_SET(ifs, j, list_for_expr); - if (NCH(ch) == 3) - ch = CHILD(ch, 2); - } - /* on exit, must guarantee that ch is a list_for */ - if (TYPE(ch) == list_iter) - ch = CHILD(ch, 0); - lc->ifs = ifs; - } - asdl_seq_SET(listcomps, i, lc); - } - - return ListComp(elt, listcomps, LINENO(n), n->n_col_offset, c->c_arena); -} - /* Count the number of 'for' loops in a comprehension. @@ -1296,6 +1275,8 @@ ast_for_comprehension(struct compiling *c, const node *n) static expr_ty ast_for_itercomp(struct compiling *c, const node *n, int type) { + /* testlist_comp: test ( comp_for | (',' test)* [','] ) + argument: [test '='] test [comp_for] # Really [keyword '='] test */ expr_ty elt; asdl_seq *comps; @@ -1311,6 +1292,8 @@ ast_for_itercomp(struct compiling *c, const node *n, int type) if (type == COMP_GENEXP) return GeneratorExp(elt, comps, LINENO(n), n->n_col_offset, c->c_arena); + else if (type == COMP_LISTCOMP) + return ListComp(elt, comps, LINENO(n), n->n_col_offset, c->c_arena); else if (type == COMP_SETCOMP) return SetComp(elt, comps, LINENO(n), n->n_col_offset, c->c_arena); else @@ -1330,7 +1313,6 @@ ast_for_dictcomp(struct compiling *c, const node *n) key = ast_for_expr(c, CHILD(n, 0)); if (!key) return NULL; - value = ast_for_expr(c, CHILD(n, 2)); if (!value) return NULL; @@ -1350,19 +1332,29 @@ ast_for_genexp(struct compiling *c, const node *n) } static expr_ty +ast_for_listcomp(struct compiling *c, const node *n) +{ + assert(TYPE(n) == (testlist_comp)); + return ast_for_itercomp(c, n, COMP_LISTCOMP); +} + +static expr_ty ast_for_setcomp(struct compiling *c, const node *n) { assert(TYPE(n) == (dictorsetmaker)); return ast_for_itercomp(c, n, COMP_SETCOMP); } + static expr_ty ast_for_atom(struct compiling *c, const node *n) { - /* atom: '(' [yield_expr|testlist_comp] ')' | '[' [listmaker] ']' - | '{' [dictmaker] '}' | '`' testlist '`' | NAME | NUMBER | STRING+ + /* atom: '(' [yield_expr|testlist_comp] ')' | '[' [testlist_comp] ']' + | '{' [dictmaker|testlist_comp] '}' | NAME | NUMBER | STRING+ + | '...' | 'None' | 'True' | 'False' */ node *ch = CHILD(n, 0); + int bytesmode = 0; switch (TYPE(ch)) { case NAME: { @@ -1374,17 +1366,16 @@ ast_for_atom(struct compiling *c, const node *n) return Name(name, Load, LINENO(n), n->n_col_offset, c->c_arena); } case STRING: { - PyObject *str = parsestrplus(c, n); + PyObject *str = parsestrplus(c, n, &bytesmode); if (!str) { -#ifdef Py_USING_UNICODE - if (PyErr_ExceptionMatches(PyExc_UnicodeError)){ + if (PyErr_ExceptionMatches(PyExc_UnicodeError)) { PyObject *type, *value, *tback, *errstr; PyErr_Fetch(&type, &value, &tback); errstr = PyObject_Str(value); if (errstr) { char *s = ""; char buf[128]; - s = PyString_AsString(errstr); + s = _PyUnicode_AsString(errstr); PyOS_snprintf(buf, sizeof(buf), "(unicode error) %s", s); ast_error(n, buf); Py_DECREF(errstr); @@ -1395,11 +1386,13 @@ ast_for_atom(struct compiling *c, const node *n) Py_DECREF(value); Py_XDECREF(tback); } -#endif return NULL; } PyArena_AddPyObject(c->c_arena, str); - return Str(str, LINENO(n), n->n_col_offset, c->c_arena); + if (bytesmode) + return Bytes(str, LINENO(n), n->n_col_offset, c->c_arena); + else + return Str(str, LINENO(n), n->n_col_offset, c->c_arena); } case NUMBER: { PyObject *pynum = parsenumber(c, STR(ch)); @@ -1409,6 +1402,8 @@ ast_for_atom(struct compiling *c, const node *n) PyArena_AddPyObject(c->c_arena, pynum); return Num(pynum, LINENO(n), n->n_col_offset, c->c_arena); } + case ELLIPSIS: /* Ellipsis */ + return Ellipsis(LINENO(n), n->n_col_offset, c->c_arena); case LPAR: /* some parenthesized expressions */ ch = CHILD(n, 1); @@ -1418,14 +1413,18 @@ ast_for_atom(struct compiling *c, const node *n) if (TYPE(ch) == yield_expr) return ast_for_expr(c, ch); - return ast_for_testlist_comp(c, ch); + /* testlist_comp: test ( comp_for | (',' test)* [','] ) */ + if ((NCH(ch) > 1) && (TYPE(CHILD(ch, 1)) == comp_for)) + return ast_for_genexp(c, ch); + + return ast_for_testlist(c, ch); case LSQB: /* list (or list comprehension) */ ch = CHILD(n, 1); if (TYPE(ch) == RSQB) return List(NULL, Load, LINENO(n), n->n_col_offset, c->c_arena); - REQ(ch, listmaker); + REQ(ch, testlist_comp); if (NCH(ch) == 1 || TYPE(CHILD(ch, 1)) == COMMA) { asdl_seq *elts = seq_for_testlist(c, ch); if (!elts) @@ -1436,10 +1435,8 @@ ast_for_atom(struct compiling *c, const node *n) else return ast_for_listcomp(c, ch); case LBRACE: { - /* dictorsetmaker: - * (test ':' test (comp_for | (',' test ':' test)* [','])) | - * (test (comp_for | (',' test)* [','])) - */ + /* dictorsetmaker: test ':' test (',' test ':' test)* [','] | + * test (gen_for | (',' test)* [',']) */ int i, size; asdl_seq *keys, *values; @@ -1496,17 +1493,6 @@ ast_for_atom(struct compiling *c, const node *n) return Dict(keys, values, LINENO(n), n->n_col_offset, c->c_arena); } } - case BACKQUOTE: { /* repr */ - expr_ty expression; - if (Py_Py3kWarningFlag && - !ast_warn(c, n, "backquote not supported in 3.x; use repr()")) - return NULL; - expression = ast_for_testlist(c, CHILD(n, 1)); - if (!expression) - return NULL; - - return Repr(expression, LINENO(n), n->n_col_offset, c->c_arena); - } default: PyErr_Format(PyExc_SystemError, "unhandled atom %d", TYPE(ch)); return NULL; @@ -1522,13 +1508,10 @@ ast_for_slice(struct compiling *c, const node *n) REQ(n, subscript); /* - subscript: '.' '.' '.' | test | [test] ':' [test] [sliceop] + subscript: test | [test] ':' [test] [sliceop] sliceop: ':' [test] */ ch = CHILD(n, 0); - if (TYPE(ch) == DOT) - return Ellipsis(c->c_arena); - if (NCH(n) == 1 && TYPE(ch) == test) { /* 'step' variable hold no significance in terms of being used over other vars */ @@ -1568,21 +1551,7 @@ ast_for_slice(struct compiling *c, const node *n) ch = CHILD(n, NCH(n) - 1); if (TYPE(ch) == sliceop) { - if (NCH(ch) == 1) { - /* - This is an extended slice (ie "x[::]") with no expression in the - step field. We set this literally to "None" in order to - disambiguate it from x[:]. (The interpreter might have to call - __getslice__ for x[:], but it must call __getitem__ for x[::].) - */ - identifier none = new_identifier("None", c->c_arena); - if (!none) - return NULL; - ch = CHILD(ch, 0); - step = Name(none, Load, LINENO(ch), ch->n_col_offset, c->c_arena); - if (!step) - return NULL; - } else { + if (NCH(ch) != 1) { ch = CHILD(ch, 1); if (TYPE(ch) == test) { step = ast_for_expr(c, ch); @@ -1598,53 +1567,53 @@ ast_for_slice(struct compiling *c, const node *n) static expr_ty ast_for_binop(struct compiling *c, const node *n) { - /* Must account for a sequence of expressions. - How should A op B op C by represented? - BinOp(BinOp(A, op, B), op, C). - */ + /* Must account for a sequence of expressions. + How should A op B op C by represented? + BinOp(BinOp(A, op, B), op, C). + */ - int i, nops; - expr_ty expr1, expr2, result; - operator_ty newoperator; + int i, nops; + expr_ty expr1, expr2, result; + operator_ty newoperator; - expr1 = ast_for_expr(c, CHILD(n, 0)); - if (!expr1) - return NULL; + expr1 = ast_for_expr(c, CHILD(n, 0)); + if (!expr1) + return NULL; - expr2 = ast_for_expr(c, CHILD(n, 2)); - if (!expr2) - return NULL; + expr2 = ast_for_expr(c, CHILD(n, 2)); + if (!expr2) + return NULL; - newoperator = get_operator(CHILD(n, 1)); - if (!newoperator) - return NULL; + newoperator = get_operator(CHILD(n, 1)); + if (!newoperator) + return NULL; - result = BinOp(expr1, newoperator, expr2, LINENO(n), n->n_col_offset, - c->c_arena); - if (!result) - return NULL; + result = BinOp(expr1, newoperator, expr2, LINENO(n), n->n_col_offset, + c->c_arena); + if (!result) + return NULL; - nops = (NCH(n) - 1) / 2; - for (i = 1; i < nops; i++) { - expr_ty tmp_result, tmp; - const node* next_oper = CHILD(n, i * 2 + 1); + nops = (NCH(n) - 1) / 2; + for (i = 1; i < nops; i++) { + expr_ty tmp_result, tmp; + const node* next_oper = CHILD(n, i * 2 + 1); - newoperator = get_operator(next_oper); - if (!newoperator) - return NULL; + newoperator = get_operator(next_oper); + if (!newoperator) + return NULL; - tmp = ast_for_expr(c, CHILD(n, i * 2 + 2)); - if (!tmp) - return NULL; + tmp = ast_for_expr(c, CHILD(n, i * 2 + 2)); + if (!tmp) + return NULL; - tmp_result = BinOp(result, newoperator, tmp, - LINENO(next_oper), next_oper->n_col_offset, - c->c_arena); - if (!tmp_result) - return NULL; - result = tmp_result; - } - return result; + tmp_result = BinOp(result, newoperator, tmp, + LINENO(next_oper), next_oper->n_col_offset, + c->c_arena); + if (!tmp_result) + return NULL; + result = tmp_result; + } + return result; } static expr_ty @@ -1688,7 +1657,7 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr) int j; slice_ty slc; expr_ty e; - bool simple = true; + int simple = 1; asdl_seq *slices, *elts; slices = asdl_seq_new((NCH(n) + 1) / 2, c->c_arena); if (!slices) @@ -1698,7 +1667,7 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr) if (!slc) return NULL; if (slc->kind != Index_kind) - simple = false; + simple = 0; asdl_seq_SET(slices, j / 2, slc); } if (!simple) { @@ -1726,39 +1695,8 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr) static expr_ty ast_for_factor(struct compiling *c, const node *n) { - node *pfactor, *ppower, *patom, *pnum; expr_ty expression; - /* If the unary - operator is applied to a constant, don't generate - a UNARY_NEGATIVE opcode. Just store the approriate value as a - constant. The peephole optimizer already does something like - this but it doesn't handle the case where the constant is - (sys.maxint - 1). In that case, we want a PyIntObject, not a - PyLongObject. - */ - if (TYPE(CHILD(n, 0)) == MINUS && - NCH(n) == 2 && - TYPE((pfactor = CHILD(n, 1))) == factor && - NCH(pfactor) == 1 && - TYPE((ppower = CHILD(pfactor, 0))) == power && - NCH(ppower) == 1 && - TYPE((patom = CHILD(ppower, 0))) == atom && - TYPE((pnum = CHILD(patom, 0))) == NUMBER) { - PyObject *pynum; - char *s = PyObject_MALLOC(strlen(STR(pnum)) + 2); - if (s == NULL) - return NULL; - s[0] = '-'; - strcpy(s + 1, STR(pnum)); - pynum = parsenumber(c, s); - PyObject_FREE(s); - if (!pynum) - return NULL; - - PyArena_AddPyObject(c->c_arena, pynum); - return Num(pynum, LINENO(n), n->n_col_offset, c->c_arena); - } - expression = ast_for_expr(c, CHILD(n, 1)); if (!expression) return NULL; @@ -1815,6 +1753,21 @@ ast_for_power(struct compiling *c, const node *n) return e; } +static expr_ty +ast_for_starred(struct compiling *c, const node *n) +{ + expr_ty tmp; + REQ(n, star_expr); + + tmp = ast_for_expr(c, CHILD(n, 1)); + if (!tmp) + return NULL; + + /* The Load context is changed later. */ + return Starred(tmp, Load, LINENO(n), n->n_col_offset, c->c_arena); +} + + /* Do not name a variable 'expr'! Will cause a compile error. */ @@ -1823,6 +1776,7 @@ ast_for_expr(struct compiling *c, const node *n) { /* handle the full range of simple expressions test: or_test ['if' or_test 'else' test] | lambdef + test_nocond: or_test | lambdef_nocond or_test: and_test ('or' and_test)* and_test: not_test ('and' not_test)* not_test: 'not' not_test | comparison @@ -1835,15 +1789,6 @@ ast_for_expr(struct compiling *c, const node *n) term: factor (('*'|'/'|'%'|'//') factor)* factor: ('+'|'-'|'~') factor | power power: atom trailer* ('**' factor)* - - As well as modified versions that exist for backward compatibility, - to explicitly allow: - [ x for x in lambda: 0, lambda: 1 ] - (which would be ambiguous without these extra rules) - - old_test: or_test | old_lambdef - old_lambdef: 'lambda' [vararglist] ':' old_test - */ asdl_seq *seq; @@ -1852,9 +1797,9 @@ ast_for_expr(struct compiling *c, const node *n) loop: switch (TYPE(n)) { case test: - case old_test: + case test_nocond: if (TYPE(CHILD(n, 0)) == lambdef || - TYPE(CHILD(n, 0)) == old_lambdef) + TYPE(CHILD(n, 0)) == lambdef_nocond) return ast_for_lambdef(c, CHILD(n, 0)); else if (NCH(n) > 1) return ast_for_ifexpr(c, n); @@ -1934,6 +1879,8 @@ ast_for_expr(struct compiling *c, const node *n) } break; + case star_expr: + return ast_for_starred(c, n); /* The next five cases all handle BinOps. The main body of code is the same in each case, but the switch turned inside out to reuse the code for each type of operator. @@ -2051,9 +1998,8 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func) } else { keyword_ty kw; - identifier key; + identifier key, tmp; int k; - char *tmp; /* CHILD(ch, 0) is test, but must be an identifier? */ e = ast_for_expr(c, CHILD(ch, 0)); @@ -2065,20 +2011,18 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func) * then is very confusing. */ if (e->kind == Lambda_kind) { - ast_error(CHILD(ch, 0), - "lambda cannot contain assignment"); - return NULL; + ast_error(CHILD(ch, 0), "lambda cannot contain assignment"); + return NULL; } else if (e->kind != Name_kind) { - ast_error(CHILD(ch, 0), "keyword can't be an expression"); - return NULL; + ast_error(CHILD(ch, 0), "keyword can't be an expression"); + return NULL; + } else if (forbidden_name(e->v.Name.id, ch, 1)) { + return NULL; } key = e->v.Name.id; - if (!forbidden_check(c, CHILD(ch, 0), PyBytes_AS_STRING(key))) - return NULL; for (k = 0; k < nkeywords; k++) { - tmp = PyString_AS_STRING( - ((keyword_ty)asdl_seq_GET(keywords, k))->arg); - if (!strcmp(tmp, PyString_AS_STRING(key))) { + tmp = ((keyword_ty)asdl_seq_GET(keywords, k))->arg; + if (!PyUnicode_Compare(tmp, key)) { ast_error(CHILD(ch, 0), "keyword argument repeated"); return NULL; } @@ -2106,17 +2050,14 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func) } } - return Call(func, args, keywords, vararg, kwarg, func->lineno, - func->col_offset, c->c_arena); + return Call(func, args, keywords, vararg, kwarg, func->lineno, func->col_offset, c->c_arena); } static expr_ty ast_for_testlist(struct compiling *c, const node* n) { - /* testlist_comp: test (',' test)* [','] */ + /* testlist_comp: test (comp_for | (',' test)* [',']) */ /* testlist: test (',' test)* [','] */ - /* testlist_safe: test (',' test)+ [','] */ - /* testlist1: test (',' test)* */ assert(NCH(n) > 0); if (TYPE(n) == testlist_comp) { if (NCH(n) > 1) @@ -2124,8 +2065,7 @@ ast_for_testlist(struct compiling *c, const node* n) } else { assert(TYPE(n) == testlist || - TYPE(n) == testlist_safe || - TYPE(n) == testlist1); + TYPE(n) == testlist_star_expr); } if (NCH(n) == 1) return ast_for_expr(c, CHILD(n, 0)); @@ -2137,46 +2077,13 @@ ast_for_testlist(struct compiling *c, const node* n) } } -static expr_ty -ast_for_testlist_comp(struct compiling *c, const node* n) -{ - /* testlist_comp: test ( comp_for | (',' test)* [','] ) */ - /* argument: test [ comp_for ] */ - assert(TYPE(n) == testlist_comp || TYPE(n) == argument); - if (NCH(n) > 1 && TYPE(CHILD(n, 1)) == comp_for) - return ast_for_genexp(c, n); - return ast_for_testlist(c, n); -} - -/* like ast_for_testlist() but returns a sequence */ -static asdl_seq* -ast_for_class_bases(struct compiling *c, const node* n) -{ - /* testlist: test (',' test)* [','] */ - assert(NCH(n) > 0); - REQ(n, testlist); - if (NCH(n) == 1) { - expr_ty base; - asdl_seq *bases = asdl_seq_new(1, c->c_arena); - if (!bases) - return NULL; - base = ast_for_expr(c, CHILD(n, 0)); - if (!base) - return NULL; - asdl_seq_SET(bases, 0, base); - return bases; - } - - return seq_for_testlist(c, n); -} - static stmt_ty ast_for_expr_stmt(struct compiling *c, const node *n) { REQ(n, expr_stmt); - /* expr_stmt: testlist (augassign (yield_expr|testlist) + /* expr_stmt: testlist_star_expr (augassign (yield_expr|testlist) | ('=' (yield_expr|testlist))*) - testlist: test (',' test)* [','] + testlist_star_expr: (test|star_expr) (',' test|star_expr)* [','] augassign: '+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' | '<<=' | '>>=' | '**=' | '//=' test: ... here starts the operator precendence dance @@ -2225,8 +2132,7 @@ ast_for_expr_stmt(struct compiling *c, const node *n) if (!newoperator) return NULL; - return AugAssign(expr1, newoperator, expr2, LINENO(n), n->n_col_offset, - c->c_arena); + return AugAssign(expr1, newoperator, expr2, LINENO(n), n->n_col_offset, c->c_arena); } else { int i; @@ -2248,59 +2154,25 @@ ast_for_expr_stmt(struct compiling *c, const node *n) } e = ast_for_testlist(c, ch); if (!e) - return NULL; + return NULL; /* set context to assign */ if (!set_context(c, e, Store, CHILD(n, i))) - return NULL; + return NULL; asdl_seq_SET(targets, i / 2, e); } value = CHILD(n, NCH(n) - 1); - if (TYPE(value) == testlist) + if (TYPE(value) == testlist_star_expr) expression = ast_for_testlist(c, value); else expression = ast_for_expr(c, value); if (!expression) return NULL; - return Assign(targets, expression, LINENO(n), n->n_col_offset, - c->c_arena); + return Assign(targets, expression, LINENO(n), n->n_col_offset, c->c_arena); } } -static stmt_ty -ast_for_print_stmt(struct compiling *c, const node *n) -{ - /* print_stmt: 'print' ( [ test (',' test)* [','] ] - | '>>' test [ (',' test)+ [','] ] ) - */ - expr_ty dest = NULL, expression; - asdl_seq *seq = NULL; - bool nl; - int i, j, values_count, start = 1; - - REQ(n, print_stmt); - if (NCH(n) >= 2 && TYPE(CHILD(n, 1)) == RIGHTSHIFT) { - dest = ast_for_expr(c, CHILD(n, 2)); - if (!dest) - return NULL; - start = 4; - } - values_count = (NCH(n) + 1 - start) / 2; - if (values_count) { - seq = asdl_seq_new(values_count, c->c_arena); - if (!seq) - return NULL; - for (i = start, j = 0; i < NCH(n); i += 2, ++j) { - expression = ast_for_expr(c, CHILD(n, i)); - if (!expression) - return NULL; - asdl_seq_SET(seq, j, expression); - } - } - nl = (TYPE(CHILD(n, NCH(n) - 1)) == COMMA) ? false : true; - return Print(dest, seq, nl, LINENO(n), n->n_col_offset, c->c_arena); -} static asdl_seq * ast_for_exprlist(struct compiling *c, const node *n, expr_context_ty context) @@ -2374,48 +2246,22 @@ ast_for_flow_stmt(struct compiling *c, const node *n) expr_ty expression = ast_for_testlist(c, CHILD(ch, 1)); if (!expression) return NULL; - return Return(expression, LINENO(n), n->n_col_offset, - c->c_arena); + return Return(expression, LINENO(n), n->n_col_offset, c->c_arena); } case raise_stmt: if (NCH(ch) == 1) - return Raise(NULL, NULL, NULL, LINENO(n), n->n_col_offset, - c->c_arena); - else if (NCH(ch) == 2) { + return Raise(NULL, NULL, LINENO(n), n->n_col_offset, c->c_arena); + else if (NCH(ch) >= 2) { + expr_ty cause = NULL; expr_ty expression = ast_for_expr(c, CHILD(ch, 1)); if (!expression) return NULL; - return Raise(expression, NULL, NULL, LINENO(n), - n->n_col_offset, c->c_arena); - } - else if (NCH(ch) == 4) { - expr_ty expr1, expr2; - - expr1 = ast_for_expr(c, CHILD(ch, 1)); - if (!expr1) - return NULL; - expr2 = ast_for_expr(c, CHILD(ch, 3)); - if (!expr2) - return NULL; - - return Raise(expr1, expr2, NULL, LINENO(n), n->n_col_offset, - c->c_arena); - } - else if (NCH(ch) == 6) { - expr_ty expr1, expr2, expr3; - - expr1 = ast_for_expr(c, CHILD(ch, 1)); - if (!expr1) - return NULL; - expr2 = ast_for_expr(c, CHILD(ch, 3)); - if (!expr2) - return NULL; - expr3 = ast_for_expr(c, CHILD(ch, 5)); - if (!expr3) - return NULL; - - return Raise(expr1, expr2, expr3, LINENO(n), n->n_col_offset, - c->c_arena); + if (NCH(ch) == 4) { + cause = ast_for_expr(c, CHILD(ch, 3)); + if (!cause) + return NULL; + } + return Raise(expression, cause, LINENO(n), n->n_col_offset, c->c_arena); } default: PyErr_Format(PyExc_SystemError, @@ -2435,28 +2281,28 @@ alias_for_import_name(struct compiling *c, const node *n, int store) dotted_as_name: dotted_name ['as' NAME] dotted_name: NAME ('.' NAME)* */ - PyObject *str, *name; + identifier str, name; loop: switch (TYPE(n)) { - case import_as_name: { + case import_as_name: { node *name_node = CHILD(n, 0); str = NULL; + name = NEW_IDENTIFIER(name_node); + if (!name) + return NULL; if (NCH(n) == 3) { node *str_node = CHILD(n, 2); - if (store && !forbidden_check(c, str_node, STR(str_node))) - return NULL; str = NEW_IDENTIFIER(str_node); if (!str) return NULL; + if (store && forbidden_name(str, str_node, 0)) + return NULL; } else { - if (!forbidden_check(c, name_node, STR(name_node))) + if (forbidden_name(name, name_node, 0)) return NULL; } - name = NEW_IDENTIFIER(name_node); - if (!name) - return NULL; return alias(name, str, c->c_arena); } case dotted_as_name: @@ -2470,22 +2316,22 @@ alias_for_import_name(struct compiling *c, const node *n, int store) if (!a) return NULL; assert(!a->asname); - if (!forbidden_check(c, asname_node, STR(asname_node))) - return NULL; a->asname = NEW_IDENTIFIER(asname_node); if (!a->asname) return NULL; + if (forbidden_name(a->asname, asname_node, 0)) + return NULL; return a; } break; case dotted_name: if (NCH(n) == 1) { node *name_node = CHILD(n, 0); - if (store && !forbidden_check(c, name_node, STR(name_node))) - return NULL; name = NEW_IDENTIFIER(name_node); if (!name) return NULL; + if (store && forbidden_name(name, name_node, 0)) + return NULL; return alias(name, NULL, c->c_arena); } else { @@ -2493,16 +2339,17 @@ alias_for_import_name(struct compiling *c, const node *n, int store) int i; size_t len; char *s; + PyObject *uni; len = 0; for (i = 0; i < NCH(n); i += 2) /* length of string plus one for the dot */ len += strlen(STR(CHILD(n, i))) + 1; len--; /* the last name doesn't have a dot */ - str = PyString_FromStringAndSize(NULL, len); + str = PyBytes_FromStringAndSize(NULL, len); if (!str) return NULL; - s = PyString_AS_STRING(str); + s = PyBytes_AS_STRING(str); if (!s) return NULL; for (i = 0; i < NCH(n); i += 2) { @@ -2513,13 +2360,20 @@ alias_for_import_name(struct compiling *c, const node *n, int store) } --s; *s = '\0'; - PyString_InternInPlace(&str); + uni = PyUnicode_DecodeUTF8(PyBytes_AS_STRING(str), + PyBytes_GET_SIZE(str), + NULL); + Py_DECREF(str); + if (!uni) + return NULL; + str = uni; + PyUnicode_InternInPlace(&str); PyArena_AddPyObject(c->c_arena, str); return alias(str, NULL, c->c_arena); } break; case STAR: - str = PyString_InternFromString("*"); + str = PyUnicode_InternFromString("*"); PyArena_AddPyObject(c->c_arena, str); return alias(str, NULL, c->c_arena); default: @@ -2538,8 +2392,8 @@ ast_for_import_stmt(struct compiling *c, const node *n) /* import_stmt: import_name | import_from import_name: 'import' dotted_as_names - import_from: 'from' ('.'* dotted_name | '.') 'import' - ('*' | '(' import_as_names ')' | import_as_names) + import_from: 'from' (('.' | '...')* dotted_name | ('.' | '...')+) + 'import' ('*' | '(' import_as_names ')' | import_as_names) */ int lineno; int col_offset; @@ -2555,7 +2409,7 @@ ast_for_import_stmt(struct compiling *c, const node *n) REQ(n, dotted_as_names); aliases = asdl_seq_new((NCH(n) + 1) / 2, c->c_arena); if (!aliases) - return NULL; + return NULL; for (i = 0; i < NCH(n); i += 2) { alias_ty import_alias = alias_for_import_name(c, CHILD(n, i), 1); if (!import_alias) @@ -2579,6 +2433,10 @@ ast_for_import_stmt(struct compiling *c, const node *n) return NULL; idx++; break; + } else if (TYPE(CHILD(n, idx)) == ELLIPSIS) { + /* three consecutive dots are tokenized as one ELLIPSIS */ + ndots += 3; + continue; } else if (TYPE(CHILD(n, idx)) != DOT) { break; } @@ -2663,35 +2521,24 @@ ast_for_global_stmt(struct compiling *c, const node *n) } static stmt_ty -ast_for_exec_stmt(struct compiling *c, const node *n) +ast_for_nonlocal_stmt(struct compiling *c, const node *n) { - expr_ty expr1, globals = NULL, locals = NULL; - int n_children = NCH(n); - if (n_children != 2 && n_children != 4 && n_children != 6) { - PyErr_Format(PyExc_SystemError, - "poorly formed 'exec' statement: %d parts to statement", - n_children); - return NULL; - } + /* nonlocal_stmt: 'nonlocal' NAME (',' NAME)* */ + identifier name; + asdl_seq *s; + int i; - /* exec_stmt: 'exec' expr ['in' test [',' test]] */ - REQ(n, exec_stmt); - expr1 = ast_for_expr(c, CHILD(n, 1)); - if (!expr1) + REQ(n, nonlocal_stmt); + s = asdl_seq_new(NCH(n) / 2, c->c_arena); + if (!s) return NULL; - if (n_children >= 4) { - globals = ast_for_expr(c, CHILD(n, 3)); - if (!globals) - return NULL; - } - if (n_children == 6) { - locals = ast_for_expr(c, CHILD(n, 5)); - if (!locals) + for (i = 1; i < NCH(n); i += 2) { + name = NEW_IDENTIFIER(CHILD(n, i)); + if (!name) return NULL; + asdl_seq_SET(s, i / 2, name); } - - return Exec(expr1, globals, locals, LINENO(n), n->n_col_offset, - c->c_arena); + return Nonlocal(s, LINENO(n), n->n_col_offset, c->c_arena); } static stmt_ty @@ -2703,8 +2550,7 @@ ast_for_assert_stmt(struct compiling *c, const node *n) expr_ty expression = ast_for_expr(c, CHILD(n, 1)); if (!expression) return NULL; - return Assert(expression, NULL, LINENO(n), n->n_col_offset, - c->c_arena); + return Assert(expression, NULL, LINENO(n), n->n_col_offset, c->c_arena); } else if (NCH(n) == 4) { expr_ty expr1, expr2; @@ -2926,8 +2772,7 @@ ast_for_while_stmt(struct compiling *c, const node *n) suite_seq = ast_for_suite(c, CHILD(n, 3)); if (!suite_seq) return NULL; - return While(expression, suite_seq, NULL, LINENO(n), n->n_col_offset, - c->c_arena); + return While(expression, suite_seq, NULL, LINENO(n), n->n_col_offset, c->c_arena); } else if (NCH(n) == 7) { expr_ty expression; @@ -2943,8 +2788,7 @@ ast_for_while_stmt(struct compiling *c, const node *n) if (!seq2) return NULL; - return While(expression, seq1, seq2, LINENO(n), n->n_col_offset, - c->c_arena); + return While(expression, seq1, seq2, LINENO(n), n->n_col_offset, c->c_arena); } PyErr_Format(PyExc_SystemError, @@ -2995,7 +2839,7 @@ ast_for_for_stmt(struct compiling *c, const node *n) static excepthandler_ty ast_for_except_clause(struct compiling *c, const node *exc, node *body) { - /* except_clause: 'except' [test [(',' | 'as') test]] */ + /* except_clause: 'except' [test ['as' test]] */ REQ(exc, except_clause); REQ(body, suite); @@ -3024,10 +2868,10 @@ ast_for_except_clause(struct compiling *c, const node *exc, node *body) else if (NCH(exc) == 4) { asdl_seq *suite_seq; expr_ty expression; - expr_ty e = ast_for_expr(c, CHILD(exc, 3)); + identifier e = NEW_IDENTIFIER(CHILD(exc, 3)); if (!e) return NULL; - if (!set_context(c, e, Store, CHILD(exc, 3))) + if (forbidden_name(e, CHILD(exc, 3), 0)) return NULL; expression = ast_for_expr(c, CHILD(exc, 1)); if (!expression) @@ -3187,50 +3031,64 @@ ast_for_with_stmt(struct compiling *c, const node *n) static stmt_ty ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq) { - /* classdef: 'class' NAME ['(' testlist ')'] ':' suite */ + /* classdef: 'class' NAME ['(' arglist ')'] ':' suite */ PyObject *classname; - asdl_seq *bases, *s; + asdl_seq *s; + expr_ty call; REQ(n, classdef); - if (!forbidden_check(c, n, STR(CHILD(n, 1)))) - return NULL; - - if (NCH(n) == 4) { + if (NCH(n) == 4) { /* class NAME ':' suite */ s = ast_for_suite(c, CHILD(n, 3)); if (!s) return NULL; classname = NEW_IDENTIFIER(CHILD(n, 1)); if (!classname) return NULL; - return ClassDef(classname, NULL, s, decorator_seq, LINENO(n), - n->n_col_offset, c->c_arena); + if (forbidden_name(classname, CHILD(n, 3), 0)) + return NULL; + return ClassDef(classname, NULL, NULL, NULL, NULL, s, decorator_seq, + LINENO(n), n->n_col_offset, c->c_arena); } - /* check for empty base list */ - if (TYPE(CHILD(n,3)) == RPAR) { + + if (TYPE(CHILD(n, 3)) == RPAR) { /* class NAME '(' ')' ':' suite */ s = ast_for_suite(c, CHILD(n,5)); if (!s) return NULL; classname = NEW_IDENTIFIER(CHILD(n, 1)); if (!classname) return NULL; - return ClassDef(classname, NULL, s, decorator_seq, LINENO(n), - n->n_col_offset, c->c_arena); + if (forbidden_name(classname, CHILD(n, 3), 0)) + return NULL; + return ClassDef(classname, NULL, NULL, NULL, NULL, s, decorator_seq, + LINENO(n), n->n_col_offset, c->c_arena); } - /* else handle the base class list */ - bases = ast_for_class_bases(c, CHILD(n, 3)); - if (!bases) - return NULL; - + /* class NAME '(' arglist ')' ':' suite */ + /* build up a fake Call node so we can extract its pieces */ + { + PyObject *dummy_name; + expr_ty dummy; + dummy_name = NEW_IDENTIFIER(CHILD(n, 1)); + if (!dummy_name) + return NULL; + dummy = Name(dummy_name, Load, LINENO(n), n->n_col_offset, c->c_arena); + call = ast_for_call(c, CHILD(n, 3), dummy); + if (!call) + return NULL; + } s = ast_for_suite(c, CHILD(n, 6)); if (!s) return NULL; classname = NEW_IDENTIFIER(CHILD(n, 1)); if (!classname) return NULL; - return ClassDef(classname, bases, s, decorator_seq, - LINENO(n), n->n_col_offset, c->c_arena); + if (forbidden_name(classname, CHILD(n, 1), 0)) + return NULL; + + return ClassDef(classname, call->v.Call.args, call->v.Call.keywords, + call->v.Call.starargs, call->v.Call.kwargs, s, + decorator_seq, LINENO(n), n->n_col_offset, c->c_arena); } static stmt_ty @@ -3246,15 +3104,12 @@ ast_for_stmt(struct compiling *c, const node *n) } if (TYPE(n) == small_stmt) { n = CHILD(n, 0); - /* small_stmt: expr_stmt | print_stmt | del_stmt | pass_stmt - | flow_stmt | import_stmt | global_stmt | exec_stmt - | assert_stmt + /* small_stmt: expr_stmt | del_stmt | pass_stmt | flow_stmt + | import_stmt | global_stmt | nonlocal_stmt | assert_stmt */ switch (TYPE(n)) { case expr_stmt: return ast_for_expr_stmt(c, n); - case print_stmt: - return ast_for_print_stmt(c, n); case del_stmt: return ast_for_del_stmt(c, n); case pass_stmt: @@ -3265,8 +3120,8 @@ ast_for_stmt(struct compiling *c, const node *n) return ast_for_import_stmt(c, n); case global_stmt: return ast_for_global_stmt(c, n); - case exec_stmt: - return ast_for_exec_stmt(c, n); + case nonlocal_stmt: + return ast_for_nonlocal_stmt(c, n); case assert_stmt: return ast_for_assert_stmt(c, n); default: @@ -3311,253 +3166,256 @@ ast_for_stmt(struct compiling *c, const node *n) static PyObject * parsenumber(struct compiling *c, const char *s) { - const char *end; - long x; - double dx; -#ifndef WITHOUT_COMPLEX - Py_complex complex; - int imflag; -#endif - - assert(s != NULL); - errno = 0; - end = s + strlen(s) - 1; -#ifndef WITHOUT_COMPLEX - imflag = *end == 'j' || *end == 'J'; -#endif - if (*end == 'l' || *end == 'L') - return PyLong_FromString((char *)s, (char **)0, 0); - x = PyOS_strtol((char *)s, (char **)&end, 0); - if (*end == '\0') { - if (errno != 0) - return PyLong_FromString((char *)s, (char **)0, 0); - return PyInt_FromLong(x); - } - /* XXX Huge floats may silently fail */ -#ifndef WITHOUT_COMPLEX - if (imflag) { - complex.real = 0.; - complex.imag = PyOS_string_to_double(s, (char **)&end, NULL); - if (complex.imag == -1.0 && PyErr_Occurred()) - return NULL; - return PyComplex_FromCComplex(complex); - } - else -#endif - { - dx = PyOS_string_to_double(s, NULL, NULL); - if (dx == -1.0 && PyErr_Occurred()) - return NULL; - return PyFloat_FromDouble(dx); + const char *end; + long x; + double dx; + Py_complex compl; + int imflag; + + assert(s != NULL); + errno = 0; + end = s + strlen(s) - 1; + imflag = *end == 'j' || *end == 'J'; + if (s[0] == '0') { + x = (long) PyOS_strtoul((char *)s, (char **)&end, 0); + if (x < 0 && errno == 0) { + return PyLong_FromString((char *)s, + (char **)0, + 0); } + } + else + x = PyOS_strtol((char *)s, (char **)&end, 0); + if (*end == '\0') { + if (errno != 0) + return PyLong_FromString((char *)s, (char **)0, 0); + return PyLong_FromLong(x); + } + /* XXX Huge floats may silently fail */ + if (imflag) { + compl.real = 0.; + compl.imag = PyOS_string_to_double(s, (char **)&end, NULL); + if (compl.imag == -1.0 && PyErr_Occurred()) + return NULL; + return PyComplex_FromCComplex(compl); + } + else + { + dx = PyOS_string_to_double(s, NULL, NULL); + if (dx == -1.0 && PyErr_Occurred()) + return NULL; + return PyFloat_FromDouble(dx); + } } static PyObject * decode_utf8(struct compiling *c, const char **sPtr, const char *end, char* encoding) { -#ifndef Py_USING_UNICODE - Py_FatalError("decode_utf8 should not be called in this build."); + PyObject *u, *v; + char *s, *t; + t = s = (char *)*sPtr; + /* while (s < end && *s != '\\') s++; */ /* inefficient for u".." */ + while (s < end && (*s & 0x80)) s++; + *sPtr = s; + u = PyUnicode_DecodeUTF8(t, s - t, NULL); + if (u == NULL) return NULL; -#else - PyObject *u, *v; - char *s, *t; - t = s = (char *)*sPtr; - /* while (s < end && *s != '\\') s++; */ /* inefficient for u".." */ - while (s < end && (*s & 0x80)) s++; - *sPtr = s; - u = PyUnicode_DecodeUTF8(t, s - t, NULL); - if (u == NULL) - return NULL; - v = PyUnicode_AsEncodedString(u, encoding, NULL); - Py_DECREF(u); - return v; -#endif + v = PyUnicode_AsEncodedString(u, encoding, NULL); + Py_DECREF(u); + return v; } -#ifdef Py_USING_UNICODE static PyObject * decode_unicode(struct compiling *c, const char *s, size_t len, int rawmode, const char *encoding) { - PyObject *v; - PyObject *u = NULL; - char *buf; - char *p; - const char *end; - if (encoding != NULL && strcmp(encoding, "iso-8859-1")) { - /* check for integer overflow */ - if (len > PY_SIZE_MAX / 6) - return NULL; - /* "<C3><A4>" (2 bytes) may become "\U000000E4" (10 bytes), or 1:5 - "\ä" (3 bytes) may become "\u005c\U000000E4" (16 bytes), or ~1:6 */ - u = PyString_FromStringAndSize((char *)NULL, len * 6); - if (u == NULL) - return NULL; - p = buf = PyString_AsString(u); - end = s + len; - while (s < end) { - if (*s == '\\') { - *p++ = *s++; - if (*s & 0x80) { - strcpy(p, "u005c"); - p += 5; - } - } - if (*s & 0x80) { /* XXX inefficient */ - PyObject *w; - char *r; - Py_ssize_t rn, i; - w = decode_utf8(c, &s, end, "utf-32-be"); - if (w == NULL) { - Py_DECREF(u); - return NULL; - } - r = PyString_AsString(w); - rn = PyString_Size(w); - assert(rn % 4 == 0); - for (i = 0; i < rn; i += 4) { - sprintf(p, "\\U%02x%02x%02x%02x", - r[i + 0] & 0xFF, - r[i + 1] & 0xFF, - r[i + 2] & 0xFF, - r[i + 3] & 0xFF); - p += 10; - } - Py_DECREF(w); - } else { - *p++ = *s++; - } + PyObject *v, *u; + char *buf; + char *p; + const char *end; + + if (encoding == NULL) { + buf = (char *)s; + u = NULL; + } else { + /* check for integer overflow */ + if (len > PY_SIZE_MAX / 6) + return NULL; + /* "ä" (2 bytes) may become "\U000000E4" (10 bytes), or 1:5 + "\ä" (3 bytes) may become "\u005c\U000000E4" (16 bytes), or ~1:6 */ + u = PyBytes_FromStringAndSize((char *)NULL, len * 6); + if (u == NULL) + return NULL; + p = buf = PyBytes_AsString(u); + end = s + len; + while (s < end) { + if (*s == '\\') { + *p++ = *s++; + if (*s & 0x80) { + strcpy(p, "u005c"); + p += 5; + } + } + if (*s & 0x80) { /* XXX inefficient */ + PyObject *w; + char *r; + Py_ssize_t rn, i; + w = decode_utf8(c, &s, end, "utf-32-be"); + if (w == NULL) { + Py_DECREF(u); + return NULL; + } + r = PyBytes_AS_STRING(w); + rn = Py_SIZE(w); + assert(rn % 4 == 0); + for (i = 0; i < rn; i += 4) { + sprintf(p, "\\U%02x%02x%02x%02x", + r[i + 0] & 0xFF, + r[i + 1] & 0xFF, + r[i + 2] & 0xFF, + r[i + 3] & 0xFF); + p += 10; } - len = p - buf; - s = buf; + /* Should be impossible to overflow */ + assert(p - buf <= Py_SIZE(u)); + Py_DECREF(w); + } else { + *p++ = *s++; + } } - if (rawmode) - v = PyUnicode_DecodeRawUnicodeEscape(s, len, NULL); - else - v = PyUnicode_DecodeUnicodeEscape(s, len, NULL); - Py_XDECREF(u); - return v; + len = p - buf; + s = buf; + } + if (rawmode) + v = PyUnicode_DecodeRawUnicodeEscape(s, len, NULL); + else + v = PyUnicode_DecodeUnicodeEscape(s, len, NULL); + Py_XDECREF(u); + return v; } -#endif /* s is a Python string literal, including the bracketing quote characters, - * and r &/or u prefixes (if any), and embedded escape sequences (if any). + * and r &/or b prefixes (if any), and embedded escape sequences (if any). * parsestr parses it, and returns the decoded Python string object. */ static PyObject * -parsestr(struct compiling *c, const char *s) -{ - size_t len; - int quote = Py_CHARMASK(*s); - int rawmode = 0; - int need_encoding; - int unicode = c->c_future_unicode; - - if (isalpha(quote) || quote == '_') { - if (quote == 'u' || quote == 'U') { - quote = *++s; - unicode = 1; - } - if (quote == 'b' || quote == 'B') { - quote = *++s; - unicode = 0; - } - if (quote == 'r' || quote == 'R') { - quote = *++s; - rawmode = 1; - } +parsestr(struct compiling *c, const node *n, int *bytesmode) +{ + size_t len; + const char *s = STR(n); + int quote = Py_CHARMASK(*s); + int rawmode = 0; + int need_encoding; + if (isalpha(quote)) { + if (quote == 'b' || quote == 'B') { + quote = *++s; + *bytesmode = 1; } - if (quote != '\'' && quote != '\"') { - PyErr_BadInternalCall(); - return NULL; + if (quote == 'r' || quote == 'R') { + quote = *++s; + rawmode = 1; } - s++; - len = strlen(s); - if (len > INT_MAX) { - PyErr_SetString(PyExc_OverflowError, - "string to parse is too long"); - return NULL; + } + if (quote != '\'' && quote != '\"') { + PyErr_BadInternalCall(); + return NULL; + } + s++; + len = strlen(s); + if (len > INT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "string to parse is too long"); + return NULL; + } + if (s[--len] != quote) { + PyErr_BadInternalCall(); + return NULL; + } + if (len >= 4 && s[0] == quote && s[1] == quote) { + s += 2; + len -= 2; + if (s[--len] != quote || s[--len] != quote) { + PyErr_BadInternalCall(); + return NULL; } - if (s[--len] != quote) { - PyErr_BadInternalCall(); + } + if (!*bytesmode && !rawmode) { + return decode_unicode(c, s, len, rawmode, c->c_encoding); + } + if (*bytesmode) { + /* Disallow non-ascii characters (but not escapes) */ + const char *c; + for (c = s; *c; c++) { + if (Py_CHARMASK(*c) >= 0x80) { + ast_error(n, "bytes can only contain ASCII " + "literal characters."); return NULL; + } } - if (len >= 4 && s[0] == quote && s[1] == quote) { - s += 2; - len -= 2; - if (s[--len] != quote || s[--len] != quote) { - PyErr_BadInternalCall(); - return NULL; - } - } -#ifdef Py_USING_UNICODE - if (unicode || Py_UnicodeFlag) { - return decode_unicode(c, s, len, rawmode, c->c_encoding); - } -#endif - need_encoding = (c->c_encoding != NULL && - strcmp(c->c_encoding, "utf-8") != 0 && - strcmp(c->c_encoding, "iso-8859-1") != 0); - if (rawmode || strchr(s, '\\') == NULL) { - if (need_encoding) { -#ifndef Py_USING_UNICODE - /* This should not happen - we never see any other - encoding. */ - Py_FatalError( - "cannot deal with encodings in this build."); -#else - PyObject *v, *u = PyUnicode_DecodeUTF8(s, len, NULL); - if (u == NULL) - return NULL; - v = PyUnicode_AsEncodedString(u, c->c_encoding, NULL); - Py_DECREF(u); - return v; -#endif - } else { - return PyString_FromStringAndSize(s, len); - } + } + need_encoding = (!*bytesmode && c->c_encoding != NULL && + strcmp(c->c_encoding, "utf-8") != 0); + if (rawmode || strchr(s, '\\') == NULL) { + if (need_encoding) { + PyObject *v, *u = PyUnicode_DecodeUTF8(s, len, NULL); + if (u == NULL || !*bytesmode) + return u; + v = PyUnicode_AsEncodedString(u, c->c_encoding, NULL); + Py_DECREF(u); + return v; + } else if (*bytesmode) { + return PyBytes_FromStringAndSize(s, len); + } else if (strcmp(c->c_encoding, "utf-8") == 0) { + return PyUnicode_FromStringAndSize(s, len); + } else { + return PyUnicode_DecodeLatin1(s, len, NULL); } - - return PyString_DecodeEscape(s, len, NULL, unicode, - need_encoding ? c->c_encoding : NULL); + } + return PyBytes_DecodeEscape(s, len, NULL, 1, + need_encoding ? c->c_encoding : NULL); } -/* Build a Python string object out of a STRING atom. This takes care of +/* Build a Python string object out of a STRING+ atom. This takes care of * compile-time literal catenation, calling parsestr() on each piece, and * pasting the intermediate results together. */ static PyObject * -parsestrplus(struct compiling *c, const node *n) +parsestrplus(struct compiling *c, const node *n, int *bytesmode) { - PyObject *v; - int i; - REQ(CHILD(n, 0), STRING); - if ((v = parsestr(c, STR(CHILD(n, 0)))) != NULL) { - /* String literal concatenation */ - for (i = 1; i < NCH(n); i++) { - PyObject *s; - s = parsestr(c, STR(CHILD(n, i))); - if (s == NULL) - goto onError; - if (PyString_Check(v) && PyString_Check(s)) { - PyString_ConcatAndDel(&v, s); - if (v == NULL) - goto onError; - } -#ifdef Py_USING_UNICODE - else { - PyObject *temp = PyUnicode_Concat(v, s); - Py_DECREF(s); - Py_DECREF(v); - v = temp; - if (v == NULL) - goto onError; - } -#endif - } + PyObject *v; + int i; + REQ(CHILD(n, 0), STRING); + v = parsestr(c, CHILD(n, 0), bytesmode); + if (v != NULL) { + /* String literal concatenation */ + for (i = 1; i < NCH(n); i++) { + PyObject *s; + int subbm = 0; + s = parsestr(c, CHILD(n, i), &subbm); + if (s == NULL) + goto onError; + if (*bytesmode != subbm) { + ast_error(n, "cannot mix bytes and nonbytes literals"); + Py_DECREF(s); + goto onError; + } + if (PyBytes_Check(v) && PyBytes_Check(s)) { + PyBytes_ConcatAndDel(&v, s); + if (v == NULL) + goto onError; + } + else { + PyObject *temp = PyUnicode_Concat(v, s); + Py_DECREF(s); + Py_DECREF(v); + v = temp; + if (v == NULL) + goto onError; + } } - return v; + } + return v; - onError: - Py_XDECREF(v); - return NULL; + onError: + Py_XDECREF(v); + return NULL; } diff --git a/Python/atof.c b/Python/atof.c deleted file mode 100644 index 562fcd688e..0000000000 --- a/Python/atof.c +++ /dev/null @@ -1,50 +0,0 @@ - -/* Just in case you haven't got an atof() around... - This one doesn't check for bad syntax or overflow, - and is slow and inaccurate. - But it's good enough for the occasional string literal... */ - -#include "pyconfig.h" - -#include <ctype.h> - -double atof(char *s) -{ - double a = 0.0; - int e = 0; - int c; - while ((c = *s++) != '\0' && isdigit(c)) { - a = a*10.0 + (c - '0'); - } - if (c == '.') { - while ((c = *s++) != '\0' && isdigit(c)) { - a = a*10.0 + (c - '0'); - e = e-1; - } - } - if (c == 'e' || c == 'E') { - int sign = 1; - int i = 0; - c = *s++; - if (c == '+') - c = *s++; - else if (c == '-') { - c = *s++; - sign = -1; - } - while (isdigit(c)) { - i = i*10 + (c - '0'); - c = *s++; - } - e += i*sign; - } - while (e > 0) { - a *= 10.0; - e--; - } - while (e < 0) { - a *= 0.1; - e++; - } - return a; -} diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index d25b14c061..35b2cdc4b2 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -5,32 +5,174 @@ #include "node.h" #include "code.h" -#include "eval.h" #include <ctype.h> -#include <float.h> /* for DBL_MANT_DIG and friends */ -#ifdef RISCOS -#include "unixstuff.h" +#ifdef HAVE_LANGINFO_H +#include <langinfo.h> /* CODESET */ #endif /* The default encoding used by the platform file system APIs Can remain NULL for all platforms that don't have such a concept + + Don't forget to modify PyUnicode_DecodeFSDefault() if you touch any of the + values for Py_FileSystemDefaultEncoding! */ #if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T) const char *Py_FileSystemDefaultEncoding = "mbcs"; +int Py_HasFileSystemDefaultEncoding = 1; #elif defined(__APPLE__) const char *Py_FileSystemDefaultEncoding = "utf-8"; +int Py_HasFileSystemDefaultEncoding = 1; +#elif defined(HAVE_LANGINFO_H) && defined(CODESET) +const char *Py_FileSystemDefaultEncoding = NULL; /* set by initfsencoding() */ +int Py_HasFileSystemDefaultEncoding = 0; #else -const char *Py_FileSystemDefaultEncoding = NULL; /* use default */ +const char *Py_FileSystemDefaultEncoding = "utf-8"; +int Py_HasFileSystemDefaultEncoding = 1; #endif -/* Forward */ -static PyObject *filterstring(PyObject *, PyObject *); -#ifdef Py_USING_UNICODE -static PyObject *filterunicode(PyObject *, PyObject *); -#endif -static PyObject *filtertuple (PyObject *, PyObject *); +static PyObject * +builtin___build_class__(PyObject *self, PyObject *args, PyObject *kwds) +{ + PyObject *func, *name, *bases, *mkw, *meta, *winner, *prep, *ns, *cell; + PyObject *cls = NULL; + Py_ssize_t nargs; + int isclass; + + assert(args != NULL); + if (!PyTuple_Check(args)) { + PyErr_SetString(PyExc_TypeError, + "__build_class__: args is not a tuple"); + return NULL; + } + nargs = PyTuple_GET_SIZE(args); + if (nargs < 2) { + PyErr_SetString(PyExc_TypeError, + "__build_class__: not enough arguments"); + return NULL; + } + func = PyTuple_GET_ITEM(args, 0); /* Better be callable */ + name = PyTuple_GET_ITEM(args, 1); + if (!PyUnicode_Check(name)) { + PyErr_SetString(PyExc_TypeError, + "__build_class__: name is not a string"); + return NULL; + } + bases = PyTuple_GetSlice(args, 2, nargs); + if (bases == NULL) + return NULL; + + if (kwds == NULL) { + meta = NULL; + mkw = NULL; + } + else { + mkw = PyDict_Copy(kwds); /* Don't modify kwds passed in! */ + if (mkw == NULL) { + Py_DECREF(bases); + return NULL; + } + meta = PyDict_GetItemString(mkw, "metaclass"); + if (meta != NULL) { + Py_INCREF(meta); + if (PyDict_DelItemString(mkw, "metaclass") < 0) { + Py_DECREF(meta); + Py_DECREF(mkw); + Py_DECREF(bases); + return NULL; + } + /* metaclass is explicitly given, check if it's indeed a class */ + isclass = PyType_Check(meta); + } + } + if (meta == NULL) { + /* if there are no bases, use type: */ + if (PyTuple_GET_SIZE(bases) == 0) { + meta = (PyObject *) (&PyType_Type); + } + /* else get the type of the first base */ + else { + PyObject *base0 = PyTuple_GET_ITEM(bases, 0); + meta = (PyObject *) (base0->ob_type); + } + Py_INCREF(meta); + isclass = 1; /* meta is really a class */ + } + if (isclass) { + /* meta is really a class, so check for a more derived + metaclass, or possible metaclass conflicts: */ + winner = (PyObject *)_PyType_CalculateMetaclass((PyTypeObject *)meta, + bases); + if (winner == NULL) { + Py_DECREF(meta); + Py_XDECREF(mkw); + Py_DECREF(bases); + return NULL; + } + if (winner != meta) { + Py_DECREF(meta); + meta = winner; + Py_INCREF(meta); + } + } + /* else: meta is not a class, so we cannot do the metaclass + calculation, so we will use the explicitly given object as it is */ + prep = PyObject_GetAttrString(meta, "__prepare__"); + if (prep == NULL) { + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + ns = PyDict_New(); + } + else { + Py_DECREF(meta); + Py_XDECREF(mkw); + Py_DECREF(bases); + return NULL; + } + } + else { + PyObject *pargs = PyTuple_Pack(2, name, bases); + if (pargs == NULL) { + Py_DECREF(prep); + Py_DECREF(meta); + Py_XDECREF(mkw); + Py_DECREF(bases); + return NULL; + } + ns = PyEval_CallObjectWithKeywords(prep, pargs, mkw); + Py_DECREF(pargs); + Py_DECREF(prep); + } + if (ns == NULL) { + Py_DECREF(meta); + Py_XDECREF(mkw); + Py_DECREF(bases); + return NULL; + } + cell = PyObject_CallFunctionObjArgs(func, ns, NULL); + if (cell != NULL) { + PyObject *margs; + margs = PyTuple_Pack(3, name, bases, ns); + if (margs != NULL) { + cls = PyEval_CallObjectWithKeywords(meta, margs, mkw); + Py_DECREF(margs); + } + if (cls != NULL && PyCell_Check(cell)) + PyCell_Set(cell, cls); + Py_DECREF(cell); + } + Py_DECREF(ns); + Py_DECREF(meta); + Py_XDECREF(mkw); + Py_DECREF(bases); + return cls; +} + +PyDoc_STRVAR(build_class_doc, +"__build_class__(func, name, *bases, metaclass=None, **kwds) -> class\n\ +\n\ +Internal helper function used by the class statement."); static PyObject * builtin___import__(PyObject *self, PyObject *args, PyObject *kwds) @@ -165,52 +307,18 @@ PyDoc_STRVAR(any_doc, Return True if bool(x) is True for any x in the iterable."); static PyObject * -builtin_apply(PyObject *self, PyObject *args) +builtin_ascii(PyObject *self, PyObject *v) { - PyObject *func, *alist = NULL, *kwdict = NULL; - PyObject *t = NULL, *retval = NULL; - - if (PyErr_WarnPy3k("apply() not supported in 3.x; " - "use func(*args, **kwargs)", 1) < 0) - return NULL; - - if (!PyArg_UnpackTuple(args, "apply", 1, 3, &func, &alist, &kwdict)) - return NULL; - if (alist != NULL) { - if (!PyTuple_Check(alist)) { - if (!PySequence_Check(alist)) { - PyErr_Format(PyExc_TypeError, - "apply() arg 2 expected sequence, found %s", - alist->ob_type->tp_name); - return NULL; - } - t = PySequence_Tuple(alist); - if (t == NULL) - return NULL; - alist = t; - } - } - if (kwdict != NULL && !PyDict_Check(kwdict)) { - PyErr_Format(PyExc_TypeError, - "apply() arg 3 expected dictionary, found %s", - kwdict->ob_type->tp_name); - goto finally; - } - retval = PyEval_CallObjectWithKeywords(func, alist, kwdict); - finally: - Py_XDECREF(t); - return retval; + return PyObject_ASCII(v); } -PyDoc_STRVAR(apply_doc, -"apply(object[, args[, kwargs]]) -> value\n\ -\n\ -Call a callable object with positional arguments taken from the tuple args,\n\ -and keyword arguments taken from the optional dictionary kwargs.\n\ -Note that classes are callable, as are instances with a __call__() method.\n\ +PyDoc_STRVAR(ascii_doc, +"ascii(object) -> string\n\ \n\ -Deprecated since release 2.3. Instead, use the extended call syntax:\n\ - function(*args, **keywords)."); +As repr(), return a string containing a printable representation of an\n\ +object, but escape the non-ASCII characters in the string returned by\n\ +repr() using \\x, \\u or \\U escapes. This generates a string similar\n\ +to that returned by repr() in Python 2."); static PyObject * @@ -222,7 +330,7 @@ builtin_bin(PyObject *self, PyObject *v) PyDoc_STRVAR(bin_doc, "bin(number) -> string\n\ \n\ -Return the binary representation of an integer or long integer."); +Return the binary representation of an integer."); static PyObject * @@ -235,126 +343,149 @@ PyDoc_STRVAR(callable_doc, "callable(object) -> bool\n\ \n\ Return whether the object is callable (i.e., some kind of function).\n\ -Note that classes are callable, as are instances with a __call__() method."); +Note that classes are callable, as are instances of classes with a\n\ +__call__() method."); + +typedef struct { + PyObject_HEAD + PyObject *func; + PyObject *it; +} filterobject; static PyObject * -builtin_filter(PyObject *self, PyObject *args) +filter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - PyObject *func, *seq, *result, *it, *arg; - Py_ssize_t len; /* guess for result list size */ - register Py_ssize_t j; + PyObject *func, *seq; + PyObject *it; + filterobject *lz; - if (!PyArg_UnpackTuple(args, "filter", 2, 2, &func, &seq)) + if (type == &PyFilter_Type && !_PyArg_NoKeywords("filter()", kwds)) return NULL; - /* Strings and tuples return a result of the same type. */ - if (PyString_Check(seq)) - return filterstring(func, seq); -#ifdef Py_USING_UNICODE - if (PyUnicode_Check(seq)) - return filterunicode(func, seq); -#endif - if (PyTuple_Check(seq)) - return filtertuple(func, seq); - - /* Pre-allocate argument list tuple. */ - arg = PyTuple_New(1); - if (arg == NULL) + if (!PyArg_UnpackTuple(args, "filter", 2, 2, &func, &seq)) return NULL; /* Get iterator. */ it = PyObject_GetIter(seq); if (it == NULL) - goto Fail_arg; - - /* Guess a result list size. */ - len = _PyObject_LengthHint(seq, 8); - if (len == -1) - goto Fail_it; + return NULL; - /* Get a result list. */ - if (PyList_Check(seq) && seq->ob_refcnt == 1) { - /* Eww - can modify the list in-place. */ - Py_INCREF(seq); - result = seq; - } - else { - result = PyList_New(len); - if (result == NULL) - goto Fail_it; + /* create filterobject structure */ + lz = (filterobject *)type->tp_alloc(type, 0); + if (lz == NULL) { + Py_DECREF(it); + return NULL; } + Py_INCREF(func); + lz->func = func; + lz->it = it; - /* Build the result list. */ - j = 0; - for (;;) { - PyObject *item; - int ok; + return (PyObject *)lz; +} - item = PyIter_Next(it); - if (item == NULL) { - if (PyErr_Occurred()) - goto Fail_result_it; - break; - } +static void +filter_dealloc(filterobject *lz) +{ + PyObject_GC_UnTrack(lz); + Py_XDECREF(lz->func); + Py_XDECREF(lz->it); + Py_TYPE(lz)->tp_free(lz); +} + +static int +filter_traverse(filterobject *lz, visitproc visit, void *arg) +{ + Py_VISIT(lz->it); + Py_VISIT(lz->func); + return 0; +} + +static PyObject * +filter_next(filterobject *lz) +{ + PyObject *item; + PyObject *it = lz->it; + long ok; + PyObject *(*iternext)(PyObject *); - if (func == (PyObject *)&PyBool_Type || func == Py_None) { + iternext = *Py_TYPE(it)->tp_iternext; + for (;;) { + item = iternext(it); + if (item == NULL) + return NULL; + + if (lz->func == Py_None || lz->func == (PyObject *)&PyBool_Type) { ok = PyObject_IsTrue(item); - } - else { + } else { PyObject *good; - PyTuple_SET_ITEM(arg, 0, item); - good = PyObject_Call(func, arg, NULL); - PyTuple_SET_ITEM(arg, 0, NULL); + good = PyObject_CallFunctionObjArgs(lz->func, + item, NULL); if (good == NULL) { Py_DECREF(item); - goto Fail_result_it; + return NULL; } ok = PyObject_IsTrue(good); Py_DECREF(good); } - if (ok > 0) { - if (j < len) - PyList_SET_ITEM(result, j, item); - else { - int status = PyList_Append(result, item); - Py_DECREF(item); - if (status < 0) - goto Fail_result_it; - } - ++j; - } - else { - Py_DECREF(item); - if (ok < 0) - goto Fail_result_it; - } + if (ok > 0) + return item; + Py_DECREF(item); + if (ok < 0) + return NULL; } - - - /* Cut back result list if len is too big. */ - if (j < len && PyList_SetSlice(result, j, len, NULL) < 0) - goto Fail_result_it; - - Py_DECREF(it); - Py_DECREF(arg); - return result; - -Fail_result_it: - Py_DECREF(result); -Fail_it: - Py_DECREF(it); -Fail_arg: - Py_DECREF(arg); - return NULL; } PyDoc_STRVAR(filter_doc, -"filter(function or None, sequence) -> list, tuple, or string\n" -"\n" -"Return those items of sequence for which function(item) is true. If\n" -"function is None, return the items that are true. If sequence is a tuple\n" -"or string, return the same type, else return a list."); +"filter(function or None, iterable) --> filter object\n\ +\n\ +Return an iterator yielding those items of iterable for which function(item)\n\ +is true. If function is None, return the items that are true."); + +PyTypeObject PyFilter_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "filter", /* tp_name */ + sizeof(filterobject), /* tp_basicsize */ + 0, /* tp_itemsize */ + /* methods */ + (destructor)filter_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_BASETYPE, /* tp_flags */ + filter_doc, /* tp_doc */ + (traverseproc)filter_traverse, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + PyObject_SelfIter, /* tp_iter */ + (iternextfunc)filter_next, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + PyType_GenericAlloc, /* tp_alloc */ + filter_new, /* tp_new */ + PyObject_GC_Del, /* tp_free */ +}; + static PyObject * builtin_format(PyObject *self, PyObject *args) @@ -362,7 +493,7 @@ builtin_format(PyObject *self, PyObject *args) PyObject *value; PyObject *format_spec = NULL; - if (!PyArg_ParseTuple(args, "O|O:format", &value, &format_spec)) + if (!PyArg_ParseTuple(args, "O|U:format", &value, &format_spec)) return NULL; return PyObject_Format(value, format_spec); @@ -377,123 +508,100 @@ format_spec defaults to \"\""); static PyObject * builtin_chr(PyObject *self, PyObject *args) { - long x; - char s[1]; - - if (!PyArg_ParseTuple(args, "l:chr", &x)) - return NULL; - if (x < 0 || x >= 256) { - PyErr_SetString(PyExc_ValueError, - "chr() arg not in range(256)"); - return NULL; - } - s[0] = (char)x; - return PyString_FromStringAndSize(s, 1); -} - -PyDoc_STRVAR(chr_doc, -"chr(i) -> character\n\ -\n\ -Return a string of one character with ordinal i; 0 <= i < 256."); - - -#ifdef Py_USING_UNICODE -static PyObject * -builtin_unichr(PyObject *self, PyObject *args) -{ int x; - if (!PyArg_ParseTuple(args, "i:unichr", &x)) + if (!PyArg_ParseTuple(args, "i:chr", &x)) return NULL; return PyUnicode_FromOrdinal(x); } -PyDoc_STRVAR(unichr_doc, -"unichr(i) -> Unicode character\n\ +PyDoc_VAR(chr_doc) = PyDoc_STR( +"chr(i) -> Unicode character\n\ \n\ -Return a Unicode string of one character with ordinal i; 0 <= i <= 0x10ffff."); +Return a Unicode string of one character with ordinal i; 0 <= i <= 0x10ffff." +) +#ifndef Py_UNICODE_WIDE +PyDoc_STR( +"\nIf 0x10000 <= i, a surrogate pair is returned." +) #endif +; -static PyObject * -builtin_cmp(PyObject *self, PyObject *args) -{ - PyObject *a, *b; - int c; - - if (!PyArg_UnpackTuple(args, "cmp", 2, 2, &a, &b)) - return NULL; - if (PyObject_Cmp(a, b, &c) < 0) - return NULL; - return PyInt_FromLong((long)c); -} - -PyDoc_STRVAR(cmp_doc, -"cmp(x, y) -> integer\n\ -\n\ -Return negative if x<y, zero if x==y, positive if x>y."); - - -static PyObject * -builtin_coerce(PyObject *self, PyObject *args) +static char * +source_as_string(PyObject *cmd, char *funcname, char *what, PyCompilerFlags *cf) { - PyObject *v, *w; - PyObject *res; + char *str; + Py_ssize_t size; - if (PyErr_WarnPy3k("coerce() not supported in 3.x", 1) < 0) + if (PyUnicode_Check(cmd)) { + cf->cf_flags |= PyCF_IGNORE_COOKIE; + cmd = _PyUnicode_AsDefaultEncodedString(cmd, NULL); + if (cmd == NULL) + return NULL; + } + else if (!PyObject_CheckReadBuffer(cmd)) { + PyErr_Format(PyExc_TypeError, + "%s() arg 1 must be a %s object", + funcname, what); return NULL; - - if (!PyArg_UnpackTuple(args, "coerce", 2, 2, &v, &w)) + } + if (PyObject_AsReadBuffer(cmd, (const void **)&str, &size) < 0) { return NULL; - if (PyNumber_Coerce(&v, &w) < 0) + } + if (strlen(str) != size) { + PyErr_SetString(PyExc_TypeError, + "source code string cannot contain null bytes"); return NULL; - res = PyTuple_Pack(2, v, w); - Py_DECREF(v); - Py_DECREF(w); - return res; + } + return str; } -PyDoc_STRVAR(coerce_doc, -"coerce(x, y) -> (x1, y1)\n\ -\n\ -Return a tuple consisting of the two numeric arguments converted to\n\ -a common type, using the same rules as used by arithmetic operations.\n\ -If coercion is not possible, raise TypeError."); - static PyObject * builtin_compile(PyObject *self, PyObject *args, PyObject *kwds) { char *str; + PyObject *filename_obj; char *filename; char *startstr; int mode = -1; int dont_inherit = 0; int supplied_flags = 0; + int optimize = -1; int is_ast; PyCompilerFlags cf; - PyObject *result = NULL, *cmd, *tmp = NULL; - Py_ssize_t length; + PyObject *cmd; static char *kwlist[] = {"source", "filename", "mode", "flags", - "dont_inherit", NULL}; + "dont_inherit", "optimize", NULL}; int start[] = {Py_file_input, Py_eval_input, Py_single_input}; + PyObject *result; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "Oss|ii:compile", - kwlist, &cmd, &filename, &startstr, - &supplied_flags, &dont_inherit)) + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO&s|iii:compile", kwlist, + &cmd, + PyUnicode_FSConverter, &filename_obj, + &startstr, &supplied_flags, + &dont_inherit, &optimize)) return NULL; - cf.cf_flags = supplied_flags; + filename = PyBytes_AS_STRING(filename_obj); + cf.cf_flags = supplied_flags | PyCF_SOURCE_IS_UTF8; if (supplied_flags & ~(PyCF_MASK | PyCF_MASK_OBSOLETE | PyCF_DONT_IMPLY_DEDENT | PyCF_ONLY_AST)) { PyErr_SetString(PyExc_ValueError, "compile(): unrecognised flags"); - return NULL; + goto error; } /* XXX Warn if (supplied_flags & PyCF_MASK_OBSOLETE) != 0? */ + if (optimize < -1 || optimize > 2) { + PyErr_SetString(PyExc_ValueError, + "compile(): invalid optimize value"); + goto error; + } + if (!dont_inherit) { PyEval_MergeCompilerFlags(&cf); } @@ -507,12 +615,12 @@ builtin_compile(PyObject *self, PyObject *args, PyObject *kwds) else { PyErr_SetString(PyExc_ValueError, "compile() arg 3 must be 'exec', 'eval' or 'single'"); - return NULL; + goto error; } is_ast = PyAST_Check(cmd); if (is_ast == -1) - return NULL; + goto error; if (is_ast) { if (supplied_flags & PyCF_ONLY_AST) { Py_INCREF(cmd); @@ -524,39 +632,30 @@ builtin_compile(PyObject *self, PyObject *args, PyObject *kwds) arena = PyArena_New(); if (arena == NULL) - return NULL; + goto error; mod = PyAST_obj2mod(cmd, arena, mode); if (mod == NULL) { PyArena_Free(arena); - return NULL; + goto error; } - result = (PyObject*)PyAST_Compile(mod, filename, - &cf, arena); + result = (PyObject*)PyAST_CompileEx(mod, filename, + &cf, optimize, arena); PyArena_Free(arena); } - return result; + goto finally; } -#ifdef Py_USING_UNICODE - if (PyUnicode_Check(cmd)) { - tmp = PyUnicode_AsUTF8String(cmd); - if (tmp == NULL) - return NULL; - cmd = tmp; - cf.cf_flags |= PyCF_SOURCE_IS_UTF8; - } -#endif + str = source_as_string(cmd, "compile", "string, bytes, AST or code", &cf); + if (str == NULL) + goto error; - if (PyObject_AsReadBuffer(cmd, (const void **)&str, &length)) - goto cleanup; - if ((size_t)length != strlen(str)) { - PyErr_SetString(PyExc_TypeError, - "compile() expected string without null bytes"); - goto cleanup; - } - result = Py_CompileStringFlags(str, filename, start[mode], &cf); -cleanup: - Py_XDECREF(tmp); + result = Py_CompileStringExFlags(str, filename, start[mode], &cf, optimize); + goto finally; + +error: + result = NULL; +finally: + Py_DECREF(filename_obj); return result; } @@ -564,7 +663,7 @@ PyDoc_STRVAR(compile_doc, "compile(source, filename, mode[, flags[, dont_inherit]]) -> code object\n\ \n\ Compile the source string (a Python module, statement or expression)\n\ -into a code object that can be executed by the exec statement or eval().\n\ +into a code object that can be executed by exec() or eval().\n\ The filename will be used for run-time error messages.\n\ The mode must be 'exec' to compile a module, 'single' to compile a\n\ single (interactive) statement, or 'eval' to compile an expression.\n\ @@ -610,7 +709,7 @@ builtin_divmod(PyObject *self, PyObject *args) } PyDoc_STRVAR(divmod_doc, -"divmod(x, y) -> (quotient, remainder)\n\ +"divmod(x, y) -> (div, mod)\n\ \n\ Return the tuple ((x-x%y)/y, x%y). Invariant: div*y + mod == x."); @@ -662,30 +761,14 @@ builtin_eval(PyObject *self, PyObject *args) "code object passed to eval() may not contain free variables"); return NULL; } - return PyEval_EvalCode((PyCodeObject *) cmd, globals, locals); + return PyEval_EvalCode(cmd, globals, locals); } - if (!PyString_Check(cmd) && - !PyUnicode_Check(cmd)) { - PyErr_SetString(PyExc_TypeError, - "eval() arg 1 must be a string or code object"); + cf.cf_flags = PyCF_SOURCE_IS_UTF8; + str = source_as_string(cmd, "eval", "string, bytes or code", &cf); + if (str == NULL) return NULL; - } - cf.cf_flags = 0; -#ifdef Py_USING_UNICODE - if (PyUnicode_Check(cmd)) { - tmp = PyUnicode_AsUTF8String(cmd); - if (tmp == NULL) - return NULL; - cmd = tmp; - cf.cf_flags |= PyCF_SOURCE_IS_UTF8; - } -#endif - if (PyString_AsStringAndSize(cmd, &str, NULL)) { - Py_XDECREF(tmp); - return NULL; - } while (*str == ' ' || *str == '\t') str++; @@ -705,108 +788,80 @@ The globals must be a dictionary and locals can be any mapping,\n\ defaulting to the current globals and locals.\n\ If only globals is given, locals defaults to it.\n"); - static PyObject * -builtin_execfile(PyObject *self, PyObject *args) +builtin_exec(PyObject *self, PyObject *args) { - char *filename; - PyObject *globals = Py_None, *locals = Py_None; - PyObject *res; - FILE* fp = NULL; - PyCompilerFlags cf; - int exists; + PyObject *v; + PyObject *prog, *globals = Py_None, *locals = Py_None; - if (PyErr_WarnPy3k("execfile() not supported in 3.x; use exec()", - 1) < 0) + if (!PyArg_UnpackTuple(args, "exec", 1, 3, &prog, &globals, &locals)) return NULL; - if (!PyArg_ParseTuple(args, "s|O!O:execfile", - &filename, - &PyDict_Type, &globals, - &locals)) - return NULL; - if (locals != Py_None && !PyMapping_Check(locals)) { - PyErr_SetString(PyExc_TypeError, "locals must be a mapping"); - return NULL; - } if (globals == Py_None) { globals = PyEval_GetGlobals(); - if (locals == Py_None) + if (locals == Py_None) { locals = PyEval_GetLocals(); + } + if (!globals || !locals) { + PyErr_SetString(PyExc_SystemError, + "globals and locals cannot be NULL"); + return NULL; + } } else if (locals == Py_None) locals = globals; + + if (!PyDict_Check(globals)) { + PyErr_Format(PyExc_TypeError, "exec() arg 2 must be a dict, not %.100s", + globals->ob_type->tp_name); + return NULL; + } + if (!PyMapping_Check(locals)) { + PyErr_Format(PyExc_TypeError, + "arg 3 must be a mapping or None, not %.100s", + locals->ob_type->tp_name); + return NULL; + } if (PyDict_GetItemString(globals, "__builtins__") == NULL) { if (PyDict_SetItemString(globals, "__builtins__", PyEval_GetBuiltins()) != 0) return NULL; } - exists = 0; - /* Test for existence or directory. */ -#if defined(PLAN9) - { - Dir *d; - - if ((d = dirstat(filename))!=nil) { - if(d->mode & DMDIR) - werrstr("is a directory"); - else - exists = 1; - free(d); + if (PyCode_Check(prog)) { + if (PyCode_GetNumFree((PyCodeObject *)prog) > 0) { + PyErr_SetString(PyExc_TypeError, + "code object passed to exec() may not " + "contain free variables"); + return NULL; } + v = PyEval_EvalCode(prog, globals, locals); } -#elif defined(RISCOS) - if (object_exists(filename)) { - if (isdir(filename)) - errno = EISDIR; + else { + char *str; + PyCompilerFlags cf; + cf.cf_flags = PyCF_SOURCE_IS_UTF8; + str = source_as_string(prog, "exec", + "string, bytes or code", &cf); + if (str == NULL) + return NULL; + if (PyEval_MergeCompilerFlags(&cf)) + v = PyRun_StringFlags(str, Py_file_input, globals, + locals, &cf); else - exists = 1; - } -#else /* standard Posix */ - { - struct stat s; - if (stat(filename, &s) == 0) { - if (S_ISDIR(s.st_mode)) -# if defined(PYOS_OS2) && defined(PYCC_VACPP) - errno = EOS2ERR; -# else - errno = EISDIR; -# endif - else - exists = 1; - } + v = PyRun_String(str, Py_file_input, globals, locals); } -#endif - - if (exists) { - Py_BEGIN_ALLOW_THREADS - fp = fopen(filename, "r" PY_STDIOTEXTMODE); - Py_END_ALLOW_THREADS - - if (fp == NULL) { - exists = 0; - } - } - - if (!exists) { - PyErr_SetFromErrnoWithFilename(PyExc_IOError, filename); + if (v == NULL) return NULL; - } - cf.cf_flags = 0; - if (PyEval_MergeCompilerFlags(&cf)) - res = PyRun_FileExFlags(fp, filename, Py_file_input, globals, - locals, 1, &cf); - else - res = PyRun_FileEx(fp, filename, Py_file_input, globals, - locals, 1); - return res; + Py_DECREF(v); + Py_RETURN_NONE; } -PyDoc_STRVAR(execfile_doc, -"execfile(filename[, globals[, locals]])\n\ +PyDoc_STRVAR(exec_doc, +"exec(object[, globals[, locals]])\n\ \n\ -Read and execute a Python script from a file.\n\ +Read and execute code from an object, which can be a string or a code\n\ +object.\n\ The globals and locals are dictionaries, defaulting to the current\n\ globals and locals. If only globals is given, locals defaults to it."); @@ -819,15 +874,8 @@ builtin_getattr(PyObject *self, PyObject *args) if (!PyArg_UnpackTuple(args, "getattr", 2, 3, &v, &name, &dflt)) return NULL; -#ifdef Py_USING_UNICODE - if (PyUnicode_Check(name)) { - name = _PyUnicode_AsDefaultEncodedString(name, NULL); - if (name == NULL) - return NULL; - } -#endif - if (!PyString_Check(name)) { + if (!PyUnicode_Check(name)) { PyErr_SetString(PyExc_TypeError, "getattr(): attribute name must be string"); return NULL; @@ -875,39 +923,28 @@ builtin_hasattr(PyObject *self, PyObject *args) if (!PyArg_UnpackTuple(args, "hasattr", 2, 2, &v, &name)) return NULL; -#ifdef Py_USING_UNICODE - if (PyUnicode_Check(name)) { - name = _PyUnicode_AsDefaultEncodedString(name, NULL); - if (name == NULL) - return NULL; - } -#endif - - if (!PyString_Check(name)) { + if (!PyUnicode_Check(name)) { PyErr_SetString(PyExc_TypeError, "hasattr(): attribute name must be string"); return NULL; } v = PyObject_GetAttr(v, name); if (v == NULL) { - if (!PyErr_ExceptionMatches(PyExc_Exception)) - return NULL; - else { + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { PyErr_Clear(); - Py_INCREF(Py_False); - return Py_False; + Py_RETURN_FALSE; } + return NULL; } Py_DECREF(v); - Py_INCREF(Py_True); - return Py_True; + Py_RETURN_TRUE; } PyDoc_STRVAR(hasattr_doc, "hasattr(object, name) -> bool\n\ \n\ Return whether the object has an attribute with the given name.\n\ -(This is done by calling getattr(object, name) and catching exceptions.)"); +(This is done by calling getattr(object, name) and catching AttributeError.)"); static PyObject * @@ -923,169 +960,151 @@ Return the identity of an object. This is guaranteed to be unique among\n\ simultaneously existing objects. (Hint: it's the object's memory address.)"); +/* map object ************************************************************/ + +typedef struct { + PyObject_HEAD + PyObject *iters; + PyObject *func; +} mapobject; + static PyObject * -builtin_map(PyObject *self, PyObject *args) +map_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - typedef struct { - PyObject *it; /* the iterator object */ - int saw_StopIteration; /* bool: did the iterator end? */ - } sequence; - - PyObject *func, *result; - sequence *seqs = NULL, *sqp; - Py_ssize_t n, len; - register int i, j; - - n = PyTuple_Size(args); - if (n < 2) { + PyObject *it, *iters, *func; + mapobject *lz; + Py_ssize_t numargs, i; + + if (type == &PyMap_Type && !_PyArg_NoKeywords("map()", kwds)) + return NULL; + + numargs = PyTuple_Size(args); + if (numargs < 2) { PyErr_SetString(PyExc_TypeError, - "map() requires at least two args"); + "map() must have at least two arguments."); return NULL; } - func = PyTuple_GetItem(args, 0); - n--; + iters = PyTuple_New(numargs-1); + if (iters == NULL) + return NULL; - if (func == Py_None) { - if (PyErr_WarnPy3k("map(None, ...) not supported in 3.x; " - "use list(...)", 1) < 0) + for (i=1 ; i<numargs ; i++) { + /* Get iterator. */ + it = PyObject_GetIter(PyTuple_GET_ITEM(args, i)); + if (it == NULL) { + Py_DECREF(iters); return NULL; - if (n == 1) { - /* map(None, S) is the same as list(S). */ - return PySequence_List(PyTuple_GetItem(args, 1)); } + PyTuple_SET_ITEM(iters, i-1, it); } - /* Get space for sequence descriptors. Must NULL out the iterator - * pointers so that jumping to Fail_2 later doesn't see trash. - */ - if ((seqs = PyMem_NEW(sequence, n)) == NULL) { - PyErr_NoMemory(); + /* create mapobject structure */ + lz = (mapobject *)type->tp_alloc(type, 0); + if (lz == NULL) { + Py_DECREF(iters); return NULL; } - for (i = 0; i < n; ++i) { - seqs[i].it = (PyObject*)NULL; - seqs[i].saw_StopIteration = 0; - } + lz->iters = iters; + func = PyTuple_GET_ITEM(args, 0); + Py_INCREF(func); + lz->func = func; - /* Do a first pass to obtain iterators for the arguments, and set len - * to the largest of their lengths. - */ - len = 0; - for (i = 0, sqp = seqs; i < n; ++i, ++sqp) { - PyObject *curseq; - Py_ssize_t curlen; - - /* Get iterator. */ - curseq = PyTuple_GetItem(args, i+1); - sqp->it = PyObject_GetIter(curseq); - if (sqp->it == NULL) { - static char errmsg[] = - "argument %d to map() must support iteration"; - char errbuf[sizeof(errmsg) + 25]; - PyOS_snprintf(errbuf, sizeof(errbuf), errmsg, i+2); - PyErr_SetString(PyExc_TypeError, errbuf); - goto Fail_2; - } - - /* Update len. */ - curlen = _PyObject_LengthHint(curseq, 8); - if (curlen > len) - len = curlen; - } - - /* Get space for the result list. */ - if ((result = (PyObject *) PyList_New(len)) == NULL) - goto Fail_2; - - /* Iterate over the sequences until all have stopped. */ - for (i = 0; ; ++i) { - PyObject *alist, *item=NULL, *value; - int numactive = 0; + return (PyObject *)lz; +} - if (func == Py_None && n == 1) - alist = NULL; - else if ((alist = PyTuple_New(n)) == NULL) - goto Fail_1; +static void +map_dealloc(mapobject *lz) +{ + PyObject_GC_UnTrack(lz); + Py_XDECREF(lz->iters); + Py_XDECREF(lz->func); + Py_TYPE(lz)->tp_free(lz); +} - for (j = 0, sqp = seqs; j < n; ++j, ++sqp) { - if (sqp->saw_StopIteration) { - Py_INCREF(Py_None); - item = Py_None; - } - else { - item = PyIter_Next(sqp->it); - if (item) - ++numactive; - else { - if (PyErr_Occurred()) { - Py_XDECREF(alist); - goto Fail_1; - } - Py_INCREF(Py_None); - item = Py_None; - sqp->saw_StopIteration = 1; - } - } - if (alist) - PyTuple_SET_ITEM(alist, j, item); - else - break; - } +static int +map_traverse(mapobject *lz, visitproc visit, void *arg) +{ + Py_VISIT(lz->iters); + Py_VISIT(lz->func); + return 0; +} - if (!alist) - alist = item; +static PyObject * +map_next(mapobject *lz) +{ + PyObject *val; + PyObject *argtuple; + PyObject *result; + Py_ssize_t numargs, i; - if (numactive == 0) { - Py_DECREF(alist); - break; - } + numargs = PyTuple_Size(lz->iters); + argtuple = PyTuple_New(numargs); + if (argtuple == NULL) + return NULL; - if (func == Py_None) - value = alist; - else { - value = PyEval_CallObject(func, alist); - Py_DECREF(alist); - if (value == NULL) - goto Fail_1; - } - if (i >= len) { - int status = PyList_Append(result, value); - Py_DECREF(value); - if (status < 0) - goto Fail_1; + for (i=0 ; i<numargs ; i++) { + val = PyIter_Next(PyTuple_GET_ITEM(lz->iters, i)); + if (val == NULL) { + Py_DECREF(argtuple); + return NULL; } - else if (PyList_SetItem(result, i, value) < 0) - goto Fail_1; + PyTuple_SET_ITEM(argtuple, i, val); } - - if (i < len && PyList_SetSlice(result, i, len, NULL) < 0) - goto Fail_1; - - goto Succeed; - -Fail_1: - Py_DECREF(result); -Fail_2: - result = NULL; -Succeed: - assert(seqs); - for (i = 0; i < n; ++i) - Py_XDECREF(seqs[i].it); - PyMem_DEL(seqs); + result = PyObject_Call(lz->func, argtuple, NULL); + Py_DECREF(argtuple); return result; } PyDoc_STRVAR(map_doc, -"map(function, sequence[, sequence, ...]) -> list\n\ +"map(func, *iterables) --> map object\n\ \n\ -Return a list of the results of applying the function to the items of\n\ -the argument sequence(s). If more than one sequence is given, the\n\ -function is called with an argument list consisting of the corresponding\n\ -item of each sequence, substituting None for missing values when not all\n\ -sequences have the same length. If the function is None, return a list of\n\ -the items of the sequence (or a list of tuples if more than one sequence)."); - +Make an iterator that computes the function using arguments from\n\ +each of the iterables. Stops when the shortest iterable is exhausted."); + +PyTypeObject PyMap_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "map", /* tp_name */ + sizeof(mapobject), /* tp_basicsize */ + 0, /* tp_itemsize */ + /* methods */ + (destructor)map_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_BASETYPE, /* tp_flags */ + map_doc, /* tp_doc */ + (traverseproc)map_traverse, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + PyObject_SelfIter, /* tp_iter */ + (iternextfunc)map_next, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + PyType_GenericAlloc, /* tp_alloc */ + map_new, /* tp_new */ + PyObject_GC_Del, /* tp_free */ +}; static PyObject * builtin_next(PyObject *self, PyObject *args) @@ -1107,7 +1126,7 @@ builtin_next(PyObject *self, PyObject *args) return res; } else if (def != NULL) { if (PyErr_Occurred()) { - if (!PyErr_ExceptionMatches(PyExc_StopIteration)) + if(!PyErr_ExceptionMatches(PyExc_StopIteration)) return NULL; PyErr_Clear(); } @@ -1174,12 +1193,12 @@ Delete a named attribute on an object; delattr(x, 'y') is equivalent to\n\ static PyObject * builtin_hash(PyObject *self, PyObject *v) { - long x; + Py_hash_t x; x = PyObject_Hash(v); if (x == -1) return NULL; - return PyInt_FromLong(x); + return PyLong_FromSsize_t(x); } PyDoc_STRVAR(hash_doc, @@ -1192,93 +1211,13 @@ the same hash value. The reverse is not necessarily true, but likely."); static PyObject * builtin_hex(PyObject *self, PyObject *v) { - PyNumberMethods *nb; - PyObject *res; - - if ((nb = v->ob_type->tp_as_number) == NULL || - nb->nb_hex == NULL) { - PyErr_SetString(PyExc_TypeError, - "hex() argument can't be converted to hex"); - return NULL; - } - res = (*nb->nb_hex)(v); - if (res && !PyString_Check(res)) { - PyErr_Format(PyExc_TypeError, - "__hex__ returned non-string (type %.200s)", - res->ob_type->tp_name); - Py_DECREF(res); - return NULL; - } - return res; + return PyNumber_ToBase(v, 16); } PyDoc_STRVAR(hex_doc, "hex(number) -> string\n\ \n\ -Return the hexadecimal representation of an integer or long integer."); - - -static PyObject *builtin_raw_input(PyObject *, PyObject *); - -static PyObject * -builtin_input(PyObject *self, PyObject *args) -{ - PyObject *line; - char *str; - PyObject *res; - PyObject *globals, *locals; - PyCompilerFlags cf; - - line = builtin_raw_input(self, args); - if (line == NULL) - return line; - if (!PyArg_Parse(line, "s;embedded '\\0' in input line", &str)) - return NULL; - while (*str == ' ' || *str == '\t') - str++; - globals = PyEval_GetGlobals(); - locals = PyEval_GetLocals(); - if (PyDict_GetItemString(globals, "__builtins__") == NULL) { - if (PyDict_SetItemString(globals, "__builtins__", - PyEval_GetBuiltins()) != 0) - return NULL; - } - cf.cf_flags = 0; - PyEval_MergeCompilerFlags(&cf); - res = PyRun_StringFlags(str, Py_eval_input, globals, locals, &cf); - Py_DECREF(line); - return res; -} - -PyDoc_STRVAR(input_doc, -"input([prompt]) -> value\n\ -\n\ -Equivalent to eval(raw_input(prompt))."); - - -static PyObject * -builtin_intern(PyObject *self, PyObject *args) -{ - PyObject *s; - if (!PyArg_ParseTuple(args, "S:intern", &s)) - return NULL; - if (!PyString_CheckExact(s)) { - PyErr_SetString(PyExc_TypeError, - "can't intern subclass of string"); - return NULL; - } - Py_INCREF(s); - PyString_InternInPlace(&s); - return s; -} - -PyDoc_STRVAR(intern_doc, -"intern(string) -> string\n\ -\n\ -``Intern'' the given string. This enters the string in the (global)\n\ -table of interned strings whose purpose is to speed up dictionary lookups.\n\ -Return the string itself or the previously interned string object with the\n\ -same value."); +Return the hexadecimal representation of an integer."); static PyObject * @@ -1299,7 +1238,7 @@ builtin_iter(PyObject *self, PyObject *args) } PyDoc_STRVAR(iter_doc, -"iter(collection) -> iterator\n\ +"iter(iterable) -> iterator\n\ iter(callable, sentinel) -> iterator\n\ \n\ Get an iterator from an object. In the first form, the argument must\n\ @@ -1315,7 +1254,7 @@ builtin_len(PyObject *self, PyObject *v) res = PyObject_Size(v); if (res < 0 && PyErr_Occurred()) return NULL; - return PyInt_FromSsize_t(res); + return PyLong_FromSsize_t(res); } PyDoc_STRVAR(len_doc, @@ -1460,43 +1399,13 @@ With two or more arguments, return the largest argument."); static PyObject * builtin_oct(PyObject *self, PyObject *v) { - PyNumberMethods *nb; - PyObject *res; - - if (v == NULL || (nb = v->ob_type->tp_as_number) == NULL || - nb->nb_oct == NULL) { - PyErr_SetString(PyExc_TypeError, - "oct() argument can't be converted to oct"); - return NULL; - } - res = (*nb->nb_oct)(v); - if (res && !PyString_Check(res)) { - PyErr_Format(PyExc_TypeError, - "__oct__ returned non-string (type %.200s)", - res->ob_type->tp_name); - Py_DECREF(res); - return NULL; - } - return res; + return PyNumber_ToBase(v, 8); } PyDoc_STRVAR(oct_doc, "oct(number) -> string\n\ \n\ -Return the octal representation of an integer or long integer."); - - -static PyObject * -builtin_open(PyObject *self, PyObject *args, PyObject *kwds) -{ - return PyObject_Call((PyObject*)&PyFile_Type, args, kwds); -} - -PyDoc_STRVAR(open_doc, -"open(name[, mode[, buffering]]) -> file object\n\ -\n\ -Open a file using the file() type, returns a file object. This is the\n\ -preferred way to open a file. See file.__doc__ for further information."); +Return the octal representation of an integer."); static PyObject * @@ -1505,28 +1414,42 @@ builtin_ord(PyObject *self, PyObject* obj) long ord; Py_ssize_t size; - if (PyString_Check(obj)) { - size = PyString_GET_SIZE(obj); + if (PyBytes_Check(obj)) { + size = PyBytes_GET_SIZE(obj); if (size == 1) { - ord = (long)((unsigned char)*PyString_AS_STRING(obj)); - return PyInt_FromLong(ord); + ord = (long)((unsigned char)*PyBytes_AS_STRING(obj)); + return PyLong_FromLong(ord); } - } else if (PyByteArray_Check(obj)) { - size = PyByteArray_GET_SIZE(obj); - if (size == 1) { - ord = (long)((unsigned char)*PyByteArray_AS_STRING(obj)); - return PyInt_FromLong(ord); - } - -#ifdef Py_USING_UNICODE - } else if (PyUnicode_Check(obj)) { + } + else if (PyUnicode_Check(obj)) { size = PyUnicode_GET_SIZE(obj); if (size == 1) { ord = (long)*PyUnicode_AS_UNICODE(obj); - return PyInt_FromLong(ord); + return PyLong_FromLong(ord); + } +#ifndef Py_UNICODE_WIDE + if (size == 2) { + /* Decode a valid surrogate pair */ + int c0 = PyUnicode_AS_UNICODE(obj)[0]; + int c1 = PyUnicode_AS_UNICODE(obj)[1]; + if (0xD800 <= c0 && c0 <= 0xDBFF && + 0xDC00 <= c1 && c1 <= 0xDFFF) { + ord = ((((c0 & 0x03FF) << 10) | (c1 & 0x03FF)) + + 0x00010000); + return PyLong_FromLong(ord); + } } #endif - } else { + } + else if (PyByteArray_Check(obj)) { + /* XXX Hopefully this is temporary */ + size = PyByteArray_GET_SIZE(obj); + if (size == 1) { + ord = (long)((unsigned char)*PyByteArray_AS_STRING(obj)); + return PyLong_FromLong(ord); + } + } + else { PyErr_Format(PyExc_TypeError, "ord() expected string of length 1, but " \ "%.200s found", obj->ob_type->tp_name); @@ -1540,10 +1463,17 @@ builtin_ord(PyObject *self, PyObject* obj) return NULL; } -PyDoc_STRVAR(ord_doc, +PyDoc_VAR(ord_doc) = PyDoc_STR( "ord(c) -> integer\n\ \n\ -Return the integer ordinal of a one-character string."); +Return the integer ordinal of a one-character string." +) +#ifndef Py_UNICODE_WIDE +PyDoc_STR( +"\nA valid surrogate pair is also accepted." +) +#endif +; static PyObject * @@ -1563,46 +1493,17 @@ With two arguments, equivalent to x**y. With three arguments,\n\ equivalent to (x**y) % z, but may be more efficient (e.g. for longs)."); + static PyObject * builtin_print(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"sep", "end", "file", 0}; - static PyObject *dummy_args = NULL; - static PyObject *unicode_newline = NULL, *unicode_space = NULL; - static PyObject *str_newline = NULL, *str_space = NULL; - PyObject *newline, *space; + static PyObject *dummy_args; PyObject *sep = NULL, *end = NULL, *file = NULL; - int i, err, use_unicode = 0; + int i, err; - if (dummy_args == NULL) { - if (!(dummy_args = PyTuple_New(0))) - return NULL; - } - if (str_newline == NULL) { - str_newline = PyString_FromString("\n"); - if (str_newline == NULL) - return NULL; - str_space = PyString_FromString(" "); - if (str_space == NULL) { - Py_CLEAR(str_newline); - return NULL; - } -#ifdef Py_USING_UNICODE - unicode_newline = PyUnicode_FromString("\n"); - if (unicode_newline == NULL) { - Py_CLEAR(str_newline); - Py_CLEAR(str_space); + if (dummy_args == NULL && !(dummy_args = PyTuple_New(0))) return NULL; - } - unicode_space = PyUnicode_FromString(" "); - if (unicode_space == NULL) { - Py_CLEAR(str_newline); - Py_CLEAR(str_space); - Py_CLEAR(unicode_space); - return NULL; - } -#endif - } if (!PyArg_ParseTupleAndKeywords(dummy_args, kwds, "|OOO:print", kwlist, &sep, &end, &file)) return NULL; @@ -1612,56 +1513,30 @@ builtin_print(PyObject *self, PyObject *args, PyObject *kwds) if (file == Py_None) Py_RETURN_NONE; } + if (sep == Py_None) { sep = NULL; } - else if (sep) { - if (PyUnicode_Check(sep)) { - use_unicode = 1; - } - else if (!PyString_Check(sep)) { - PyErr_Format(PyExc_TypeError, - "sep must be None, str or unicode, not %.200s", - sep->ob_type->tp_name); - return NULL; - } + else if (sep && !PyUnicode_Check(sep)) { + PyErr_Format(PyExc_TypeError, + "sep must be None or a string, not %.200s", + sep->ob_type->tp_name); + return NULL; } - if (end == Py_None) + if (end == Py_None) { end = NULL; - else if (end) { - if (PyUnicode_Check(end)) { - use_unicode = 1; - } - else if (!PyString_Check(end)) { - PyErr_Format(PyExc_TypeError, - "end must be None, str or unicode, not %.200s", - end->ob_type->tp_name); - return NULL; - } } - - if (!use_unicode) { - for (i = 0; i < PyTuple_Size(args); i++) { - if (PyUnicode_Check(PyTuple_GET_ITEM(args, i))) { - use_unicode = 1; - break; - } - } - } - if (use_unicode) { - newline = unicode_newline; - space = unicode_space; - } - else { - newline = str_newline; - space = str_space; + else if (end && !PyUnicode_Check(end)) { + PyErr_Format(PyExc_TypeError, + "end must be None or a string, not %.200s", + end->ob_type->tp_name); + return NULL; } for (i = 0; i < PyTuple_Size(args); i++) { if (i > 0) { if (sep == NULL) - err = PyFile_WriteObject(space, file, - Py_PRINT_RAW); + err = PyFile_WriteString(" ", file); else err = PyFile_WriteObject(sep, file, Py_PRINT_RAW); @@ -1675,7 +1550,7 @@ builtin_print(PyObject *self, PyObject *args, PyObject *kwds) } if (end == NULL) - err = PyFile_WriteObject(newline, file, Py_PRINT_RAW); + err = PyFile_WriteString("\n", file); else err = PyFile_WriteObject(end, file, Py_PRINT_RAW); if (err) @@ -1694,402 +1569,184 @@ sep: string inserted between values, default a space.\n\ end: string appended after the last value, default a newline."); -/* Return number of items in range (lo, hi, step), when arguments are - * PyInt or PyLong objects. step > 0 required. Return a value < 0 if - * & only if the true value is too large to fit in a signed long. - * Arguments MUST return 1 with either PyInt_Check() or - * PyLong_Check(). Return -1 when there is an error. - */ -static long -get_len_of_range_longs(PyObject *lo, PyObject *hi, PyObject *step) -{ - /* ------------------------------------------------------------- - Algorithm is equal to that of get_len_of_range(), but it operates - on PyObjects (which are assumed to be PyLong or PyInt objects). - ---------------------------------------------------------------*/ - long n; - PyObject *diff = NULL; - PyObject *one = NULL; - PyObject *tmp1 = NULL, *tmp2 = NULL, *tmp3 = NULL; - /* holds sub-expression evaluations */ - - /* if (lo >= hi), return length of 0. */ - if (PyObject_Compare(lo, hi) >= 0) - return 0; - - if ((one = PyLong_FromLong(1L)) == NULL) - goto Fail; - - if ((tmp1 = PyNumber_Subtract(hi, lo)) == NULL) - goto Fail; - - if ((diff = PyNumber_Subtract(tmp1, one)) == NULL) - goto Fail; - - if ((tmp2 = PyNumber_FloorDivide(diff, step)) == NULL) - goto Fail; - - if ((tmp3 = PyNumber_Add(tmp2, one)) == NULL) - goto Fail; - - n = PyLong_AsLong(tmp3); - if (PyErr_Occurred()) { /* Check for Overflow */ - PyErr_Clear(); - goto Fail; - } - - Py_DECREF(tmp3); - Py_DECREF(tmp2); - Py_DECREF(diff); - Py_DECREF(tmp1); - Py_DECREF(one); - return n; - - Fail: - Py_XDECREF(tmp3); - Py_XDECREF(tmp2); - Py_XDECREF(diff); - Py_XDECREF(tmp1); - Py_XDECREF(one); - return -1; -} - -/* Helper function for handle_range_longs. If arg is int or long - object, returns it with incremented reference count. If arg is - float, raises type error. As a last resort, creates a new int by - calling arg type's nb_int method if it is defined. Returns NULL - and sets exception on error. - - Returns a new reference to an int object. */ -static PyObject * -get_range_long_argument(PyObject *arg, const char *name) -{ - PyObject *v; - PyNumberMethods *nb; - if (PyInt_Check(arg) || PyLong_Check(arg)) { - Py_INCREF(arg); - return arg; - } - if (PyFloat_Check(arg) || - (nb = Py_TYPE(arg)->tp_as_number) == NULL || - nb->nb_int == NULL) { - PyErr_Format(PyExc_TypeError, - "range() integer %s argument expected, got %s.", - name, arg->ob_type->tp_name); - return NULL; - } - v = nb->nb_int(arg); - if (v == NULL) - return NULL; - if (PyInt_Check(v) || PyLong_Check(v)) - return v; - Py_DECREF(v); - PyErr_SetString(PyExc_TypeError, - "__int__ should return int object"); - return NULL; -} - -/* An extension of builtin_range() that handles the case when PyLong - * arguments are given. */ static PyObject * -handle_range_longs(PyObject *self, PyObject *args) +builtin_input(PyObject *self, PyObject *args) { - PyObject *ilow = NULL; - PyObject *ihigh = NULL; - PyObject *istep = NULL; - - PyObject *low = NULL; - PyObject *high = NULL; - PyObject *step = NULL; - - PyObject *curnum = NULL; - PyObject *v = NULL; - long bign; - Py_ssize_t i, n; - int cmp_result; - - PyObject *zero = PyLong_FromLong(0); + PyObject *promptarg = NULL; + PyObject *fin = PySys_GetObject("stdin"); + PyObject *fout = PySys_GetObject("stdout"); + PyObject *ferr = PySys_GetObject("stderr"); + PyObject *tmp; + long fd; + int tty; - if (zero == NULL) + /* Parse arguments */ + if (!PyArg_UnpackTuple(args, "input", 0, 1, &promptarg)) return NULL; - if (!PyArg_UnpackTuple(args, "range", 1, 3, &ilow, &ihigh, &istep)) { - Py_DECREF(zero); + /* Check that stdin/out/err are intact */ + if (fin == NULL || fin == Py_None) { + PyErr_SetString(PyExc_RuntimeError, + "input(): lost sys.stdin"); return NULL; } - - /* Figure out which way we were called, supply defaults, and be - * sure to incref everything so that the decrefs at the end - * are correct. NB: ilow, ihigh and istep are borrowed references. - */ - assert(ilow != NULL); - if (ihigh == NULL) { - /* only 1 arg -- it's the upper limit */ - ihigh = ilow; - ilow = NULL; - } - - /* convert ihigh if necessary */ - assert(ihigh != NULL); - high = get_range_long_argument(ihigh, "end"); - if (high == NULL) - goto Fail; - - /* ihigh correct now; do ilow */ - if (ilow == NULL) { - Py_INCREF(zero); - low = zero; + if (fout == NULL || fout == Py_None) { + PyErr_SetString(PyExc_RuntimeError, + "input(): lost sys.stdout"); + return NULL; } - else { - low = get_range_long_argument(ilow, "start"); - if (low == NULL) - goto Fail; + if (ferr == NULL || ferr == Py_None) { + PyErr_SetString(PyExc_RuntimeError, + "input(): lost sys.stderr"); + return NULL; } - /* ilow and ihigh correct now; do istep */ - if (istep == NULL) - step = PyLong_FromLong(1); + /* First of all, flush stderr */ + tmp = PyObject_CallMethod(ferr, "flush", ""); + if (tmp == NULL) + PyErr_Clear(); else - step = get_range_long_argument(istep, "step"); - if (step == NULL) - goto Fail; + Py_DECREF(tmp); - if (PyObject_Cmp(step, zero, &cmp_result) == -1) - goto Fail; - - if (cmp_result == 0) { - PyErr_SetString(PyExc_ValueError, - "range() step argument must not be zero"); - goto Fail; + /* We should only use (GNU) readline if Python's sys.stdin and + sys.stdout are the same as C's stdin and stdout, because we + need to pass it those. */ + tmp = PyObject_CallMethod(fin, "fileno", ""); + if (tmp == NULL) { + PyErr_Clear(); + tty = 0; } - - if (cmp_result > 0) - bign = get_len_of_range_longs(low, high, step); else { - PyObject *neg_step = PyNumber_Negative(step); - if (neg_step == NULL) - goto Fail; - bign = get_len_of_range_longs(high, low, neg_step); - Py_DECREF(neg_step); - } - - n = (Py_ssize_t)bign; - if (bign < 0 || (long)n != bign) { - PyErr_SetString(PyExc_OverflowError, - "range() result has too many items"); - goto Fail; - } - - v = PyList_New(n); - if (v == NULL) - goto Fail; - - curnum = low; - Py_INCREF(curnum); - - for (i = 0; i < n; i++) { - PyObject *w = PyNumber_Long(curnum); - PyObject *tmp_num; - if (w == NULL) - goto Fail; - - PyList_SET_ITEM(v, i, w); - - tmp_num = PyNumber_Add(curnum, step); - if (tmp_num == NULL) - goto Fail; - - Py_DECREF(curnum); - curnum = tmp_num; - } - Py_DECREF(low); - Py_DECREF(high); - Py_DECREF(step); - Py_DECREF(zero); - Py_DECREF(curnum); - return v; - - Fail: - Py_XDECREF(low); - Py_XDECREF(high); - Py_XDECREF(step); - Py_DECREF(zero); - Py_XDECREF(curnum); - Py_XDECREF(v); - return NULL; -} - -/* Return number of items in range/xrange (lo, hi, step). step > 0 - * required. Return a value < 0 if & only if the true value is too - * large to fit in a signed long. - */ -static long -get_len_of_range(long lo, long hi, long step) -{ - /* ------------------------------------------------------------- - If lo >= hi, the range is empty. - Else if n values are in the range, the last one is - lo + (n-1)*step, which must be <= hi-1. Rearranging, - n <= (hi - lo - 1)/step + 1, so taking the floor of the RHS gives - the proper value. Since lo < hi in this case, hi-lo-1 >= 0, so - the RHS is non-negative and so truncation is the same as the - floor. Letting M be the largest positive long, the worst case - for the RHS numerator is hi=M, lo=-M-1, and then - hi-lo-1 = M-(-M-1)-1 = 2*M. Therefore unsigned long has enough - precision to compute the RHS exactly. - ---------------------------------------------------------------*/ - long n = 0; - if (lo < hi) { - unsigned long uhi = (unsigned long)hi; - unsigned long ulo = (unsigned long)lo; - unsigned long diff = uhi - ulo - 1; - n = (long)(diff / (unsigned long)step + 1); - } - return n; -} - -static PyObject * -builtin_range(PyObject *self, PyObject *args) -{ - long ilow = 0, ihigh = 0, istep = 1; - long bign; - Py_ssize_t i, n; - - PyObject *v; - - if (PyTuple_Size(args) <= 1) { - if (!PyArg_ParseTuple(args, - "l;range() requires 1-3 int arguments", - &ihigh)) { - PyErr_Clear(); - return handle_range_longs(self, args); - } + fd = PyLong_AsLong(tmp); + Py_DECREF(tmp); + if (fd < 0 && PyErr_Occurred()) + return NULL; + tty = fd == fileno(stdin) && isatty(fd); } - else { - if (!PyArg_ParseTuple(args, - "ll|l;range() requires 1-3 int arguments", - &ilow, &ihigh, &istep)) { + if (tty) { + tmp = PyObject_CallMethod(fout, "fileno", ""); + if (tmp == NULL) PyErr_Clear(); - return handle_range_longs(self, args); - } - } - if (istep == 0) { - PyErr_SetString(PyExc_ValueError, - "range() step argument must not be zero"); - return NULL; - } - if (istep > 0) - bign = get_len_of_range(ilow, ihigh, istep); - else - bign = get_len_of_range(ihigh, ilow, -istep); - n = (Py_ssize_t)bign; - if (bign < 0 || (long)n != bign) { - PyErr_SetString(PyExc_OverflowError, - "range() result has too many items"); - return NULL; - } - v = PyList_New(n); - if (v == NULL) - return NULL; - for (i = 0; i < n; i++) { - PyObject *w = PyInt_FromLong(ilow); - if (w == NULL) { - Py_DECREF(v); - return NULL; + else { + fd = PyLong_AsLong(tmp); + Py_DECREF(tmp); + if (fd < 0 && PyErr_Occurred()) + return NULL; + tty = fd == fileno(stdout) && isatty(fd); } - PyList_SET_ITEM(v, i, w); - ilow += istep; } - return v; -} - -PyDoc_STRVAR(range_doc, -"range(stop) -> list of integers\n\ -range(start, stop[, step]) -> list of integers\n\ -\n\ -Return a list containing an arithmetic progression of integers.\n\ -range(i, j) returns [i, i+1, i+2, ..., j-1]; start (!) defaults to 0.\n\ -When step is given, it specifies the increment (or decrement).\n\ -For example, range(4) returns [0, 1, 2, 3]. The end point is omitted!\n\ -These are exactly the valid indices for a list of 4 elements."); - -static PyObject * -builtin_raw_input(PyObject *self, PyObject *args) -{ - PyObject *v = NULL; - PyObject *fin = PySys_GetObject("stdin"); - PyObject *fout = PySys_GetObject("stdout"); - - if (!PyArg_UnpackTuple(args, "[raw_]input", 0, 1, &v)) - return NULL; - - if (fin == NULL) { - PyErr_SetString(PyExc_RuntimeError, "[raw_]input: lost sys.stdin"); - return NULL; - } - if (fout == NULL) { - PyErr_SetString(PyExc_RuntimeError, "[raw_]input: lost sys.stdout"); - return NULL; - } - if (PyFile_SoftSpace(fout, 0)) { - if (PyFile_WriteString(" ", fout) != 0) - return NULL; - } - if (PyFile_AsFile(fin) && PyFile_AsFile(fout) - && isatty(fileno(PyFile_AsFile(fin))) - && isatty(fileno(PyFile_AsFile(fout)))) { - PyObject *po; + /* If we're interactive, use (GNU) readline */ + if (tty) { + PyObject *po = NULL; char *prompt; - char *s; + char *s = NULL; + PyObject *stdin_encoding = NULL, *stdin_errors = NULL; + PyObject *stdout_encoding = NULL, *stdout_errors = NULL; + char *stdin_encoding_str, *stdin_errors_str; PyObject *result; - if (v != NULL) { - po = PyObject_Str(v); + size_t len; + + stdin_encoding = PyObject_GetAttrString(fin, "encoding"); + stdin_errors = PyObject_GetAttrString(fin, "errors"); + if (!stdin_encoding || !stdin_errors) + /* stdin is a text stream, so it must have an + encoding. */ + goto _readline_errors; + stdin_encoding_str = _PyUnicode_AsString(stdin_encoding); + stdin_errors_str = _PyUnicode_AsString(stdin_errors); + if (!stdin_encoding_str || !stdin_errors_str) + goto _readline_errors; + tmp = PyObject_CallMethod(fout, "flush", ""); + if (tmp == NULL) + PyErr_Clear(); + else + Py_DECREF(tmp); + if (promptarg != NULL) { + /* We have a prompt, encode it as stdout would */ + char *stdout_encoding_str, *stdout_errors_str; + PyObject *stringpo; + stdout_encoding = PyObject_GetAttrString(fout, "encoding"); + stdout_errors = PyObject_GetAttrString(fout, "errors"); + if (!stdout_encoding || !stdout_errors) + goto _readline_errors; + stdout_encoding_str = _PyUnicode_AsString(stdout_encoding); + stdout_errors_str = _PyUnicode_AsString(stdout_errors); + if (!stdout_encoding_str || !stdout_errors_str) + goto _readline_errors; + stringpo = PyObject_Str(promptarg); + if (stringpo == NULL) + goto _readline_errors; + po = PyUnicode_AsEncodedString(stringpo, + stdout_encoding_str, stdout_errors_str); + Py_CLEAR(stdout_encoding); + Py_CLEAR(stdout_errors); + Py_CLEAR(stringpo); if (po == NULL) - return NULL; - prompt = PyString_AsString(po); + goto _readline_errors; + prompt = PyBytes_AsString(po); if (prompt == NULL) - return NULL; + goto _readline_errors; } else { po = NULL; prompt = ""; } - s = PyOS_Readline(PyFile_AsFile(fin), PyFile_AsFile(fout), - prompt); - Py_XDECREF(po); + s = PyOS_Readline(stdin, stdout, prompt); if (s == NULL) { if (!PyErr_Occurred()) PyErr_SetNone(PyExc_KeyboardInterrupt); - return NULL; + goto _readline_errors; } - if (*s == '\0') { + + len = strlen(s); + if (len == 0) { PyErr_SetNone(PyExc_EOFError); result = NULL; } - else { /* strip trailing '\n' */ - size_t len = strlen(s); + else { if (len > PY_SSIZE_T_MAX) { PyErr_SetString(PyExc_OverflowError, - "[raw_]input: input too long"); + "input: input too long"); result = NULL; } else { - result = PyString_FromStringAndSize(s, len-1); + len--; /* strip trailing '\n' */ + if (len != 0 && s[len-1] == '\r') + len--; /* strip trailing '\r' */ + result = PyUnicode_Decode(s, len, stdin_encoding_str, + stdin_errors_str); } } + Py_DECREF(stdin_encoding); + Py_DECREF(stdin_errors); + Py_XDECREF(po); PyMem_FREE(s); return result; + _readline_errors: + Py_XDECREF(stdin_encoding); + Py_XDECREF(stdout_encoding); + Py_XDECREF(stdin_errors); + Py_XDECREF(stdout_errors); + Py_XDECREF(po); + return NULL; } - if (v != NULL) { - if (PyFile_WriteObject(v, fout, Py_PRINT_RAW) != 0) + + /* Fallback if we're not interactive */ + if (promptarg != NULL) { + if (PyFile_WriteObject(promptarg, fout, Py_PRINT_RAW) != 0) return NULL; } + tmp = PyObject_CallMethod(fout, "flush", ""); + if (tmp == NULL) + PyErr_Clear(); + else + Py_DECREF(tmp); return PyFile_GetLine(fin, -1); } -PyDoc_STRVAR(raw_input_doc, -"raw_input([prompt]) -> string\n\ +PyDoc_STRVAR(input_doc, +"input([prompt]) -> string\n\ \n\ Read a string from standard input. The trailing newline is stripped.\n\ If the user hits EOF (Unix: Ctl-D, Windows: Ctl-Z+Return), raise EOFError.\n\ @@ -2098,54 +1755,6 @@ is printed without a trailing newline before reading."); static PyObject * -builtin_reduce(PyObject *self, PyObject *args) -{ - static PyObject *functools_reduce = NULL; - - if (PyErr_WarnPy3k("reduce() not supported in 3.x; " - "use functools.reduce()", 1) < 0) - return NULL; - - if (functools_reduce == NULL) { - PyObject *functools = PyImport_ImportModule("functools"); - if (functools == NULL) - return NULL; - functools_reduce = PyObject_GetAttrString(functools, "reduce"); - Py_DECREF(functools); - if (functools_reduce == NULL) - return NULL; - } - return PyObject_Call(functools_reduce, args, NULL); -} - -PyDoc_STRVAR(reduce_doc, -"reduce(function, sequence[, initial]) -> value\n\ -\n\ -Apply a function of two arguments cumulatively to the items of a sequence,\n\ -from left to right, so as to reduce the sequence to a single value.\n\ -For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5]) calculates\n\ -((((1+2)+3)+4)+5). If initial is present, it is placed before the items\n\ -of the sequence in the calculation, and serves as a default when the\n\ -sequence is empty."); - - -static PyObject * -builtin_reload(PyObject *self, PyObject *v) -{ - if (PyErr_WarnPy3k("In 3.x, reload() is renamed to imp.reload()", - 1) < 0) - return NULL; - - return PyImport_ReloadModule(v); -} - -PyDoc_STRVAR(reload_doc, -"reload(module) -> module\n\ -\n\ -Reload the module. The module must have been successfully imported before."); - - -static PyObject * builtin_repr(PyObject *self, PyObject *v) { return PyObject_Repr(v); @@ -2161,66 +1770,59 @@ For most object types, eval(repr(object)) == object."); static PyObject * builtin_round(PyObject *self, PyObject *args, PyObject *kwds) { - double x; - PyObject *o_ndigits = NULL; - Py_ssize_t ndigits; + static PyObject *round_str = NULL; + PyObject *ndigits = NULL; static char *kwlist[] = {"number", "ndigits", 0}; + PyObject *number, *round; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "d|O:round", - kwlist, &x, &o_ndigits)) + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|O:round", + kwlist, &number, &ndigits)) return NULL; - if (o_ndigits == NULL) { - /* second argument defaults to 0 */ - ndigits = 0; + if (Py_TYPE(number)->tp_dict == NULL) { + if (PyType_Ready(Py_TYPE(number)) < 0) + return NULL; } - else { - /* interpret 2nd argument as a Py_ssize_t; clip on overflow */ - ndigits = PyNumber_AsSsize_t(o_ndigits, NULL); - if (ndigits == -1 && PyErr_Occurred()) + + if (round_str == NULL) { + round_str = PyUnicode_InternFromString("__round__"); + if (round_str == NULL) return NULL; } - /* nans, infinities and zeros round to themselves */ - if (!Py_IS_FINITE(x) || x == 0.0) - return PyFloat_FromDouble(x); - - /* Deal with extreme values for ndigits. For ndigits > NDIGITS_MAX, x - always rounds to itself. For ndigits < NDIGITS_MIN, x always - rounds to +-0.0. Here 0.30103 is an upper bound for log10(2). */ -#define NDIGITS_MAX ((int)((DBL_MANT_DIG-DBL_MIN_EXP) * 0.30103)) -#define NDIGITS_MIN (-(int)((DBL_MAX_EXP + 1) * 0.30103)) - if (ndigits > NDIGITS_MAX) - /* return x */ - return PyFloat_FromDouble(x); - else if (ndigits < NDIGITS_MIN) - /* return 0.0, but with sign of x */ - return PyFloat_FromDouble(0.0*x); + round = _PyType_Lookup(Py_TYPE(number), round_str); + if (round == NULL) { + PyErr_Format(PyExc_TypeError, + "type %.100s doesn't define __round__ method", + Py_TYPE(number)->tp_name); + return NULL; + } + + if (ndigits == NULL) + return PyObject_CallFunction(round, "O", number); else - /* finite x, and ndigits is not unreasonably large */ - /* _Py_double_round is defined in floatobject.c */ - return _Py_double_round(x, (int)ndigits); -#undef NDIGITS_MAX -#undef NDIGITS_MIN + return PyObject_CallFunction(round, "OO", number, ndigits); } PyDoc_STRVAR(round_doc, -"round(number[, ndigits]) -> floating point number\n\ +"round(number[, ndigits]) -> number\n\ \n\ Round a number to a given precision in decimal digits (default 0 digits).\n\ -This always returns a floating point number. Precision may be negative."); +This returns an int when called with one argument, otherwise the\n\ +same type as the number. ndigits may be negative."); + static PyObject * builtin_sorted(PyObject *self, PyObject *args, PyObject *kwds) { - PyObject *newlist, *v, *seq, *compare=NULL, *keyfunc=NULL, *newargs; + PyObject *newlist, *v, *seq, *keyfunc=NULL, *newargs; PyObject *callable; - static char *kwlist[] = {"iterable", "cmp", "key", "reverse", 0}; + static char *kwlist[] = {"iterable", "key", "reverse", 0}; int reverse; - /* args 1-4 should match listsort in Objects/listobject.c */ - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|OOi:sorted", - kwlist, &seq, &compare, &keyfunc, &reverse)) + /* args 1-3 should match listsort in Objects/listobject.c */ + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|Oi:sorted", + kwlist, &seq, &keyfunc, &reverse)) return NULL; newlist = PySequence_List(seq); @@ -2252,7 +1854,7 @@ builtin_sorted(PyObject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(sorted_doc, -"sorted(iterable, cmp=None, key=None, reverse=False) --> new sorted list"); +"sorted(iterable, key=None, reverse=False) --> new sorted list"); static PyObject * builtin_vars(PyObject *self, PyObject *args) @@ -2289,7 +1891,6 @@ PyDoc_STRVAR(vars_doc, Without arguments, equivalent to locals().\n\ With an argument, equivalent to object.__dict__."); - static PyObject* builtin_sum(PyObject *self, PyObject *args) { @@ -2305,19 +1906,26 @@ builtin_sum(PyObject *self, PyObject *args) return NULL; if (result == NULL) { - result = PyInt_FromLong(0); + result = PyLong_FromLong(0); if (result == NULL) { Py_DECREF(iter); return NULL; } } else { /* reject string values for 'start' parameter */ - if (PyObject_TypeCheck(result, &PyBaseString_Type)) { + if (PyUnicode_Check(result)) { PyErr_SetString(PyExc_TypeError, "sum() can't sum strings [use ''.join(seq) instead]"); Py_DECREF(iter); return NULL; } + if (PyByteArray_Check(result)) { + PyErr_SetString(PyExc_TypeError, + "sum() can't sum bytes [use b''.join(seq) instead]"); + Py_DECREF(iter); + return NULL; + } + Py_INCREF(result); } @@ -2326,29 +1934,33 @@ builtin_sum(PyObject *self, PyObject *args) Assumes all inputs are the same type. If the assumption fails, default to the more general routine. */ - if (PyInt_CheckExact(result)) { - long i_result = PyInt_AS_LONG(result); - Py_DECREF(result); - result = NULL; + if (PyLong_CheckExact(result)) { + int overflow; + long i_result = PyLong_AsLongAndOverflow(result, &overflow); + /* If this already overflowed, don't even enter the loop. */ + if (overflow == 0) { + Py_DECREF(result); + result = NULL; + } while(result == NULL) { item = PyIter_Next(iter); if (item == NULL) { Py_DECREF(iter); if (PyErr_Occurred()) return NULL; - return PyInt_FromLong(i_result); + return PyLong_FromLong(i_result); } - if (PyInt_CheckExact(item)) { - long b = PyInt_AS_LONG(item); + if (PyLong_CheckExact(item)) { + long b = PyLong_AsLongAndOverflow(item, &overflow); long x = i_result + b; - if ((x^i_result) >= 0 || (x^b) >= 0) { + if (overflow == 0 && ((x^i_result) >= 0 || (x^b) >= 0)) { i_result = x; Py_DECREF(item); continue; } } /* Either overflowed or is not an int. Restore real objects and process normally */ - result = PyInt_FromLong(i_result); + result = PyLong_FromLong(i_result); temp = PyNumber_Add(result, item); Py_DECREF(result); Py_DECREF(item); @@ -2379,12 +1991,17 @@ builtin_sum(PyObject *self, PyObject *args) Py_DECREF(item); continue; } - if (PyInt_CheckExact(item)) { - PyFPE_START_PROTECT("add", Py_DECREF(item); Py_DECREF(iter); return 0) - f_result += (double)PyInt_AS_LONG(item); - PyFPE_END_PROTECT(f_result) - Py_DECREF(item); - continue; + if (PyLong_CheckExact(item)) { + long value; + int overflow; + value = PyLong_AsLongAndOverflow(item, &overflow); + if (!overflow) { + PyFPE_START_PROTECT("add", Py_DECREF(item); Py_DECREF(iter); return 0) + f_result += (double)value; + PyFPE_END_PROTECT(f_result) + Py_DECREF(item); + continue; + } } result = PyFloat_FromDouble(f_result); temp = PyNumber_Add(result, item); @@ -2430,10 +2047,10 @@ builtin_sum(PyObject *self, PyObject *args) } PyDoc_STRVAR(sum_doc, -"sum(sequence[, start]) -> value\n\ +"sum(iterable[, start]) -> value\n\ \n\ -Returns the sum of a sequence of numbers (NOT strings) plus the value\n\ -of parameter 'start' (which defaults to 0). When the sequence is\n\ +Returns the sum of an iterable of numbers (NOT strings) plus the value\n\ +of parameter 'start' (which defaults to 0). When the iterable is\n\ empty, returns start."); @@ -2486,49 +2103,33 @@ When using a tuple as the second argument issubclass(X, (A, B, ...)),\n\ is a shortcut for issubclass(X, A) or issubclass(X, B) or ... (etc.)."); -static PyObject* -builtin_zip(PyObject *self, PyObject *args) +typedef struct { + PyObject_HEAD + Py_ssize_t tuplesize; + PyObject *ittuple; /* tuple of iterators */ + PyObject *result; +} zipobject; + +static PyObject * +zip_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - PyObject *ret; - const Py_ssize_t itemsize = PySequence_Length(args); + zipobject *lz; Py_ssize_t i; - PyObject *itlist; /* tuple of iterators */ - Py_ssize_t len; /* guess at result length */ + PyObject *ittuple; /* tuple of iterators */ + PyObject *result; + Py_ssize_t tuplesize = PySequence_Length(args); - if (itemsize == 0) - return PyList_New(0); + if (type == &PyZip_Type && !_PyArg_NoKeywords("zip()", kwds)) + return NULL; /* args must be a tuple */ assert(PyTuple_Check(args)); - /* Guess at result length: the shortest of the input lengths. - If some argument refuses to say, we refuse to guess too, lest - an argument like xrange(sys.maxint) lead us astray.*/ - len = -1; /* unknown */ - for (i = 0; i < itemsize; ++i) { - PyObject *item = PyTuple_GET_ITEM(args, i); - Py_ssize_t thislen = _PyObject_LengthHint(item, -2); - if (thislen < 0) { - if (thislen == -1) - return NULL; - len = -1; - break; - } - else if (len < 0 || thislen < len) - len = thislen; - } - - /* allocate result list */ - if (len < 0) - len = 10; /* arbitrary */ - if ((ret = PyList_New(len)) == NULL) - return NULL; - /* obtain iterators */ - itlist = PyTuple_New(itemsize); - if (itlist == NULL) - goto Fail_ret; - for (i = 0; i < itemsize; ++i) { + ittuple = PyTuple_New(tuplesize); + if (ittuple == NULL) + return NULL; + for (i=0; i < tuplesize; ++i) { PyObject *item = PyTuple_GET_ITEM(args, i); PyObject *it = PyObject_GetIter(item); if (it == NULL) { @@ -2536,86 +2137,166 @@ builtin_zip(PyObject *self, PyObject *args) PyErr_Format(PyExc_TypeError, "zip argument #%zd must support iteration", i+1); - goto Fail_ret_itlist; - } - PyTuple_SET_ITEM(itlist, i, it); - } - - /* build result into ret list */ - for (i = 0; ; ++i) { - int j; - PyObject *next = PyTuple_New(itemsize); - if (!next) - goto Fail_ret_itlist; - - for (j = 0; j < itemsize; j++) { - PyObject *it = PyTuple_GET_ITEM(itlist, j); - PyObject *item = PyIter_Next(it); - if (!item) { - if (PyErr_Occurred()) { - Py_DECREF(ret); - ret = NULL; - } - Py_DECREF(next); - Py_DECREF(itlist); - goto Done; - } - PyTuple_SET_ITEM(next, j, item); + Py_DECREF(ittuple); + return NULL; } + PyTuple_SET_ITEM(ittuple, i, it); + } - if (i < len) - PyList_SET_ITEM(ret, i, next); - else { - int status = PyList_Append(ret, next); - Py_DECREF(next); - ++len; - if (status < 0) - goto Fail_ret_itlist; - } + /* create a result holder */ + result = PyTuple_New(tuplesize); + if (result == NULL) { + Py_DECREF(ittuple); + return NULL; + } + for (i=0 ; i < tuplesize ; i++) { + Py_INCREF(Py_None); + PyTuple_SET_ITEM(result, i, Py_None); } -Done: - if (ret != NULL && i < len) { - /* The list is too big. */ - if (PyList_SetSlice(ret, i, len, NULL) < 0) - return NULL; + /* create zipobject structure */ + lz = (zipobject *)type->tp_alloc(type, 0); + if (lz == NULL) { + Py_DECREF(ittuple); + Py_DECREF(result); + return NULL; } - return ret; + lz->ittuple = ittuple; + lz->tuplesize = tuplesize; + lz->result = result; -Fail_ret_itlist: - Py_DECREF(itlist); -Fail_ret: - Py_DECREF(ret); - return NULL; + return (PyObject *)lz; +} + +static void +zip_dealloc(zipobject *lz) +{ + PyObject_GC_UnTrack(lz); + Py_XDECREF(lz->ittuple); + Py_XDECREF(lz->result); + Py_TYPE(lz)->tp_free(lz); +} + +static int +zip_traverse(zipobject *lz, visitproc visit, void *arg) +{ + Py_VISIT(lz->ittuple); + Py_VISIT(lz->result); + return 0; } +static PyObject * +zip_next(zipobject *lz) +{ + Py_ssize_t i; + Py_ssize_t tuplesize = lz->tuplesize; + PyObject *result = lz->result; + PyObject *it; + PyObject *item; + PyObject *olditem; + + if (tuplesize == 0) + return NULL; + if (Py_REFCNT(result) == 1) { + Py_INCREF(result); + for (i=0 ; i < tuplesize ; i++) { + it = PyTuple_GET_ITEM(lz->ittuple, i); + item = (*Py_TYPE(it)->tp_iternext)(it); + if (item == NULL) { + Py_DECREF(result); + return NULL; + } + olditem = PyTuple_GET_ITEM(result, i); + PyTuple_SET_ITEM(result, i, item); + Py_DECREF(olditem); + } + } else { + result = PyTuple_New(tuplesize); + if (result == NULL) + return NULL; + for (i=0 ; i < tuplesize ; i++) { + it = PyTuple_GET_ITEM(lz->ittuple, i); + item = (*Py_TYPE(it)->tp_iternext)(it); + if (item == NULL) { + Py_DECREF(result); + return NULL; + } + PyTuple_SET_ITEM(result, i, item); + } + } + return result; +} PyDoc_STRVAR(zip_doc, -"zip(seq1 [, seq2 [...]]) -> [(seq1[0], seq2[0] ...), (...)]\n\ +"zip(iter1 [,iter2 [...]]) --> zip object\n\ \n\ -Return a list of tuples, where each tuple contains the i-th element\n\ -from each of the argument sequences. The returned list is truncated\n\ -in length to the length of the shortest argument sequence."); +Return a zip object whose .__next__() method returns a tuple where\n\ +the i-th element comes from the i-th iterable argument. The .__next__()\n\ +method continues until the shortest iterable in the argument sequence\n\ +is exhausted and then it raises StopIteration."); + +PyTypeObject PyZip_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "zip", /* tp_name */ + sizeof(zipobject), /* tp_basicsize */ + 0, /* tp_itemsize */ + /* methods */ + (destructor)zip_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_BASETYPE, /* tp_flags */ + zip_doc, /* tp_doc */ + (traverseproc)zip_traverse, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + PyObject_SelfIter, /* tp_iter */ + (iternextfunc)zip_next, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + PyType_GenericAlloc, /* tp_alloc */ + zip_new, /* tp_new */ + PyObject_GC_Del, /* tp_free */ +}; static PyMethodDef builtin_methods[] = { + {"__build_class__", (PyCFunction)builtin___build_class__, + METH_VARARGS | METH_KEYWORDS, build_class_doc}, {"__import__", (PyCFunction)builtin___import__, METH_VARARGS | METH_KEYWORDS, import_doc}, {"abs", builtin_abs, METH_O, abs_doc}, {"all", builtin_all, METH_O, all_doc}, {"any", builtin_any, METH_O, any_doc}, - {"apply", builtin_apply, METH_VARARGS, apply_doc}, + {"ascii", builtin_ascii, METH_O, ascii_doc}, {"bin", builtin_bin, METH_O, bin_doc}, {"callable", builtin_callable, METH_O, callable_doc}, {"chr", builtin_chr, METH_VARARGS, chr_doc}, - {"cmp", builtin_cmp, METH_VARARGS, cmp_doc}, - {"coerce", builtin_coerce, METH_VARARGS, coerce_doc}, {"compile", (PyCFunction)builtin_compile, METH_VARARGS | METH_KEYWORDS, compile_doc}, {"delattr", builtin_delattr, METH_VARARGS, delattr_doc}, {"dir", builtin_dir, METH_VARARGS, dir_doc}, {"divmod", builtin_divmod, METH_VARARGS, divmod_doc}, {"eval", builtin_eval, METH_VARARGS, eval_doc}, - {"execfile", builtin_execfile, METH_VARARGS, execfile_doc}, - {"filter", builtin_filter, METH_VARARGS, filter_doc}, + {"exec", builtin_exec, METH_VARARGS, exec_doc}, {"format", builtin_format, METH_VARARGS, format_doc}, {"getattr", builtin_getattr, METH_VARARGS, getattr_doc}, {"globals", (PyCFunction)builtin_globals, METH_NOARGS, globals_doc}, @@ -2624,35 +2305,24 @@ static PyMethodDef builtin_methods[] = { {"hex", builtin_hex, METH_O, hex_doc}, {"id", builtin_id, METH_O, id_doc}, {"input", builtin_input, METH_VARARGS, input_doc}, - {"intern", builtin_intern, METH_VARARGS, intern_doc}, {"isinstance", builtin_isinstance, METH_VARARGS, isinstance_doc}, {"issubclass", builtin_issubclass, METH_VARARGS, issubclass_doc}, {"iter", builtin_iter, METH_VARARGS, iter_doc}, {"len", builtin_len, METH_O, len_doc}, {"locals", (PyCFunction)builtin_locals, METH_NOARGS, locals_doc}, - {"map", builtin_map, METH_VARARGS, map_doc}, {"max", (PyCFunction)builtin_max, METH_VARARGS | METH_KEYWORDS, max_doc}, {"min", (PyCFunction)builtin_min, METH_VARARGS | METH_KEYWORDS, min_doc}, - {"next", builtin_next, METH_VARARGS, next_doc}, + {"next", (PyCFunction)builtin_next, METH_VARARGS, next_doc}, {"oct", builtin_oct, METH_O, oct_doc}, - {"open", (PyCFunction)builtin_open, METH_VARARGS | METH_KEYWORDS, open_doc}, {"ord", builtin_ord, METH_O, ord_doc}, {"pow", builtin_pow, METH_VARARGS, pow_doc}, {"print", (PyCFunction)builtin_print, METH_VARARGS | METH_KEYWORDS, print_doc}, - {"range", builtin_range, METH_VARARGS, range_doc}, - {"raw_input", builtin_raw_input, METH_VARARGS, raw_input_doc}, - {"reduce", builtin_reduce, METH_VARARGS, reduce_doc}, - {"reload", builtin_reload, METH_O, reload_doc}, {"repr", builtin_repr, METH_O, repr_doc}, {"round", (PyCFunction)builtin_round, METH_VARARGS | METH_KEYWORDS, round_doc}, {"setattr", builtin_setattr, METH_VARARGS, setattr_doc}, {"sorted", (PyCFunction)builtin_sorted, METH_VARARGS | METH_KEYWORDS, sorted_doc}, {"sum", builtin_sum, METH_VARARGS, sum_doc}, -#ifdef Py_USING_UNICODE - {"unichr", builtin_unichr, METH_VARARGS, unichr_doc}, -#endif {"vars", builtin_vars, METH_VARARGS, vars_doc}, - {"zip", builtin_zip, METH_VARARGS, zip_doc}, {NULL, NULL}, }; @@ -2661,19 +2331,30 @@ PyDoc_STRVAR(builtin_doc, \n\ Noteworthy: None is the `nil' object; Ellipsis represents `...' in slices."); +static struct PyModuleDef builtinsmodule = { + PyModuleDef_HEAD_INIT, + "builtins", + builtin_doc, + -1, /* multiple "initialization" just copies the module dict. */ + builtin_methods, + NULL, + NULL, + NULL, + NULL +}; + + PyObject * _PyBuiltin_Init(void) { PyObject *mod, *dict, *debug; - mod = Py_InitModule4("__builtin__", builtin_methods, - builtin_doc, (PyObject *)NULL, - PYTHON_API_VERSION); + mod = PyModule_Create(&builtinsmodule); if (mod == NULL) return NULL; dict = PyModule_GetDict(mod); #ifdef Py_TRACE_REFS - /* __builtin__ exposes a number of statically allocated objects + /* "builtins" exposes a number of statically allocated objects * that, before this code was added in 2.3, never showed up in * the list of "all objects" maintained by Py_TRACE_REFS. As a * result, programs leaking references to None and False (etc) @@ -2694,38 +2375,32 @@ _PyBuiltin_Init(void) SETBUILTIN("NotImplemented", Py_NotImplemented); SETBUILTIN("False", Py_False); SETBUILTIN("True", Py_True); - SETBUILTIN("basestring", &PyBaseString_Type); SETBUILTIN("bool", &PyBool_Type); SETBUILTIN("memoryview", &PyMemoryView_Type); SETBUILTIN("bytearray", &PyByteArray_Type); - SETBUILTIN("bytes", &PyString_Type); - SETBUILTIN("buffer", &PyBuffer_Type); + SETBUILTIN("bytes", &PyBytes_Type); SETBUILTIN("classmethod", &PyClassMethod_Type); -#ifndef WITHOUT_COMPLEX SETBUILTIN("complex", &PyComplex_Type); -#endif SETBUILTIN("dict", &PyDict_Type); SETBUILTIN("enumerate", &PyEnum_Type); - SETBUILTIN("file", &PyFile_Type); + SETBUILTIN("filter", &PyFilter_Type); SETBUILTIN("float", &PyFloat_Type); SETBUILTIN("frozenset", &PyFrozenSet_Type); SETBUILTIN("property", &PyProperty_Type); - SETBUILTIN("int", &PyInt_Type); + SETBUILTIN("int", &PyLong_Type); SETBUILTIN("list", &PyList_Type); - SETBUILTIN("long", &PyLong_Type); + SETBUILTIN("map", &PyMap_Type); SETBUILTIN("object", &PyBaseObject_Type); + SETBUILTIN("range", &PyRange_Type); SETBUILTIN("reversed", &PyReversed_Type); SETBUILTIN("set", &PySet_Type); SETBUILTIN("slice", &PySlice_Type); SETBUILTIN("staticmethod", &PyStaticMethod_Type); - SETBUILTIN("str", &PyString_Type); + SETBUILTIN("str", &PyUnicode_Type); SETBUILTIN("super", &PySuper_Type); SETBUILTIN("tuple", &PyTuple_Type); SETBUILTIN("type", &PyType_Type); - SETBUILTIN("xrange", &PyRange_Type); -#ifdef Py_USING_UNICODE - SETBUILTIN("unicode", &PyUnicode_Type); -#endif + SETBUILTIN("zip", &PyZip_Type); debug = PyBool_FromLong(Py_OptimizeFlag == 0); if (PyDict_SetItemString(dict, "__debug__", debug) < 0) { Py_XDECREF(debug); @@ -2737,323 +2412,3 @@ _PyBuiltin_Init(void) #undef ADD_TO_ALL #undef SETBUILTIN } - -/* Helper for filter(): filter a tuple through a function */ - -static PyObject * -filtertuple(PyObject *func, PyObject *tuple) -{ - PyObject *result; - Py_ssize_t i, j; - Py_ssize_t len = PyTuple_Size(tuple); - - if (len == 0) { - if (PyTuple_CheckExact(tuple)) - Py_INCREF(tuple); - else - tuple = PyTuple_New(0); - return tuple; - } - - if ((result = PyTuple_New(len)) == NULL) - return NULL; - - for (i = j = 0; i < len; ++i) { - PyObject *item, *good; - int ok; - - if (tuple->ob_type->tp_as_sequence && - tuple->ob_type->tp_as_sequence->sq_item) { - item = tuple->ob_type->tp_as_sequence->sq_item(tuple, i); - if (item == NULL) - goto Fail_1; - } else { - PyErr_SetString(PyExc_TypeError, "filter(): unsubscriptable tuple"); - goto Fail_1; - } - if (func == Py_None) { - Py_INCREF(item); - good = item; - } - else { - PyObject *arg = PyTuple_Pack(1, item); - if (arg == NULL) { - Py_DECREF(item); - goto Fail_1; - } - good = PyEval_CallObject(func, arg); - Py_DECREF(arg); - if (good == NULL) { - Py_DECREF(item); - goto Fail_1; - } - } - ok = PyObject_IsTrue(good); - Py_DECREF(good); - if (ok > 0) { - if (PyTuple_SetItem(result, j++, item) < 0) - goto Fail_1; - } - else { - Py_DECREF(item); - if (ok < 0) - goto Fail_1; - } - } - - if (_PyTuple_Resize(&result, j) < 0) - return NULL; - - return result; - -Fail_1: - Py_DECREF(result); - return NULL; -} - - -/* Helper for filter(): filter a string through a function */ - -static PyObject * -filterstring(PyObject *func, PyObject *strobj) -{ - PyObject *result; - Py_ssize_t i, j; - Py_ssize_t len = PyString_Size(strobj); - Py_ssize_t outlen = len; - - if (func == Py_None) { - /* If it's a real string we can return the original, - * as no character is ever false and __getitem__ - * does return this character. If it's a subclass - * we must go through the __getitem__ loop */ - if (PyString_CheckExact(strobj)) { - Py_INCREF(strobj); - return strobj; - } - } - if ((result = PyString_FromStringAndSize(NULL, len)) == NULL) - return NULL; - - for (i = j = 0; i < len; ++i) { - PyObject *item; - int ok; - - item = (*strobj->ob_type->tp_as_sequence->sq_item)(strobj, i); - if (item == NULL) - goto Fail_1; - if (func==Py_None) { - ok = 1; - } else { - PyObject *arg, *good; - arg = PyTuple_Pack(1, item); - if (arg == NULL) { - Py_DECREF(item); - goto Fail_1; - } - good = PyEval_CallObject(func, arg); - Py_DECREF(arg); - if (good == NULL) { - Py_DECREF(item); - goto Fail_1; - } - ok = PyObject_IsTrue(good); - Py_DECREF(good); - } - if (ok > 0) { - Py_ssize_t reslen; - if (!PyString_Check(item)) { - PyErr_SetString(PyExc_TypeError, "can't filter str to str:" - " __getitem__ returned different type"); - Py_DECREF(item); - goto Fail_1; - } - reslen = PyString_GET_SIZE(item); - if (reslen == 1) { - PyString_AS_STRING(result)[j++] = - PyString_AS_STRING(item)[0]; - } else { - /* do we need more space? */ - Py_ssize_t need = j; - - /* calculate space requirements while checking for overflow */ - if (need > PY_SSIZE_T_MAX - reslen) { - Py_DECREF(item); - goto Fail_1; - } - - need += reslen; - - if (need > PY_SSIZE_T_MAX - len) { - Py_DECREF(item); - goto Fail_1; - } - - need += len; - - if (need <= i) { - Py_DECREF(item); - goto Fail_1; - } - - need = need - i - 1; - - assert(need >= 0); - assert(outlen >= 0); - - if (need > outlen) { - /* overallocate, to avoid reallocations */ - if (outlen > PY_SSIZE_T_MAX / 2) { - Py_DECREF(item); - return NULL; - } - - if (need<2*outlen) { - need = 2*outlen; - } - if (_PyString_Resize(&result, need)) { - Py_DECREF(item); - return NULL; - } - outlen = need; - } - memcpy( - PyString_AS_STRING(result) + j, - PyString_AS_STRING(item), - reslen - ); - j += reslen; - } - } - Py_DECREF(item); - if (ok < 0) - goto Fail_1; - } - - if (j < outlen) - _PyString_Resize(&result, j); - - return result; - -Fail_1: - Py_DECREF(result); - return NULL; -} - -#ifdef Py_USING_UNICODE -/* Helper for filter(): filter a Unicode object through a function */ - -static PyObject * -filterunicode(PyObject *func, PyObject *strobj) -{ - PyObject *result; - register Py_ssize_t i, j; - Py_ssize_t len = PyUnicode_GetSize(strobj); - Py_ssize_t outlen = len; - - if (func == Py_None) { - /* If it's a real string we can return the original, - * as no character is ever false and __getitem__ - * does return this character. If it's a subclass - * we must go through the __getitem__ loop */ - if (PyUnicode_CheckExact(strobj)) { - Py_INCREF(strobj); - return strobj; - } - } - if ((result = PyUnicode_FromUnicode(NULL, len)) == NULL) - return NULL; - - for (i = j = 0; i < len; ++i) { - PyObject *item, *arg, *good; - int ok; - - item = (*strobj->ob_type->tp_as_sequence->sq_item)(strobj, i); - if (item == NULL) - goto Fail_1; - if (func == Py_None) { - ok = 1; - } else { - arg = PyTuple_Pack(1, item); - if (arg == NULL) { - Py_DECREF(item); - goto Fail_1; - } - good = PyEval_CallObject(func, arg); - Py_DECREF(arg); - if (good == NULL) { - Py_DECREF(item); - goto Fail_1; - } - ok = PyObject_IsTrue(good); - Py_DECREF(good); - } - if (ok > 0) { - Py_ssize_t reslen; - if (!PyUnicode_Check(item)) { - PyErr_SetString(PyExc_TypeError, - "can't filter unicode to unicode:" - " __getitem__ returned different type"); - Py_DECREF(item); - goto Fail_1; - } - reslen = PyUnicode_GET_SIZE(item); - if (reslen == 1) - PyUnicode_AS_UNICODE(result)[j++] = - PyUnicode_AS_UNICODE(item)[0]; - else { - /* do we need more space? */ - Py_ssize_t need = j + reslen + len - i - 1; - - /* check that didnt overflow */ - if ((j > PY_SSIZE_T_MAX - reslen) || - ((j + reslen) > PY_SSIZE_T_MAX - len) || - ((j + reslen + len) < i) || - ((j + reslen + len - i) <= 0)) { - Py_DECREF(item); - return NULL; - } - - assert(need >= 0); - assert(outlen >= 0); - - if (need > outlen) { - /* overallocate, - to avoid reallocations */ - if (need < 2 * outlen) { - if (outlen > PY_SSIZE_T_MAX / 2) { - Py_DECREF(item); - return NULL; - } else { - need = 2 * outlen; - } - } - - if (PyUnicode_Resize( - &result, need) < 0) { - Py_DECREF(item); - goto Fail_1; - } - outlen = need; - } - memcpy(PyUnicode_AS_UNICODE(result) + j, - PyUnicode_AS_UNICODE(item), - reslen*sizeof(Py_UNICODE)); - j += reslen; - } - } - Py_DECREF(item); - if (ok < 0) - goto Fail_1; - } - - if (j < outlen) - PyUnicode_Resize(&result, j); - - return result; - -Fail_1: - Py_DECREF(result); - return NULL; -} -#endif diff --git a/Python/ceval.c b/Python/ceval.c index 06ada97274..f0ea7c90dc 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -13,7 +13,6 @@ #include "code.h" #include "frameobject.h" -#include "eval.h" #include "opcode.h" #include "structmember.h" @@ -132,21 +131,13 @@ static void call_exc_trace(Py_tracefunc, PyObject *, PyFrameObject *); static int maybe_call_line_trace(Py_tracefunc, PyObject *, PyFrameObject *, int *, int *, int *); -static PyObject * apply_slice(PyObject *, PyObject *, PyObject *); -static int assign_slice(PyObject *, PyObject *, - PyObject *, PyObject *); static PyObject * cmp_outcome(int, PyObject *, PyObject *); static PyObject * import_from(PyObject *, PyObject *); static int import_all_from(PyObject *, PyObject *); -static PyObject * build_class(PyObject *, PyObject *, PyObject *); -static int exec_statement(PyFrameObject *, - PyObject *, PyObject *, PyObject *); -static void set_exc_info(PyThreadState *, PyObject *, PyObject *, PyObject *); -static void reset_exc_info(PyThreadState *); -static void format_exc_check_arg(PyObject *, char *, PyObject *); -static PyObject * string_concatenate(PyObject *, PyObject *, - PyFrameObject *, unsigned char *); -static PyObject * kwd_as_string(PyObject *); +static void format_exc_check_arg(PyObject *, const char *, PyObject *); +static void format_exc_unbound(PyCodeObject *co, int oparg); +static PyObject * unicode_concatenate(PyObject *, PyObject *, + PyFrameObject *, unsigned char *); static PyObject * special_lookup(PyObject *, char *, PyObject **); #define NAME_ERROR_MSG \ @@ -227,42 +218,127 @@ PyEval_GetCallStats(PyObject *self) #ifdef WITH_THREAD +#define GIL_REQUEST _Py_atomic_load_relaxed(&gil_drop_request) +#else +#define GIL_REQUEST 0 +#endif + +/* This can set eval_breaker to 0 even though gil_drop_request became + 1. We believe this is all right because the eval loop will release + the GIL eventually anyway. */ +#define COMPUTE_EVAL_BREAKER() \ + _Py_atomic_store_relaxed( \ + &eval_breaker, \ + GIL_REQUEST | \ + _Py_atomic_load_relaxed(&pendingcalls_to_do) | \ + pending_async_exc) + +#ifdef WITH_THREAD + +#define SET_GIL_DROP_REQUEST() \ + do { \ + _Py_atomic_store_relaxed(&gil_drop_request, 1); \ + _Py_atomic_store_relaxed(&eval_breaker, 1); \ + } while (0) + +#define RESET_GIL_DROP_REQUEST() \ + do { \ + _Py_atomic_store_relaxed(&gil_drop_request, 0); \ + COMPUTE_EVAL_BREAKER(); \ + } while (0) + +#endif + +/* Pending calls are only modified under pending_lock */ +#define SIGNAL_PENDING_CALLS() \ + do { \ + _Py_atomic_store_relaxed(&pendingcalls_to_do, 1); \ + _Py_atomic_store_relaxed(&eval_breaker, 1); \ + } while (0) + +#define UNSIGNAL_PENDING_CALLS() \ + do { \ + _Py_atomic_store_relaxed(&pendingcalls_to_do, 0); \ + COMPUTE_EVAL_BREAKER(); \ + } while (0) + +#define SIGNAL_ASYNC_EXC() \ + do { \ + pending_async_exc = 1; \ + _Py_atomic_store_relaxed(&eval_breaker, 1); \ + } while (0) + +#define UNSIGNAL_ASYNC_EXC() \ + do { pending_async_exc = 0; COMPUTE_EVAL_BREAKER(); } while (0) + + +#ifdef WITH_THREAD #ifdef HAVE_ERRNO_H #include <errno.h> #endif #include "pythread.h" -static PyThread_type_lock interpreter_lock = 0; /* This is the GIL */ static PyThread_type_lock pending_lock = 0; /* for pending calls */ static long main_thread = 0; +/* This single variable consolidates all requests to break out of the fast path + in the eval loop. */ +static _Py_atomic_int eval_breaker = {0}; +/* Request for dropping the GIL */ +static _Py_atomic_int gil_drop_request = {0}; +/* Request for running pending calls. */ +static _Py_atomic_int pendingcalls_to_do = {0}; +/* Request for looking at the `async_exc` field of the current thread state. + Guarded by the GIL. */ +static int pending_async_exc = 0; + +#include "ceval_gil.h" int PyEval_ThreadsInitialized(void) { - return interpreter_lock != 0; + return gil_created(); } void PyEval_InitThreads(void) { - if (interpreter_lock) + if (gil_created()) return; - interpreter_lock = PyThread_allocate_lock(); - PyThread_acquire_lock(interpreter_lock, 1); + create_gil(); + take_gil(PyThreadState_GET()); main_thread = PyThread_get_thread_ident(); + if (!pending_lock) + pending_lock = PyThread_allocate_lock(); +} + +void +_PyEval_FiniThreads(void) +{ + if (!gil_created()) + return; + destroy_gil(); + assert(!gil_created()); } void PyEval_AcquireLock(void) { - PyThread_acquire_lock(interpreter_lock, 1); + PyThreadState *tstate = PyThreadState_GET(); + if (tstate == NULL) + Py_FatalError("PyEval_AcquireLock: current thread state is NULL"); + take_gil(tstate); } void PyEval_ReleaseLock(void) { - PyThread_release_lock(interpreter_lock); + /* This function must succeed when the current thread state is NULL. + We therefore avoid PyThreadState_GET() which dumps a fatal error + in debug mode. + */ + drop_gil((PyThreadState*)_Py_atomic_load_relaxed( + &_PyThreadState_Current)); } void @@ -271,8 +347,8 @@ PyEval_AcquireThread(PyThreadState *tstate) if (tstate == NULL) Py_FatalError("PyEval_AcquireThread: NULL new thread state"); /* Check someone has called PyEval_InitThreads() to create the lock */ - assert(interpreter_lock); - PyThread_acquire_lock(interpreter_lock, 1); + assert(gil_created()); + take_gil(tstate); if (PyThreadState_Swap(tstate) != NULL) Py_FatalError( "PyEval_AcquireThread: non-NULL old thread state"); @@ -285,7 +361,7 @@ PyEval_ReleaseThread(PyThreadState *tstate) Py_FatalError("PyEval_ReleaseThread: NULL thread state"); if (PyThreadState_Swap(NULL) != tstate) Py_FatalError("PyEval_ReleaseThread: wrong thread state"); - PyThread_release_lock(interpreter_lock); + drop_gil(tstate); } /* This function is called from PyOS_AfterFork to ensure that newly @@ -297,17 +373,13 @@ void PyEval_ReInitThreads(void) { PyObject *threading, *result; - PyThreadState *tstate; + PyThreadState *tstate = PyThreadState_GET(); - if (!interpreter_lock) + if (!gil_created()) return; - /*XXX Can't use PyThread_free_lock here because it does too - much error-checking. Doing this cleanly would require - adding a new function to each thread_*.h. Instead, just - create a new lock and waste a little bit of memory */ - interpreter_lock = PyThread_allocate_lock(); + recreate_gil(); pending_lock = PyThread_allocate_lock(); - PyThread_acquire_lock(interpreter_lock, 1); + take_gil(tstate); main_thread = PyThread_get_thread_ident(); /* Update the threading module with the new state. @@ -327,7 +399,20 @@ PyEval_ReInitThreads(void) Py_DECREF(result); Py_DECREF(threading); } -#endif + +#else +static _Py_atomic_int eval_breaker = {0}; +static int pending_async_exc = 0; +#endif /* WITH_THREAD */ + +/* This function is used to signal that async exceptions are waiting to be + raised, therefore it is also useful in non-threaded builds. */ + +void +_PyEval_SignalAsyncExc(void) +{ + SIGNAL_ASYNC_EXC(); +} /* Functions save_thread and restore_thread are always defined so dynamically loaded modules needn't be compiled separately for use @@ -340,8 +425,8 @@ PyEval_SaveThread(void) if (tstate == NULL) Py_FatalError("PyEval_SaveThread: NULL tstate"); #ifdef WITH_THREAD - if (interpreter_lock) - PyThread_release_lock(interpreter_lock); + if (gil_created()) + drop_gil(tstate); #endif return tstate; } @@ -352,9 +437,15 @@ PyEval_RestoreThread(PyThreadState *tstate) if (tstate == NULL) Py_FatalError("PyEval_RestoreThread: NULL tstate"); #ifdef WITH_THREAD - if (interpreter_lock) { + if (gil_created()) { int err = errno; - PyThread_acquire_lock(interpreter_lock, 1); + take_gil(tstate); + /* _Py_Finalizing is protected by the GIL */ + if (_Py_Finalizing && tstate != _Py_Finalizing) { + drop_gil(tstate); + PyThread_exit_thread(); + assert(0); /* unreachable */ + } errno = err; } #endif @@ -400,7 +491,6 @@ static struct { } pendingcalls[NPENDINGCALLS]; static int pendingfirst = 0; static int pendinglast = 0; -static volatile int pendingcalls_to_do = 1; /* trigger initialization of lock */ static char pendingbusy = 0; int @@ -439,8 +529,7 @@ Py_AddPendingCall(int (*func)(void *), void *arg) pendinglast = j; } /* signal main loop */ - _Py_Ticker = 0; - pendingcalls_to_do = 1; + SIGNAL_PENDING_CALLS(); if (lock != NULL) PyThread_release_lock(lock); return result; @@ -482,7 +571,10 @@ Py_MakePendingCalls(void) arg = pendingcalls[j].arg; pendingfirst = (j + 1) % NPENDINGCALLS; } - pendingcalls_to_do = pendingfirst != pendinglast; + if (pendingfirst != pendinglast) + SIGNAL_PENDING_CALLS(); + else + UNSIGNAL_PENDING_CALLS(); PyThread_release_lock(pending_lock); /* having released the lock, perform the callback */ if (func == NULL) @@ -527,7 +619,7 @@ static struct { } pendingcalls[NPENDINGCALLS]; static volatile int pendingfirst = 0; static volatile int pendinglast = 0; -static volatile int pendingcalls_to_do = 0; +static _Py_atomic_int pendingcalls_to_do = {0}; int Py_AddPendingCall(int (*func)(void *), void *arg) @@ -548,8 +640,7 @@ Py_AddPendingCall(int (*func)(void *), void *arg) pendingcalls[i].arg = arg; pendinglast = j; - _Py_Ticker = 0; - pendingcalls_to_do = 1; /* Signal main loop */ + SIGNAL_PENDING_CALLS(); busy = 0; /* XXX End critical section */ return 0; @@ -562,7 +653,7 @@ Py_MakePendingCalls(void) if (busy) return 0; busy = 1; - pendingcalls_to_do = 0; + UNSIGNAL_PENDING_CALLS(); for (;;) { int i; int (*func)(void *); @@ -575,7 +666,7 @@ Py_MakePendingCalls(void) pendingfirst = (i + 1) % NPENDINGCALLS; if (func(arg) < 0) { busy = 0; - pendingcalls_to_do = 1; /* We're not done yet */ + SIGNAL_PENDING_CALLS(); /* We're not done yet */ return -1; } } @@ -624,14 +715,25 @@ _Py_CheckRecursiveCall(char *where) return -1; } #endif + _Py_CheckRecursionLimit = recursion_limit; + if (tstate->recursion_critical) + /* Somebody asked that we don't check for recursion. */ + return 0; + if (tstate->overflowed) { + if (tstate->recursion_depth > recursion_limit + 50) { + /* Overflowing while handling an overflow. Give up. */ + Py_FatalError("Cannot recover from stack overflow."); + } + return 0; + } if (tstate->recursion_depth > recursion_limit) { --tstate->recursion_depth; + tstate->overflowed = 1; PyErr_Format(PyExc_RuntimeError, "maximum recursion depth exceeded%s", where); return -1; } - _Py_CheckRecursionLimit = recursion_limit; return 0; } @@ -643,11 +745,12 @@ enum why_code { WHY_RETURN = 0x0008, /* 'return' statement */ WHY_BREAK = 0x0010, /* 'break' statement */ WHY_CONTINUE = 0x0020, /* 'continue' statement */ - WHY_YIELD = 0x0040 /* 'yield' operator */ + WHY_YIELD = 0x0040, /* 'yield' operator */ + WHY_SILENCED = 0x0080 /* Exception silenced by 'with' */ }; -static enum why_code do_raise(PyObject *, PyObject *, PyObject *); -static int unpack_iterable(PyObject *, int, PyObject **); +static enum why_code do_raise(PyObject *, PyObject *); +static int unpack_iterable(PyObject *, int, int, PyObject **); /* Records whether tracing is on for any thread. Counts the number of threads for which tstate->c_tracefunc is non-NULL, so if the value @@ -656,20 +759,17 @@ static int unpack_iterable(PyObject *, int, PyObject **); fast_next_opcode*/ static int _Py_TracingPossible = 0; -/* for manipulating the thread switch and periodic "stuff" - used to be - per thread, now just a pair o' globals */ -int _Py_CheckInterval = 100; -volatile int _Py_Ticker = 0; /* so that we hit a "tick" first thing */ + PyObject * -PyEval_EvalCode(PyCodeObject *co, PyObject *globals, PyObject *locals) +PyEval_EvalCode(PyObject *co, PyObject *globals, PyObject *locals) { return PyEval_EvalCodeEx(co, globals, locals, (PyObject **)NULL, 0, (PyObject **)NULL, 0, (PyObject **)NULL, 0, - NULL); + NULL, NULL); } @@ -700,7 +800,6 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) register PyObject *w; register PyObject *u; register PyObject *t; - register PyObject *stream = NULL; /* for PRINT opcodes */ register PyObject **fastlocals, **freevars; PyObject *retval = NULL; /* Return value */ PyThreadState *tstate = PyThreadState_GET(); @@ -723,6 +822,124 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) char *filename; #endif +/* Computed GOTOs, or + the-optimization-commonly-but-improperly-known-as-"threaded code" + using gcc's labels-as-values extension + (http://gcc.gnu.org/onlinedocs/gcc/Labels-as-Values.html). + + The traditional bytecode evaluation loop uses a "switch" statement, which + decent compilers will optimize as a single indirect branch instruction + combined with a lookup table of jump addresses. However, since the + indirect jump instruction is shared by all opcodes, the CPU will have a + hard time making the right prediction for where to jump next (actually, + it will be always wrong except in the uncommon case of a sequence of + several identical opcodes). + + "Threaded code" in contrast, uses an explicit jump table and an explicit + indirect jump instruction at the end of each opcode. Since the jump + instruction is at a different address for each opcode, the CPU will make a + separate prediction for each of these instructions, which is equivalent to + predicting the second opcode of each opcode pair. These predictions have + a much better chance to turn out valid, especially in small bytecode loops. + + A mispredicted branch on a modern CPU flushes the whole pipeline and + can cost several CPU cycles (depending on the pipeline depth), + and potentially many more instructions (depending on the pipeline width). + A correctly predicted branch, however, is nearly free. + + At the time of this writing, the "threaded code" version is up to 15-20% + faster than the normal "switch" version, depending on the compiler and the + CPU architecture. + + We disable the optimization if DYNAMIC_EXECUTION_PROFILE is defined, + because it would render the measurements invalid. + + + NOTE: care must be taken that the compiler doesn't try to "optimize" the + indirect jumps by sharing them between all opcodes. Such optimizations + can be disabled on gcc by using the -fno-gcse flag (or possibly + -fno-crossjumping). +*/ + +#ifdef DYNAMIC_EXECUTION_PROFILE +#undef USE_COMPUTED_GOTOS +#define USE_COMPUTED_GOTOS 0 +#endif + +#ifdef HAVE_COMPUTED_GOTOS + #ifndef USE_COMPUTED_GOTOS + #define USE_COMPUTED_GOTOS 1 + #endif +#else + #if defined(USE_COMPUTED_GOTOS) && USE_COMPUTED_GOTOS + #error "Computed gotos are not supported on this compiler." + #endif + #undef USE_COMPUTED_GOTOS + #define USE_COMPUTED_GOTOS 0 +#endif + +#if USE_COMPUTED_GOTOS +/* Import the static jump table */ +#include "opcode_targets.h" + +/* This macro is used when several opcodes defer to the same implementation + (e.g. SETUP_LOOP, SETUP_FINALLY) */ +#define TARGET_WITH_IMPL(op, impl) \ + TARGET_##op: \ + opcode = op; \ + if (HAS_ARG(op)) \ + oparg = NEXTARG(); \ + case op: \ + goto impl; \ + +#define TARGET(op) \ + TARGET_##op: \ + opcode = op; \ + if (HAS_ARG(op)) \ + oparg = NEXTARG(); \ + case op: + + +#define DISPATCH() \ + { \ + if (!_Py_atomic_load_relaxed(&eval_breaker)) { \ + FAST_DISPATCH(); \ + } \ + continue; \ + } + +#ifdef LLTRACE +#define FAST_DISPATCH() \ + { \ + if (!lltrace && !_Py_TracingPossible) { \ + f->f_lasti = INSTR_OFFSET(); \ + goto *opcode_targets[*next_instr++]; \ + } \ + goto fast_next_opcode; \ + } +#else +#define FAST_DISPATCH() \ + { \ + if (!_Py_TracingPossible) { \ + f->f_lasti = INSTR_OFFSET(); \ + goto *opcode_targets[*next_instr++]; \ + } \ + goto fast_next_opcode; \ + } +#endif + +#else +#define TARGET(op) \ + case op: +#define TARGET_WITH_IMPL(op, impl) \ + /* silence compiler warnings about `impl` unused */ \ + if (0) goto impl; \ + case op: +#define DISPATCH() continue +#define FAST_DISPATCH() goto fast_next_opcode +#endif + + /* Tuple access macros */ #ifndef Py_DEBUG @@ -754,7 +971,6 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) It's a case-by-case judgement. I'll use intr1 for the following cases: - EXEC_STMT IMPORT_STAR IMPORT_FROM CALL_FUNCTION (and friends) @@ -784,8 +1000,8 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) /* OpCode prediction macros Some opcodes tend to come in pairs thus making it possible to predict the second code when the first is run. For example, - GET_ITER is often followed by FOR_ITER. And FOR_ITER is often - followed by STORE_FAST or UNPACK_SEQUENCE. + COMPARE_OP is often followed by JUMP_IF_FALSE or JUMP_IF_TRUE. And, + those opcodes are often followed by a POP_TOP. Verifying the prediction costs a single high-speed test of a register variable against a constant. If the pairing was good, then the @@ -801,16 +1017,23 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) predictions turned-on and interpret the results as if some opcodes had been combined or turn-off predictions so that the opcode frequency counter updates for both opcodes. + + Opcode prediction is disabled with threaded code, since the latter allows + the CPU to record separate branch prediction information for each + opcode. + */ -#ifdef DYNAMIC_EXECUTION_PROFILE +#if defined(DYNAMIC_EXECUTION_PROFILE) || USE_COMPUTED_GOTOS #define PREDICT(op) if (0) goto PRED_##op +#define PREDICTED(op) PRED_##op: +#define PREDICTED_WITH_ARG(op) PRED_##op: #else #define PREDICT(op) if (*next_instr == op) goto PRED_##op -#endif - #define PREDICTED(op) PRED_##op: next_instr++ #define PREDICTED_WITH_ARG(op) PRED_##op: oparg = PEEKARG(); next_instr += 3 +#endif + /* Stack manipulation macros */ @@ -865,6 +1088,80 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) GETLOCAL(i) = value; \ Py_XDECREF(tmp); } while (0) + +#define UNWIND_BLOCK(b) \ + while (STACK_LEVEL() > (b)->b_level) { \ + PyObject *v = POP(); \ + Py_XDECREF(v); \ + } + +#define UNWIND_EXCEPT_HANDLER(b) \ + { \ + PyObject *type, *value, *traceback; \ + assert(STACK_LEVEL() >= (b)->b_level + 3); \ + while (STACK_LEVEL() > (b)->b_level + 3) { \ + value = POP(); \ + Py_XDECREF(value); \ + } \ + type = tstate->exc_type; \ + value = tstate->exc_value; \ + traceback = tstate->exc_traceback; \ + tstate->exc_type = POP(); \ + tstate->exc_value = POP(); \ + tstate->exc_traceback = POP(); \ + Py_XDECREF(type); \ + Py_XDECREF(value); \ + Py_XDECREF(traceback); \ + } + +#define SAVE_EXC_STATE() \ + { \ + PyObject *type, *value, *traceback; \ + Py_XINCREF(tstate->exc_type); \ + Py_XINCREF(tstate->exc_value); \ + Py_XINCREF(tstate->exc_traceback); \ + type = f->f_exc_type; \ + value = f->f_exc_value; \ + traceback = f->f_exc_traceback; \ + f->f_exc_type = tstate->exc_type; \ + f->f_exc_value = tstate->exc_value; \ + f->f_exc_traceback = tstate->exc_traceback; \ + Py_XDECREF(type); \ + Py_XDECREF(value); \ + Py_XDECREF(traceback); \ + } + +#define SWAP_EXC_STATE() \ + { \ + PyObject *tmp; \ + tmp = tstate->exc_type; \ + tstate->exc_type = f->f_exc_type; \ + f->f_exc_type = tmp; \ + tmp = tstate->exc_value; \ + tstate->exc_value = f->f_exc_value; \ + f->f_exc_value = tmp; \ + tmp = tstate->exc_traceback; \ + tstate->exc_traceback = f->f_exc_traceback; \ + f->f_exc_traceback = tmp; \ + } + +#define RESTORE_AND_CLEAR_EXC_STATE() \ + { \ + PyObject *type, *value, *tb; \ + type = tstate->exc_type; \ + value = tstate->exc_value; \ + tb = tstate->exc_traceback; \ + tstate->exc_type = f->f_exc_type; \ + tstate->exc_value = f->f_exc_value; \ + tstate->exc_traceback = f->f_exc_traceback; \ + f->f_exc_type = NULL; \ + f->f_exc_value = NULL; \ + f->f_exc_traceback = NULL; \ + Py_XDECREF(type); \ + Py_XDECREF(value); \ + Py_XDECREF(tb); \ + } + /* Start of code */ if (f == NULL) @@ -915,7 +1212,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) consts = co->co_consts; fastlocals = f->f_localsplus; freevars = f->f_localsplus + co->co_nlocals; - first_instr = (unsigned char*) PyString_AS_STRING(co->co_code); + first_instr = (unsigned char*) PyBytes_AS_STRING(co->co_code); /* An explanation is in order for the next line. f->f_lasti now refers to the index of the last instruction @@ -938,11 +1235,32 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) assert(stack_pointer != NULL); f->f_stacktop = NULL; /* remains NULL unless yield suspends frame */ + if (co->co_flags & CO_GENERATOR && !throwflag) { + if (f->f_exc_type != NULL && f->f_exc_type != Py_None) { + /* We were in an except handler when we left, + restore the exception state which was put aside + (see YIELD_VALUE). */ + SWAP_EXC_STATE(); + } + else { + SAVE_EXC_STATE(); + } + } + #ifdef LLTRACE lltrace = PyDict_GetItemString(f->f_globals, "__lltrace__") != NULL; #endif #if defined(Py_DEBUG) || defined(LLTRACE) - filename = PyString_AsString(co->co_filename); + { + PyObject *error_type, *error_value, *error_traceback; + PyErr_Fetch(&error_type, &error_value, &error_traceback); + filename = _PyUnicode_AsString(co->co_filename); + if (filename == NULL && tstate->overflowed) { + /* maximum recursion depth exceeded */ + goto exit_eval_frame; + } + PyErr_Restore(error_type, error_value, error_traceback); + } #endif why = WHY_NOT; @@ -984,55 +1302,46 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) async I/O handler); see Py_AddPendingCall() and Py_MakePendingCalls() above. */ - if (--_Py_Ticker < 0) { + if (_Py_atomic_load_relaxed(&eval_breaker)) { if (*next_instr == SETUP_FINALLY) { /* Make the last opcode before a try: finally: block uninterruptible. */ goto fast_next_opcode; } - _Py_Ticker = _Py_CheckInterval; tstate->tick_counter++; #ifdef WITH_TSC ticked = 1; #endif - if (pendingcalls_to_do) { + if (_Py_atomic_load_relaxed(&pendingcalls_to_do)) { if (Py_MakePendingCalls() < 0) { why = WHY_EXCEPTION; goto on_error; } - if (pendingcalls_to_do) - /* MakePendingCalls() didn't succeed. - Force early re-execution of this - "periodic" code, possibly after - a thread switch */ - _Py_Ticker = 0; } #ifdef WITH_THREAD - if (interpreter_lock) { + if (_Py_atomic_load_relaxed(&gil_drop_request)) { /* Give another thread a chance */ - if (PyThreadState_Swap(NULL) != tstate) Py_FatalError("ceval: tstate mix-up"); - PyThread_release_lock(interpreter_lock); + drop_gil(tstate); /* Other threads may run now */ - PyThread_acquire_lock(interpreter_lock, 1); + take_gil(tstate); if (PyThreadState_Swap(tstate) != NULL) Py_FatalError("ceval: orphan tstate"); - - /* Check for thread interrupts */ - - if (tstate->async_exc != NULL) { - x = tstate->async_exc; - tstate->async_exc = NULL; - PyErr_SetNone(x); - Py_DECREF(x); - why = WHY_EXCEPTION; - goto on_error; - } } #endif + /* Check for asynchronous exceptions. */ + if (tstate->async_exc != NULL) { + x = tstate->async_exc; + tstate->async_exc = NULL; + UNSIGNAL_ASYNC_EXC(); + PyErr_SetNone(x); + Py_DECREF(x); + why = WHY_EXCEPTION; + goto on_error; + } } fast_next_opcode: @@ -1105,599 +1414,406 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) /* case STOP_CODE: this is an error! */ - case NOP: - goto fast_next_opcode; + TARGET(NOP) + FAST_DISPATCH(); - case LOAD_FAST: + TARGET(LOAD_FAST) x = GETLOCAL(oparg); if (x != NULL) { Py_INCREF(x); PUSH(x); - goto fast_next_opcode; + FAST_DISPATCH(); } format_exc_check_arg(PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(co->co_varnames, oparg)); break; - case LOAD_CONST: + TARGET(LOAD_CONST) x = GETITEM(consts, oparg); Py_INCREF(x); PUSH(x); - goto fast_next_opcode; + FAST_DISPATCH(); PREDICTED_WITH_ARG(STORE_FAST); - case STORE_FAST: + TARGET(STORE_FAST) v = POP(); SETLOCAL(oparg, v); - goto fast_next_opcode; + FAST_DISPATCH(); - case POP_TOP: + TARGET(POP_TOP) v = POP(); Py_DECREF(v); - goto fast_next_opcode; + FAST_DISPATCH(); - case ROT_TWO: + TARGET(ROT_TWO) v = TOP(); w = SECOND(); SET_TOP(w); SET_SECOND(v); - goto fast_next_opcode; + FAST_DISPATCH(); - case ROT_THREE: + TARGET(ROT_THREE) v = TOP(); w = SECOND(); x = THIRD(); SET_TOP(w); SET_SECOND(x); SET_THIRD(v); - goto fast_next_opcode; + FAST_DISPATCH(); - case ROT_FOUR: - u = TOP(); - v = SECOND(); - w = THIRD(); - x = FOURTH(); - SET_TOP(v); - SET_SECOND(w); - SET_THIRD(x); - SET_FOURTH(u); - goto fast_next_opcode; - - case DUP_TOP: + TARGET(DUP_TOP) v = TOP(); Py_INCREF(v); PUSH(v); - goto fast_next_opcode; + FAST_DISPATCH(); - case DUP_TOPX: - if (oparg == 2) { - x = TOP(); - Py_INCREF(x); - w = SECOND(); - Py_INCREF(w); - STACKADJ(2); - SET_TOP(x); - SET_SECOND(w); - goto fast_next_opcode; - } else if (oparg == 3) { - x = TOP(); - Py_INCREF(x); - w = SECOND(); - Py_INCREF(w); - v = THIRD(); - Py_INCREF(v); - STACKADJ(3); - SET_TOP(x); - SET_SECOND(w); - SET_THIRD(v); - goto fast_next_opcode; - } - Py_FatalError("invalid argument to DUP_TOPX" - " (bytecode corruption?)"); - /* Never returns, so don't bother to set why. */ - break; + TARGET(DUP_TOP_TWO) + x = TOP(); + Py_INCREF(x); + w = SECOND(); + Py_INCREF(w); + STACKADJ(2); + SET_TOP(x); + SET_SECOND(w); + FAST_DISPATCH(); - case UNARY_POSITIVE: + TARGET(UNARY_POSITIVE) v = TOP(); x = PyNumber_Positive(v); Py_DECREF(v); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case UNARY_NEGATIVE: + TARGET(UNARY_NEGATIVE) v = TOP(); x = PyNumber_Negative(v); Py_DECREF(v); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case UNARY_NOT: + TARGET(UNARY_NOT) v = TOP(); err = PyObject_IsTrue(v); Py_DECREF(v); if (err == 0) { Py_INCREF(Py_True); SET_TOP(Py_True); - continue; + DISPATCH(); } else if (err > 0) { Py_INCREF(Py_False); SET_TOP(Py_False); err = 0; - continue; + DISPATCH(); } STACKADJ(-1); break; - case UNARY_CONVERT: - v = TOP(); - x = PyObject_Repr(v); - Py_DECREF(v); - SET_TOP(x); - if (x != NULL) continue; - break; - - case UNARY_INVERT: + TARGET(UNARY_INVERT) v = TOP(); x = PyNumber_Invert(v); Py_DECREF(v); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case BINARY_POWER: + TARGET(BINARY_POWER) w = POP(); v = TOP(); x = PyNumber_Power(v, w, Py_None); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case BINARY_MULTIPLY: + TARGET(BINARY_MULTIPLY) w = POP(); v = TOP(); x = PyNumber_Multiply(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case BINARY_DIVIDE: - if (!_Py_QnewFlag) { - w = POP(); - v = TOP(); - x = PyNumber_Divide(v, w); - Py_DECREF(v); - Py_DECREF(w); - SET_TOP(x); - if (x != NULL) continue; - break; - } - /* -Qnew is in effect: fall through to - BINARY_TRUE_DIVIDE */ - case BINARY_TRUE_DIVIDE: + TARGET(BINARY_TRUE_DIVIDE) w = POP(); v = TOP(); x = PyNumber_TrueDivide(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case BINARY_FLOOR_DIVIDE: + TARGET(BINARY_FLOOR_DIVIDE) w = POP(); v = TOP(); x = PyNumber_FloorDivide(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case BINARY_MODULO: + TARGET(BINARY_MODULO) w = POP(); v = TOP(); - if (PyString_CheckExact(v)) - x = PyString_Format(v, w); + if (PyUnicode_CheckExact(v)) + x = PyUnicode_Format(v, w); else x = PyNumber_Remainder(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case BINARY_ADD: + TARGET(BINARY_ADD) w = POP(); v = TOP(); - if (PyInt_CheckExact(v) && PyInt_CheckExact(w)) { - /* INLINE: int + int */ - register long a, b, i; - a = PyInt_AS_LONG(v); - b = PyInt_AS_LONG(w); - /* cast to avoid undefined behaviour - on overflow */ - i = (long)((unsigned long)a + b); - if ((i^a) < 0 && (i^b) < 0) - goto slow_add; - x = PyInt_FromLong(i); - } - else if (PyString_CheckExact(v) && - PyString_CheckExact(w)) { - x = string_concatenate(v, w, f, next_instr); - /* string_concatenate consumed the ref to v */ + if (PyUnicode_CheckExact(v) && + PyUnicode_CheckExact(w)) { + x = unicode_concatenate(v, w, f, next_instr); + /* unicode_concatenate consumed the ref to v */ goto skip_decref_vx; } else { - slow_add: x = PyNumber_Add(v, w); } Py_DECREF(v); skip_decref_vx: Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case BINARY_SUBTRACT: + TARGET(BINARY_SUBTRACT) w = POP(); v = TOP(); - if (PyInt_CheckExact(v) && PyInt_CheckExact(w)) { - /* INLINE: int - int */ - register long a, b, i; - a = PyInt_AS_LONG(v); - b = PyInt_AS_LONG(w); - /* cast to avoid undefined behaviour - on overflow */ - i = (long)((unsigned long)a - b); - if ((i^a) < 0 && (i^~b) < 0) - goto slow_sub; - x = PyInt_FromLong(i); - } - else { - slow_sub: - x = PyNumber_Subtract(v, w); - } + x = PyNumber_Subtract(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case BINARY_SUBSCR: + TARGET(BINARY_SUBSCR) w = POP(); v = TOP(); - if (PyList_CheckExact(v) && PyInt_CheckExact(w)) { - /* INLINE: list[int] */ - Py_ssize_t i = PyInt_AsSsize_t(w); - if (i < 0) - i += PyList_GET_SIZE(v); - if (i >= 0 && i < PyList_GET_SIZE(v)) { - x = PyList_GET_ITEM(v, i); - Py_INCREF(x); - } - else - goto slow_get; - } - else - slow_get: - x = PyObject_GetItem(v, w); + x = PyObject_GetItem(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case BINARY_LSHIFT: + TARGET(BINARY_LSHIFT) w = POP(); v = TOP(); x = PyNumber_Lshift(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case BINARY_RSHIFT: + TARGET(BINARY_RSHIFT) w = POP(); v = TOP(); x = PyNumber_Rshift(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case BINARY_AND: + TARGET(BINARY_AND) w = POP(); v = TOP(); x = PyNumber_And(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case BINARY_XOR: + TARGET(BINARY_XOR) w = POP(); v = TOP(); x = PyNumber_Xor(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case BINARY_OR: + TARGET(BINARY_OR) w = POP(); v = TOP(); x = PyNumber_Or(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case LIST_APPEND: + TARGET(LIST_APPEND) w = POP(); v = PEEK(oparg); err = PyList_Append(v, w); Py_DECREF(w); if (err == 0) { PREDICT(JUMP_ABSOLUTE); - continue; + DISPATCH(); } break; - case SET_ADD: + TARGET(SET_ADD) w = POP(); v = stack_pointer[-oparg]; err = PySet_Add(v, w); Py_DECREF(w); if (err == 0) { PREDICT(JUMP_ABSOLUTE); - continue; + DISPATCH(); } break; - case INPLACE_POWER: + TARGET(INPLACE_POWER) w = POP(); v = TOP(); x = PyNumber_InPlacePower(v, w, Py_None); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case INPLACE_MULTIPLY: + TARGET(INPLACE_MULTIPLY) w = POP(); v = TOP(); x = PyNumber_InPlaceMultiply(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case INPLACE_DIVIDE: - if (!_Py_QnewFlag) { - w = POP(); - v = TOP(); - x = PyNumber_InPlaceDivide(v, w); - Py_DECREF(v); - Py_DECREF(w); - SET_TOP(x); - if (x != NULL) continue; - break; - } - /* -Qnew is in effect: fall through to - INPLACE_TRUE_DIVIDE */ - case INPLACE_TRUE_DIVIDE: + TARGET(INPLACE_TRUE_DIVIDE) w = POP(); v = TOP(); x = PyNumber_InPlaceTrueDivide(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case INPLACE_FLOOR_DIVIDE: + TARGET(INPLACE_FLOOR_DIVIDE) w = POP(); v = TOP(); x = PyNumber_InPlaceFloorDivide(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case INPLACE_MODULO: + TARGET(INPLACE_MODULO) w = POP(); v = TOP(); x = PyNumber_InPlaceRemainder(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case INPLACE_ADD: + TARGET(INPLACE_ADD) w = POP(); v = TOP(); - if (PyInt_CheckExact(v) && PyInt_CheckExact(w)) { - /* INLINE: int + int */ - register long a, b, i; - a = PyInt_AS_LONG(v); - b = PyInt_AS_LONG(w); - i = a + b; - if ((i^a) < 0 && (i^b) < 0) - goto slow_iadd; - x = PyInt_FromLong(i); - } - else if (PyString_CheckExact(v) && - PyString_CheckExact(w)) { - x = string_concatenate(v, w, f, next_instr); - /* string_concatenate consumed the ref to v */ + if (PyUnicode_CheckExact(v) && + PyUnicode_CheckExact(w)) { + x = unicode_concatenate(v, w, f, next_instr); + /* unicode_concatenate consumed the ref to v */ goto skip_decref_v; } else { - slow_iadd: x = PyNumber_InPlaceAdd(v, w); } Py_DECREF(v); skip_decref_v: Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case INPLACE_SUBTRACT: + TARGET(INPLACE_SUBTRACT) w = POP(); v = TOP(); - if (PyInt_CheckExact(v) && PyInt_CheckExact(w)) { - /* INLINE: int - int */ - register long a, b, i; - a = PyInt_AS_LONG(v); - b = PyInt_AS_LONG(w); - i = a - b; - if ((i^a) < 0 && (i^~b) < 0) - goto slow_isub; - x = PyInt_FromLong(i); - } - else { - slow_isub: - x = PyNumber_InPlaceSubtract(v, w); - } + x = PyNumber_InPlaceSubtract(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case INPLACE_LSHIFT: + TARGET(INPLACE_LSHIFT) w = POP(); v = TOP(); x = PyNumber_InPlaceLshift(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case INPLACE_RSHIFT: + TARGET(INPLACE_RSHIFT) w = POP(); v = TOP(); x = PyNumber_InPlaceRshift(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case INPLACE_AND: + TARGET(INPLACE_AND) w = POP(); v = TOP(); x = PyNumber_InPlaceAnd(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case INPLACE_XOR: + TARGET(INPLACE_XOR) w = POP(); v = TOP(); x = PyNumber_InPlaceXor(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case INPLACE_OR: + TARGET(INPLACE_OR) w = POP(); v = TOP(); x = PyNumber_InPlaceOr(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; - break; - - case SLICE+0: - case SLICE+1: - case SLICE+2: - case SLICE+3: - if ((opcode-SLICE) & 2) - w = POP(); - else - w = NULL; - if ((opcode-SLICE) & 1) - v = POP(); - else - v = NULL; - u = TOP(); - x = apply_slice(u, v, w); - Py_DECREF(u); - Py_XDECREF(v); - Py_XDECREF(w); - SET_TOP(x); - if (x != NULL) continue; - break; - - case STORE_SLICE+0: - case STORE_SLICE+1: - case STORE_SLICE+2: - case STORE_SLICE+3: - if ((opcode-STORE_SLICE) & 2) - w = POP(); - else - w = NULL; - if ((opcode-STORE_SLICE) & 1) - v = POP(); - else - v = NULL; - u = POP(); - t = POP(); - err = assign_slice(u, v, w, t); /* u[v:w] = t */ - Py_DECREF(t); - Py_DECREF(u); - Py_XDECREF(v); - Py_XDECREF(w); - if (err == 0) continue; - break; - - case DELETE_SLICE+0: - case DELETE_SLICE+1: - case DELETE_SLICE+2: - case DELETE_SLICE+3: - if ((opcode-DELETE_SLICE) & 2) - w = POP(); - else - w = NULL; - if ((opcode-DELETE_SLICE) & 1) - v = POP(); - else - v = NULL; - u = POP(); - err = assign_slice(u, v, w, (PyObject *)NULL); - /* del u[v:w] */ - Py_DECREF(u); - Py_XDECREF(v); - Py_XDECREF(w); - if (err == 0) continue; + if (x != NULL) DISPATCH(); break; - case STORE_SUBSCR: + TARGET(STORE_SUBSCR) w = TOP(); v = SECOND(); u = THIRD(); @@ -1707,10 +1823,10 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) Py_DECREF(u); Py_DECREF(v); Py_DECREF(w); - if (err == 0) continue; + if (err == 0) DISPATCH(); break; - case DELETE_SUBSCR: + TARGET(DELETE_SUBSCR) w = TOP(); v = SECOND(); STACKADJ(-2); @@ -1718,10 +1834,10 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) err = PyObject_DelItem(v, w); Py_DECREF(v); Py_DECREF(w); - if (err == 0) continue; + if (err == 0) DISPATCH(); break; - case PRINT_EXPR: + TARGET(PRINT_EXPR) v = POP(); w = PySys_GetObject("displayhook"); if (w == NULL) { @@ -1745,103 +1861,18 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) Py_XDECREF(x); break; - case PRINT_ITEM_TO: - w = stream = POP(); - /* fall through to PRINT_ITEM */ - - case PRINT_ITEM: - v = POP(); - if (stream == NULL || stream == Py_None) { - w = PySys_GetObject("stdout"); - if (w == NULL) { - PyErr_SetString(PyExc_RuntimeError, - "lost sys.stdout"); - err = -1; - } - } - /* PyFile_SoftSpace() can exececute arbitrary code - if sys.stdout is an instance with a __getattr__. - If __getattr__ raises an exception, w will - be freed, so we need to prevent that temporarily. */ - Py_XINCREF(w); - if (w != NULL && PyFile_SoftSpace(w, 0)) - err = PyFile_WriteString(" ", w); - if (err == 0) - err = PyFile_WriteObject(v, w, Py_PRINT_RAW); - if (err == 0) { - /* XXX move into writeobject() ? */ - if (PyString_Check(v)) { - char *s = PyString_AS_STRING(v); - Py_ssize_t len = PyString_GET_SIZE(v); - if (len == 0 || - !isspace(Py_CHARMASK(s[len-1])) || - s[len-1] == ' ') - PyFile_SoftSpace(w, 1); - } -#ifdef Py_USING_UNICODE - else if (PyUnicode_Check(v)) { - Py_UNICODE *s = PyUnicode_AS_UNICODE(v); - Py_ssize_t len = PyUnicode_GET_SIZE(v); - if (len == 0 || - !Py_UNICODE_ISSPACE(s[len-1]) || - s[len-1] == ' ') - PyFile_SoftSpace(w, 1); - } -#endif - else - PyFile_SoftSpace(w, 1); - } - Py_XDECREF(w); - Py_DECREF(v); - Py_XDECREF(stream); - stream = NULL; - if (err == 0) - continue; - break; - - case PRINT_NEWLINE_TO: - w = stream = POP(); - /* fall through to PRINT_NEWLINE */ - - case PRINT_NEWLINE: - if (stream == NULL || stream == Py_None) { - w = PySys_GetObject("stdout"); - if (w == NULL) { - PyErr_SetString(PyExc_RuntimeError, - "lost sys.stdout"); - why = WHY_EXCEPTION; - } - } - if (w != NULL) { - /* w.write() may replace sys.stdout, so we - * have to keep our reference to it */ - Py_INCREF(w); - err = PyFile_WriteString("\n", w); - if (err == 0) - PyFile_SoftSpace(w, 0); - Py_DECREF(w); - } - Py_XDECREF(stream); - stream = NULL; - break; - - #ifdef CASE_TOO_BIG default: switch (opcode) { #endif - case RAISE_VARARGS: - u = v = w = NULL; + TARGET(RAISE_VARARGS) + v = w = NULL; switch (oparg) { - case 3: - u = POP(); /* traceback */ - /* Fallthrough */ case 2: - v = POP(); /* value */ - /* Fallthrough */ + v = POP(); /* cause */ case 1: w = POP(); /* exc */ case 0: /* Fallthrough */ - why = do_raise(w, v, u); + why = do_raise(w, v); break; default: PyErr_SetString(PyExc_SystemError, @@ -1851,61 +1882,65 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) } break; - case LOAD_LOCALS: - if ((x = f->f_locals) != NULL) { - Py_INCREF(x); - PUSH(x); - continue; - } - PyErr_SetString(PyExc_SystemError, "no locals"); - break; + TARGET(STORE_LOCALS) + x = POP(); + v = f->f_locals; + Py_XDECREF(v); + f->f_locals = x; + DISPATCH(); - case RETURN_VALUE: + TARGET(RETURN_VALUE) retval = POP(); why = WHY_RETURN; goto fast_block_end; - case YIELD_VALUE: + TARGET(YIELD_VALUE) retval = POP(); f->f_stacktop = stack_pointer; why = WHY_YIELD; goto fast_yield; - case EXEC_STMT: - w = TOP(); - v = SECOND(); - u = THIRD(); - STACKADJ(-3); - READ_TIMESTAMP(intr0); - err = exec_statement(f, u, v, w); - READ_TIMESTAMP(intr1); - Py_DECREF(u); - Py_DECREF(v); - Py_DECREF(w); - break; - - case POP_BLOCK: + TARGET(POP_EXCEPT) { PyTryBlock *b = PyFrame_BlockPop(f); - while (STACK_LEVEL() > b->b_level) { - v = POP(); - Py_DECREF(v); + if (b->b_type != EXCEPT_HANDLER) { + PyErr_SetString(PyExc_SystemError, + "popped block is not an except handler"); + why = WHY_EXCEPTION; + break; } + UNWIND_EXCEPT_HANDLER(b); } - continue; + DISPATCH(); + + TARGET(POP_BLOCK) + { + PyTryBlock *b = PyFrame_BlockPop(f); + UNWIND_BLOCK(b); + } + DISPATCH(); PREDICTED(END_FINALLY); - case END_FINALLY: + TARGET(END_FINALLY) v = POP(); - if (PyInt_Check(v)) { - why = (enum why_code) PyInt_AS_LONG(v); + if (PyLong_Check(v)) { + why = (enum why_code) PyLong_AS_LONG(v); assert(why != WHY_YIELD); if (why == WHY_RETURN || why == WHY_CONTINUE) retval = POP(); + if (why == WHY_SILENCED) { + /* An exception was silenced by 'with', we must + manually unwind the EXCEPT_HANDLER block which was + created when the exception was caught, otherwise + the stack will be in an inconsistent state. */ + PyTryBlock *b = PyFrame_BlockPop(f); + assert(b->b_type == EXCEPT_HANDLER); + UNWIND_EXCEPT_HANDLER(b); + why = WHY_NOT; + } } - else if (PyExceptionClass_Check(v) || - PyString_Check(v)) { + else if (PyExceptionClass_Check(v)) { w = POP(); u = POP(); PyErr_Restore(v, w, u); @@ -1920,19 +1955,19 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) Py_DECREF(v); break; - case BUILD_CLASS: - u = TOP(); - v = SECOND(); - w = THIRD(); - STACKADJ(-2); - x = build_class(u, v, w); - SET_TOP(x); - Py_DECREF(u); - Py_DECREF(v); - Py_DECREF(w); + TARGET(LOAD_BUILD_CLASS) + x = PyDict_GetItemString(f->f_builtins, + "__build_class__"); + if (x == NULL) { + PyErr_SetString(PyExc_ImportError, + "__build_class__ not found"); + break; + } + Py_INCREF(x); + PUSH(x); break; - case STORE_NAME: + TARGET(STORE_NAME) w = GETITEM(names, oparg); v = POP(); if ((x = f->f_locals) != NULL) { @@ -1941,15 +1976,14 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) else err = PyObject_SetItem(x, w, v); Py_DECREF(v); - if (err == 0) continue; + if (err == 0) DISPATCH(); break; } PyErr_Format(PyExc_SystemError, - "no locals found when storing %s", - PyObject_REPR(w)); + "no locals found when storing %R", w); break; - case DELETE_NAME: + TARGET(DELETE_NAME) w = GETITEM(names, oparg); if ((x = f->f_locals) != NULL) { if ((err = PyObject_DelItem(x, w)) != 0) @@ -1959,12 +1993,11 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) break; } PyErr_Format(PyExc_SystemError, - "no locals when deleting %s", - PyObject_REPR(w)); + "no locals when deleting %R", w); break; PREDICTED_WITH_ARG(UNPACK_SEQUENCE); - case UNPACK_SEQUENCE: + TARGET(UNPACK_SEQUENCE) v = POP(); if (PyTuple_CheckExact(v) && PyTuple_GET_SIZE(v) == oparg) { @@ -1976,7 +2009,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) PUSH(w); } Py_DECREF(v); - continue; + DISPATCH(); } else if (PyList_CheckExact(v) && PyList_GET_SIZE(v) == oparg) { PyObject **items = \ @@ -1986,7 +2019,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) Py_INCREF(w); PUSH(w); } - } else if (unpack_iterable(v, oparg, + } else if (unpack_iterable(v, oparg, -1, stack_pointer + oparg)) { STACKADJ(oparg); } else { @@ -1996,7 +2029,22 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) Py_DECREF(v); break; - case STORE_ATTR: + TARGET(UNPACK_EX) + { + int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); + v = POP(); + + if (unpack_iterable(v, oparg & 0xFF, oparg >> 8, + stack_pointer + totalargs)) { + stack_pointer += totalargs; + } else { + why = WHY_EXCEPTION; + } + Py_DECREF(v); + break; + } + + TARGET(STORE_ATTR) w = GETITEM(names, oparg); v = TOP(); u = SECOND(); @@ -2004,10 +2052,10 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) err = PyObject_SetAttr(v, w, u); /* v.w = u */ Py_DECREF(v); Py_DECREF(u); - if (err == 0) continue; + if (err == 0) DISPATCH(); break; - case DELETE_ATTR: + TARGET(DELETE_ATTR) w = GETITEM(names, oparg); v = POP(); err = PyObject_SetAttr(v, w, (PyObject *)NULL); @@ -2015,27 +2063,26 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) Py_DECREF(v); break; - case STORE_GLOBAL: + TARGET(STORE_GLOBAL) w = GETITEM(names, oparg); v = POP(); err = PyDict_SetItem(f->f_globals, w, v); Py_DECREF(v); - if (err == 0) continue; + if (err == 0) DISPATCH(); break; - case DELETE_GLOBAL: + TARGET(DELETE_GLOBAL) w = GETITEM(names, oparg); if ((err = PyDict_DelItem(f->f_globals, w)) != 0) format_exc_check_arg( PyExc_NameError, GLOBAL_NAME_ERROR_MSG, w); break; - case LOAD_NAME: + TARGET(LOAD_NAME) w = GETITEM(names, oparg); if ((v = f->f_locals) == NULL) { PyErr_Format(PyExc_SystemError, - "no locals when loading %s", - PyObject_REPR(w)); + "no locals when loading %R", w); why = WHY_EXCEPTION; break; } @@ -2066,15 +2113,15 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) Py_INCREF(x); } PUSH(x); - continue; + DISPATCH(); - case LOAD_GLOBAL: + TARGET(LOAD_GLOBAL) w = GETITEM(names, oparg); - if (PyString_CheckExact(w)) { + if (PyUnicode_CheckExact(w)) { /* Inline the PyDict_GetItem() calls. WARNING: this is an extreme speed hack. Do not try this at home. */ - long hash = ((PyStringObject *)w)->ob_shash; + Py_hash_t hash = ((PyUnicodeObject *)w)->hash; if (hash != -1) { PyDictObject *d; PyDictEntry *e; @@ -2088,7 +2135,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) if (x != NULL) { Py_INCREF(x); PUSH(x); - continue; + DISPATCH(); } d = (PyDictObject *)(f->f_builtins); e = d->ma_lookup(d, w, hash); @@ -2100,7 +2147,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) if (x != NULL) { Py_INCREF(x); PUSH(x); - continue; + DISPATCH(); } goto load_global_error; } @@ -2119,13 +2166,13 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) } Py_INCREF(x); PUSH(x); - continue; + DISPATCH(); - case DELETE_FAST: + TARGET(DELETE_FAST) x = GETLOCAL(oparg); if (x != NULL) { SETLOCAL(oparg, NULL); - continue; + DISPATCH(); } format_exc_check_arg( PyExc_UnboundLocalError, @@ -2134,47 +2181,42 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) ); break; - case LOAD_CLOSURE: + TARGET(DELETE_DEREF) + x = freevars[oparg]; + if (PyCell_GET(x) != NULL) { + PyCell_Set(x, NULL); + DISPATCH(); + } + err = -1; + format_exc_unbound(co, oparg); + break; + + TARGET(LOAD_CLOSURE) x = freevars[oparg]; Py_INCREF(x); PUSH(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case LOAD_DEREF: + TARGET(LOAD_DEREF) x = freevars[oparg]; w = PyCell_Get(x); if (w != NULL) { PUSH(w); - continue; + DISPATCH(); } err = -1; - /* Don't stomp existing exception */ - if (PyErr_Occurred()) - break; - if (oparg < PyTuple_GET_SIZE(co->co_cellvars)) { - v = PyTuple_GET_ITEM(co->co_cellvars, - oparg); - format_exc_check_arg( - PyExc_UnboundLocalError, - UNBOUNDLOCAL_ERROR_MSG, - v); - } else { - v = PyTuple_GET_ITEM(co->co_freevars, oparg - - PyTuple_GET_SIZE(co->co_cellvars)); - format_exc_check_arg(PyExc_NameError, - UNBOUNDFREE_ERROR_MSG, v); - } + format_exc_unbound(co, oparg); break; - case STORE_DEREF: + TARGET(STORE_DEREF) w = POP(); x = freevars[oparg]; PyCell_Set(x, w); Py_DECREF(w); - continue; + DISPATCH(); - case BUILD_TUPLE: + TARGET(BUILD_TUPLE) x = PyTuple_New(oparg); if (x != NULL) { for (; --oparg >= 0;) { @@ -2182,11 +2224,11 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) PyTuple_SET_ITEM(x, oparg, w); } PUSH(x); - continue; + DISPATCH(); } break; - case BUILD_LIST: + TARGET(BUILD_LIST) x = PyList_New(oparg); if (x != NULL) { for (; --oparg >= 0;) { @@ -2194,11 +2236,11 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) PyList_SET_ITEM(x, oparg, w); } PUSH(x); - continue; + DISPATCH(); } break; - case BUILD_SET: + TARGET(BUILD_SET) x = PySet_New(NULL); if (x != NULL) { for (; --oparg >= 0;) { @@ -2212,18 +2254,17 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) break; } PUSH(x); - continue; + DISPATCH(); } break; - - case BUILD_MAP: + TARGET(BUILD_MAP) x = _PyDict_NewPresized((Py_ssize_t)oparg); PUSH(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case STORE_MAP: + TARGET(STORE_MAP) w = TOP(); /* key */ u = SECOND(); /* value */ v = THIRD(); /* dict */ @@ -2232,10 +2273,10 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) err = PyDict_SetItem(v, w, u); /* v[w] = u */ Py_DECREF(u); Py_DECREF(w); - if (err == 0) continue; + if (err == 0) DISPATCH(); break; - case MAP_ADD: + TARGET(MAP_ADD) w = TOP(); /* key */ u = SECOND(); /* value */ STACKADJ(-2); @@ -2246,55 +2287,32 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) Py_DECREF(w); if (err == 0) { PREDICT(JUMP_ABSOLUTE); - continue; + DISPATCH(); } break; - case LOAD_ATTR: + TARGET(LOAD_ATTR) w = GETITEM(names, oparg); v = TOP(); x = PyObject_GetAttr(v, w); Py_DECREF(v); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case COMPARE_OP: + TARGET(COMPARE_OP) w = POP(); v = TOP(); - if (PyInt_CheckExact(w) && PyInt_CheckExact(v)) { - /* INLINE: cmp(int, int) */ - register long a, b; - register int res; - a = PyInt_AS_LONG(v); - b = PyInt_AS_LONG(w); - switch (oparg) { - case PyCmp_LT: res = a < b; break; - case PyCmp_LE: res = a <= b; break; - case PyCmp_EQ: res = a == b; break; - case PyCmp_NE: res = a != b; break; - case PyCmp_GT: res = a > b; break; - case PyCmp_GE: res = a >= b; break; - case PyCmp_IS: res = v == w; break; - case PyCmp_IS_NOT: res = v != w; break; - default: goto slow_compare; - } - x = res ? Py_True : Py_False; - Py_INCREF(x); - } - else { - slow_compare: - x = cmp_outcome(oparg, v, w); - } + x = cmp_outcome(oparg, v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); if (x == NULL) break; PREDICT(POP_JUMP_IF_FALSE); PREDICT(POP_JUMP_IF_TRUE); - continue; + DISPATCH(); - case IMPORT_NAME: + TARGET(IMPORT_NAME) w = GETITEM(names, oparg); x = PyDict_GetItemString(f->f_builtins, "__import__"); if (x == NULL) { @@ -2305,7 +2323,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) Py_INCREF(x); v = POP(); u = TOP(); - if (PyInt_AsLong(u) != -1 || PyErr_Occurred()) + if (PyLong_AsLong(u) != -1 || PyErr_Occurred()) w = PyTuple_Pack(5, w, f->f_globals, @@ -2335,10 +2353,10 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) READ_TIMESTAMP(intr1); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case IMPORT_STAR: + TARGET(IMPORT_STAR) v = POP(); PyFrame_FastToLocals(f); if ((x = f->f_locals) == NULL) { @@ -2351,34 +2369,34 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) READ_TIMESTAMP(intr1); PyFrame_LocalsToFast(f, 0); Py_DECREF(v); - if (err == 0) continue; + if (err == 0) DISPATCH(); break; - case IMPORT_FROM: + TARGET(IMPORT_FROM) w = GETITEM(names, oparg); v = TOP(); READ_TIMESTAMP(intr0); x = import_from(v, w); READ_TIMESTAMP(intr1); PUSH(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case JUMP_FORWARD: + TARGET(JUMP_FORWARD) JUMPBY(oparg); - goto fast_next_opcode; + FAST_DISPATCH(); PREDICTED_WITH_ARG(POP_JUMP_IF_FALSE); - case POP_JUMP_IF_FALSE: + TARGET(POP_JUMP_IF_FALSE) w = POP(); if (w == Py_True) { Py_DECREF(w); - goto fast_next_opcode; + FAST_DISPATCH(); } if (w == Py_False) { Py_DECREF(w); JUMPTO(oparg); - goto fast_next_opcode; + FAST_DISPATCH(); } err = PyObject_IsTrue(w); Py_DECREF(w); @@ -2388,19 +2406,19 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) JUMPTO(oparg); else break; - continue; + DISPATCH(); PREDICTED_WITH_ARG(POP_JUMP_IF_TRUE); - case POP_JUMP_IF_TRUE: + TARGET(POP_JUMP_IF_TRUE) w = POP(); if (w == Py_False) { Py_DECREF(w); - goto fast_next_opcode; + FAST_DISPATCH(); } if (w == Py_True) { Py_DECREF(w); JUMPTO(oparg); - goto fast_next_opcode; + FAST_DISPATCH(); } err = PyObject_IsTrue(w); Py_DECREF(w); @@ -2412,18 +2430,18 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) ; else break; - continue; + DISPATCH(); - case JUMP_IF_FALSE_OR_POP: + TARGET(JUMP_IF_FALSE_OR_POP) w = TOP(); if (w == Py_True) { STACKADJ(-1); Py_DECREF(w); - goto fast_next_opcode; + FAST_DISPATCH(); } if (w == Py_False) { JUMPTO(oparg); - goto fast_next_opcode; + FAST_DISPATCH(); } err = PyObject_IsTrue(w); if (err > 0) { @@ -2435,18 +2453,18 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) JUMPTO(oparg); else break; - continue; + DISPATCH(); - case JUMP_IF_TRUE_OR_POP: + TARGET(JUMP_IF_TRUE_OR_POP) w = TOP(); if (w == Py_False) { STACKADJ(-1); Py_DECREF(w); - goto fast_next_opcode; + FAST_DISPATCH(); } if (w == Py_True) { JUMPTO(oparg); - goto fast_next_opcode; + FAST_DISPATCH(); } err = PyObject_IsTrue(w); if (err > 0) { @@ -2459,10 +2477,10 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) } else break; - continue; + DISPATCH(); PREDICTED_WITH_ARG(JUMP_ABSOLUTE); - case JUMP_ABSOLUTE: + TARGET(JUMP_ABSOLUTE) JUMPTO(oparg); #if FAST_LOOPS /* Enabling this path speeds-up all while and for-loops by bypassing @@ -2470,14 +2488,14 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) because it prevents detection of a control-break in tight loops like "while 1: pass". Compile with this option turned-on when you need the speed-up and do not need break checking inside tight loops (ones - that contain only instructions ending with goto fast_next_opcode). + that contain only instructions ending with FAST_DISPATCH). */ - goto fast_next_opcode; + FAST_DISPATCH(); #else - continue; + DISPATCH(); #endif - case GET_ITER: + TARGET(GET_ITER) /* before: [obj]; after [getiter(obj)] */ v = TOP(); x = PyObject_GetIter(v); @@ -2485,13 +2503,13 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) if (x != NULL) { SET_TOP(x); PREDICT(FOR_ITER); - continue; + DISPATCH(); } STACKADJ(-1); break; PREDICTED_WITH_ARG(FOR_ITER); - case FOR_ITER: + TARGET(FOR_ITER) /* before: [iter]; after: [iter, iter()] *or* [] */ v = TOP(); x = (*v->ob_type->tp_iternext)(v); @@ -2499,7 +2517,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) PUSH(x); PREDICT(STORE_FAST); PREDICT(UNPACK_SEQUENCE); - continue; + DISPATCH(); } if (PyErr_Occurred()) { if (!PyErr_ExceptionMatches( @@ -2511,14 +2529,14 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) x = v = POP(); Py_DECREF(v); JUMPBY(oparg); - continue; + DISPATCH(); - case BREAK_LOOP: + TARGET(BREAK_LOOP) why = WHY_BREAK; goto fast_block_end; - case CONTINUE_LOOP: - retval = PyInt_FromLong(oparg); + TARGET(CONTINUE_LOOP) + retval = PyLong_FromLong(oparg); if (!retval) { x = NULL; break; @@ -2526,9 +2544,10 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) why = WHY_CONTINUE; goto fast_block_end; - case SETUP_LOOP: - case SETUP_EXCEPT: - case SETUP_FINALLY: + TARGET_WITH_IMPL(SETUP_LOOP, _setup_finally) + TARGET_WITH_IMPL(SETUP_EXCEPT, _setup_finally) + TARGET(SETUP_FINALLY) + _setup_finally: /* NOTE: If you add any new block-setup opcodes that are not try/except/finally handlers, you may need to update the PyGen_NeedsFinalizing() function. @@ -2536,9 +2555,9 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) PyFrame_BlockSetup(f, opcode, INSTR_OFFSET() + oparg, STACK_LEVEL()); - continue; + DISPATCH(); - case SETUP_WITH: + TARGET(SETUP_WITH) { static PyObject *exit, *enter; w = TOP(); @@ -2556,18 +2575,16 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) Py_DECREF(u); if (!x) break; - /* Setup a finally block (SETUP_WITH as a block is - equivalent to SETUP_FINALLY except it normalizes - the exception) before pushing the result of - __enter__ on the stack. */ - PyFrame_BlockSetup(f, SETUP_WITH, INSTR_OFFSET() + oparg, + /* Setup the finally block before pushing the result + of __enter__ on the stack. */ + PyFrame_BlockSetup(f, SETUP_FINALLY, INSTR_OFFSET() + oparg, STACK_LEVEL()); PUSH(x); - continue; + DISPATCH(); } - case WITH_CLEANUP: + TARGET(WITH_CLEANUP) { /* At the top of the stack are 1-3 values indicating how/why we entered the finally clause: @@ -2575,32 +2592,36 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) - (TOP, SECOND) = (WHY_{RETURN,CONTINUE}), retval - TOP = WHY_*; no retval below it - (TOP, SECOND, THIRD) = exc_info() + (FOURTH, FITH, SIXTH) = previous exception for EXCEPT_HANDLER Below them is EXIT, the context.__exit__ bound method. In the last case, we must call EXIT(TOP, SECOND, THIRD) otherwise we must call EXIT(None, None, None) - In all cases, we remove EXIT from the stack, leaving - the rest in the same order. + In the first two cases, we remove EXIT from the + stack, leaving the rest in the same order. In the + third case, we shift the bottom 3 values of the + stack down, and replace the empty spot with NULL. In addition, if the stack represents an exception, *and* the function call returns a 'true' value, we - "zap" this information, to prevent END_FINALLY from - re-raising the exception. (But non-local gotos - should still be resumed.) + push WHY_SILENCED onto the stack. END_FINALLY will + then not re-raise the exception. (But non-local + gotos should still be resumed.) */ PyObject *exit_func; - - u = POP(); + u = TOP(); if (u == Py_None) { + (void)POP(); exit_func = TOP(); SET_TOP(u); v = w = Py_None; } - else if (PyInt_Check(u)) { - switch(PyInt_AS_LONG(u)) { + else if (PyLong_Check(u)) { + (void)POP(); + switch(PyLong_AsLong(u)) { case WHY_RETURN: case WHY_CONTINUE: /* Retval in TOP. */ @@ -2616,12 +2637,25 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) u = v = w = Py_None; } else { - v = TOP(); - w = SECOND(); - exit_func = THIRD(); - SET_TOP(u); - SET_SECOND(v); - SET_THIRD(w); + PyObject *tp, *exc, *tb; + PyTryBlock *block; + v = SECOND(); + w = THIRD(); + tp = FOURTH(); + exc = PEEK(5); + tb = PEEK(6); + exit_func = PEEK(7); + SET_VALUE(7, tb); + SET_VALUE(6, exc); + SET_VALUE(5, tp); + /* UNWIND_EXCEPT_HANDLER will pop this off. */ + SET_FOURTH(NULL); + /* We just shifted the stack down, so we have + to tell the except handler block that the + values are lower than it expects. */ + block = &f->f_blockstack[f->f_iblock - 1]; + assert(block->b_type == EXCEPT_HANDLER); + block->b_level--; } /* XXX Not the fastest way to call it... */ x = PyObject_CallFunctionObjArgs(exit_func, u, v, w, @@ -2640,22 +2674,14 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) break; /* Go to error exit */ else if (err > 0) { err = 0; - /* There was an exception and a true return */ - STACKADJ(-2); - Py_INCREF(Py_None); - SET_TOP(Py_None); - Py_DECREF(u); - Py_DECREF(v); - Py_DECREF(w); - } else { - /* The stack was rearranged to remove EXIT - above. Let END_FINALLY do its thing */ + /* There was an exception and a True return */ + PUSH(PyLong_FromLong((long) WHY_SILENCED)); } PREDICT(END_FINALLY); break; } - case CALL_FUNCTION: + TARGET(CALL_FUNCTION) { PyObject **sp; PCALL(PCALL_ALL); @@ -2668,13 +2694,14 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) stack_pointer = sp; PUSH(x); if (x != NULL) - continue; + DISPATCH(); break; } - case CALL_FUNCTION_VAR: - case CALL_FUNCTION_KW: - case CALL_FUNCTION_VAR_KW: + TARGET_WITH_IMPL(CALL_FUNCTION_VAR, _call_function_var_kw) + TARGET_WITH_IMPL(CALL_FUNCTION_KW, _call_function_var_kw) + TARGET(CALL_FUNCTION_VAR_KW) + _call_function_var_kw: { int na = oparg & 0xff; int nk = (oparg>>8) & 0xff; @@ -2698,6 +2725,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) Py_DECREF(*pfunc); *pfunc = self; na++; + n++; } else Py_INCREF(func); sp = stack_pointer; @@ -2713,55 +2741,71 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) } PUSH(x); if (x != NULL) - continue; + DISPATCH(); break; } - case MAKE_FUNCTION: + TARGET_WITH_IMPL(MAKE_CLOSURE, _make_function) + TARGET(MAKE_FUNCTION) + _make_function: + { + int posdefaults = oparg & 0xff; + int kwdefaults = (oparg>>8) & 0xff; + int num_annotations = (oparg >> 16) & 0x7fff; + v = POP(); /* code object */ x = PyFunction_New(v, f->f_globals); Py_DECREF(v); - /* XXX Maybe this should be a separate opcode? */ - if (x != NULL && oparg > 0) { - v = PyTuple_New(oparg); + + if (x != NULL && opcode == MAKE_CLOSURE) { + v = POP(); + if (PyFunction_SetClosure(x, v) != 0) { + /* Can't happen unless bytecode is corrupt. */ + why = WHY_EXCEPTION; + } + Py_DECREF(v); + } + + if (x != NULL && num_annotations > 0) { + Py_ssize_t name_ix; + u = POP(); /* names of args with annotations */ + v = PyDict_New(); if (v == NULL) { Py_DECREF(x); x = NULL; break; } - while (--oparg >= 0) { + name_ix = PyTuple_Size(u); + assert(num_annotations == name_ix+1); + while (name_ix > 0) { + --name_ix; + t = PyTuple_GET_ITEM(u, name_ix); w = POP(); - PyTuple_SET_ITEM(v, oparg, w); + /* XXX(nnorwitz): check for errors */ + PyDict_SetItem(v, t, w); + Py_DECREF(w); } - err = PyFunction_SetDefaults(x, v); - Py_DECREF(v); - } - PUSH(x); - break; - case MAKE_CLOSURE: - { - v = POP(); /* code object */ - x = PyFunction_New(v, f->f_globals); - Py_DECREF(v); - if (x != NULL) { - v = POP(); - if (PyFunction_SetClosure(x, v) != 0) { - /* Can't happen unless bytecode is corrupt. */ + if (PyFunction_SetAnnotations(x, v) != 0) { + /* Can't happen unless + PyFunction_SetAnnotations changes. */ why = WHY_EXCEPTION; } Py_DECREF(v); + Py_DECREF(u); } - if (x != NULL && oparg > 0) { - v = PyTuple_New(oparg); + + /* XXX Maybe this should be a separate opcode? */ + if (x != NULL && posdefaults > 0) { + v = PyTuple_New(posdefaults); if (v == NULL) { Py_DECREF(x); x = NULL; break; } - while (--oparg >= 0) { + while (--posdefaults >= 0) { w = POP(); - PyTuple_SET_ITEM(v, oparg, w); + PyTuple_SET_ITEM(v, posdefaults, w); } if (PyFunction_SetDefaults(x, v) != 0) { /* Can't happen unless @@ -2770,11 +2814,33 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) } Py_DECREF(v); } + if (x != NULL && kwdefaults > 0) { + v = PyDict_New(); + if (v == NULL) { + Py_DECREF(x); + x = NULL; + break; + } + while (--kwdefaults >= 0) { + w = POP(); /* default value */ + u = POP(); /* kw only arg name */ + /* XXX(nnorwitz): check for errors */ + PyDict_SetItem(v, u, w); + Py_DECREF(w); + Py_DECREF(u); + } + if (PyFunction_SetKwDefaults(x, v) != 0) { + /* Can't happen unless + PyFunction_SetKwDefaults changes. */ + why = WHY_EXCEPTION; + } + Py_DECREF(v); + } PUSH(x); break; } - case BUILD_SLICE: + TARGET(BUILD_SLICE) if (oparg == 3) w = POP(); else @@ -2786,14 +2852,17 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) Py_DECREF(v); Py_XDECREF(w); SET_TOP(x); - if (x != NULL) continue; + if (x != NULL) DISPATCH(); break; - case EXTENDED_ARG: + TARGET(EXTENDED_ARG) opcode = NEXTOP(); oparg = oparg<<16 | NEXTARG(); goto dispatch_opcode; +#if USE_COMPUTED_GOTOS + _unknown_opcode: +#endif default: fprintf(stderr, "XXX lineno: %d, opcode: %d\n", @@ -2881,60 +2950,65 @@ fast_block_end: assert(why != WHY_YIELD); if (b->b_type == SETUP_LOOP && why == WHY_CONTINUE) { why = WHY_NOT; - JUMPTO(PyInt_AS_LONG(retval)); + JUMPTO(PyLong_AS_LONG(retval)); Py_DECREF(retval); break; } - /* Now we have to pop the block. */ f->f_iblock--; - while (STACK_LEVEL() > b->b_level) { - v = POP(); - Py_XDECREF(v); + if (b->b_type == EXCEPT_HANDLER) { + UNWIND_EXCEPT_HANDLER(b); + continue; } + UNWIND_BLOCK(b); if (b->b_type == SETUP_LOOP && why == WHY_BREAK) { why = WHY_NOT; JUMPTO(b->b_handler); break; } - if (b->b_type == SETUP_FINALLY || - (b->b_type == SETUP_EXCEPT && - why == WHY_EXCEPTION) || - b->b_type == SETUP_WITH) { - if (why == WHY_EXCEPTION) { - PyObject *exc, *val, *tb; - PyErr_Fetch(&exc, &val, &tb); - if (val == NULL) { - val = Py_None; - Py_INCREF(val); - } - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. Don't do - this for 'finally'. */ - if (b->b_type == SETUP_EXCEPT || - b->b_type == SETUP_WITH) { - PyErr_NormalizeException( - &exc, &val, &tb); - set_exc_info(tstate, - exc, val, tb); - } - if (tb == NULL) { - Py_INCREF(Py_None); - PUSH(Py_None); - } else - PUSH(tb); - PUSH(val); - PUSH(exc); + if (why == WHY_EXCEPTION && (b->b_type == SETUP_EXCEPT + || b->b_type == SETUP_FINALLY)) { + PyObject *exc, *val, *tb; + int handler = b->b_handler; + /* Beware, this invalidates all b->b_* fields */ + PyFrame_BlockSetup(f, EXCEPT_HANDLER, -1, STACK_LEVEL()); + PUSH(tstate->exc_traceback); + PUSH(tstate->exc_value); + if (tstate->exc_type != NULL) { + PUSH(tstate->exc_type); } else { - if (why & (WHY_RETURN | WHY_CONTINUE)) - PUSH(retval); - v = PyInt_FromLong((long)why); - PUSH(v); + Py_INCREF(Py_None); + PUSH(Py_None); } + PyErr_Fetch(&exc, &val, &tb); + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyErr_NormalizeException( + &exc, &val, &tb); + PyException_SetTraceback(val, tb); + Py_INCREF(exc); + tstate->exc_type = exc; + Py_INCREF(val); + tstate->exc_value = val; + tstate->exc_traceback = tb; + if (tb == NULL) + tb = Py_None; + Py_INCREF(tb); + PUSH(tb); + PUSH(val); + PUSH(exc); + why = WHY_NOT; + JUMPTO(handler); + break; + } + if (b->b_type == SETUP_FINALLY) { + if (why & (WHY_RETURN | WHY_CONTINUE)) + PUSH(retval); + PUSH(PyLong_FromLong((long)why)); why = WHY_NOT; JUMPTO(b->b_handler); break; @@ -2960,6 +3034,26 @@ fast_block_end: retval = NULL; fast_yield: + if (co->co_flags & CO_GENERATOR && (why == WHY_YIELD || why == WHY_RETURN)) { + /* The purpose of this block is to put aside the generator's exception + state and restore that of the calling frame. If the current + exception state is from the caller, we clear the exception values + on the generator frame, so they are not swapped back in latter. The + origin of the current exception state is determined by checking for + except handler blocks, which we must be in iff a new exception + state came into existence in this frame. (An uncaught exception + would have why == WHY_EXCEPTION, and we wouldn't be here). */ + int i; + for (i = 0; i < f->f_iblock; i++) + if (f->f_blockstack[i].b_type == EXCEPT_HANDLER) + break; + if (i == f->f_iblock) + /* We did not create this exception. */ + RESTORE_AND_CLEAR_EXC_STATE() + else + SWAP_EXC_STATE() + } + if (tstate->use_tracing) { if (tstate->c_tracefunc) { if (why == WHY_RETURN || why == WHY_YIELD) { @@ -2992,13 +3086,6 @@ fast_yield: } } - if (tstate->frame->f_exc_type != NULL) - reset_exc_info(tstate); - else { - assert(tstate->frame->f_exc_value == NULL); - assert(tstate->frame->f_exc_traceback == NULL); - } - /* pop frame */ exit_eval_frame: Py_LeaveRecursiveCall(); @@ -3012,15 +3099,17 @@ exit_eval_frame: the test in the if statements in Misc/gdbinit (pystack and pystackv). */ PyObject * -PyEval_EvalCodeEx(PyCodeObject *co, PyObject *globals, PyObject *locals, +PyEval_EvalCodeEx(PyObject *_co, PyObject *globals, PyObject *locals, PyObject **args, int argcount, PyObject **kws, int kwcount, - PyObject **defs, int defcount, PyObject *closure) + PyObject **defs, int defcount, PyObject *kwdefs, PyObject *closure) { + PyCodeObject* co = (PyCodeObject*)_co; register PyFrameObject *f; register PyObject *retval = NULL; register PyObject **fastlocals, **freevars; PyThreadState *tstate = PyThreadState_GET(); PyObject *x, *u; + int total_args = co->co_argcount + co->co_kwonlyargcount; if (globals == NULL) { PyErr_SetString(PyExc_SystemError, @@ -3037,8 +3126,7 @@ PyEval_EvalCodeEx(PyCodeObject *co, PyObject *globals, PyObject *locals, fastlocals = f->f_localsplus; freevars = f->f_localsplus + co->co_nlocals; - if (co->co_argcount > 0 || - co->co_flags & (CO_VARARGS | CO_VARKEYWORDS)) { + if (total_args || co->co_flags & (CO_VARARGS | CO_VARKEYWORDS)) { int i; int n = argcount; PyObject *kwdict = NULL; @@ -3046,7 +3134,7 @@ PyEval_EvalCodeEx(PyCodeObject *co, PyObject *globals, PyObject *locals, kwdict = PyDict_New(); if (kwdict == NULL) goto fail; - i = co->co_argcount; + i = total_args; if (co->co_flags & CO_VARARGS) i++; SETLOCAL(i, kwdict); @@ -3054,9 +3142,9 @@ PyEval_EvalCodeEx(PyCodeObject *co, PyObject *globals, PyObject *locals, if (argcount > co->co_argcount) { if (!(co->co_flags & CO_VARARGS)) { PyErr_Format(PyExc_TypeError, - "%.200s() takes %s %d " - "argument%s (%d given)", - PyString_AsString(co->co_name), + "%U() takes %s %d " + "positional argument%s (%d given)", + co->co_name, defcount ? "at most" : "exactly", co->co_argcount, co->co_argcount == 1 ? "" : "s", @@ -3074,7 +3162,7 @@ PyEval_EvalCodeEx(PyCodeObject *co, PyObject *globals, PyObject *locals, u = PyTuple_New(argcount - n); if (u == NULL) goto fail; - SETLOCAL(co->co_argcount, u); + SETLOCAL(total_args, u); for (i = n; i < argcount; i++) { x = args[i]; Py_INCREF(x); @@ -3086,26 +3174,22 @@ PyEval_EvalCodeEx(PyCodeObject *co, PyObject *globals, PyObject *locals, PyObject *keyword = kws[2*i]; PyObject *value = kws[2*i + 1]; int j; - if (keyword == NULL || !(PyString_Check(keyword) -#ifdef Py_USING_UNICODE - || PyUnicode_Check(keyword) -#endif - )) { + if (keyword == NULL || !PyUnicode_Check(keyword)) { PyErr_Format(PyExc_TypeError, - "%.200s() keywords must be strings", - PyString_AsString(co->co_name)); + "%U() keywords must be strings", + co->co_name); goto fail; } /* Speed hack: do raw pointer compares. As names are normally interned this should almost always hit. */ co_varnames = ((PyTupleObject *)(co->co_varnames))->ob_item; - for (j = 0; j < co->co_argcount; j++) { + for (j = 0; j < total_args; j++) { PyObject *nm = co_varnames[j]; if (nm == keyword) goto kw_found; } /* Slow fallback, just in case */ - for (j = 0; j < co->co_argcount; j++) { + for (j = 0; j < total_args; j++) { PyObject *nm = co_varnames[j]; int cmp = PyObject_RichCompareBool( keyword, nm, Py_EQ); @@ -3114,37 +3198,49 @@ PyEval_EvalCodeEx(PyCodeObject *co, PyObject *globals, PyObject *locals, else if (cmp < 0) goto fail; } - if (kwdict == NULL) { - PyObject *kwd_str = kwd_as_string(keyword); - if (kwd_str) { - PyErr_Format(PyExc_TypeError, - "%.200s() got an unexpected " - "keyword argument '%.400s'", - PyString_AsString(co->co_name), - PyString_AsString(kwd_str)); - Py_DECREF(kwd_str); - } + if (j >= total_args && kwdict == NULL) { + PyErr_Format(PyExc_TypeError, + "%U() got an unexpected " + "keyword argument '%S'", + co->co_name, + keyword); goto fail; } PyDict_SetItem(kwdict, keyword, value); continue; kw_found: if (GETLOCAL(j) != NULL) { - PyObject *kwd_str = kwd_as_string(keyword); - if (kwd_str) { - PyErr_Format(PyExc_TypeError, - "%.200s() got multiple " - "values for keyword " - "argument '%.400s'", - PyString_AsString(co->co_name), - PyString_AsString(kwd_str)); - Py_DECREF(kwd_str); - } + PyErr_Format(PyExc_TypeError, + "%U() got multiple " + "values for keyword " + "argument '%S'", + co->co_name, + keyword); goto fail; } Py_INCREF(value); SETLOCAL(j, value); } + if (co->co_kwonlyargcount > 0) { + for (i = co->co_argcount; i < total_args; i++) { + PyObject *name; + if (GETLOCAL(i) != NULL) + continue; + name = PyTuple_GET_ITEM(co->co_varnames, i); + if (kwdefs != NULL) { + PyObject *def = PyDict_GetItem(kwdefs, name); + if (def) { + Py_INCREF(def); + SETLOCAL(i, def); + continue; + } + } + PyErr_Format(PyExc_TypeError, + "%U() needs keyword-only argument %S", + co->co_name, name); + goto fail; + } + } if (argcount < co->co_argcount) { int m = co->co_argcount - defcount; for (i = argcount; i < m; i++) { @@ -3154,13 +3250,14 @@ PyEval_EvalCodeEx(PyCodeObject *co, PyObject *globals, PyObject *locals, if (GETLOCAL(j)) given++; PyErr_Format(PyExc_TypeError, - "%.200s() takes %s %d " - "argument%s (%d given)", - PyString_AsString(co->co_name), + "%U() takes %s %d " + "argument%s " + "(%d given)", + co->co_name, ((co->co_flags & CO_VARARGS) || defcount) ? "at least" : "exactly", - m, m == 1 ? "" : "s", given); + m, m == 1 ? "" : "s", given); goto fail; } } @@ -3179,8 +3276,8 @@ PyEval_EvalCodeEx(PyCodeObject *co, PyObject *globals, PyObject *locals, } else if (argcount > 0 || kwcount > 0) { PyErr_Format(PyExc_TypeError, - "%.200s() takes no arguments (%d given)", - PyString_AsString(co->co_name), + "%U() takes no arguments (%d given)", + co->co_name, argcount + kwcount); goto fail; } @@ -3188,10 +3285,10 @@ PyEval_EvalCodeEx(PyCodeObject *co, PyObject *globals, PyObject *locals, vars into frame. This isn't too efficient right now. */ if (PyTuple_GET_SIZE(co->co_cellvars)) { int i, j, nargs, found; - char *cellname, *argname; + Py_UNICODE *cellname, *argname; PyObject *c; - nargs = co->co_argcount; + nargs = total_args; if (co->co_flags & CO_VARARGS) nargs++; if (co->co_flags & CO_VARKEYWORDS) @@ -3205,13 +3302,13 @@ PyEval_EvalCodeEx(PyCodeObject *co, PyObject *globals, PyObject *locals, list so that we can march over it more efficiently? */ for (i = 0; i < PyTuple_GET_SIZE(co->co_cellvars); ++i) { - cellname = PyString_AS_STRING( + cellname = PyUnicode_AS_UNICODE( PyTuple_GET_ITEM(co->co_cellvars, i)); found = 0; for (j = 0; j < nargs; j++) { - argname = PyString_AS_STRING( + argname = PyUnicode_AS_UNICODE( PyTuple_GET_ITEM(co->co_varnames, j)); - if (strcmp(cellname, argname) == 0) { + if (Py_UNICODE_strcmp(cellname, argname) == 0) { c = PyCell_New(GETLOCAL(j)); if (c == NULL) goto fail; @@ -3271,12 +3368,6 @@ static PyObject * special_lookup(PyObject *o, char *meth, PyObject **cache) { PyObject *res; - if (PyInstance_Check(o)) { - if (!*cache) - return PyObject_GetAttrString(o, meth); - else - return PyObject_GetAttr(o, *cache); - } res = _PyObject_LookupSpecial(o, meth, cache); if (res == NULL && !PyErr_Occurred()) { PyErr_SetObject(PyExc_AttributeError, *cache); @@ -3286,300 +3377,112 @@ special_lookup(PyObject *o, char *meth, PyObject **cache) } -static PyObject * -kwd_as_string(PyObject *kwd) { -#ifdef Py_USING_UNICODE - if (PyString_Check(kwd)) { -#else - assert(PyString_Check(kwd)); -#endif - Py_INCREF(kwd); - return kwd; -#ifdef Py_USING_UNICODE - } - return _PyUnicode_AsDefaultEncodedString(kwd, "replace"); -#endif -} - - -/* Implementation notes for set_exc_info() and reset_exc_info(): - -- Below, 'exc_ZZZ' stands for 'exc_type', 'exc_value' and - 'exc_traceback'. These always travel together. - -- tstate->curexc_ZZZ is the "hot" exception that is set by - PyErr_SetString(), cleared by PyErr_Clear(), and so on. - -- Once an exception is caught by an except clause, it is transferred - from tstate->curexc_ZZZ to tstate->exc_ZZZ, from which sys.exc_info() - can pick it up. This is the primary task of set_exc_info(). - XXX That can't be right: set_exc_info() doesn't look at tstate->curexc_ZZZ. - -- Now let me explain the complicated dance with frame->f_exc_ZZZ. - - Long ago, when none of this existed, there were just a few globals: - one set corresponding to the "hot" exception, and one set - corresponding to sys.exc_ZZZ. (Actually, the latter weren't C - globals; they were simply stored as sys.exc_ZZZ. For backwards - compatibility, they still are!) The problem was that in code like - this: - - try: - "something that may fail" - except "some exception": - "do something else first" - "print the exception from sys.exc_ZZZ." - - if "do something else first" invoked something that raised and caught - an exception, sys.exc_ZZZ were overwritten. That was a frequent - cause of subtle bugs. I fixed this by changing the semantics as - follows: - - - Within one frame, sys.exc_ZZZ will hold the last exception caught - *in that frame*. - - - But initially, and as long as no exception is caught in a given - frame, sys.exc_ZZZ will hold the last exception caught in the - previous frame (or the frame before that, etc.). - - The first bullet fixed the bug in the above example. The second - bullet was for backwards compatibility: it was (and is) common to - have a function that is called when an exception is caught, and to - have that function access the caught exception via sys.exc_ZZZ. - (Example: traceback.print_exc()). - - At the same time I fixed the problem that sys.exc_ZZZ weren't - thread-safe, by introducing sys.exc_info() which gets it from tstate; - but that's really a separate improvement. - - The reset_exc_info() function in ceval.c restores the tstate->exc_ZZZ - variables to what they were before the current frame was called. The - set_exc_info() function saves them on the frame so that - reset_exc_info() can restore them. The invariant is that - frame->f_exc_ZZZ is NULL iff the current frame never caught an - exception (where "catching" an exception applies only to successful - except clauses); and if the current frame ever caught an exception, - frame->f_exc_ZZZ is the exception that was stored in tstate->exc_ZZZ - at the start of the current frame. - -*/ - -static void -set_exc_info(PyThreadState *tstate, - PyObject *type, PyObject *value, PyObject *tb) -{ - PyFrameObject *frame = tstate->frame; - PyObject *tmp_type, *tmp_value, *tmp_tb; - - assert(type != NULL); - assert(frame != NULL); - if (frame->f_exc_type == NULL) { - assert(frame->f_exc_value == NULL); - assert(frame->f_exc_traceback == NULL); - /* This frame didn't catch an exception before. */ - /* Save previous exception of this thread in this frame. */ - if (tstate->exc_type == NULL) { - /* XXX Why is this set to Py_None? */ - Py_INCREF(Py_None); - tstate->exc_type = Py_None; - } - Py_INCREF(tstate->exc_type); - Py_XINCREF(tstate->exc_value); - Py_XINCREF(tstate->exc_traceback); - frame->f_exc_type = tstate->exc_type; - frame->f_exc_value = tstate->exc_value; - frame->f_exc_traceback = tstate->exc_traceback; - } - /* Set new exception for this thread. */ - tmp_type = tstate->exc_type; - tmp_value = tstate->exc_value; - tmp_tb = tstate->exc_traceback; - Py_INCREF(type); - Py_XINCREF(value); - Py_XINCREF(tb); - tstate->exc_type = type; - tstate->exc_value = value; - tstate->exc_traceback = tb; - Py_XDECREF(tmp_type); - Py_XDECREF(tmp_value); - Py_XDECREF(tmp_tb); - /* For b/w compatibility */ - PySys_SetObject("exc_type", type); - PySys_SetObject("exc_value", value); - PySys_SetObject("exc_traceback", tb); -} - -static void -reset_exc_info(PyThreadState *tstate) -{ - PyFrameObject *frame; - PyObject *tmp_type, *tmp_value, *tmp_tb; - - /* It's a precondition that the thread state's frame caught an - * exception -- verify in a debug build. - */ - assert(tstate != NULL); - frame = tstate->frame; - assert(frame != NULL); - assert(frame->f_exc_type != NULL); - - /* Copy the frame's exception info back to the thread state. */ - tmp_type = tstate->exc_type; - tmp_value = tstate->exc_value; - tmp_tb = tstate->exc_traceback; - Py_INCREF(frame->f_exc_type); - Py_XINCREF(frame->f_exc_value); - Py_XINCREF(frame->f_exc_traceback); - tstate->exc_type = frame->f_exc_type; - tstate->exc_value = frame->f_exc_value; - tstate->exc_traceback = frame->f_exc_traceback; - Py_XDECREF(tmp_type); - Py_XDECREF(tmp_value); - Py_XDECREF(tmp_tb); - - /* For b/w compatibility */ - PySys_SetObject("exc_type", frame->f_exc_type); - PySys_SetObject("exc_value", frame->f_exc_value); - PySys_SetObject("exc_traceback", frame->f_exc_traceback); - - /* Clear the frame's exception info. */ - tmp_type = frame->f_exc_type; - tmp_value = frame->f_exc_value; - tmp_tb = frame->f_exc_traceback; - frame->f_exc_type = NULL; - frame->f_exc_value = NULL; - frame->f_exc_traceback = NULL; - Py_DECREF(tmp_type); - Py_XDECREF(tmp_value); - Py_XDECREF(tmp_tb); -} - /* Logic for the raise statement (too complicated for inlining). This *consumes* a reference count to each of its arguments. */ static enum why_code -do_raise(PyObject *type, PyObject *value, PyObject *tb) +do_raise(PyObject *exc, PyObject *cause) { - if (type == NULL) { + PyObject *type = NULL, *value = NULL; + + if (exc == NULL) { /* Reraise */ PyThreadState *tstate = PyThreadState_GET(); - type = tstate->exc_type == NULL ? Py_None : tstate->exc_type; + PyObject *tb; + type = tstate->exc_type; value = tstate->exc_value; tb = tstate->exc_traceback; + if (type == Py_None) { + PyErr_SetString(PyExc_RuntimeError, + "No active exception to reraise"); + return WHY_EXCEPTION; + } Py_XINCREF(type); Py_XINCREF(value); Py_XINCREF(tb); + PyErr_Restore(type, value, tb); + return WHY_RERAISE; } /* We support the following forms of raise: - raise <class>, <classinstance> - raise <class>, <argument tuple> - raise <class>, None - raise <class>, <argument> - raise <classinstance>, None - raise <string>, <object> - raise <string>, None - - An omitted second argument is the same as None. - - In addition, raise <tuple>, <anything> is the same as - raising the tuple's first item (and it better have one!); - this rule is applied recursively. - - Finally, an optional third argument can be supplied, which - gives the traceback to be substituted (useful when - re-raising an exception after examining it). */ - - /* First, check the traceback argument, replacing None with - NULL. */ - if (tb == Py_None) { - Py_DECREF(tb); - tb = NULL; - } - else if (tb != NULL && !PyTraceBack_Check(tb)) { - PyErr_SetString(PyExc_TypeError, - "raise: arg 3 must be a traceback or None"); - goto raise_error; - } - - /* Next, replace a missing value with None */ - if (value == NULL) { - value = Py_None; - Py_INCREF(value); - } + raise + raise <instance> + raise <type> */ - /* Next, repeatedly, replace a tuple exception with its first item */ - while (PyTuple_Check(type) && PyTuple_Size(type) > 0) { - PyObject *tmp = type; - type = PyTuple_GET_ITEM(type, 0); - Py_INCREF(type); - Py_DECREF(tmp); - } - - if (PyExceptionClass_Check(type)) { - PyErr_NormalizeException(&type, &value, &tb); + if (PyExceptionClass_Check(exc)) { + type = exc; + value = PyObject_CallObject(exc, NULL); + if (value == NULL) + goto raise_error; if (!PyExceptionInstance_Check(value)) { PyErr_Format(PyExc_TypeError, - "calling %s() should have returned an instance of " - "BaseException, not '%s'", - ((PyTypeObject *)type)->tp_name, - Py_TYPE(value)->tp_name); + "calling %R should have returned an instance of " + "BaseException, not %R", + type, Py_TYPE(value)); goto raise_error; } } - else if (PyExceptionInstance_Check(type)) { - /* Raising an instance. The value should be a dummy. */ - if (value != Py_None) { - PyErr_SetString(PyExc_TypeError, - "instance exception may not have a separate value"); - goto raise_error; - } - else { - /* Normalize to raise <class>, <instance> */ - Py_DECREF(value); - value = type; - type = PyExceptionInstance_Class(type); - Py_INCREF(type); - } + else if (PyExceptionInstance_Check(exc)) { + value = exc; + type = PyExceptionInstance_Class(exc); + Py_INCREF(type); } else { /* Not something you can raise. You get an exception anyway, just not what you specified :-) */ - PyErr_Format(PyExc_TypeError, - "exceptions must be old-style classes or " - "derived from BaseException, not %s", - type->ob_type->tp_name); + Py_DECREF(exc); + PyErr_SetString(PyExc_TypeError, + "exceptions must derive from BaseException"); goto raise_error; } - assert(PyExceptionClass_Check(type)); - if (Py_Py3kWarningFlag && PyClass_Check(type)) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "exceptions must derive from BaseException " - "in 3.x", 1) < 0) + if (cause) { + PyObject *fixed_cause; + if (PyExceptionClass_Check(cause)) { + fixed_cause = PyObject_CallObject(cause, NULL); + if (fixed_cause == NULL) + goto raise_error; + Py_DECREF(cause); + } + else if (PyExceptionInstance_Check(cause)) { + fixed_cause = cause; + } + else { + PyErr_SetString(PyExc_TypeError, + "exception causes must derive from " + "BaseException"); goto raise_error; + } + PyException_SetCause(value, fixed_cause); } - PyErr_Restore(type, value, tb); - if (tb == NULL) - return WHY_EXCEPTION; - else - return WHY_RERAISE; - raise_error: + PyErr_SetObject(type, value); + /* PyErr_SetObject incref's its arguments */ + Py_XDECREF(value); + Py_XDECREF(type); + return WHY_EXCEPTION; + +raise_error: Py_XDECREF(value); Py_XDECREF(type); - Py_XDECREF(tb); + Py_XDECREF(cause); return WHY_EXCEPTION; } /* Iterate v argcnt times and store the results on the stack (via decreasing - sp). Return 1 for success, 0 if error. */ + sp). Return 1 for success, 0 if error. + + If argcntafter == -1, do a simple unpack. If it is >= 0, do an unpack + with a variable target. +*/ static int -unpack_iterable(PyObject *v, int argcnt, PyObject **sp) +unpack_iterable(PyObject *v, int argcnt, int argcntafter, PyObject **sp) { - int i = 0; + int i = 0, j = 0; + Py_ssize_t ll = 0; PyObject *it; /* iter(v) */ PyObject *w; + PyObject *l = NULL; /* variable list */ assert(v != NULL); @@ -3601,17 +3504,43 @@ unpack_iterable(PyObject *v, int argcnt, PyObject **sp) *--sp = w; } - /* We better have exhausted the iterator now. */ - w = PyIter_Next(it); - if (w == NULL) { - if (PyErr_Occurred()) - goto Error; - Py_DECREF(it); - return 1; + if (argcntafter == -1) { + /* We better have exhausted the iterator now. */ + w = PyIter_Next(it); + if (w == NULL) { + if (PyErr_Occurred()) + goto Error; + Py_DECREF(it); + return 1; + } + Py_DECREF(w); + PyErr_Format(PyExc_ValueError, "too many values to unpack " + "(expected %d)", argcnt); + goto Error; } - Py_DECREF(w); - PyErr_SetString(PyExc_ValueError, "too many values to unpack"); - /* fall through */ + + l = PySequence_List(it); + if (l == NULL) + goto Error; + *--sp = l; + i++; + + ll = PyList_GET_SIZE(l); + if (ll < argcntafter) { + PyErr_Format(PyExc_ValueError, "need more than %zd values to unpack", + argcnt + ll); + goto Error; + } + + /* Pop the "after-variable" args off the list. */ + for (j = argcntafter; j > 0; j--, i++) { + *--sp = PyList_GET_ITEM(l, ll - j); + } + /* Resize the list. */ + Py_SIZE(l) = ll - argcntafter; + Py_DECREF(it); + return 1; + Error: for (; i > 0; i--, sp++) Py_DECREF(*sp); @@ -3818,13 +3747,6 @@ PyEval_GetFrame(void) } int -PyEval_GetRestricted(void) -{ - PyFrameObject *current_frame = PyEval_GetFrame(); - return current_frame == NULL ? 0 : PyFrame_IsRestricted(current_frame); -} - -int PyEval_MergeCompilerFlags(PyCompilerFlags *cf) { PyFrameObject *current_frame = PyEval_GetFrame(); @@ -3847,17 +3769,6 @@ PyEval_MergeCompilerFlags(PyCompilerFlags *cf) return result; } -int -Py_FlushLine(void) -{ - PyObject *f = PySys_GetObject("stdout"); - if (f == NULL) - return 0; - if (!PyFile_SoftSpace(f, 0)) - return 0; - return PyFile_WriteString("\n", f); -} - /* External interface to call any callable object. The arg must be a tuple or NULL. The kw must be a dict or NULL. */ @@ -3898,17 +3809,11 @@ PyEval_GetFuncName(PyObject *func) if (PyMethod_Check(func)) return PyEval_GetFuncName(PyMethod_GET_FUNCTION(func)); else if (PyFunction_Check(func)) - return PyString_AsString(((PyFunctionObject*)func)->func_name); + return _PyUnicode_AsString(((PyFunctionObject*)func)->func_name); else if (PyCFunction_Check(func)) return ((PyCFunctionObject*)func)->m_ml->ml_name; - else if (PyClass_Check(func)) - return PyString_AsString(((PyClassObject*)func)->cl_name); - else if (PyInstance_Check(func)) { - return PyString_AsString( - ((PyInstanceObject*)func)->in_class->cl_name); - } else { + else return func->ob_type->tp_name; - } } const char * @@ -3920,13 +3825,8 @@ PyEval_GetFuncDesc(PyObject *func) return "()"; else if (PyCFunction_Check(func)) return "()"; - else if (PyClass_Check(func)) - return " constructor"; - else if (PyInstance_Check(func)) { - return " instance"; - } else { + else return " object"; - } } static void @@ -4073,12 +3973,14 @@ fast_function(PyObject *func, PyObject ***pp_stack, int n, int na, int nk) PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); PyObject *globals = PyFunction_GET_GLOBALS(func); PyObject *argdefs = PyFunction_GET_DEFAULTS(func); + PyObject *kwdefs = PyFunction_GET_KW_DEFAULTS(func); PyObject **d = NULL; int nd = 0; PCALL(PCALL_FUNCTION); PCALL(PCALL_FAST_FUNCTION); - if (argdefs == NULL && co->co_argcount == n && nk==0 && + if (argdefs == NULL && co->co_argcount == n && + co->co_kwonlyargcount == 0 && nk==0 && co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) { PyFrameObject *f; PyObject *retval = NULL; @@ -4114,9 +4016,9 @@ fast_function(PyObject *func, PyObject ***pp_stack, int n, int na, int nk) d = &PyTuple_GET_ITEM(argdefs, 0); nd = Py_SIZE(argdefs); } - return PyEval_EvalCodeEx(co, globals, + return PyEval_EvalCodeEx((PyObject*)co, globals, (PyObject *)NULL, (*pp_stack)-n, na, - (*pp_stack)-2*nk, nk, d, nd, + (*pp_stack)-2*nk, nk, d, nd, kwdefs, PyFunction_GET_CLOSURE(func)); } @@ -4140,10 +4042,10 @@ update_keyword_args(PyObject *orig_kwdict, int nk, PyObject ***pp_stack, if (PyDict_GetItem(kwdict, key) != NULL) { PyErr_Format(PyExc_TypeError, "%.200s%s got multiple values " - "for keyword argument '%.200s'", + "for keyword argument '%U'", PyEval_GetFuncName(func), PyEval_GetFuncDesc(func), - PyString_AsString(key)); + key); Py_DECREF(key); Py_DECREF(value); Py_DECREF(kwdict); @@ -4237,7 +4139,7 @@ do_call(PyObject *func, PyObject ***pp_stack, int na, int nk) } else result = PyObject_Call(func, callargs, kwdict); - call_fail: +call_fail: Py_XDECREF(callargs); Py_XDECREF(kwdict); return result; @@ -4354,14 +4256,7 @@ _PyEval_SliceIndex(PyObject *v, Py_ssize_t *pi) { if (v != NULL) { Py_ssize_t x; - if (PyInt_Check(v)) { - /* XXX(nnorwitz): I think PyInt_AS_LONG is correct, - however, it looks like it should be AsSsize_t. - There should be a comment here explaining why. - */ - x = PyInt_AS_LONG(v); - } - else if (PyIndex_Check(v)) { + if (PyIndex_Check(v)) { x = PyNumber_AsSsize_t(v, NULL); if (x == -1 && PyErr_Occurred()) return 0; @@ -4377,76 +4272,8 @@ _PyEval_SliceIndex(PyObject *v, Py_ssize_t *pi) return 1; } -#undef ISINDEX -#define ISINDEX(x) ((x) == NULL || \ - PyInt_Check(x) || PyLong_Check(x) || PyIndex_Check(x)) - -static PyObject * -apply_slice(PyObject *u, PyObject *v, PyObject *w) /* return u[v:w] */ -{ - PyTypeObject *tp = u->ob_type; - PySequenceMethods *sq = tp->tp_as_sequence; - - if (sq && sq->sq_slice && ISINDEX(v) && ISINDEX(w)) { - Py_ssize_t ilow = 0, ihigh = PY_SSIZE_T_MAX; - if (!_PyEval_SliceIndex(v, &ilow)) - return NULL; - if (!_PyEval_SliceIndex(w, &ihigh)) - return NULL; - return PySequence_GetSlice(u, ilow, ihigh); - } - else { - PyObject *slice = PySlice_New(v, w, NULL); - if (slice != NULL) { - PyObject *res = PyObject_GetItem(u, slice); - Py_DECREF(slice); - return res; - } - else - return NULL; - } -} - -static int -assign_slice(PyObject *u, PyObject *v, PyObject *w, PyObject *x) - /* u[v:w] = x */ -{ - PyTypeObject *tp = u->ob_type; - PySequenceMethods *sq = tp->tp_as_sequence; - - if (sq && sq->sq_ass_slice && ISINDEX(v) && ISINDEX(w)) { - Py_ssize_t ilow = 0, ihigh = PY_SSIZE_T_MAX; - if (!_PyEval_SliceIndex(v, &ilow)) - return -1; - if (!_PyEval_SliceIndex(w, &ihigh)) - return -1; - if (x == NULL) - return PySequence_DelSlice(u, ilow, ihigh); - else - return PySequence_SetSlice(u, ilow, ihigh, x); - } - else { - PyObject *slice = PySlice_New(v, w, NULL); - if (slice != NULL) { - int res; - if (x != NULL) - res = PyObject_SetItem(u, slice, x); - else - res = PyObject_DelItem(u, slice); - Py_DECREF(slice); - return res; - } - else - return -1; - } -} - -#define Py3kExceptionClass_Check(x) \ - (PyType_Check((x)) && \ - PyType_FastSubclass((PyTypeObject*)(x), Py_TPFLAGS_BASE_EXC_SUBCLASS)) - -#define CANNOT_CATCH_MSG "catching classes that don't inherit from " \ - "BaseException is not allowed in 3.x" +#define CANNOT_CATCH_MSG "catching classes that do not inherit from "\ + "BaseException is not allowed" static PyObject * cmp_outcome(int op, register PyObject *v, register PyObject *w) @@ -4476,48 +4303,18 @@ cmp_outcome(int op, register PyObject *v, register PyObject *w) length = PyTuple_Size(w); for (i = 0; i < length; i += 1) { PyObject *exc = PyTuple_GET_ITEM(w, i); - if (PyString_Check(exc)) { - int ret_val; - ret_val = PyErr_WarnEx( - PyExc_DeprecationWarning, - "catching of string " - "exceptions is deprecated", 1); - if (ret_val < 0) - return NULL; - } - else if (Py_Py3kWarningFlag && - !PyTuple_Check(exc) && - !Py3kExceptionClass_Check(exc)) - { - int ret_val; - ret_val = PyErr_WarnEx( - PyExc_DeprecationWarning, - CANNOT_CATCH_MSG, 1); - if (ret_val < 0) - return NULL; + if (!PyExceptionClass_Check(exc)) { + PyErr_SetString(PyExc_TypeError, + CANNOT_CATCH_MSG); + return NULL; } } } else { - if (PyString_Check(w)) { - int ret_val; - ret_val = PyErr_WarnEx( - PyExc_DeprecationWarning, - "catching of string " - "exceptions is deprecated", 1); - if (ret_val < 0) - return NULL; - } - else if (Py_Py3kWarningFlag && - !PyTuple_Check(w) && - !Py3kExceptionClass_Check(w)) - { - int ret_val; - ret_val = PyErr_WarnEx( - PyExc_DeprecationWarning, - CANNOT_CATCH_MSG, 1); - if (ret_val < 0) - return NULL; + if (!PyExceptionClass_Check(w)) { + PyErr_SetString(PyExc_TypeError, + CANNOT_CATCH_MSG); + return NULL; } } res = PyErr_GivenExceptionMatches(v, w); @@ -4537,9 +4334,7 @@ import_from(PyObject *v, PyObject *name) x = PyObject_GetAttr(v, name); if (x == NULL && PyErr_ExceptionMatches(PyExc_AttributeError)) { - PyErr_Format(PyExc_ImportError, - "cannot import name %.230s", - PyString_AsString(name)); + PyErr_Format(PyExc_ImportError, "cannot import name %S", name); } return x; } @@ -4581,8 +4376,8 @@ import_all_from(PyObject *locals, PyObject *v) break; } if (skip_leading_underscores && - PyString_Check(name) && - PyString_AS_STRING(name)[0] == '_') + PyUnicode_Check(name) && + PyUnicode_AS_UNICODE(name)[0] == '_') { Py_DECREF(name); continue; @@ -4603,188 +4398,51 @@ import_all_from(PyObject *locals, PyObject *v) return err; } -static PyObject * -build_class(PyObject *methods, PyObject *bases, PyObject *name) -{ - PyObject *metaclass = NULL, *result, *base; - - if (PyDict_Check(methods)) - metaclass = PyDict_GetItemString(methods, "__metaclass__"); - if (metaclass != NULL) - Py_INCREF(metaclass); - else if (PyTuple_Check(bases) && PyTuple_GET_SIZE(bases) > 0) { - base = PyTuple_GET_ITEM(bases, 0); - metaclass = PyObject_GetAttrString(base, "__class__"); - if (metaclass == NULL) { - PyErr_Clear(); - metaclass = (PyObject *)base->ob_type; - Py_INCREF(metaclass); - } - } - else { - PyObject *g = PyEval_GetGlobals(); - if (g != NULL && PyDict_Check(g)) - metaclass = PyDict_GetItemString(g, "__metaclass__"); - if (metaclass == NULL) - metaclass = (PyObject *) &PyClass_Type; - Py_INCREF(metaclass); - } - result = PyObject_CallFunctionObjArgs(metaclass, name, bases, methods, - NULL); - Py_DECREF(metaclass); - if (result == NULL && PyErr_ExceptionMatches(PyExc_TypeError)) { - /* A type error here likely means that the user passed - in a base that was not a class (such the random module - instead of the random.random type). Help them out with - by augmenting the error message with more information.*/ - - PyObject *ptype, *pvalue, *ptraceback; - - PyErr_Fetch(&ptype, &pvalue, &ptraceback); - if (PyString_Check(pvalue)) { - PyObject *newmsg; - newmsg = PyString_FromFormat( - "Error when calling the metaclass bases\n" - " %s", - PyString_AS_STRING(pvalue)); - if (newmsg != NULL) { - Py_DECREF(pvalue); - pvalue = newmsg; - } - } - PyErr_Restore(ptype, pvalue, ptraceback); - } - return result; -} - -static int -exec_statement(PyFrameObject *f, PyObject *prog, PyObject *globals, - PyObject *locals) -{ - int n; - PyObject *v; - int plain = 0; - - if (PyTuple_Check(prog) && globals == Py_None && locals == Py_None && - ((n = PyTuple_Size(prog)) == 2 || n == 3)) { - /* Backward compatibility hack */ - globals = PyTuple_GetItem(prog, 1); - if (n == 3) - locals = PyTuple_GetItem(prog, 2); - prog = PyTuple_GetItem(prog, 0); - } - if (globals == Py_None) { - globals = PyEval_GetGlobals(); - if (locals == Py_None) { - locals = PyEval_GetLocals(); - plain = 1; - } - if (!globals || !locals) { - PyErr_SetString(PyExc_SystemError, - "globals and locals cannot be NULL"); - return -1; - } - } - else if (locals == Py_None) - locals = globals; - if (!PyString_Check(prog) && -#ifdef Py_USING_UNICODE - !PyUnicode_Check(prog) && -#endif - !PyCode_Check(prog) && - !PyFile_Check(prog)) { - PyErr_SetString(PyExc_TypeError, - "exec: arg 1 must be a string, file, or code object"); - return -1; - } - if (!PyDict_Check(globals)) { - PyErr_SetString(PyExc_TypeError, - "exec: arg 2 must be a dictionary or None"); - return -1; - } - if (!PyMapping_Check(locals)) { - PyErr_SetString(PyExc_TypeError, - "exec: arg 3 must be a mapping or None"); - return -1; - } - if (PyDict_GetItemString(globals, "__builtins__") == NULL) - PyDict_SetItemString(globals, "__builtins__", f->f_builtins); - if (PyCode_Check(prog)) { - if (PyCode_GetNumFree((PyCodeObject *)prog) > 0) { - PyErr_SetString(PyExc_TypeError, - "code object passed to exec may not contain free variables"); - return -1; - } - v = PyEval_EvalCode((PyCodeObject *) prog, globals, locals); - } - else if (PyFile_Check(prog)) { - FILE *fp = PyFile_AsFile(prog); - char *name = PyString_AsString(PyFile_Name(prog)); - PyCompilerFlags cf; - if (name == NULL) - return -1; - cf.cf_flags = 0; - if (PyEval_MergeCompilerFlags(&cf)) - v = PyRun_FileFlags(fp, name, Py_file_input, globals, - locals, &cf); - else - v = PyRun_File(fp, name, Py_file_input, globals, - locals); - } - else { - PyObject *tmp = NULL; - char *str; - PyCompilerFlags cf; - cf.cf_flags = 0; -#ifdef Py_USING_UNICODE - if (PyUnicode_Check(prog)) { - tmp = PyUnicode_AsUTF8String(prog); - if (tmp == NULL) - return -1; - prog = tmp; - cf.cf_flags |= PyCF_SOURCE_IS_UTF8; - } -#endif - if (PyString_AsStringAndSize(prog, &str, NULL)) - return -1; - if (PyEval_MergeCompilerFlags(&cf)) - v = PyRun_StringFlags(str, Py_file_input, globals, - locals, &cf); - else - v = PyRun_String(str, Py_file_input, globals, locals); - Py_XDECREF(tmp); - } - if (plain) - PyFrame_LocalsToFast(f, 0); - if (v == NULL) - return -1; - Py_DECREF(v); - return 0; -} - static void -format_exc_check_arg(PyObject *exc, char *format_str, PyObject *obj) +format_exc_check_arg(PyObject *exc, const char *format_str, PyObject *obj) { - char *obj_str; + const char *obj_str; if (!obj) return; - obj_str = PyString_AsString(obj); + obj_str = _PyUnicode_AsString(obj); if (!obj_str) return; PyErr_Format(exc, format_str, obj_str); } +static void +format_exc_unbound(PyCodeObject *co, int oparg) +{ + PyObject *name; + /* Don't stomp existing exception */ + if (PyErr_Occurred()) + return; + if (oparg < PyTuple_GET_SIZE(co->co_cellvars)) { + name = PyTuple_GET_ITEM(co->co_cellvars, + oparg); + format_exc_check_arg( + PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + name); + } else { + name = PyTuple_GET_ITEM(co->co_freevars, oparg - + PyTuple_GET_SIZE(co->co_cellvars)); + format_exc_check_arg(PyExc_NameError, + UNBOUNDFREE_ERROR_MSG, name); + } +} + static PyObject * -string_concatenate(PyObject *v, PyObject *w, +unicode_concatenate(PyObject *v, PyObject *w, PyFrameObject *f, unsigned char *next_instr) { /* This function implements 'variable += expr' when both arguments - are strings. */ - Py_ssize_t v_len = PyString_GET_SIZE(v); - Py_ssize_t w_len = PyString_GET_SIZE(w); + are (Unicode) strings. */ + Py_ssize_t v_len = PyUnicode_GET_SIZE(v); + Py_ssize_t w_len = PyUnicode_GET_SIZE(w); Py_ssize_t new_len = v_len + w_len; if (new_len < 0) { PyErr_SetString(PyExc_OverflowError, @@ -4792,7 +4450,7 @@ string_concatenate(PyObject *v, PyObject *w, return NULL; } - if (v->ob_refcnt == 2) { + if (Py_REFCNT(v) == 2) { /* In the common case, there are 2 references to the value * stored in 'variable' when the += is performed: one on the * value stack (in 'v') and one still stored in the @@ -4833,12 +4491,12 @@ string_concatenate(PyObject *v, PyObject *w, } } - if (v->ob_refcnt == 1 && !PyString_CHECK_INTERNED(v)) { + if (Py_REFCNT(v) == 1 && !PyUnicode_CHECK_INTERNED(v)) { /* Now we own the last reference to 'v', so we can resize it * in-place. */ - if (_PyString_Resize(&v, new_len) != 0) { - /* XXX if _PyString_Resize() fails, 'v' has been + if (PyUnicode_Resize(&v, new_len) != 0) { + /* XXX if PyUnicode_Resize() fails, 'v' has been * deallocated so it cannot be put back into * 'variable'. The MemoryError is raised when there * is no value in 'variable', which might (very @@ -4847,14 +4505,15 @@ string_concatenate(PyObject *v, PyObject *w, return NULL; } /* copy 'w' into the newly allocated area of 'v' */ - memcpy(PyString_AS_STRING(v) + v_len, - PyString_AS_STRING(w), w_len); + memcpy(PyUnicode_AS_UNICODE(v) + v_len, + PyUnicode_AS_UNICODE(w), w_len*sizeof(Py_UNICODE)); return v; } else { /* When in-place resizing is not an option. */ - PyString_Concat(&v, w); - return v; + w = PyUnicode_Concat(v, w); + Py_DECREF(v); + return w; } } @@ -4867,7 +4526,7 @@ getarray(long a[256]) PyObject *l = PyList_New(256); if (l == NULL) return NULL; for (i = 0; i < 256; i++) { - PyObject *x = PyInt_FromLong(a[i]); + PyObject *x = PyLong_FromLong(a[i]); if (x == NULL) { Py_DECREF(l); return NULL; diff --git a/Python/ceval_gil.h b/Python/ceval_gil.h new file mode 100644 index 0000000000..bf7a350791 --- /dev/null +++ b/Python/ceval_gil.h @@ -0,0 +1,433 @@ +/* + * Implementation of the Global Interpreter Lock (GIL). + */ + +#include <stdlib.h> +#include <errno.h> + + +/* First some general settings */ + +/* microseconds (the Python API uses seconds, though) */ +#define DEFAULT_INTERVAL 5000 +static unsigned long gil_interval = DEFAULT_INTERVAL; +#define INTERVAL (gil_interval >= 1 ? gil_interval : 1) + +/* Enable if you want to force the switching of threads at least every `gil_interval` */ +#undef FORCE_SWITCHING +#define FORCE_SWITCHING + + +/* + Notes about the implementation: + + - The GIL is just a boolean variable (gil_locked) whose access is protected + by a mutex (gil_mutex), and whose changes are signalled by a condition + variable (gil_cond). gil_mutex is taken for short periods of time, + and therefore mostly uncontended. + + - In the GIL-holding thread, the main loop (PyEval_EvalFrameEx) must be + able to release the GIL on demand by another thread. A volatile boolean + variable (gil_drop_request) is used for that purpose, which is checked + at every turn of the eval loop. That variable is set after a wait of + `interval` microseconds on `gil_cond` has timed out. + + [Actually, another volatile boolean variable (eval_breaker) is used + which ORs several conditions into one. Volatile booleans are + sufficient as inter-thread signalling means since Python is run + on cache-coherent architectures only.] + + - A thread wanting to take the GIL will first let pass a given amount of + time (`interval` microseconds) before setting gil_drop_request. This + encourages a defined switching period, but doesn't enforce it since + opcodes can take an arbitrary time to execute. + + The `interval` value is available for the user to read and modify + using the Python API `sys.{get,set}switchinterval()`. + + - When a thread releases the GIL and gil_drop_request is set, that thread + ensures that another GIL-awaiting thread gets scheduled. + It does so by waiting on a condition variable (switch_cond) until + the value of gil_last_holder is changed to something else than its + own thread state pointer, indicating that another thread was able to + take the GIL. + + This is meant to prohibit the latency-adverse behaviour on multi-core + machines where one thread would speculatively release the GIL, but still + run and end up being the first to re-acquire it, making the "timeslices" + much longer than expected. + (Note: this mechanism is enabled with FORCE_SWITCHING above) +*/ + +#ifndef _POSIX_THREADS +/* This means pthreads are not implemented in libc headers, hence the macro + not present in unistd.h. But they still can be implemented as an external + library (e.g. gnu pth in pthread emulation) */ +# ifdef HAVE_PTHREAD_H +# include <pthread.h> /* _POSIX_THREADS */ +# endif +#endif + + +#ifdef _POSIX_THREADS + +/* + * POSIX support + */ + +#include <pthread.h> + +#define ADD_MICROSECONDS(tv, interval) \ +do { \ + tv.tv_usec += (long) interval; \ + tv.tv_sec += tv.tv_usec / 1000000; \ + tv.tv_usec %= 1000000; \ +} while (0) + +/* We assume all modern POSIX systems have gettimeofday() */ +#ifdef GETTIMEOFDAY_NO_TZ +#define GETTIMEOFDAY(ptv) gettimeofday(ptv) +#else +#define GETTIMEOFDAY(ptv) gettimeofday(ptv, (struct timezone *)NULL) +#endif + +#define MUTEX_T pthread_mutex_t +#define MUTEX_INIT(mut) \ + if (pthread_mutex_init(&mut, NULL)) { \ + Py_FatalError("pthread_mutex_init(" #mut ") failed"); }; +#define MUTEX_FINI(mut) \ + if (pthread_mutex_destroy(&mut)) { \ + Py_FatalError("pthread_mutex_destroy(" #mut ") failed"); }; +#define MUTEX_LOCK(mut) \ + if (pthread_mutex_lock(&mut)) { \ + Py_FatalError("pthread_mutex_lock(" #mut ") failed"); }; +#define MUTEX_UNLOCK(mut) \ + if (pthread_mutex_unlock(&mut)) { \ + Py_FatalError("pthread_mutex_unlock(" #mut ") failed"); }; + +#define COND_T pthread_cond_t +#define COND_INIT(cond) \ + if (pthread_cond_init(&cond, NULL)) { \ + Py_FatalError("pthread_cond_init(" #cond ") failed"); }; +#define COND_FINI(cond) \ + if (pthread_cond_destroy(&cond)) { \ + Py_FatalError("pthread_cond_destroy(" #cond ") failed"); }; +#define COND_SIGNAL(cond) \ + if (pthread_cond_signal(&cond)) { \ + Py_FatalError("pthread_cond_signal(" #cond ") failed"); }; +#define COND_WAIT(cond, mut) \ + if (pthread_cond_wait(&cond, &mut)) { \ + Py_FatalError("pthread_cond_wait(" #cond ") failed"); }; +#define COND_TIMED_WAIT(cond, mut, microseconds, timeout_result) \ + { \ + int r; \ + struct timespec ts; \ + struct timeval deadline; \ + \ + GETTIMEOFDAY(&deadline); \ + ADD_MICROSECONDS(deadline, microseconds); \ + ts.tv_sec = deadline.tv_sec; \ + ts.tv_nsec = deadline.tv_usec * 1000; \ + \ + r = pthread_cond_timedwait(&cond, &mut, &ts); \ + if (r == ETIMEDOUT) \ + timeout_result = 1; \ + else if (r) \ + Py_FatalError("pthread_cond_timedwait(" #cond ") failed"); \ + else \ + timeout_result = 0; \ + } \ + +#elif defined(NT_THREADS) + +/* + * Windows (2000 and later, as well as (hopefully) CE) support + */ + +#include <windows.h> + +#define MUTEX_T CRITICAL_SECTION +#define MUTEX_INIT(mut) do { \ + if (!(InitializeCriticalSectionAndSpinCount(&(mut), 4000))) \ + Py_FatalError("CreateMutex(" #mut ") failed"); \ +} while (0) +#define MUTEX_FINI(mut) \ + DeleteCriticalSection(&(mut)) +#define MUTEX_LOCK(mut) \ + EnterCriticalSection(&(mut)) +#define MUTEX_UNLOCK(mut) \ + LeaveCriticalSection(&(mut)) + +/* We emulate condition variables with a semaphore. + We use a Semaphore rather than an auto-reset event, because although + an auto-resent event might appear to solve the lost-wakeup bug (race + condition between releasing the outer lock and waiting) because it + maintains state even though a wait hasn't happened, there is still + a lost wakeup problem if more than one thread are interrupted in the + critical place. A semaphore solves that. + Because it is ok to signal a condition variable with no one + waiting, we need to keep track of the number of + waiting threads. Otherwise, the semaphore's state could rise + without bound. + + Generic emulations of the pthread_cond_* API using + Win32 functions can be found on the Web. + The following read can be edificating (or not): + http://www.cse.wustl.edu/~schmidt/win32-cv-1.html +*/ +typedef struct COND_T +{ + HANDLE sem; /* the semaphore */ + int n_waiting; /* how many are unreleased */ +} COND_T; + +__inline static void _cond_init(COND_T *cond) +{ + /* A semaphore with a large max value, The positive value + * is only needed to catch those "lost wakeup" events and + * race conditions when a timed wait elapses. + */ + if (!(cond->sem = CreateSemaphore(NULL, 0, 1000, NULL))) + Py_FatalError("CreateSemaphore() failed"); + cond->n_waiting = 0; +} + +__inline static void _cond_fini(COND_T *cond) +{ + BOOL ok = CloseHandle(cond->sem); + if (!ok) + Py_FatalError("CloseHandle() failed"); +} + +__inline static void _cond_wait(COND_T *cond, MUTEX_T *mut) +{ + ++cond->n_waiting; + MUTEX_UNLOCK(*mut); + /* "lost wakeup bug" would occur if the caller were interrupted here, + * but we are safe because we are using a semaphore wich has an internal + * count. + */ + if (WaitForSingleObject(cond->sem, INFINITE) == WAIT_FAILED) + Py_FatalError("WaitForSingleObject() failed"); + MUTEX_LOCK(*mut); +} + +__inline static int _cond_timed_wait(COND_T *cond, MUTEX_T *mut, + int us) +{ + DWORD r; + ++cond->n_waiting; + MUTEX_UNLOCK(*mut); + r = WaitForSingleObject(cond->sem, us / 1000); + if (r == WAIT_FAILED) + Py_FatalError("WaitForSingleObject() failed"); + MUTEX_LOCK(*mut); + if (r == WAIT_TIMEOUT) + --cond->n_waiting; + /* Here we have a benign race condition with _cond_signal. If the + * wait operation has timed out, but before we can acquire the + * mutex again to decrement n_waiting, a thread holding the mutex + * still sees a positive n_waiting value and may call + * ReleaseSemaphore and decrement n_waiting. + * This will cause n_waiting to be decremented twice. + * This is benign, though, because ReleaseSemaphore will also have + * been called, leaving the semaphore state positive. We may + * thus end up with semaphore in state 1, and n_waiting == -1, and + * the next time someone calls _cond_wait(), that thread will + * pass right through, decrementing the semaphore state and + * incrementing n_waiting, thus correcting the extra _cond_signal. + */ + return r == WAIT_TIMEOUT; +} + +__inline static void _cond_signal(COND_T *cond) { + /* NOTE: This must be called with the mutex held */ + if (cond->n_waiting > 0) { + if (!ReleaseSemaphore(cond->sem, 1, NULL)) + Py_FatalError("ReleaseSemaphore() failed"); + --cond->n_waiting; + } +} + +#define COND_INIT(cond) \ + _cond_init(&(cond)) +#define COND_FINI(cond) \ + _cond_fini(&(cond)) +#define COND_SIGNAL(cond) \ + _cond_signal(&(cond)) +#define COND_WAIT(cond, mut) \ + _cond_wait(&(cond), &(mut)) +#define COND_TIMED_WAIT(cond, mut, us, timeout_result) do { \ + (timeout_result) = _cond_timed_wait(&(cond), &(mut), us); \ +} while (0) + +#else + +#error You need either a POSIX-compatible or a Windows system! + +#endif /* _POSIX_THREADS, NT_THREADS */ + + +/* Whether the GIL is already taken (-1 if uninitialized). This is atomic + because it can be read without any lock taken in ceval.c. */ +static _Py_atomic_int gil_locked = {-1}; +/* Number of GIL switches since the beginning. */ +static unsigned long gil_switch_number = 0; +/* Last PyThreadState holding / having held the GIL. This helps us know + whether anyone else was scheduled after we dropped the GIL. */ +static _Py_atomic_address gil_last_holder = {NULL}; + +/* This condition variable allows one or several threads to wait until + the GIL is released. In addition, the mutex also protects the above + variables. */ +static COND_T gil_cond; +static MUTEX_T gil_mutex; + +#ifdef FORCE_SWITCHING +/* This condition variable helps the GIL-releasing thread wait for + a GIL-awaiting thread to be scheduled and take the GIL. */ +static COND_T switch_cond; +static MUTEX_T switch_mutex; +#endif + + +static int gil_created(void) +{ + return _Py_atomic_load_explicit(&gil_locked, _Py_memory_order_acquire) >= 0; +} + +static void create_gil(void) +{ + MUTEX_INIT(gil_mutex); +#ifdef FORCE_SWITCHING + MUTEX_INIT(switch_mutex); +#endif + COND_INIT(gil_cond); +#ifdef FORCE_SWITCHING + COND_INIT(switch_cond); +#endif + _Py_atomic_store_relaxed(&gil_last_holder, NULL); + _Py_ANNOTATE_RWLOCK_CREATE(&gil_locked); + _Py_atomic_store_explicit(&gil_locked, 0, _Py_memory_order_release); +} + +static void destroy_gil(void) +{ + MUTEX_FINI(gil_mutex); +#ifdef FORCE_SWITCHING + MUTEX_FINI(switch_mutex); +#endif + COND_FINI(gil_cond); +#ifdef FORCE_SWITCHING + COND_FINI(switch_cond); +#endif + _Py_atomic_store_explicit(&gil_locked, -1, _Py_memory_order_release); + _Py_ANNOTATE_RWLOCK_DESTROY(&gil_locked); +} + +static void recreate_gil(void) +{ + _Py_ANNOTATE_RWLOCK_DESTROY(&gil_locked); + /* XXX should we destroy the old OS resources here? */ + create_gil(); +} + +static void drop_gil(PyThreadState *tstate) +{ + if (!_Py_atomic_load_relaxed(&gil_locked)) + Py_FatalError("drop_gil: GIL is not locked"); + /* tstate is allowed to be NULL (early interpreter init) */ + if (tstate != NULL) { + /* Sub-interpreter support: threads might have been switched + under our feet using PyThreadState_Swap(). Fix the GIL last + holder variable so that our heuristics work. */ + _Py_atomic_store_relaxed(&gil_last_holder, tstate); + } + + MUTEX_LOCK(gil_mutex); + _Py_ANNOTATE_RWLOCK_RELEASED(&gil_locked, /*is_write=*/1); + _Py_atomic_store_relaxed(&gil_locked, 0); + COND_SIGNAL(gil_cond); + MUTEX_UNLOCK(gil_mutex); + +#ifdef FORCE_SWITCHING + if (_Py_atomic_load_relaxed(&gil_drop_request) && tstate != NULL) { + MUTEX_LOCK(switch_mutex); + /* Not switched yet => wait */ + if (_Py_atomic_load_relaxed(&gil_last_holder) == tstate) { + RESET_GIL_DROP_REQUEST(); + /* NOTE: if COND_WAIT does not atomically start waiting when + releasing the mutex, another thread can run through, take + the GIL and drop it again, and reset the condition + before we even had a chance to wait for it. */ + COND_WAIT(switch_cond, switch_mutex); + } + MUTEX_UNLOCK(switch_mutex); + } +#endif +} + +static void take_gil(PyThreadState *tstate) +{ + int err; + if (tstate == NULL) + Py_FatalError("take_gil: NULL tstate"); + + err = errno; + MUTEX_LOCK(gil_mutex); + + if (!_Py_atomic_load_relaxed(&gil_locked)) + goto _ready; + + while (_Py_atomic_load_relaxed(&gil_locked)) { + int timed_out = 0; + unsigned long saved_switchnum; + + saved_switchnum = gil_switch_number; + COND_TIMED_WAIT(gil_cond, gil_mutex, INTERVAL, timed_out); + /* If we timed out and no switch occurred in the meantime, it is time + to ask the GIL-holding thread to drop it. */ + if (timed_out && + _Py_atomic_load_relaxed(&gil_locked) && + gil_switch_number == saved_switchnum) { + SET_GIL_DROP_REQUEST(); + } + } +_ready: +#ifdef FORCE_SWITCHING + /* This mutex must be taken before modifying gil_last_holder (see drop_gil()). */ + MUTEX_LOCK(switch_mutex); +#endif + /* We now hold the GIL */ + _Py_atomic_store_relaxed(&gil_locked, 1); + _Py_ANNOTATE_RWLOCK_ACQUIRED(&gil_locked, /*is_write=*/1); + + if (tstate != _Py_atomic_load_relaxed(&gil_last_holder)) { + _Py_atomic_store_relaxed(&gil_last_holder, tstate); + ++gil_switch_number; + } + +#ifdef FORCE_SWITCHING + COND_SIGNAL(switch_cond); + MUTEX_UNLOCK(switch_mutex); +#endif + if (_Py_atomic_load_relaxed(&gil_drop_request)) { + RESET_GIL_DROP_REQUEST(); + } + if (tstate->async_exc != NULL) { + _PyEval_SignalAsyncExc(); + } + + MUTEX_UNLOCK(gil_mutex); + errno = err; +} + +void _PyEval_SetSwitchInterval(unsigned long microseconds) +{ + gil_interval = microseconds; +} + +unsigned long _PyEval_GetSwitchInterval() +{ + return gil_interval; +} diff --git a/Python/codecs.c b/Python/codecs.c index 7334eb3e36..e21834a5c1 100644 --- a/Python/codecs.c +++ b/Python/codecs.c @@ -61,10 +61,9 @@ PyObject *normalizestring(const char *string) return NULL; } - v = PyString_FromStringAndSize(NULL, len); - if (v == NULL) + p = PyMem_Malloc(len + 1); + if (p == NULL) return NULL; - p = PyString_AS_STRING(v); for (i = 0; i < len; i++) { register char ch = string[i]; if (ch == ' ') @@ -73,6 +72,11 @@ PyObject *normalizestring(const char *string) ch = Py_TOLOWER(Py_CHARMASK(ch)); p[i] = ch; } + p[i] = '\0'; + v = PyUnicode_FromString(p); + if (v == NULL) + return NULL; + PyMem_Free(p); return v; } @@ -112,7 +116,7 @@ PyObject *_PyCodec_Lookup(const char *encoding) v = normalizestring(encoding); if (v == NULL) goto onError; - PyString_InternInPlace(&v); + PyUnicode_InternInPlace(&v); /* First, try to lookup the name in the registry dictionary */ result = PyDict_GetItem(interp->codec_search_cache, v); @@ -167,7 +171,10 @@ PyObject *_PyCodec_Lookup(const char *encoding) } /* Cache and return the result */ - PyDict_SetItem(interp->codec_search_cache, v, result); + if (PyDict_SetItem(interp->codec_search_cache, v, result) < 0) { + Py_DECREF(result); + goto onError; + } Py_DECREF(args); return result; @@ -176,6 +183,23 @@ PyObject *_PyCodec_Lookup(const char *encoding) return NULL; } +/* Codec registry encoding check API. */ + +int PyCodec_KnownEncoding(const char *encoding) +{ + PyObject *codecs; + + codecs = _PyCodec_Lookup(encoding); + if (!codecs) { + PyErr_Clear(); + return 0; + } + else { + Py_DECREF(codecs); + return 1; + } +} + static PyObject *args_tuple(PyObject *object, const char *errors) @@ -190,7 +214,7 @@ PyObject *args_tuple(PyObject *object, if (errors) { PyObject *v; - v = PyString_FromString(errors); + v = PyUnicode_FromString(errors); if (v == NULL) { Py_DECREF(args); return NULL; @@ -317,7 +341,7 @@ PyObject *PyCodec_Encode(PyObject *object, { PyObject *encoder = NULL; PyObject *args = NULL, *result = NULL; - PyObject *v; + PyObject *v = NULL; encoder = PyCodec_Encoder(encoding); if (encoder == NULL) @@ -327,14 +351,14 @@ PyObject *PyCodec_Encode(PyObject *object, if (args == NULL) goto onError; - result = PyEval_CallObject(encoder,args); + result = PyEval_CallObject(encoder, args); if (result == NULL) goto onError; if (!PyTuple_Check(result) || PyTuple_GET_SIZE(result) != 2) { PyErr_SetString(PyExc_TypeError, - "encoder must return a tuple (object,integer)"); + "encoder must return a tuple (object, integer)"); goto onError; } v = PyTuple_GET_ITEM(result,0); @@ -446,14 +470,9 @@ static void wrong_exception_type(PyObject *exc) PyObject *name = PyObject_GetAttrString(type, "__name__"); Py_DECREF(type); if (name != NULL) { - PyObject *string = PyObject_Str(name); + PyErr_Format(PyExc_TypeError, + "don't know how to handle %S in error callback", name); Py_DECREF(name); - if (string != NULL) { - PyErr_Format(PyExc_TypeError, - "don't know how to handle %.400s in error callback", - PyString_AS_STRING(string)); - Py_DECREF(string); - } } } } @@ -468,7 +487,6 @@ PyObject *PyCodec_StrictErrors(PyObject *exc) } -#ifdef Py_USING_UNICODE PyObject *PyCodec_IgnoreErrors(PyObject *exc) { Py_ssize_t end; @@ -660,6 +678,13 @@ static Py_UNICODE hexdigits[] = { PyObject *PyCodec_BackslashReplaceErrors(PyObject *exc) { +#ifndef Py_UNICODE_WIDE +#define IS_SURROGATE_PAIR(p, end) \ + (*p >= 0xD800 && *p <= 0xDBFF && (p + 1) < end && \ + *(p + 1) >= 0xDC00 && *(p + 1) <= 0xDFFF) +#else +#define IS_SURROGATE_PAIR(p, end) 0 +#endif if (PyObject_IsInstance(exc, PyExc_UnicodeEncodeError)) { PyObject *restuple; PyObject *object; @@ -684,7 +709,12 @@ PyObject *PyCodec_BackslashReplaceErrors(PyObject *exc) else #endif if (*p >= 0x100) { - ressize += 1+1+4; + if (IS_SURROGATE_PAIR(p, startp+end)) { + ressize += 1+1+8; + ++p; + } + else + ressize += 1+1+4; } else ressize += 1+1+2; @@ -694,9 +724,12 @@ PyObject *PyCodec_BackslashReplaceErrors(PyObject *exc) return NULL; for (p = startp+start, outp = PyUnicode_AS_UNICODE(res); p < startp+end; ++p) { - Py_UNICODE c = *p; + Py_UCS4 c = (Py_UCS4) *p; *outp++ = '\\'; -#ifdef Py_UNICODE_WIDE + if (IS_SURROGATE_PAIR(p, startp+end)) { + c = ((*p & 0x3FF) << 10) + (*(p + 1) & 0x3FF) + 0x10000; + ++p; + } if (c >= 0x00010000) { *outp++ = 'U'; *outp++ = hexdigits[(c>>28)&0xf]; @@ -706,9 +739,7 @@ PyObject *PyCodec_BackslashReplaceErrors(PyObject *exc) *outp++ = hexdigits[(c>>12)&0xf]; *outp++ = hexdigits[(c>>8)&0xf]; } - else -#endif - if (c >= 0x100) { + else if (c >= 0x100) { *outp++ = 'u'; *outp++ = hexdigits[(c>>12)&0xf]; *outp++ = hexdigits[(c>>8)&0xf]; @@ -728,8 +759,167 @@ PyObject *PyCodec_BackslashReplaceErrors(PyObject *exc) wrong_exception_type(exc); return NULL; } +#undef IS_SURROGATE_PAIR } -#endif + +/* This handler is declared static until someone demonstrates + a need to call it directly. */ +static PyObject * +PyCodec_SurrogatePassErrors(PyObject *exc) +{ + PyObject *restuple; + PyObject *object; + Py_ssize_t start; + Py_ssize_t end; + PyObject *res; + if (PyObject_IsInstance(exc, PyExc_UnicodeEncodeError)) { + Py_UNICODE *p; + Py_UNICODE *startp; + char *outp; + if (PyUnicodeEncodeError_GetStart(exc, &start)) + return NULL; + if (PyUnicodeEncodeError_GetEnd(exc, &end)) + return NULL; + if (!(object = PyUnicodeEncodeError_GetObject(exc))) + return NULL; + startp = PyUnicode_AS_UNICODE(object); + res = PyBytes_FromStringAndSize(NULL, 3*(end-start)); + if (!res) { + Py_DECREF(object); + return NULL; + } + outp = PyBytes_AsString(res); + for (p = startp+start; p < startp+end; p++) { + Py_UNICODE ch = *p; + if (ch < 0xd800 || ch > 0xdfff) { + /* Not a surrogate, fail with original exception */ + PyErr_SetObject(PyExceptionInstance_Class(exc), exc); + Py_DECREF(res); + Py_DECREF(object); + return NULL; + } + *outp++ = (char)(0xe0 | (ch >> 12)); + *outp++ = (char)(0x80 | ((ch >> 6) & 0x3f)); + *outp++ = (char)(0x80 | (ch & 0x3f)); + } + restuple = Py_BuildValue("(On)", res, end); + Py_DECREF(res); + Py_DECREF(object); + return restuple; + } + else if (PyObject_IsInstance(exc, PyExc_UnicodeDecodeError)) { + unsigned char *p; + Py_UNICODE ch = 0; + if (PyUnicodeDecodeError_GetStart(exc, &start)) + return NULL; + if (!(object = PyUnicodeDecodeError_GetObject(exc))) + return NULL; + if (!(p = (unsigned char*)PyBytes_AsString(object))) { + Py_DECREF(object); + return NULL; + } + /* Try decoding a single surrogate character. If + there are more, let the codec call us again. */ + p += start; + if (PyBytes_GET_SIZE(object) - start >= 3 && + (p[0] & 0xf0) == 0xe0 && + (p[1] & 0xc0) == 0x80 && + (p[2] & 0xc0) == 0x80) { + /* it's a three-byte code */ + ch = ((p[0] & 0x0f) << 12) + ((p[1] & 0x3f) << 6) + (p[2] & 0x3f); + if (ch < 0xd800 || ch > 0xdfff) + /* it's not a surrogate - fail */ + ch = 0; + } + Py_DECREF(object); + if (ch == 0) { + PyErr_SetObject(PyExceptionInstance_Class(exc), exc); + return NULL; + } + return Py_BuildValue("(u#n)", &ch, 1, start+3); + } + else { + wrong_exception_type(exc); + return NULL; + } +} + +static PyObject * +PyCodec_SurrogateEscapeErrors(PyObject *exc) +{ + PyObject *restuple; + PyObject *object; + Py_ssize_t start; + Py_ssize_t end; + PyObject *res; + if (PyObject_IsInstance(exc, PyExc_UnicodeEncodeError)) { + Py_UNICODE *p; + Py_UNICODE *startp; + char *outp; + if (PyUnicodeEncodeError_GetStart(exc, &start)) + return NULL; + if (PyUnicodeEncodeError_GetEnd(exc, &end)) + return NULL; + if (!(object = PyUnicodeEncodeError_GetObject(exc))) + return NULL; + startp = PyUnicode_AS_UNICODE(object); + res = PyBytes_FromStringAndSize(NULL, end-start); + if (!res) { + Py_DECREF(object); + return NULL; + } + outp = PyBytes_AsString(res); + for (p = startp+start; p < startp+end; p++) { + Py_UNICODE ch = *p; + if (ch < 0xdc80 || ch > 0xdcff) { + /* Not a UTF-8b surrogate, fail with original exception */ + PyErr_SetObject(PyExceptionInstance_Class(exc), exc); + Py_DECREF(res); + Py_DECREF(object); + return NULL; + } + *outp++ = ch - 0xdc00; + } + restuple = Py_BuildValue("(On)", res, end); + Py_DECREF(res); + Py_DECREF(object); + return restuple; + } + else if (PyObject_IsInstance(exc, PyExc_UnicodeDecodeError)) { + unsigned char *p; + Py_UNICODE ch[4]; /* decode up to 4 bad bytes. */ + int consumed = 0; + if (PyUnicodeDecodeError_GetStart(exc, &start)) + return NULL; + if (PyUnicodeDecodeError_GetEnd(exc, &end)) + return NULL; + if (!(object = PyUnicodeDecodeError_GetObject(exc))) + return NULL; + if (!(p = (unsigned char*)PyBytes_AsString(object))) { + Py_DECREF(object); + return NULL; + } + while (consumed < 4 && consumed < end-start) { + /* Refuse to escape ASCII bytes. */ + if (p[start+consumed] < 128) + break; + ch[consumed] = 0xdc00 + p[start+consumed]; + consumed++; + } + Py_DECREF(object); + if (!consumed) { + /* codec complained about ASCII byte. */ + PyErr_SetObject(PyExceptionInstance_Class(exc), exc); + return NULL; + } + return Py_BuildValue("(u#n)", ch, consumed, start+consumed); + } + else { + wrong_exception_type(exc); + return NULL; + } +} + static PyObject *strict_errors(PyObject *self, PyObject *exc) { @@ -737,7 +927,6 @@ static PyObject *strict_errors(PyObject *self, PyObject *exc) } -#ifdef Py_USING_UNICODE static PyObject *ignore_errors(PyObject *self, PyObject *exc) { return PyCodec_IgnoreErrors(exc); @@ -760,7 +949,16 @@ static PyObject *backslashreplace_errors(PyObject *self, PyObject *exc) { return PyCodec_BackslashReplaceErrors(exc); } -#endif + +static PyObject *surrogatepass_errors(PyObject *self, PyObject *exc) +{ + return PyCodec_SurrogatePassErrors(exc); +} + +static PyObject *surrogateescape_errors(PyObject *self, PyObject *exc) +{ + return PyCodec_SurrogateEscapeErrors(exc); +} static int _PyCodecRegistry_Init(void) { @@ -779,7 +977,6 @@ static int _PyCodecRegistry_Init(void) "raises a UnicodeError on coding errors.") } }, -#ifdef Py_USING_UNICODE { "ignore", { @@ -821,8 +1018,23 @@ static int _PyCodecRegistry_Init(void) "which replaces an unencodable character with a " "backslashed escape sequence.") } + }, + { + "surrogatepass", + { + "surrogatepass", + surrogatepass_errors, + METH_O + } + }, + { + "surrogateescape", + { + "surrogateescape", + surrogateescape_errors, + METH_O + } } -#endif }; PyInterpreterState *interp = PyThreadState_GET()->interp; @@ -854,19 +1066,11 @@ static int _PyCodecRegistry_Init(void) interp->codec_error_registry == NULL) Py_FatalError("can't initialize codec registry"); - mod = PyImport_ImportModuleLevel("encodings", NULL, NULL, NULL, 0); + mod = PyImport_ImportModuleNoBlock("encodings"); if (mod == NULL) { - if (PyErr_ExceptionMatches(PyExc_ImportError)) { - /* Ignore ImportErrors... this is done so that - distributions can disable the encodings package. Note - that other errors are not masked, e.g. SystemErrors - raised to inform the user of an error in the Python - configuration are still reported back to the user. */ - PyErr_Clear(); - return 0; - } return -1; } Py_DECREF(mod); + interp->codecs_initialized = 1; return 0; } diff --git a/Python/compile.c b/Python/compile.c index 531bed43c4..d6f640fabb 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -25,10 +25,8 @@ #include "Python-ast.h" #include "node.h" -#include "pyarena.h" #include "ast.h" #include "code.h" -#include "compile.h" #include "symtable.h" #include "opcode.h" @@ -40,8 +38,9 @@ int Py_OptimizeFlag = 0; #define DEFAULT_LNOTAB_SIZE 16 #define COMP_GENEXP 0 -#define COMP_SETCOMP 1 -#define COMP_DICTCOMP 2 +#define COMP_LISTCOMP 1 +#define COMP_SETCOMP 2 +#define COMP_DICTCOMP 3 struct instr { unsigned i_jabs : 1; @@ -111,6 +110,7 @@ struct compiler_unit { PyObject *u_private; /* for private name mangling */ int u_argcount; /* number of arguments for block */ + int u_kwonlyargcount; /* number of keyword only arguments for block */ /* Pointer to the most recently allocated block. By following b_list members, you can reach all early allocated blocks. */ basicblock *u_blocks; @@ -121,7 +121,8 @@ struct compiler_unit { int u_firstlineno; /* the first lineno of the block */ int u_lineno; /* the lineno for the current stmt */ - bool u_lineno_set; /* boolean to indicate whether instr + int u_col_offset; /* the offset of the current stmt */ + int u_lineno_set; /* boolean to indicate whether instr has been generated with current lineno */ }; @@ -138,6 +139,7 @@ struct compiler { PyFutureFeatures *c_future; /* pointer to module's __future__ */ PyCompilerFlags *c_flags; + int c_optimize; /* optimization level */ int c_interactive; /* true if in interactive mode */ int c_nestlevel; @@ -174,9 +176,14 @@ static void compiler_pop_fblock(struct compiler *, enum fblocktype, static int compiler_in_loop(struct compiler *); static int inplace_binop(struct compiler *, operator_ty); -static int expr_constant(expr_ty e); +static int expr_constant(struct compiler *, expr_ty); static int compiler_with(struct compiler *, stmt_ty); +static int compiler_call_helper(struct compiler *c, int n, + asdl_seq *args, + asdl_seq *keywords, + expr_ty starargs, + expr_ty kwargs); static PyCodeObject *assemble(struct compiler *, int addNone); static PyObject *__doc__; @@ -188,16 +195,16 @@ _Py_Mangle(PyObject *privateobj, PyObject *ident) { /* Name mangling: __private becomes _classname__private. This is independent from how the name is used. */ - const char *p, *name = PyString_AsString(ident); - char *buffer; + const Py_UNICODE *p, *name = PyUnicode_AS_UNICODE(ident); + Py_UNICODE *buffer; size_t nlen, plen; - if (privateobj == NULL || !PyString_Check(privateobj) || + if (privateobj == NULL || !PyUnicode_Check(privateobj) || name == NULL || name[0] != '_' || name[1] != '_') { Py_INCREF(ident); return ident; } - p = PyString_AsString(privateobj); - nlen = strlen(name); + p = PyUnicode_AS_UNICODE(privateobj); + nlen = Py_UNICODE_strlen(name); /* Don't mangle __id__ or names with dots. The only time a name with a dot can occur is when @@ -208,30 +215,30 @@ _Py_Mangle(PyObject *privateobj, PyObject *ident) mangling of the module name, e.g. __M.X. */ if ((name[nlen-1] == '_' && name[nlen-2] == '_') - || strchr(name, '.')) { + || Py_UNICODE_strchr(name, '.')) { Py_INCREF(ident); return ident; /* Don't mangle __whatever__ */ } /* Strip leading underscores from class name */ while (*p == '_') p++; - if (*p == '\0') { + if (*p == 0) { Py_INCREF(ident); return ident; /* Don't mangle if class is just underscores */ } - plen = strlen(p); + plen = Py_UNICODE_strlen(p); assert(1 <= PY_SSIZE_T_MAX - nlen); assert(1 + nlen <= PY_SSIZE_T_MAX - plen); - ident = PyString_FromStringAndSize(NULL, 1 + nlen + plen); + ident = PyUnicode_FromStringAndSize(NULL, 1 + nlen + plen); if (!ident) return 0; /* ident = "_" + p[:plen] + name # i.e. 1+plen+nlen bytes */ - buffer = PyString_AS_STRING(ident); + buffer = PyUnicode_AS_UNICODE(ident); buffer[0] = '_'; - strncpy(buffer+1, p, plen); - strcpy(buffer+1+plen, name); + Py_UNICODE_strncpy(buffer+1, p, plen); + Py_UNICODE_strcpy(buffer+1+plen, name); return ident; } @@ -248,8 +255,8 @@ compiler_init(struct compiler *c) } PyCodeObject * -PyAST_Compile(mod_ty mod, const char *filename, PyCompilerFlags *flags, - PyArena *arena) +PyAST_CompileEx(mod_ty mod, const char *filename, PyCompilerFlags *flags, + int optimize, PyArena *arena) { struct compiler c; PyCodeObject *co = NULL; @@ -257,7 +264,7 @@ PyAST_Compile(mod_ty mod, const char *filename, PyCompilerFlags *flags, int merged; if (!__doc__) { - __doc__ = PyString_InternFromString("__doc__"); + __doc__ = PyUnicode_InternFromString("__doc__"); if (!__doc__) return NULL; } @@ -277,6 +284,7 @@ PyAST_Compile(mod_ty mod, const char *filename, PyCompilerFlags *flags, c.c_future->ff_features = merged; flags->cf_flags = merged; c.c_flags = flags; + c.c_optimize = (optimize == -1) ? Py_OptimizeFlag : optimize; c.c_nestlevel = 0; c.c_st = PySymtable_Build(mod, filename, c.c_future); @@ -329,7 +337,7 @@ list2dict(PyObject *list) n = PyList_Size(list); for (i = 0; i < n; i++) { - v = PyInt_FromLong(i); + v = PyLong_FromLong(i); if (!v) { Py_DECREF(dict); return NULL; @@ -382,14 +390,16 @@ dictbytype(PyObject *src, int scope_type, int flag, int offset) num_keys = PyList_GET_SIZE(sorted_keys); for (key_i = 0; key_i < num_keys; key_i++) { + /* XXX this should probably be a macro in symtable.h */ + long vi; k = PyList_GET_ITEM(sorted_keys, key_i); v = PyDict_GetItem(src, k); - /* XXX this should probably be a macro in symtable.h */ - assert(PyInt_Check(v)); - scope = (PyInt_AS_LONG(v) >> SCOPE_OFF) & SCOPE_MASK; + assert(PyLong_Check(v)); + vi = PyLong_AS_LONG(v); + scope = (vi >> SCOPE_OFFSET) & SCOPE_MASK; - if (scope == scope_type || PyInt_AS_LONG(v) & flag) { - PyObject *tuple, *item = PyInt_FromLong(i); + if (scope == scope_type || vi & flag) { + PyObject *tuple, *item = PyLong_FromLong(i); if (item == NULL) { Py_DECREF(sorted_keys); Py_DECREF(dest); @@ -471,6 +481,7 @@ compiler_enter_scope(struct compiler *c, identifier name, void *key, } memset(u, 0, sizeof(struct compiler_unit)); u->u_argcount = 0; + u->u_kwonlyargcount = 0; u->u_ste = PySymtable_Lookup(c->c_st, key); if (!u->u_ste) { compiler_unit_free(u); @@ -496,7 +507,8 @@ compiler_enter_scope(struct compiler *c, identifier name, void *key, u->u_nfblocks = 0; u->u_firstlineno = lineno; u->u_lineno = 0; - u->u_lineno_set = false; + u->u_col_offset = 0; + u->u_lineno_set = 0; u->u_consts = PyDict_New(); if (!u->u_consts) { compiler_unit_free(u); @@ -674,7 +686,7 @@ compiler_set_lineno(struct compiler *c, int off) basicblock *b; if (c->u->u_lineno_set) return; - c->u->u_lineno_set = true; + c->u->u_lineno_set = 1; b = c->u->u_curblock; b->b_instr[off].i_lineno = c->u->u_lineno; } @@ -690,26 +702,23 @@ opcode_stack_effect(int opcode, int oparg) return 0; case DUP_TOP: return 1; - case ROT_FOUR: - return 0; + case DUP_TOP_TWO: + return 2; case UNARY_POSITIVE: case UNARY_NEGATIVE: case UNARY_NOT: - case UNARY_CONVERT: case UNARY_INVERT: return 0; case SET_ADD: case LIST_APPEND: return -1; - case MAP_ADD: return -2; case BINARY_POWER: case BINARY_MULTIPLY: - case BINARY_DIVIDE: case BINARY_MODULO: case BINARY_ADD: case BINARY_SUBTRACT: @@ -721,37 +730,9 @@ opcode_stack_effect(int opcode, int oparg) case INPLACE_TRUE_DIVIDE: return -1; - case SLICE+0: - return 0; - case SLICE+1: - return -1; - case SLICE+2: - return -1; - case SLICE+3: - return -2; - - case STORE_SLICE+0: - return -2; - case STORE_SLICE+1: - return -3; - case STORE_SLICE+2: - return -3; - case STORE_SLICE+3: - return -4; - - case DELETE_SLICE+0: - return -1; - case DELETE_SLICE+1: - return -2; - case DELETE_SLICE+2: - return -2; - case DELETE_SLICE+3: - return -3; - case INPLACE_ADD: case INPLACE_SUBTRACT: case INPLACE_MULTIPLY: - case INPLACE_DIVIDE: case INPLACE_MODULO: return -1; case STORE_SUBSCR: @@ -774,14 +755,8 @@ opcode_stack_effect(int opcode, int oparg) case PRINT_EXPR: return -1; - case PRINT_ITEM: - return -1; - case PRINT_NEWLINE: - return 0; - case PRINT_ITEM_TO: - return -2; - case PRINT_NEWLINE_TO: - return -1; + case LOAD_BUILD_CLASS: + return 1; case INPLACE_LSHIFT: case INPLACE_RSHIFT: case INPLACE_AND: @@ -791,27 +766,24 @@ opcode_stack_effect(int opcode, int oparg) case BREAK_LOOP: return 0; case SETUP_WITH: - return 4; + return 7; case WITH_CLEANUP: return -1; /* XXX Sometimes more */ - case LOAD_LOCALS: - return 1; + case STORE_LOCALS: + return -1; case RETURN_VALUE: return -1; case IMPORT_STAR: return -1; - case EXEC_STMT: - return -3; case YIELD_VALUE: return 0; case POP_BLOCK: return 0; + case POP_EXCEPT: + return 0; /* -3 except if bad bytecode */ case END_FINALLY: - return -3; /* or -1 or -2 if no exception occurred or - return/break/continue */ - case BUILD_CLASS: - return -2; + return -1; /* or -2 or -3 if exception occurred */ case STORE_NAME: return -1; @@ -819,6 +791,8 @@ opcode_stack_effect(int opcode, int oparg) return 0; case UNPACK_SEQUENCE: return oparg-1; + case UNPACK_EX: + return (oparg&0xFF) + (oparg>>8); case FOR_ITER: return 1; /* or -1, at end of iterator */ @@ -830,8 +804,6 @@ opcode_stack_effect(int opcode, int oparg) return -1; case DELETE_GLOBAL: return 0; - case DUP_TOPX: - return oparg; case LOAD_CONST: return 1; case LOAD_NAME: @@ -867,9 +839,11 @@ opcode_stack_effect(int opcode, int oparg) case CONTINUE_LOOP: return 0; case SETUP_LOOP: + return 0; case SETUP_EXCEPT: case SETUP_FINALLY: - return 0; + return 6; /* can push 3 values for the new exception + + 3 others for the previous exception state */ case LOAD_FAST: return 1; @@ -880,7 +854,7 @@ opcode_stack_effect(int opcode, int oparg) case RAISE_VARARGS: return -oparg; -#define NARGS(o) (((o) % 256) + 2*((o) / 256)) +#define NARGS(o) (((o) % 256) + 2*(((o) / 256) % 256)) case CALL_FUNCTION: return -NARGS(oparg); case CALL_FUNCTION_VAR: @@ -888,23 +862,25 @@ opcode_stack_effect(int opcode, int oparg) return -NARGS(oparg)-1; case CALL_FUNCTION_VAR_KW: return -NARGS(oparg)-2; -#undef NARGS case MAKE_FUNCTION: - return -oparg; + return -NARGS(oparg) - ((oparg >> 16) & 0xffff); + case MAKE_CLOSURE: + return -1 - NARGS(oparg) - ((oparg >> 16) & 0xffff); +#undef NARGS case BUILD_SLICE: if (oparg == 3) return -2; else return -1; - case MAKE_CLOSURE: - return -oparg-1; case LOAD_CLOSURE: return 1; case LOAD_DEREF: return 1; case STORE_DEREF: return -1; + case DELETE_DEREF: + return 0; default: fprintf(stderr, "opcode = %d\n", opcode); Py_FatalError("opcode_stack_effect()"); @@ -955,7 +931,6 @@ compiler_add_o(struct compiler *c, PyObject *dict, PyObject *o) else t = PyTuple_Pack(2, o, o->ob_type); } -#ifndef WITHOUT_COMPLEX else if (PyComplex_Check(o)) { Py_complex z; int real_negzero, imag_negzero; @@ -980,7 +955,6 @@ compiler_add_o(struct compiler *c, PyObject *dict, PyObject *o) t = PyTuple_Pack(2, o, o->ob_type); } } -#endif /* WITHOUT_COMPLEX */ else { t = PyTuple_Pack(2, o, o->ob_type); } @@ -989,8 +963,10 @@ compiler_add_o(struct compiler *c, PyObject *dict, PyObject *o) v = PyDict_GetItem(dict, t); if (!v) { + if (PyErr_Occurred()) + return -1; arg = PyDict_Size(dict); - v = PyInt_FromLong(arg); + v = PyLong_FromLong(arg); if (!v) { Py_DECREF(t); return -1; @@ -1003,7 +979,7 @@ compiler_add_o(struct compiler *c, PyObject *dict, PyObject *o) Py_DECREF(v); } else - arg = PyInt_AsLong(v); + arg = PyLong_AsLong(v); Py_DECREF(t); return arg; } @@ -1195,7 +1171,7 @@ compiler_body(struct compiler *c, asdl_seq *stmts) if (!asdl_seq_LEN(stmts)) return 1; st = (stmt_ty)asdl_seq_GET(stmts, 0); - if (compiler_isdocstring(st) && Py_OptimizeFlag < 2) { + if (compiler_isdocstring(st) && c->c_optimize < 2) { /* don't generate docstrings if -OO */ i = 1; VISIT(c, expr, st->v.Expr.value); @@ -1214,7 +1190,7 @@ compiler_mod(struct compiler *c, mod_ty mod) int addNone = 1; static PyObject *module; if (!module) { - module = PyString_InternFromString("<module>"); + module = PyUnicode_InternFromString("<module>"); if (!module) return NULL; } @@ -1266,8 +1242,8 @@ get_ref_type(struct compiler *c, PyObject *name) PyOS_snprintf(buf, sizeof(buf), "unknown scope for %.100s in %.100s(%s) in %s\n" "symbols: %s\nlocals: %s\nglobals: %s", - PyString_AS_STRING(name), - PyString_AS_STRING(c->u->u_name), + PyBytes_AS_STRING(name), + PyBytes_AS_STRING(c->u->u_name), PyObject_REPR(c->u->u_ste->ste_id), c->c_filename, PyObject_REPR(c->u->u_ste->ste_symbols), @@ -1291,7 +1267,7 @@ compiler_lookup_arg(PyObject *dict, PyObject *name) Py_DECREF(k); if (v == NULL) return -1; - return PyInt_AS_LONG(v); + return PyLong_AS_LONG(v); } static int @@ -1322,12 +1298,13 @@ compiler_make_closure(struct compiler *c, PyCodeObject *co, int args) else /* (reftype == FREE) */ arg = compiler_lookup_arg(c->u->u_freevars, name); if (arg == -1) { - printf("lookup %s in %s %d %d\n" + fprintf(stderr, + "lookup %s in %s %d %d\n" "freevars of %s: %s\n", PyObject_REPR(name), - PyString_AS_STRING(c->u->u_name), + PyBytes_AS_STRING(c->u->u_name), reftype, arg, - PyString_AS_STRING(co->co_name), + _PyUnicode_AsString(co->co_name), PyObject_REPR(co->co_freevars)); Py_FatalError("compiler_make_closure()"); } @@ -1354,27 +1331,126 @@ compiler_decorators(struct compiler *c, asdl_seq* decos) } static int -compiler_arguments(struct compiler *c, arguments_ty args) +compiler_visit_kwonlydefaults(struct compiler *c, asdl_seq *kwonlyargs, + asdl_seq *kw_defaults) { - int i; - int n = asdl_seq_LEN(args->args); - /* Correctly handle nested argument lists */ - for (i = 0; i < n; i++) { - expr_ty arg = (expr_ty)asdl_seq_GET(args->args, i); - if (arg->kind == Tuple_kind) { - PyObject *id = PyString_FromFormat(".%d", i); - if (id == NULL) { - return 0; - } - if (!compiler_nameop(c, id, Load)) { - Py_DECREF(id); - return 0; + int i, default_count = 0; + for (i = 0; i < asdl_seq_LEN(kwonlyargs); i++) { + arg_ty arg = asdl_seq_GET(kwonlyargs, i); + expr_ty default_ = asdl_seq_GET(kw_defaults, i); + if (default_) { + PyObject *mangled = _Py_Mangle(c->u->u_private, arg->arg); + if (!mangled) + return -1; + ADDOP_O(c, LOAD_CONST, mangled, consts); + Py_DECREF(mangled); + if (!compiler_visit_expr(c, default_)) { + return -1; } - Py_DECREF(id); - VISIT(c, expr, arg); + default_count++; } } - return 1; + return default_count; +} + +static int +compiler_visit_argannotation(struct compiler *c, identifier id, + expr_ty annotation, PyObject *names) +{ + if (annotation) { + VISIT(c, expr, annotation); + if (PyList_Append(names, id)) + return -1; + } + return 0; +} + +static int +compiler_visit_argannotations(struct compiler *c, asdl_seq* args, + PyObject *names) +{ + int i, error; + for (i = 0; i < asdl_seq_LEN(args); i++) { + arg_ty arg = (arg_ty)asdl_seq_GET(args, i); + error = compiler_visit_argannotation( + c, + arg->arg, + arg->annotation, + names); + if (error) + return error; + } + return 0; +} + +static int +compiler_visit_annotations(struct compiler *c, arguments_ty args, + expr_ty returns) +{ + /* Push arg annotations and a list of the argument names. Return the # + of items pushed. The expressions are evaluated out-of-order wrt the + source code. + + More than 2^16-1 annotations is a SyntaxError. Returns -1 on error. + */ + static identifier return_str; + PyObject *names; + int len; + names = PyList_New(0); + if (!names) + return -1; + + if (compiler_visit_argannotations(c, args->args, names)) + goto error; + if (args->varargannotation && + compiler_visit_argannotation(c, args->vararg, + args->varargannotation, names)) + goto error; + if (compiler_visit_argannotations(c, args->kwonlyargs, names)) + goto error; + if (args->kwargannotation && + compiler_visit_argannotation(c, args->kwarg, + args->kwargannotation, names)) + goto error; + + if (!return_str) { + return_str = PyUnicode_InternFromString("return"); + if (!return_str) + goto error; + } + if (compiler_visit_argannotation(c, return_str, returns, names)) { + goto error; + } + + len = PyList_GET_SIZE(names); + if (len > 65534) { + /* len must fit in 16 bits, and len is incremented below */ + PyErr_SetString(PyExc_SyntaxError, + "too many annotations"); + goto error; + } + if (len) { + /* convert names to a tuple and place on stack */ + PyObject *elt; + int i; + PyObject *s = PyTuple_New(len); + if (!s) + goto error; + for (i = 0; i < len; i++) { + elt = PyList_GET_ITEM(names, i); + Py_INCREF(elt); + PyTuple_SET_ITEM(s, i, elt); + } + ADDOP_O(c, LOAD_CONST, s, consts); + Py_DECREF(s); + len++; /* include the just-pushed tuple */ + } + Py_DECREF(names); + return len; + +error: + Py_DECREF(names); + return -1; } static int @@ -1383,33 +1459,45 @@ compiler_function(struct compiler *c, stmt_ty s) PyCodeObject *co; PyObject *first_const = Py_None; arguments_ty args = s->v.FunctionDef.args; + expr_ty returns = s->v.FunctionDef.returns; asdl_seq* decos = s->v.FunctionDef.decorator_list; stmt_ty st; - int i, n, docstring; + int i, n, docstring, kw_default_count = 0, arglength; + int num_annotations; assert(s->kind == FunctionDef_kind); if (!compiler_decorators(c, decos)) return 0; + if (args->kwonlyargs) { + int res = compiler_visit_kwonlydefaults(c, args->kwonlyargs, + args->kw_defaults); + if (res < 0) + return 0; + kw_default_count = res; + } if (args->defaults) VISIT_SEQ(c, expr, args->defaults); + num_annotations = compiler_visit_annotations(c, args, returns); + if (num_annotations < 0) + return 0; + assert((num_annotations & 0xFFFF) == num_annotations); + if (!compiler_enter_scope(c, s->v.FunctionDef.name, (void *)s, s->lineno)) return 0; st = (stmt_ty)asdl_seq_GET(s->v.FunctionDef.body, 0); docstring = compiler_isdocstring(st); - if (docstring && Py_OptimizeFlag < 2) + if (docstring && c->c_optimize < 2) first_const = st->v.Expr.value->v.Str.s; if (compiler_add_o(c, c->u->u_consts, first_const) < 0) { compiler_exit_scope(c); return 0; } - /* unpack nested arguments */ - compiler_arguments(c, args); - c->u->u_argcount = asdl_seq_LEN(args->args); + c->u->u_kwonlyargcount = asdl_seq_LEN(args->kwonlyargs); n = asdl_seq_LEN(s->v.FunctionDef.body); /* if there was a docstring, we need to skip the first statement */ for (i = docstring; i < n; i++) { @@ -1421,9 +1509,13 @@ compiler_function(struct compiler *c, stmt_ty s) if (co == NULL) return 0; - compiler_make_closure(c, co, asdl_seq_LEN(args->defaults)); + arglength = asdl_seq_LEN(args->defaults); + arglength |= kw_default_count << 8; + arglength |= num_annotations << 16; + compiler_make_closure(c, co, arglength); Py_DECREF(co); + /* decorators */ for (i = 0; i < asdl_seq_LEN(decos); i++) { ADDOP_I(c, CALL_FUNCTION, 1); } @@ -1434,64 +1526,113 @@ compiler_function(struct compiler *c, stmt_ty s) static int compiler_class(struct compiler *c, stmt_ty s) { - int n, i; PyCodeObject *co; PyObject *str; + int i; asdl_seq* decos = s->v.ClassDef.decorator_list; if (!compiler_decorators(c, decos)) return 0; - /* push class name on stack, needed by BUILD_CLASS */ - ADDOP_O(c, LOAD_CONST, s->v.ClassDef.name, consts); - /* push the tuple of base classes on the stack */ - n = asdl_seq_LEN(s->v.ClassDef.bases); - if (n > 0) - VISIT_SEQ(c, expr, s->v.ClassDef.bases); - ADDOP_I(c, BUILD_TUPLE, n); - if (!compiler_enter_scope(c, s->v.ClassDef.name, (void *)s, - s->lineno)) - return 0; - Py_XDECREF(c->u->u_private); - c->u->u_private = s->v.ClassDef.name; - Py_INCREF(c->u->u_private); - str = PyString_InternFromString("__name__"); - if (!str || !compiler_nameop(c, str, Load)) { - Py_XDECREF(str); - compiler_exit_scope(c); - return 0; - } + /* ultimately generate code for: + <name> = __build_class__(<func>, <name>, *<bases>, **<keywords>) + where: + <func> is a function/closure created from the class body; + it has a single argument (__locals__) where the dict + (or MutableSequence) representing the locals is passed + <name> is the class name + <bases> is the positional arguments and *varargs argument + <keywords> is the keyword arguments and **kwds argument + This borrows from compiler_call. + */ - Py_DECREF(str); - str = PyString_InternFromString("__module__"); - if (!str || !compiler_nameop(c, str, Store)) { - Py_XDECREF(str); - compiler_exit_scope(c); - return 0; - } - Py_DECREF(str); - - if (!compiler_body(c, s->v.ClassDef.body)) { - compiler_exit_scope(c); + /* 1. compile the class body into a code object */ + if (!compiler_enter_scope(c, s->v.ClassDef.name, (void *)s, s->lineno)) return 0; + /* this block represents what we do in the new scope */ + { + /* use the class name for name mangling */ + Py_INCREF(s->v.ClassDef.name); + Py_XDECREF(c->u->u_private); + c->u->u_private = s->v.ClassDef.name; + /* force it to have one mandatory argument */ + c->u->u_argcount = 1; + /* load the first argument (__locals__) ... */ + ADDOP_I(c, LOAD_FAST, 0); + /* ... and store it into f_locals */ + ADDOP_IN_SCOPE(c, STORE_LOCALS); + /* load (global) __name__ ... */ + str = PyUnicode_InternFromString("__name__"); + if (!str || !compiler_nameop(c, str, Load)) { + Py_XDECREF(str); + compiler_exit_scope(c); + return 0; + } + Py_DECREF(str); + /* ... and store it as __module__ */ + str = PyUnicode_InternFromString("__module__"); + if (!str || !compiler_nameop(c, str, Store)) { + Py_XDECREF(str); + compiler_exit_scope(c); + return 0; + } + Py_DECREF(str); + /* compile the body proper */ + if (!compiler_body(c, s->v.ClassDef.body)) { + compiler_exit_scope(c); + return 0; + } + /* return the (empty) __class__ cell */ + str = PyUnicode_InternFromString("__class__"); + if (str == NULL) { + compiler_exit_scope(c); + return 0; + } + i = compiler_lookup_arg(c->u->u_cellvars, str); + Py_DECREF(str); + if (i == -1) { + /* This happens when nobody references the cell */ + PyErr_Clear(); + /* Return None */ + ADDOP_O(c, LOAD_CONST, Py_None, consts); + } + else { + /* Return the cell where to store __class__ */ + ADDOP_I(c, LOAD_CLOSURE, i); + } + ADDOP_IN_SCOPE(c, RETURN_VALUE); + /* create the code object */ + co = assemble(c, 1); } - - ADDOP_IN_SCOPE(c, LOAD_LOCALS); - ADDOP_IN_SCOPE(c, RETURN_VALUE); - co = assemble(c, 1); + /* leave the new scope */ compiler_exit_scope(c); if (co == NULL) return 0; + /* 2. load the 'build_class' function */ + ADDOP(c, LOAD_BUILD_CLASS); + + /* 3. load a function (or closure) made from the code object */ compiler_make_closure(c, co, 0); Py_DECREF(co); - ADDOP_I(c, CALL_FUNCTION, 0); - ADDOP(c, BUILD_CLASS); - /* apply decorators */ + /* 4. load class name */ + ADDOP_O(c, LOAD_CONST, s->v.ClassDef.name, consts); + + /* 5. generate the rest of the code for the call */ + if (!compiler_call_helper(c, 2, + s->v.ClassDef.bases, + s->v.ClassDef.keywords, + s->v.ClassDef.starargs, + s->v.ClassDef.kwargs)) + return 0; + + /* 6. apply decorators */ for (i = 0; i < asdl_seq_LEN(decos); i++) { ADDOP_I(c, CALL_FUNCTION, 1); } + + /* 7. store into <name> */ if (!compiler_nameop(c, s->v.ClassDef.name, Store)) return 0; return 1; @@ -1524,29 +1665,34 @@ compiler_lambda(struct compiler *c, expr_ty e) { PyCodeObject *co; static identifier name; + int kw_default_count = 0, arglength; arguments_ty args = e->v.Lambda.args; assert(e->kind == Lambda_kind); if (!name) { - name = PyString_InternFromString("<lambda>"); + name = PyUnicode_InternFromString("<lambda>"); if (!name) return 0; } + if (args->kwonlyargs) { + int res = compiler_visit_kwonlydefaults(c, args->kwonlyargs, + args->kw_defaults); + if (res < 0) return 0; + kw_default_count = res; + } if (args->defaults) VISIT_SEQ(c, expr, args->defaults); if (!compiler_enter_scope(c, name, (void *)e, e->lineno)) return 0; - /* unpack nested arguments */ - compiler_arguments(c, args); - /* Make None the first constant, so the lambda can't have a docstring. */ if (compiler_add_o(c, c->u->u_consts, Py_None) < 0) return 0; c->u->u_argcount = asdl_seq_LEN(args->args); + c->u->u_kwonlyargcount = asdl_seq_LEN(args->kwonlyargs); VISIT_IN_SCOPE(c, expr, e->v.Lambda.body); if (c->u->u_ste->ste_generator) { ADDOP_IN_SCOPE(c, POP_TOP); @@ -1559,50 +1705,15 @@ compiler_lambda(struct compiler *c, expr_ty e) if (co == NULL) return 0; - compiler_make_closure(c, co, asdl_seq_LEN(args->defaults)); + arglength = asdl_seq_LEN(args->defaults); + arglength |= kw_default_count << 8; + compiler_make_closure(c, co, arglength); Py_DECREF(co); return 1; } static int -compiler_print(struct compiler *c, stmt_ty s) -{ - int i, n; - bool dest; - - assert(s->kind == Print_kind); - n = asdl_seq_LEN(s->v.Print.values); - dest = false; - if (s->v.Print.dest) { - VISIT(c, expr, s->v.Print.dest); - dest = true; - } - for (i = 0; i < n; i++) { - expr_ty e = (expr_ty)asdl_seq_GET(s->v.Print.values, i); - if (dest) { - ADDOP(c, DUP_TOP); - VISIT(c, expr, e); - ADDOP(c, ROT_TWO); - ADDOP(c, PRINT_ITEM_TO); - } - else { - VISIT(c, expr, e); - ADDOP(c, PRINT_ITEM); - } - } - if (s->v.Print.nl) { - if (dest) - ADDOP(c, PRINT_NEWLINE_TO) - else - ADDOP(c, PRINT_NEWLINE) - } - else if (dest) - ADDOP(c, POP_TOP); - return 1; -} - -static int compiler_if(struct compiler *c, stmt_ty s) { basicblock *end, *next; @@ -1612,7 +1723,7 @@ compiler_if(struct compiler *c, stmt_ty s) if (end == NULL) return 0; - constant = expr_constant(s->v.If.test); + constant = expr_constant(c, s->v.If.test); /* constant = 0: "if 0" * constant = 1: "if 1", "if 2", ... * constant = -1: rest */ @@ -1674,7 +1785,7 @@ static int compiler_while(struct compiler *c, stmt_ty s) { basicblock *loop, *orelse, *end, *anchor = NULL; - int constant = expr_constant(s->v.While.test); + int constant = expr_constant(c, s->v.While.test); if (constant == 0) { if (s->v.While.orelse) @@ -1821,7 +1932,7 @@ compiler_try_finally(struct compiler *c, stmt_ty s) } /* - Code generated for "try: S except E1, V1: S1 except E2, V2: S2 ...": + Code generated for "try: S except E1 as V1: S1 except E2 as V2: S2 ...": (The contents of the value stack is shown in [], with the top at the right; 'tb' is trace-back info, 'val' the exception's associated value, and 'exc' the exception.) @@ -1875,11 +1986,12 @@ compiler_try_except(struct compiler *c, stmt_ty s) compiler_use_next_block(c, except); for (i = 0; i < n; i++) { excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET( - s->v.TryExcept.handlers, i); + s->v.TryExcept.handlers, i); if (!handler->v.ExceptHandler.type && i < n-1) return compiler_error(c, "default 'except:' must be last"); - c->u->u_lineno_set = false; + c->u->u_lineno_set = 0; c->u->u_lineno = handler->lineno; + c->u->u_col_offset = handler->col_offset; except = compiler_new_block(c); if (except == NULL) return 0; @@ -1891,13 +2003,71 @@ compiler_try_except(struct compiler *c, stmt_ty s) } ADDOP(c, POP_TOP); if (handler->v.ExceptHandler.name) { - VISIT(c, expr, handler->v.ExceptHandler.name); + basicblock *cleanup_end, *cleanup_body; + + cleanup_end = compiler_new_block(c); + cleanup_body = compiler_new_block(c); + if (!(cleanup_end || cleanup_body)) + return 0; + + compiler_nameop(c, handler->v.ExceptHandler.name, Store); + ADDOP(c, POP_TOP); + + /* + try: + # body + except type as name: + try: + # body + finally: + name = None + del name + */ + + /* second try: */ + ADDOP_JREL(c, SETUP_FINALLY, cleanup_end); + compiler_use_next_block(c, cleanup_body); + if (!compiler_push_fblock(c, FINALLY_TRY, cleanup_body)) + return 0; + + /* second # body */ + VISIT_SEQ(c, stmt, handler->v.ExceptHandler.body); + ADDOP(c, POP_BLOCK); + ADDOP(c, POP_EXCEPT); + compiler_pop_fblock(c, FINALLY_TRY, cleanup_body); + + /* finally: */ + ADDOP_O(c, LOAD_CONST, Py_None, consts); + compiler_use_next_block(c, cleanup_end); + if (!compiler_push_fblock(c, FINALLY_END, cleanup_end)) + return 0; + + /* name = None */ + ADDOP_O(c, LOAD_CONST, Py_None, consts); + compiler_nameop(c, handler->v.ExceptHandler.name, Store); + + /* del name */ + compiler_nameop(c, handler->v.ExceptHandler.name, Del); + + ADDOP(c, END_FINALLY); + compiler_pop_fblock(c, FINALLY_END, cleanup_end); } else { + basicblock *cleanup_body; + + cleanup_body = compiler_new_block(c); + if (!cleanup_body) + return 0; + + ADDOP(c, POP_TOP); ADDOP(c, POP_TOP); + compiler_use_next_block(c, cleanup_body); + if (!compiler_push_fblock(c, FINALLY_TRY, cleanup_body)) + return 0; + VISIT_SEQ(c, stmt, handler->v.ExceptHandler.body); + ADDOP(c, POP_EXCEPT); + compiler_pop_fblock(c, FINALLY_TRY, cleanup_body); } - ADDOP(c, POP_TOP); - VISIT_SEQ(c, stmt, handler->v.ExceptHandler.body); ADDOP_JREL(c, JUMP_FORWARD, end); compiler_use_next_block(c, except); } @@ -1917,17 +2087,17 @@ compiler_import_as(struct compiler *c, identifier name, identifier asname) If there is a dot in name, we need to split it and emit a LOAD_ATTR for each name. */ - const char *src = PyString_AS_STRING(name); - const char *dot = strchr(src, '.'); + const Py_UNICODE *src = PyUnicode_AS_UNICODE(name); + const Py_UNICODE *dot = Py_UNICODE_strchr(src, '.'); if (dot) { /* Consume the base module name to get the first attribute */ src = dot + 1; while (dot) { /* NB src is only defined when dot != NULL */ PyObject *attr; - dot = strchr(src, '.'); - attr = PyString_FromStringAndSize(src, - dot ? dot - src : strlen(src)); + dot = Py_UNICODE_strchr(src, '.'); + attr = PyUnicode_FromUnicode(src, + dot ? dot - src : Py_UNICODE_strlen(src)); if (!attr) return -1; ADDOP_O(c, LOAD_ATTR, attr, names); @@ -1955,11 +2125,7 @@ compiler_import(struct compiler *c, stmt_ty s) int r; PyObject *level; - if (c->c_flags && (c->c_flags->cf_flags & CO_FUTURE_ABSOLUTE_IMPORT)) - level = PyInt_FromLong(0); - else - level = PyInt_FromLong(-1); - + level = PyLong_FromLong(0); if (level == NULL) return 0; @@ -1975,11 +2141,11 @@ compiler_import(struct compiler *c, stmt_ty s) } else { identifier tmp = alias->name; - const char *base = PyString_AS_STRING(alias->name); - char *dot = strchr(base, '.'); + const Py_UNICODE *base = PyUnicode_AS_UNICODE(alias->name); + Py_UNICODE *dot = Py_UNICODE_strchr(base, '.'); if (dot) - tmp = PyString_FromStringAndSize(base, - dot - base); + tmp = PyUnicode_FromUnicode(base, + dot - base); r = compiler_nameop(c, tmp, Store); if (dot) { Py_DECREF(tmp); @@ -2001,7 +2167,7 @@ compiler_from_import(struct compiler *c, stmt_ty s) static PyObject *empty_string; if (!empty_string) { - empty_string = PyString_FromString(""); + empty_string = PyUnicode_FromString(""); if (!empty_string) return 0; } @@ -2009,12 +2175,7 @@ compiler_from_import(struct compiler *c, stmt_ty s) if (!names) return 0; - if (s->v.ImportFrom.level == 0 && c->c_flags && - !(c->c_flags->cf_flags & CO_FUTURE_ABSOLUTE_IMPORT)) - level = PyInt_FromLong(-1); - else - level = PyInt_FromLong(s->v.ImportFrom.level); - + level = PyLong_FromLong(s->v.ImportFrom.level); if (!level) { Py_DECREF(names); return 0; @@ -2028,7 +2189,7 @@ compiler_from_import(struct compiler *c, stmt_ty s) } if (s->lineno > c->c_future->ff_lineno && s->v.ImportFrom.module && - !strcmp(PyString_AS_STRING(s->v.ImportFrom.module), "__future__")) { + !PyUnicode_CompareWithASCIIString(s->v.ImportFrom.module, "__future__")) { Py_DECREF(level); Py_DECREF(names); return compiler_error(c, "from __future__ imports must occur " @@ -2049,7 +2210,7 @@ compiler_from_import(struct compiler *c, stmt_ty s) alias_ty alias = (alias_ty)asdl_seq_GET(s->v.ImportFrom.names, i); identifier store_name; - if (i == 0 && *PyString_AS_STRING(alias->name) == '*') { + if (i == 0 && *PyUnicode_AS_UNICODE(alias->name) == '*') { assert(n == 1); ADDOP(c, IMPORT_STAR); return 1; @@ -2076,10 +2237,10 @@ compiler_assert(struct compiler *c, stmt_ty s) static PyObject *assertion_error = NULL; basicblock *end; - if (Py_OptimizeFlag) + if (c->c_optimize) return 1; if (assertion_error == NULL) { - assertion_error = PyString_InternFromString("AssertionError"); + assertion_error = PyUnicode_InternFromString("AssertionError"); if (assertion_error == NULL) return 0; } @@ -2113,7 +2274,8 @@ compiler_visit_stmt(struct compiler *c, stmt_ty s) /* Always assign a lineno to the next instruction for a stmt. */ c->u->u_lineno = s->lineno; - c->u->u_lineno_set = false; + c->u->u_col_offset = s->col_offset; + c->u->u_lineno_set = 0; switch (s->kind) { case FunctionDef_kind: @@ -2145,8 +2307,6 @@ compiler_visit_stmt(struct compiler *c, stmt_ty s) break; case AugAssign_kind: return compiler_augassign(c, s); - case Print_kind: - return compiler_print(c, s); case For_kind: return compiler_for(c, s); case While_kind: @@ -2155,17 +2315,13 @@ compiler_visit_stmt(struct compiler *c, stmt_ty s) return compiler_if(c, s); case Raise_kind: n = 0; - if (s->v.Raise.type) { - VISIT(c, expr, s->v.Raise.type); + if (s->v.Raise.exc) { + VISIT(c, expr, s->v.Raise.exc); n++; - if (s->v.Raise.inst) { - VISIT(c, expr, s->v.Raise.inst); - n++; - if (s->v.Raise.tback) { - VISIT(c, expr, s->v.Raise.tback); - n++; - } - } + if (s->v.Raise.cause) { + VISIT(c, expr, s->v.Raise.cause); + n++; + } } ADDOP_I(c, RAISE_VARARGS, n); break; @@ -2179,22 +2335,8 @@ compiler_visit_stmt(struct compiler *c, stmt_ty s) return compiler_import(c, s); case ImportFrom_kind: return compiler_from_import(c, s); - case Exec_kind: - VISIT(c, expr, s->v.Exec.body); - if (s->v.Exec.globals) { - VISIT(c, expr, s->v.Exec.globals); - if (s->v.Exec.locals) { - VISIT(c, expr, s->v.Exec.locals); - } else { - ADDOP(c, DUP_TOP); - } - } else { - ADDOP_O(c, LOAD_CONST, Py_None, consts); - ADDOP(c, DUP_TOP); - } - ADDOP(c, EXEC_STMT); - break; case Global_kind: + case Nonlocal_kind: break; case Expr_kind: if (c->c_interactive && c->c_nestlevel <= 1) { @@ -2252,10 +2394,7 @@ binop(struct compiler *c, operator_ty op) case Mult: return BINARY_MULTIPLY; case Div: - if (c->c_flags && c->c_flags->cf_flags & CO_FUTURE_DIVISION) - return BINARY_TRUE_DIVIDE; - else - return BINARY_DIVIDE; + return BINARY_TRUE_DIVIDE; case Mod: return BINARY_MODULO; case Pow: @@ -2319,10 +2458,7 @@ inplace_binop(struct compiler *c, operator_ty op) case Mult: return INPLACE_MULTIPLY; case Div: - if (c->c_flags && c->c_flags->cf_flags & CO_FUTURE_DIVISION) - return INPLACE_TRUE_DIVIDE; - else - return INPLACE_DIVIDE; + return INPLACE_TRUE_DIVIDE; case Mod: return INPLACE_MODULO; case Pow: @@ -2390,7 +2526,7 @@ compiler_nameop(struct compiler *c, identifier name, expr_context_ty ctx) } /* XXX Leave assert here, but handle __doc__ and the like better */ - assert(scope || PyString_AS_STRING(name)[0] == '_'); + assert(scope || PyUnicode_AS_UNICODE(name)[0] == '_'); switch (optype) { case OP_DEREF: @@ -2400,13 +2536,7 @@ compiler_nameop(struct compiler *c, identifier name, expr_context_ty ctx) case AugLoad: case AugStore: break; - case Del: - PyErr_Format(PyExc_SyntaxError, - "can not delete variable '%s' referenced " - "in nested scope", - PyString_AS_STRING(name)); - Py_DECREF(mangled); - return 0; + case Del: op = DELETE_DEREF; break; case Param: default: PyErr_SetString(PyExc_SystemError, @@ -2503,7 +2633,26 @@ compiler_list(struct compiler *c, expr_ty e) { int n = asdl_seq_LEN(e->v.List.elts); if (e->v.List.ctx == Store) { - ADDOP_I(c, UNPACK_SEQUENCE, n); + int i, seen_star = 0; + for (i = 0; i < n; i++) { + expr_ty elt = asdl_seq_GET(e->v.List.elts, i); + if (elt->kind == Starred_kind && !seen_star) { + if ((i >= (1 << 8)) || + (n-i-1 >= (INT_MAX >> 8))) + return compiler_error(c, + "too many expressions in " + "star-unpacking assignment"); + ADDOP_I(c, UNPACK_EX, (i + ((n-i-1) << 8))); + seen_star = 1; + asdl_seq_SET(e->v.List.elts, i, elt->v.Starred.value); + } else if (elt->kind == Starred_kind) { + return compiler_error(c, + "two starred expressions in assignment"); + } + } + if (!seen_star) { + ADDOP_I(c, UNPACK_SEQUENCE, n); + } } VISIT_SEQ(c, expr, e->v.List.elts); if (e->v.List.ctx == Load) { @@ -2517,7 +2666,26 @@ compiler_tuple(struct compiler *c, expr_ty e) { int n = asdl_seq_LEN(e->v.Tuple.elts); if (e->v.Tuple.ctx == Store) { - ADDOP_I(c, UNPACK_SEQUENCE, n); + int i, seen_star = 0; + for (i = 0; i < n; i++) { + expr_ty elt = asdl_seq_GET(e->v.Tuple.elts, i); + if (elt->kind == Starred_kind && !seen_star) { + if ((i >= (1 << 8)) || + (n-i-1 >= (INT_MAX >> 8))) + return compiler_error(c, + "too many expressions in " + "star-unpacking assignment"); + ADDOP_I(c, UNPACK_EX, (i + ((n-i-1) << 8))); + seen_star = 1; + asdl_seq_SET(e->v.Tuple.elts, i, elt->v.Starred.value); + } else if (elt->kind == Starred_kind) { + return compiler_error(c, + "two starred expressions in assignment"); + } + } + if (!seen_star) { + ADDOP_I(c, UNPACK_SEQUENCE, n); + } } VISIT_SEQ(c, expr, e->v.Tuple.elts); if (e->v.Tuple.ctx == Load) { @@ -2574,21 +2742,37 @@ compiler_compare(struct compiler *c, expr_ty e) static int compiler_call(struct compiler *c, expr_ty e) { - int n, code = 0; - VISIT(c, expr, e->v.Call.func); - n = asdl_seq_LEN(e->v.Call.args); - VISIT_SEQ(c, expr, e->v.Call.args); - if (e->v.Call.keywords) { - VISIT_SEQ(c, keyword, e->v.Call.keywords); - n |= asdl_seq_LEN(e->v.Call.keywords) << 8; - } - if (e->v.Call.starargs) { - VISIT(c, expr, e->v.Call.starargs); + return compiler_call_helper(c, 0, + e->v.Call.args, + e->v.Call.keywords, + e->v.Call.starargs, + e->v.Call.kwargs); +} + +/* shared code between compiler_call and compiler_class */ +static int +compiler_call_helper(struct compiler *c, + int n, /* Args already pushed */ + asdl_seq *args, + asdl_seq *keywords, + expr_ty starargs, + expr_ty kwargs) +{ + int code = 0; + + n += asdl_seq_LEN(args); + VISIT_SEQ(c, expr, args); + if (keywords) { + VISIT_SEQ(c, keyword, keywords); + n |= asdl_seq_LEN(keywords) << 8; + } + if (starargs) { + VISIT(c, expr, starargs); code |= 1; } - if (e->v.Call.kwargs) { - VISIT(c, expr, e->v.Call.kwargs); + if (kwargs) { + VISIT(c, expr, kwargs); code |= 2; } switch (code) { @@ -2608,82 +2792,19 @@ compiler_call(struct compiler *c, expr_ty e) return 1; } -static int -compiler_listcomp_generator(struct compiler *c, asdl_seq *generators, - int gen_index, expr_ty elt) -{ - /* generate code for the iterator, then each of the ifs, - and then write to the element */ - - comprehension_ty l; - basicblock *start, *anchor, *skip, *if_cleanup; - int i, n; - - start = compiler_new_block(c); - skip = compiler_new_block(c); - if_cleanup = compiler_new_block(c); - anchor = compiler_new_block(c); - - if (start == NULL || skip == NULL || if_cleanup == NULL || - anchor == NULL) - return 0; - - l = (comprehension_ty)asdl_seq_GET(generators, gen_index); - VISIT(c, expr, l->iter); - ADDOP(c, GET_ITER); - compiler_use_next_block(c, start); - ADDOP_JREL(c, FOR_ITER, anchor); - NEXT_BLOCK(c); - VISIT(c, expr, l->target); - - /* XXX this needs to be cleaned up...a lot! */ - n = asdl_seq_LEN(l->ifs); - for (i = 0; i < n; i++) { - expr_ty e = (expr_ty)asdl_seq_GET(l->ifs, i); - VISIT(c, expr, e); - ADDOP_JABS(c, POP_JUMP_IF_FALSE, if_cleanup); - NEXT_BLOCK(c); - } - - if (++gen_index < asdl_seq_LEN(generators)) - if (!compiler_listcomp_generator(c, generators, gen_index, elt)) - return 0; - - /* only append after the last for generator */ - if (gen_index >= asdl_seq_LEN(generators)) { - VISIT(c, expr, elt); - ADDOP_I(c, LIST_APPEND, gen_index+1); - - compiler_use_next_block(c, skip); - } - compiler_use_next_block(c, if_cleanup); - ADDOP_JABS(c, JUMP_ABSOLUTE, start); - compiler_use_next_block(c, anchor); - return 1; -} +/* List and set comprehensions and generator expressions work by creating a + nested function to perform the actual iteration. This means that the + iteration variables don't leak into the current scope. + The defined function is called immediately following its definition, with the + result of that call being the result of the expression. + The LC/SC version returns the populated container, while the GE version is + flagged in symtable.c as a generator, so it returns the generator object + when the function is called. + This code *knows* that the loop cannot contain break, continue, or return, + so it cheats and skips the SETUP_LOOP/POP_BLOCK steps used in normal loops. -static int -compiler_listcomp(struct compiler *c, expr_ty e) -{ - assert(e->kind == ListComp_kind); - ADDOP_I(c, BUILD_LIST, 0); - return compiler_listcomp_generator(c, e->v.ListComp.generators, 0, - e->v.ListComp.elt); -} - -/* Dict and set comprehensions and generator expressions work by creating a - nested function to perform the actual iteration. This means that the - iteration variables don't leak into the current scope. - The defined function is called immediately following its definition, with the - result of that call being the result of the expression. - The LC/SC version returns the populated container, while the GE version is - flagged in symtable.c as a generator, so it returns the generator object - when the function is called. - This code *knows* that the loop cannot contain break, continue, or return, - so it cheats and skips the SETUP_LOOP/POP_BLOCK steps used in normal loops. - - Possible cleanups: + Possible cleanups: - iterate over the generator sequence instead of using recursion */ @@ -2749,6 +2870,10 @@ compiler_comprehension_generator(struct compiler *c, ADDOP(c, YIELD_VALUE); ADDOP(c, POP_TOP); break; + case COMP_LISTCOMP: + VISIT(c, expr, elt); + ADDOP_I(c, LIST_APPEND, gen_index + 1); + break; case COMP_SETCOMP: VISIT(c, expr, elt); ADDOP_I(c, SET_ADD, gen_index + 1); @@ -2789,6 +2914,9 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, identifier name, if (type != COMP_GENEXP) { int op; switch (type) { + case COMP_LISTCOMP: + op = BUILD_LIST; + break; case COMP_SETCOMP: op = BUILD_SET; break; @@ -2837,7 +2965,7 @@ compiler_genexp(struct compiler *c, expr_ty e) { static identifier name; if (!name) { - name = PyString_FromString("<genexpr>"); + name = PyUnicode_FromString("<genexpr>"); if (!name) return 0; } @@ -2848,11 +2976,26 @@ compiler_genexp(struct compiler *c, expr_ty e) } static int +compiler_listcomp(struct compiler *c, expr_ty e) +{ + static identifier name; + if (!name) { + name = PyUnicode_FromString("<listcomp>"); + if (!name) + return 0; + } + assert(e->kind == ListComp_kind); + return compiler_comprehension(c, e, COMP_LISTCOMP, name, + e->v.ListComp.generators, + e->v.ListComp.elt, NULL); +} + +static int compiler_setcomp(struct compiler *c, expr_ty e) { static identifier name; if (!name) { - name = PyString_FromString("<setcomp>"); + name = PyUnicode_FromString("<setcomp>"); if (!name) return 0; } @@ -2862,12 +3005,13 @@ compiler_setcomp(struct compiler *c, expr_ty e) e->v.SetComp.elt, NULL); } + static int compiler_dictcomp(struct compiler *c, expr_ty e) { static identifier name; if (!name) { - name = PyString_FromString("<dictcomp>"); + name = PyUnicode_FromString("<dictcomp>"); if (!name) return 0; } @@ -2877,6 +3021,7 @@ compiler_dictcomp(struct compiler *c, expr_ty e) e->v.DictComp.key, e->v.DictComp.value); } + static int compiler_visit_keyword(struct compiler *c, keyword_ty k) { @@ -2892,19 +3037,25 @@ compiler_visit_keyword(struct compiler *c, keyword_ty k) */ static int -expr_constant(expr_ty e) +expr_constant(struct compiler *c, expr_ty e) { + char *id; switch (e->kind) { + case Ellipsis_kind: + return 1; case Num_kind: return PyObject_IsTrue(e->v.Num.n); case Str_kind: return PyObject_IsTrue(e->v.Str.s); case Name_kind: - /* __debug__ is not assignable, so we can optimize - * it away in if and while statements */ - if (strcmp(PyString_AS_STRING(e->v.Name.id), - "__debug__") == 0) - return ! Py_OptimizeFlag; + /* optimize away names that can't be reassigned */ + id = PyBytes_AS_STRING( + _PyUnicode_AsDefaultEncodedString(e->v.Name.id, NULL)); + if (strcmp(id, "True") == 0) return 1; + if (strcmp(id, "False") == 0) return 0; + if (strcmp(id, "None") == 0) return 0; + if (strcmp(id, "__debug__") == 0) + return ! c->c_optimize; /* fall through */ default: return -1; @@ -2952,9 +3103,6 @@ compiler_with(struct compiler *c, stmt_ty s) /* SETUP_WITH pushes a finally block. */ compiler_use_next_block(c, block); - /* Note that the block is actually called SETUP_WITH in ceval.c, but - functions the same as SETUP_FINALLY except that exceptions are - normalized. */ if (!compiler_push_fblock(c, FINALLY_TRY, block)) { return 0; } @@ -3000,8 +3148,10 @@ compiler_visit_expr(struct compiler *c, expr_ty e) */ if (e->lineno > c->u->u_lineno) { c->u->u_lineno = e->lineno; - c->u->u_lineno_set = false; + c->u->u_lineno_set = 0; } + /* Updating the column offset is always harmless. */ + c->u->u_col_offset = e->col_offset; switch (e->kind) { case BoolOp_kind: return compiler_boolop(c, e); @@ -3034,14 +3184,14 @@ compiler_visit_expr(struct compiler *c, expr_ty e) VISIT_SEQ(c, expr, e->v.Set.elts); ADDOP_I(c, BUILD_SET, n); break; + case GeneratorExp_kind: + return compiler_genexp(c, e); case ListComp_kind: return compiler_listcomp(c, e); case SetComp_kind: return compiler_setcomp(c, e); case DictComp_kind: return compiler_dictcomp(c, e); - case GeneratorExp_kind: - return compiler_genexp(c, e); case Yield_kind: if (c->u->u_ste->ste_type != FunctionBlock) return compiler_error(c, "'yield' outside function"); @@ -3057,16 +3207,18 @@ compiler_visit_expr(struct compiler *c, expr_ty e) return compiler_compare(c, e); case Call_kind: return compiler_call(c, e); - case Repr_kind: - VISIT(c, expr, e->v.Repr.value); - ADDOP(c, UNARY_CONVERT); - break; case Num_kind: ADDOP_O(c, LOAD_CONST, e->v.Num.n, consts); break; case Str_kind: ADDOP_O(c, LOAD_CONST, e->v.Str.s, consts); break; + case Bytes_kind: + ADDOP_O(c, LOAD_CONST, e->v.Bytes.s, consts); + break; + case Ellipsis_kind: + ADDOP_O(c, LOAD_CONST, Py_Ellipsis, consts); + break; /* The following exprs can be assignment targets. */ case Attribute_kind: if (e->v.Attribute.ctx != AugStore) @@ -3122,6 +3274,18 @@ compiler_visit_expr(struct compiler *c, expr_ty e) return 0; } break; + case Starred_kind: + switch (e->v.Starred.ctx) { + case Store: + /* In all legitimate cases, the Starred node was already replaced + * by compiler_list/compiler_tuple. XXX: is that okay? */ + return compiler_error(c, + "starred assignment target must be in a list or tuple"); + default: + return compiler_error(c, + "can use starred expression only as assignment target"); + } + break; case Name_kind: return compiler_nameop(c, e->v.Name.id, e->v.Name.ctx); /* child nodes of List and Tuple will have expr_context set */ @@ -3221,7 +3385,7 @@ compiler_in_loop(struct compiler *c) { static int compiler_error(struct compiler *c, const char *errstr) { - PyObject *loc; + PyObject *loc, *filename; PyObject *u = NULL, *v = NULL; loc = PyErr_ProgramText(c->c_filename, c->u->u_lineno); @@ -3229,8 +3393,17 @@ compiler_error(struct compiler *c, const char *errstr) Py_INCREF(Py_None); loc = Py_None; } - u = Py_BuildValue("(ziOO)", c->c_filename, c->u->u_lineno, - Py_None, loc); + if (c->c_filename != NULL) { + filename = PyUnicode_DecodeFSDefault(c->c_filename); + if (!filename) + goto exit; + } + else { + Py_INCREF(Py_None); + filename = Py_None; + } + u = Py_BuildValue("(NiiO)", filename, c->u->u_lineno, + c->u->u_col_offset, loc); if (!u) goto exit; v = Py_BuildValue("(zO)", errstr, u); @@ -3264,7 +3437,7 @@ compiler_handle_subscr(struct compiler *c, const char *kind, return 0; } if (ctx == AugLoad) { - ADDOP_I(c, DUP_TOPX, 2); + ADDOP(c, DUP_TOP_TWO); } else if (ctx == AugStore) { ADDOP(c, ROT_THREE); @@ -3303,64 +3476,10 @@ compiler_slice(struct compiler *c, slice_ty s, expr_context_ty ctx) } static int -compiler_simple_slice(struct compiler *c, slice_ty s, expr_context_ty ctx) -{ - int op = 0, slice_offset = 0, stack_count = 0; - - assert(s->v.Slice.step == NULL); - if (s->v.Slice.lower) { - slice_offset++; - stack_count++; - if (ctx != AugStore) - VISIT(c, expr, s->v.Slice.lower); - } - if (s->v.Slice.upper) { - slice_offset += 2; - stack_count++; - if (ctx != AugStore) - VISIT(c, expr, s->v.Slice.upper); - } - - if (ctx == AugLoad) { - switch (stack_count) { - case 0: ADDOP(c, DUP_TOP); break; - case 1: ADDOP_I(c, DUP_TOPX, 2); break; - case 2: ADDOP_I(c, DUP_TOPX, 3); break; - } - } - else if (ctx == AugStore) { - switch (stack_count) { - case 0: ADDOP(c, ROT_TWO); break; - case 1: ADDOP(c, ROT_THREE); break; - case 2: ADDOP(c, ROT_FOUR); break; - } - } - - switch (ctx) { - case AugLoad: /* fall through to Load */ - case Load: op = SLICE; break; - case AugStore:/* fall through to Store */ - case Store: op = STORE_SLICE; break; - case Del: op = DELETE_SLICE; break; - case Param: - default: - PyErr_SetString(PyExc_SystemError, - "param invalid in simple slice"); - return 0; - } - - ADDOP(c, op + slice_offset); - return 1; -} - -static int compiler_visit_nested_slice(struct compiler *c, slice_ty s, expr_context_ty ctx) { switch (s->kind) { - case Ellipsis_kind: - ADDOP_O(c, LOAD_CONST, Py_Ellipsis, consts); - break; case Slice_kind: return compiler_slice(c, s, ctx); case Index_kind: @@ -3386,16 +3505,8 @@ compiler_visit_slice(struct compiler *c, slice_ty s, expr_context_ty ctx) VISIT(c, expr, s->v.Index.value); } break; - case Ellipsis_kind: - kindname = "ellipsis"; - if (ctx != AugStore) { - ADDOP_O(c, LOAD_CONST, Py_Ellipsis, consts); - } - break; case Slice_kind: kindname = "slice"; - if (!s->v.Slice.step) - return compiler_simple_slice(c, s, ctx); if (ctx != AugStore) { if (!compiler_slice(c, s, ctx)) return 0; @@ -3422,7 +3533,6 @@ compiler_visit_slice(struct compiler *c, slice_ty s, expr_context_ty ctx) return compiler_handle_subscr(c, kindname, ctx); } - /* End of the compiler section, beginning of the assembler section */ /* do depth-first search of basic block graph, starting with block. @@ -3524,10 +3634,10 @@ assemble_init(struct assembler *a, int nblocks, int firstlineno) { memset(a, 0, sizeof(struct assembler)); a->a_lineno = firstlineno; - a->a_bytecode = PyString_FromStringAndSize(NULL, DEFAULT_CODE_SIZE); + a->a_bytecode = PyBytes_FromStringAndSize(NULL, DEFAULT_CODE_SIZE); if (!a->a_bytecode) return 0; - a->a_lnotab = PyString_FromStringAndSize(NULL, DEFAULT_LNOTAB_SIZE); + a->a_lnotab = PyBytes_FromStringAndSize(NULL, DEFAULT_LNOTAB_SIZE); if (!a->a_lnotab) return 0; if (nblocks > PY_SIZE_MAX / sizeof(basicblock *)) { @@ -3598,7 +3708,7 @@ assemble_lnotab(struct assembler *a, struct instr *i) if (d_bytecode > 255) { int j, nbytes, ncodes = d_bytecode / 255; nbytes = a->a_lnotab_off + 2 * ncodes; - len = PyString_GET_SIZE(a->a_lnotab); + len = PyBytes_GET_SIZE(a->a_lnotab); if (nbytes >= len) { if ((len <= INT_MAX / 2) && (len * 2 < nbytes)) len = nbytes; @@ -3608,11 +3718,11 @@ assemble_lnotab(struct assembler *a, struct instr *i) PyErr_NoMemory(); return 0; } - if (_PyString_Resize(&a->a_lnotab, len) < 0) + if (_PyBytes_Resize(&a->a_lnotab, len) < 0) return 0; } lnotab = (unsigned char *) - PyString_AS_STRING(a->a_lnotab) + a->a_lnotab_off; + PyBytes_AS_STRING(a->a_lnotab) + a->a_lnotab_off; for (j = 0; j < ncodes; j++) { *lnotab++ = 255; *lnotab++ = 0; @@ -3624,7 +3734,7 @@ assemble_lnotab(struct assembler *a, struct instr *i) if (d_lineno > 255) { int j, nbytes, ncodes = d_lineno / 255; nbytes = a->a_lnotab_off + 2 * ncodes; - len = PyString_GET_SIZE(a->a_lnotab); + len = PyBytes_GET_SIZE(a->a_lnotab); if (nbytes >= len) { if ((len <= INT_MAX / 2) && len * 2 < nbytes) len = nbytes; @@ -3634,11 +3744,11 @@ assemble_lnotab(struct assembler *a, struct instr *i) PyErr_NoMemory(); return 0; } - if (_PyString_Resize(&a->a_lnotab, len) < 0) + if (_PyBytes_Resize(&a->a_lnotab, len) < 0) return 0; } lnotab = (unsigned char *) - PyString_AS_STRING(a->a_lnotab) + a->a_lnotab_off; + PyBytes_AS_STRING(a->a_lnotab) + a->a_lnotab_off; *lnotab++ = d_bytecode; *lnotab++ = 255; d_bytecode = 0; @@ -3650,13 +3760,13 @@ assemble_lnotab(struct assembler *a, struct instr *i) a->a_lnotab_off += ncodes * 2; } - len = PyString_GET_SIZE(a->a_lnotab); + len = PyBytes_GET_SIZE(a->a_lnotab); if (a->a_lnotab_off + 2 >= len) { - if (_PyString_Resize(&a->a_lnotab, len * 2) < 0) + if (_PyBytes_Resize(&a->a_lnotab, len * 2) < 0) return 0; } lnotab = (unsigned char *) - PyString_AS_STRING(a->a_lnotab) + a->a_lnotab_off; + PyBytes_AS_STRING(a->a_lnotab) + a->a_lnotab_off; a->a_lnotab_off += 2; if (d_bytecode) { @@ -3681,7 +3791,7 @@ static int assemble_emit(struct assembler *a, struct instr *i) { int size, arg = 0, ext = 0; - Py_ssize_t len = PyString_GET_SIZE(a->a_bytecode); + Py_ssize_t len = PyBytes_GET_SIZE(a->a_bytecode); char *code; size = instrsize(i); @@ -3694,10 +3804,10 @@ assemble_emit(struct assembler *a, struct instr *i) if (a->a_offset + size >= len) { if (len > PY_SSIZE_T_MAX / 2) return 0; - if (_PyString_Resize(&a->a_bytecode, len * 2) < 0) + if (_PyBytes_Resize(&a->a_bytecode, len * 2) < 0) return 0; } - code = PyString_AS_STRING(a->a_bytecode) + a->a_offset; + code = PyBytes_AS_STRING(a->a_bytecode) + a->a_offset; a->a_offset += size; if (size == 6) { assert(i->i_hasarg); @@ -3783,7 +3893,7 @@ dict_keys_inorder(PyObject *dict, int offset) if (tuple == NULL) return NULL; while (PyDict_Next(dict, &pos, &k, &v)) { - i = PyInt_AS_LONG(v); + i = PyLong_AS_LONG(v); /* The keys of the dictionary are tuples. (see compiler_add_o) The object we want is always first, though. */ k = PyTuple_GET_ITEM(k, 0); @@ -3865,7 +3975,7 @@ makecode(struct compiler *c, struct assembler *a) freevars = dict_keys_inorder(c->u->u_freevars, PyTuple_Size(cellvars)); if (!freevars) goto error; - filename = PyString_FromString(c->c_filename); + filename = PyUnicode_DecodeFSDefault(c->c_filename); if (!filename) goto error; @@ -3884,7 +3994,8 @@ makecode(struct compiler *c, struct assembler *a) Py_DECREF(consts); consts = tmp; - co = PyCode_New(c->u->u_argcount, nlocals, stackdepth(c), flags, + co = PyCode_New(c->u->u_argcount, c->u->u_kwonlyargcount, + nlocals, stackdepth(c), flags, bytecode, consts, names, varnames, freevars, cellvars, filename, c->u->u_name, @@ -3985,9 +4096,9 @@ assemble(struct compiler *c, int addNone) goto error; } - if (_PyString_Resize(&a.a_lnotab, a.a_lnotab_off) < 0) + if (_PyBytes_Resize(&a.a_lnotab, a.a_lnotab_off) < 0) goto error; - if (_PyString_Resize(&a.a_bytecode, a.a_offset) < 0) + if (_PyBytes_Resize(&a.a_bytecode, a.a_offset) < 0) goto error; co = makecode(c, &a); @@ -3995,3 +4106,13 @@ assemble(struct compiler *c, int addNone) assemble_free(&a); return co; } + +#undef PyAST_Compile +PyAPI_FUNC(PyCodeObject *) +PyAST_Compile(mod_ty mod, const char *filename, PyCompilerFlags *flags, + PyArena *arena) +{ + return PyAST_CompileEx(mod, filename, flags, -1, arena); +} + + diff --git a/Python/dynamic_annotations.c b/Python/dynamic_annotations.c new file mode 100644 index 0000000000..10511da466 --- /dev/null +++ b/Python/dynamic_annotations.c @@ -0,0 +1,154 @@ +/* Copyright (c) 2008-2009, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * --- + * Author: Kostya Serebryany + */ + +#ifdef _MSC_VER +# include <windows.h> +#endif + +#ifdef __cplusplus +# error "This file should be built as pure C to avoid name mangling" +#endif + +#include <stdlib.h> +#include <string.h> + +#include "dynamic_annotations.h" + +/* Each function is empty and called (via a macro) only in debug mode. + The arguments are captured by dynamic tools at runtime. */ + +#if DYNAMIC_ANNOTATIONS_ENABLED == 1 + +void AnnotateRWLockCreate(const char *file, int line, + const volatile void *lock){} +void AnnotateRWLockDestroy(const char *file, int line, + const volatile void *lock){} +void AnnotateRWLockAcquired(const char *file, int line, + const volatile void *lock, long is_w){} +void AnnotateRWLockReleased(const char *file, int line, + const volatile void *lock, long is_w){} +void AnnotateBarrierInit(const char *file, int line, + const volatile void *barrier, long count, + long reinitialization_allowed) {} +void AnnotateBarrierWaitBefore(const char *file, int line, + const volatile void *barrier) {} +void AnnotateBarrierWaitAfter(const char *file, int line, + const volatile void *barrier) {} +void AnnotateBarrierDestroy(const char *file, int line, + const volatile void *barrier) {} + +void AnnotateCondVarWait(const char *file, int line, + const volatile void *cv, + const volatile void *lock){} +void AnnotateCondVarSignal(const char *file, int line, + const volatile void *cv){} +void AnnotateCondVarSignalAll(const char *file, int line, + const volatile void *cv){} +void AnnotatePublishMemoryRange(const char *file, int line, + const volatile void *address, + long size){} +void AnnotateUnpublishMemoryRange(const char *file, int line, + const volatile void *address, + long size){} +void AnnotatePCQCreate(const char *file, int line, + const volatile void *pcq){} +void AnnotatePCQDestroy(const char *file, int line, + const volatile void *pcq){} +void AnnotatePCQPut(const char *file, int line, + const volatile void *pcq){} +void AnnotatePCQGet(const char *file, int line, + const volatile void *pcq){} +void AnnotateNewMemory(const char *file, int line, + const volatile void *mem, + long size){} +void AnnotateExpectRace(const char *file, int line, + const volatile void *mem, + const char *description){} +void AnnotateBenignRace(const char *file, int line, + const volatile void *mem, + const char *description){} +void AnnotateBenignRaceSized(const char *file, int line, + const volatile void *mem, + long size, + const char *description) {} +void AnnotateMutexIsUsedAsCondVar(const char *file, int line, + const volatile void *mu){} +void AnnotateTraceMemory(const char *file, int line, + const volatile void *arg){} +void AnnotateThreadName(const char *file, int line, + const char *name){} +void AnnotateIgnoreReadsBegin(const char *file, int line){} +void AnnotateIgnoreReadsEnd(const char *file, int line){} +void AnnotateIgnoreWritesBegin(const char *file, int line){} +void AnnotateIgnoreWritesEnd(const char *file, int line){} +void AnnotateIgnoreSyncBegin(const char *file, int line){} +void AnnotateIgnoreSyncEnd(const char *file, int line){} +void AnnotateEnableRaceDetection(const char *file, int line, int enable){} +void AnnotateNoOp(const char *file, int line, + const volatile void *arg){} +void AnnotateFlushState(const char *file, int line){} + +static int GetRunningOnValgrind(void) { +#ifdef RUNNING_ON_VALGRIND + if (RUNNING_ON_VALGRIND) return 1; +#endif + +#ifndef _MSC_VER + char *running_on_valgrind_str = getenv("RUNNING_ON_VALGRIND"); + if (running_on_valgrind_str) { + return strcmp(running_on_valgrind_str, "0") != 0; + } +#else + /* Visual Studio issues warnings if we use getenv, + * so we use GetEnvironmentVariableA instead. + */ + char value[100] = "1"; + int res = GetEnvironmentVariableA("RUNNING_ON_VALGRIND", + value, sizeof(value)); + /* value will remain "1" if res == 0 or res >= sizeof(value). The latter + * can happen only if the given value is long, in this case it can't be "0". + */ + if (res > 0 && !strcmp(value, "0")) + return 1; +#endif + return 0; +} + +/* See the comments in dynamic_annotations.h */ +int RunningOnValgrind(void) { + static volatile int running_on_valgrind = -1; + /* C doesn't have thread-safe initialization of statics, and we + don't want to depend on pthread_once here, so hack it. */ + int local_running_on_valgrind = running_on_valgrind; + if (local_running_on_valgrind == -1) + running_on_valgrind = local_running_on_valgrind = GetRunningOnValgrind(); + return local_running_on_valgrind; +} + +#endif /* DYNAMIC_ANNOTATIONS_ENABLED == 1 */ diff --git a/Python/dynload_atheos.c b/Python/dynload_atheos.c deleted file mode 100644 index 65e4136380..0000000000 --- a/Python/dynload_atheos.c +++ /dev/null @@ -1,47 +0,0 @@ - -/* Support for dynamic loading of extension modules */ - -#include <atheos/image.h> -#include <errno.h> - -#include "Python.h" -#include "importdl.h" - - -const struct filedescr _PyImport_DynLoadFiletab[] = { - {".so", "rb", C_EXTENSION}, - {"module.so", "rb", C_EXTENSION}, - {0, 0} -}; - -dl_funcptr _PyImport_GetDynLoadFunc(const char *fqname, const char *shortname, - const char *pathname, FILE *fp) -{ - void *p; - int lib; - char funcname[258]; - - if (Py_VerboseFlag) - printf("load_library %s\n", pathname); - - lib = load_library(pathname, 0); - if (lib < 0) { - char buf[512]; - if (Py_VerboseFlag) - perror(pathname); - PyOS_snprintf(buf, sizeof(buf), "Failed to load %.200s: %.200s", - pathname, strerror(errno)); - PyErr_SetString(PyExc_ImportError, buf); - return NULL; - } - PyOS_snprintf(funcname, sizeof(funcname), "init%.200s", shortname); - if (Py_VerboseFlag) - printf("get_symbol_address %s\n", funcname); - if (get_symbol_address(lib, funcname, -1, &p) < 0) { - p = NULL; - if (Py_VerboseFlag) - perror(funcname); - } - - return (dl_funcptr) p; -} diff --git a/Python/dynload_beos.c b/Python/dynload_beos.c deleted file mode 100644 index f5ca1ec3aa..0000000000 --- a/Python/dynload_beos.c +++ /dev/null @@ -1,254 +0,0 @@ - -/* Support for dynamic loading of extension modules */ - -#include <kernel/image.h> -#include <kernel/OS.h> -#include <stdlib.h> - -#include "Python.h" -#include "importdl.h" - -const struct filedescr _PyImport_DynLoadFiletab[] = { - {".so", "rb", C_EXTENSION}, - {"module.so", "rb", C_EXTENSION}, - {0, 0} -}; - -#if defined(MAXPATHLEN) && !defined(_SYS_PARAM_H) -#undef MAXPATHLEN -#endif - -#ifdef WITH_THREAD -#include "pythread.h" -static PyThread_type_lock beos_dyn_lock; -#endif - -static PyObject *beos_dyn_images = NULL; - -/* ---------------------------------------------------------------------- - * BeOS dynamic loading support - * - * This uses shared libraries, but BeOS has its own way of doing things - * (much easier than dlfnc.h, from the look of things). We'll use a - * Python Dictionary object to store the images_ids so we can be very - * nice and unload them when we exit. - * - * Note that this is thread-safe. Probably irrelevent, because of losing - * systems... Python probably disables threads while loading modules. - * Note the use of "probably"! Better to be safe than sorry. [chrish] - * - * As of 1.5.1 this should also work properly when you've configured - * Python without thread support; the 1.5 version required it, which wasn't - * very friendly. Note that I haven't tested it without threading... why - * would you want to avoid threads on BeOS? [chrish] - * - * As of 1.5.2, the PyImport_BeImageID() function has been removed; Donn - * tells me it's not necessary anymore because of PyCObject_Import(). - * [chrish] - */ - -/* Whack an item; the item is an image_id in disguise, so we'll call - * unload_add_on() for it. - */ -static void beos_nuke_dyn( PyObject *item ) -{ - status_t retval; - - if( item ) { - image_id id = (image_id)PyInt_AsLong( item ); - - retval = unload_add_on( id ); - } -} - -/* atexit() handler that'll call unload_add_on() for every item in the - * dictionary. - */ -static void beos_cleanup_dyn( void ) -{ - if( beos_dyn_images ) { - int idx; - int list_size; - PyObject *id_list; - -#ifdef WITH_THREAD - PyThread_acquire_lock( beos_dyn_lock, 1 ); -#endif - - id_list = PyDict_Values( beos_dyn_images ); - - list_size = PyList_Size( id_list ); - for( idx = 0; idx < list_size; idx++ ) { - PyObject *the_item; - - the_item = PyList_GetItem( id_list, idx ); - beos_nuke_dyn( the_item ); - } - - PyDict_Clear( beos_dyn_images ); - -#ifdef WITH_THREAD - PyThread_free_lock( beos_dyn_lock ); -#endif - } -} - -/* - * Initialize our dictionary, and the dictionary mutex. - */ -static void beos_init_dyn( void ) -{ - /* We're protected from a race condition here by the atomic init_count - * variable. - */ - static int32 init_count = 0; - int32 val; - - val = atomic_add( &init_count, 1 ); - if( beos_dyn_images == NULL && val == 0 ) { - beos_dyn_images = PyDict_New(); -#ifdef WITH_THREAD - beos_dyn_lock = PyThread_allocate_lock(); -#endif - atexit( beos_cleanup_dyn ); - } -} - -/* - * Add an image_id to the dictionary; the module name of the loaded image - * is the key. Note that if the key is already in the dict, we unload - * that image; this should allow reload() to work on dynamically loaded - * modules (super-keen!). - */ -static void beos_add_dyn( char *name, image_id id ) -{ - int retval; - PyObject *py_id; - - if( beos_dyn_images == NULL ) { - beos_init_dyn(); - } - -#ifdef WITH_THREAD - retval = PyThread_acquire_lock( beos_dyn_lock, 1 ); -#endif - - /* If there's already an object with this key in the dictionary, - * we're doing a reload(), so let's nuke it. - */ - py_id = PyDict_GetItemString( beos_dyn_images, name ); - if( py_id ) { - beos_nuke_dyn( py_id ); - retval = PyDict_DelItemString( beos_dyn_images, name ); - } - - py_id = PyInt_FromLong( (long)id ); - if( py_id ) { - retval = PyDict_SetItemString( beos_dyn_images, name, py_id ); - } - -#ifdef WITH_THREAD - PyThread_release_lock( beos_dyn_lock ); -#endif -} - - - -dl_funcptr _PyImport_GetDynLoadFunc(const char *fqname, const char *shortname, - const char *pathname, FILE *fp) -{ - dl_funcptr p; - image_id the_id; - status_t retval; - char fullpath[PATH_MAX]; - char funcname[258]; - - if( Py_VerboseFlag ) { - printf( "load_add_on( %s )\n", pathname ); - } - - /* Hmm, this old bug appears to have regenerated itself; if the - * path isn't absolute, load_add_on() will fail. Reported to Be - * April 21, 1998. - */ - if( pathname[0] != '/' ) { - (void)getcwd( fullpath, PATH_MAX ); - (void)strncat( fullpath, "/", PATH_MAX ); - (void)strncat( fullpath, pathname, PATH_MAX ); - - if( Py_VerboseFlag ) { - printf( "load_add_on( %s )\n", fullpath ); - } - } else { - (void)strcpy( fullpath, pathname ); - } - - the_id = load_add_on( fullpath ); - if( the_id < B_NO_ERROR ) { - /* It's too bad load_add_on() doesn't set errno or something... - */ - char buff[256]; /* hate hard-coded string sizes... */ - - if( Py_VerboseFlag ) { - printf( "load_add_on( %s ) failed", fullpath ); - } - - if( the_id == B_ERROR ) - PyOS_snprintf( buff, sizeof(buff), - "BeOS: Failed to load %.200s", - fullpath ); - else - PyOS_snprintf( buff, sizeof(buff), - "Unknown error loading %.200s", - fullpath ); - - PyErr_SetString( PyExc_ImportError, buff ); - return NULL; - } - - PyOS_snprintf(funcname, sizeof(funcname), "init%.200s", shortname); - if( Py_VerboseFlag ) { - printf( "get_image_symbol( %s )\n", funcname ); - } - - retval = get_image_symbol( the_id, funcname, B_SYMBOL_TYPE_TEXT, &p ); - if( retval != B_NO_ERROR || p == NULL ) { - /* That's bad, we can't find that symbol in the module... - */ - char buff[256]; /* hate hard-coded string sizes... */ - - if( Py_VerboseFlag ) { - printf( "get_image_symbol( %s ) failed", funcname ); - } - - switch( retval ) { - case B_BAD_IMAGE_ID: - PyOS_snprintf( buff, sizeof(buff), - "can't load init function for dynamic module: " - "Invalid image ID for %.180s", fullpath ); - break; - case B_BAD_INDEX: - PyOS_snprintf( buff, sizeof(buff), - "can't load init function for dynamic module: " - "Bad index for %.180s", funcname ); - break; - default: - PyOS_snprintf( buff, sizeof(buff), - "can't load init function for dynamic module: " - "Unknown error looking up %.180s", funcname ); - break; - } - - retval = unload_add_on( the_id ); - - PyErr_SetString( PyExc_ImportError, buff ); - return NULL; - } - - /* Save the module name and image ID for later so we can clean up - * gracefully. - */ - beos_add_dyn( fqname, the_id ); - - return p; -} diff --git a/Python/dynload_dl.c b/Python/dynload_dl.c index 4675a6722a..2606e1e32f 100644 --- a/Python/dynload_dl.c +++ b/Python/dynload_dl.c @@ -21,6 +21,6 @@ dl_funcptr _PyImport_GetDynLoadFunc(const char *fqname, const char *shortname, { char funcname[258]; - PyOS_snprintf(funcname, sizeof(funcname), "init%.200s", shortname); + PyOS_snprintf(funcname, sizeof(funcname), "PyInit_%.200s", shortname); return dl_loadmod(Py_GetProgramName(), pathname, funcname); } diff --git a/Python/dynload_hpux.c b/Python/dynload_hpux.c index c6ce16ef7e..18a81377f2 100644 --- a/Python/dynload_hpux.c +++ b/Python/dynload_hpux.c @@ -8,9 +8,9 @@ #include "importdl.h" #if defined(__hp9000s300) -#define FUNCNAME_PATTERN "_init%.200s" +#define FUNCNAME_PATTERN "_PyInit_%.200s" #else -#define FUNCNAME_PATTERN "init%.200s" +#define FUNCNAME_PATTERN "PyInit_%.200s" #endif const struct filedescr _PyImport_DynLoadFiletab[] = { diff --git a/Python/dynload_next.c b/Python/dynload_next.c index 7e61b5d570..cabf9b9663 100644 --- a/Python/dynload_next.c +++ b/Python/dynload_next.c @@ -43,7 +43,7 @@ dl_funcptr _PyImport_GetDynLoadFunc(const char *fqname, const char *shortname, const char *errString; char errBuf[512]; - PyOS_snprintf(funcname, sizeof(funcname), "_init%.200s", shortname); + PyOS_snprintf(funcname, sizeof(funcname), "_PyInit_%.200s", shortname); #ifdef USE_DYLD_GLOBAL_NAMESPACE if (NSIsSymbolNameDefined(funcname)) { diff --git a/Python/dynload_os2.c b/Python/dynload_os2.c index e3fdb70ab6..101c024bbe 100644 --- a/Python/dynload_os2.c +++ b/Python/dynload_os2.c @@ -38,7 +38,7 @@ dl_funcptr _PyImport_GetDynLoadFunc(const char *fqname, const char *shortname, return NULL; } - PyOS_snprintf(funcname, sizeof(funcname), "init%.200s", shortname); + PyOS_snprintf(funcname, sizeof(funcname), "PyInit_%.200s", shortname); rc = DosQueryProcAddr(hDLL, 0L, funcname, &p); if (rc != NO_ERROR) p = NULL; /* Signify Failure to Acquire Entrypoint */ diff --git a/Python/dynload_shlib.c b/Python/dynload_shlib.c index 17ebab16ba..7ea510e862 100644 --- a/Python/dynload_shlib.c +++ b/Python/dynload_shlib.c @@ -30,27 +30,36 @@ #define LEAD_UNDERSCORE "" #endif +/* The .so extension module ABI tag, supplied by the Makefile via + Makefile.pre.in and configure. This is used to discriminate between + incompatible .so files so that extensions for different Python builds can + live in the same directory. E.g. foomodule.cpython-32.so +*/ const struct filedescr _PyImport_DynLoadFiletab[] = { #ifdef __CYGWIN__ {".dll", "rb", C_EXTENSION}, {"module.dll", "rb", C_EXTENSION}, -#else +#else /* !__CYGWIN__ */ #if defined(PYOS_OS2) && defined(PYCC_GCC) {".pyd", "rb", C_EXTENSION}, {".dll", "rb", C_EXTENSION}, -#else +#else /* !(defined(PYOS_OS2) && defined(PYCC_GCC)) */ #ifdef __VMS {".exe", "rb", C_EXTENSION}, {".EXE", "rb", C_EXTENSION}, {"module.exe", "rb", C_EXTENSION}, {"MODULE.EXE", "rb", C_EXTENSION}, -#else +#else /* !__VMS */ + {"." SOABI ".so", "rb", C_EXTENSION}, + {"module." SOABI ".so", "rb", C_EXTENSION}, + {".abi" PYTHON_ABI_STRING ".so", "rb", C_EXTENSION}, + {"module.abi" PYTHON_ABI_STRING ".so", "rb", C_EXTENSION}, {".so", "rb", C_EXTENSION}, {"module.so", "rb", C_EXTENSION}, -#endif -#endif -#endif +#endif /* __VMS */ +#endif /* defined(PYOS_OS2) && defined(PYCC_GCC) */ +#endif /* __CYGWIN__ */ {0, 0} }; @@ -82,7 +91,7 @@ dl_funcptr _PyImport_GetDynLoadFunc(const char *fqname, const char *shortname, } PyOS_snprintf(funcname, sizeof(funcname), - LEAD_UNDERSCORE "init%.200s", shortname); + LEAD_UNDERSCORE "PyInit_%.200s", shortname); if (fp != NULL) { int i; diff --git a/Python/dynload_win.c b/Python/dynload_win.c index 4e5555e917..73a1dcf897 100644 --- a/Python/dynload_win.c +++ b/Python/dynload_win.c @@ -134,6 +134,15 @@ static char *GetPythonImport (HINSTANCE hModule) !strncmp(import_name,"python",6)) { char *pch; +#ifndef _DEBUG + /* In a release version, don't claim that python3.dll is + a Python DLL. */ + if (strcmp(import_name, "python3.dll") == 0) { + import_data += 20; + continue; + } +#endif + /* Ensure python prefix is followed only by numbers to the end of the basename */ pch = import_name + 6; @@ -162,14 +171,17 @@ static char *GetPythonImport (HINSTANCE hModule) return NULL; } - dl_funcptr _PyImport_GetDynLoadFunc(const char *fqname, const char *shortname, const char *pathname, FILE *fp) { dl_funcptr p; char funcname[258], *import_python; - PyOS_snprintf(funcname, sizeof(funcname), "init%.200s", shortname); +#ifndef _DEBUG + _Py_CheckPython3(); +#endif + + PyOS_snprintf(funcname, sizeof(funcname), "PyInit_%.200s", shortname); { HINSTANCE hDLL = NULL; @@ -201,34 +213,35 @@ dl_funcptr _PyImport_GetDynLoadFunc(const char *fqname, const char *shortname, SetErrorMode(old_mode); if (hDLL==NULL){ - char errBuf[256]; + PyObject *message; unsigned int errorCode; /* Get an error string from Win32 error code */ - char theInfo[256]; /* Pointer to error text + wchar_t theInfo[256]; /* Pointer to error text from system */ int theLength; /* Length of error text */ errorCode = GetLastError(); - theLength = FormatMessage( + theLength = FormatMessageW( FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, /* flags */ NULL, /* message source */ errorCode, /* the message (error) ID */ - 0, /* default language environment */ - (LPTSTR) theInfo, /* the buffer */ + MAKELANGID(LANG_NEUTRAL, + SUBLANG_DEFAULT), + /* Default language */ + theInfo, /* the buffer */ sizeof(theInfo), /* the buffer size */ NULL); /* no additional format args. */ /* Problem: could not get the error message. This should not happen if called correctly. */ if (theLength == 0) { - PyOS_snprintf(errBuf, sizeof(errBuf), - "DLL load failed with error code %d", - errorCode); + message = PyUnicode_FromFormat( + "DLL load failed with error code %d", + errorCode); } else { - size_t len; /* For some reason a \r\n is appended to the text */ if (theLength >= 2 && @@ -237,13 +250,16 @@ dl_funcptr _PyImport_GetDynLoadFunc(const char *fqname, const char *shortname, theLength -= 2; theInfo[theLength] = '\0'; } - strcpy(errBuf, "DLL load failed: "); - len = strlen(errBuf); - strncpy(errBuf+len, theInfo, - sizeof(errBuf)-len); - errBuf[sizeof(errBuf)-1] = '\0'; + message = PyUnicode_FromString( + "DLL load failed: "); + + PyUnicode_AppendAndDel(&message, + PyUnicode_FromUnicode( + theInfo, + theLength)); } - PyErr_SetString(PyExc_ImportError, errBuf); + PyErr_SetObject(PyExc_ImportError, message); + Py_XDECREF(message); return NULL; } else { char buffer[256]; diff --git a/Python/errors.c b/Python/errors.c index 64ba05dd6c..5a9a624279 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -10,8 +10,8 @@ extern char *strerror(int); #endif #ifdef MS_WINDOWS -#include "windows.h" -#include "winbase.h" +#include <windows.h> +#include <winbase.h> #endif #include <ctype.h> @@ -52,9 +52,64 @@ PyErr_Restore(PyObject *type, PyObject *value, PyObject *traceback) void PyErr_SetObject(PyObject *exception, PyObject *value) { - Py_XINCREF(exception); + PyThreadState *tstate = PyThreadState_GET(); + PyObject *exc_value; + PyObject *tb = NULL; + + if (exception != NULL && + !PyExceptionClass_Check(exception)) { + PyErr_Format(PyExc_SystemError, + "exception %R not a BaseException subclass", + exception); + return; + } Py_XINCREF(value); - PyErr_Restore(exception, value, (PyObject *)NULL); + exc_value = tstate->exc_value; + if (exc_value != NULL && exc_value != Py_None) { + /* Implicit exception chaining */ + Py_INCREF(exc_value); + if (value == NULL || !PyExceptionInstance_Check(value)) { + /* We must normalize the value right now */ + PyObject *args, *fixed_value; + if (value == NULL || value == Py_None) + args = PyTuple_New(0); + else if (PyTuple_Check(value)) { + Py_INCREF(value); + args = value; + } + else + args = PyTuple_Pack(1, value); + fixed_value = args ? + PyEval_CallObject(exception, args) : NULL; + Py_XDECREF(args); + Py_XDECREF(value); + if (fixed_value == NULL) + return; + value = fixed_value; + } + /* Avoid reference cycles through the context chain. + This is O(chain length) but context chains are + usually very short. Sensitive readers may try + to inline the call to PyException_GetContext. */ + if (exc_value != value) { + PyObject *o = exc_value, *context; + while ((context = PyException_GetContext(o))) { + Py_DECREF(context); + if (context == value) { + PyException_SetContext(o, NULL); + break; + } + o = context; + } + PyException_SetContext(value, exc_value); + } else { + Py_DECREF(exc_value); + } + } + if (value != NULL && PyExceptionInstance_Check(value)) + tb = PyException_GetTraceback(value); + Py_XINCREF(exception); + PyErr_Restore(exception, value, tb); } void @@ -66,7 +121,7 @@ PyErr_SetNone(PyObject *exception) void PyErr_SetString(PyObject *exception, const char *string) { - PyObject *value = PyString_FromString(string); + PyObject *value = PyUnicode_FromString(string); PyErr_SetObject(exception, value); Py_XDECREF(value); } @@ -75,9 +130,14 @@ PyErr_SetString(PyObject *exception, const char *string) PyObject * PyErr_Occurred(void) { - PyThreadState *tstate = PyThreadState_GET(); + /* If there is no thread state, PyThreadState_GET calls + Py_FatalError, which calls PyErr_Occurred. To avoid the + resulting infinite loop, we inline PyThreadState_GET here and + treat no thread as no error. */ + PyThreadState *tstate = + ((PyThreadState*)_Py_atomic_load_relaxed(&_PyThreadState_Current)); - return tstate->curexc_type; + return tstate == NULL ? NULL : tstate->curexc_type; } @@ -106,18 +166,12 @@ PyErr_GivenExceptionMatches(PyObject *err, PyObject *exc) err = PyExceptionInstance_Class(err); if (PyExceptionClass_Check(err) && PyExceptionClass_Check(exc)) { - int res = 0, reclimit; + int res = 0; PyObject *exception, *value, *tb; PyErr_Fetch(&exception, &value, &tb); - /* Temporarily bump the recursion limit, so that in the most - common case PyObject_IsSubclass will not raise a recursion - error we have to ignore anyway. Don't do it when the limit - is already insanely high, to avoid overflow */ - reclimit = Py_GetRecursionLimit(); - if (reclimit < (1 << 30)) - Py_SetRecursionLimit(reclimit + 5); - res = PyObject_IsSubclass(err, exc); - Py_SetRecursionLimit(reclimit); + /* PyObject_IsSubclass() can recurse and therefore is + not safe (see test_bad_getattr in test.pickletester). */ + res = PyType_IsSubtype((PyTypeObject *)err, (PyTypeObject *)exc); /* This function must not fail, so print the error here */ if (res == -1) { PyErr_WriteUnraisable(err); @@ -140,6 +194,9 @@ PyErr_ExceptionMatches(PyObject *exc) /* Used in many places to normalize a raised exception, including in eval_code2(), do_raise(), and PyErr_Print() + + XXX: should PyErr_NormalizeException() also call + PyException_SetTraceback() with the resulting value and tb? */ void PyErr_NormalizeException(PyObject **exc, PyObject **val, PyObject **tb) @@ -276,49 +333,37 @@ PyErr_BadArgument(void) PyObject * PyErr_NoMemory(void) { - if (PyErr_ExceptionMatches(PyExc_MemoryError)) - /* already current */ - return NULL; - - /* raise the pre-allocated instance if it still exists */ - if (PyExc_MemoryErrorInst) - PyErr_SetObject(PyExc_MemoryError, PyExc_MemoryErrorInst); - else - /* this will probably fail since there's no memory and hee, - hee, we have to instantiate this class - */ - PyErr_SetNone(PyExc_MemoryError); - + PyErr_SetNone(PyExc_MemoryError); return NULL; } PyObject * PyErr_SetFromErrnoWithFilenameObject(PyObject *exc, PyObject *filenameObject) { + PyObject *message; PyObject *v; - char *s; int i = errno; -#ifdef PLAN9 - char errbuf[ERRMAX]; -#endif -#ifdef MS_WINDOWS - char *s_buf = NULL; - char s_small_buf[28]; /* Room for "Windows Error 0xFFFFFFFF" */ -#endif +#ifndef MS_WINDOWS + char *s; +#else + WCHAR *s_buf = NULL; +#endif /* Unix/Windows */ + #ifdef EINTR if (i == EINTR && PyErr_CheckSignals()) return NULL; #endif -#ifdef PLAN9 - rerrstr(errbuf, sizeof errbuf); - s = errbuf; -#else + +#ifndef MS_WINDOWS if (i == 0) s = "Error"; /* Sometimes errno didn't get set */ else -#ifndef MS_WINDOWS s = strerror(i); + message = PyUnicode_DecodeUTF8(s, strlen(s), "ignore"); #else + if (i == 0) + message = PyUnicode_FromString("Error"); /* Sometimes errno didn't get set */ + else { /* Note that the Win32 errors do not lineup with the errno error. So if the error is in the MSVC error @@ -326,10 +371,10 @@ PyErr_SetFromErrnoWithFilenameObject(PyObject *exc, PyObject *filenameObject) a Win32 error code */ if (i > 0 && i < _sys_nerr) { - s = _sys_errlist[i]; + message = PyUnicode_FromString(_sys_errlist[i]); } else { - int len = FormatMessage( + int len = FormatMessageW( FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, @@ -338,29 +383,38 @@ PyErr_SetFromErrnoWithFilenameObject(PyObject *exc, PyObject *filenameObject) MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), /* Default language */ - (LPTSTR) &s_buf, + (LPWSTR) &s_buf, 0, /* size not used */ NULL); /* no args */ if (len==0) { /* Only ever seen this in out-of-mem situations */ - sprintf(s_small_buf, "Windows Error 0x%X", i); - s = s_small_buf; s_buf = NULL; + message = PyUnicode_FromFormat("Windows Error 0x%X", i); } else { - s = s_buf; /* remove trailing cr/lf and dots */ - while (len > 0 && (s[len-1] <= ' ' || s[len-1] == '.')) - s[--len] = '\0'; + while (len > 0 && (s_buf[len-1] <= L' ' || s_buf[len-1] == L'.')) + s_buf[--len] = L'\0'; + message = PyUnicode_FromUnicode(s_buf, len); } } } #endif /* Unix/Windows */ -#endif /* PLAN 9*/ + + if (message == NULL) + { +#ifdef MS_WINDOWS + LocalFree(s_buf); +#endif + return NULL; + } + if (filenameObject != NULL) - v = Py_BuildValue("(isO)", i, s, filenameObject); + v = Py_BuildValue("(iOO)", i, message, filenameObject); else - v = Py_BuildValue("(is)", i, s); + v = Py_BuildValue("(iO)", i, message); + Py_DECREF(message); + if (v != NULL) { PyErr_SetObject(exc, v); Py_DECREF(v); @@ -375,7 +429,7 @@ PyErr_SetFromErrnoWithFilenameObject(PyObject *exc, PyObject *filenameObject) PyObject * PyErr_SetFromErrnoWithFilename(PyObject *exc, const char *filename) { - PyObject *name = filename ? PyString_FromString(filename) : NULL; + PyObject *name = filename ? PyUnicode_DecodeFSDefault(filename) : NULL; PyObject *result = PyErr_SetFromErrnoWithFilenameObject(exc, name); Py_XDECREF(name); return result; @@ -408,13 +462,12 @@ PyObject *PyErr_SetExcFromWindowsErrWithFilenameObject( PyObject *filenameObject) { int len; - char *s; - char *s_buf = NULL; /* Free via LocalFree */ - char s_small_buf[28]; /* Room for "Windows Error 0xFFFFFFFF" */ + WCHAR *s_buf = NULL; /* Free via LocalFree */ + PyObject *message; PyObject *v; DWORD err = (DWORD)ierr; if (err==0) err = GetLastError(); - len = FormatMessage( + len = FormatMessageW( /* Error API error */ FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | @@ -423,24 +476,32 @@ PyObject *PyErr_SetExcFromWindowsErrWithFilenameObject( err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), /* Default language */ - (LPTSTR) &s_buf, + (LPWSTR) &s_buf, 0, /* size not used */ NULL); /* no args */ if (len==0) { /* Only seen this in out of mem situations */ - sprintf(s_small_buf, "Windows Error 0x%X", err); - s = s_small_buf; + message = PyUnicode_FromFormat("Windows Error 0x%X", err); s_buf = NULL; } else { - s = s_buf; /* remove trailing cr/lf and dots */ - while (len > 0 && (s[len-1] <= ' ' || s[len-1] == '.')) - s[--len] = '\0'; + while (len > 0 && (s_buf[len-1] <= L' ' || s_buf[len-1] == L'.')) + s_buf[--len] = L'\0'; + message = PyUnicode_FromUnicode(s_buf, len); } + + if (message == NULL) + { + LocalFree(s_buf); + return NULL; + } + if (filenameObject != NULL) - v = Py_BuildValue("(isO)", err, s, filenameObject); + v = Py_BuildValue("(iOO)", err, message, filenameObject); else - v = Py_BuildValue("(is)", err, s); + v = Py_BuildValue("(iO)", err, message); + Py_DECREF(message); + if (v != NULL) { PyErr_SetObject(exc, v); Py_DECREF(v); @@ -454,7 +515,7 @@ PyObject *PyErr_SetExcFromWindowsErrWithFilename( int ierr, const char *filename) { - PyObject *name = filename ? PyString_FromString(filename) : NULL; + PyObject *name = filename ? PyUnicode_DecodeFSDefault(filename) : NULL; PyObject *ret = PyErr_SetExcFromWindowsErrWithFilenameObject(exc, ierr, name); @@ -491,7 +552,7 @@ PyObject *PyErr_SetFromWindowsErrWithFilename( int ierr, const char *filename) { - PyObject *name = filename ? PyString_FromString(filename) : NULL; + PyObject *name = filename ? PyUnicode_DecodeFSDefault(filename) : NULL; PyObject *result = PyErr_SetExcFromWindowsErrWithFilenameObject( PyExc_WindowsError, ierr, name); @@ -515,7 +576,7 @@ PyObject *PyErr_SetFromWindowsErrWithUnicodeFilename( #endif /* MS_WINDOWS */ void -_PyErr_BadInternalCall(char *filename, int lineno) +_PyErr_BadInternalCall(const char *filename, int lineno) { PyErr_Format(PyExc_SystemError, "%s:%d: bad argument to internal function", @@ -547,7 +608,7 @@ PyErr_Format(PyObject *exception, const char *format, ...) va_start(vargs); #endif - string = PyString_FromFormatV(format, vargs); + string = PyUnicode_FromFormatV(format, vargs); PyErr_SetObject(exception, string); Py_XDECREF(string); va_end(vargs); @@ -557,9 +618,9 @@ PyErr_Format(PyObject *exception, const char *format, ...) PyObject * -PyErr_NewException(char *name, PyObject *base, PyObject *dict) +PyErr_NewException(const char *name, PyObject *base, PyObject *dict) { - char *dot; + const char *dot; PyObject *modulename = NULL; PyObject *classname = NULL; PyObject *mydict = NULL; @@ -579,7 +640,7 @@ PyErr_NewException(char *name, PyObject *base, PyObject *dict) goto failure; } if (PyDict_GetItemString(dict, "__module__") == NULL) { - modulename = PyString_FromStringAndSize(name, + modulename = PyUnicode_FromStringAndSize(name, (Py_ssize_t)(dot-name)); if (modulename == NULL) goto failure; @@ -609,7 +670,8 @@ PyErr_NewException(char *name, PyObject *base, PyObject *dict) /* Create an exception with docstring */ PyObject * -PyErr_NewExceptionWithDoc(char *name, char *doc, PyObject *base, PyObject *dict) +PyErr_NewExceptionWithDoc(const char *name, const char *doc, + PyObject *base, PyObject *dict) { int result; PyObject *ret = NULL; @@ -624,7 +686,7 @@ PyErr_NewExceptionWithDoc(char *name, char *doc, PyObject *base, PyObject *dict) } if (doc != NULL) { - docobj = PyString_FromString(doc); + docobj = PyUnicode_FromString(doc); if (docobj == NULL) goto failure; result = PyDict_SetItemString(dict, "__doc__", docobj); @@ -648,7 +710,7 @@ PyErr_WriteUnraisable(PyObject *obj) PyObject *f, *t, *v, *tb; PyErr_Fetch(&t, &v, &tb); f = PySys_GetObject("stderr"); - if (f != NULL) { + if (f != NULL && f != Py_None) { PyFile_WriteString("Exception ", f); if (t) { PyObject* moduleName; @@ -665,9 +727,9 @@ PyErr_WriteUnraisable(PyObject *obj) if (moduleName == NULL) PyFile_WriteString("<unknown>", f); else { - char* modstr = PyString_AsString(moduleName); + char* modstr = _PyUnicode_AsString(moduleName); if (modstr && - strcmp(modstr, "exceptions") != 0) + strcmp(modstr, "builtins") != 0) { PyFile_WriteString(modstr, f); PyFile_WriteString(".", f); @@ -683,8 +745,10 @@ PyErr_WriteUnraisable(PyObject *obj) } Py_XDECREF(moduleName); } - PyFile_WriteString(" in ", f); - PyFile_WriteObject(obj, f, 0); + if (obj) { + PyFile_WriteString(" in ", f); + PyFile_WriteObject(obj, f, 0); + } PyFile_WriteString(" ignored\n", f); PyErr_Clear(); /* Just in case */ } @@ -696,12 +760,18 @@ PyErr_WriteUnraisable(PyObject *obj) extern PyObject *PyModule_GetWarningsModule(void); +void +PyErr_SyntaxLocation(const char *filename, int lineno) { + PyErr_SyntaxLocationEx(filename, lineno, -1); +} + + /* Set file and line information for the current exception. If the exception is not a SyntaxError, also sets additional attributes to make printing of exceptions believe it is a syntax error. */ void -PyErr_SyntaxLocation(const char *filename, int lineno) +PyErr_SyntaxLocationEx(const char *filename, int lineno, int col_offset) { PyObject *exc, *v, *tb, *tmp; @@ -710,7 +780,7 @@ PyErr_SyntaxLocation(const char *filename, int lineno) PyErr_NormalizeException(&exc, &v, &tb); /* XXX check that it is, indeed, a syntax error. It might not * be, though. */ - tmp = PyInt_FromLong(lineno); + tmp = PyLong_FromLong(lineno); if (tmp == NULL) PyErr_Clear(); else { @@ -718,8 +788,18 @@ PyErr_SyntaxLocation(const char *filename, int lineno) PyErr_Clear(); Py_DECREF(tmp); } + if (col_offset >= 0) { + tmp = PyLong_FromLong(col_offset); + if (tmp == NULL) + PyErr_Clear(); + else { + if (PyObject_SetAttrString(v, "offset", tmp)) + PyErr_Clear(); + Py_DECREF(tmp); + } + } if (filename != NULL) { - tmp = PyString_FromString(filename); + tmp = PyUnicode_DecodeFSDefault(filename); if (tmp == NULL) PyErr_Clear(); else { @@ -758,13 +838,11 @@ PyErr_SyntaxLocation(const char *filename, int lineno) PyErr_Restore(exc, v, tb); } -/* com_fetch_program_text will attempt to load the line of text that - the exception refers to. If it fails, it will return NULL but will - not set an exception. +/* Attempt to load the line of text that the exception refers to. If it + fails, it will return NULL but will not set an exception. XXX The functionality of this function is quite similar to the - functionality in tb_displayline() in traceback.c. -*/ + functionality in tb_displayline() in traceback.c. */ PyObject * PyErr_ProgramText(const char *filename, int lineno) @@ -782,7 +860,8 @@ PyErr_ProgramText(const char *filename, int lineno) char *pLastChar = &linebuf[sizeof(linebuf) - 2]; do { *pLastChar = '\0'; - if (Py_UniversalNewlineFgets(linebuf, sizeof linebuf, fp, NULL) == NULL) + if (Py_UniversalNewlineFgets(linebuf, sizeof linebuf, + fp, NULL) == NULL) break; /* fgets read *something*; if it didn't get as far as pLastChar, it must have found a newline @@ -794,9 +873,13 @@ PyErr_ProgramText(const char *filename, int lineno) fclose(fp); if (i == lineno) { char *p = linebuf; + PyObject *res; while (*p == ' ' || *p == '\t' || *p == '\014') p++; - return PyString_FromString(p); + res = PyUnicode_FromString(p); + if (res == NULL) + PyErr_Clear(); + return res; } return NULL; } @@ -804,4 +887,3 @@ PyErr_ProgramText(const char *filename, int lineno) #ifdef __cplusplus } #endif - diff --git a/Python/fileutils.c b/Python/fileutils.c new file mode 100644 index 0000000000..94ab8e4251 --- /dev/null +++ b/Python/fileutils.c @@ -0,0 +1,481 @@ +#include "Python.h" +#ifdef MS_WINDOWS +# include <windows.h> +#endif + +#ifdef __APPLE__ +extern wchar_t* _Py_DecodeUTF8_surrogateescape(const char *s, Py_ssize_t size); +#endif + +#ifdef HAVE_STAT + +/* Decode a byte string from the locale encoding with the + surrogateescape error handler (undecodable bytes are decoded as characters + in range U+DC80..U+DCFF). If a byte sequence can be decoded as a surrogate + character, escape the bytes using the surrogateescape error handler instead + of decoding them. + + Use _Py_wchar2char() to encode the character string back to a byte string. + + Return a pointer to a newly allocated wide character string (use + PyMem_Free() to free the memory) and write the number of written wide + characters excluding the null character into *size if size is not NULL, or + NULL on error (conversion or memory allocation error). + + Conversion errors should never happen, unless there is a bug in the C + library. */ +wchar_t* +_Py_char2wchar(const char* arg, size_t *size) +{ +#ifdef __APPLE__ + wchar_t *wstr; + wstr = _Py_DecodeUTF8_surrogateescape(arg, strlen(arg)); + if (size != NULL) { + if (wstr != NULL) + *size = wcslen(wstr); + else + *size = (size_t)-1; + } + return wstr; +#else + wchar_t *res; +#ifdef HAVE_BROKEN_MBSTOWCS + /* Some platforms have a broken implementation of + * mbstowcs which does not count the characters that + * would result from conversion. Use an upper bound. + */ + size_t argsize = strlen(arg); +#else + size_t argsize = mbstowcs(NULL, arg, 0); +#endif + size_t count; + unsigned char *in; + wchar_t *out; +#ifdef HAVE_MBRTOWC + mbstate_t mbs; +#endif + if (argsize != (size_t)-1) { + res = (wchar_t *)PyMem_Malloc((argsize+1)*sizeof(wchar_t)); + if (!res) + goto oom; + count = mbstowcs(res, arg, argsize+1); + if (count != (size_t)-1) { + wchar_t *tmp; + /* Only use the result if it contains no + surrogate characters. */ + for (tmp = res; *tmp != 0 && + (*tmp < 0xd800 || *tmp > 0xdfff); tmp++) + ; + if (*tmp == 0) { + if (size != NULL) + *size = count; + return res; + } + } + PyMem_Free(res); + } + /* Conversion failed. Fall back to escaping with surrogateescape. */ +#ifdef HAVE_MBRTOWC + /* Try conversion with mbrtwoc (C99), and escape non-decodable bytes. */ + + /* Overallocate; as multi-byte characters are in the argument, the + actual output could use less memory. */ + argsize = strlen(arg) + 1; + res = (wchar_t*)PyMem_Malloc(argsize*sizeof(wchar_t)); + if (!res) + goto oom; + in = (unsigned char*)arg; + out = res; + memset(&mbs, 0, sizeof mbs); + while (argsize) { + size_t converted = mbrtowc(out, (char*)in, argsize, &mbs); + if (converted == 0) + /* Reached end of string; null char stored. */ + break; + if (converted == (size_t)-2) { + /* Incomplete character. This should never happen, + since we provide everything that we have - + unless there is a bug in the C library, or I + misunderstood how mbrtowc works. */ + fprintf(stderr, "unexpected mbrtowc result -2\n"); + PyMem_Free(res); + return NULL; + } + if (converted == (size_t)-1) { + /* Conversion error. Escape as UTF-8b, and start over + in the initial shift state. */ + *out++ = 0xdc00 + *in++; + argsize--; + memset(&mbs, 0, sizeof mbs); + continue; + } + if (*out >= 0xd800 && *out <= 0xdfff) { + /* Surrogate character. Escape the original + byte sequence with surrogateescape. */ + argsize -= converted; + while (converted--) + *out++ = 0xdc00 + *in++; + continue; + } + /* successfully converted some bytes */ + in += converted; + argsize -= converted; + out++; + } +#else /* HAVE_MBRTOWC */ + /* Cannot use C locale for escaping; manually escape as if charset + is ASCII (i.e. escape all bytes > 128. This will still roundtrip + correctly in the locale's charset, which must be an ASCII superset. */ + res = PyMem_Malloc((strlen(arg)+1)*sizeof(wchar_t)); + if (!res) goto oom; + in = (unsigned char*)arg; + out = res; + while(*in) + if(*in < 128) + *out++ = *in++; + else + *out++ = 0xdc00 + *in++; + *out = 0; +#endif /* HAVE_MBRTOWC */ + if (size != NULL) + *size = out - res; + return res; +oom: + fprintf(stderr, "out of memory\n"); + return NULL; +#endif /* __APPLE__ */ +} + +/* Encode a (wide) character string to the locale encoding with the + surrogateescape error handler (characters in range U+DC80..U+DCFF are + converted to bytes 0x80..0xFF). + + This function is the reverse of _Py_char2wchar(). + + Return a pointer to a newly allocated byte string (use PyMem_Free() to free + the memory), or NULL on conversion or memory allocation error. + + If error_pos is not NULL: *error_pos is the index of the invalid character + on conversion error, or (size_t)-1 otherwise. */ +char* +_Py_wchar2char(const wchar_t *text, size_t *error_pos) +{ +#ifdef __APPLE__ + Py_ssize_t len; + PyObject *unicode, *bytes = NULL; + char *cpath; + + unicode = PyUnicode_FromWideChar(text, wcslen(text)); + if (unicode == NULL) + return NULL; + + bytes = PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(unicode), + PyUnicode_GET_SIZE(unicode), + "surrogateescape"); + Py_DECREF(unicode); + if (bytes == NULL) { + PyErr_Clear(); + if (error_pos != NULL) + *error_pos = (size_t)-1; + return NULL; + } + + len = PyBytes_GET_SIZE(bytes); + cpath = PyMem_Malloc(len+1); + if (cpath == NULL) { + PyErr_Clear(); + Py_DECREF(bytes); + if (error_pos != NULL) + *error_pos = (size_t)-1; + return NULL; + } + memcpy(cpath, PyBytes_AsString(bytes), len + 1); + Py_DECREF(bytes); + return cpath; +#else /* __APPLE__ */ + const size_t len = wcslen(text); + char *result = NULL, *bytes = NULL; + size_t i, size, converted; + wchar_t c, buf[2]; + + /* The function works in two steps: + 1. compute the length of the output buffer in bytes (size) + 2. outputs the bytes */ + size = 0; + buf[1] = 0; + while (1) { + for (i=0; i < len; i++) { + c = text[i]; + if (c >= 0xdc80 && c <= 0xdcff) { + /* UTF-8b surrogate */ + if (bytes != NULL) { + *bytes++ = c - 0xdc00; + size--; + } + else + size++; + continue; + } + else { + buf[0] = c; + if (bytes != NULL) + converted = wcstombs(bytes, buf, size); + else + converted = wcstombs(NULL, buf, 0); + if (converted == (size_t)-1) { + if (result != NULL) + PyMem_Free(result); + if (error_pos != NULL) + *error_pos = i; + return NULL; + } + if (bytes != NULL) { + bytes += converted; + size -= converted; + } + else + size += converted; + } + } + if (result != NULL) { + *bytes = 0; + break; + } + + size += 1; /* nul byte at the end */ + result = PyMem_Malloc(size); + if (result == NULL) { + if (error_pos != NULL) + *error_pos = (size_t)-1; + return NULL; + } + bytes = result; + } + return result; +#endif /* __APPLE__ */ +} + +/* In principle, this should use HAVE__WSTAT, and _wstat + should be detected by autoconf. However, no current + POSIX system provides that function, so testing for + it is pointless. + Not sure whether the MS_WINDOWS guards are necessary: + perhaps for cygwin/mingw builds? +*/ +#if defined(HAVE_STAT) && !defined(MS_WINDOWS) + +/* Get file status. Encode the path to the locale encoding. */ + +int +_Py_wstat(const wchar_t* path, struct stat *buf) +{ + int err; + char *fname; + fname = _Py_wchar2char(path, NULL); + if (fname == NULL) { + errno = EINVAL; + return -1; + } + err = stat(fname, buf); + PyMem_Free(fname); + return err; +} +#endif + +/* Call _wstat() on Windows, or encode the path to the filesystem encoding and + call stat() otherwise. Only fill st_mode attribute on Windows. + + Return 0 on success, -1 on _wstat() / stat() error or (if PyErr_Occurred()) + unicode error. */ + +int +_Py_stat(PyObject *path, struct stat *statbuf) +{ +#ifdef MS_WINDOWS + int err; + struct _stat wstatbuf; + + err = _wstat(PyUnicode_AS_UNICODE(path), &wstatbuf); + if (!err) + statbuf->st_mode = wstatbuf.st_mode; + return err; +#else + int ret; + PyObject *bytes = PyUnicode_EncodeFSDefault(path); + if (bytes == NULL) + return -1; + ret = stat(PyBytes_AS_STRING(bytes), statbuf); + Py_DECREF(bytes); + return ret; +#endif +} + +/* Open a file. Use _wfopen() on Windows, encode the path to the locale + encoding and use fopen() otherwise. */ + +FILE * +_Py_wfopen(const wchar_t *path, const wchar_t *mode) +{ +#ifndef MS_WINDOWS + FILE *f; + char *cpath; + char cmode[10]; + size_t r; + r = wcstombs(cmode, mode, 10); + if (r == (size_t)-1 || r >= 10) { + errno = EINVAL; + return NULL; + } + cpath = _Py_wchar2char(path, NULL); + if (cpath == NULL) + return NULL; + f = fopen(cpath, cmode); + PyMem_Free(cpath); + return f; +#else + return _wfopen(path, mode); +#endif +} + +/* Call _wfopen() on Windows, or encode the path to the filesystem encoding and + call fopen() otherwise. + + Return the new file object on success, or NULL if the file cannot be open or + (if PyErr_Occurred()) on unicode error */ + +FILE* +_Py_fopen(PyObject *path, const char *mode) +{ +#ifdef MS_WINDOWS + wchar_t wmode[10]; + int usize; + + usize = MultiByteToWideChar(CP_ACP, 0, mode, -1, wmode, sizeof(wmode)); + if (usize == 0) + return NULL; + + return _wfopen(PyUnicode_AS_UNICODE(path), wmode); +#else + FILE *f; + PyObject *bytes = PyUnicode_EncodeFSDefault(path); + if (bytes == NULL) + return NULL; + f = fopen(PyBytes_AS_STRING(bytes), mode); + Py_DECREF(bytes); + return f; +#endif +} + +#ifdef HAVE_READLINK + +/* Read value of symbolic link. Encode the path to the locale encoding, decode + the result from the locale encoding. */ + +int +_Py_wreadlink(const wchar_t *path, wchar_t *buf, size_t bufsiz) +{ + char *cpath; + char cbuf[PATH_MAX]; + wchar_t *wbuf; + int res; + size_t r1; + + cpath = _Py_wchar2char(path, NULL); + if (cpath == NULL) { + errno = EINVAL; + return -1; + } + res = (int)readlink(cpath, cbuf, PATH_MAX); + PyMem_Free(cpath); + if (res == -1) + return -1; + if (res == PATH_MAX) { + errno = EINVAL; + return -1; + } + cbuf[res] = '\0'; /* buf will be null terminated */ + wbuf = _Py_char2wchar(cbuf, &r1); + if (wbuf == NULL) { + errno = EINVAL; + return -1; + } + if (bufsiz <= r1) { + PyMem_Free(wbuf); + errno = EINVAL; + return -1; + } + wcsncpy(buf, wbuf, bufsiz); + PyMem_Free(wbuf); + return (int)r1; +} +#endif + +#ifdef HAVE_REALPATH + +/* Return the canonicalized absolute pathname. Encode path to the locale + encoding, decode the result from the locale encoding. */ + +wchar_t* +_Py_wrealpath(const wchar_t *path, + wchar_t *resolved_path, size_t resolved_path_size) +{ + char *cpath; + char cresolved_path[PATH_MAX]; + wchar_t *wresolved_path; + char *res; + size_t r; + cpath = _Py_wchar2char(path, NULL); + if (cpath == NULL) { + errno = EINVAL; + return NULL; + } + res = realpath(cpath, cresolved_path); + PyMem_Free(cpath); + if (res == NULL) + return NULL; + + wresolved_path = _Py_char2wchar(cresolved_path, &r); + if (wresolved_path == NULL) { + errno = EINVAL; + return NULL; + } + if (resolved_path_size <= r) { + PyMem_Free(wresolved_path); + errno = EINVAL; + return NULL; + } + wcsncpy(resolved_path, wresolved_path, resolved_path_size); + PyMem_Free(wresolved_path); + return resolved_path; +} +#endif + +/* Get the current directory. size is the buffer size in wide characters + including the null character. Decode the path from the locale encoding. */ + +wchar_t* +_Py_wgetcwd(wchar_t *buf, size_t size) +{ +#ifdef MS_WINDOWS + return _wgetcwd(buf, size); +#else + char fname[PATH_MAX]; + wchar_t *wname; + size_t len; + + if (getcwd(fname, PATH_MAX) == NULL) + return NULL; + wname = _Py_char2wchar(fname, &len); + if (wname == NULL) + return NULL; + if (size <= len) { + PyMem_Free(wname); + return NULL; + } + wcsncpy(buf, wname, size); + PyMem_Free(wname); + return buf; +#endif +} + +#endif diff --git a/Python/formatter_string.c b/Python/formatter_string.c deleted file mode 100644 index 06a2ef5903..0000000000 --- a/Python/formatter_string.c +++ /dev/null @@ -1,17 +0,0 @@ -/***********************************************************************/ -/* Implements the string (as opposed to unicode) version of the - built-in formatters for string, int, float. That is, the versions - of int.__format__, etc., that take and return string objects */ - -#include "Python.h" -#include "../Objects/stringlib/stringdefs.h" - -#define FORMAT_STRING _PyBytes_FormatAdvanced -#define FORMAT_LONG _PyLong_FormatAdvanced -#define FORMAT_INT _PyInt_FormatAdvanced -#define FORMAT_FLOAT _PyFloat_FormatAdvanced -#ifndef WITHOUT_COMPLEX -#define FORMAT_COMPLEX _PyComplex_FormatAdvanced -#endif - -#include "../Objects/stringlib/formatter.h" diff --git a/Python/formatter_unicode.c b/Python/formatter_unicode.c index 6e3685d16e..c350907da1 100644 --- a/Python/formatter_unicode.c +++ b/Python/formatter_unicode.c @@ -1,18 +1,14 @@ -/* Implements the unicode (as opposed to string) version of the - built-in formatter for unicode. That is, unicode.__format__(). */ +/* implements the unicode (as opposed to string) version of the + built-in formatters for string, int, float. that is, the versions + of int.__float__, etc., that take and return unicode objects */ #include "Python.h" - -#ifdef Py_USING_UNICODE - #include "../Objects/stringlib/unicodedefs.h" -#define FORMAT_STRING _PyUnicode_FormatAdvanced -/* don't define FORMAT_LONG, FORMAT_FLOAT, and FORMAT_COMPLEX, since - we can live with only the string versions of those. The builtin - format() will convert them to unicode. */ +#define FORMAT_STRING _PyUnicode_FormatAdvanced +#define FORMAT_LONG _PyLong_FormatAdvanced +#define FORMAT_FLOAT _PyFloat_FormatAdvanced +#define FORMAT_COMPLEX _PyComplex_FormatAdvanced #include "../Objects/stringlib/formatter.h" - -#endif diff --git a/Python/frozen.c b/Python/frozen.c index ae7e9a4a93..ddf6224e6d 100644 --- a/Python/frozen.c +++ b/Python/frozen.c @@ -8,17 +8,21 @@ some famous words... */ /* To regenerate this data after the bytecode or marshal format has changed, - go to ../Tools/freeze/ and freeze the hello.py file; then copy and paste + go to ../Tools/freeze/ and freeze the flag.py file; then copy and paste the appropriate bytes from M___main__.c. */ static unsigned char M___hello__[] = { - 99,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0, - 0,115,9,0,0,0,100,0,0,71,72,100,1,0,83,40, - 2,0,0,0,115,14,0,0,0,72,101,108,108,111,32,119, - 111,114,108,100,46,46,46,78,40,0,0,0,0,40,0,0, - 0,0,40,0,0,0,0,40,0,0,0,0,115,8,0,0, - 0,104,101,108,108,111,46,112,121,115,1,0,0,0,63,1, - 0,0,0,115,0,0,0,0, + 99,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0, + 0,64,0,0,0,115,20,0,0,0,100,2,0,90,1,0, + 101,2,0,100,0,0,131,1,0,1,100,1,0,83,40,3, + 0,0,0,117,12,0,0,0,72,101,108,108,111,32,119,111, + 114,108,100,33,78,84,40,3,0,0,0,117,4,0,0,0, + 84,114,117,101,117,11,0,0,0,105,110,105,116,105,97,108, + 105,122,101,100,117,5,0,0,0,112,114,105,110,116,40,0, + 0,0,0,40,0,0,0,0,40,0,0,0,0,117,7,0, + 0,0,102,108,97,103,46,112,121,117,8,0,0,0,60,109, + 111,100,117,108,101,62,1,0,0,0,115,2,0,0,0,6, + 1, }; #define SIZE (int)sizeof(M___hello__) diff --git a/Python/frozenmain.c b/Python/frozenmain.c index d31da07708..f08caf23b9 100644 --- a/Python/frozenmain.c +++ b/Python/frozenmain.c @@ -2,6 +2,7 @@ /* Python interpreter main program for frozen scripts */ #include "Python.h" +#include <locale.h> #ifdef MS_WINDOWS extern void PyWinFreeze_ExeInit(void); @@ -15,9 +16,13 @@ int Py_FrozenMain(int argc, char **argv) { char *p; - int n, sts; + int i, n, sts; int inspect = 0; int unbuffered = 0; + char *oldloc; + wchar_t **argv_copy = PyMem_Malloc(sizeof(wchar_t*)*argc); + /* We need a second copies, as Python might modify the first one. */ + wchar_t **argv_copy2 = PyMem_Malloc(sizeof(wchar_t*)*argc); Py_FrozenFlag = 1; /* Suppress errors from getpath.c */ @@ -32,10 +37,42 @@ Py_FrozenMain(int argc, char **argv) setbuf(stderr, (char *)NULL); } + if (!argv_copy) { + fprintf(stderr, "out of memory\n"); + return 1; + } + + oldloc = setlocale(LC_ALL, NULL); + setlocale(LC_ALL, ""); + for (i = 0; i < argc; i++) { +#ifdef HAVE_BROKEN_MBSTOWCS + size_t argsize = strlen(argv[i]); +#else + size_t argsize = mbstowcs(NULL, argv[i], 0); +#endif + size_t count; + if (argsize == (size_t)-1) { + fprintf(stderr, "Could not convert argument %d to string\n", i); + return 1; + } + argv_copy[i] = PyMem_Malloc((argsize+1)*sizeof(wchar_t)); + argv_copy2[i] = argv_copy[i]; + if (!argv_copy[i]) { + fprintf(stderr, "out of memory\n"); + return 1; + } + count = mbstowcs(argv_copy[i], argv[i], argsize+1); + if (count == (size_t)-1) { + fprintf(stderr, "Could not convert argument %d to string\n", i); + return 1; + } + } + setlocale(LC_ALL, oldloc); + #ifdef MS_WINDOWS PyInitFrozenExtensions(); #endif /* MS_WINDOWS */ - Py_SetProgramName(argv[0]); + Py_SetProgramName(argv_copy[0]); Py_Initialize(); #ifdef MS_WINDOWS PyWinFreeze_ExeInit(); @@ -45,7 +82,7 @@ Py_FrozenMain(int argc, char **argv) fprintf(stderr, "Python %s\n%s\n", Py_GetVersion(), Py_GetCopyright()); - PySys_SetArgv(argc, argv); + PySys_SetArgv(argc, argv_copy); n = PyImport_ImportFrozenModule("__main__"); if (n == 0) @@ -64,5 +101,10 @@ Py_FrozenMain(int argc, char **argv) PyWinFreeze_ExeTerm(); #endif Py_Finalize(); + for (i = 0; i < argc; i++) { + PyMem_Free(argv_copy2[i]); + } + PyMem_Free(argv_copy); + PyMem_Free(argv_copy2); return sts; } diff --git a/Python/future.c b/Python/future.c index 0e68845981..d24ae416ff 100644 --- a/Python/future.c +++ b/Python/future.c @@ -4,7 +4,6 @@ #include "token.h" #include "graminit.h" #include "code.h" -#include "compile.h" #include "symtable.h" #define UNDEFINED_FUTURE_FEATURE "future feature %.100s is not defined" @@ -22,7 +21,7 @@ future_check_features(PyFutureFeatures *ff, stmt_ty s, const char *filename) names = s->v.ImportFrom.names; for (i = 0; i < asdl_seq_LEN(names); i++) { alias_ty name = (alias_ty)asdl_seq_GET(names, i); - const char *feature = PyString_AsString(name->name); + const char *feature = _PyUnicode_AsString(name->name); if (!feature) return 0; if (strcmp(feature, FUTURE_NESTED_SCOPES) == 0) { @@ -30,24 +29,26 @@ future_check_features(PyFutureFeatures *ff, stmt_ty s, const char *filename) } else if (strcmp(feature, FUTURE_GENERATORS) == 0) { continue; } else if (strcmp(feature, FUTURE_DIVISION) == 0) { - ff->ff_features |= CO_FUTURE_DIVISION; + continue; } else if (strcmp(feature, FUTURE_ABSOLUTE_IMPORT) == 0) { - ff->ff_features |= CO_FUTURE_ABSOLUTE_IMPORT; + continue; } else if (strcmp(feature, FUTURE_WITH_STATEMENT) == 0) { - ff->ff_features |= CO_FUTURE_WITH_STATEMENT; + continue; } else if (strcmp(feature, FUTURE_PRINT_FUNCTION) == 0) { - ff->ff_features |= CO_FUTURE_PRINT_FUNCTION; + continue; } else if (strcmp(feature, FUTURE_UNICODE_LITERALS) == 0) { - ff->ff_features |= CO_FUTURE_UNICODE_LITERALS; + continue; + } else if (strcmp(feature, FUTURE_BARRY_AS_BDFL) == 0) { + ff->ff_features |= CO_FUTURE_BARRY_AS_BDFL; } else if (strcmp(feature, "braces") == 0) { PyErr_SetString(PyExc_SyntaxError, "not a chance"); - PyErr_SyntaxLocation(filename, s->lineno); + PyErr_SyntaxLocationEx(filename, s->lineno, s->col_offset); return 0; } else { PyErr_Format(PyExc_SyntaxError, UNDEFINED_FUTURE_FEATURE, feature); - PyErr_SyntaxLocation(filename, s->lineno); + PyErr_SyntaxLocationEx(filename, s->lineno, s->col_offset); return 0; } } @@ -86,13 +87,12 @@ future_parse(PyFutureFeatures *ff, mod_ty mod, const char *filename) if (s->kind == ImportFrom_kind) { identifier modname = s->v.ImportFrom.module; - if (modname && PyString_GET_SIZE(modname) == 10 && - !strcmp(PyString_AS_STRING(modname), "__future__")) { + if (modname && + !PyUnicode_CompareWithASCIIString(modname, "__future__")) { if (done) { PyErr_SetString(PyExc_SyntaxError, ERR_LATE_FUTURE); - PyErr_SyntaxLocation(filename, - s->lineno); + PyErr_SyntaxLocationEx(filename, s->lineno, s->col_offset); return 0; } if (!future_check_features(ff, s, filename)) diff --git a/Python/getargs.c b/Python/getargs.c index 81a27217bb..a77bb05529 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -105,15 +105,7 @@ PyArg_VaParse(PyObject *args, const char *format, va_list va) { va_list lva; -#ifdef VA_LIST_IS_ARRAY - memcpy(lva, va, sizeof(va_list)); -#else -#ifdef __va_copy - __va_copy(lva, va); -#else - lva = va; -#endif -#endif + Py_VA_COPY(lva, va); return vgetargs1(args, format, &lva, 0); } @@ -123,15 +115,7 @@ _PyArg_VaParse_SizeT(PyObject *args, char *format, va_list va) { va_list lva; -#ifdef VA_LIST_IS_ARRAY - memcpy(lva, va, sizeof(va_list)); -#else -#ifdef __va_copy - __va_copy(lva, va); -#else - lva = va; -#endif -#endif + Py_VA_COPY(lva, va); return vgetargs1(args, format, &lva, FLAG_SIZE_T); } @@ -141,13 +125,14 @@ _PyArg_VaParse_SizeT(PyObject *args, char *format, va_list va) #define GETARGS_CAPSULE_NAME_CLEANUP_PTR "getargs.cleanup_ptr" #define GETARGS_CAPSULE_NAME_CLEANUP_BUFFER "getargs.cleanup_buffer" +#define GETARGS_CAPSULE_NAME_CLEANUP_CONVERT "getargs.cleanup_convert" static void cleanup_ptr(PyObject *self) { void *ptr = PyCapsule_GetPointer(self, GETARGS_CAPSULE_NAME_CLEANUP_PTR); if (ptr) { - PyMem_FREE(ptr); + PyMem_FREE(ptr); } } @@ -161,10 +146,19 @@ cleanup_buffer(PyObject *self) } static int -addcleanup(void *ptr, PyObject **freelist, PyCapsule_Destructor destr) +addcleanup(void *ptr, PyObject **freelist, int is_buffer) { PyObject *cobj; const char *name; + PyCapsule_Destructor destr; + + if (is_buffer) { + destr = cleanup_buffer; + name = GETARGS_CAPSULE_NAME_CLEANUP_BUFFER; + } else { + destr = cleanup_ptr; + name = GETARGS_CAPSULE_NAME_CLEANUP_PTR; + } if (!*freelist) { *freelist = PyList_New(0); @@ -174,13 +168,6 @@ addcleanup(void *ptr, PyObject **freelist, PyCapsule_Destructor destr) } } - if (destr == cleanup_ptr) { - name = GETARGS_CAPSULE_NAME_CLEANUP_PTR; - } else if (destr == cleanup_buffer) { - name = GETARGS_CAPSULE_NAME_CLEANUP_BUFFER; - } else { - return -1; - } cobj = PyCapsule_New(ptr, name, destr); if (!cobj) { destr(ptr); @@ -194,6 +181,46 @@ addcleanup(void *ptr, PyObject **freelist, PyCapsule_Destructor destr) return 0; } +static void +cleanup_convert(PyObject *self) +{ + typedef int (*destr_t)(PyObject *, void *); + destr_t destr = (destr_t)PyCapsule_GetContext(self); + void *ptr = PyCapsule_GetPointer(self, + GETARGS_CAPSULE_NAME_CLEANUP_CONVERT); + if (ptr && destr) + destr(NULL, ptr); +} + +static int +addcleanup_convert(void *ptr, PyObject **freelist, int (*destr)(PyObject*,void*)) +{ + PyObject *cobj; + if (!*freelist) { + *freelist = PyList_New(0); + if (!*freelist) { + destr(NULL, ptr); + return -1; + } + } + cobj = PyCapsule_New(ptr, GETARGS_CAPSULE_NAME_CLEANUP_CONVERT, + cleanup_convert); + if (!cobj) { + destr(NULL, ptr); + return -1; + } + if (PyCapsule_SetContext(cobj, destr) == -1) { + /* This really should not happen. */ + Py_FatalError("capsule refused setting of context."); + } + if (PyList_Append(*freelist, cobj)) { + Py_DECREF(cobj); /* This will also call destr. */ + return -1; + } + Py_DECREF(cobj); + return 0; +} + static int cleanreturn(int retval, PyObject *freelist) { @@ -448,7 +475,7 @@ converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, n++; } - if (!PySequence_Check(arg) || PyString_Check(arg)) { + if (!PySequence_Check(arg) || PyBytes_Check(arg)) { levels[0] = 0; PyOS_snprintf(msgbuf, bufsize, toplevel ? "expected %d arguments, not %.50s" : @@ -541,27 +568,14 @@ converterr(const char *expected, PyObject *arg, char *msgbuf, size_t bufsize) #define CONV_UNICODE "(unicode conversion error)" -/* explicitly check for float arguments when integers are expected. For now - * signal a warning. Returns true if an exception was raised. */ -static int -float_argument_warning(PyObject *arg) -{ - if (PyFloat_Check(arg) && - PyErr_Warn(PyExc_DeprecationWarning, - "integer argument expected, got float" )) - return 1; - else - return 0; -} - -/* explicitly check for float arguments when integers are expected. Raises - TypeError and returns true for float arguments. */ +/* Explicitly check for float arguments when integers are expected. + Return 1 for error, 0 if ok. */ static int float_argument_error(PyObject *arg) { if (PyFloat_Check(arg)) { PyErr_SetString(PyExc_TypeError, - "integer argument expected, got float"); + "integer argument expected, got float" ); return 1; } else @@ -597,12 +611,11 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, *q=s; \ } #define BUFFER_LEN ((flags & FLAG_SIZE_T) ? *q2:*q) +#define RETURN_ERR_OCCURRED return msgbuf const char *format = *p_format; char c = *format++; -#ifdef Py_USING_UNICODE PyObject *uarg; -#endif switch (c) { @@ -610,19 +623,19 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, char *p = va_arg(*p_va, char *); long ival; if (float_argument_error(arg)) - return converterr("integer<b>", arg, msgbuf, bufsize); - ival = PyInt_AsLong(arg); + RETURN_ERR_OCCURRED; + ival = PyLong_AsLong(arg); if (ival == -1 && PyErr_Occurred()) - return converterr("integer<b>", arg, msgbuf, bufsize); + RETURN_ERR_OCCURRED; else if (ival < 0) { PyErr_SetString(PyExc_OverflowError, - "unsigned byte integer is less than minimum"); - return converterr("integer<b>", arg, msgbuf, bufsize); + "unsigned byte integer is less than minimum"); + RETURN_ERR_OCCURRED; } else if (ival > UCHAR_MAX) { PyErr_SetString(PyExc_OverflowError, - "unsigned byte integer is greater than maximum"); - return converterr("integer<b>", arg, msgbuf, bufsize); + "unsigned byte integer is greater than maximum"); + RETURN_ERR_OCCURRED; } else *p = (unsigned char) ival; @@ -634,10 +647,10 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, char *p = va_arg(*p_va, char *); long ival; if (float_argument_error(arg)) - return converterr("integer<B>", arg, msgbuf, bufsize); - ival = PyInt_AsUnsignedLongMask(arg); + RETURN_ERR_OCCURRED; + ival = PyLong_AsUnsignedLongMask(arg); if (ival == -1 && PyErr_Occurred()) - return converterr("integer<B>", arg, msgbuf, bufsize); + RETURN_ERR_OCCURRED; else *p = (unsigned char) ival; break; @@ -647,19 +660,19 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, short *p = va_arg(*p_va, short *); long ival; if (float_argument_error(arg)) - return converterr("integer<h>", arg, msgbuf, bufsize); - ival = PyInt_AsLong(arg); + RETURN_ERR_OCCURRED; + ival = PyLong_AsLong(arg); if (ival == -1 && PyErr_Occurred()) - return converterr("integer<h>", arg, msgbuf, bufsize); + RETURN_ERR_OCCURRED; else if (ival < SHRT_MIN) { PyErr_SetString(PyExc_OverflowError, - "signed short integer is less than minimum"); - return converterr("integer<h>", arg, msgbuf, bufsize); + "signed short integer is less than minimum"); + RETURN_ERR_OCCURRED; } else if (ival > SHRT_MAX) { PyErr_SetString(PyExc_OverflowError, - "signed short integer is greater than maximum"); - return converterr("integer<h>", arg, msgbuf, bufsize); + "signed short integer is greater than maximum"); + RETURN_ERR_OCCURRED; } else *p = (short) ival; @@ -671,10 +684,10 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, unsigned short *p = va_arg(*p_va, unsigned short *); long ival; if (float_argument_error(arg)) - return converterr("integer<H>", arg, msgbuf, bufsize); - ival = PyInt_AsUnsignedLongMask(arg); + RETURN_ERR_OCCURRED; + ival = PyLong_AsUnsignedLongMask(arg); if (ival == -1 && PyErr_Occurred()) - return converterr("integer<H>", arg, msgbuf, bufsize); + RETURN_ERR_OCCURRED; else *p = (unsigned short) ival; break; @@ -684,19 +697,19 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, int *p = va_arg(*p_va, int *); long ival; if (float_argument_error(arg)) - return converterr("integer<i>", arg, msgbuf, bufsize); - ival = PyInt_AsLong(arg); + RETURN_ERR_OCCURRED; + ival = PyLong_AsLong(arg); if (ival == -1 && PyErr_Occurred()) - return converterr("integer<i>", arg, msgbuf, bufsize); + RETURN_ERR_OCCURRED; else if (ival > INT_MAX) { PyErr_SetString(PyExc_OverflowError, - "signed integer is greater than maximum"); - return converterr("integer<i>", arg, msgbuf, bufsize); + "signed integer is greater than maximum"); + RETURN_ERR_OCCURRED; } else if (ival < INT_MIN) { PyErr_SetString(PyExc_OverflowError, - "signed integer is less than minimum"); - return converterr("integer<i>", arg, msgbuf, bufsize); + "signed integer is less than minimum"); + RETURN_ERR_OCCURRED; } else *p = ival; @@ -708,38 +721,40 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, unsigned int *p = va_arg(*p_va, unsigned int *); unsigned int ival; if (float_argument_error(arg)) - return converterr("integer<I>", arg, msgbuf, bufsize); - ival = (unsigned int)PyInt_AsUnsignedLongMask(arg); + RETURN_ERR_OCCURRED; + ival = (unsigned int)PyLong_AsUnsignedLongMask(arg); if (ival == (unsigned int)-1 && PyErr_Occurred()) - return converterr("integer<I>", arg, msgbuf, bufsize); + RETURN_ERR_OCCURRED; else *p = ival; break; } case 'n': /* Py_ssize_t */ -#if SIZEOF_SIZE_T != SIZEOF_LONG { + PyObject *iobj; Py_ssize_t *p = va_arg(*p_va, Py_ssize_t *); - Py_ssize_t ival; + Py_ssize_t ival = -1; if (float_argument_error(arg)) - return converterr("integer<n>", arg, msgbuf, bufsize); - ival = PyInt_AsSsize_t(arg); + RETURN_ERR_OCCURRED; + iobj = PyNumber_Index(arg); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } if (ival == -1 && PyErr_Occurred()) - return converterr("integer<n>", arg, msgbuf, bufsize); + RETURN_ERR_OCCURRED; *p = ival; break; } -#endif - /* Fall through from 'n' to 'l' if Py_ssize_t is int */ case 'l': {/* long int */ long *p = va_arg(*p_va, long *); long ival; if (float_argument_error(arg)) - return converterr("integer<l>", arg, msgbuf, bufsize); - ival = PyInt_AsLong(arg); + RETURN_ERR_OCCURRED; + ival = PyLong_AsLong(arg); if (ival == -1 && PyErr_Occurred()) - return converterr("integer<l>", arg, msgbuf, bufsize); + RETURN_ERR_OCCURRED; else *p = ival; break; @@ -748,9 +763,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'k': { /* long sized bitfield */ unsigned long *p = va_arg(*p_va, unsigned long *); unsigned long ival; - if (PyInt_Check(arg)) - ival = PyInt_AsUnsignedLongMask(arg); - else if (PyLong_Check(arg)) + if (PyLong_Check(arg)) ival = PyLong_AsUnsignedLongMask(arg); else return converterr("integer<k>", arg, msgbuf, bufsize); @@ -762,23 +775,20 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'L': {/* PY_LONG_LONG */ PY_LONG_LONG *p = va_arg( *p_va, PY_LONG_LONG * ); PY_LONG_LONG ival; - if (float_argument_warning(arg)) - return converterr("long<L>", arg, msgbuf, bufsize); + if (float_argument_error(arg)) + RETURN_ERR_OCCURRED; ival = PyLong_AsLongLong(arg); - if (ival == (PY_LONG_LONG)-1 && PyErr_Occurred() ) { - return converterr("long<L>", arg, msgbuf, bufsize); - } else { + if (ival == (PY_LONG_LONG)-1 && PyErr_Occurred()) + RETURN_ERR_OCCURRED; + else *p = ival; - } break; } case 'K': { /* long long sized bitfield */ unsigned PY_LONG_LONG *p = va_arg(*p_va, unsigned PY_LONG_LONG *); unsigned PY_LONG_LONG ival; - if (PyInt_Check(arg)) - ival = PyInt_AsUnsignedLongMask(arg); - else if (PyLong_Check(arg)) + if (PyLong_Check(arg)) ival = PyLong_AsUnsignedLongLongMask(arg); else return converterr("integer<K>", arg, msgbuf, bufsize); @@ -791,7 +801,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, float *p = va_arg(*p_va, float *); double dval = PyFloat_AsDouble(arg); if (PyErr_Occurred()) - return converterr("float<f>", arg, msgbuf, bufsize); + RETURN_ERR_OCCURRED; else *p = (float) dval; break; @@ -801,170 +811,124 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, double *p = va_arg(*p_va, double *); double dval = PyFloat_AsDouble(arg); if (PyErr_Occurred()) - return converterr("float<d>", arg, msgbuf, bufsize); + RETURN_ERR_OCCURRED; else *p = dval; break; } -#ifndef WITHOUT_COMPLEX case 'D': {/* complex double */ Py_complex *p = va_arg(*p_va, Py_complex *); Py_complex cval; cval = PyComplex_AsCComplex(arg); if (PyErr_Occurred()) - return converterr("complex<D>", arg, msgbuf, bufsize); + RETURN_ERR_OCCURRED; else *p = cval; break; } -#endif /* WITHOUT_COMPLEX */ case 'c': {/* char */ char *p = va_arg(*p_va, char *); - if (PyString_Check(arg) && PyString_Size(arg) == 1) - *p = PyString_AS_STRING(arg)[0]; + if (PyBytes_Check(arg) && PyBytes_Size(arg) == 1) + *p = PyBytes_AS_STRING(arg)[0]; else - return converterr("char", arg, msgbuf, bufsize); + return converterr("a byte string of length 1", arg, msgbuf, bufsize); break; } - case 's': {/* string */ - if (*format == '*') { - Py_buffer *p = (Py_buffer *)va_arg(*p_va, Py_buffer *); + case 'C': {/* unicode char */ + int *p = va_arg(*p_va, int *); + if (PyUnicode_Check(arg) && + PyUnicode_GET_SIZE(arg) == 1) + *p = PyUnicode_AS_UNICODE(arg)[0]; + else + return converterr("a unicode character", arg, msgbuf, bufsize); + break; + } - if (PyString_Check(arg)) { - PyBuffer_FillInfo(p, arg, - PyString_AS_STRING(arg), PyString_GET_SIZE(arg), - 1, 0); - } -#ifdef Py_USING_UNICODE - else if (PyUnicode_Check(arg)) { - uarg = UNICODE_DEFAULT_ENCODING(arg); - if (uarg == NULL) - return converterr(CONV_UNICODE, - arg, msgbuf, bufsize); - PyBuffer_FillInfo(p, arg, - PyString_AS_STRING(uarg), PyString_GET_SIZE(uarg), - 1, 0); - } -#endif - else { /* any buffer-like object */ - char *buf; - if (getbuffer(arg, p, &buf) < 0) - return converterr(buf, arg, msgbuf, bufsize); - } - if (addcleanup(p, freelist, cleanup_buffer)) { + /* XXX WAAAAH! 's', 'y', 'z', 'u', 'Z', 'e', 'w' codes all + need to be cleaned up! */ + + case 'y': {/* any buffer-like object, but not PyUnicode */ + void **p = (void **)va_arg(*p_va, char **); + char *buf; + Py_ssize_t count; + if (*format == '*') { + if (getbuffer(arg, (Py_buffer*)p, &buf) < 0) + return converterr(buf, arg, msgbuf, bufsize); + format++; + if (addcleanup(p, freelist, 1)) { return converterr( "(cleanup problem)", arg, msgbuf, bufsize); } - format++; - } else if (*format == '#') { - void **p = (void **)va_arg(*p_va, char **); + break; + } + count = convertbuffer(arg, p, &buf); + if (count < 0) + return converterr(buf, arg, msgbuf, bufsize); + if (*format == '#') { FETCH_SIZE; - - if (PyString_Check(arg)) { - *p = PyString_AS_STRING(arg); - STORE_SIZE(PyString_GET_SIZE(arg)); - } -#ifdef Py_USING_UNICODE - else if (PyUnicode_Check(arg)) { - uarg = UNICODE_DEFAULT_ENCODING(arg); - if (uarg == NULL) - return converterr(CONV_UNICODE, - arg, msgbuf, bufsize); - *p = PyString_AS_STRING(uarg); - STORE_SIZE(PyString_GET_SIZE(uarg)); - } -#endif - else { /* any buffer-like object */ - char *buf; - Py_ssize_t count = convertbuffer(arg, p, &buf); - if (count < 0) - return converterr(buf, arg, msgbuf, bufsize); - STORE_SIZE(count); - } + STORE_SIZE(count); format++; } else { - char **p = va_arg(*p_va, char **); - - if (PyString_Check(arg)) - *p = PyString_AS_STRING(arg); -#ifdef Py_USING_UNICODE - else if (PyUnicode_Check(arg)) { - uarg = UNICODE_DEFAULT_ENCODING(arg); - if (uarg == NULL) - return converterr(CONV_UNICODE, - arg, msgbuf, bufsize); - *p = PyString_AS_STRING(uarg); - } -#endif - else - return converterr("string", arg, msgbuf, bufsize); - if ((Py_ssize_t)strlen(*p) != PyString_Size(arg)) - return converterr("string without null bytes", - arg, msgbuf, bufsize); + if (strlen(*p) != count) + return converterr( + "bytes without null bytes", + arg, msgbuf, bufsize); } break; } - case 'z': {/* string, may be NULL (None) */ + case 's': /* text string */ + case 'z': /* text string or None */ + { if (*format == '*') { + /* "s*" or "z*" */ Py_buffer *p = (Py_buffer *)va_arg(*p_va, Py_buffer *); - if (arg == Py_None) + if (c == 'z' && arg == Py_None) PyBuffer_FillInfo(p, NULL, NULL, 0, 1, 0); - else if (PyString_Check(arg)) { - PyBuffer_FillInfo(p, arg, - PyString_AS_STRING(arg), PyString_GET_SIZE(arg), - 1, 0); - } -#ifdef Py_USING_UNICODE else if (PyUnicode_Check(arg)) { uarg = UNICODE_DEFAULT_ENCODING(arg); if (uarg == NULL) return converterr(CONV_UNICODE, arg, msgbuf, bufsize); PyBuffer_FillInfo(p, arg, - PyString_AS_STRING(uarg), PyString_GET_SIZE(uarg), + PyBytes_AS_STRING(uarg), PyBytes_GET_SIZE(uarg), 1, 0); } -#endif else { /* any buffer-like object */ char *buf; if (getbuffer(arg, p, &buf) < 0) return converterr(buf, arg, msgbuf, bufsize); } - if (addcleanup(p, freelist, cleanup_buffer)) { + if (addcleanup(p, freelist, 1)) { return converterr( "(cleanup problem)", arg, msgbuf, bufsize); } format++; } else if (*format == '#') { /* any buffer-like object */ + /* "s#" or "z#" */ void **p = (void **)va_arg(*p_va, char **); FETCH_SIZE; - if (arg == Py_None) { - *p = 0; + if (c == 'z' && arg == Py_None) { + *p = NULL; STORE_SIZE(0); } - else if (PyString_Check(arg)) { - *p = PyString_AS_STRING(arg); - STORE_SIZE(PyString_GET_SIZE(arg)); - } -#ifdef Py_USING_UNICODE else if (PyUnicode_Check(arg)) { uarg = UNICODE_DEFAULT_ENCODING(arg); if (uarg == NULL) return converterr(CONV_UNICODE, arg, msgbuf, bufsize); - *p = PyString_AS_STRING(uarg); - STORE_SIZE(PyString_GET_SIZE(uarg)); + *p = PyBytes_AS_STRING(uarg); + STORE_SIZE(PyBytes_GET_SIZE(uarg)); } -#endif else { /* any buffer-like object */ + /* XXX Really? */ char *buf; Py_ssize_t count = convertbuffer(arg, p, &buf); if (count < 0) @@ -973,56 +937,82 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } format++; } else { + /* "s" or "z" */ char **p = va_arg(*p_va, char **); + uarg = NULL; - if (arg == Py_None) - *p = 0; - else if (PyString_Check(arg)) - *p = PyString_AS_STRING(arg); -#ifdef Py_USING_UNICODE + if (c == 'z' && arg == Py_None) + *p = NULL; else if (PyUnicode_Check(arg)) { uarg = UNICODE_DEFAULT_ENCODING(arg); if (uarg == NULL) return converterr(CONV_UNICODE, arg, msgbuf, bufsize); - *p = PyString_AS_STRING(uarg); + *p = PyBytes_AS_STRING(uarg); } -#endif else - return converterr("string or None", + return converterr(c == 'z' ? "str or None" : "str", arg, msgbuf, bufsize); - if (*format == '#') { - FETCH_SIZE; - assert(0); /* XXX redundant with if-case */ - if (arg == Py_None) { - STORE_SIZE(0); - } else { - STORE_SIZE(PyString_Size(arg)); - } - format++; - } - else if (*p != NULL && - (Py_ssize_t)strlen(*p) != PyString_Size(arg)) + if (*p != NULL && uarg != NULL && + (Py_ssize_t) strlen(*p) != PyBytes_GET_SIZE(uarg)) return converterr( - "string without null bytes or None", + c == 'z' ? "str without null bytes or None" + : "str without null bytes", arg, msgbuf, bufsize); } break; } + case 'u': /* raw unicode buffer (Py_UNICODE *) */ + case 'Z': /* raw unicode buffer or None */ + { + if (*format == '#') { /* any buffer-like object */ + /* "s#" or "Z#" */ + Py_UNICODE **p = va_arg(*p_va, Py_UNICODE **); + FETCH_SIZE; + + if (c == 'Z' && arg == Py_None) { + *p = NULL; + STORE_SIZE(0); + } + else if (PyUnicode_Check(arg)) { + *p = PyUnicode_AS_UNICODE(arg); + STORE_SIZE(PyUnicode_GET_SIZE(arg)); + } + else + return converterr("str or None", arg, msgbuf, bufsize); + format++; + } else { + /* "s" or "Z" */ + Py_UNICODE **p = va_arg(*p_va, Py_UNICODE **); + + if (c == 'Z' && arg == Py_None) + *p = NULL; + else if (PyUnicode_Check(arg)) { + *p = PyUnicode_AS_UNICODE(arg); + if (Py_UNICODE_strlen(*p) != PyUnicode_GET_SIZE(arg)) + return converterr( + "str without null character or None", + arg, msgbuf, bufsize); + } else + return converterr(c == 'Z' ? "str or None" : "str", + arg, msgbuf, bufsize); + } + break; + } + case 'e': {/* encoded string */ char **buffer; const char *encoding; PyObject *s; - Py_ssize_t size; int recode_strings; + Py_ssize_t size; + const char *ptr; /* Get 'e' parameter: the encoding name */ encoding = (const char *)va_arg(*p_va, const char *); -#ifdef Py_USING_UNICODE if (encoding == NULL) encoding = PyUnicode_GetDefaultEncoding(); -#endif /* Get output buffer parameter: 's' (recode all objects via Unicode) or @@ -1043,12 +1033,15 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, arg, msgbuf, bufsize); /* Encode object */ - if (!recode_strings && PyString_Check(arg)) { + if (!recode_strings && + (PyBytes_Check(arg) || PyByteArray_Check(arg))) { s = arg; Py_INCREF(s); + if (PyObject_AsCharBuffer(s, &ptr, &size) < 0) + return converterr("(AsCharBuffer failed)", + arg, msgbuf, bufsize); } else { -#ifdef Py_USING_UNICODE PyObject *u; /* Convert object to Unicode */ @@ -1066,17 +1059,17 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, if (s == NULL) return converterr("(encoding failed)", arg, msgbuf, bufsize); - if (!PyString_Check(s)) { + if (!PyBytes_Check(s)) { Py_DECREF(s); return converterr( - "(encoder failed to return a string)", + "(encoder failed to return bytes)", arg, msgbuf, bufsize); } -#else - return converterr("string<e>", arg, msgbuf, bufsize); -#endif + size = PyBytes_GET_SIZE(s); + ptr = PyBytes_AS_STRING(s); + if (ptr == NULL) + ptr = ""; } - size = PyString_GET_SIZE(s); /* Write output; output is guaranteed to be 0-terminated */ if (*format == '#') { @@ -1114,11 +1107,10 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, *buffer = PyMem_NEW(char, size + 1); if (*buffer == NULL) { Py_DECREF(s); - return converterr( - "(memory error)", - arg, msgbuf, bufsize); + PyErr_NoMemory(); + RETURN_ERR_OCCURRED; } - if (addcleanup(*buffer, freelist, cleanup_ptr)) { + if (addcleanup(*buffer, freelist, 0)) { Py_DECREF(s); return converterr( "(cleanup problem)", @@ -1132,9 +1124,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, arg, msgbuf, bufsize); } } - memcpy(*buffer, - PyString_AS_STRING(s), - size + 1); + memcpy(*buffer, ptr, size+1); STORE_SIZE(size); } else { /* Using a 0-terminated buffer: @@ -1150,8 +1140,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, PyMem_Free()ing it after usage */ - if ((Py_ssize_t)strlen(PyString_AS_STRING(s)) - != size) { + if ((Py_ssize_t)strlen(ptr) != size) { Py_DECREF(s); return converterr( "encoded string without NULL bytes", @@ -1160,66 +1149,46 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, *buffer = PyMem_NEW(char, size + 1); if (*buffer == NULL) { Py_DECREF(s); - return converterr("(memory error)", - arg, msgbuf, bufsize); + PyErr_NoMemory(); + RETURN_ERR_OCCURRED; } - if (addcleanup(*buffer, freelist, cleanup_ptr)) { + if (addcleanup(*buffer, freelist, 0)) { Py_DECREF(s); return converterr("(cleanup problem)", arg, msgbuf, bufsize); } - memcpy(*buffer, - PyString_AS_STRING(s), - size + 1); + memcpy(*buffer, ptr, size+1); } Py_DECREF(s); break; } -#ifdef Py_USING_UNICODE - case 'u': {/* raw unicode buffer (Py_UNICODE *) */ - if (*format == '#') { /* any buffer-like object */ - void **p = (void **)va_arg(*p_va, char **); - FETCH_SIZE; - if (PyUnicode_Check(arg)) { - *p = PyUnicode_AS_UNICODE(arg); - STORE_SIZE(PyUnicode_GET_SIZE(arg)); - } - else { - return converterr("cannot convert raw buffers", - arg, msgbuf, bufsize); - } - format++; - } else { - Py_UNICODE **p = va_arg(*p_va, Py_UNICODE **); - if (PyUnicode_Check(arg)) - *p = PyUnicode_AS_UNICODE(arg); - else - return converterr("unicode", arg, msgbuf, bufsize); - } + case 'S': { /* PyBytes object */ + PyObject **p = va_arg(*p_va, PyObject **); + if (PyBytes_Check(arg)) + *p = arg; + else + return converterr("bytes", arg, msgbuf, bufsize); break; } -#endif - case 'S': { /* string object */ + case 'Y': { /* PyByteArray object */ PyObject **p = va_arg(*p_va, PyObject **); - if (PyString_Check(arg)) + if (PyByteArray_Check(arg)) *p = arg; else - return converterr("string", arg, msgbuf, bufsize); + return converterr("bytearray", arg, msgbuf, bufsize); break; } -#ifdef Py_USING_UNICODE - case 'U': { /* Unicode object */ + case 'U': { /* PyUnicode object */ PyObject **p = va_arg(*p_va, PyObject **); if (PyUnicode_Check(arg)) *p = arg; else - return converterr("unicode", arg, msgbuf, bufsize); + return converterr("str", arg, msgbuf, bufsize); break; } -#endif case 'O': { /* object */ PyTypeObject *type; @@ -1234,25 +1203,19 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, return converterr(type->tp_name, arg, msgbuf, bufsize); } - else if (*format == '?') { - inquiry pred = va_arg(*p_va, inquiry); - p = va_arg(*p_va, PyObject **); - format++; - if ((*pred)(arg)) - *p = arg; - else - return converterr("(unspecified)", - arg, msgbuf, bufsize); - - } else if (*format == '&') { typedef int (*converter)(PyObject *, void *); converter convert = va_arg(*p_va, converter); void *addr = va_arg(*p_va, void *); + int res; format++; - if (! (*convert)(arg, addr)) + if (! (res = (*convert)(arg, addr))) return converterr("(unspecified)", arg, msgbuf, bufsize); + if (res == Py_CLEANUP_SUPPORTED && + addcleanup_convert(addr, freelist, convert) == -1) + return converterr("(cleanup problem)", + arg, msgbuf, bufsize); } else { p = va_arg(*p_va, PyObject **); @@ -1262,92 +1225,29 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } - case 'w': { /* memory buffer, read-write access */ + case 'w': { /* "w*": memory buffer, read-write access */ void **p = va_arg(*p_va, void **); - void *res; - PyBufferProcs *pb = arg->ob_type->tp_as_buffer; - Py_ssize_t count; - if (pb && pb->bf_releasebuffer && *format != '*') - /* Buffer must be released, yet caller does not use - the Py_buffer protocol. */ - return converterr("pinned buffer", arg, msgbuf, bufsize); - - if (pb && pb->bf_getbuffer && *format == '*') { - /* Caller is interested in Py_buffer, and the object - supports it directly. */ - format++; - if (pb->bf_getbuffer(arg, (Py_buffer*)p, PyBUF_WRITABLE) < 0) { - PyErr_Clear(); - return converterr("read-write buffer", arg, msgbuf, bufsize); - } - if (addcleanup(p, freelist, cleanup_buffer)) { - return converterr( - "(cleanup problem)", - arg, msgbuf, bufsize); - } - if (!PyBuffer_IsContiguous((Py_buffer*)p, 'C')) - return converterr("contiguous buffer", arg, msgbuf, bufsize); - break; - } + if (*format != '*') + return converterr( + "invalid use of 'w' format character", + arg, msgbuf, bufsize); + format++; - if (pb == NULL || - pb->bf_getwritebuffer == NULL || - pb->bf_getsegcount == NULL) + /* Caller is interested in Py_buffer, and the object + supports it directly. */ + if (PyObject_GetBuffer(arg, (Py_buffer*)p, PyBUF_WRITABLE) < 0) { + PyErr_Clear(); return converterr("read-write buffer", arg, msgbuf, bufsize); - if ((*pb->bf_getsegcount)(arg, NULL) != 1) - return converterr("single-segment read-write buffer", - arg, msgbuf, bufsize); - if ((count = pb->bf_getwritebuffer(arg, 0, &res)) < 0) - return converterr("(unspecified)", arg, msgbuf, bufsize); - if (*format == '*') { - PyBuffer_FillInfo((Py_buffer*)p, arg, res, count, 1, 0); - format++; } - else { - *p = res; - if (*format == '#') { - FETCH_SIZE; - STORE_SIZE(count); - format++; - } + if (!PyBuffer_IsContiguous((Py_buffer*)p, 'C')) { + PyBuffer_Release((Py_buffer*)p); + return converterr("contiguous buffer", arg, msgbuf, bufsize); } - break; - } - - case 't': { /* 8-bit character buffer, read-only access */ - char **p = va_arg(*p_va, char **); - PyBufferProcs *pb = arg->ob_type->tp_as_buffer; - Py_ssize_t count; - - if (*format++ != '#') - return converterr( - "invalid use of 't' format character", - arg, msgbuf, bufsize); - if (!PyType_HasFeature(arg->ob_type, - Py_TPFLAGS_HAVE_GETCHARBUFFER) || - pb == NULL || pb->bf_getcharbuffer == NULL || - pb->bf_getsegcount == NULL) - return converterr( - "string or read-only character buffer", - arg, msgbuf, bufsize); - - if (pb->bf_getsegcount(arg, NULL) != 1) - return converterr( - "string or single-segment read-only buffer", - arg, msgbuf, bufsize); - - if (pb->bf_releasebuffer) + if (addcleanup(p, freelist, 1)) { return converterr( - "string or pinned buffer", + "(cleanup problem)", arg, msgbuf, bufsize); - - count = pb->bf_getcharbuffer(arg, 0, p); - if (count < 0) - return converterr("(unspecified)", arg, msgbuf, bufsize); - { - FETCH_SIZE; - STORE_SIZE(count); } break; } @@ -1359,58 +1259,47 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, *p_format = format; return NULL; + +#undef FETCH_SIZE +#undef STORE_SIZE +#undef BUFFER_LEN +#undef RETURN_ERR_OCCURRED } static Py_ssize_t convertbuffer(PyObject *arg, void **p, char **errmsg) { - PyBufferProcs *pb = arg->ob_type->tp_as_buffer; + PyBufferProcs *pb = Py_TYPE(arg)->tp_as_buffer; Py_ssize_t count; - if (pb == NULL || - pb->bf_getreadbuffer == NULL || - pb->bf_getsegcount == NULL || - pb->bf_releasebuffer != NULL) { - *errmsg = "string or read-only buffer"; + Py_buffer view; + + *errmsg = NULL; + *p = NULL; + if (pb != NULL && pb->bf_releasebuffer != NULL) { + *errmsg = "read-only pinned buffer"; return -1; } - if ((*pb->bf_getsegcount)(arg, NULL) != 1) { - *errmsg = "string or single-segment read-only buffer"; + + if (getbuffer(arg, &view, errmsg) < 0) return -1; - } - if ((count = (*pb->bf_getreadbuffer)(arg, 0, p)) < 0) { - *errmsg = "(unspecified)"; - } + count = view.len; + *p = view.buf; + PyBuffer_Release(&view); return count; } static int getbuffer(PyObject *arg, Py_buffer *view, char **errmsg) { - void *buf; - Py_ssize_t count; - PyBufferProcs *pb = arg->ob_type->tp_as_buffer; - if (pb == NULL) { - *errmsg = "string or buffer"; + if (PyObject_GetBuffer(arg, view, PyBUF_SIMPLE) != 0) { + *errmsg = "bytes or buffer"; return -1; } - if (pb->bf_getbuffer) { - if (pb->bf_getbuffer(arg, view, 0) < 0) { - *errmsg = "convertible to a buffer"; - return -1; - } - if (!PyBuffer_IsContiguous(view, 'C')) { - *errmsg = "contiguous buffer"; - return -1; - } - return 0; - } - - count = convertbuffer(arg, &buf, errmsg); - if (count < 0) { - *errmsg = "convertible to a buffer"; - return count; + if (!PyBuffer_IsContiguous(view, 'C')) { + PyBuffer_Release(view); + *errmsg = "contiguous buffer"; + return -1; } - PyBuffer_FillInfo(view, arg, buf, count, 1, 0); return 0; } @@ -1486,15 +1375,7 @@ PyArg_VaParseTupleAndKeywords(PyObject *args, return 0; } -#ifdef VA_LIST_IS_ARRAY - memcpy(lva, va, sizeof(va_list)); -#else -#ifdef __va_copy - __va_copy(lva, va); -#else - lva = va; -#endif -#endif + Py_VA_COPY(lva, va); retval = vgetargskeywords(args, keywords, format, kwlist, &lva, 0); return retval; @@ -1518,21 +1399,28 @@ _PyArg_VaParseTupleAndKeywords_SizeT(PyObject *args, return 0; } -#ifdef VA_LIST_IS_ARRAY - memcpy(lva, va, sizeof(va_list)); -#else -#ifdef __va_copy - __va_copy(lva, va); -#else - lva = va; -#endif -#endif + Py_VA_COPY(lva, va); retval = vgetargskeywords(args, keywords, format, kwlist, &lva, FLAG_SIZE_T); return retval; } +int +PyArg_ValidateKeywordArguments(PyObject *kwargs) +{ + if (!PyDict_Check(kwargs)) { + PyErr_BadInternalCall(); + return 0; + } + if (!_PyDict_HasOnlyStringKeys(kwargs)) { + PyErr_SetString(PyExc_TypeError, + "keyword arguments must be strings"); + return 0; + } + return 1; +} + #define IS_END_OF_FORMAT(c) (c == '\0' || c == ';' || c == ':') static int @@ -1661,23 +1549,26 @@ vgetargskeywords(PyObject *args, PyObject *keywords, const char *format, while (PyDict_Next(keywords, &pos, &key, &value)) { int match = 0; char *ks; - if (!PyString_Check(key)) { + if (!PyUnicode_Check(key)) { PyErr_SetString(PyExc_TypeError, "keywords must be strings"); return cleanreturn(0, freelist); } - ks = PyString_AsString(key); - for (i = 0; i < len; i++) { - if (!strcmp(ks, kwlist[i])) { - match = 1; - break; + /* check that _PyUnicode_AsString() result is not NULL */ + ks = _PyUnicode_AsString(key); + if (ks != NULL) { + for (i = 0; i < len; i++) { + if (!strcmp(ks, kwlist[i])) { + match = 1; + break; + } } } if (!match) { PyErr_Format(PyExc_TypeError, - "'%s' is an invalid keyword " + "'%U' is an invalid keyword " "argument for this function", - ks); + key); return cleanreturn(0, freelist); } } @@ -1712,10 +1603,9 @@ skipitem(const char **p_format, va_list *p_va, int flags) #endif case 'f': /* float */ case 'd': /* double */ -#ifndef WITHOUT_COMPLEX case 'D': /* complex double */ -#endif case 'c': /* char */ + case 'C': /* unicode char */ { (void) va_arg(*p_va, void *); break; @@ -1741,10 +1631,9 @@ skipitem(const char **p_format, va_list *p_va, int flags) case 's': /* string */ case 'z': /* string or None */ -#ifdef Py_USING_UNICODE + case 'y': /* bytes */ case 'u': /* unicode string */ -#endif - case 't': /* buffer, read-only */ + case 'Z': /* unicode string or None */ case 'w': /* buffer, read-write */ { (void) va_arg(*p_va, char **); @@ -1754,7 +1643,7 @@ skipitem(const char **p_format, va_list *p_va, int flags) else (void) va_arg(*p_va, int *); format++; - } else if ((c == 's' || c == 'z') && *format == '*') { + } else if ((c == 's' || c == 'z' || c == 'y') && *format == '*') { format++; } break; @@ -1763,9 +1652,8 @@ skipitem(const char **p_format, va_list *p_va, int flags) /* object codes */ case 'S': /* string object */ -#ifdef Py_USING_UNICODE + case 'Y': /* string object */ case 'U': /* unicode string object */ -#endif { (void) va_arg(*p_va, PyObject **); break; diff --git a/Python/getcopyright.c b/Python/getcopyright.c index d91a879ae3..d1e2578747 100644 --- a/Python/getcopyright.c +++ b/Python/getcopyright.c @@ -2,7 +2,7 @@ #include "Python.h" -static char cprt[] = +static char cprt[] = "\ Copyright (c) 2001-2012 Python Software Foundation.\n\ All Rights Reserved.\n\ diff --git a/Python/getcwd.c b/Python/getcwd.c index 44dc9e69ab..4bedbd1f75 100644 --- a/Python/getcwd.c +++ b/Python/getcwd.c @@ -59,13 +59,14 @@ getcwd(char *buf, int size) { FILE *fp; char *p; + int sts; if (size <= 0) { errno = EINVAL; return NULL; } if ((fp = popen(PWD_CMD, "r")) == NULL) return NULL; - if (fgets(buf, size, fp) == NULL || pclose(fp) != 0) { + if (fgets(buf, size, fp) == NULL || (sts = pclose(fp)) != 0) { errno = EACCES; /* Most likely error */ return NULL; } diff --git a/Python/getopt.c b/Python/getopt.c index af5b03c4f4..5cf4cbd7bb 100644 --- a/Python/getopt.c +++ b/Python/getopt.c @@ -27,8 +27,11 @@ /* Modified to support --help and --version, as well as /? on Windows * by Georg Brandl. */ +#include <Python.h> #include <stdio.h> #include <string.h> +#include <wchar.h> +#include <pygetopt.h> #ifdef __cplusplus extern "C" { @@ -36,48 +39,49 @@ extern "C" { int _PyOS_opterr = 1; /* generate error messages */ int _PyOS_optind = 1; /* index into argv array */ -char *_PyOS_optarg = NULL; /* optional argument */ -static char *opt_ptr = ""; +wchar_t *_PyOS_optarg = NULL; /* optional argument */ + +static wchar_t *opt_ptr = L""; void _PyOS_ResetGetOpt(void) { _PyOS_opterr = 1; _PyOS_optind = 1; _PyOS_optarg = NULL; - opt_ptr = ""; + opt_ptr = L""; } -int _PyOS_GetOpt(int argc, char **argv, char *optstring) +int _PyOS_GetOpt(int argc, wchar_t **argv, wchar_t *optstring) { - char *ptr; - int option; + wchar_t *ptr; + wchar_t option; if (*opt_ptr == '\0') { if (_PyOS_optind >= argc) return -1; #ifdef MS_WINDOWS - else if (strcmp(argv[_PyOS_optind], "/?") == 0) { + else if (wcscmp(argv[_PyOS_optind], L"/?") == 0) { ++_PyOS_optind; return 'h'; } #endif - else if (argv[_PyOS_optind][0] != '-' || - argv[_PyOS_optind][1] == '\0' /* lone dash */ ) + else if (argv[_PyOS_optind][0] != L'-' || + argv[_PyOS_optind][1] == L'\0' /* lone dash */ ) return -1; - else if (strcmp(argv[_PyOS_optind], "--") == 0) { + else if (wcscmp(argv[_PyOS_optind], L"--") == 0) { ++_PyOS_optind; return -1; } - else if (strcmp(argv[_PyOS_optind], "--help") == 0) { + else if (wcscmp(argv[_PyOS_optind], L"--help") == 0) { ++_PyOS_optind; return 'h'; } - else if (strcmp(argv[_PyOS_optind], "--version") == 0) { + else if (wcscmp(argv[_PyOS_optind], L"--version") == 0) { ++_PyOS_optind; return 'V'; } @@ -86,7 +90,7 @@ int _PyOS_GetOpt(int argc, char **argv, char *optstring) opt_ptr = &argv[_PyOS_optind++][1]; } - if ((option = *opt_ptr++) == '\0') + if ((option = *opt_ptr++) == L'\0') return -1; if (option == 'J') { @@ -95,31 +99,23 @@ int _PyOS_GetOpt(int argc, char **argv, char *optstring) return '_'; } - if (option == 'X') { - if (_PyOS_opterr) - fprintf(stderr, - "-X is reserved for implementation-specific arguments\n"); - return '_'; - } - - if ((ptr = strchr(optstring, option)) == NULL) { + if ((ptr = wcschr(optstring, option)) == NULL) { if (_PyOS_opterr) - fprintf(stderr, "Unknown option: -%c\n", option); - + fprintf(stderr, "Unknown option: -%c\n", (char)option); return '_'; } - if (*(ptr + 1) == ':') { - if (*opt_ptr != '\0') { + if (*(ptr + 1) == L':') { + if (*opt_ptr != L'\0') { _PyOS_optarg = opt_ptr; - opt_ptr = ""; + opt_ptr = L""; } else { if (_PyOS_optind >= argc) { if (_PyOS_opterr) fprintf(stderr, - "Argument expected for the -%c option\n", option); + "Argument expected for the -%c option\n", (char)option); return '_'; } diff --git a/Python/graminit.c b/Python/graminit.c index 5db6bb5e53..cabc4d63b3 100644 --- a/Python/graminit.c +++ b/Python/graminit.c @@ -113,28 +113,37 @@ static arc arcs_6_1[1] = { static arc arcs_6_2[1] = { {22, 3}, }; -static arc arcs_6_3[1] = { +static arc arcs_6_3[2] = { {23, 4}, + {25, 5}, }; static arc arcs_6_4[1] = { - {24, 5}, + {24, 6}, }; static arc arcs_6_5[1] = { - {0, 5}, + {26, 7}, +}; +static arc arcs_6_6[1] = { + {25, 5}, +}; +static arc arcs_6_7[1] = { + {0, 7}, }; -static state states_6[6] = { +static state states_6[8] = { {1, arcs_6_0}, {1, arcs_6_1}, {1, arcs_6_2}, - {1, arcs_6_3}, + {2, arcs_6_3}, {1, arcs_6_4}, {1, arcs_6_5}, + {1, arcs_6_6}, + {1, arcs_6_7}, }; static arc arcs_7_0[1] = { {13, 1}, }; static arc arcs_7_1[2] = { - {25, 2}, + {27, 2}, {15, 3}, }; static arc arcs_7_2[1] = { @@ -150,125 +159,258 @@ static state states_7[4] = { {1, arcs_7_3}, }; static arc arcs_8_0[3] = { - {26, 1}, - {30, 2}, - {31, 3}, + {28, 1}, + {31, 2}, + {32, 3}, }; static arc arcs_8_1[3] = { - {27, 4}, - {29, 5}, + {29, 4}, + {30, 5}, {0, 1}, }; -static arc arcs_8_2[1] = { - {21, 6}, +static arc arcs_8_2[3] = { + {28, 6}, + {30, 7}, + {0, 2}, }; static arc arcs_8_3[1] = { - {21, 7}, + {28, 8}, }; static arc arcs_8_4[1] = { - {28, 8}, + {24, 9}, }; static arc arcs_8_5[4] = { - {26, 1}, - {30, 2}, - {31, 3}, + {28, 10}, + {31, 11}, + {32, 3}, {0, 5}, }; static arc arcs_8_6[2] = { - {29, 9}, + {30, 7}, {0, 6}, }; -static arc arcs_8_7[1] = { - {0, 7}, +static arc arcs_8_7[2] = { + {28, 12}, + {32, 3}, }; -static arc arcs_8_8[2] = { - {29, 5}, +static arc arcs_8_8[1] = { {0, 8}, }; -static arc arcs_8_9[1] = { - {31, 3}, +static arc arcs_8_9[2] = { + {30, 5}, + {0, 9}, +}; +static arc arcs_8_10[3] = { + {30, 5}, + {29, 4}, + {0, 10}, +}; +static arc arcs_8_11[3] = { + {28, 13}, + {30, 14}, + {0, 11}, }; -static state states_8[10] = { +static arc arcs_8_12[3] = { + {30, 7}, + {29, 15}, + {0, 12}, +}; +static arc arcs_8_13[2] = { + {30, 14}, + {0, 13}, +}; +static arc arcs_8_14[2] = { + {28, 16}, + {32, 3}, +}; +static arc arcs_8_15[1] = { + {24, 6}, +}; +static arc arcs_8_16[3] = { + {30, 14}, + {29, 17}, + {0, 16}, +}; +static arc arcs_8_17[1] = { + {24, 13}, +}; +static state states_8[18] = { {3, arcs_8_0}, {3, arcs_8_1}, - {1, arcs_8_2}, + {3, arcs_8_2}, {1, arcs_8_3}, {1, arcs_8_4}, {4, arcs_8_5}, {2, arcs_8_6}, - {1, arcs_8_7}, - {2, arcs_8_8}, - {1, arcs_8_9}, -}; -static arc arcs_9_0[2] = { + {2, arcs_8_7}, + {1, arcs_8_8}, + {2, arcs_8_9}, + {3, arcs_8_10}, + {3, arcs_8_11}, + {3, arcs_8_12}, + {2, arcs_8_13}, + {2, arcs_8_14}, + {1, arcs_8_15}, + {3, arcs_8_16}, + {1, arcs_8_17}, +}; +static arc arcs_9_0[1] = { {21, 1}, - {13, 2}, }; -static arc arcs_9_1[1] = { +static arc arcs_9_1[2] = { + {25, 2}, {0, 1}, }; static arc arcs_9_2[1] = { - {32, 3}, + {24, 3}, }; static arc arcs_9_3[1] = { - {15, 1}, + {0, 3}, }; static state states_9[4] = { - {2, arcs_9_0}, - {1, arcs_9_1}, + {1, arcs_9_0}, + {2, arcs_9_1}, {1, arcs_9_2}, {1, arcs_9_3}, }; -static arc arcs_10_0[1] = { - {26, 1}, +static arc arcs_10_0[3] = { + {34, 1}, + {31, 2}, + {32, 3}, }; -static arc arcs_10_1[2] = { - {29, 2}, +static arc arcs_10_1[3] = { + {29, 4}, + {30, 5}, {0, 1}, }; -static arc arcs_10_2[2] = { - {26, 1}, +static arc arcs_10_2[3] = { + {34, 6}, + {30, 7}, {0, 2}, }; -static state states_10[3] = { - {1, arcs_10_0}, - {2, arcs_10_1}, - {2, arcs_10_2}, +static arc arcs_10_3[1] = { + {34, 8}, }; -static arc arcs_11_0[2] = { - {3, 1}, - {4, 1}, +static arc arcs_10_4[1] = { + {24, 9}, +}; +static arc arcs_10_5[4] = { + {34, 10}, + {31, 11}, + {32, 3}, + {0, 5}, +}; +static arc arcs_10_6[2] = { + {30, 7}, + {0, 6}, +}; +static arc arcs_10_7[2] = { + {34, 12}, + {32, 3}, +}; +static arc arcs_10_8[1] = { + {0, 8}, +}; +static arc arcs_10_9[2] = { + {30, 5}, + {0, 9}, +}; +static arc arcs_10_10[3] = { + {30, 5}, + {29, 4}, + {0, 10}, +}; +static arc arcs_10_11[3] = { + {34, 13}, + {30, 14}, + {0, 11}, +}; +static arc arcs_10_12[3] = { + {30, 7}, + {29, 15}, + {0, 12}, +}; +static arc arcs_10_13[2] = { + {30, 14}, + {0, 13}, +}; +static arc arcs_10_14[2] = { + {34, 16}, + {32, 3}, +}; +static arc arcs_10_15[1] = { + {24, 6}, +}; +static arc arcs_10_16[3] = { + {30, 14}, + {29, 17}, + {0, 16}, +}; +static arc arcs_10_17[1] = { + {24, 13}, +}; +static state states_10[18] = { + {3, arcs_10_0}, + {3, arcs_10_1}, + {3, arcs_10_2}, + {1, arcs_10_3}, + {1, arcs_10_4}, + {4, arcs_10_5}, + {2, arcs_10_6}, + {2, arcs_10_7}, + {1, arcs_10_8}, + {2, arcs_10_9}, + {3, arcs_10_10}, + {3, arcs_10_11}, + {3, arcs_10_12}, + {2, arcs_10_13}, + {2, arcs_10_14}, + {1, arcs_10_15}, + {3, arcs_10_16}, + {1, arcs_10_17}, +}; +static arc arcs_11_0[1] = { + {21, 1}, }; static arc arcs_11_1[1] = { {0, 1}, }; static state states_11[2] = { - {2, arcs_11_0}, + {1, arcs_11_0}, {1, arcs_11_1}, }; -static arc arcs_12_0[1] = { - {33, 1}, +static arc arcs_12_0[2] = { + {3, 1}, + {4, 1}, +}; +static arc arcs_12_1[1] = { + {0, 1}, +}; +static state states_12[2] = { + {2, arcs_12_0}, + {1, arcs_12_1}, }; -static arc arcs_12_1[2] = { - {34, 2}, +static arc arcs_13_0[1] = { + {35, 1}, +}; +static arc arcs_13_1[2] = { + {36, 2}, {2, 3}, }; -static arc arcs_12_2[2] = { - {33, 1}, +static arc arcs_13_2[2] = { + {35, 1}, {2, 3}, }; -static arc arcs_12_3[1] = { +static arc arcs_13_3[1] = { {0, 3}, }; -static state states_12[4] = { - {1, arcs_12_0}, - {2, arcs_12_1}, - {2, arcs_12_2}, - {1, arcs_12_3}, +static state states_13[4] = { + {1, arcs_13_0}, + {2, arcs_13_1}, + {2, arcs_13_2}, + {1, arcs_13_3}, }; -static arc arcs_13_0[9] = { - {35, 1}, - {36, 1}, +static arc arcs_14_0[8] = { {37, 1}, {38, 1}, {39, 1}, @@ -276,49 +418,65 @@ static arc arcs_13_0[9] = { {41, 1}, {42, 1}, {43, 1}, + {44, 1}, }; -static arc arcs_13_1[1] = { +static arc arcs_14_1[1] = { {0, 1}, }; -static state states_13[2] = { - {9, arcs_13_0}, - {1, arcs_13_1}, +static state states_14[2] = { + {8, arcs_14_0}, + {1, arcs_14_1}, }; -static arc arcs_14_0[1] = { - {9, 1}, +static arc arcs_15_0[1] = { + {45, 1}, }; -static arc arcs_14_1[3] = { - {44, 2}, - {27, 3}, +static arc arcs_15_1[3] = { + {46, 2}, + {29, 3}, {0, 1}, }; -static arc arcs_14_2[2] = { - {45, 4}, +static arc arcs_15_2[2] = { + {47, 4}, {9, 4}, }; -static arc arcs_14_3[2] = { +static arc arcs_15_3[2] = { + {47, 5}, {45, 5}, - {9, 5}, }; -static arc arcs_14_4[1] = { +static arc arcs_15_4[1] = { {0, 4}, }; -static arc arcs_14_5[2] = { - {27, 3}, +static arc arcs_15_5[2] = { + {29, 3}, {0, 5}, }; -static state states_14[6] = { - {1, arcs_14_0}, - {3, arcs_14_1}, - {2, arcs_14_2}, - {2, arcs_14_3}, - {1, arcs_14_4}, - {2, arcs_14_5}, +static state states_15[6] = { + {1, arcs_15_0}, + {3, arcs_15_1}, + {2, arcs_15_2}, + {2, arcs_15_3}, + {1, arcs_15_4}, + {2, arcs_15_5}, }; -static arc arcs_15_0[12] = { - {46, 1}, - {47, 1}, +static arc arcs_16_0[2] = { + {24, 1}, + {48, 1}, +}; +static arc arcs_16_1[2] = { + {30, 2}, + {0, 1}, +}; +static arc arcs_16_2[3] = { + {24, 1}, {48, 1}, + {0, 2}, +}; +static state states_16[3] = { + {2, arcs_16_0}, + {2, arcs_16_1}, + {3, arcs_16_2}, +}; +static arc arcs_17_0[12] = { {49, 1}, {50, 1}, {51, 1}, @@ -328,105 +486,53 @@ static arc arcs_15_0[12] = { {55, 1}, {56, 1}, {57, 1}, -}; -static arc arcs_15_1[1] = { - {0, 1}, -}; -static state states_15[2] = { - {12, arcs_15_0}, - {1, arcs_15_1}, -}; -static arc arcs_16_0[1] = { {58, 1}, -}; -static arc arcs_16_1[3] = { - {28, 2}, - {59, 3}, - {0, 1}, -}; -static arc arcs_16_2[2] = { - {29, 4}, - {0, 2}, -}; -static arc arcs_16_3[1] = { - {28, 5}, -}; -static arc arcs_16_4[2] = { - {28, 2}, - {0, 4}, -}; -static arc arcs_16_5[2] = { - {29, 6}, - {0, 5}, -}; -static arc arcs_16_6[1] = { - {28, 7}, -}; -static arc arcs_16_7[2] = { - {29, 8}, - {0, 7}, -}; -static arc arcs_16_8[2] = { - {28, 7}, - {0, 8}, -}; -static state states_16[9] = { - {1, arcs_16_0}, - {3, arcs_16_1}, - {2, arcs_16_2}, - {1, arcs_16_3}, - {2, arcs_16_4}, - {2, arcs_16_5}, - {1, arcs_16_6}, - {2, arcs_16_7}, - {2, arcs_16_8}, -}; -static arc arcs_17_0[1] = { + {59, 1}, {60, 1}, }; static arc arcs_17_1[1] = { - {61, 2}, -}; -static arc arcs_17_2[1] = { - {0, 2}, + {0, 1}, }; -static state states_17[3] = { - {1, arcs_17_0}, +static state states_17[2] = { + {12, arcs_17_0}, {1, arcs_17_1}, - {1, arcs_17_2}, }; static arc arcs_18_0[1] = { - {62, 1}, + {61, 1}, }; static arc arcs_18_1[1] = { - {0, 1}, + {62, 2}, +}; +static arc arcs_18_2[1] = { + {0, 2}, }; -static state states_18[2] = { +static state states_18[3] = { {1, arcs_18_0}, {1, arcs_18_1}, + {1, arcs_18_2}, }; -static arc arcs_19_0[5] = { +static arc arcs_19_0[1] = { {63, 1}, - {64, 1}, - {65, 1}, - {66, 1}, - {67, 1}, }; static arc arcs_19_1[1] = { {0, 1}, }; static state states_19[2] = { - {5, arcs_19_0}, + {1, arcs_19_0}, {1, arcs_19_1}, }; -static arc arcs_20_0[1] = { +static arc arcs_20_0[5] = { + {64, 1}, + {65, 1}, + {66, 1}, + {67, 1}, {68, 1}, }; static arc arcs_20_1[1] = { {0, 1}, }; static state states_20[2] = { - {1, arcs_20_0}, + {5, arcs_20_0}, {1, arcs_20_1}, }; static arc arcs_21_0[1] = { @@ -442,149 +548,133 @@ static state states_21[2] = { static arc arcs_22_0[1] = { {70, 1}, }; -static arc arcs_22_1[2] = { - {9, 2}, +static arc arcs_22_1[1] = { {0, 1}, }; -static arc arcs_22_2[1] = { - {0, 2}, -}; -static state states_22[3] = { +static state states_22[2] = { {1, arcs_22_0}, - {2, arcs_22_1}, - {1, arcs_22_2}, + {1, arcs_22_1}, }; static arc arcs_23_0[1] = { - {45, 1}, + {71, 1}, }; -static arc arcs_23_1[1] = { +static arc arcs_23_1[2] = { + {9, 2}, {0, 1}, }; -static state states_23[2] = { +static arc arcs_23_2[1] = { + {0, 2}, +}; +static state states_23[3] = { {1, arcs_23_0}, - {1, arcs_23_1}, + {2, arcs_23_1}, + {1, arcs_23_2}, }; static arc arcs_24_0[1] = { - {71, 1}, + {47, 1}, }; -static arc arcs_24_1[2] = { - {28, 2}, +static arc arcs_24_1[1] = { {0, 1}, }; -static arc arcs_24_2[2] = { - {29, 3}, - {0, 2}, -}; -static arc arcs_24_3[1] = { - {28, 4}, +static state states_24[2] = { + {1, arcs_24_0}, + {1, arcs_24_1}, }; -static arc arcs_24_4[2] = { - {29, 5}, - {0, 4}, +static arc arcs_25_0[1] = { + {72, 1}, }; -static arc arcs_24_5[1] = { - {28, 6}, +static arc arcs_25_1[2] = { + {24, 2}, + {0, 1}, }; -static arc arcs_24_6[1] = { - {0, 6}, +static arc arcs_25_2[2] = { + {73, 3}, + {0, 2}, }; -static state states_24[7] = { - {1, arcs_24_0}, - {2, arcs_24_1}, - {2, arcs_24_2}, - {1, arcs_24_3}, - {2, arcs_24_4}, - {1, arcs_24_5}, - {1, arcs_24_6}, -}; -static arc arcs_25_0[2] = { - {72, 1}, - {73, 1}, +static arc arcs_25_3[1] = { + {24, 4}, }; -static arc arcs_25_1[1] = { - {0, 1}, +static arc arcs_25_4[1] = { + {0, 4}, }; -static state states_25[2] = { - {2, arcs_25_0}, - {1, arcs_25_1}, +static state states_25[5] = { + {1, arcs_25_0}, + {2, arcs_25_1}, + {2, arcs_25_2}, + {1, arcs_25_3}, + {1, arcs_25_4}, }; -static arc arcs_26_0[1] = { +static arc arcs_26_0[2] = { {74, 1}, + {75, 1}, }; static arc arcs_26_1[1] = { - {75, 2}, -}; -static arc arcs_26_2[1] = { - {0, 2}, + {0, 1}, }; -static state states_26[3] = { - {1, arcs_26_0}, +static state states_26[2] = { + {2, arcs_26_0}, {1, arcs_26_1}, - {1, arcs_26_2}, }; static arc arcs_27_0[1] = { {76, 1}, }; -static arc arcs_27_1[2] = { +static arc arcs_27_1[1] = { {77, 2}, +}; +static arc arcs_27_2[1] = { + {0, 2}, +}; +static state states_27[3] = { + {1, arcs_27_0}, + {1, arcs_27_1}, + {1, arcs_27_2}, +}; +static arc arcs_28_0[1] = { + {73, 1}, +}; +static arc arcs_28_1[3] = { + {78, 2}, + {79, 2}, {12, 3}, }; -static arc arcs_27_2[3] = { - {77, 2}, +static arc arcs_28_2[4] = { + {78, 2}, + {79, 2}, {12, 3}, - {74, 4}, + {76, 4}, }; -static arc arcs_27_3[1] = { - {74, 4}, +static arc arcs_28_3[1] = { + {76, 4}, }; -static arc arcs_27_4[3] = { - {30, 5}, +static arc arcs_28_4[3] = { + {31, 5}, {13, 6}, - {78, 5}, + {80, 5}, }; -static arc arcs_27_5[1] = { +static arc arcs_28_5[1] = { {0, 5}, }; -static arc arcs_27_6[1] = { - {78, 7}, +static arc arcs_28_6[1] = { + {80, 7}, }; -static arc arcs_27_7[1] = { +static arc arcs_28_7[1] = { {15, 5}, }; -static state states_27[8] = { - {1, arcs_27_0}, - {2, arcs_27_1}, - {3, arcs_27_2}, - {1, arcs_27_3}, - {3, arcs_27_4}, - {1, arcs_27_5}, - {1, arcs_27_6}, - {1, arcs_27_7}, -}; -static arc arcs_28_0[1] = { - {21, 1}, -}; -static arc arcs_28_1[2] = { - {80, 2}, - {0, 1}, -}; -static arc arcs_28_2[1] = { - {21, 3}, -}; -static arc arcs_28_3[1] = { - {0, 3}, -}; -static state states_28[4] = { +static state states_28[8] = { {1, arcs_28_0}, - {2, arcs_28_1}, - {1, arcs_28_2}, + {3, arcs_28_1}, + {4, arcs_28_2}, {1, arcs_28_3}, + {3, arcs_28_4}, + {1, arcs_28_5}, + {1, arcs_28_6}, + {1, arcs_28_7}, }; static arc arcs_29_0[1] = { - {12, 1}, + {21, 1}, }; static arc arcs_29_1[2] = { - {80, 2}, + {82, 2}, {0, 1}, }; static arc arcs_29_2[1] = { @@ -600,37 +690,45 @@ static state states_29[4] = { {1, arcs_29_3}, }; static arc arcs_30_0[1] = { - {79, 1}, + {12, 1}, }; static arc arcs_30_1[2] = { - {29, 2}, + {82, 2}, {0, 1}, }; -static arc arcs_30_2[2] = { - {79, 1}, - {0, 2}, +static arc arcs_30_2[1] = { + {21, 3}, }; -static state states_30[3] = { +static arc arcs_30_3[1] = { + {0, 3}, +}; +static state states_30[4] = { {1, arcs_30_0}, {2, arcs_30_1}, - {2, arcs_30_2}, + {1, arcs_30_2}, + {1, arcs_30_3}, }; static arc arcs_31_0[1] = { {81, 1}, }; static arc arcs_31_1[2] = { - {29, 0}, + {30, 2}, {0, 1}, }; -static state states_31[2] = { +static arc arcs_31_2[2] = { + {81, 1}, + {0, 2}, +}; +static state states_31[3] = { {1, arcs_31_0}, {2, arcs_31_1}, + {2, arcs_31_2}, }; static arc arcs_32_0[1] = { - {21, 1}, + {83, 1}, }; static arc arcs_32_1[2] = { - {77, 0}, + {30, 0}, {0, 1}, }; static state states_32[2] = { @@ -638,149 +736,108 @@ static state states_32[2] = { {2, arcs_32_1}, }; static arc arcs_33_0[1] = { - {82, 1}, -}; -static arc arcs_33_1[1] = { - {21, 2}, + {21, 1}, }; -static arc arcs_33_2[2] = { - {29, 1}, - {0, 2}, +static arc arcs_33_1[2] = { + {78, 0}, + {0, 1}, }; -static state states_33[3] = { +static state states_33[2] = { {1, arcs_33_0}, - {1, arcs_33_1}, - {2, arcs_33_2}, + {2, arcs_33_1}, }; static arc arcs_34_0[1] = { - {83, 1}, + {84, 1}, }; static arc arcs_34_1[1] = { - {84, 2}, + {21, 2}, }; static arc arcs_34_2[2] = { - {85, 3}, + {30, 1}, {0, 2}, }; -static arc arcs_34_3[1] = { - {28, 4}, -}; -static arc arcs_34_4[2] = { - {29, 5}, - {0, 4}, -}; -static arc arcs_34_5[1] = { - {28, 6}, -}; -static arc arcs_34_6[1] = { - {0, 6}, -}; -static state states_34[7] = { +static state states_34[3] = { {1, arcs_34_0}, {1, arcs_34_1}, {2, arcs_34_2}, - {1, arcs_34_3}, - {2, arcs_34_4}, - {1, arcs_34_5}, - {1, arcs_34_6}, }; static arc arcs_35_0[1] = { - {86, 1}, + {85, 1}, }; static arc arcs_35_1[1] = { - {28, 2}, + {21, 2}, }; static arc arcs_35_2[2] = { - {29, 3}, + {30, 1}, {0, 2}, }; -static arc arcs_35_3[1] = { - {28, 4}, -}; -static arc arcs_35_4[1] = { - {0, 4}, -}; -static state states_35[5] = { +static state states_35[3] = { {1, arcs_35_0}, {1, arcs_35_1}, {2, arcs_35_2}, - {1, arcs_35_3}, - {1, arcs_35_4}, }; -static arc arcs_36_0[8] = { - {87, 1}, - {88, 1}, - {89, 1}, - {90, 1}, - {91, 1}, - {19, 1}, - {18, 1}, - {17, 1}, +static arc arcs_36_0[1] = { + {86, 1}, }; static arc arcs_36_1[1] = { - {0, 1}, -}; -static state states_36[2] = { - {8, arcs_36_0}, - {1, arcs_36_1}, -}; -static arc arcs_37_0[1] = { - {92, 1}, -}; -static arc arcs_37_1[1] = { - {28, 2}, + {24, 2}, }; -static arc arcs_37_2[1] = { - {23, 3}, +static arc arcs_36_2[2] = { + {30, 3}, + {0, 2}, }; -static arc arcs_37_3[1] = { +static arc arcs_36_3[1] = { {24, 4}, }; -static arc arcs_37_4[3] = { - {93, 1}, - {94, 5}, +static arc arcs_36_4[1] = { {0, 4}, }; -static arc arcs_37_5[1] = { - {23, 6}, +static state states_36[5] = { + {1, arcs_36_0}, + {1, arcs_36_1}, + {2, arcs_36_2}, + {1, arcs_36_3}, + {1, arcs_36_4}, }; -static arc arcs_37_6[1] = { - {24, 7}, +static arc arcs_37_0[8] = { + {87, 1}, + {88, 1}, + {89, 1}, + {90, 1}, + {91, 1}, + {19, 1}, + {18, 1}, + {17, 1}, }; -static arc arcs_37_7[1] = { - {0, 7}, +static arc arcs_37_1[1] = { + {0, 1}, }; -static state states_37[8] = { - {1, arcs_37_0}, +static state states_37[2] = { + {8, arcs_37_0}, {1, arcs_37_1}, - {1, arcs_37_2}, - {1, arcs_37_3}, - {3, arcs_37_4}, - {1, arcs_37_5}, - {1, arcs_37_6}, - {1, arcs_37_7}, }; static arc arcs_38_0[1] = { - {95, 1}, + {92, 1}, }; static arc arcs_38_1[1] = { - {28, 2}, + {24, 2}, }; static arc arcs_38_2[1] = { - {23, 3}, + {25, 3}, }; static arc arcs_38_3[1] = { - {24, 4}, + {26, 4}, }; -static arc arcs_38_4[2] = { +static arc arcs_38_4[3] = { + {93, 1}, {94, 5}, {0, 4}, }; static arc arcs_38_5[1] = { - {23, 6}, + {25, 6}, }; static arc arcs_38_6[1] = { - {24, 7}, + {26, 7}, }; static arc arcs_38_7[1] = { {0, 7}, @@ -790,394 +847,418 @@ static state states_38[8] = { {1, arcs_38_1}, {1, arcs_38_2}, {1, arcs_38_3}, - {2, arcs_38_4}, + {3, arcs_38_4}, {1, arcs_38_5}, {1, arcs_38_6}, {1, arcs_38_7}, }; static arc arcs_39_0[1] = { - {96, 1}, + {95, 1}, }; static arc arcs_39_1[1] = { - {61, 2}, + {24, 2}, }; static arc arcs_39_2[1] = { - {85, 3}, + {25, 3}, }; static arc arcs_39_3[1] = { - {9, 4}, + {26, 4}, }; -static arc arcs_39_4[1] = { - {23, 5}, +static arc arcs_39_4[2] = { + {94, 5}, + {0, 4}, }; static arc arcs_39_5[1] = { - {24, 6}, + {25, 6}, }; -static arc arcs_39_6[2] = { - {94, 7}, - {0, 6}, +static arc arcs_39_6[1] = { + {26, 7}, }; static arc arcs_39_7[1] = { - {23, 8}, -}; -static arc arcs_39_8[1] = { - {24, 9}, -}; -static arc arcs_39_9[1] = { - {0, 9}, + {0, 7}, }; -static state states_39[10] = { +static state states_39[8] = { {1, arcs_39_0}, {1, arcs_39_1}, {1, arcs_39_2}, {1, arcs_39_3}, - {1, arcs_39_4}, + {2, arcs_39_4}, {1, arcs_39_5}, - {2, arcs_39_6}, + {1, arcs_39_6}, {1, arcs_39_7}, - {1, arcs_39_8}, - {1, arcs_39_9}, }; static arc arcs_40_0[1] = { - {97, 1}, + {96, 1}, }; static arc arcs_40_1[1] = { - {23, 2}, + {62, 2}, }; static arc arcs_40_2[1] = { - {24, 3}, + {97, 3}, }; -static arc arcs_40_3[2] = { - {98, 4}, - {99, 5}, +static arc arcs_40_3[1] = { + {9, 4}, }; static arc arcs_40_4[1] = { - {23, 6}, + {25, 5}, }; static arc arcs_40_5[1] = { - {23, 7}, + {26, 6}, }; -static arc arcs_40_6[1] = { - {24, 8}, +static arc arcs_40_6[2] = { + {94, 7}, + {0, 6}, }; static arc arcs_40_7[1] = { - {24, 9}, + {25, 8}, }; -static arc arcs_40_8[4] = { - {98, 4}, - {94, 10}, - {99, 5}, - {0, 8}, +static arc arcs_40_8[1] = { + {26, 9}, }; static arc arcs_40_9[1] = { {0, 9}, }; -static arc arcs_40_10[1] = { - {23, 11}, -}; -static arc arcs_40_11[1] = { - {24, 12}, -}; -static arc arcs_40_12[2] = { - {99, 5}, - {0, 12}, -}; -static state states_40[13] = { +static state states_40[10] = { {1, arcs_40_0}, {1, arcs_40_1}, {1, arcs_40_2}, - {2, arcs_40_3}, + {1, arcs_40_3}, {1, arcs_40_4}, {1, arcs_40_5}, - {1, arcs_40_6}, + {2, arcs_40_6}, {1, arcs_40_7}, - {4, arcs_40_8}, + {1, arcs_40_8}, {1, arcs_40_9}, - {1, arcs_40_10}, - {1, arcs_40_11}, - {2, arcs_40_12}, }; static arc arcs_41_0[1] = { - {100, 1}, + {98, 1}, }; static arc arcs_41_1[1] = { - {101, 2}, + {25, 2}, }; -static arc arcs_41_2[2] = { - {29, 1}, - {23, 3}, +static arc arcs_41_2[1] = { + {26, 3}, }; -static arc arcs_41_3[1] = { - {24, 4}, +static arc arcs_41_3[2] = { + {99, 4}, + {100, 5}, }; static arc arcs_41_4[1] = { - {0, 4}, + {25, 6}, +}; +static arc arcs_41_5[1] = { + {25, 7}, +}; +static arc arcs_41_6[1] = { + {26, 8}, +}; +static arc arcs_41_7[1] = { + {26, 9}, }; -static state states_41[5] = { +static arc arcs_41_8[4] = { + {99, 4}, + {94, 10}, + {100, 5}, + {0, 8}, +}; +static arc arcs_41_9[1] = { + {0, 9}, +}; +static arc arcs_41_10[1] = { + {25, 11}, +}; +static arc arcs_41_11[1] = { + {26, 12}, +}; +static arc arcs_41_12[2] = { + {100, 5}, + {0, 12}, +}; +static state states_41[13] = { {1, arcs_41_0}, {1, arcs_41_1}, - {2, arcs_41_2}, - {1, arcs_41_3}, + {1, arcs_41_2}, + {2, arcs_41_3}, {1, arcs_41_4}, + {1, arcs_41_5}, + {1, arcs_41_6}, + {1, arcs_41_7}, + {4, arcs_41_8}, + {1, arcs_41_9}, + {1, arcs_41_10}, + {1, arcs_41_11}, + {2, arcs_41_12}, }; static arc arcs_42_0[1] = { - {28, 1}, + {101, 1}, }; -static arc arcs_42_1[2] = { - {80, 2}, - {0, 1}, +static arc arcs_42_1[1] = { + {102, 2}, }; -static arc arcs_42_2[1] = { - {84, 3}, +static arc arcs_42_2[2] = { + {30, 1}, + {25, 3}, }; static arc arcs_42_3[1] = { - {0, 3}, + {26, 4}, }; -static state states_42[4] = { +static arc arcs_42_4[1] = { + {0, 4}, +}; +static state states_42[5] = { {1, arcs_42_0}, - {2, arcs_42_1}, - {1, arcs_42_2}, + {1, arcs_42_1}, + {2, arcs_42_2}, {1, arcs_42_3}, + {1, arcs_42_4}, }; static arc arcs_43_0[1] = { - {102, 1}, + {24, 1}, }; static arc arcs_43_1[2] = { - {28, 2}, + {82, 2}, {0, 1}, }; -static arc arcs_43_2[3] = { - {80, 3}, - {29, 3}, - {0, 2}, +static arc arcs_43_2[1] = { + {103, 3}, }; static arc arcs_43_3[1] = { - {28, 4}, -}; -static arc arcs_43_4[1] = { - {0, 4}, + {0, 3}, }; -static state states_43[5] = { +static state states_43[4] = { {1, arcs_43_0}, {2, arcs_43_1}, - {3, arcs_43_2}, + {1, arcs_43_2}, {1, arcs_43_3}, - {1, arcs_43_4}, }; -static arc arcs_44_0[2] = { - {3, 1}, - {2, 2}, +static arc arcs_44_0[1] = { + {104, 1}, }; -static arc arcs_44_1[1] = { +static arc arcs_44_1[2] = { + {24, 2}, {0, 1}, }; -static arc arcs_44_2[1] = { - {103, 3}, +static arc arcs_44_2[2] = { + {82, 3}, + {0, 2}, }; static arc arcs_44_3[1] = { - {6, 4}, + {21, 4}, }; -static arc arcs_44_4[2] = { - {6, 4}, - {104, 1}, +static arc arcs_44_4[1] = { + {0, 4}, }; static state states_44[5] = { - {2, arcs_44_0}, - {1, arcs_44_1}, - {1, arcs_44_2}, + {1, arcs_44_0}, + {2, arcs_44_1}, + {2, arcs_44_2}, {1, arcs_44_3}, - {2, arcs_44_4}, + {1, arcs_44_4}, }; -static arc arcs_45_0[1] = { - {106, 1}, +static arc arcs_45_0[2] = { + {3, 1}, + {2, 2}, }; -static arc arcs_45_1[2] = { - {29, 2}, +static arc arcs_45_1[1] = { {0, 1}, }; static arc arcs_45_2[1] = { - {106, 3}, + {105, 3}, }; -static arc arcs_45_3[2] = { - {29, 4}, - {0, 3}, +static arc arcs_45_3[1] = { + {6, 4}, }; static arc arcs_45_4[2] = { - {106, 3}, - {0, 4}, + {6, 4}, + {106, 1}, }; static state states_45[5] = { - {1, arcs_45_0}, - {2, arcs_45_1}, + {2, arcs_45_0}, + {1, arcs_45_1}, {1, arcs_45_2}, - {2, arcs_45_3}, + {1, arcs_45_3}, {2, arcs_45_4}, }; static arc arcs_46_0[2] = { {107, 1}, - {108, 1}, + {108, 2}, }; -static arc arcs_46_1[1] = { +static arc arcs_46_1[2] = { + {92, 3}, {0, 1}, }; -static state states_46[2] = { - {2, arcs_46_0}, - {1, arcs_46_1}, +static arc arcs_46_2[1] = { + {0, 2}, }; -static arc arcs_47_0[1] = { - {109, 1}, +static arc arcs_46_3[1] = { + {107, 4}, }; -static arc arcs_47_1[2] = { - {25, 2}, - {23, 3}, +static arc arcs_46_4[1] = { + {94, 5}, +}; +static arc arcs_46_5[1] = { + {24, 2}, }; -static arc arcs_47_2[1] = { - {23, 3}, +static state states_46[6] = { + {2, arcs_46_0}, + {2, arcs_46_1}, + {1, arcs_46_2}, + {1, arcs_46_3}, + {1, arcs_46_4}, + {1, arcs_46_5}, }; -static arc arcs_47_3[1] = { - {106, 4}, +static arc arcs_47_0[2] = { + {107, 1}, + {110, 1}, }; -static arc arcs_47_4[1] = { - {0, 4}, +static arc arcs_47_1[1] = { + {0, 1}, }; -static state states_47[5] = { - {1, arcs_47_0}, - {2, arcs_47_1}, - {1, arcs_47_2}, - {1, arcs_47_3}, - {1, arcs_47_4}, +static state states_47[2] = { + {2, arcs_47_0}, + {1, arcs_47_1}, }; -static arc arcs_48_0[2] = { - {107, 1}, - {110, 2}, +static arc arcs_48_0[1] = { + {111, 1}, }; static arc arcs_48_1[2] = { - {92, 3}, - {0, 1}, + {33, 2}, + {25, 3}, }; static arc arcs_48_2[1] = { - {0, 2}, + {25, 3}, }; static arc arcs_48_3[1] = { - {107, 4}, + {24, 4}, }; static arc arcs_48_4[1] = { - {94, 5}, -}; -static arc arcs_48_5[1] = { - {28, 2}, + {0, 4}, }; -static state states_48[6] = { - {2, arcs_48_0}, +static state states_48[5] = { + {1, arcs_48_0}, {2, arcs_48_1}, {1, arcs_48_2}, {1, arcs_48_3}, {1, arcs_48_4}, - {1, arcs_48_5}, }; static arc arcs_49_0[1] = { {111, 1}, }; static arc arcs_49_1[2] = { - {112, 0}, - {0, 1}, + {33, 2}, + {25, 3}, }; -static state states_49[2] = { +static arc arcs_49_2[1] = { + {25, 3}, +}; +static arc arcs_49_3[1] = { + {109, 4}, +}; +static arc arcs_49_4[1] = { + {0, 4}, +}; +static state states_49[5] = { {1, arcs_49_0}, {2, arcs_49_1}, + {1, arcs_49_2}, + {1, arcs_49_3}, + {1, arcs_49_4}, }; static arc arcs_50_0[1] = { - {113, 1}, + {112, 1}, }; static arc arcs_50_1[2] = { - {114, 0}, + {113, 0}, {0, 1}, }; static state states_50[2] = { {1, arcs_50_0}, {2, arcs_50_1}, }; -static arc arcs_51_0[2] = { - {115, 1}, - {116, 2}, +static arc arcs_51_0[1] = { + {114, 1}, }; -static arc arcs_51_1[1] = { - {113, 2}, +static arc arcs_51_1[2] = { + {115, 0}, + {0, 1}, +}; +static state states_51[2] = { + {1, arcs_51_0}, + {2, arcs_51_1}, +}; +static arc arcs_52_0[2] = { + {116, 1}, + {117, 2}, }; -static arc arcs_51_2[1] = { +static arc arcs_52_1[1] = { + {114, 2}, +}; +static arc arcs_52_2[1] = { {0, 2}, }; -static state states_51[3] = { - {2, arcs_51_0}, - {1, arcs_51_1}, - {1, arcs_51_2}, +static state states_52[3] = { + {2, arcs_52_0}, + {1, arcs_52_1}, + {1, arcs_52_2}, }; -static arc arcs_52_0[1] = { - {84, 1}, +static arc arcs_53_0[1] = { + {103, 1}, }; -static arc arcs_52_1[2] = { - {117, 0}, +static arc arcs_53_1[2] = { + {118, 0}, {0, 1}, }; -static state states_52[2] = { - {1, arcs_52_0}, - {2, arcs_52_1}, +static state states_53[2] = { + {1, arcs_53_0}, + {2, arcs_53_1}, }; -static arc arcs_53_0[10] = { - {118, 1}, +static arc arcs_54_0[10] = { {119, 1}, {120, 1}, {121, 1}, {122, 1}, {123, 1}, {124, 1}, - {85, 1}, - {115, 2}, - {125, 3}, + {125, 1}, + {97, 1}, + {116, 2}, + {126, 3}, }; -static arc arcs_53_1[1] = { +static arc arcs_54_1[1] = { {0, 1}, }; -static arc arcs_53_2[1] = { - {85, 1}, +static arc arcs_54_2[1] = { + {97, 1}, }; -static arc arcs_53_3[2] = { - {115, 1}, +static arc arcs_54_3[2] = { + {116, 1}, {0, 3}, }; -static state states_53[4] = { - {10, arcs_53_0}, - {1, arcs_53_1}, - {1, arcs_53_2}, - {2, arcs_53_3}, -}; -static arc arcs_54_0[1] = { - {126, 1}, -}; -static arc arcs_54_1[2] = { - {127, 0}, - {0, 1}, -}; -static state states_54[2] = { - {1, arcs_54_0}, - {2, arcs_54_1}, +static state states_54[4] = { + {10, arcs_54_0}, + {1, arcs_54_1}, + {1, arcs_54_2}, + {2, arcs_54_3}, }; static arc arcs_55_0[1] = { - {128, 1}, + {31, 1}, }; -static arc arcs_55_1[2] = { - {129, 0}, - {0, 1}, +static arc arcs_55_1[1] = { + {103, 2}, }; -static state states_55[2] = { +static arc arcs_55_2[1] = { + {0, 2}, +}; +static state states_55[3] = { {1, arcs_55_0}, - {2, arcs_55_1}, + {1, arcs_55_1}, + {1, arcs_55_2}, }; static arc arcs_56_0[1] = { - {130, 1}, + {127, 1}, }; static arc arcs_56_1[2] = { - {131, 0}, + {128, 0}, {0, 1}, }; static state states_56[2] = { @@ -1185,223 +1266,193 @@ static state states_56[2] = { {2, arcs_56_1}, }; static arc arcs_57_0[1] = { - {132, 1}, + {129, 1}, }; -static arc arcs_57_1[3] = { - {133, 0}, - {59, 0}, +static arc arcs_57_1[2] = { + {130, 0}, {0, 1}, }; static state states_57[2] = { {1, arcs_57_0}, - {3, arcs_57_1}, + {2, arcs_57_1}, }; static arc arcs_58_0[1] = { - {134, 1}, + {131, 1}, }; -static arc arcs_58_1[3] = { - {135, 0}, - {136, 0}, +static arc arcs_58_1[2] = { + {132, 0}, {0, 1}, }; static state states_58[2] = { {1, arcs_58_0}, - {3, arcs_58_1}, + {2, arcs_58_1}, }; static arc arcs_59_0[1] = { - {137, 1}, + {133, 1}, }; -static arc arcs_59_1[5] = { - {30, 0}, - {138, 0}, - {139, 0}, - {140, 0}, +static arc arcs_59_1[3] = { + {134, 0}, + {135, 0}, {0, 1}, }; static state states_59[2] = { {1, arcs_59_0}, - {5, arcs_59_1}, + {3, arcs_59_1}, }; -static arc arcs_60_0[4] = { - {135, 1}, +static arc arcs_60_0[1] = { {136, 1}, - {141, 1}, - {142, 2}, }; -static arc arcs_60_1[1] = { - {137, 2}, -}; -static arc arcs_60_2[1] = { - {0, 2}, +static arc arcs_60_1[3] = { + {137, 0}, + {138, 0}, + {0, 1}, }; -static state states_60[3] = { - {4, arcs_60_0}, - {1, arcs_60_1}, - {1, arcs_60_2}, +static state states_60[2] = { + {1, arcs_60_0}, + {3, arcs_60_1}, }; static arc arcs_61_0[1] = { - {143, 1}, + {139, 1}, }; -static arc arcs_61_1[3] = { - {144, 1}, - {31, 2}, +static arc arcs_61_1[5] = { + {31, 0}, + {140, 0}, + {141, 0}, + {142, 0}, {0, 1}, }; -static arc arcs_61_2[1] = { - {137, 3}, -}; -static arc arcs_61_3[1] = { - {0, 3}, -}; -static state states_61[4] = { +static state states_61[2] = { {1, arcs_61_0}, - {3, arcs_61_1}, - {1, arcs_61_2}, - {1, arcs_61_3}, -}; -static arc arcs_62_0[7] = { - {13, 1}, - {146, 2}, - {149, 3}, - {152, 4}, - {21, 5}, - {154, 5}, - {155, 6}, -}; -static arc arcs_62_1[3] = { - {45, 7}, - {145, 7}, - {15, 5}, -}; -static arc arcs_62_2[2] = { - {147, 8}, - {148, 5}, -}; -static arc arcs_62_3[2] = { - {150, 9}, - {151, 5}, -}; -static arc arcs_62_4[1] = { - {153, 10}, -}; -static arc arcs_62_5[1] = { - {0, 5}, -}; -static arc arcs_62_6[2] = { - {155, 6}, - {0, 6}, -}; -static arc arcs_62_7[1] = { - {15, 5}, + {5, arcs_61_1}, }; -static arc arcs_62_8[1] = { - {148, 5}, +static arc arcs_62_0[4] = { + {137, 1}, + {138, 1}, + {143, 1}, + {144, 2}, }; -static arc arcs_62_9[1] = { - {151, 5}, +static arc arcs_62_1[1] = { + {139, 2}, }; -static arc arcs_62_10[1] = { - {152, 5}, +static arc arcs_62_2[1] = { + {0, 2}, }; -static state states_62[11] = { - {7, arcs_62_0}, - {3, arcs_62_1}, - {2, arcs_62_2}, - {2, arcs_62_3}, - {1, arcs_62_4}, - {1, arcs_62_5}, - {2, arcs_62_6}, - {1, arcs_62_7}, - {1, arcs_62_8}, - {1, arcs_62_9}, - {1, arcs_62_10}, +static state states_62[3] = { + {4, arcs_62_0}, + {1, arcs_62_1}, + {1, arcs_62_2}, }; static arc arcs_63_0[1] = { - {28, 1}, + {145, 1}, }; static arc arcs_63_1[3] = { - {156, 2}, - {29, 3}, + {146, 1}, + {32, 2}, {0, 1}, }; static arc arcs_63_2[1] = { - {0, 2}, + {139, 3}, }; -static arc arcs_63_3[2] = { - {28, 4}, +static arc arcs_63_3[1] = { {0, 3}, }; -static arc arcs_63_4[2] = { - {29, 3}, - {0, 4}, -}; -static state states_63[5] = { +static state states_63[4] = { {1, arcs_63_0}, {3, arcs_63_1}, {1, arcs_63_2}, - {2, arcs_63_3}, - {2, arcs_63_4}, + {1, arcs_63_3}, }; -static arc arcs_64_0[1] = { - {28, 1}, +static arc arcs_64_0[10] = { + {13, 1}, + {148, 2}, + {150, 3}, + {21, 4}, + {153, 4}, + {154, 5}, + {79, 4}, + {155, 4}, + {156, 4}, + {157, 4}, }; static arc arcs_64_1[3] = { - {157, 2}, - {29, 3}, - {0, 1}, + {47, 6}, + {147, 6}, + {15, 4}, }; -static arc arcs_64_2[1] = { - {0, 2}, +static arc arcs_64_2[2] = { + {147, 7}, + {149, 4}, }; static arc arcs_64_3[2] = { - {28, 4}, - {0, 3}, + {151, 8}, + {152, 4}, }; -static arc arcs_64_4[2] = { - {29, 3}, +static arc arcs_64_4[1] = { {0, 4}, }; -static state states_64[5] = { - {1, arcs_64_0}, +static arc arcs_64_5[2] = { + {154, 5}, + {0, 5}, +}; +static arc arcs_64_6[1] = { + {15, 4}, +}; +static arc arcs_64_7[1] = { + {149, 4}, +}; +static arc arcs_64_8[1] = { + {152, 4}, +}; +static state states_64[9] = { + {10, arcs_64_0}, {3, arcs_64_1}, - {1, arcs_64_2}, + {2, arcs_64_2}, {2, arcs_64_3}, - {2, arcs_64_4}, -}; -static arc arcs_65_0[1] = { - {109, 1}, + {1, arcs_64_4}, + {2, arcs_64_5}, + {1, arcs_64_6}, + {1, arcs_64_7}, + {1, arcs_64_8}, +}; +static arc arcs_65_0[2] = { + {24, 1}, + {48, 1}, }; -static arc arcs_65_1[2] = { - {25, 2}, - {23, 3}, +static arc arcs_65_1[3] = { + {158, 2}, + {30, 3}, + {0, 1}, }; static arc arcs_65_2[1] = { - {23, 3}, + {0, 2}, }; -static arc arcs_65_3[1] = { - {28, 4}, +static arc arcs_65_3[3] = { + {24, 4}, + {48, 4}, + {0, 3}, }; -static arc arcs_65_4[1] = { +static arc arcs_65_4[2] = { + {30, 3}, {0, 4}, }; static state states_65[5] = { - {1, arcs_65_0}, - {2, arcs_65_1}, + {2, arcs_65_0}, + {3, arcs_65_1}, {1, arcs_65_2}, - {1, arcs_65_3}, - {1, arcs_65_4}, + {3, arcs_65_3}, + {2, arcs_65_4}, }; static arc arcs_66_0[3] = { {13, 1}, - {146, 2}, - {77, 3}, + {148, 2}, + {78, 3}, }; static arc arcs_66_1[2] = { {14, 4}, {15, 5}, }; static arc arcs_66_2[1] = { - {158, 6}, + {159, 6}, }; static arc arcs_66_3[1] = { {21, 5}, @@ -1413,7 +1464,7 @@ static arc arcs_66_5[1] = { {0, 5}, }; static arc arcs_66_6[1] = { - {148, 5}, + {149, 5}, }; static state states_66[7] = { {3, arcs_66_0}, @@ -1425,14 +1476,14 @@ static state states_66[7] = { {1, arcs_66_6}, }; static arc arcs_67_0[1] = { - {159, 1}, + {160, 1}, }; static arc arcs_67_1[2] = { - {29, 2}, + {30, 2}, {0, 1}, }; static arc arcs_67_2[2] = { - {159, 1}, + {160, 1}, {0, 2}, }; static state states_67[3] = { @@ -1440,47 +1491,38 @@ static state states_67[3] = { {2, arcs_67_1}, {2, arcs_67_2}, }; -static arc arcs_68_0[3] = { - {77, 1}, - {28, 2}, - {23, 3}, +static arc arcs_68_0[2] = { + {24, 1}, + {25, 2}, }; -static arc arcs_68_1[1] = { - {77, 4}, +static arc arcs_68_1[2] = { + {25, 2}, + {0, 1}, }; -static arc arcs_68_2[2] = { - {23, 3}, +static arc arcs_68_2[3] = { + {24, 3}, + {161, 4}, {0, 2}, }; -static arc arcs_68_3[3] = { - {28, 5}, - {160, 6}, +static arc arcs_68_3[2] = { + {161, 4}, {0, 3}, }; static arc arcs_68_4[1] = { - {77, 6}, -}; -static arc arcs_68_5[2] = { - {160, 6}, - {0, 5}, -}; -static arc arcs_68_6[1] = { - {0, 6}, + {0, 4}, }; -static state states_68[7] = { - {3, arcs_68_0}, - {1, arcs_68_1}, - {2, arcs_68_2}, - {3, arcs_68_3}, +static state states_68[5] = { + {2, arcs_68_0}, + {2, arcs_68_1}, + {3, arcs_68_2}, + {2, arcs_68_3}, {1, arcs_68_4}, - {2, arcs_68_5}, - {1, arcs_68_6}, }; static arc arcs_69_0[1] = { - {23, 1}, + {25, 1}, }; static arc arcs_69_1[2] = { - {28, 2}, + {24, 2}, {0, 1}, }; static arc arcs_69_2[1] = { @@ -1491,31 +1533,33 @@ static state states_69[3] = { {2, arcs_69_1}, {1, arcs_69_2}, }; -static arc arcs_70_0[1] = { - {84, 1}, +static arc arcs_70_0[2] = { + {103, 1}, + {48, 1}, }; static arc arcs_70_1[2] = { - {29, 2}, + {30, 2}, {0, 1}, }; -static arc arcs_70_2[2] = { - {84, 1}, +static arc arcs_70_2[3] = { + {103, 1}, + {48, 1}, {0, 2}, }; static state states_70[3] = { - {1, arcs_70_0}, + {2, arcs_70_0}, {2, arcs_70_1}, - {2, arcs_70_2}, + {3, arcs_70_2}, }; static arc arcs_71_0[1] = { - {28, 1}, + {24, 1}, }; static arc arcs_71_1[2] = { - {29, 2}, + {30, 2}, {0, 1}, }; static arc arcs_71_2[2] = { - {28, 1}, + {24, 1}, {0, 2}, }; static state states_71[3] = { @@ -1524,45 +1568,45 @@ static state states_71[3] = { {2, arcs_71_2}, }; static arc arcs_72_0[1] = { - {28, 1}, + {24, 1}, }; static arc arcs_72_1[4] = { - {23, 2}, - {157, 3}, - {29, 4}, + {25, 2}, + {158, 3}, + {30, 4}, {0, 1}, }; static arc arcs_72_2[1] = { - {28, 5}, + {24, 5}, }; static arc arcs_72_3[1] = { {0, 3}, }; static arc arcs_72_4[2] = { - {28, 6}, + {24, 6}, {0, 4}, }; static arc arcs_72_5[3] = { - {157, 3}, - {29, 7}, + {158, 3}, + {30, 7}, {0, 5}, }; static arc arcs_72_6[2] = { - {29, 4}, + {30, 4}, {0, 6}, }; static arc arcs_72_7[2] = { - {28, 8}, + {24, 8}, {0, 7}, }; static arc arcs_72_8[1] = { - {23, 9}, + {25, 9}, }; static arc arcs_72_9[1] = { - {28, 10}, + {24, 10}, }; static arc arcs_72_10[2] = { - {29, 7}, + {30, 7}, {0, 10}, }; static state states_72[11] = { @@ -1579,27 +1623,27 @@ static state states_72[11] = { {2, arcs_72_10}, }; static arc arcs_73_0[1] = { - {161, 1}, + {162, 1}, }; static arc arcs_73_1[1] = { {21, 2}, }; static arc arcs_73_2[2] = { {13, 3}, - {23, 4}, + {25, 4}, }; static arc arcs_73_3[2] = { - {9, 5}, + {14, 5}, {15, 6}, }; static arc arcs_73_4[1] = { - {24, 7}, + {26, 7}, }; static arc arcs_73_5[1] = { {15, 6}, }; static arc arcs_73_6[1] = { - {23, 4}, + {25, 4}, }; static arc arcs_73_7[1] = { {0, 7}, @@ -1615,36 +1659,36 @@ static state states_73[8] = { {1, arcs_73_7}, }; static arc arcs_74_0[3] = { - {162, 1}, - {30, 2}, - {31, 3}, + {163, 1}, + {31, 2}, + {32, 3}, }; static arc arcs_74_1[2] = { - {29, 4}, + {30, 4}, {0, 1}, }; static arc arcs_74_2[1] = { - {28, 5}, + {24, 5}, }; static arc arcs_74_3[1] = { - {28, 6}, + {24, 6}, }; static arc arcs_74_4[4] = { - {162, 1}, - {30, 2}, - {31, 3}, + {163, 1}, + {31, 2}, + {32, 3}, {0, 4}, }; static arc arcs_74_5[2] = { - {29, 7}, + {30, 7}, {0, 5}, }; static arc arcs_74_6[1] = { {0, 6}, }; static arc arcs_74_7[2] = { - {162, 5}, - {31, 3}, + {163, 5}, + {32, 3}, }; static state states_74[8] = { {3, arcs_74_0}, @@ -1657,18 +1701,18 @@ static state states_74[8] = { {2, arcs_74_7}, }; static arc arcs_75_0[1] = { - {28, 1}, + {24, 1}, }; static arc arcs_75_1[3] = { - {157, 2}, - {27, 3}, + {158, 2}, + {29, 3}, {0, 1}, }; static arc arcs_75_2[1] = { {0, 2}, }; static arc arcs_75_3[1] = { - {28, 2}, + {24, 2}, }; static state states_75[4] = { {1, arcs_75_0}, @@ -1677,8 +1721,8 @@ static state states_75[4] = { {1, arcs_75_3}, }; static arc arcs_76_0[2] = { - {156, 1}, - {164, 1}, + {158, 1}, + {165, 1}, }; static arc arcs_76_1[1] = { {0, 1}, @@ -1691,16 +1735,16 @@ static arc arcs_77_0[1] = { {96, 1}, }; static arc arcs_77_1[1] = { - {61, 2}, + {62, 2}, }; static arc arcs_77_2[1] = { - {85, 3}, + {97, 3}, }; static arc arcs_77_3[1] = { - {105, 4}, + {107, 4}, }; static arc arcs_77_4[2] = { - {163, 5}, + {164, 5}, {0, 4}, }; static arc arcs_77_5[1] = { @@ -1718,10 +1762,10 @@ static arc arcs_78_0[1] = { {92, 1}, }; static arc arcs_78_1[1] = { - {106, 2}, + {109, 2}, }; static arc arcs_78_2[2] = { - {163, 3}, + {164, 3}, {0, 2}, }; static arc arcs_78_3[1] = { @@ -1733,285 +1777,209 @@ static state states_78[4] = { {2, arcs_78_2}, {1, arcs_78_3}, }; -static arc arcs_79_0[2] = { - {157, 1}, - {166, 1}, +static arc arcs_79_0[1] = { + {21, 1}, }; static arc arcs_79_1[1] = { {0, 1}, }; static state states_79[2] = { - {2, arcs_79_0}, + {1, arcs_79_0}, {1, arcs_79_1}, }; static arc arcs_80_0[1] = { - {96, 1}, -}; -static arc arcs_80_1[1] = { - {61, 2}, -}; -static arc arcs_80_2[1] = { - {85, 3}, -}; -static arc arcs_80_3[1] = { - {107, 4}, -}; -static arc arcs_80_4[2] = { - {165, 5}, - {0, 4}, -}; -static arc arcs_80_5[1] = { - {0, 5}, -}; -static state states_80[6] = { - {1, arcs_80_0}, - {1, arcs_80_1}, - {1, arcs_80_2}, - {1, arcs_80_3}, - {2, arcs_80_4}, - {1, arcs_80_5}, -}; -static arc arcs_81_0[1] = { - {92, 1}, -}; -static arc arcs_81_1[1] = { - {106, 2}, -}; -static arc arcs_81_2[2] = { - {165, 3}, - {0, 2}, -}; -static arc arcs_81_3[1] = { - {0, 3}, -}; -static state states_81[4] = { - {1, arcs_81_0}, - {1, arcs_81_1}, - {2, arcs_81_2}, - {1, arcs_81_3}, -}; -static arc arcs_82_0[1] = { - {28, 1}, -}; -static arc arcs_82_1[2] = { - {29, 0}, - {0, 1}, -}; -static state states_82[2] = { - {1, arcs_82_0}, - {2, arcs_82_1}, -}; -static arc arcs_83_0[1] = { - {21, 1}, -}; -static arc arcs_83_1[1] = { - {0, 1}, -}; -static state states_83[2] = { - {1, arcs_83_0}, - {1, arcs_83_1}, + {167, 1}, }; -static arc arcs_84_0[1] = { - {168, 1}, -}; -static arc arcs_84_1[2] = { +static arc arcs_80_1[2] = { {9, 2}, {0, 1}, }; -static arc arcs_84_2[1] = { +static arc arcs_80_2[1] = { {0, 2}, }; -static state states_84[3] = { - {1, arcs_84_0}, - {2, arcs_84_1}, - {1, arcs_84_2}, +static state states_80[3] = { + {1, arcs_80_0}, + {2, arcs_80_1}, + {1, arcs_80_2}, }; -static dfa dfas[85] = { +static dfa dfas[81] = { {256, "single_input", 0, 3, states_0, - "\004\050\060\000\000\000\000\124\360\024\114\220\023\040\010\000\200\041\044\015\002\001"}, + "\004\050\060\200\000\000\000\240\340\223\160\220\045\200\020\000\000\206\120\076\204"}, {257, "file_input", 0, 2, states_1, - "\204\050\060\000\000\000\000\124\360\024\114\220\023\040\010\000\200\041\044\015\002\001"}, + "\204\050\060\200\000\000\000\240\340\223\160\220\045\200\020\000\000\206\120\076\204"}, {258, "eval_input", 0, 3, states_2, - "\000\040\040\000\000\000\000\000\000\000\000\000\000\040\010\000\200\041\044\015\000\000"}, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000"}, {259, "decorator", 0, 7, states_3, - "\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {260, "decorators", 0, 2, states_4, - "\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {261, "decorated", 0, 3, states_5, - "\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, - {262, "funcdef", 0, 6, states_6, - "\000\000\020\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + {262, "funcdef", 0, 8, states_6, + "\000\000\020\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {263, "parameters", 0, 4, states_7, - "\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, - {264, "varargslist", 0, 10, states_8, - "\000\040\040\300\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, - {265, "fpdef", 0, 4, states_9, - "\000\040\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, - {266, "fplist", 0, 3, states_10, - "\000\040\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, - {267, "stmt", 0, 2, states_11, - "\000\050\060\000\000\000\000\124\360\024\114\220\023\040\010\000\200\041\044\015\002\001"}, - {268, "simple_stmt", 0, 4, states_12, - "\000\040\040\000\000\000\000\124\360\024\114\000\000\040\010\000\200\041\044\015\000\001"}, - {269, "small_stmt", 0, 2, states_13, - "\000\040\040\000\000\000\000\124\360\024\114\000\000\040\010\000\200\041\044\015\000\001"}, - {270, "expr_stmt", 0, 6, states_14, - "\000\040\040\000\000\000\000\000\000\000\000\000\000\040\010\000\200\041\044\015\000\000"}, - {271, "augassign", 0, 2, states_15, - "\000\000\000\000\000\300\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, - {272, "print_stmt", 0, 9, states_16, - "\000\000\000\000\000\000\000\004\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, - {273, "del_stmt", 0, 3, states_17, - "\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, - {274, "pass_stmt", 0, 2, states_18, - "\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, - {275, "flow_stmt", 0, 2, states_19, - "\000\000\000\000\000\000\000\000\360\000\000\000\000\000\000\000\000\000\000\000\000\001"}, - {276, "break_stmt", 0, 2, states_20, - "\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000\000\000\000"}, - {277, "continue_stmt", 0, 2, states_21, - "\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000"}, - {278, "return_stmt", 0, 3, states_22, - "\000\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000\000\000\000\000\000"}, - {279, "yield_stmt", 0, 2, states_23, - "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\001"}, - {280, "raise_stmt", 0, 7, states_24, - "\000\000\000\000\000\000\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000"}, - {281, "import_stmt", 0, 2, states_25, - "\000\000\000\000\000\000\000\000\000\024\000\000\000\000\000\000\000\000\000\000\000\000"}, - {282, "import_name", 0, 3, states_26, - "\000\000\000\000\000\000\000\000\000\004\000\000\000\000\000\000\000\000\000\000\000\000"}, - {283, "import_from", 0, 8, states_27, - "\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000\000\000"}, - {284, "import_as_name", 0, 4, states_28, - "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, - {285, "dotted_as_name", 0, 4, states_29, - "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, - {286, "import_as_names", 0, 3, states_30, - "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, - {287, "dotted_as_names", 0, 2, states_31, - "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, - {288, "dotted_name", 0, 2, states_32, - "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, - {289, "global_stmt", 0, 3, states_33, - "\000\000\000\000\000\000\000\000\000\000\004\000\000\000\000\000\000\000\000\000\000\000"}, - {290, "exec_stmt", 0, 7, states_34, - "\000\000\000\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\000\000\000\000"}, - {291, "assert_stmt", 0, 5, states_35, - "\000\000\000\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000\000\000\000"}, - {292, "compound_stmt", 0, 2, states_36, - "\000\010\020\000\000\000\000\000\000\000\000\220\023\000\000\000\000\000\000\000\002\000"}, - {293, "if_stmt", 0, 8, states_37, - "\000\000\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000"}, - {294, "while_stmt", 0, 8, states_38, - "\000\000\000\000\000\000\000\000\000\000\000\200\000\000\000\000\000\000\000\000\000\000"}, - {295, "for_stmt", 0, 10, states_39, - "\000\000\000\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000\000"}, - {296, "try_stmt", 0, 13, states_40, - "\000\000\000\000\000\000\000\000\000\000\000\000\002\000\000\000\000\000\000\000\000\000"}, - {297, "with_stmt", 0, 5, states_41, - "\000\000\000\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000"}, - {298, "with_item", 0, 4, states_42, - "\000\040\040\000\000\000\000\000\000\000\000\000\000\040\010\000\200\041\044\015\000\000"}, - {299, "except_clause", 0, 5, states_43, - "\000\000\000\000\000\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000\000"}, - {300, "suite", 0, 5, states_44, - "\004\040\040\000\000\000\000\124\360\024\114\000\000\040\010\000\200\041\044\015\000\001"}, - {301, "testlist_safe", 0, 5, states_45, - "\000\040\040\000\000\000\000\000\000\000\000\000\000\040\010\000\200\041\044\015\000\000"}, - {302, "old_test", 0, 2, states_46, - "\000\040\040\000\000\000\000\000\000\000\000\000\000\040\010\000\200\041\044\015\000\000"}, - {303, "old_lambdef", 0, 5, states_47, - "\000\000\000\000\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000"}, - {304, "test", 0, 6, states_48, - "\000\040\040\000\000\000\000\000\000\000\000\000\000\040\010\000\200\041\044\015\000\000"}, - {305, "or_test", 0, 2, states_49, - "\000\040\040\000\000\000\000\000\000\000\000\000\000\000\010\000\200\041\044\015\000\000"}, - {306, "and_test", 0, 2, states_50, - "\000\040\040\000\000\000\000\000\000\000\000\000\000\000\010\000\200\041\044\015\000\000"}, - {307, "not_test", 0, 3, states_51, - "\000\040\040\000\000\000\000\000\000\000\000\000\000\000\010\000\200\041\044\015\000\000"}, - {308, "comparison", 0, 2, states_52, - "\000\040\040\000\000\000\000\000\000\000\000\000\000\000\000\000\200\041\044\015\000\000"}, - {309, "comp_op", 0, 4, states_53, - "\000\000\000\000\000\000\000\000\000\000\040\000\000\000\310\077\000\000\000\000\000\000"}, - {310, "expr", 0, 2, states_54, - "\000\040\040\000\000\000\000\000\000\000\000\000\000\000\000\000\200\041\044\015\000\000"}, - {311, "xor_expr", 0, 2, states_55, - "\000\040\040\000\000\000\000\000\000\000\000\000\000\000\000\000\200\041\044\015\000\000"}, - {312, "and_expr", 0, 2, states_56, - "\000\040\040\000\000\000\000\000\000\000\000\000\000\000\000\000\200\041\044\015\000\000"}, - {313, "shift_expr", 0, 2, states_57, - "\000\040\040\000\000\000\000\000\000\000\000\000\000\000\000\000\200\041\044\015\000\000"}, - {314, "arith_expr", 0, 2, states_58, - "\000\040\040\000\000\000\000\000\000\000\000\000\000\000\000\000\200\041\044\015\000\000"}, - {315, "term", 0, 2, states_59, - "\000\040\040\000\000\000\000\000\000\000\000\000\000\000\000\000\200\041\044\015\000\000"}, - {316, "factor", 0, 3, states_60, - "\000\040\040\000\000\000\000\000\000\000\000\000\000\000\000\000\200\041\044\015\000\000"}, - {317, "power", 0, 4, states_61, - "\000\040\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\044\015\000\000"}, - {318, "atom", 0, 11, states_62, - "\000\040\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\044\015\000\000"}, - {319, "listmaker", 0, 5, states_63, - "\000\040\040\000\000\000\000\000\000\000\000\000\000\040\010\000\200\041\044\015\000\000"}, - {320, "testlist_comp", 0, 5, states_64, - "\000\040\040\000\000\000\000\000\000\000\000\000\000\040\010\000\200\041\044\015\000\000"}, - {321, "lambdef", 0, 5, states_65, - "\000\000\000\000\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000"}, + "\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + {264, "typedargslist", 0, 18, states_8, + "\000\000\040\200\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + {265, "tfpdef", 0, 4, states_9, + "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + {266, "varargslist", 0, 18, states_10, + "\000\000\040\200\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + {267, "vfpdef", 0, 2, states_11, + "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + {268, "stmt", 0, 2, states_12, + "\000\050\060\200\000\000\000\240\340\223\160\220\045\200\020\000\000\206\120\076\204"}, + {269, "simple_stmt", 0, 4, states_13, + "\000\040\040\200\000\000\000\240\340\223\160\000\000\200\020\000\000\206\120\076\200"}, + {270, "small_stmt", 0, 2, states_14, + "\000\040\040\200\000\000\000\240\340\223\160\000\000\200\020\000\000\206\120\076\200"}, + {271, "expr_stmt", 0, 6, states_15, + "\000\040\040\200\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000"}, + {272, "testlist_star_expr", 0, 3, states_16, + "\000\040\040\200\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000"}, + {273, "augassign", 0, 2, states_17, + "\000\000\000\000\000\000\376\037\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + {274, "del_stmt", 0, 3, states_18, + "\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + {275, "pass_stmt", 0, 2, states_19, + "\000\000\000\000\000\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + {276, "flow_stmt", 0, 2, states_20, + "\000\000\000\000\000\000\000\000\340\001\000\000\000\000\000\000\000\000\000\000\200"}, + {277, "break_stmt", 0, 2, states_21, + "\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000"}, + {278, "continue_stmt", 0, 2, states_22, + "\000\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000\000\000\000\000"}, + {279, "return_stmt", 0, 3, states_23, + "\000\000\000\000\000\000\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000"}, + {280, "yield_stmt", 0, 2, states_24, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\200"}, + {281, "raise_stmt", 0, 5, states_25, + "\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000\000\000\000"}, + {282, "import_stmt", 0, 2, states_26, + "\000\000\000\000\000\000\000\000\000\022\000\000\000\000\000\000\000\000\000\000\000"}, + {283, "import_name", 0, 3, states_27, + "\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000\000"}, + {284, "import_from", 0, 8, states_28, + "\000\000\000\000\000\000\000\000\000\002\000\000\000\000\000\000\000\000\000\000\000"}, + {285, "import_as_name", 0, 4, states_29, + "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + {286, "dotted_as_name", 0, 4, states_30, + "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + {287, "import_as_names", 0, 3, states_31, + "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + {288, "dotted_as_names", 0, 2, states_32, + "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + {289, "dotted_name", 0, 2, states_33, + "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + {290, "global_stmt", 0, 3, states_34, + "\000\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000"}, + {291, "nonlocal_stmt", 0, 3, states_35, + "\000\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\000\000"}, + {292, "assert_stmt", 0, 5, states_36, + "\000\000\000\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000\000\000"}, + {293, "compound_stmt", 0, 2, states_37, + "\000\010\020\000\000\000\000\000\000\000\000\220\045\000\000\000\000\000\000\000\004"}, + {294, "if_stmt", 0, 8, states_38, + "\000\000\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000"}, + {295, "while_stmt", 0, 8, states_39, + "\000\000\000\000\000\000\000\000\000\000\000\200\000\000\000\000\000\000\000\000\000"}, + {296, "for_stmt", 0, 10, states_40, + "\000\000\000\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000"}, + {297, "try_stmt", 0, 13, states_41, + "\000\000\000\000\000\000\000\000\000\000\000\000\004\000\000\000\000\000\000\000\000"}, + {298, "with_stmt", 0, 5, states_42, + "\000\000\000\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000"}, + {299, "with_item", 0, 4, states_43, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000"}, + {300, "except_clause", 0, 5, states_44, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000"}, + {301, "suite", 0, 5, states_45, + "\004\040\040\200\000\000\000\240\340\223\160\000\000\200\020\000\000\206\120\076\200"}, + {302, "test", 0, 6, states_46, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000"}, + {303, "test_nocond", 0, 2, states_47, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000"}, + {304, "lambdef", 0, 5, states_48, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\200\000\000\000\000\000\000\000"}, + {305, "lambdef_nocond", 0, 5, states_49, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\200\000\000\000\000\000\000\000"}, + {306, "or_test", 0, 2, states_50, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\020\000\000\206\120\076\000"}, + {307, "and_test", 0, 2, states_51, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\020\000\000\206\120\076\000"}, + {308, "not_test", 0, 3, states_52, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\020\000\000\206\120\076\000"}, + {309, "comparison", 0, 2, states_53, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000"}, + {310, "comp_op", 0, 4, states_54, + "\000\000\000\000\000\000\000\000\000\000\000\000\002\000\220\177\000\000\000\000\000"}, + {311, "star_expr", 0, 3, states_55, + "\000\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + {312, "expr", 0, 2, states_56, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000"}, + {313, "xor_expr", 0, 2, states_57, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000"}, + {314, "and_expr", 0, 2, states_58, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000"}, + {315, "shift_expr", 0, 2, states_59, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000"}, + {316, "arith_expr", 0, 2, states_60, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000"}, + {317, "term", 0, 2, states_61, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000"}, + {318, "factor", 0, 3, states_62, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000"}, + {319, "power", 0, 4, states_63, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\000\120\076\000"}, + {320, "atom", 0, 9, states_64, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\000\120\076\000"}, + {321, "testlist_comp", 0, 5, states_65, + "\000\040\040\200\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000"}, {322, "trailer", 0, 7, states_66, - "\000\040\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\004\000\000\000"}, + "\000\040\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000\020\000\000"}, {323, "subscriptlist", 0, 3, states_67, - "\000\040\240\000\000\000\000\000\000\040\000\000\000\040\010\000\200\041\044\015\000\000"}, - {324, "subscript", 0, 7, states_68, - "\000\040\240\000\000\000\000\000\000\040\000\000\000\040\010\000\200\041\044\015\000\000"}, + "\000\040\040\002\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000"}, + {324, "subscript", 0, 5, states_68, + "\000\040\040\002\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000"}, {325, "sliceop", 0, 3, states_69, - "\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\002\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {326, "exprlist", 0, 3, states_70, - "\000\040\040\000\000\000\000\000\000\000\000\000\000\000\000\000\200\041\044\015\000\000"}, + "\000\040\040\200\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000"}, {327, "testlist", 0, 3, states_71, - "\000\040\040\000\000\000\000\000\000\000\000\000\000\040\010\000\200\041\044\015\000\000"}, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000"}, {328, "dictorsetmaker", 0, 11, states_72, - "\000\040\040\000\000\000\000\000\000\000\000\000\000\040\010\000\200\041\044\015\000\000"}, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000"}, {329, "classdef", 0, 8, states_73, - "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\002\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\004"}, {330, "arglist", 0, 8, states_74, - "\000\040\040\300\000\000\000\000\000\000\000\000\000\040\010\000\200\041\044\015\000\000"}, + "\000\040\040\200\001\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000"}, {331, "argument", 0, 4, states_75, - "\000\040\040\000\000\000\000\000\000\000\000\000\000\040\010\000\200\041\044\015\000\000"}, - {332, "list_iter", 0, 2, states_76, - "\000\000\000\000\000\000\000\000\000\000\000\020\001\000\000\000\000\000\000\000\000\000"}, - {333, "list_for", 0, 6, states_77, - "\000\000\000\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000\000"}, - {334, "list_if", 0, 4, states_78, - "\000\000\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000"}, - {335, "comp_iter", 0, 2, states_79, - "\000\000\000\000\000\000\000\000\000\000\000\020\001\000\000\000\000\000\000\000\000\000"}, - {336, "comp_for", 0, 6, states_80, - "\000\000\000\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000\000"}, - {337, "comp_if", 0, 4, states_81, - "\000\000\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000"}, - {338, "testlist1", 0, 2, states_82, - "\000\040\040\000\000\000\000\000\000\000\000\000\000\040\010\000\200\041\044\015\000\000"}, - {339, "encoding_decl", 0, 2, states_83, - "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, - {340, "yield_expr", 0, 3, states_84, - "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\001"}, -}; -static label labels[169] = { + "\000\040\040\000\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000"}, + {332, "comp_iter", 0, 2, states_76, + "\000\000\000\000\000\000\000\000\000\000\000\020\001\000\000\000\000\000\000\000\000"}, + {333, "comp_for", 0, 6, states_77, + "\000\000\000\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000"}, + {334, "comp_if", 0, 4, states_78, + "\000\000\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000"}, + {335, "encoding_decl", 0, 2, states_79, + "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + {336, "yield_expr", 0, 3, states_80, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\200"}, +}; +static label labels[168] = { {0, "EMPTY"}, {256, 0}, {4, 0}, - {268, 0}, - {292, 0}, + {269, 0}, + {293, 0}, {257, 0}, - {267, 0}, + {268, 0}, {0, 0}, {258, 0}, {327, 0}, {259, 0}, - {50, 0}, - {288, 0}, + {49, 0}, + {289, 0}, {7, 0}, {330, 0}, {8, 0}, @@ -2022,29 +1990,33 @@ static label labels[169] = { {1, "def"}, {1, 0}, {263, 0}, + {50, 0}, + {302, 0}, {11, 0}, - {300, 0}, + {301, 0}, {264, 0}, {265, 0}, {22, 0}, - {304, 0}, {12, 0}, {16, 0}, - {36, 0}, + {35, 0}, {266, 0}, - {269, 0}, - {13, 0}, + {267, 0}, {270, 0}, - {272, 0}, - {273, 0}, + {13, 0}, + {271, 0}, {274, 0}, {275, 0}, - {281, 0}, - {289, 0}, + {276, 0}, + {282, 0}, {290, 0}, {291, 0}, - {271, 0}, - {340, 0}, + {292, 0}, + {272, 0}, + {273, 0}, + {336, 0}, + {311, 0}, + {36, 0}, {37, 0}, {38, 0}, {39, 0}, @@ -2055,108 +2027,105 @@ static label labels[169] = { {44, 0}, {45, 0}, {46, 0}, - {47, 0}, - {49, 0}, - {1, "print"}, - {35, 0}, + {48, 0}, {1, "del"}, {326, 0}, {1, "pass"}, - {276, 0}, {277, 0}, {278, 0}, - {280, 0}, {279, 0}, + {281, 0}, + {280, 0}, {1, "break"}, {1, "continue"}, {1, "return"}, {1, "raise"}, - {282, 0}, + {1, "from"}, {283, 0}, + {284, 0}, {1, "import"}, - {287, 0}, - {1, "from"}, + {288, 0}, {23, 0}, - {286, 0}, - {284, 0}, - {1, "as"}, + {51, 0}, + {287, 0}, {285, 0}, + {1, "as"}, + {286, 0}, {1, "global"}, - {1, "exec"}, - {310, 0}, - {1, "in"}, + {1, "nonlocal"}, {1, "assert"}, - {293, 0}, {294, 0}, {295, 0}, {296, 0}, {297, 0}, + {298, 0}, {1, "if"}, {1, "elif"}, {1, "else"}, {1, "while"}, {1, "for"}, + {1, "in"}, {1, "try"}, - {299, 0}, + {300, 0}, {1, "finally"}, {1, "with"}, - {298, 0}, + {299, 0}, + {312, 0}, {1, "except"}, {5, 0}, {6, 0}, - {301, 0}, - {302, 0}, - {305, 0}, + {306, 0}, + {304, 0}, {303, 0}, + {305, 0}, {1, "lambda"}, - {321, 0}, - {306, 0}, - {1, "or"}, {307, 0}, + {1, "or"}, + {308, 0}, {1, "and"}, {1, "not"}, - {308, 0}, {309, 0}, + {310, 0}, {20, 0}, {21, 0}, - {28, 0}, - {31, 0}, + {27, 0}, {30, 0}, {29, 0}, - {29, 0}, + {28, 0}, + {28, 0}, {1, "is"}, - {311, 0}, - {18, 0}, - {312, 0}, - {33, 0}, {313, 0}, - {19, 0}, + {18, 0}, {314, 0}, - {34, 0}, + {32, 0}, {315, 0}, + {19, 0}, + {316, 0}, + {33, 0}, + {34, 0}, + {317, 0}, {14, 0}, {15, 0}, - {316, 0}, + {318, 0}, {17, 0}, {24, 0}, - {48, 0}, - {32, 0}, - {317, 0}, - {318, 0}, - {322, 0}, + {47, 0}, + {31, 0}, + {319, 0}, {320, 0}, + {322, 0}, + {321, 0}, {9, 0}, - {319, 0}, {10, 0}, - {26, 0}, - {328, 0}, - {27, 0}, {25, 0}, - {338, 0}, + {328, 0}, + {26, 0}, {2, 0}, {3, 0}, + {1, "None"}, + {1, "True"}, + {1, "False"}, {333, 0}, - {336, 0}, {323, 0}, {324, 0}, {325, 0}, @@ -2165,13 +2134,11 @@ static label labels[169] = { {332, 0}, {334, 0}, {335, 0}, - {337, 0}, - {339, 0}, {1, "yield"}, }; grammar _PyParser_Grammar = { - 85, + 81, dfas, - {169, labels}, + {168, labels}, 256 }; diff --git a/Python/import.c b/Python/import.c index 92363b3937..f655e514d1 100644 --- a/Python/import.c +++ b/Python/import.c @@ -5,13 +5,9 @@ #include "Python-ast.h" #undef Yield /* undefine macro conflicting with winbase.h */ -#include "pyarena.h" -#include "pythonrun.h" #include "errcode.h" #include "marshal.h" #include "code.h" -#include "compile.h" -#include "eval.h" #include "osdefs.h" #include "importdl.h" @@ -25,6 +21,8 @@ extern "C" { #ifdef MS_WINDOWS /* for stat.st_mode */ typedef unsigned short mode_t; +/* for _mkdir */ +#include <direct.h> #endif @@ -35,7 +33,7 @@ typedef unsigned short mode_t; a .pyc file in text mode the magic number will be wrong; also, the Apple MPW compiler swaps their values, botching string constants. - The magic numbers must be spaced apart atleast 2 values, as the + The magic numbers must be spaced apart at least 2 values, as the -U interpeter flag will cause MAGIC+1 being used. They have been odd numbers for some time now. @@ -43,6 +41,15 @@ typedef unsigned short mode_t; The current working scheme is to increment the previous value by 10. + Starting with the adoption of PEP 3147 in Python 3.2, every bump in magic + number also includes a new "magic tag", i.e. a human readable string used + to represent the magic number in __pycache__ directories. When you change + the magic number, you must also set a new unique magic tag. Generally this + can be named after the Python major version of the magic number bump, but + it can really be anything, as long as it's different than anything else + that's come before. The tags are included in the following table, starting + with Python 3.2a0. + Known values: Python 1.5: 20121 Python 1.5.1: 20121 @@ -71,39 +78,59 @@ typedef unsigned short mode_t; Python 2.5c2: 62131 (fix wrong code: for x, in ... in listcomp/genexp) Python 2.6a0: 62151 (peephole optimizations and STORE_MAP opcode) Python 2.6a1: 62161 (WITH_CLEANUP optimization) - Python 2.7a0: 62171 (optimize list comprehensions/change LIST_APPEND) - Python 2.7a0: 62181 (optimize conditional branches: - introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE) - Python 2.7a0 62191 (introduce SETUP_WITH) - Python 2.7a0 62201 (introduce BUILD_SET) - Python 2.7a0 62211 (introduce MAP_ADD and SET_ADD) -. + Python 3000: 3000 + 3010 (removed UNARY_CONVERT) + 3020 (added BUILD_SET) + 3030 (added keyword-only parameters) + 3040 (added signature annotations) + 3050 (print becomes a function) + 3060 (PEP 3115 metaclass syntax) + 3061 (string literals become unicode) + 3071 (PEP 3109 raise changes) + 3081 (PEP 3137 make __file__ and __name__ unicode) + 3091 (kill str8 interning) + 3101 (merge from 2.6a0, see 62151) + 3103 (__file__ points to source file) + Python 3.0a4: 3111 (WITH_CLEANUP optimization). + Python 3.0a5: 3131 (lexical exception stacking, including POP_EXCEPT) + Python 3.1a0: 3141 (optimize list, set and dict comprehensions: + change LIST_APPEND and SET_ADD, add MAP_ADD) + Python 3.1a0: 3151 (optimize conditional branches: + introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE) + Python 3.2a0: 3160 (add SETUP_WITH) + tag: cpython-32 + Python 3.2a1: 3170 (add DUP_TOP_TWO, remove DUP_TOPX and ROT_FOUR) + tag: cpython-32 + Python 3.2a2 3180 (add DELETE_DEREF) */ -#define MAGIC (62211 | ((long)'\r'<<16) | ((long)'\n'<<24)) -/* Magic word as global; note that _PyImport_Init() can change the - value of this global to accommodate for alterations of how the - compiler works which are enabled by command line switches. */ +/* MAGIC must change whenever the bytecode emitted by the compiler may no + longer be understood by older implementations of the eval loop (usually + due to the addition of new opcodes) + TAG must change for each major Python release. The magic number will take + care of any bytecode changes that occur during development. +*/ +#define MAGIC (3180 | ((long)'\r'<<16) | ((long)'\n'<<24)) +#define TAG "cpython-32" +#define CACHEDIR "__pycache__" +/* Current magic word and string tag as globals. */ static long pyc_magic = MAGIC; +static const char *pyc_tag = TAG; -/* See _PyImport_FixupExtension() below */ +/* See _PyImport_FixupExtensionUnicode() below */ static PyObject *extensions = NULL; /* This table is defined in config.c: */ extern struct _inittab _PyImport_Inittab[]; +/* Method from Parser/tokenizer.c */ +extern char * PyTokenizer_FindEncoding(int); + struct _inittab *PyImport_Inittab = _PyImport_Inittab; /* these tables define the module suffixes that Python recognizes */ struct filedescr * _PyImport_Filetab = NULL; -#ifdef RISCOS -static const struct filedescr _PyImport_StandardFiletab[] = { - {"/py", "U", PY_SOURCE}, - {"/pyc", "rb", PY_COMPILED}, - {0, 0} -}; -#else static const struct filedescr _PyImport_StandardFiletab[] = { {".py", "U", PY_SOURCE}, #ifdef MS_WINDOWS @@ -112,36 +139,7 @@ static const struct filedescr _PyImport_StandardFiletab[] = { {".pyc", "rb", PY_COMPILED}, {0, 0} }; -#endif -#ifdef MS_WINDOWS -static int isdir(char *path) { - DWORD rv; - /* see issue1293 and issue3677: - * stat() on Windows doesn't recognise paths like - * "e:\\shared\\" and "\\\\whiterab-c2znlh\\shared" as dirs. - * Also reference issue6727: - * stat() on Windows is broken and doesn't resolve symlinks properly. - */ - rv = GetFileAttributesA(path); - return rv != INVALID_FILE_ATTRIBUTES && rv & FILE_ATTRIBUTE_DIRECTORY; -} -#else -#ifdef HAVE_STAT -static int isdir(char *path) { - struct stat statbuf; - return stat(path, &statbuf) == 0 && S_ISDIR(statbuf.st_mode); -} -#else -#ifdef RISCOS -/* with RISCOS, isdir is in unixstuff */ -#else -int isdir(char *path) { - return 0; -} -#endif /* RISCOS */ -#endif /* HAVE_STAT */ -#endif /* MS_WINDOWS */ /* Initialize things */ @@ -178,22 +176,10 @@ _PyImport_Init(void) if (Py_OptimizeFlag) { /* Replace ".pyc" with ".pyo" in _PyImport_Filetab */ for (; filetab->suffix != NULL; filetab++) { -#ifndef RISCOS if (strcmp(filetab->suffix, ".pyc") == 0) filetab->suffix = ".pyo"; -#else - if (strcmp(filetab->suffix, "/pyc") == 0) - filetab->suffix = "/pyo"; -#endif } } - - if (Py_UnicodeFlag) { - /* Fix the pyc_magic so that byte compiled code created - using the all-Unicode method doesn't interfere with - code created in normal operation mode. */ - pyc_magic = MAGIC + 1; - } } void @@ -266,16 +252,6 @@ _PyImportHooks_Init(void) Py_DECREF(path_hooks); } -void -_PyImport_Fini(void) -{ - Py_XDECREF(extensions); - extensions = NULL; - PyMem_DEL(_PyImport_Filetab); - _PyImport_Filetab = NULL; -} - - /* Locking primitives to prevent parallel imports of the same module in different threads to return with a partially loaded module. These calls are serialized by the global interpreter lock. */ @@ -339,8 +315,17 @@ _PyImport_ReInitLock(void) { if (import_lock != NULL) import_lock = PyThread_allocate_lock(); - import_lock_thread = -1; - import_lock_level = 0; + if (import_lock_level > 1) { + /* Forked as a side effect of import */ + long me = PyThread_get_thread_ident(); + PyThread_acquire_lock(import_lock, 0); + /* XXX: can the previous line fail? */ + import_lock_thread = me; + import_lock_level--; + } else { + import_lock_thread = -1; + import_lock_level = 0; + } } #endif @@ -379,6 +364,21 @@ imp_release_lock(PyObject *self, PyObject *noargs) return Py_None; } +void +_PyImport_Fini(void) +{ + Py_XDECREF(extensions); + extensions = NULL; + PyMem_DEL(_PyImport_Filetab); + _PyImport_Filetab = NULL; +#ifdef WITH_THREAD + if (import_lock != NULL) { + PyThread_free_lock(import_lock); + import_lock = NULL; + } +#endif +} + static void imp_modules_reloading_clear(void) { @@ -401,8 +401,7 @@ PyImport_GetModuleDict(void) /* List of names to clear in sys */ static char* sys_deletes[] = { - "path", "argv", "ps1", "ps2", "exitfunc", - "exc_type", "exc_value", "exc_traceback", + "path", "argv", "ps1", "ps2", "last_type", "last_value", "last_traceback", "path_hooks", "path_importer_cache", "meta_path", /* misc stuff */ @@ -438,11 +437,11 @@ PyImport_Cleanup(void) deleted *last* of all, they would come too late in the normal destruction order. Sigh. */ - value = PyDict_GetItemString(modules, "__builtin__"); + value = PyDict_GetItemString(modules, "builtins"); if (value != NULL && PyModule_Check(value)) { dict = PyModule_GetDict(value); if (Py_VerboseFlag) - PySys_WriteStderr("# clear __builtin__._\n"); + PySys_WriteStderr("# clear builtins._\n"); PyDict_SetItemString(dict, "_", Py_None); } value = PyDict_GetItemString(modules, "sys"); @@ -474,11 +473,11 @@ PyImport_Cleanup(void) PyDict_SetItemString(modules, "__main__", Py_None); } - /* The special treatment of __builtin__ here is because even + /* The special treatment of "builtins" here is because even when it's not referenced as a module, its dictionary is referenced by almost every module's __builtins__. Since deleting a module clears its dictionary (even if there are - references left to it), we need to delete the __builtin__ + references left to it), we need to delete the "builtins" module last. Likewise, we don't delete sys until the very end because it is implicitly referenced (e.g. by print). @@ -489,16 +488,16 @@ PyImport_Cleanup(void) re-imported. */ /* Next, repeatedly delete modules with a reference count of - one (skipping __builtin__ and sys) and delete them */ + one (skipping builtins and sys) and delete them */ do { ndone = 0; pos = 0; while (PyDict_Next(modules, &pos, &key, &value)) { if (value->ob_refcnt != 1) continue; - if (PyString_Check(key) && PyModule_Check(value)) { - name = PyString_AS_STRING(key); - if (strcmp(name, "__builtin__") == 0) + if (PyUnicode_Check(key) && PyModule_Check(value)) { + name = _PyUnicode_AsString(key); + if (strcmp(name, "builtins") == 0) continue; if (strcmp(name, "sys") == 0) continue; @@ -512,12 +511,12 @@ PyImport_Cleanup(void) } } while (ndone > 0); - /* Next, delete all modules (still skipping __builtin__ and sys) */ + /* Next, delete all modules (still skipping builtins and sys) */ pos = 0; while (PyDict_Next(modules, &pos, &key, &value)) { - if (PyString_Check(key) && PyModule_Check(value)) { - name = PyString_AS_STRING(key); - if (strcmp(name, "__builtin__") == 0) + if (PyUnicode_Check(key) && PyModule_Check(value)) { + name = _PyUnicode_AsString(key); + if (strcmp(name, "builtins") == 0) continue; if (strcmp(name, "sys") == 0) continue; @@ -528,7 +527,7 @@ PyImport_Cleanup(void) } } - /* Next, delete sys and __builtin__ (in that order) */ + /* Next, delete sys and builtins (in that order) */ value = PyDict_GetItemString(modules, "sys"); if (value != NULL && PyModule_Check(value)) { if (Py_VerboseFlag) @@ -536,12 +535,12 @@ PyImport_Cleanup(void) _PyModule_Clear(value); PyDict_SetItemString(modules, "sys", Py_None); } - value = PyDict_GetItemString(modules, "__builtin__"); + value = PyDict_GetItemString(modules, "builtins"); if (value != NULL && PyModule_Check(value)) { if (Py_VerboseFlag) - PySys_WriteStderr("# cleanup __builtin__\n"); + PySys_WriteStderr("# cleanup builtins\n"); _PyModule_Clear(value); - PyDict_SetItemString(modules, "__builtin__", Py_None); + PyDict_SetItemString(modules, "builtins", Py_None); } /* Finally, clear and delete the modules directory */ @@ -552,7 +551,7 @@ PyImport_Cleanup(void) } -/* Helper for pythonrun.c -- return magic number */ +/* Helper for pythonrun.c -- return magic number and tag. */ long PyImport_GetMagicNumber(void) @@ -561,66 +560,141 @@ PyImport_GetMagicNumber(void) } +const char * +PyImport_GetMagicTag(void) +{ + return pyc_tag; +} + /* Magic for extension modules (built-in as well as dynamically loaded). To prevent initializing an extension module more than once, we keep a static dictionary 'extensions' keyed by module name (for built-in modules) or by filename (for dynamically loaded modules), containing these modules. A copy of the module's - dictionary is stored by calling _PyImport_FixupExtension() + dictionary is stored by calling _PyImport_FixupExtensionUnicode() immediately after the module initialization function succeeds. A copy can be retrieved from there by calling - _PyImport_FindExtension(). */ + _PyImport_FindExtensionUnicode(). -PyObject * -_PyImport_FixupExtension(char *name, char *filename) + Modules which do support multiple initialization set their m_size + field to a non-negative number (indicating the size of the + module-specific state). They are still recorded in the extensions + dictionary, to avoid loading shared libraries twice. +*/ + +int +_PyImport_FixupExtensionUnicode(PyObject *mod, char *name, PyObject *filename) { - PyObject *modules, *mod, *dict, *copy; + PyObject *modules, *dict; + struct PyModuleDef *def; if (extensions == NULL) { extensions = PyDict_New(); if (extensions == NULL) - return NULL; + return -1; } - modules = PyImport_GetModuleDict(); - mod = PyDict_GetItemString(modules, name); if (mod == NULL || !PyModule_Check(mod)) { - PyErr_Format(PyExc_SystemError, - "_PyImport_FixupExtension: module %.200s not loaded", name); - return NULL; + PyErr_BadInternalCall(); + return -1; } - dict = PyModule_GetDict(mod); - if (dict == NULL) - return NULL; - copy = PyDict_Copy(dict); - if (copy == NULL) - return NULL; - PyDict_SetItemString(extensions, filename, copy); - Py_DECREF(copy); - return copy; + def = PyModule_GetDef(mod); + if (!def) { + PyErr_BadInternalCall(); + return -1; + } + modules = PyImport_GetModuleDict(); + if (PyDict_SetItemString(modules, name, mod) < 0) + return -1; + if (_PyState_AddModule(mod, def) < 0) { + PyDict_DelItemString(modules, name); + return -1; + } + if (def->m_size == -1) { + if (def->m_base.m_copy) { + /* Somebody already imported the module, + likely under a different name. + XXX this should really not happen. */ + Py_DECREF(def->m_base.m_copy); + def->m_base.m_copy = NULL; + } + dict = PyModule_GetDict(mod); + if (dict == NULL) + return -1; + def->m_base.m_copy = PyDict_Copy(dict); + if (def->m_base.m_copy == NULL) + return -1; + } + PyDict_SetItem(extensions, filename, (PyObject*)def); + return 0; +} + +int +_PyImport_FixupBuiltin(PyObject *mod, char *name) +{ + int res; + PyObject *filename; + filename = PyUnicode_FromString(name); + if (filename == NULL) + return -1; + res = _PyImport_FixupExtensionUnicode(mod, name, filename); + Py_DECREF(filename); + return res; } PyObject * -_PyImport_FindExtension(char *name, char *filename) +_PyImport_FindExtensionUnicode(char *name, PyObject *filename) { - PyObject *dict, *mod, *mdict; + PyObject *mod, *mdict; + PyModuleDef* def; if (extensions == NULL) return NULL; - dict = PyDict_GetItemString(extensions, filename); - if (dict == NULL) + def = (PyModuleDef*)PyDict_GetItem(extensions, filename); + if (def == NULL) return NULL; - mod = PyImport_AddModule(name); - if (mod == NULL) - return NULL; - mdict = PyModule_GetDict(mod); - if (mdict == NULL) - return NULL; - if (PyDict_Update(mdict, dict)) + if (def->m_size == -1) { + /* Module does not support repeated initialization */ + if (def->m_base.m_copy == NULL) + return NULL; + mod = PyImport_AddModule(name); + if (mod == NULL) + return NULL; + mdict = PyModule_GetDict(mod); + if (mdict == NULL) + return NULL; + if (PyDict_Update(mdict, def->m_base.m_copy)) + return NULL; + } + else { + if (def->m_base.m_init == NULL) + return NULL; + mod = def->m_base.m_init(); + if (mod == NULL) + return NULL; + PyDict_SetItemString(PyImport_GetModuleDict(), name, mod); + Py_DECREF(mod); + } + if (_PyState_AddModule(mod, def) < 0) { + PyDict_DelItemString(PyImport_GetModuleDict(), name); + Py_DECREF(mod); return NULL; + } if (Py_VerboseFlag) - PySys_WriteStderr("import %s # previously loaded (%s)\n", - name, filename); + PySys_FormatStderr("import %s # previously loaded (%U)\n", + name, filename); return mod; + } +PyObject * +_PyImport_FindBuiltin(char *name) +{ + PyObject *res, *filename; + filename = PyUnicode_FromString(name); + if (filename == NULL) + return NULL; + res = _PyImport_FindExtensionUnicode(name, filename); + Py_DECREF(filename); + return res; +} /* Get the module object corresponding to a module name. First check the modules dictionary if there's one there, @@ -661,22 +735,39 @@ remove_module(const char *name) "sys.modules failed"); } +static PyObject * get_sourcefile(char *file); +static char *make_source_pathname(char *pathname, char *buf); +static char *make_compiled_pathname(char *pathname, char *buf, size_t buflen, + int debug); + /* Execute a code object in a module and return the module object * WITH INCREMENTED REFERENCE COUNT. If an error occurs, name is * removed from sys.modules, to avoid leaving damaged module objects * in sys.modules. The caller may wish to restore the original * module object (if any) in this case; PyImport_ReloadModule is an * example. + * + * Note that PyImport_ExecCodeModuleWithPathnames() is the preferred, richer + * interface. The other two exist primarily for backward compatibility. */ PyObject * PyImport_ExecCodeModule(char *name, PyObject *co) { - return PyImport_ExecCodeModuleEx(name, co, (char *)NULL); + return PyImport_ExecCodeModuleWithPathnames( + name, co, (char *)NULL, (char *)NULL); } PyObject * PyImport_ExecCodeModuleEx(char *name, PyObject *co, char *pathname) { + return PyImport_ExecCodeModuleWithPathnames( + name, co, pathname, (char *)NULL); +} + +PyObject * +PyImport_ExecCodeModuleWithPathnames(char *name, PyObject *co, char *pathname, + char *cpathname) +{ PyObject *modules = PyImport_GetModuleDict(); PyObject *m, *d, *v; @@ -694,7 +785,7 @@ PyImport_ExecCodeModuleEx(char *name, PyObject *co, char *pathname) /* Remember the filename as the __file__ attribute */ v = NULL; if (pathname != NULL) { - v = PyString_FromString(pathname); + v = get_sourcefile(pathname); if (v == NULL) PyErr_Clear(); } @@ -706,7 +797,21 @@ PyImport_ExecCodeModuleEx(char *name, PyObject *co, char *pathname) PyErr_Clear(); /* Not important enough to report */ Py_DECREF(v); - v = PyEval_EvalCode((PyCodeObject *)co, d, d); + /* Remember the pyc path name as the __cached__ attribute. */ + if (cpathname == NULL) { + v = Py_None; + Py_INCREF(v); + } + else if ((v = PyUnicode_FromString(cpathname)) == NULL) { + PyErr_Clear(); /* Not important enough to report */ + v = Py_None; + Py_INCREF(v); + } + if (PyDict_SetItemString(d, "__cached__", v) != 0) + PyErr_Clear(); /* Not important enough to report */ + Py_DECREF(v); + + v = PyEval_EvalCode(co, d, d); if (v == NULL) goto error; Py_DECREF(v); @@ -728,32 +833,189 @@ PyImport_ExecCodeModuleEx(char *name, PyObject *co, char *pathname) } +/* Like strrchr(string, '/') but searches for the rightmost of either SEP + or ALTSEP, if the latter is defined. +*/ +static char * +rightmost_sep(char *s) +{ + char *found, c; + for (found = NULL; (c = *s); s++) { + if (c == SEP +#ifdef ALTSEP + || c == ALTSEP +#endif + ) + { + found = s; + } + } + return found; +} + + /* Given a pathname for a Python source file, fill a buffer with the pathname for the corresponding compiled file. Return the pathname for the compiled file, or NULL if there's no space in the buffer. Doesn't set an exception. */ static char * -make_compiled_pathname(char *pathname, char *buf, size_t buflen) +make_compiled_pathname(char *pathname, char *buf, size_t buflen, int debug) { + /* foo.py -> __pycache__/foo.<tag>.pyc */ size_t len = strlen(pathname); - if (len+2 > buflen) + size_t i, save; + char *pos; + int sep = SEP; + + /* Sanity check that the buffer has roughly enough space to hold what + will eventually be the full path to the compiled file. The 5 extra + bytes include the slash afer __pycache__, the two extra dots, the + extra trailing character ('c' or 'o') and null. This isn't exact + because the contents of the buffer can affect how many actual + characters of the string get into the buffer. We'll do a final + sanity check before writing the extension to ensure we do not + overflow the buffer. + */ + if (len + strlen(CACHEDIR) + strlen(pyc_tag) + 5 > buflen) return NULL; -#ifdef MS_WINDOWS - /* Treat .pyw as if it were .py. The case of ".pyw" must match - that used in _PyImport_StandardFiletab. */ - if (len >= 4 && strcmp(&pathname[len-4], ".pyw") == 0) - --len; /* pretend 'w' isn't there */ + /* Find the last path separator and copy everything from the start of + the source string up to and including the separator. + */ + if ((pos = rightmost_sep(pathname)) == NULL) { + i = 0; + } + else { + sep = *pos; + i = pos - pathname + 1; + strncpy(buf, pathname, i); + } + + save = i; + buf[i++] = '\0'; + /* Add __pycache__/ */ + strcat(buf, CACHEDIR); + i += strlen(CACHEDIR) - 1; + buf[i++] = sep; + buf[i++] = '\0'; + /* Add the base filename, but remove the .py or .pyw extension, since + the tag name must go before the extension. + */ + strcat(buf, pathname + save); + if ((pos = strrchr(buf, '.')) != NULL) + *++pos = '\0'; + strcat(buf, pyc_tag); + /* The length test above assumes that we're only adding one character + to the end of what would normally be the extension. What if there + is no extension, or the string ends in '.' or '.p', and otherwise + fills the buffer? By appending 4 more characters onto the string + here, we could overrun the buffer. + + As a simple example, let's say buflen=32 and the input string is + 'xxx.py'. strlen() would be 6 and the test above would yield: + + (6 + 11 + 10 + 5 == 32) > 32 + + which is false and so the name mangling would continue. This would + be fine because we'd end up with this string in buf: + + __pycache__/xxx.cpython-32.pyc\0 + + strlen(of that) == 30 + the nul fits inside a 32 character buffer. + We can even handle an input string of say 'xxxxx' above because + that's (5 + 11 + 10 + 5 == 31) > 32 which is also false. Name + mangling that yields: + + __pycache__/xxxxxcpython-32.pyc\0 + + which is 32 characters including the nul, and thus fits in the + buffer. However, an input string of 'xxxxxx' would yield a result + string of: + + __pycache__/xxxxxxcpython-32.pyc\0 + + which is 33 characters long (including the nul), thus overflowing + the buffer, even though the first test would fail, i.e.: the input + string is also 6 characters long, so 32 > 32 is false. + + The reason the first test fails but we still overflow the buffer is + that the test above only expects to add one extra character to be + added to the extension, and here we're adding three (pyc). We + don't add the first dot, so that reclaims one of expected + positions, leaving us overflowing by 1 byte (3 extra - 1 reclaimed + dot - 1 expected extra == 1 overflowed). + + The best we can do is ensure that we still have enough room in the + target buffer before we write the extension. Because it's always + only the extension that can cause the overflow, and never the other + path bytes we've written, it's sufficient to just do one more test + here. Still, the assertion that follows can't hurt. + */ +#if 0 + printf("strlen(buf): %d; buflen: %d\n", (int)strlen(buf), (int)buflen); #endif - memcpy(buf, pathname, len); - buf[len] = Py_OptimizeFlag ? 'o' : 'c'; - buf[len+1] = '\0'; - + if (strlen(buf) + 5 > buflen) + return NULL; + strcat(buf, debug ? ".pyc" : ".pyo"); + assert(strlen(buf) < buflen); return buf; } +/* Given a pathname to a Python byte compiled file, return the path to the + source file, if the path matches the PEP 3147 format. This does not check + for any file existence, however, if the pyc file name does not match PEP + 3147 style, NULL is returned. buf must be at least as big as pathname; + the resulting path will always be shorter. */ + +static char * +make_source_pathname(char *pathname, char *buf) +{ + /* __pycache__/foo.<tag>.pyc -> foo.py */ + size_t i, j; + char *left, *right, *dot0, *dot1, sep; + + /* Look back two slashes from the end. In between these two slashes + must be the string __pycache__ or this is not a PEP 3147 style + path. It's possible for there to be only one slash. + */ + if ((right = rightmost_sep(pathname)) == NULL) + return NULL; + sep = *right; + *right = '\0'; + left = rightmost_sep(pathname); + *right = sep; + if (left == NULL) + left = pathname; + else + left++; + if (right-left != strlen(CACHEDIR) || + strncmp(left, CACHEDIR, right-left) != 0) + return NULL; + + /* Now verify that the path component to the right of the last slash + has two dots in it. + */ + if ((dot0 = strchr(right + 1, '.')) == NULL) + return NULL; + if ((dot1 = strchr(dot0 + 1, '.')) == NULL) + return NULL; + /* Too many dots? */ + if (strchr(dot1 + 1, '.') != NULL) + return NULL; + + /* This is a PEP 3147 path. Start by copying everything from the + start of pathname up to and including the leftmost slash. Then + copy the file's basename, removing the magic tag and adding a .py + suffix. + */ + strncpy(buf, pathname, (i=left-pathname)); + strncpy(buf+i, right+1, (j=dot0-right)); + strcpy(buf+i+j, "py"); + return buf; +} + /* Given a pathname for a Python source file, its time of last modification, and a pathname for a compiled file, check whether the compiled file represents the same version of the source. If so, @@ -834,7 +1096,8 @@ load_compiled_module(char *name, char *cpathname, FILE *fp) if (Py_VerboseFlag) PySys_WriteStderr("import %s # precompiled from %s\n", name, cpathname); - m = PyImport_ExecCodeModuleEx(name, (PyObject *)co, cpathname); + m = PyImport_ExecCodeModuleWithPathnames( + name, (PyObject *)co, cpathname, cpathname); Py_DECREF(co); return m; @@ -853,8 +1116,8 @@ parse_source_module(const char *pathname, FILE *fp) return NULL; flags.cf_flags = 0; - - mod = PyParser_ASTFromFile(fp, pathname, Py_file_input, 0, 0, &flags, + mod = PyParser_ASTFromFile(fp, pathname, NULL, + Py_file_input, 0, 0, &flags, NULL, arena); if (mod) { co = PyAST_Compile(mod, pathname, NULL, arena); @@ -904,9 +1167,12 @@ open_exclusive(char *filename, mode_t mode) remove the file. */ static void -write_compiled_module(PyCodeObject *co, char *cpathname, struct stat *srcstat, time_t mtime) +write_compiled_module(PyCodeObject *co, char *cpathname, struct stat *srcstat) { FILE *fp; + char *dirpath; + time_t mtime = srcstat->st_mtime; + int saved; #ifdef MS_WINDOWS /* since Windows uses different permissions */ mode_t mode = srcstat->st_mode & ~S_IEXEC; /* Issue #6074: We ensure user write access, so we can delete it later @@ -917,8 +1183,36 @@ write_compiled_module(PyCodeObject *co, char *cpathname, struct stat *srcstat, t mode |= _S_IWRITE; #else mode_t mode = srcstat->st_mode & ~S_IXUSR & ~S_IXGRP & ~S_IXOTH; + mode_t dirmode = (srcstat->st_mode | + S_IXUSR | S_IXGRP | S_IXOTH | + S_IWUSR | S_IWGRP | S_IWOTH); #endif + /* Ensure that the __pycache__ directory exists. */ + dirpath = rightmost_sep(cpathname); + if (dirpath == NULL) { + if (Py_VerboseFlag) + PySys_WriteStderr( + "# no %s path found %s\n", + CACHEDIR, cpathname); + return; + } + saved = *dirpath; + *dirpath = '\0'; + +#ifdef MS_WINDOWS + if (_mkdir(cpathname) < 0 && errno != EEXIST) { +#else + if (mkdir(cpathname, dirmode) < 0 && errno != EEXIST) { +#endif + *dirpath = saved; + if (Py_VerboseFlag) + PySys_WriteStderr( + "# cannot create cache dir %s\n", cpathname); + return; + } + *dirpath = saved; + fp = open_exclusive(cpathname, mode); if (fp == NULL) { if (Py_VerboseFlag) @@ -954,7 +1248,7 @@ update_code_filenames(PyCodeObject *co, PyObject *oldname, PyObject *newname) PyObject *constants, *tmp; Py_ssize_t i, n; - if (!_PyString_Eq(co->co_filename, oldname)) + if (PyUnicode_Compare(co->co_filename, oldname)) return; tmp = co->co_filename; @@ -977,13 +1271,15 @@ update_compiled_module(PyCodeObject *co, char *pathname) { PyObject *oldname, *newname; - if (strcmp(PyString_AsString(co->co_filename), pathname) == 0) - return 0; - - newname = PyString_FromString(pathname); + newname = PyUnicode_DecodeFSDefault(pathname); if (newname == NULL) return -1; + if (!PyUnicode_Compare(co->co_filename, newname)) { + Py_DECREF(newname); + return 0; + } + oldname = co->co_filename; Py_INCREF(oldname); update_code_filenames(co, oldname, newname); @@ -992,38 +1288,6 @@ update_compiled_module(PyCodeObject *co, char *pathname) return 1; } -#ifdef MS_WINDOWS - -/* Seconds between 1.1.1601 and 1.1.1970 */ -static __int64 secs_between_epochs = 11644473600; - -/* Get mtime from file pointer. */ - -static time_t -win32_mtime(FILE *fp, char *pathname) -{ - __int64 filetime; - HANDLE fh; - BY_HANDLE_FILE_INFORMATION file_information; - - fh = (HANDLE)_get_osfhandle(fileno(fp)); - if (fh == INVALID_HANDLE_VALUE || - !GetFileInformationByHandle(fh, &file_information)) { - PyErr_Format(PyExc_RuntimeError, - "unable to get file status from '%s'", - pathname); - return -1; - } - /* filetime represents the number of 100ns intervals since - 1.1.1601 (UTC). Convert to seconds since 1.1.1970 (UTC). */ - filetime = (__int64)file_information.ftLastWriteTime.dwHighDateTime << 32 | - file_information.ftLastWriteTime.dwLowDateTime; - return filetime / 10000000 - secs_between_epochs; -} - -#endif /* #ifdef MS_WINDOWS */ - - /* Load a source module from a given file and return its module object WITH INCREMENTED REFERENCE COUNT. If there's a matching byte-compiled file, use that instead. */ @@ -1037,7 +1301,6 @@ load_source_module(char *name, char *pathname, FILE *fp) char *cpathname; PyCodeObject *co = NULL; PyObject *m; - time_t mtime; if (fstat(fileno(fp), &st) != 0) { PyErr_Format(PyExc_RuntimeError, @@ -1045,30 +1308,22 @@ load_source_module(char *name, char *pathname, FILE *fp) pathname); return NULL; } - -#ifdef MS_WINDOWS - mtime = win32_mtime(fp, pathname); - if (mtime == (time_t)-1 && PyErr_Occurred()) - return NULL; -#else - mtime = st.st_mtime; -#endif - if (sizeof mtime > 4) { + if (sizeof st.st_mtime > 4) { /* Python's .pyc timestamp handling presumes that the timestamp fits in 4 bytes. Since the code only does an equality comparison, ordering is not important and we can safely ignore the higher bits (collisions are extremely unlikely). */ - mtime &= 0xFFFFFFFF; + st.st_mtime &= 0xFFFFFFFF; } buf = PyMem_MALLOC(MAXPATHLEN+1); if (buf == NULL) { return PyErr_NoMemory(); } - cpathname = make_compiled_pathname(pathname, buf, - (size_t)MAXPATHLEN + 1); + cpathname = make_compiled_pathname( + pathname, buf, (size_t)MAXPATHLEN + 1, !Py_OptimizeFlag); if (cpathname != NULL && - (fpc = check_compiled_module(pathname, mtime, cpathname))) { + (fpc = check_compiled_module(pathname, st.st_mtime, cpathname))) { co = read_compiled_module(cpathname, fpc); fclose(fpc); if (co == NULL) @@ -1093,10 +1348,11 @@ load_source_module(char *name, char *pathname, FILE *fp) if (b < 0) goto error_exit; if (!b) - write_compiled_module(co, cpathname, &st, mtime); + write_compiled_module(co, cpathname, &st); } } - m = PyImport_ExecCodeModuleEx(name, (PyObject *)co, pathname); + m = PyImport_ExecCodeModuleWithPathnames( + name, (PyObject *)co, pathname, cpathname); Py_DECREF(co); PyMem_FREE(buf); @@ -1108,12 +1364,56 @@ error_exit: return NULL; } +/* Get source file -> unicode or None + * Returns the path to the py file if available, else the given path + */ +static PyObject * +get_sourcefile(char *file) +{ + char *py = NULL; + Py_ssize_t len; + PyObject *u; + struct stat statbuf; + + if (!file || !*file) { + Py_RETURN_NONE; + } + + len = strlen(file); + /* match '*.py?' */ + if (len > MAXPATHLEN || PyOS_strnicmp(&file[len-4], ".py", 3) != 0) { + return PyUnicode_DecodeFSDefault(file); + } + + py = PyMem_MALLOC(MAXPATHLEN+1); + if (py == NULL) { + return PyErr_NoMemory(); + } + /* Start by trying to turn PEP 3147 path into source path. If that + * fails, just chop off the trailing character, i.e. legacy pyc path + * to py. + */ + if (make_source_pathname(file, py) == NULL) { + strncpy(py, file, len-1); + py[len-1] = '\0'; + } + + if (stat(py, &statbuf) == 0 && + S_ISREG(statbuf.st_mode)) { + u = PyUnicode_DecodeFSDefault(py); + } + else { + u = PyUnicode_DecodeFSDefault(file); + } + PyMem_FREE(py); + return u; +} /* Forward */ static PyObject *load_module(char *, FILE *, char *, int, PyObject *); static struct filedescr *find_module(char *, char *, PyObject *, char *, size_t, FILE **, PyObject **); -static struct _frozen *find_frozen(char *name); +static struct _frozen * find_frozen(char *); /* Load a package and return its module object WITH INCREMENTED REFERENCE COUNT */ @@ -1136,7 +1436,7 @@ load_package(char *name, char *pathname) PySys_WriteStderr("import %s # directory %s\n", name, pathname); d = PyModule_GetDict(m); - file = PyString_FromString(pathname); + file = get_sourcefile(pathname); if (file == NULL) goto error; path = Py_BuildValue("[O]", file); @@ -1298,6 +1598,7 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, char *filemode; FILE *fp = NULL; PyObject *path_hooks, *path_importer_cache; + struct stat statbuf; static struct filedescr fd_frozen = {"", "", PY_FROZEN}; static struct filedescr fd_builtin = {"", "", C_BUILTIN}; static struct filedescr fd_package = {"", "", PKG_DIRECTORY}; @@ -1358,39 +1659,18 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, Py_DECREF(meta_path); } - if (path != NULL && PyString_Check(path)) { - /* The only type of submodule allowed inside a "frozen" - package are other frozen modules or packages. */ - if (PyString_Size(path) + 1 + strlen(name) >= (size_t)buflen) { - PyErr_SetString(PyExc_ImportError, - "full frozen module name too long"); - goto error_exit; - } - strcpy(buf, PyString_AsString(path)); - strcat(buf, "."); - strcat(buf, name); - strcpy(name, buf); - if (find_frozen(name) != NULL) { - strcpy(buf, name); - PyMem_FREE(name); - return &fd_frozen; - } - PyErr_Format(PyExc_ImportError, - "No frozen submodule named %.200s", name); - goto error_exit; + if (find_frozen(fullname) != NULL) { + strcpy(buf, fullname); + PyMem_FREE(name); + return &fd_frozen; } + if (path == NULL) { if (is_builtin(name)) { strcpy(buf, name); PyMem_FREE(name); return &fd_builtin; } - if ((find_frozen(name)) != NULL) { - strcpy(buf, name); - PyMem_FREE(name); - return &fd_frozen; - } - #ifdef MS_COREDLL fp = PyWin_FindRegisteredModule(name, &fdp, buf, buflen); if (fp != NULL) { @@ -1401,6 +1681,7 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, #endif path = PySys_GetObject("path"); } + if (path == NULL || !PyList_Check(path)) { PyErr_SetString(PyExc_RuntimeError, "sys.path must be a list of directory names"); @@ -1425,30 +1706,33 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, npath = PyList_Size(path); namelen = strlen(name); for (i = 0; i < npath; i++) { - PyObject *copy = NULL; PyObject *v = PyList_GetItem(path, i); + PyObject *origv = v; + const char *base; + Py_ssize_t size; if (!v) goto error_exit; -#ifdef Py_USING_UNICODE if (PyUnicode_Check(v)) { - copy = PyUnicode_Encode(PyUnicode_AS_UNICODE(v), - PyUnicode_GET_SIZE(v), Py_FileSystemDefaultEncoding, NULL); - if (copy == NULL) + v = PyUnicode_EncodeFSDefault(v); + if (v == NULL) goto error_exit; - v = copy; } - else -#endif - if (!PyString_Check(v)) + else if (!PyBytes_Check(v)) continue; - len = PyString_GET_SIZE(v); + else + Py_INCREF(v); + + base = PyBytes_AS_STRING(v); + size = PyBytes_GET_SIZE(v); + len = size; if (len + 2 + namelen + MAXSUFFIXSIZE >= buflen) { - Py_XDECREF(copy); + Py_DECREF(v); continue; /* Too long */ } - strcpy(buf, PyString_AS_STRING(v)); + strcpy(buf, base); + Py_DECREF(v); + if (strlen(buf) != len) { - Py_XDECREF(copy); continue; /* v contains '\0' */ } @@ -1457,9 +1741,8 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, PyObject *importer; importer = get_path_importer(path_importer_cache, - path_hooks, v); + path_hooks, origv); if (importer == NULL) { - Py_XDECREF(copy); goto error_exit; } /* Note: importer is a borrowed reference */ @@ -1468,7 +1751,6 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, loader = PyObject_CallMethod(importer, "find_module", "s", fullname); - Py_XDECREF(copy); if (loader == NULL) goto error_exit; /* error */ if (loader != Py_None) { @@ -1494,25 +1776,28 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, /* Check for package import (buf holds a directory name, and there's an __init__ module in that directory */ - if (isdir(buf) && /* it's an existing directory */ +#ifdef HAVE_STAT + if (stat(buf, &statbuf) == 0 && /* it exists */ + S_ISDIR(statbuf.st_mode) && /* it's a directory */ case_ok(buf, len, namelen, name)) { /* case matches */ if (find_init_module(buf)) { /* and has __init__.py */ - Py_XDECREF(copy); PyMem_FREE(name); return &fd_package; } else { - char warnstr[MAXPATHLEN+80]; - sprintf(warnstr, "Not importing directory " - "'%.*s': missing __init__.py", - MAXPATHLEN, buf); - if (PyErr_Warn(PyExc_ImportWarning, - warnstr)) { - Py_XDECREF(copy); + int err; + PyObject *unicode = PyUnicode_DecodeFSDefault(buf); + if (unicode == NULL) + goto error_exit; + err = PyErr_WarnFormat(PyExc_ImportWarning, 1, + "Not importing directory '%U': missing __init__.py", + unicode); + Py_DECREF(unicode); + if (err) goto error_exit; - } } } +#endif #if defined(PYOS_OS2) /* take a snapshot of the module spec for restoration * after the 8 character DLL hackery @@ -1522,6 +1807,7 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, saved_namelen = namelen; #endif /* PYOS_OS2 */ for (fdp = _PyImport_Filetab; fdp->suffix != NULL; fdp++) { + struct stat statbuf; #if defined(PYOS_OS2) && defined(HAVE_DYNAMIC_LOADING) /* OS/2 limits DLLs to 8 character names (w/o extension) @@ -1550,10 +1836,16 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, strcpy(buf+len, fdp->suffix); if (Py_VerboseFlag > 1) PySys_WriteStderr("# trying %s\n", buf); + filemode = fdp->mode; if (filemode[0] == 'U') filemode = "r" PY_STDIOTEXTMODE; - fp = fopen(buf, filemode); + + if (stat(buf, &statbuf) == 0 && S_ISDIR(statbuf.st_mode)) + /* it's a directory */ + fp = NULL; + else + fp = fopen(buf, filemode); if (fp != NULL) { if (case_ok(buf, len, namelen, name)) break; @@ -1577,7 +1869,6 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, saved_buf = NULL; } #endif - Py_XDECREF(copy); if (fp != NULL) break; } @@ -1595,22 +1886,6 @@ error_exit: return NULL; } -/* Helpers for main.c - * Find the source file corresponding to a named module - */ -struct filedescr * -_PyImport_FindModule(const char *name, PyObject *path, char *buf, - size_t buflen, FILE **p_fp, PyObject **p_loader) -{ - return find_module((char *) name, (char *) name, path, - buf, buflen, p_fp, p_loader); -} - -PyAPI_FUNC(int) _PyImport_IsScript(struct filedescr * fd) -{ - return fd->type == PY_SOURCE || fd->type == PY_COMPILED; -} - /* case_ok(char* buf, Py_ssize_t len, Py_ssize_t namelen, char* name) * The arguments here are tricky, best shown by example: * /a/b/c/d/e/f/g/h/i/j/k/some_long_module_name.py\0 @@ -1653,9 +1928,6 @@ PyAPI_FUNC(int) _PyImport_IsScript(struct filedescr * fd) #define INCL_DOSERRORS #define INCL_NOPMAPI #include <os2.h> - -#elif defined(RISCOS) -#include "oslib/osfscontrol.h" #endif static int @@ -1741,31 +2013,6 @@ case_ok(char *buf, Py_ssize_t len, Py_ssize_t namelen, char *name) } return 0 ; /* Not found */ -/* RISC OS */ -#elif defined(RISCOS) - char canon[MAXPATHLEN+1]; /* buffer for the canonical form of the path */ - char buf2[MAXPATHLEN+2]; - char *nameWithExt = buf+len-namelen; - int canonlen; - os_error *e; - - if (Py_GETENV("PYTHONCASEOK") != NULL) - return 1; - - /* workaround: - append wildcard, otherwise case of filename wouldn't be touched */ - strcpy(buf2, buf); - strcat(buf2, "*"); - - e = xosfscontrol_canonicalise_path(buf2,canon,0,0,MAXPATHLEN+1,&canonlen); - canonlen = MAXPATHLEN+1-canonlen; - if (e || canonlen<=0 || canonlen>(MAXPATHLEN+1) ) - return 0; - if (strcmp(nameWithExt, canon+canonlen-strlen(nameWithExt))==0) - return 1; /* match */ - - return 0; - /* OS/2 */ #elif defined(PYOS_OS2) HDIR hdir = 1; @@ -1793,8 +2040,8 @@ case_ok(char *buf, Py_ssize_t len, Py_ssize_t namelen, char *name) #endif } - #ifdef HAVE_STAT + /* Helper to look for __init__.py or __init__.py[co] in potential package */ static int find_init_module(char *buf) @@ -1841,42 +2088,49 @@ find_init_module(char *buf) return 0; } -#else +#endif /* HAVE_STAT */ -#ifdef RISCOS -static int -find_init_module(buf) - char *buf; + +static int init_builtin(char *); /* Forward */ + +static PyObject* +load_builtin(char *name, char *pathname, int type) { - int save_len = strlen(buf); - int i = save_len; + PyObject *m, *modules; + int err; - if (save_len + 13 >= MAXPATHLEN) - return 0; - buf[i++] = SEP; - strcpy(buf+i, "__init__/py"); - if (isfile(buf)) { - buf[save_len] = '\0'; - return 1; - } + if (pathname != NULL && pathname[0] != '\0') + name = pathname; - if (Py_OptimizeFlag) - strcpy(buf+i, "o"); + if (type == C_BUILTIN) + err = init_builtin(name); else - strcpy(buf+i, "c"); - if (isfile(buf)) { - buf[save_len] = '\0'; - return 1; + err = PyImport_ImportFrozenModule(name); + if (err < 0) + return NULL; + if (err == 0) { + PyErr_Format(PyExc_ImportError, + "Purported %s module %.200s not found", + type == C_BUILTIN ? + "builtin" : "frozen", + name); + return NULL; } - buf[save_len] = '\0'; - return 0; -} -#endif /*RISCOS*/ - -#endif /* HAVE_STAT */ - -static int init_builtin(char *); /* Forward */ + modules = PyImport_GetModuleDict(); + m = PyDict_GetItemString(modules, name); + if (m == NULL) { + PyErr_Format( + PyExc_ImportError, + "%s module %.200s not properly initialized", + type == C_BUILTIN ? + "builtin" : "frozen", + name); + return NULL; + } + Py_INCREF(m); + return m; +} /* Load an external module using the default search path and return its module object WITH INCREMENTED REFERENCE COUNT */ @@ -1884,9 +2138,7 @@ static int init_builtin(char *); /* Forward */ static PyObject * load_module(char *name, FILE *fp, char *pathname, int type, PyObject *loader) { - PyObject *modules; PyObject *m; - int err; /* First check that there's an open file (if we need one) */ switch (type) { @@ -1922,34 +2174,7 @@ load_module(char *name, FILE *fp, char *pathname, int type, PyObject *loader) case C_BUILTIN: case PY_FROZEN: - if (pathname != NULL && pathname[0] != '\0') - name = pathname; - if (type == C_BUILTIN) - err = init_builtin(name); - else - err = PyImport_ImportFrozenModule(name); - if (err < 0) - return NULL; - if (err == 0) { - PyErr_Format(PyExc_ImportError, - "Purported %s module %.200s not found", - type == C_BUILTIN ? - "builtin" : "frozen", - name); - return NULL; - } - modules = PyImport_GetModuleDict(); - m = PyDict_GetItemString(modules, name); - if (m == NULL) { - PyErr_Format( - PyExc_ImportError, - "%s module %.200s not properly initialized", - type == C_BUILTIN ? - "builtin" : "frozen", - name); - return NULL; - } - Py_INCREF(m); + m = load_builtin(name, pathname, type); break; case IMP_HOOK: { @@ -1983,10 +2208,12 @@ init_builtin(char *name) { struct _inittab *p; - if (_PyImport_FindExtension(name, name) != NULL) + if (_PyImport_FindBuiltin(name) != NULL) return 1; for (p = PyImport_Inittab; p->name != NULL; p++) { + PyObject *mod; + PyModuleDef *def; if (strcmp(name, p->name) == 0) { if (p->initfunc == NULL) { PyErr_Format(PyExc_ImportError, @@ -1996,11 +2223,17 @@ init_builtin(char *name) } if (Py_VerboseFlag) PySys_WriteStderr("import %s # builtin\n", name); - (*p->initfunc)(); - if (PyErr_Occurred()) + mod = (*p->initfunc)(); + if (mod == 0) return -1; - if (_PyImport_FixupExtension(name, name) == NULL) + /* Remember pointer to module init function. */ + def = PyModule_GetDef(mod); + def->m_base.m_init = p->initfunc; + if (_PyImport_FixupBuiltin(mod, name) < 0) return -1; + /* FixupExtension has put the module into sys.modules, + so we can release our own reference. */ + Py_DECREF(mod); return 1; } } @@ -2015,6 +2248,9 @@ find_frozen(char *name) { struct _frozen *p; + if (!name) + return NULL; + for (p = PyImport_FrozenModules; ; p++) { if (p->name == NULL) return NULL; @@ -2048,8 +2284,30 @@ get_frozen_object(char *name) return PyMarshal_ReadObjectFromString((char *)p->code, size); } +static PyObject * +is_frozen_package(char *name) +{ + struct _frozen *p = find_frozen(name); + int size; + + if (p == NULL) { + PyErr_Format(PyExc_ImportError, + "No such frozen object named %.200s", + name); + return NULL; + } + + size = p->size; + + if (size < 0) + Py_RETURN_TRUE; + else + Py_RETURN_FALSE; +} + + /* Initialize a frozen module. - Return 1 for succes, 0 if the module is not found, and -1 with + Return 1 for success, 0 if the module is not found, and -1 with an exception set if the initialization failed. This function is also used from frozenmain.c */ @@ -2088,17 +2346,23 @@ PyImport_ImportFrozenModule(char *name) } if (ispackage) { /* Set __path__ to the package name */ - PyObject *d, *s; + PyObject *d, *s, *l; int err; m = PyImport_AddModule(name); if (m == NULL) goto err_return; d = PyModule_GetDict(m); - s = PyString_InternFromString(name); + s = PyUnicode_InternFromString(name); if (s == NULL) goto err_return; - err = PyDict_SetItemString(d, "__path__", s); - Py_DECREF(s); + l = PyList_New(1); + if (l == NULL) { + Py_DECREF(s); + goto err_return; + } + PyList_SET_ITEM(l, 0, s); + err = PyDict_SetItemString(d, "__path__", l); + Py_DECREF(l); if (err != 0) goto err_return; } @@ -2123,7 +2387,7 @@ PyImport_ImportModule(const char *name) PyObject *pname; PyObject *result; - pname = PyString_FromString(name); + pname = PyUnicode_FromString(name); if (pname == NULL) return NULL; result = PyImport_Import(pname); @@ -2319,17 +2583,17 @@ get_parent(PyObject *globals, char *buf, Py_ssize_t *p_buflen, int level) return Py_None; if (namestr == NULL) { - namestr = PyString_InternFromString("__name__"); + namestr = PyUnicode_InternFromString("__name__"); if (namestr == NULL) return NULL; } if (pathstr == NULL) { - pathstr = PyString_InternFromString("__path__"); + pathstr = PyUnicode_InternFromString("__path__"); if (pathstr == NULL) return NULL; } if (pkgstr == NULL) { - pkgstr = PyString_InternFromString("__package__"); + pkgstr = PyUnicode_InternFromString("__package__"); if (pkgstr == NULL) return NULL; } @@ -2340,13 +2604,15 @@ get_parent(PyObject *globals, char *buf, Py_ssize_t *p_buflen, int level) if ((pkgname != NULL) && (pkgname != Py_None)) { /* __package__ is set, so use it */ + char *pkgname_str; Py_ssize_t len; - if (!PyString_Check(pkgname)) { + + if (!PyUnicode_Check(pkgname)) { PyErr_SetString(PyExc_ValueError, "__package__ set to non-string"); return NULL; } - len = PyString_GET_SIZE(pkgname); + pkgname_str = _PyUnicode_AsStringAndSize(pkgname, &len); if (len == 0) { if (level > 0) { PyErr_SetString(PyExc_ValueError, @@ -2360,24 +2626,27 @@ get_parent(PyObject *globals, char *buf, Py_ssize_t *p_buflen, int level) "Package name too long"); return NULL; } - strcpy(buf, PyString_AS_STRING(pkgname)); + strcpy(buf, pkgname_str); } else { /* __package__ not set, so figure it out and set it */ modname = PyDict_GetItem(globals, namestr); - if (modname == NULL || !PyString_Check(modname)) + if (modname == NULL || !PyUnicode_Check(modname)) return Py_None; modpath = PyDict_GetItem(globals, pathstr); if (modpath != NULL) { /* __path__ is set, so modname is already the package name */ - Py_ssize_t len = PyString_GET_SIZE(modname); + char *modname_str; + Py_ssize_t len; int error; + + modname_str = _PyUnicode_AsStringAndSize(modname, &len); if (len > MAXPATHLEN) { PyErr_SetString(PyExc_ValueError, "Module name too long"); return NULL; } - strcpy(buf, PyString_AS_STRING(modname)); + strcpy(buf, modname_str); error = PyDict_SetItem(globals, pkgstr, modname); if (error) { PyErr_SetString(PyExc_ValueError, @@ -2386,7 +2655,7 @@ get_parent(PyObject *globals, char *buf, Py_ssize_t *p_buflen, int level) } } else { /* Normal module, so work out the package name if any */ - char *start = PyString_AS_STRING(modname); + char *start = _PyUnicode_AsString(modname); char *lastdot = strrchr(start, '.'); size_t len; int error; @@ -2412,7 +2681,7 @@ get_parent(PyObject *globals, char *buf, Py_ssize_t *p_buflen, int level) } strncpy(buf, start, len); buf[len] = '\0'; - pkgname = PyString_FromString(buf); + pkgname = PyUnicode_FromString(buf); if (pkgname == NULL) { return NULL; } @@ -2441,14 +2710,14 @@ get_parent(PyObject *globals, char *buf, Py_ssize_t *p_buflen, int level) parent = PyDict_GetItemString(modules, buf); if (parent == NULL) { if (orig_level < 1) { - PyObject *err_msg = PyString_FromFormat( + PyObject *err_msg = PyBytes_FromFormat( "Parent module '%.200s' not found " "while handling absolute import", buf); if (err_msg == NULL) { return NULL; } if (!PyErr_WarnEx(PyExc_RuntimeWarning, - PyString_AsString(err_msg), 1)) { + PyBytes_AsString(err_msg), 1)) { *buf = '\0'; *p_buflen = 0; parent = Py_None; @@ -2566,13 +2835,13 @@ ensure_fromlist(PyObject *mod, PyObject *fromlist, char *buf, Py_ssize_t buflen, } return 0; } - if (!PyString_Check(item)) { + if (!PyUnicode_Check(item)) { PyErr_SetString(PyExc_TypeError, "Item in ``from list'' not a string"); Py_DECREF(item); return 0; } - if (PyString_AS_STRING(item)[0] == '*') { + if (PyUnicode_AS_UNICODE(item)[0] == '*') { PyObject *all; Py_DECREF(item); /* See if the package defines __all__ */ @@ -2591,9 +2860,16 @@ ensure_fromlist(PyObject *mod, PyObject *fromlist, char *buf, Py_ssize_t buflen, } hasit = PyObject_HasAttr(mod, item); if (!hasit) { - char *subname = PyString_AS_STRING(item); + PyObject *item8; + char *subname; PyObject *submod; char *p; + item8 = PyUnicode_EncodeFSDefault(item); + if (!item8) { + PyErr_SetString(PyExc_ValueError, "Cannot encode path item"); + return 0; + } + subname = PyBytes_AS_STRING(item8); if (buflen + strlen(subname) >= MAXPATHLEN) { PyErr_SetString(PyExc_ValueError, "Module name too long"); @@ -2604,6 +2880,7 @@ ensure_fromlist(PyObject *mod, PyObject *fromlist, char *buf, Py_ssize_t buflen, *p++ = '.'; strcpy(p, subname); submod = import_submodule(mod, subname, buf); + Py_DECREF(item8); Py_XDECREF(submod); if (submod == NULL) { Py_DECREF(item); @@ -2739,7 +3016,7 @@ PyImport_ReloadModule(PyObject *m) "reload() argument must be module"); return NULL; } - name = PyModule_GetName(m); + name = (char*)PyModule_GetName(m); if (name == NULL) return NULL; if (m != PyDict_GetItemString(modules, name)) { @@ -2763,7 +3040,7 @@ PyImport_ReloadModule(PyObject *m) subname = name; else { PyObject *parentname, *parent; - parentname = PyString_FromStringAndSize(name, (subname-name)); + parentname = PyUnicode_FromStringAndSize(name, (subname-name)); if (parentname == NULL) { imp_modules_reloading_clear(); return NULL; @@ -2771,8 +3048,8 @@ PyImport_ReloadModule(PyObject *m) parent = PyDict_GetItem(modules, parentname); if (parent == NULL) { PyErr_Format(PyExc_ImportError, - "reload(): parent %.200s not in sys.modules", - PyString_AS_STRING(parentname)); + "reload(): parent %U not in sys.modules", + parentname); Py_DECREF(parentname); imp_modules_reloading_clear(); return NULL; @@ -2822,9 +3099,9 @@ PyImport_ReloadModule(PyObject *m) more accurately -- it invokes the __import__() function from the builtins of the current globals. This means that the import is done using whatever import hooks are installed in the current - environment, e.g. by "rexec". + environment. A dummy list ["__doc__"] is passed as the 4th argument so that - e.g. PyImport_Import(PyString_FromString("win32com.client.gencache")) + e.g. PyImport_Import(PyUnicode_FromString("win32com.client.gencache")) will return <module "gencache"> instead of <module "win32com">. */ PyObject * @@ -2836,17 +3113,18 @@ PyImport_Import(PyObject *module_name) PyObject *globals = NULL; PyObject *import = NULL; PyObject *builtins = NULL; + PyObject *modules = NULL; PyObject *r = NULL; /* Initialize constant string objects */ if (silly_list == NULL) { - import_str = PyString_InternFromString("__import__"); + import_str = PyUnicode_InternFromString("__import__"); if (import_str == NULL) return NULL; - builtins_str = PyString_InternFromString("__builtins__"); + builtins_str = PyUnicode_InternFromString("__builtins__"); if (builtins_str == NULL) return NULL; - silly_list = Py_BuildValue("[s]", "__doc__"); + silly_list = PyList_New(0); if (silly_list == NULL) return NULL; } @@ -2861,7 +3139,7 @@ PyImport_Import(PyObject *module_name) } else { /* No globals -- use standard builtins, and fake globals */ - builtins = PyImport_ImportModuleLevel("__builtin__", + builtins = PyImport_ImportModuleLevel("builtins", NULL, NULL, NULL, 0); if (builtins == NULL) return NULL; @@ -2882,9 +3160,18 @@ PyImport_Import(PyObject *module_name) goto err; /* Call the __import__ function with the proper argument list - * Always use absolute import here. */ + Always use absolute import here. + Calling for side-effect of import. */ r = PyObject_CallFunction(import, "OOOOi", module_name, globals, globals, silly_list, 0, NULL); + if (r == NULL) + goto err; + Py_DECREF(r); + + modules = PyImport_GetModuleDict(); + r = PyDict_GetItem(modules, module_name); + if (r != NULL) + Py_INCREF(r); err: Py_XDECREF(globals); @@ -2900,16 +3187,28 @@ PyImport_Import(PyObject *module_name) */ static PyObject * -imp_get_magic(PyObject *self, PyObject *noargs) +imp_make_magic(long magic) { char buf[4]; - buf[0] = (char) ((pyc_magic >> 0) & 0xff); - buf[1] = (char) ((pyc_magic >> 8) & 0xff); - buf[2] = (char) ((pyc_magic >> 16) & 0xff); - buf[3] = (char) ((pyc_magic >> 24) & 0xff); + buf[0] = (char) ((magic >> 0) & 0xff); + buf[1] = (char) ((magic >> 8) & 0xff); + buf[2] = (char) ((magic >> 16) & 0xff); + buf[3] = (char) ((magic >> 24) & 0xff); - return PyString_FromStringAndSize(buf, 4); + return PyBytes_FromStringAndSize(buf, 4); +} + +static PyObject * +imp_get_magic(PyObject *self, PyObject *noargs) +{ + return imp_make_magic(pyc_magic); +} + +static PyObject * +imp_get_tag(PyObject *self, PyObject *noargs) +{ + return PyUnicode_FromString(pyc_tag); } static PyObject * @@ -2943,9 +3242,13 @@ call_find_module(char *name, PyObject *path) { extern int fclose(FILE *); PyObject *fob, *ret; + PyObject *pathobj; struct filedescr *fdp; char *pathname; FILE *fp = NULL; + int fd = -1; + char *found_encoding = NULL; + char *encoding = NULL; pathname = PyMem_MALLOC(MAXPATHLEN+1); if (pathname == NULL) { @@ -2955,36 +3258,68 @@ call_find_module(char *name, PyObject *path) if (path == Py_None) path = NULL; fdp = find_module(NULL, name, path, pathname, MAXPATHLEN+1, &fp, NULL); - if (fdp == NULL) { - PyMem_FREE(pathname); - return NULL; - } + if (fdp == NULL) + goto error_exit; if (fp != NULL) { - fob = PyFile_FromFile(fp, pathname, fdp->mode, fclose); + fd = fileno(fp); + if (fd != -1) + fd = dup(fd); + fclose(fp); + fp = NULL; + if (fd == -1) { + PyErr_SetFromErrno(PyExc_OSError); + goto error_exit; + } + } + if (fd != -1) { + if (strchr(fdp->mode, 'b') == NULL) { + /* PyTokenizer_FindEncoding() returns PyMem_MALLOC'ed + memory. */ + found_encoding = PyTokenizer_FindEncoding(fd); + lseek(fd, 0, 0); /* Reset position */ + if (found_encoding == NULL && PyErr_Occurred()) { + close(fd); + goto error_exit; + } + encoding = (found_encoding != NULL) ? found_encoding : + (char*)PyUnicode_GetDefaultEncoding(); + } + fob = PyFile_FromFd(fd, pathname, fdp->mode, -1, + (char*)encoding, NULL, NULL, 1); if (fob == NULL) { - PyMem_FREE(pathname); - return NULL; + close(fd); + PyMem_FREE(found_encoding); + goto error_exit; } } else { fob = Py_None; Py_INCREF(fob); } - ret = Py_BuildValue("Os(ssi)", - fob, pathname, fdp->suffix, fdp->mode, fdp->type); - Py_DECREF(fob); + pathobj = PyUnicode_DecodeFSDefault(pathname); + ret = Py_BuildValue("NN(ssi)", + fob, pathobj, fdp->suffix, fdp->mode, fdp->type); + PyMem_FREE(found_encoding); PyMem_FREE(pathname); return ret; + +error_exit: + PyMem_FREE(pathname); + return NULL; } static PyObject * imp_find_module(PyObject *self, PyObject *args) { - char *name; - PyObject *path = NULL; - if (!PyArg_ParseTuple(args, "s|O:find_module", &name, &path)) + PyObject *name; + PyObject *ret, *path = NULL; + if (!PyArg_ParseTuple(args, "O&|O:find_module", + PyUnicode_FSConverter, &name, + &path)) return NULL; - return call_find_module(name, path); + ret = call_find_module(PyBytes_AS_STRING(name), path); + Py_DECREF(name); + return ret; } static PyObject * @@ -3038,12 +3373,22 @@ imp_get_frozen_object(PyObject *self, PyObject *args) } static PyObject * +imp_is_frozen_package(PyObject *self, PyObject *args) +{ + char *name; + + if (!PyArg_ParseTuple(args, "s:is_frozen_package", &name)) + return NULL; + return is_frozen_package(name); +} + +static PyObject * imp_is_builtin(PyObject *self, PyObject *args) { char *name; if (!PyArg_ParseTuple(args, "s:is_builtin", &name)) return NULL; - return PyInt_FromLong(is_builtin(name)); + return PyLong_FromLong(is_builtin(name)); } static PyObject * @@ -3061,39 +3406,53 @@ static FILE * get_file(char *pathname, PyObject *fob, char *mode) { FILE *fp; + if (mode[0] == 'U') + mode = "r" PY_STDIOTEXTMODE; if (fob == NULL) { - if (mode[0] == 'U') - mode = "r" PY_STDIOTEXTMODE; fp = fopen(pathname, mode); - if (fp == NULL) - PyErr_SetFromErrno(PyExc_IOError); } else { - fp = PyFile_AsFile(fob); - if (fp == NULL) - PyErr_SetString(PyExc_ValueError, - "bad/closed file object"); + int fd = PyObject_AsFileDescriptor(fob); + if (fd == -1) + return NULL; + if (!_PyVerify_fd(fd)) + goto error; + /* the FILE struct gets a new fd, so that it can be closed + * independently of the file descriptor given + */ + fd = dup(fd); + if (fd == -1) + goto error; + fp = fdopen(fd, mode); } - return fp; + if (fp) + return fp; +error: + PyErr_SetFromErrno(PyExc_IOError); + return NULL; } static PyObject * imp_load_compiled(PyObject *self, PyObject *args) { char *name; - char *pathname; + PyObject *pathname; PyObject *fob = NULL; PyObject *m; FILE *fp; - if (!PyArg_ParseTuple(args, "ss|O!:load_compiled", &name, &pathname, - &PyFile_Type, &fob)) + if (!PyArg_ParseTuple(args, "sO&|O:load_compiled", + &name, + PyUnicode_FSConverter, &pathname, + &fob)) return NULL; - fp = get_file(pathname, fob, "rb"); - if (fp == NULL) + fp = get_file(PyBytes_AS_STRING(pathname), fob, "rb"); + if (fp == NULL) { + Py_DECREF(pathname); return NULL; - m = load_compiled_module(name, pathname, fp); - if (fob == NULL) - fclose(fp); + } + m = load_compiled_module(name, PyBytes_AS_STRING(pathname), fp); + fclose(fp); + Py_DECREF(pathname); return m; } @@ -3103,19 +3462,26 @@ static PyObject * imp_load_dynamic(PyObject *self, PyObject *args) { char *name; + PyObject *pathbytes; char *pathname; PyObject *fob = NULL; PyObject *m; FILE *fp = NULL; - if (!PyArg_ParseTuple(args, "ss|O!:load_dynamic", &name, &pathname, - &PyFile_Type, &fob)) + if (!PyArg_ParseTuple(args, "sO&|O:load_dynamic", + &name, PyUnicode_FSConverter, &pathbytes, &fob)) return NULL; + pathname = PyBytes_AS_STRING(pathbytes); if (fob) { fp = get_file(pathname, fob, "r"); - if (fp == NULL) + if (fp == NULL) { + Py_DECREF(pathbytes); return NULL; + } } m = _PyImport_LoadDynamicModule(name, pathname, fp); + Py_DECREF(pathbytes); + if (fp) + fclose(fp); return m; } @@ -3125,19 +3491,23 @@ static PyObject * imp_load_source(PyObject *self, PyObject *args) { char *name; - char *pathname; + PyObject *pathname; PyObject *fob = NULL; PyObject *m; FILE *fp; - if (!PyArg_ParseTuple(args, "ss|O!:load_source", &name, &pathname, - &PyFile_Type, &fob)) + if (!PyArg_ParseTuple(args, "sO&|O:load_source", + &name, + PyUnicode_FSConverter, &pathname, + &fob)) return NULL; - fp = get_file(pathname, fob, "r"); - if (fp == NULL) + fp = get_file(PyBytes_AS_STRING(pathname), fob, "r"); + if (fp == NULL) { + Py_DECREF(pathname); return NULL; - m = load_source_module(name, pathname, fp); - if (fob == NULL) - fclose(fp); + } + m = load_source_module(name, PyBytes_AS_STRING(pathname), fp); + Py_DECREF(pathname); + fclose(fp); return m; } @@ -3146,14 +3516,16 @@ imp_load_module(PyObject *self, PyObject *args) { char *name; PyObject *fob; - char *pathname; + PyObject *pathname; + PyObject * ret; char *suffix; /* Unused */ char *mode; int type; FILE *fp; - if (!PyArg_ParseTuple(args, "sOs(ssi):load_module", - &name, &fob, &pathname, + if (!PyArg_ParseTuple(args, "sOO&(ssi):load_module", + &name, &fob, + PyUnicode_FSConverter, &pathname, &suffix, &mode, &type)) return NULL; if (*mode) { @@ -3164,32 +3536,38 @@ imp_load_module(PyObject *self, PyObject *args) if (!(*mode == 'r' || *mode == 'U') || strchr(mode, '+')) { PyErr_Format(PyExc_ValueError, "invalid file open mode %.200s", mode); + Py_DECREF(pathname); return NULL; } } if (fob == Py_None) fp = NULL; else { - if (!PyFile_Check(fob)) { - PyErr_SetString(PyExc_ValueError, - "load_module arg#2 should be a file or None"); + fp = get_file(NULL, fob, mode); + if (fp == NULL) { + Py_DECREF(pathname); return NULL; } - fp = get_file(pathname, fob, mode); - if (fp == NULL) - return NULL; } - return load_module(name, fp, pathname, type, NULL); + ret = load_module(name, fp, PyBytes_AS_STRING(pathname), type, NULL); + Py_DECREF(pathname); + if (fp) + fclose(fp); + return ret; } static PyObject * imp_load_package(PyObject *self, PyObject *args) { char *name; - char *pathname; - if (!PyArg_ParseTuple(args, "ss:load_package", &name, &pathname)) + PyObject *pathname; + PyObject * ret; + if (!PyArg_ParseTuple(args, "sO&:load_package", + &name, PyUnicode_FSConverter, &pathname)) return NULL; - return load_package(name, pathname); + ret = load_package(name, PyBytes_AS_STRING(pathname)); + Py_DECREF(pathname); + return ret; } static PyObject * @@ -3207,6 +3585,106 @@ imp_reload(PyObject *self, PyObject *v) return PyImport_ReloadModule(v); } +PyDoc_STRVAR(doc_reload, +"reload(module) -> module\n\ +\n\ +Reload the module. The module must have been successfully imported before."); + +static PyObject * +imp_cache_from_source(PyObject *self, PyObject *args, PyObject *kws) +{ + static char *kwlist[] = {"path", "debug_override", NULL}; + + char *buf; + PyObject *pathbytes; + char *cpathname; + PyObject *debug_override = NULL; + int debug = !Py_OptimizeFlag; + + if (!PyArg_ParseTupleAndKeywords( + args, kws, "O&|O", kwlist, + PyUnicode_FSConverter, &pathbytes, &debug_override)) + return NULL; + + if (debug_override != NULL && + (debug = PyObject_IsTrue(debug_override)) < 0) { + Py_DECREF(pathbytes); + return NULL; + } + + buf = PyMem_MALLOC(MAXPATHLEN+1); + if (buf == NULL) { + return PyErr_NoMemory(); + } + cpathname = make_compiled_pathname( + PyBytes_AS_STRING(pathbytes), + buf, MAXPATHLEN+1, debug); + Py_DECREF(pathbytes); + + if (cpathname == NULL) { + PyErr_Format(PyExc_SystemError, "path buffer too short"); + PyMem_FREE(buf); + return NULL; + } + { + PyObject *ret = PyUnicode_DecodeFSDefault(buf); + PyMem_FREE(buf); + return ret; + } +} + +PyDoc_STRVAR(doc_cache_from_source, +"cache_from_source(path, [debug_override]) -> path\n\ +Given the path to a .py file, return the path to its .pyc/.pyo file.\n\ +\n\ +The .py file does not need to exist; this simply returns the path to the\n\ +.pyc/.pyo file calculated as if the .py file were imported. The extension\n\ +will be .pyc unless __debug__ is not defined, then it will be .pyo.\n\ +\n\ +If debug_override is not None, then it must be a boolean and is taken as\n\ +the value of __debug__ instead."); + +static PyObject * +imp_source_from_cache(PyObject *self, PyObject *args, PyObject *kws) +{ + static char *kwlist[] = {"path", NULL}; + + PyObject *pathname_obj; + char *pathname; + char *buf; + + if (!PyArg_ParseTupleAndKeywords( + args, kws, "O&", kwlist, + PyUnicode_FSConverter, &pathname_obj)) + return NULL; + + pathname = PyBytes_AS_STRING(pathname_obj); + buf = PyMem_MALLOC(MAXPATHLEN+1); + if (buf == NULL) { + return PyErr_NoMemory(); + } + if (make_source_pathname(pathname, buf) == NULL) { + PyErr_Format(PyExc_ValueError, "Not a PEP 3147 pyc path: %s", + pathname); + Py_DECREF(pathname_obj); + PyMem_FREE(buf); + return NULL; + } + Py_DECREF(pathname_obj); + { + PyObject *ret = PyUnicode_FromString(buf); + PyMem_FREE(buf); + return ret; + } +} + +PyDoc_STRVAR(doc_source_from_cache, +"source_from_cache(path) -> path\n\ +Given the path to a .pyc./.pyo file, return the path to its .py file.\n\ +\n\ +The .pyc/.pyo file does not need to exist; this simply returns the path to\n\ +the .py file calculated to correspond to the .pyc/.pyo file. If path\n\ +does not conform to PEP 3147 format, ValueError will be raised."); /* Doc strings */ @@ -3214,11 +3692,6 @@ PyDoc_STRVAR(doc_imp, "This module provides the components needed to build your own\n\ __import__ function. Undocumented functions are obsolete."); -PyDoc_STRVAR(doc_reload, -"reload(module) -> module\n\ -\n\ -Reload the module. The module must have been successfully imported before."); - PyDoc_STRVAR(doc_find_module, "find_module(name, [path]) -> (file, filename, (suffix, mode, type))\n\ Search for a module. If path is omitted or None, search for a\n\ @@ -3235,6 +3708,10 @@ PyDoc_STRVAR(doc_get_magic, "get_magic() -> string\n\ Return the magic number for .pyc or .pyo files."); +PyDoc_STRVAR(doc_get_tag, +"get_tag() -> string\n\ +Return the magic tag for .pyc or .pyo files."); + PyDoc_STRVAR(doc_get_suffixes, "get_suffixes() -> [(suffix, mode, type), ...]\n\ Return a list of (suffix, mode, type) tuples describing the files\n\ @@ -3263,17 +3740,23 @@ Release the interpreter's import lock.\n\ On platforms without threads, this function does nothing."); static PyMethodDef imp_methods[] = { - {"reload", imp_reload, METH_O, doc_reload}, {"find_module", imp_find_module, METH_VARARGS, doc_find_module}, {"get_magic", imp_get_magic, METH_NOARGS, doc_get_magic}, + {"get_tag", imp_get_tag, METH_NOARGS, doc_get_tag}, {"get_suffixes", imp_get_suffixes, METH_NOARGS, doc_get_suffixes}, {"load_module", imp_load_module, METH_VARARGS, doc_load_module}, {"new_module", imp_new_module, METH_VARARGS, doc_new_module}, {"lock_held", imp_lock_held, METH_NOARGS, doc_lock_held}, {"acquire_lock", imp_acquire_lock, METH_NOARGS, doc_acquire_lock}, {"release_lock", imp_release_lock, METH_NOARGS, doc_release_lock}, + {"reload", imp_reload, METH_O, doc_reload}, + {"cache_from_source", (PyCFunction)imp_cache_from_source, + METH_VARARGS | METH_KEYWORDS, doc_cache_from_source}, + {"source_from_cache", (PyCFunction)imp_source_from_cache, + METH_VARARGS | METH_KEYWORDS, doc_source_from_cache}, /* The rest are obsolete */ {"get_frozen_object", imp_get_frozen_object, METH_VARARGS}, + {"is_frozen_package", imp_is_frozen_package, METH_VARARGS}, {"init_builtin", imp_init_builtin, METH_VARARGS}, {"init_frozen", imp_init_frozen, METH_VARARGS}, {"is_builtin", imp_is_builtin, METH_VARARGS}, @@ -3293,7 +3776,7 @@ setint(PyObject *d, char *name, int value) PyObject *v; int err; - v = PyInt_FromLong((long)value); + v = PyLong_FromLong((long)value); err = PyDict_SetItemString(d, name, v); Py_XDECREF(v); return err; @@ -3306,27 +3789,69 @@ typedef struct { static int NullImporter_init(NullImporter *self, PyObject *args, PyObject *kwds) { - char *path; - Py_ssize_t pathlen; +#ifndef MS_WINDOWS + PyObject *path; + struct stat statbuf; + int rv; if (!_PyArg_NoKeywords("NullImporter()", kwds)) return -1; - if (!PyArg_ParseTuple(args, "s:NullImporter", - &path)) + if (!PyArg_ParseTuple(args, "O&:NullImporter", + PyUnicode_FSConverter, &path)) return -1; - pathlen = strlen(path); - if (pathlen == 0) { + if (PyBytes_GET_SIZE(path) == 0) { + Py_DECREF(path); PyErr_SetString(PyExc_ImportError, "empty pathname"); return -1; - } else { - if(isdir(path)) { - PyErr_SetString(PyExc_ImportError, - "existing directory"); + } + + rv = stat(PyBytes_AS_STRING(path), &statbuf); + Py_DECREF(path); + if (rv == 0) { + /* it exists */ + if (S_ISDIR(statbuf.st_mode)) { + /* it's a directory */ + PyErr_SetString(PyExc_ImportError, "existing directory"); return -1; } } +#else /* MS_WINDOWS */ + PyObject *pathobj; + DWORD rv; + wchar_t *path; + + if (!_PyArg_NoKeywords("NullImporter()", kwds)) + return -1; + + if (!PyArg_ParseTuple(args, "U:NullImporter", + &pathobj)) + return -1; + + if (PyUnicode_GET_SIZE(pathobj) == 0) { + PyErr_SetString(PyExc_ImportError, "empty pathname"); + return -1; + } + + path = PyUnicode_AsWideCharString(pathobj, NULL); + if (path == NULL) + return -1; + /* see issue1293 and issue3677: + * stat() on Windows doesn't recognise paths like + * "e:\\shared\\" and "\\\\whiterab-c2znlh\\shared" as dirs. + */ + rv = GetFileAttributesW(path); + PyMem_Free(path); + if (rv != INVALID_FILE_ATTRIBUTES) { + /* it exists */ + if (rv & FILE_ATTRIBUTE_DIRECTORY) { + /* it's a directory */ + PyErr_SetString(PyExc_ImportError, "existing directory"); + return -1; + } + } +#endif return 0; } @@ -3353,7 +3878,7 @@ PyTypeObject PyNullImporter_Type = { 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ - 0, /*tp_compare*/ + 0, /*tp_reserved*/ 0, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ @@ -3385,17 +3910,27 @@ PyTypeObject PyNullImporter_Type = { PyType_GenericNew /* tp_new */ }; +static struct PyModuleDef impmodule = { + PyModuleDef_HEAD_INIT, + "imp", + doc_imp, + 0, + imp_methods, + NULL, + NULL, + NULL, + NULL +}; PyMODINIT_FUNC -initimp(void) +PyInit_imp(void) { PyObject *m, *d; if (PyType_Ready(&PyNullImporter_Type) < 0) - goto failure; + return NULL; - m = Py_InitModule4("imp", imp_methods, doc_imp, - NULL, PYTHON_API_VERSION); + m = PyModule_Create(&impmodule); if (m == NULL) goto failure; d = PyModule_GetDict(m); @@ -3415,8 +3950,10 @@ initimp(void) Py_INCREF(&PyNullImporter_Type); PyModule_AddObject(m, "NullImporter", (PyObject *)&PyNullImporter_Type); + return m; failure: - ; + Py_XDECREF(m); + return NULL; } @@ -3460,7 +3997,7 @@ PyImport_ExtendInittab(struct _inittab *newtab) /* Shorthand to add a single entry given a name and a function */ int -PyImport_AppendInittab(const char *name, void (*initfunc)(void)) +PyImport_AppendInittab(const char *name, PyObject* (*initfunc)(void)) { struct _inittab newtab[2]; diff --git a/Python/importdl.c b/Python/importdl.c index 14c9be87d3..9caed453aa 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -22,12 +22,21 @@ PyObject * _PyImport_LoadDynamicModule(char *name, char *pathname, FILE *fp) { PyObject *m; + PyObject *path; char *lastdot, *shortname, *packagecontext, *oldcontext; - dl_funcptr p; + dl_funcptr p0; + PyObject* (*p)(void); + struct PyModuleDef *def; + PyObject *result; - if ((m = _PyImport_FindExtension(name, pathname)) != NULL) { + path = PyUnicode_DecodeFSDefault(pathname); + if (path == NULL) + return NULL; + + if ((m = _PyImport_FindExtensionUnicode(name, path)) != NULL) { Py_INCREF(m); - return m; + result = m; + goto finally; } lastdot = strrchr(name, '.'); if (lastdot == NULL) { @@ -39,40 +48,55 @@ _PyImport_LoadDynamicModule(char *name, char *pathname, FILE *fp) shortname = lastdot+1; } - p = _PyImport_GetDynLoadFunc(name, shortname, pathname, fp); + p0 = _PyImport_GetDynLoadFunc(name, shortname, pathname, fp); + p = (PyObject*(*)(void))p0; if (PyErr_Occurred()) - return NULL; + goto error; if (p == NULL) { PyErr_Format(PyExc_ImportError, - "dynamic module does not define init function (init%.200s)", + "dynamic module does not define init function (PyInit_%.200s)", shortname); - return NULL; + goto error; } oldcontext = _Py_PackageContext; _Py_PackageContext = packagecontext; - (*p)(); + m = (*p)(); _Py_PackageContext = oldcontext; - if (PyErr_Occurred()) - return NULL; + if (m == NULL) + goto error; - m = PyDict_GetItemString(PyImport_GetModuleDict(), name); - if (m == NULL) { - PyErr_SetString(PyExc_SystemError, - "dynamic module not initialized properly"); - return NULL; + if (PyErr_Occurred()) { + Py_DECREF(m); + PyErr_Format(PyExc_SystemError, + "initialization of %s raised unreported exception", + shortname); + goto error; } + + /* Remember pointer to module init function. */ + def = PyModule_GetDef(m); + def->m_base.m_init = p; + /* Remember the filename as the __file__ attribute */ - if (PyModule_AddStringConstant(m, "__file__", pathname) < 0) + if (PyModule_AddObject(m, "__file__", path) < 0) PyErr_Clear(); /* Not important enough to report */ + else + Py_INCREF(path); - if (_PyImport_FixupExtension(name, pathname) == NULL) - return NULL; + if (_PyImport_FixupExtensionUnicode(m, name, path) < 0) + goto error; if (Py_VerboseFlag) PySys_WriteStderr( "import %s # dynamically loaded from %s\n", name, pathname); - Py_INCREF(m); - return m; + result = m; + goto finally; + +error: + result = NULL; +finally: + Py_DECREF(path); + return result; } #endif /* HAVE_DYNAMIC_LOADING */ diff --git a/Python/mactoolboxglue.c b/Python/mactoolboxglue.c deleted file mode 100644 index 92bf3e8587..0000000000 --- a/Python/mactoolboxglue.c +++ /dev/null @@ -1,474 +0,0 @@ -/*********************************************************** -Copyright 1991-1997 by Stichting Mathematisch Centrum, Amsterdam, -The Netherlands. - - All Rights Reserved - -Permission to use, copy, modify, and distribute this software and its -documentation for any purpose and without fee is hereby granted, -provided that the above copyright notice appear in all copies and that -both that copyright notice and this permission notice appear in -supporting documentation, and that the names of Stichting Mathematisch -Centrum or CWI not be used in advertising or publicity pertaining to -distribution of the software without specific, written prior permission. - -STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO -THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND -FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE -FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -******************************************************************/ - - -#include "Python.h" -#include "pymactoolbox.h" -#include <arpa/inet.h> /* for ntohl, htonl */ - - -/* Like strerror() but for Mac OS error numbers */ -char * -PyMac_StrError(int err) -{ - static char buf[256]; - PyObject *m; - PyObject *rv; - - m = PyImport_ImportModuleNoBlock("MacOS"); - if (!m) { - if (Py_VerboseFlag) - PyErr_Print(); - PyErr_Clear(); - rv = NULL; - } - else { - rv = PyObject_CallMethod(m, "GetErrorString", "i", err); - if (!rv) - PyErr_Clear(); - } - if (!rv) { - buf[0] = '\0'; - } - else { - char *input = PyString_AsString(rv); - if (!input) { - PyErr_Clear(); - buf[0] = '\0'; - } else { - strncpy(buf, input, sizeof(buf) - 1); - buf[sizeof(buf) - 1] = '\0'; - } - Py_DECREF(rv); - } - Py_XDECREF(m); - return buf; -} - -/* Exception object shared by all Mac specific modules for Mac OS errors */ -PyObject *PyMac_OSErrException; - -/* Initialize and return PyMac_OSErrException */ -PyObject * -PyMac_GetOSErrException(void) -{ - if (PyMac_OSErrException == NULL) - PyMac_OSErrException = PyErr_NewException("MacOS.Error", NULL, NULL); - return PyMac_OSErrException; -} - -/* Set a MAC-specific error from errno, and return NULL; return None if no error */ -PyObject * -PyErr_Mac(PyObject *eobj, int err) -{ - char *msg; - PyObject *v; - - if (err == 0 && !PyErr_Occurred()) { - Py_INCREF(Py_None); - return Py_None; - } - if (err == -1 && PyErr_Occurred()) - return NULL; - msg = PyMac_StrError(err); - v = Py_BuildValue("(is)", err, msg); - PyErr_SetObject(eobj, v); - Py_DECREF(v); - return NULL; -} - -/* Call PyErr_Mac with PyMac_OSErrException */ -PyObject * -PyMac_Error(OSErr err) -{ - return PyErr_Mac(PyMac_GetOSErrException(), err); -} - - -#ifndef __LP64__ -OSErr -PyMac_GetFullPathname(FSSpec *fss, char *path, int len) -{ - PyObject *fs, *exc; - PyObject *rv = NULL; - char *input; - OSErr err = noErr; - - *path = '\0'; - - fs = PyMac_BuildFSSpec(fss); - if (!fs) - goto error; - - rv = PyObject_CallMethod(fs, "as_pathname", ""); - if (!rv) - goto error; - - input = PyString_AsString(rv); - if (!input) - goto error; - - strncpy(path, input, len - 1); - path[len - 1] = '\0'; - - Py_XDECREF(rv); - Py_XDECREF(fs); - return err; - - error: - exc = PyErr_Occurred(); - if (exc && PyErr_GivenExceptionMatches(exc, - PyMac_GetOSErrException())) { - PyObject *args = PyObject_GetAttrString(exc, "args"); - if (args) { - char *ignore; - PyArg_ParseTuple(args, "is", &err, &ignore); - Py_XDECREF(args); - } - } - if (err == noErr) - err = -1; - PyErr_Clear(); - Py_XDECREF(rv); - Py_XDECREF(fs); - return err; -} -#endif /* !__LP64__ */ - -/* Convert a 4-char string object argument to an OSType value */ -int -PyMac_GetOSType(PyObject *v, OSType *pr) -{ - uint32_t tmp; - if (!PyString_Check(v) || PyString_Size(v) != 4) { - PyErr_SetString(PyExc_TypeError, - "OSType arg must be string of 4 chars"); - return 0; - } - memcpy((char *)&tmp, PyString_AsString(v), 4); - *pr = (OSType)ntohl(tmp); - return 1; -} - -/* Convert an OSType value to a 4-char string object */ -PyObject * -PyMac_BuildOSType(OSType t) -{ - uint32_t tmp = htonl((uint32_t)t); - return PyString_FromStringAndSize((char *)&tmp, 4); -} - -/* Convert an NumVersion value to a 4-element tuple */ -PyObject * -PyMac_BuildNumVersion(NumVersion t) -{ - return Py_BuildValue("(hhhh)", t.majorRev, t.minorAndBugRev, t.stage, t.nonRelRev); -} - - -/* Convert a Python string object to a Str255 */ -int -PyMac_GetStr255(PyObject *v, Str255 pbuf) -{ - int len; - if (!PyString_Check(v) || (len = PyString_Size(v)) > 255) { - PyErr_SetString(PyExc_TypeError, - "Str255 arg must be string of at most 255 chars"); - return 0; - } - pbuf[0] = len; - memcpy((char *)(pbuf+1), PyString_AsString(v), len); - return 1; -} - -/* Convert a Str255 to a Python string object */ -PyObject * -PyMac_BuildStr255(Str255 s) -{ - if ( s == NULL ) { - PyErr_SetString(PyExc_SystemError, "Str255 pointer is NULL"); - return NULL; - } - return PyString_FromStringAndSize((char *)&s[1], (int)s[0]); -} - -PyObject * -PyMac_BuildOptStr255(Str255 s) -{ - if ( s == NULL ) { - Py_INCREF(Py_None); - return Py_None; - } - return PyString_FromStringAndSize((char *)&s[1], (int)s[0]); -} - - - -/* Convert a Python object to a Rect. - The object must be a (left, top, right, bottom) tuple. - (This differs from the order in the struct but is consistent with - the arguments to SetRect(), and also with STDWIN). */ -int -PyMac_GetRect(PyObject *v, Rect *r) -{ - return PyArg_Parse(v, "(hhhh)", &r->left, &r->top, &r->right, &r->bottom); -} - -/* Convert a Rect to a Python object */ -PyObject * -PyMac_BuildRect(Rect *r) -{ - return Py_BuildValue("(hhhh)", r->left, r->top, r->right, r->bottom); -} - - -/* Convert a Python object to a Point. - The object must be a (h, v) tuple. - (This differs from the order in the struct but is consistent with - the arguments to SetPoint(), and also with STDWIN). */ -int -PyMac_GetPoint(PyObject *v, Point *p) -{ - return PyArg_Parse(v, "(hh)", &p->h, &p->v); -} - -/* Convert a Point to a Python object */ -PyObject * -PyMac_BuildPoint(Point p) -{ - return Py_BuildValue("(hh)", p.h, p.v); -} - - -/* Convert a Python object to an EventRecord. - The object must be a (what, message, when, (v, h), modifiers) tuple. */ -int -PyMac_GetEventRecord(PyObject *v, EventRecord *e) -{ - return PyArg_Parse(v, "(Hkk(hh)H)", - &e->what, - &e->message, - &e->when, - &e->where.h, - &e->where.v, - &e->modifiers); -} - -/* Convert a Rect to an EventRecord object */ -PyObject * -PyMac_BuildEventRecord(EventRecord *e) -{ - return Py_BuildValue("(hll(hh)h)", - e->what, - e->message, - e->when, - e->where.h, - e->where.v, - e->modifiers); -} - -/* Convert Python object to Fixed */ -int -PyMac_GetFixed(PyObject *v, Fixed *f) -{ - double d; - - if( !PyArg_Parse(v, "d", &d)) - return 0; - *f = (Fixed)(d * 0x10000); - return 1; -} - -/* Convert a Fixed to a Python object */ -PyObject * -PyMac_BuildFixed(Fixed f) -{ - double d; - - d = f; - d = d / 0x10000; - return Py_BuildValue("d", d); -} - -/* Convert wide to/from Python int or (hi, lo) tuple. XXXX Should use Python longs */ -int -PyMac_Getwide(PyObject *v, wide *rv) -{ - if (PyInt_Check(v)) { - rv->hi = 0; - rv->lo = PyInt_AsLong(v); - if( rv->lo & 0x80000000 ) - rv->hi = -1; - return 1; - } - return PyArg_Parse(v, "(kk)", &rv->hi, &rv->lo); -} - - -PyObject * -PyMac_Buildwide(wide *w) -{ - if ( (w->hi == 0 && (w->lo & 0x80000000) == 0) || - (w->hi == -1 && (w->lo & 0x80000000) ) ) - return PyInt_FromLong(w->lo); - return Py_BuildValue("(ll)", w->hi, w->lo); -} - -#ifdef USE_TOOLBOX_OBJECT_GLUE -/* -** Glue together the toolbox objects. -** -** Because toolbox modules interdepend on each other, they use each others -** object types, on MacOSX/MachO this leads to the situation that they -** cannot be dynamically loaded (or they would all have to be lumped into -** a single .so, but this would be bad for extensibility). -** -** This file defines wrappers for all the _New and _Convert functions, -** which are the Py_BuildValue and PyArg_ParseTuple helpers. The wrappers -** check an indirection function pointer, and if it isn't filled in yet -** they import the appropriate module, whose init routine should fill in -** the pointer. -*/ - -#define GLUE_NEW(object, routinename, module) \ -PyObject *(*PyMacGluePtr_##routinename)(object); \ -\ -PyObject *routinename(object cobj) { \ - if (!PyMacGluePtr_##routinename) { \ - if (!PyImport_ImportModule(module)) return NULL; \ - if (!PyMacGluePtr_##routinename) { \ - PyErr_SetString(PyExc_ImportError, "Module did not provide routine: " module ": " #routinename); \ - return NULL; \ - } \ - } \ - return (*PyMacGluePtr_##routinename)(cobj); \ -} - -#define GLUE_CONVERT(object, routinename, module) \ -int (*PyMacGluePtr_##routinename)(PyObject *, object *); \ -\ -int routinename(PyObject *pyobj, object *cobj) { \ - if (!PyMacGluePtr_##routinename) { \ - if (!PyImport_ImportModule(module)) return 0; \ - if (!PyMacGluePtr_##routinename) { \ - PyErr_SetString(PyExc_ImportError, "Module did not provide routine: " module ": " #routinename); \ - return 0; \ - } \ - } \ - return (*PyMacGluePtr_##routinename)(pyobj, cobj); \ -} - -GLUE_NEW(FSSpec *, PyMac_BuildFSSpec, "Carbon.File") -GLUE_CONVERT(FSSpec, PyMac_GetFSSpec, "Carbon.File") -GLUE_NEW(FSRef *, PyMac_BuildFSRef, "Carbon.File") -GLUE_CONVERT(FSRef, PyMac_GetFSRef, "Carbon.File") - -GLUE_NEW(AppleEvent *, AEDesc_New, "Carbon.AE") /* XXXX Why by address? */ -GLUE_NEW(AppleEvent *, AEDesc_NewBorrowed, "Carbon.AE") -GLUE_CONVERT(AppleEvent, AEDesc_Convert, "Carbon.AE") - -GLUE_NEW(Component, CmpObj_New, "Carbon.Cm") -GLUE_CONVERT(Component, CmpObj_Convert, "Carbon.Cm") -GLUE_NEW(ComponentInstance, CmpInstObj_New, "Carbon.Cm") -GLUE_CONVERT(ComponentInstance, CmpInstObj_Convert, "Carbon.Cm") - -GLUE_NEW(ControlHandle, CtlObj_New, "Carbon.Ctl") -GLUE_CONVERT(ControlHandle, CtlObj_Convert, "Carbon.Ctl") - -GLUE_NEW(DialogPtr, DlgObj_New, "Carbon.Dlg") -GLUE_CONVERT(DialogPtr, DlgObj_Convert, "Carbon.Dlg") -GLUE_NEW(DialogPtr, DlgObj_WhichDialog, "Carbon.Dlg") - -GLUE_NEW(DragReference, DragObj_New, "Carbon.Drag") -GLUE_CONVERT(DragReference, DragObj_Convert, "Carbon.Drag") - -GLUE_NEW(ListHandle, ListObj_New, "Carbon.List") -GLUE_CONVERT(ListHandle, ListObj_Convert, "Carbon.List") - -GLUE_NEW(MenuHandle, MenuObj_New, "Carbon.Menu") -GLUE_CONVERT(MenuHandle, MenuObj_Convert, "Carbon.Menu") - -GLUE_NEW(GrafPtr, GrafObj_New, "Carbon.Qd") -GLUE_CONVERT(GrafPtr, GrafObj_Convert, "Carbon.Qd") -GLUE_NEW(BitMapPtr, BMObj_New, "Carbon.Qd") -GLUE_CONVERT(BitMapPtr, BMObj_Convert, "Carbon.Qd") -GLUE_NEW(RGBColor *, QdRGB_New, "Carbon.Qd") /* XXXX Why? */ -GLUE_CONVERT(RGBColor, QdRGB_Convert, "Carbon.Qd") - -GLUE_NEW(GWorldPtr, GWorldObj_New, "Carbon.Qdoffs") -GLUE_CONVERT(GWorldPtr, GWorldObj_Convert, "Carbon.Qdoffs") - -#ifndef __LP64__ -GLUE_NEW(Track, TrackObj_New, "Carbon.Qt") -GLUE_CONVERT(Track, TrackObj_Convert, "Carbon.Qt") -GLUE_NEW(Movie, MovieObj_New, "Carbon.Qt") -GLUE_CONVERT(Movie, MovieObj_Convert, "Carbon.Qt") -GLUE_NEW(MovieController, MovieCtlObj_New, "Carbon.Qt") -GLUE_CONVERT(MovieController, MovieCtlObj_Convert, "Carbon.Qt") -GLUE_NEW(TimeBase, TimeBaseObj_New, "Carbon.Qt") -GLUE_CONVERT(TimeBase, TimeBaseObj_Convert, "Carbon.Qt") -GLUE_NEW(UserData, UserDataObj_New, "Carbon.Qt") -GLUE_CONVERT(UserData, UserDataObj_Convert, "Carbon.Qt") -GLUE_NEW(Media, MediaObj_New, "Carbon.Qt") -GLUE_CONVERT(Media, MediaObj_Convert, "Carbon.Qt") -#endif /* !__LP64__ */ - -GLUE_NEW(Handle, ResObj_New, "Carbon.Res") -GLUE_CONVERT(Handle, ResObj_Convert, "Carbon.Res") -GLUE_NEW(Handle, OptResObj_New, "Carbon.Res") -GLUE_CONVERT(Handle, OptResObj_Convert, "Carbon.Res") - -GLUE_NEW(TEHandle, TEObj_New, "Carbon.TE") -GLUE_CONVERT(TEHandle, TEObj_Convert, "Carbon.TE") - -GLUE_NEW(WindowPtr, WinObj_New, "Carbon.Win") -GLUE_CONVERT(WindowPtr, WinObj_Convert, "Carbon.Win") -GLUE_NEW(WindowPtr, WinObj_WhichWindow, "Carbon.Win") - -GLUE_CONVERT(CFTypeRef, CFObj_Convert, "Carbon.CF") -GLUE_NEW(CFTypeRef, CFObj_New, "Carbon.CF") - -GLUE_CONVERT(CFTypeRef, CFTypeRefObj_Convert, "Carbon.CF") -GLUE_NEW(CFTypeRef, CFTypeRefObj_New, "Carbon.CF") - -GLUE_CONVERT(CFStringRef, CFStringRefObj_Convert, "Carbon.CF") -GLUE_NEW(CFStringRef, CFStringRefObj_New, "Carbon.CF") -GLUE_CONVERT(CFMutableStringRef, CFMutableStringRefObj_Convert, "Carbon.CF") -GLUE_NEW(CFMutableStringRef, CFMutableStringRefObj_New, "Carbon.CF") - -GLUE_CONVERT(CFArrayRef, CFArrayRefObj_Convert, "Carbon.CF") -GLUE_NEW(CFArrayRef, CFArrayRefObj_New, "Carbon.CF") -GLUE_CONVERT(CFMutableArrayRef, CFMutableArrayRefObj_Convert, "Carbon.CF") -GLUE_NEW(CFMutableArrayRef, CFMutableArrayRefObj_New, "Carbon.CF") - -GLUE_CONVERT(CFDictionaryRef, CFDictionaryRefObj_Convert, "Carbon.CF") -GLUE_NEW(CFDictionaryRef, CFDictionaryRefObj_New, "Carbon.CF") -GLUE_CONVERT(CFMutableDictionaryRef, CFMutableDictionaryRefObj_Convert, "Carbon.CF") -GLUE_NEW(CFMutableDictionaryRef, CFMutableDictionaryRefObj_New, "Carbon.CF") - -GLUE_CONVERT(CFURLRef, CFURLRefObj_Convert, "Carbon.CF") -GLUE_CONVERT(CFURLRef, OptionalCFURLRefObj_Convert, "Carbon.CF") -GLUE_NEW(CFURLRef, CFURLRefObj_New, "Carbon.CF") - -#endif /* USE_TOOLBOX_OBJECT_GLUE */ diff --git a/Python/makeopcodetargets.py b/Python/makeopcodetargets.py new file mode 100755 index 0000000000..5d8e5a9a46 --- /dev/null +++ b/Python/makeopcodetargets.py @@ -0,0 +1,46 @@ +#! /usr/bin/env python +"""Generate C code for the jump table of the threaded code interpreter +(for compilers supporting computed gotos or "labels-as-values", such as gcc). +""" + +# This code should stay compatible with Python 2.3, at least while +# some of the buildbots have Python 2.3 as their system Python. + +import imp +import os + + +def find_module(modname): + """Finds and returns a module in the local dist/checkout. + """ + modpath = os.path.join( + os.path.dirname(os.path.dirname(__file__)), "Lib") + return imp.load_module(modname, *imp.find_module(modname, [modpath])) + +def write_contents(f): + """Write C code contents to the target file object. + """ + opcode = find_module("opcode") + targets = ['_unknown_opcode'] * 256 + for opname, op in opcode.opmap.items(): + if opname == "STOP_CODE": + # XXX opcode not implemented + continue + targets[op] = "TARGET_%s" % opname + f.write("static void *opcode_targets[256] = {\n") + f.write(",\n".join([" &&%s" % s for s in targets])) + f.write("\n};\n") + + +if __name__ == "__main__": + import sys + assert len(sys.argv) < 3, "Too many arguments" + if len(sys.argv) == 2: + target = sys.argv[1] + else: + target = "Python/opcode_targets.h" + f = open(target, "w") + try: + write_contents(f) + finally: + f.close() diff --git a/Python/marshal.c b/Python/marshal.c index faca027d4c..3e2fbeb499 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -16,8 +16,13 @@ /* High water mark to determine when the marshalled object is dangerously deep * and risks coring the interpreter. When the object stack gets this deep, * raise an exception instead of continuing. + * On Windows debug builds, reduce this value. */ +#if defined(MS_WINDOWS) && defined(_DEBUG) +#define MAX_MARSHAL_STACK_DEPTH 1500 +#else #define MAX_MARSHAL_STACK_DEPTH 2000 +#endif #define TYPE_NULL '0' #define TYPE_NONE 'N' @@ -33,8 +38,6 @@ #define TYPE_BINARY_COMPLEX 'y' #define TYPE_LONG 'l' #define TYPE_STRING 's' -#define TYPE_INTERNED 't' -#define TYPE_STRINGREF 'R' #define TYPE_TUPLE '(' #define TYPE_LIST '[' #define TYPE_DICT '{' @@ -54,6 +57,7 @@ typedef struct { int error; /* see WFERR_* values */ int depth; /* If fp == NULL, the following are valid: */ + PyObject *readable; /* Stream-like object being read from */ PyObject *str; char *ptr; char *end; @@ -71,18 +75,18 @@ w_more(int c, WFILE *p) Py_ssize_t size, newsize; if (p->str == NULL) return; /* An error already occurred */ - size = PyString_Size(p->str); + size = PyBytes_Size(p->str); newsize = size + size + 1024; if (newsize > 32*1024*1024) { newsize = size + (size >> 3); /* 12.5% overallocation */ } - if (_PyString_Resize(&p->str, newsize) != 0) { + if (_PyBytes_Resize(&p->str, newsize) != 0) { p->ptr = p->end = NULL; } else { - p->ptr = PyString_AS_STRING((PyStringObject *)p->str) + size; + p->ptr = PyBytes_AS_STRING((PyBytesObject *)p->str) + size; p->end = - PyString_AS_STRING((PyStringObject *)p->str) + newsize; + PyBytes_AS_STRING((PyBytesObject *)p->str) + newsize; *p->ptr++ = Py_SAFE_DOWNCAST(c, int, char); } } @@ -204,25 +208,28 @@ w_object(PyObject *v, WFILE *p) else if (v == Py_True) { w_byte(TYPE_TRUE, p); } - else if (PyInt_CheckExact(v)) { - long x = PyInt_AS_LONG((PyIntObject *)v); -#if SIZEOF_LONG > 4 - long y = Py_ARITHMETIC_RIGHT_SHIFT(long, x, 31); - if (y && y != -1) { - w_byte(TYPE_INT64, p); - w_long64(x, p); + else if (PyLong_CheckExact(v)) { + long x = PyLong_AsLong(v); + if ((x == -1) && PyErr_Occurred()) { + PyLongObject *ob = (PyLongObject *)v; + PyErr_Clear(); + w_PyLong(ob, p); } - else + else { +#if SIZEOF_LONG > 4 + long y = Py_ARITHMETIC_RIGHT_SHIFT(long, x, 31); + if (y && y != -1) { + w_byte(TYPE_INT64, p); + w_long64(x, p); + } + else #endif { - w_byte(TYPE_INT, p); - w_long(x, p); + w_byte(TYPE_INT, p); + w_long(x, p); + } } } - else if (PyLong_CheckExact(v)) { - PyLongObject *ob = (PyLongObject *)v; - w_PyLong(ob, p); - } else if (PyFloat_CheckExact(v)) { if (p->version > 1) { unsigned char buf[8]; @@ -248,7 +255,6 @@ w_object(PyObject *v, WFILE *p) PyMem_Free(buf); } } -#ifndef WITHOUT_COMPLEX else if (PyComplex_CheckExact(v)) { if (p->version > 1) { unsigned char buf[8]; @@ -291,34 +297,9 @@ w_object(PyObject *v, WFILE *p) PyMem_Free(buf); } } -#endif - else if (PyString_CheckExact(v)) { - if (p->strings && PyString_CHECK_INTERNED(v)) { - PyObject *o = PyDict_GetItem(p->strings, v); - if (o) { - long w = PyInt_AsLong(o); - w_byte(TYPE_STRINGREF, p); - w_long(w, p); - goto exit; - } - else { - int ok; - o = PyInt_FromSsize_t(PyDict_Size(p->strings)); - ok = o && - PyDict_SetItem(p->strings, v, o) >= 0; - Py_XDECREF(o); - if (!ok) { - p->depth--; - p->error = WFERR_UNMARSHALLABLE; - return; - } - w_byte(TYPE_INTERNED, p); - } - } - else { - w_byte(TYPE_STRING, p); - } - n = PyString_GET_SIZE(v); + else if (PyBytes_CheckExact(v)) { + w_byte(TYPE_STRING, p); + n = PyBytes_GET_SIZE(v); if (n > INT_MAX) { /* huge strings are not supported */ p->depth--; @@ -326,29 +307,29 @@ w_object(PyObject *v, WFILE *p) return; } w_long((long)n, p); - w_string(PyString_AS_STRING(v), (int)n, p); + w_string(PyBytes_AS_STRING(v), (int)n, p); } -#ifdef Py_USING_UNICODE else if (PyUnicode_CheckExact(v)) { PyObject *utf8; - utf8 = PyUnicode_AsUTF8String(v); + utf8 = PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(v), + PyUnicode_GET_SIZE(v), + "surrogatepass"); if (utf8 == NULL) { p->depth--; p->error = WFERR_UNMARSHALLABLE; return; } w_byte(TYPE_UNICODE, p); - n = PyString_GET_SIZE(utf8); + n = PyBytes_GET_SIZE(utf8); if (n > INT_MAX) { p->depth--; p->error = WFERR_UNMARSHALLABLE; return; } w_long((long)n, p); - w_string(PyString_AS_STRING(utf8), (int)n, p); + w_string(PyBytes_AS_STRING(utf8), (int)n, p); Py_DECREF(utf8); } -#endif else if (PyTuple_CheckExact(v)) { w_byte(TYPE_TUPLE, p); n = PyTuple_Size(v); @@ -412,6 +393,7 @@ w_object(PyObject *v, WFILE *p) PyCodeObject *co = (PyCodeObject *)v; w_byte(TYPE_CODE, p); w_long(co->co_argcount, p); + w_long(co->co_kwonlyargcount, p); w_long(co->co_nlocals, p); w_long(co->co_stacksize, p); w_long(co->co_flags, p); @@ -426,12 +408,19 @@ w_object(PyObject *v, WFILE *p) w_long(co->co_firstlineno, p); w_object(co->co_lnotab, p); } - else if (PyObject_CheckReadBuffer(v)) { + else if (PyObject_CheckBuffer(v)) { /* Write unknown buffer-style objects as a string */ char *s; - PyBufferProcs *pb = v->ob_type->tp_as_buffer; + Py_buffer view; + if (PyObject_GetBuffer(v, &view, PyBUF_SIMPLE) != 0) { + w_byte(TYPE_UNKNOWN, p); + p->depth--; + p->error = WFERR_UNMARSHALLABLE; + return; + } w_byte(TYPE_STRING, p); - n = (*pb->bf_getreadbuffer)(v, 0, (void **)&s); + n = view.len; + s = view.buf; if (n > INT_MAX) { p->depth--; p->error = WFERR_UNMARSHALLABLE; @@ -439,12 +428,12 @@ w_object(PyObject *v, WFILE *p) } w_long((long)n, p); w_string(s, (int)n, p); + PyBuffer_Release(&view); } else { w_byte(TYPE_UNKNOWN, p); p->error = WFERR_UNMARSHALLABLE; } - exit: p->depth--; } @@ -478,27 +467,75 @@ typedef WFILE RFILE; /* Same struct with different invariants */ #define rs_byte(p) (((p)->ptr < (p)->end) ? (unsigned char)*(p)->ptr++ : EOF) -#define r_byte(p) ((p)->fp ? getc((p)->fp) : rs_byte(p)) - static int r_string(char *s, int n, RFILE *p) { - if (p->fp != NULL) - /* The result fits into int because it must be <=n. */ - return (int)fread(s, 1, n, p->fp); - if (p->end - p->ptr < n) - n = (int)(p->end - p->ptr); - memcpy(s, p->ptr, n); - p->ptr += n; - return n; + char *ptr; + int read, left; + + if (!p->readable) { + if (p->fp != NULL) + /* The result fits into int because it must be <=n. */ + read = (int) fread(s, 1, n, p->fp); + else { + left = (int)(p->end - p->ptr); + read = (left < n) ? left : n; + memcpy(s, p->ptr, read); + p->ptr += read; + } + } + else { + PyObject *data = PyObject_CallMethod(p->readable, "read", "i", n); + read = 0; + if (data != NULL) { + if (!PyBytes_Check(data)) { + PyErr_Format(PyExc_TypeError, + "f.read() returned not bytes but %.100s", + data->ob_type->tp_name); + } + else { + read = PyBytes_GET_SIZE(data); + if (read > 0) { + ptr = PyBytes_AS_STRING(data); + memcpy(s, ptr, read); + } + } + Py_DECREF(data); + } + } + if (!PyErr_Occurred() && (read < n)) { + PyErr_SetString(PyExc_EOFError, "EOF read where not expected"); + } + return read; +} + + +static int +r_byte(RFILE *p) +{ + int c = EOF; + unsigned char ch; + int n; + + if (!p->readable) + c = p->fp ? getc(p->fp) : rs_byte(p); + else { + n = r_string((char *) &ch, 1, p); + if (n > 0) + c = ch; + } + return c; } static int r_short(RFILE *p) { register short x; - x = r_byte(p); - x |= r_byte(p) << 8; + unsigned char buffer[2]; + + r_string((char *) buffer, 2, p); + x = buffer[0]; + x |= buffer[1] << 8; /* Sign-extension, in case short greater than 16 bits */ x |= -(x & 0x8000); return x; @@ -508,19 +545,13 @@ static long r_long(RFILE *p) { register long x; - register FILE *fp = p->fp; - if (fp) { - x = getc(fp); - x |= (long)getc(fp) << 8; - x |= (long)getc(fp) << 16; - x |= (long)getc(fp) << 24; - } - else { - x = rs_byte(p); - x |= (long)rs_byte(p) << 8; - x |= (long)rs_byte(p) << 16; - x |= (long)rs_byte(p) << 24; - } + unsigned char buffer[4]; + + r_string((char *) buffer, 4, p); + x = buffer[0]; + x |= (long)buffer[1] << 8; + x |= (long)buffer[2] << 16; + x |= (long)buffer[3] << 24; #if SIZEOF_LONG > 4 /* Sign extension for 64-bit machines */ x |= -(x & 0x80000000L); @@ -538,25 +569,30 @@ r_long(RFILE *p) static PyObject * r_long64(RFILE *p) { + PyObject *result = NULL; long lo4 = r_long(p); long hi4 = r_long(p); + + if (!PyErr_Occurred()) { #if SIZEOF_LONG > 4 - long x = (hi4 << 32) | (lo4 & 0xFFFFFFFFL); - return PyInt_FromLong(x); + long x = (hi4 << 32) | (lo4 & 0xFFFFFFFFL); + result = PyLong_FromLong(x); #else - unsigned char buf[8]; - int one = 1; - int is_little_endian = (int)*(char*)&one; - if (is_little_endian) { - memcpy(buf, &lo4, 4); - memcpy(buf+4, &hi4, 4); - } - else { - memcpy(buf, &hi4, 4); - memcpy(buf+4, &lo4, 4); - } - return _PyLong_FromByteArray(buf, 8, is_little_endian, 1); + unsigned char buf[8]; + int one = 1; + int is_little_endian = (int)*(char*)&one; + if (is_little_endian) { + memcpy(buf, &lo4, 4); + memcpy(buf+4, &hi4, 4); + } + else { + memcpy(buf, &hi4, 4); + memcpy(buf+4, &lo4, 4); + } + result = _PyLong_FromByteArray(buf, 8, is_little_endian, 1); #endif + } + return result; } static PyObject * @@ -568,6 +604,8 @@ r_PyLong(RFILE *p) digit d; n = r_long(p); + if (PyErr_Occurred()) + return NULL; if (n == 0) return (PyObject *)_PyLong_New(0); if (n < -INT_MAX || n > INT_MAX) { @@ -587,6 +625,8 @@ r_PyLong(RFILE *p) d = 0; for (j=0; j < PyLong_MARSHAL_RATIO; j++) { md = r_short(p); + if (PyErr_Occurred()) + break; if (md < 0 || md > PyLong_MARSHAL_BASE) goto bad_digit; d += (digit)md << j*PyLong_MARSHAL_SHIFT; @@ -596,6 +636,8 @@ r_PyLong(RFILE *p) d = 0; for (j=0; j < shorts_in_top_digit; j++) { md = r_short(p); + if (PyErr_Occurred()) + break; if (md < 0 || md > PyLong_MARSHAL_BASE) goto bad_digit; /* topmost marshal digit should be nonzero */ @@ -607,6 +649,10 @@ r_PyLong(RFILE *p) } d += (digit)md << j*PyLong_MARSHAL_SHIFT; } + if (PyErr_Occurred()) { + Py_DECREF(ob); + return NULL; + } /* top digit should be nonzero, else the resulting PyLong won't be normalized */ ob->ob_digit[size-1] = d; @@ -675,7 +721,8 @@ r_object(RFILE *p) break; case TYPE_INT: - retval = PyInt_FromLong(r_long(p)); + n = r_long(p); + retval = PyErr_Occurred() ? NULL : PyLong_FromLong(n); break; case TYPE_INT64: @@ -690,19 +737,17 @@ r_object(RFILE *p) { char buf[256]; double dx; + retval = NULL; n = r_byte(p); if (n == EOF || r_string(buf, (int)n, p) != n) { PyErr_SetString(PyExc_EOFError, "EOF read where object expected"); - retval = NULL; break; } buf[n] = '\0'; dx = PyOS_string_to_double(buf, NULL, NULL); - if (dx == -1.0 && PyErr_Occurred()) { - retval = NULL; + if (dx == -1.0 && PyErr_Occurred()) break; - } retval = PyFloat_FromDouble(dx); break; } @@ -726,37 +771,31 @@ r_object(RFILE *p) break; } -#ifndef WITHOUT_COMPLEX case TYPE_COMPLEX: { char buf[256]; Py_complex c; + retval = NULL; n = r_byte(p); if (n == EOF || r_string(buf, (int)n, p) != n) { PyErr_SetString(PyExc_EOFError, "EOF read where object expected"); - retval = NULL; break; } buf[n] = '\0'; c.real = PyOS_string_to_double(buf, NULL, NULL); - if (c.real == -1.0 && PyErr_Occurred()) { - retval = NULL; + if (c.real == -1.0 && PyErr_Occurred()) break; - } n = r_byte(p); if (n == EOF || r_string(buf, (int)n, p) != n) { PyErr_SetString(PyExc_EOFError, "EOF read where object expected"); - retval = NULL; break; } buf[n] = '\0'; c.imag = PyOS_string_to_double(buf, NULL, NULL); - if (c.imag == -1.0 && PyErr_Occurred()) { - retval = NULL; + if (c.imag == -1.0 && PyErr_Occurred()) break; - } retval = PyComplex_FromCComplex(c); break; } @@ -790,56 +829,42 @@ r_object(RFILE *p) retval = PyComplex_FromCComplex(c); break; } -#endif - case TYPE_INTERNED: case TYPE_STRING: n = r_long(p); + if (PyErr_Occurred()) { + retval = NULL; + break; + } if (n < 0 || n > INT_MAX) { PyErr_SetString(PyExc_ValueError, "bad marshal data (string size out of range)"); retval = NULL; break; } - v = PyString_FromStringAndSize((char *)NULL, n); + v = PyBytes_FromStringAndSize((char *)NULL, n); if (v == NULL) { retval = NULL; break; } - if (r_string(PyString_AS_STRING(v), (int)n, p) != n) { + if (r_string(PyBytes_AS_STRING(v), (int)n, p) != n) { Py_DECREF(v); PyErr_SetString(PyExc_EOFError, "EOF read where object expected"); retval = NULL; break; } - if (type == TYPE_INTERNED) { - PyString_InternInPlace(&v); - if (PyList_Append(p->strings, v) < 0) { - retval = NULL; - break; - } - } retval = v; break; - case TYPE_STRINGREF: - n = r_long(p); - if (n < 0 || n >= PyList_GET_SIZE(p->strings)) { - PyErr_SetString(PyExc_ValueError, "bad marshal data (string ref out of range)"); - retval = NULL; - break; - } - v = PyList_GET_ITEM(p->strings, n); - Py_INCREF(v); - retval = v; - break; - -#ifdef Py_USING_UNICODE case TYPE_UNICODE: { char *buffer; n = r_long(p); + if (PyErr_Occurred()) { + retval = NULL; + break; + } if (n < 0 || n > INT_MAX) { PyErr_SetString(PyExc_ValueError, "bad marshal data (unicode size out of range)"); retval = NULL; @@ -857,15 +882,18 @@ r_object(RFILE *p) retval = NULL; break; } - v = PyUnicode_DecodeUTF8(buffer, n, NULL); + v = PyUnicode_DecodeUTF8(buffer, n, "surrogatepass"); PyMem_DEL(buffer); retval = v; break; } -#endif case TYPE_TUPLE: n = r_long(p); + if (PyErr_Occurred()) { + retval = NULL; + break; + } if (n < 0 || n > INT_MAX) { PyErr_SetString(PyExc_ValueError, "bad marshal data (tuple size out of range)"); retval = NULL; @@ -893,6 +921,10 @@ r_object(RFILE *p) case TYPE_LIST: n = r_long(p); + if (PyErr_Occurred()) { + retval = NULL; + break; + } if (n < 0 || n > INT_MAX) { PyErr_SetString(PyExc_ValueError, "bad marshal data (list size out of range)"); retval = NULL; @@ -945,6 +977,10 @@ r_object(RFILE *p) case TYPE_SET: case TYPE_FROZENSET: n = r_long(p); + if (PyErr_Occurred()) { + retval = NULL; + break; + } if (n < 0 || n > INT_MAX) { PyErr_SetString(PyExc_ValueError, "bad marshal data (set size out of range)"); retval = NULL; @@ -977,15 +1013,9 @@ r_object(RFILE *p) break; case TYPE_CODE: - if (PyEval_GetRestricted()) { - PyErr_SetString(PyExc_RuntimeError, - "cannot unmarshal code objects in " - "restricted execution mode"); - retval = NULL; - break; - } - else { + { int argcount; + int kwonlyargcount; int nlocals; int stacksize; int flags; @@ -1004,9 +1034,20 @@ r_object(RFILE *p) /* XXX ignore long->int overflows for now */ argcount = (int)r_long(p); + if (PyErr_Occurred()) + goto code_error; + kwonlyargcount = (int)r_long(p); + if (PyErr_Occurred()) + goto code_error; nlocals = (int)r_long(p); + if (PyErr_Occurred()) + goto code_error; stacksize = (int)r_long(p); + if (PyErr_Occurred()) + goto code_error; flags = (int)r_long(p); + if (PyErr_Occurred()) + goto code_error; code = r_object(p); if (code == NULL) goto code_error; @@ -1037,7 +1078,8 @@ r_object(RFILE *p) goto code_error; v = (PyObject *) PyCode_New( - argcount, nlocals, stacksize, flags, + argcount, kwonlyargcount, + nlocals, stacksize, flags, code, consts, names, varnames, freevars, cellvars, filename, name, firstlineno, lnotab); @@ -1052,7 +1094,6 @@ r_object(RFILE *p) Py_XDECREF(filename); Py_XDECREF(name); Py_XDECREF(lnotab); - } retval = v; break; @@ -1088,6 +1129,7 @@ PyMarshal_ReadShortFromFile(FILE *fp) { RFILE rf; assert(fp); + rf.readable = NULL; rf.fp = fp; rf.strings = NULL; rf.end = rf.ptr = NULL; @@ -1099,6 +1141,7 @@ PyMarshal_ReadLongFromFile(FILE *fp) { RFILE rf; rf.fp = fp; + rf.readable = NULL; rf.strings = NULL; rf.ptr = rf.end = NULL; return r_long(&rf); @@ -1160,6 +1203,7 @@ PyMarshal_ReadObjectFromFile(FILE *fp) RFILE rf; PyObject *result; rf.fp = fp; + rf.readable = NULL; rf.strings = PyList_New(0); rf.depth = 0; rf.ptr = rf.end = NULL; @@ -1174,6 +1218,7 @@ PyMarshal_ReadObjectFromString(char *str, Py_ssize_t len) RFILE rf; PyObject *result; rf.fp = NULL; + rf.readable = NULL; rf.ptr = str; rf.end = str + len; rf.strings = PyList_New(0); @@ -1183,34 +1228,19 @@ PyMarshal_ReadObjectFromString(char *str, Py_ssize_t len) return result; } -static void -set_error(int error) -{ - switch (error) { - case WFERR_NOMEMORY: - PyErr_NoMemory(); - break; - case WFERR_UNMARSHALLABLE: - PyErr_SetString(PyExc_ValueError, "unmarshallable object"); - break; - case WFERR_NESTEDTOODEEP: - default: - PyErr_SetString(PyExc_ValueError, - "object too deeply nested to marshal"); - break; - } -} - PyObject * PyMarshal_WriteObjectToString(PyObject *x, int version) { WFILE wf; + PyObject *res = NULL; + wf.fp = NULL; - wf.str = PyString_FromStringAndSize((char *)NULL, 50); + wf.readable = NULL; + wf.str = PyBytes_FromStringAndSize((char *)NULL, 50); if (wf.str == NULL) return NULL; - wf.ptr = PyString_AS_STRING((PyStringObject *)wf.str); - wf.end = wf.ptr + PyString_Size(wf.str); + wf.ptr = PyBytes_AS_STRING((PyBytesObject *)wf.str); + wf.end = wf.ptr + PyBytes_Size(wf.str); wf.error = WFERR_OK; wf.depth = 0; wf.version = version; @@ -1218,22 +1248,32 @@ PyMarshal_WriteObjectToString(PyObject *x, int version) w_object(x, &wf); Py_XDECREF(wf.strings); if (wf.str != NULL) { - char *base = PyString_AS_STRING((PyStringObject *)wf.str); + char *base = PyBytes_AS_STRING((PyBytesObject *)wf.str); if (wf.ptr - base > PY_SSIZE_T_MAX) { Py_DECREF(wf.str); PyErr_SetString(PyExc_OverflowError, - "too much marshall data for a string"); + "too much marshal data for a string"); return NULL; } - if (_PyString_Resize(&wf.str, (Py_ssize_t)(wf.ptr - base))) + if (_PyBytes_Resize(&wf.str, (Py_ssize_t)(wf.ptr - base)) < 0) return NULL; } if (wf.error != WFERR_OK) { Py_XDECREF(wf.str); - set_error(wf.error); + if (wf.error == WFERR_NOMEMORY) + PyErr_NoMemory(); + else + PyErr_SetString(PyExc_ValueError, + (wf.error==WFERR_UNMARSHALLABLE)?"unmarshallable object" + :"object too deeply nested to marshal"); return NULL; } - return wf.str; + if (wf.str != NULL) { + /* XXX Quick hack -- need to do this differently */ + res = PyBytes_FromObject(wf.str); + Py_DECREF(wf.str); + } + return res; } /* And an interface for Python programs... */ @@ -1241,32 +1281,20 @@ PyMarshal_WriteObjectToString(PyObject *x, int version) static PyObject * marshal_dump(PyObject *self, PyObject *args) { - WFILE wf; + /* XXX Quick hack -- need to do this differently */ PyObject *x; PyObject *f; int version = Py_MARSHAL_VERSION; + PyObject *s; + PyObject *res; if (!PyArg_ParseTuple(args, "OO|i:dump", &x, &f, &version)) return NULL; - if (!PyFile_Check(f)) { - PyErr_SetString(PyExc_TypeError, - "marshal.dump() 2nd arg must be file"); + s = PyMarshal_WriteObjectToString(x, version); + if (s == NULL) return NULL; - } - wf.fp = PyFile_AsFile(f); - wf.str = NULL; - wf.ptr = wf.end = NULL; - wf.error = WFERR_OK; - wf.depth = 0; - wf.strings = (version > 0) ? PyDict_New() : 0; - wf.version = version; - w_object(x, &wf); - Py_XDECREF(wf.strings); - if (wf.error != WFERR_OK) { - set_error(wf.error); - return NULL; - } - Py_INCREF(Py_None); - return Py_None; + res = PyObject_CallMethod(f, "write", "O", s); + Py_DECREF(s); + return res; } PyDoc_STRVAR(dump_doc, @@ -1280,24 +1308,37 @@ If the value has (or contains an object that has) an unsupported type, a\n\ ValueError exception is raised — but garbage data will also be written\n\ to the file. The object will not be properly read back by load()\n\ \n\ -New in version 2.4: The version argument indicates the data format that\n\ -dump should use."); +The version argument indicates the data format that dump should use."); static PyObject * marshal_load(PyObject *self, PyObject *f) { + PyObject *data, *result; RFILE rf; - PyObject *result; - if (!PyFile_Check(f)) { - PyErr_SetString(PyExc_TypeError, - "marshal.load() arg must be file"); + + /* + * Make a call to the read method, but read zero bytes. + * This is to ensure that the object passed in at least + * has a read method which returns bytes. + */ + data = PyObject_CallMethod(f, "read", "i", 0); + if (data == NULL) return NULL; + if (!PyBytes_Check(data)) { + PyErr_Format(PyExc_TypeError, + "f.read() returned not bytes but %.100s", + data->ob_type->tp_name); + result = NULL; } - rf.fp = PyFile_AsFile(f); - rf.strings = PyList_New(0); - rf.depth = 0; - result = read_object(&rf); - Py_DECREF(rf.strings); + else { + rf.strings = PyList_New(0); + rf.depth = 0; + rf.fp = NULL; + rf.readable = f; + result = read_object(&rf); + Py_DECREF(rf.strings); + } + Py_DECREF(data); return result; } @@ -1331,34 +1372,38 @@ Return the string that would be written to a file by dump(value, file).\n\ The value must be a supported type. Raise a ValueError exception if\n\ value has (or contains an object that has) an unsupported type.\n\ \n\ -New in version 2.4: The version argument indicates the data format that\n\ -dumps should use."); +The version argument indicates the data format that dumps should use."); static PyObject * marshal_loads(PyObject *self, PyObject *args) { RFILE rf; + Py_buffer p; char *s; Py_ssize_t n; PyObject* result; - if (!PyArg_ParseTuple(args, "s#:loads", &s, &n)) + if (!PyArg_ParseTuple(args, "y*:loads", &p)) return NULL; + s = p.buf; + n = p.len; rf.fp = NULL; + rf.readable = NULL; rf.ptr = s; rf.end = s + n; rf.strings = PyList_New(0); rf.depth = 0; result = read_object(&rf); Py_DECREF(rf.strings); + PyBuffer_Release(&p); return result; } PyDoc_STRVAR(loads_doc, -"loads(string)\n\ +"loads(bytes)\n\ \n\ -Convert the string to a value. If no valid value is found, raise\n\ -EOFError, ValueError or TypeError. Extra characters in the string are\n\ +Convert the bytes object to a value. If no valid value is found, raise\n\ +EOFError, ValueError or TypeError. Extra characters in the input are\n\ ignored."); static PyMethodDef marshal_methods[] = { @@ -1369,7 +1414,8 @@ static PyMethodDef marshal_methods[] = { {NULL, NULL} /* sentinel */ }; -PyDoc_STRVAR(marshal_doc, + +PyDoc_STRVAR(module_doc, "This module contains functions that can read and write Python values in\n\ a binary format. The format is specific to Python, but independent of\n\ machine architecture issues.\n\ @@ -1377,8 +1423,8 @@ machine architecture issues.\n\ Not all Python object types are supported; in general, only objects\n\ whose value is independent from a particular invocation of Python can be\n\ written and read by this module. The following types are supported:\n\ -None, integers, long integers, floating point numbers, strings, Unicode\n\ -objects, tuples, lists, sets, dictionaries, and code objects, where it\n\ +None, integers, floating point numbers, strings, bytes, bytearrays,\n\ +tuples, lists, sets, dictionaries, and code objects, where it\n\ should be understood that tuples, lists and dictionaries are only\n\ supported as long as the values contained therein are themselves\n\ supported; and recursive lists and dictionaries should not be written\n\ @@ -1387,9 +1433,8 @@ supported; and recursive lists and dictionaries should not be written\n\ Variables:\n\ \n\ version -- indicates the format that the module uses. Version 0 is the\n\ - historical format, version 1 (added in Python 2.4) shares interned\n\ - strings and version 2 (added in Python 2.5) uses a binary format for\n\ - floating point numbers. (New in version 2.4)\n\ + historical format, version 1 shares interned strings and version 2\n\ + uses a binary format for floating point numbers.\n\ \n\ Functions:\n\ \n\ @@ -1399,12 +1444,25 @@ dumps() -- write value to a string\n\ loads() -- read value from a string"); + +static struct PyModuleDef marshalmodule = { + PyModuleDef_HEAD_INIT, + "marshal", + module_doc, + 0, + marshal_methods, + NULL, + NULL, + NULL, + NULL +}; + PyMODINIT_FUNC PyMarshal_Init(void) { - PyObject *mod = Py_InitModule3("marshal", marshal_methods, - marshal_doc); + PyObject *mod = PyModule_Create(&marshalmodule); if (mod == NULL) - return; + return NULL; PyModule_AddIntConstant(mod, "version", Py_MARSHAL_VERSION); + return mod; } diff --git a/Python/modsupport.c b/Python/modsupport.c index 8bdec8b792..85b0d66358 100644 --- a/Python/modsupport.c +++ b/Python/modsupport.c @@ -11,99 +11,6 @@ static PyObject *va_build_value(const char *, va_list, int); /* Package context -- the full module name for package imports */ char *_Py_PackageContext = NULL; -/* Py_InitModule4() parameters: - - name is the module name - - methods is the list of top-level functions - - doc is the documentation string - - passthrough is passed as self to functions defined in the module - - api_version is the value of PYTHON_API_VERSION at the time the - module was compiled - - Return value is a borrowed reference to the module object; or NULL - if an error occurred (in Python 1.4 and before, errors were fatal). - Errors may still leak memory. -*/ - -static char api_version_warning[] = -"Python C API version mismatch for module %.100s:\ - This Python has API version %d, module %.100s has version %d."; - -PyObject * -Py_InitModule4(const char *name, PyMethodDef *methods, const char *doc, - PyObject *passthrough, int module_api_version) -{ - PyObject *m, *d, *v, *n; - PyMethodDef *ml; - PyInterpreterState *interp = PyThreadState_Get()->interp; - if (interp->modules == NULL) - Py_FatalError("Python import machinery not initialized"); - if (module_api_version != PYTHON_API_VERSION) { - char message[512]; - PyOS_snprintf(message, sizeof(message), - api_version_warning, name, - PYTHON_API_VERSION, name, - module_api_version); - if (PyErr_Warn(PyExc_RuntimeWarning, message)) - return NULL; - } - /* Make sure name is fully qualified. - - This is a bit of a hack: when the shared library is loaded, - the module name is "package.module", but the module calls - Py_InitModule*() with just "module" for the name. The shared - library loader squirrels away the true name of the module in - _Py_PackageContext, and Py_InitModule*() will substitute this - (if the name actually matches). - */ - if (_Py_PackageContext != NULL) { - char *p = strrchr(_Py_PackageContext, '.'); - if (p != NULL && strcmp(name, p+1) == 0) { - name = _Py_PackageContext; - _Py_PackageContext = NULL; - } - } - if ((m = PyImport_AddModule(name)) == NULL) - return NULL; - d = PyModule_GetDict(m); - if (methods != NULL) { - n = PyString_FromString(name); - if (n == NULL) - return NULL; - for (ml = methods; ml->ml_name != NULL; ml++) { - if ((ml->ml_flags & METH_CLASS) || - (ml->ml_flags & METH_STATIC)) { - PyErr_SetString(PyExc_ValueError, - "module functions cannot set" - " METH_CLASS or METH_STATIC"); - Py_DECREF(n); - return NULL; - } - v = PyCFunction_NewEx(ml, passthrough, n); - if (v == NULL) { - Py_DECREF(n); - return NULL; - } - if (PyDict_SetItemString(d, ml->ml_name, v) != 0) { - Py_DECREF(v); - Py_DECREF(n); - return NULL; - } - Py_DECREF(v); - } - Py_DECREF(n); - } - if (doc != NULL) { - v = PyString_FromString(doc); - if (v == NULL || PyDict_SetItemString(d, "__doc__", v) != 0) { - Py_XDECREF(v); - return NULL; - } - Py_DECREF(v); - } - return m; -} - - /* Helper for mkvalue() to scan the length of a format */ static int @@ -241,7 +148,6 @@ do_mklist(const char **p_format, va_list *p_va, int endchar, int n, int flags) return v; } -#ifdef Py_USING_UNICODE static int _ustrlen(Py_UNICODE *u) { @@ -250,7 +156,6 @@ _ustrlen(Py_UNICODE *u) while (*v != 0) { i++; v++; } return i; } -#endif static PyObject * do_mktuple(const char **p_format, va_list *p_va, int endchar, int n, int flags) @@ -310,37 +215,31 @@ do_mkvalue(const char **p_format, va_list *p_va, int flags) case 'B': case 'h': case 'i': - return PyInt_FromLong((long)va_arg(*p_va, int)); + return PyLong_FromLong((long)va_arg(*p_va, int)); case 'H': - return PyInt_FromLong((long)va_arg(*p_va, unsigned int)); + return PyLong_FromLong((long)va_arg(*p_va, unsigned int)); case 'I': { unsigned int n; n = va_arg(*p_va, unsigned int); - if (n > (unsigned long)PyInt_GetMax()) - return PyLong_FromUnsignedLong((unsigned long)n); - else - return PyInt_FromLong(n); + return PyLong_FromUnsignedLong(n); } case 'n': #if SIZEOF_SIZE_T!=SIZEOF_LONG - return PyInt_FromSsize_t(va_arg(*p_va, Py_ssize_t)); + return PyLong_FromSsize_t(va_arg(*p_va, Py_ssize_t)); #endif /* Fall through from 'n' to 'l' if Py_ssize_t is long */ case 'l': - return PyInt_FromLong(va_arg(*p_va, long)); + return PyLong_FromLong(va_arg(*p_va, long)); case 'k': { unsigned long n; n = va_arg(*p_va, unsigned long); - if (n > (unsigned long)PyInt_GetMax()) - return PyLong_FromUnsignedLong(n); - else - return PyInt_FromLong(n); + return PyLong_FromUnsignedLong(n); } #ifdef HAVE_LONG_LONG @@ -350,7 +249,6 @@ do_mkvalue(const char **p_format, va_list *p_va, int flags) case 'K': return PyLong_FromUnsignedLongLong((PY_LONG_LONG)va_arg(*p_va, unsigned PY_LONG_LONG)); #endif -#ifdef Py_USING_UNICODE case 'u': { PyObject *v; @@ -376,27 +274,35 @@ do_mkvalue(const char **p_format, va_list *p_va, int flags) } return v; } -#endif case 'f': case 'd': return PyFloat_FromDouble( (double)va_arg(*p_va, va_double)); -#ifndef WITHOUT_COMPLEX case 'D': return PyComplex_FromCComplex( *((Py_complex *)va_arg(*p_va, Py_complex *))); -#endif /* WITHOUT_COMPLEX */ case 'c': { char p[1]; p[0] = (char)va_arg(*p_va, int); - return PyString_FromStringAndSize(p, 1); + return PyBytes_FromStringAndSize(p, 1); + } + case 'C': + { + int i = va_arg(*p_va, int); + if (i < 0 || i > PyUnicode_GetMax()) { + PyErr_SetString(PyExc_OverflowError, + "%c arg not in range(0x110000)"); + return NULL; + } + return PyUnicode_FromOrdinal(i); } case 's': case 'z': + case 'U': /* XXX deprecated alias */ { PyObject *v; char *str = va_arg(*p_va, char *); @@ -424,7 +330,40 @@ do_mkvalue(const char **p_format, va_list *p_va, int flags) } n = (Py_ssize_t)m; } - v = PyString_FromStringAndSize(str, n); + v = PyUnicode_FromStringAndSize(str, n); + } + return v; + } + + case 'y': + { + PyObject *v; + char *str = va_arg(*p_va, char *); + Py_ssize_t n; + if (**p_format == '#') { + ++*p_format; + if (flags & FLAG_SIZE_T) + n = va_arg(*p_va, Py_ssize_t); + else + n = va_arg(*p_va, int); + } + else + n = -1; + if (str == NULL) { + v = Py_None; + Py_INCREF(v); + } + else { + if (n < 0) { + size_t m = strlen(str); + if (m > PY_SSIZE_T_MAX) { + PyErr_SetString(PyExc_OverflowError, + "string too long for Python bytes"); + return NULL; + } + n = (Py_ssize_t)m; + } + v = PyBytes_FromStringAndSize(str, n); } return v; } @@ -517,15 +456,7 @@ va_build_value(const char *format, va_list va, int flags) int n = countformat(f, '\0'); va_list lva; -#ifdef VA_LIST_IS_ARRAY - memcpy(lva, va, sizeof(va_list)); -#else -#ifdef __va_copy - __va_copy(lva, va); -#else - lva = va; -#endif -#endif + Py_VA_COPY(lva, va); if (n < 0) return NULL; @@ -622,7 +553,7 @@ PyModule_AddObject(PyObject *m, const char *name, PyObject *o) int PyModule_AddIntConstant(PyObject *m, const char *name, long value) { - PyObject *o = PyInt_FromLong(value); + PyObject *o = PyLong_FromLong(value); if (!o) return -1; if (PyModule_AddObject(m, name, o) == 0) @@ -634,7 +565,7 @@ PyModule_AddIntConstant(PyObject *m, const char *name, long value) int PyModule_AddStringConstant(PyObject *m, const char *name, const char *value) { - PyObject *o = PyString_FromString(value); + PyObject *o = PyUnicode_FromString(value); if (!o) return -1; if (PyModule_AddObject(m, name, o) == 0) diff --git a/Python/mysnprintf.c b/Python/mysnprintf.c index 39e3ff1a4d..a08e249b42 100644 --- a/Python/mysnprintf.c +++ b/Python/mysnprintf.c @@ -1,5 +1,4 @@ #include "Python.h" -#include <ctype.h> /* snprintf() wrappers. If the platform has vsnprintf, we use it, else we emulate it in a half-hearted way. Even if the platform has it, we wrap diff --git a/Python/mystrtoul.c b/Python/mystrtoul.c index c35be819f6..52502cba46 100644 --- a/Python/mystrtoul.c +++ b/Python/mystrtoul.c @@ -83,9 +83,9 @@ static int digitlimit[] = { ** This is a general purpose routine for converting ** an ascii string to an integer in an arbitrary base. ** Leading white space is ignored. If 'base' is zero -** it looks for a leading 0, 0b, 0B, 0o, 0O, 0x or 0X -** to tell which base. If these are absent it defaults -** to 10. Base must be 0 or between 2 and 36 (inclusive). +** it looks for a leading 0b, 0o or 0x to tell which +** base. If these are absent it defaults to 10. +** Base must be 0 or between 2 and 36 (inclusive). ** If 'ptr' is non-NULL it will contain a pointer to ** the end of the scan. ** Errors due to bad pointers will probably result in @@ -102,9 +102,9 @@ PyOS_strtoul(register char *str, char **ptr, int base) while (*str && isspace(Py_CHARMASK(*str))) ++str; - /* check for leading 0 or 0x for auto-base or base 16 */ + /* check for leading 0b, 0o or 0x for auto-base or base 16 */ switch (base) { - case 0: /* look for leading 0, 0b, 0o or 0x */ + case 0: /* look for leading 0b, 0o or 0x */ if (*str == '0') { ++str; if (*str == 'x' || *str == 'X') { @@ -135,19 +135,27 @@ PyOS_strtoul(register char *str, char **ptr, int base) ++str; base = 2; } else { - base = 8; + /* skip all zeroes... */ + while (*str == '0') + ++str; + while (isspace(Py_CHARMASK(*str))) + ++str; + if (ptr) + *ptr = str; + return 0; } } else base = 10; break; - case 2: /* skip leading 0b or 0B */ + /* even with explicit base, skip leading 0? prefix */ + case 16: if (*str == '0') { ++str; - if (*str == 'b' || *str == 'B') { - /* there must be at least one digit after 0b */ - if (_PyLong_DigitValue[Py_CHARMASK(str[1])] >= 2) { + if (*str == 'x' || *str == 'X') { + /* there must be at least one digit after 0x */ + if (_PyLong_DigitValue[Py_CHARMASK(str[1])] >= 16) { if (ptr) *ptr = str; return 0; @@ -156,8 +164,7 @@ PyOS_strtoul(register char *str, char **ptr, int base) } } break; - - case 8: /* skip leading 0o or 0O */ + case 8: if (*str == '0') { ++str; if (*str == 'o' || *str == 'O') { @@ -171,13 +178,12 @@ PyOS_strtoul(register char *str, char **ptr, int base) } } break; - - case 16: /* skip leading 0x or 0X */ - if (*str == '0') { + case 2: + if(*str == '0') { ++str; - if (*str == 'x' || *str == 'X') { - /* there must be at least one digit after 0x */ - if (_PyLong_DigitValue[Py_CHARMASK(str[1])] >= 16) { + if (*str == 'b' || *str == 'B') { + /* there must be at least one digit after 0b */ + if (_PyLong_DigitValue[Py_CHARMASK(str[1])] >= 2) { if (ptr) *ptr = str; return 0; diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h new file mode 100644 index 0000000000..a91da79be6 --- /dev/null +++ b/Python/opcode_targets.h @@ -0,0 +1,258 @@ +static void *opcode_targets[256] = { + &&_unknown_opcode, + &&TARGET_POP_TOP, + &&TARGET_ROT_TWO, + &&TARGET_ROT_THREE, + &&TARGET_DUP_TOP, + &&TARGET_DUP_TOP_TWO, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&TARGET_NOP, + &&TARGET_UNARY_POSITIVE, + &&TARGET_UNARY_NEGATIVE, + &&TARGET_UNARY_NOT, + &&_unknown_opcode, + &&_unknown_opcode, + &&TARGET_UNARY_INVERT, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&TARGET_BINARY_POWER, + &&TARGET_BINARY_MULTIPLY, + &&_unknown_opcode, + &&TARGET_BINARY_MODULO, + &&TARGET_BINARY_ADD, + &&TARGET_BINARY_SUBTRACT, + &&TARGET_BINARY_SUBSCR, + &&TARGET_BINARY_FLOOR_DIVIDE, + &&TARGET_BINARY_TRUE_DIVIDE, + &&TARGET_INPLACE_FLOOR_DIVIDE, + &&TARGET_INPLACE_TRUE_DIVIDE, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&TARGET_STORE_MAP, + &&TARGET_INPLACE_ADD, + &&TARGET_INPLACE_SUBTRACT, + &&TARGET_INPLACE_MULTIPLY, + &&_unknown_opcode, + &&TARGET_INPLACE_MODULO, + &&TARGET_STORE_SUBSCR, + &&TARGET_DELETE_SUBSCR, + &&TARGET_BINARY_LSHIFT, + &&TARGET_BINARY_RSHIFT, + &&TARGET_BINARY_AND, + &&TARGET_BINARY_XOR, + &&TARGET_BINARY_OR, + &&TARGET_INPLACE_POWER, + &&TARGET_GET_ITER, + &&TARGET_STORE_LOCALS, + &&TARGET_PRINT_EXPR, + &&TARGET_LOAD_BUILD_CLASS, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&TARGET_INPLACE_LSHIFT, + &&TARGET_INPLACE_RSHIFT, + &&TARGET_INPLACE_AND, + &&TARGET_INPLACE_XOR, + &&TARGET_INPLACE_OR, + &&TARGET_BREAK_LOOP, + &&TARGET_WITH_CLEANUP, + &&_unknown_opcode, + &&TARGET_RETURN_VALUE, + &&TARGET_IMPORT_STAR, + &&_unknown_opcode, + &&TARGET_YIELD_VALUE, + &&TARGET_POP_BLOCK, + &&TARGET_END_FINALLY, + &&TARGET_POP_EXCEPT, + &&TARGET_STORE_NAME, + &&TARGET_DELETE_NAME, + &&TARGET_UNPACK_SEQUENCE, + &&TARGET_FOR_ITER, + &&TARGET_UNPACK_EX, + &&TARGET_STORE_ATTR, + &&TARGET_DELETE_ATTR, + &&TARGET_STORE_GLOBAL, + &&TARGET_DELETE_GLOBAL, + &&_unknown_opcode, + &&TARGET_LOAD_CONST, + &&TARGET_LOAD_NAME, + &&TARGET_BUILD_TUPLE, + &&TARGET_BUILD_LIST, + &&TARGET_BUILD_SET, + &&TARGET_BUILD_MAP, + &&TARGET_LOAD_ATTR, + &&TARGET_COMPARE_OP, + &&TARGET_IMPORT_NAME, + &&TARGET_IMPORT_FROM, + &&TARGET_JUMP_FORWARD, + &&TARGET_JUMP_IF_FALSE_OR_POP, + &&TARGET_JUMP_IF_TRUE_OR_POP, + &&TARGET_JUMP_ABSOLUTE, + &&TARGET_POP_JUMP_IF_FALSE, + &&TARGET_POP_JUMP_IF_TRUE, + &&TARGET_LOAD_GLOBAL, + &&_unknown_opcode, + &&_unknown_opcode, + &&TARGET_CONTINUE_LOOP, + &&TARGET_SETUP_LOOP, + &&TARGET_SETUP_EXCEPT, + &&TARGET_SETUP_FINALLY, + &&_unknown_opcode, + &&TARGET_LOAD_FAST, + &&TARGET_STORE_FAST, + &&TARGET_DELETE_FAST, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&TARGET_RAISE_VARARGS, + &&TARGET_CALL_FUNCTION, + &&TARGET_MAKE_FUNCTION, + &&TARGET_BUILD_SLICE, + &&TARGET_MAKE_CLOSURE, + &&TARGET_LOAD_CLOSURE, + &&TARGET_LOAD_DEREF, + &&TARGET_STORE_DEREF, + &&TARGET_DELETE_DEREF, + &&_unknown_opcode, + &&TARGET_CALL_FUNCTION_VAR, + &&TARGET_CALL_FUNCTION_KW, + &&TARGET_CALL_FUNCTION_VAR_KW, + &&TARGET_SETUP_WITH, + &&TARGET_EXTENDED_ARG, + &&TARGET_LIST_APPEND, + &&TARGET_SET_ADD, + &&TARGET_MAP_ADD, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode +}; diff --git a/Python/peephole.c b/Python/peephole.c index fb6cd03c86..7ae599bb11 100644 --- a/Python/peephole.c +++ b/Python/peephole.c @@ -4,10 +4,8 @@ #include "Python-ast.h" #include "node.h" -#include "pyarena.h" #include "ast.h" #include "code.h" -#include "compile.h" #include "symtable.h" #include "opcode.h" @@ -31,7 +29,8 @@ new constant (c1, c2, ... cn) can be appended. Called with codestr pointing to the first LOAD_CONST. Bails out with no change if one or more of the LOAD_CONSTs is missing. - Also works for BUILD_LIST when followed by an "in" or "not in" test. + Also works for BUILD_LIST and BUILT_SET when followed by an "in" or "not in" + test; for BUILD_SET it assembles a frozenset rather than a tuple. */ static int tuple_of_constants(unsigned char *codestr, Py_ssize_t n, PyObject *consts) @@ -41,7 +40,7 @@ tuple_of_constants(unsigned char *codestr, Py_ssize_t n, PyObject *consts) /* Pre-conditions */ assert(PyList_CheckExact(consts)); - assert(codestr[n*3] == BUILD_TUPLE || codestr[n*3] == BUILD_LIST); + assert(codestr[n*3] == BUILD_TUPLE || codestr[n*3] == BUILD_LIST || codestr[n*3] == BUILD_SET); assert(GETARG(codestr, (n*3)) == n); for (i=0 ; i<n ; i++) assert(codestr[i*3] == LOAD_CONST); @@ -59,6 +58,16 @@ tuple_of_constants(unsigned char *codestr, Py_ssize_t n, PyObject *consts) PyTuple_SET_ITEM(newconst, i, constant); } + /* If it's a BUILD_SET, use the PyTuple we just built to create a + PyFrozenSet, and use that as the constant instead: */ + if (codestr[n*3] == BUILD_SET) { + PyObject *tuple = newconst; + newconst = PyFrozenSet_New(tuple); + Py_DECREF(tuple); + if (newconst == NULL) + return 0; + } + /* Append folded constant onto consts */ if (PyList_Append(consts, newconst)) { Py_DECREF(newconst); @@ -107,11 +116,6 @@ fold_binops_on_constants(unsigned char *codestr, PyObject *consts) case BINARY_MULTIPLY: newconst = PyNumber_Multiply(v, w); break; - case BINARY_DIVIDE: - /* Cannot fold this operation statically since - the result can depend on the run-time presence - of the -Qnew flag */ - return 0; case BINARY_TRUE_DIVIDE: newconst = PyNumber_TrueDivide(v, w); break; @@ -160,13 +164,16 @@ fold_binops_on_constants(unsigned char *codestr, PyObject *consts) return 0; } if (newconst == NULL) { - PyErr_Clear(); + if(!PyErr_ExceptionMatches(PyExc_KeyboardInterrupt)) + PyErr_Clear(); return 0; } size = PyObject_Size(newconst); - if (size == -1) + if (size == -1) { + if (PyErr_ExceptionMatches(PyExc_KeyboardInterrupt)) + return 0; PyErr_Clear(); - else if (size > 20) { + } else if (size > 20) { Py_DECREF(newconst); return 0; } @@ -206,12 +213,12 @@ fold_unaryops_on_constants(unsigned char *codestr, PyObject *consts) if (PyObject_IsTrue(v) == 1) newconst = PyNumber_Negative(v); break; - case UNARY_CONVERT: - newconst = PyObject_Repr(v); - break; case UNARY_INVERT: newconst = PyNumber_Invert(v); break; + case UNARY_POSITIVE: + newconst = PyNumber_Positive(v); + break; default: /* Called with an unknown opcode */ PyErr_Format(PyExc_SystemError, @@ -220,7 +227,8 @@ fold_unaryops_on_constants(unsigned char *codestr, PyObject *consts) return 0; } if (newconst == NULL) { - PyErr_Clear(); + if(!PyErr_ExceptionMatches(PyExc_KeyboardInterrupt)) + PyErr_Clear(); return 0; } @@ -280,14 +288,47 @@ markblocks(unsigned char *code, Py_ssize_t len) return blocks; } +/* Helper to replace LOAD_NAME None/True/False with LOAD_CONST + Returns: 0 if no change, 1 if change, -1 if error */ +static int +load_global(unsigned char *codestr, Py_ssize_t i, char *name, PyObject *consts) +{ + Py_ssize_t j; + PyObject *obj; + if (name == NULL) + return 0; + if (strcmp(name, "None") == 0) + obj = Py_None; + else if (strcmp(name, "True") == 0) + obj = Py_True; + else if (strcmp(name, "False") == 0) + obj = Py_False; + else + return 0; + for (j = 0; j < PyList_GET_SIZE(consts); j++) { + if (PyList_GET_ITEM(consts, j) == obj) + break; + } + if (j == PyList_GET_SIZE(consts)) { + if (PyList_Append(consts, obj) < 0) + return -1; + } + assert(PyList_GET_ITEM(consts, j) == obj); + codestr[i] = LOAD_CONST; + SETARG(codestr, i, j); + return 1; +} + /* Perform basic peephole optimizations to components of a code object. The consts object should still be in list form to allow new constants to be appended. - To keep the optimizer simple, it bails out (does nothing) for code - containing extended arguments or that has a length over 32,700. That - allows us to avoid overflow and sign issues. Likewise, it bails when - the lineno table has complex encoding for gaps >= 255. + To keep the optimizer simple, it bails out (does nothing) for code that + has a length over 32,700, and does not calculate extended arguments. + That allows us to avoid overflow and sign issues. Likewise, it bails when + the lineno table has complex encoding for gaps >= 255. EXTENDED_ARG can + appear before MAKE_FUNCTION; in this case both opcodes are skipped. + EXTENDED_ARG preceding any other opcode causes the optimizer to bail. Optimizations are restricted to simple transformations occuring within a single basic block. All transformations keep the code size the same or @@ -315,15 +356,15 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, goto exitError; /* Bypass optimization when the lineno table is too complex */ - assert(PyString_Check(lineno_obj)); - lineno = (unsigned char*)PyString_AS_STRING(lineno_obj); - tabsiz = PyString_GET_SIZE(lineno_obj); + assert(PyBytes_Check(lineno_obj)); + lineno = (unsigned char*)PyBytes_AS_STRING(lineno_obj); + tabsiz = PyBytes_GET_SIZE(lineno_obj); if (memchr(lineno, 255, tabsiz) != NULL) goto exitUnchanged; /* Avoid situations where jump retargeting could overflow */ - assert(PyString_Check(code)); - codelen = PyString_GET_SIZE(code); + assert(PyBytes_Check(code)); + codelen = PyBytes_GET_SIZE(code); if (codelen > 32700) goto exitUnchanged; @@ -332,9 +373,9 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, if (codestr == NULL) goto exitError; codestr = (unsigned char *)memcpy(codestr, - PyString_AS_STRING(code), codelen); + PyBytes_AS_STRING(code), codelen); - /* Verify that RETURN_VALUE terminates the codestring. This allows + /* Verify that RETURN_VALUE terminates the codestring. This allows the various transformation patterns to look ahead several instructions without additional checks to make sure they are not looking beyond the end of the code string. @@ -387,25 +428,17 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, codestr[i+3] = NOP; break; - /* Replace LOAD_GLOBAL/LOAD_NAME None - with LOAD_CONST None */ + /* Replace LOAD_GLOBAL/LOAD_NAME None/True/False + with LOAD_CONST None/True/False */ case LOAD_NAME: case LOAD_GLOBAL: j = GETARG(codestr, i); - name = PyString_AsString(PyTuple_GET_ITEM(names, j)); - if (name == NULL || strcmp(name, "None") != 0) + name = _PyUnicode_AsString(PyTuple_GET_ITEM(names, j)); + h = load_global(codestr, i, name, consts); + if (h < 0) + goto exitError; + else if (h == 0) continue; - for (j=0 ; j < PyList_GET_SIZE(consts) ; j++) { - if (PyList_GET_ITEM(consts, j) == Py_None) - break; - } - if (j == PyList_GET_SIZE(consts)) { - if (PyList_Append(consts, Py_None) == -1) - goto exitError; - } - assert(PyList_GET_ITEM(consts, j) == Py_None); - codestr[i] = LOAD_CONST; - SETARG(codestr, i, j); cumlc = lastlc + 1; break; @@ -423,20 +456,21 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, cumlc = 0; break; - /* Try to fold tuples of constants (includes a case for lists + /* Try to fold tuples of constants (includes a case for lists and sets which are only used for "in" and "not in" tests). Skip over BUILD_SEQN 1 UNPACK_SEQN 1. Replace BUILD_SEQN 2 UNPACK_SEQN 2 with ROT2. Replace BUILD_SEQN 3 UNPACK_SEQN 3 with ROT3 ROT2. */ case BUILD_TUPLE: case BUILD_LIST: + case BUILD_SET: j = GETARG(codestr, i); h = i - 3 * j; - if (h >= 0 && - j <= lastlc && + if (h >= 0 && + j <= lastlc && ((opcode == BUILD_TUPLE && ISBASICBLOCK(blocks, h, 3*(j+1))) || - (opcode == BUILD_LIST && + ((opcode == BUILD_LIST || opcode == BUILD_SET) && codestr[i+3]==COMPARE_OP && ISBASICBLOCK(blocks, h, 3*(j+2)) && (GETARG(codestr,i+3)==6 || @@ -448,7 +482,8 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, } if (codestr[i+3] != UNPACK_SEQUENCE || !ISBASICBLOCK(blocks,i,6) || - j != GETARG(codestr, i+3)) + j != GETARG(codestr, i+3) || + opcode == BUILD_SET) continue; if (j == 1) { memset(codestr+i, NOP, 6); @@ -477,8 +512,8 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, case BINARY_AND: case BINARY_XOR: case BINARY_OR: - if (lastlc >= 2 && - ISBASICBLOCK(blocks, i-6, 7) && + if (lastlc >= 2 && + ISBASICBLOCK(blocks, i-6, 7) && fold_binops_on_constants(&codestr[i-6], consts)) { i -= 2; assert(codestr[i] == LOAD_CONST); @@ -487,13 +522,13 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, break; /* Fold unary ops on constants. - LOAD_CONST c1 UNARY_OP --> LOAD_CONST unary_op(c) */ + LOAD_CONST c1 UNARY_OP --> LOAD_CONST unary_op(c) */ case UNARY_NEGATIVE: - case UNARY_CONVERT: case UNARY_INVERT: - if (lastlc >= 1 && - ISBASICBLOCK(blocks, i-3, 4) && - fold_unaryops_on_constants(&codestr[i-3], consts)) { + case UNARY_POSITIVE: + if (lastlc >= 1 && + ISBASICBLOCK(blocks, i-3, 4) && + fold_unaryops_on_constants(&codestr[i-3], consts)) { i -= 2; assert(codestr[i] == LOAD_CONST); cumlc = 1; @@ -519,7 +554,8 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, tgt = GETJUMPTGT(codestr, i); j = codestr[tgt]; if (CONDITIONAL_JUMP(j)) { - /* NOTE: all possible jumps here are absolute! */ + /* NOTE: all possible jumps here are + absolute! */ if (JUMPS_ON_TRUE(j) == JUMPS_ON_TRUE(opcode)) { /* The second jump will be taken iff the first is. */ @@ -530,10 +566,13 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, SETARG(codestr, i, tgttgt); goto reoptimize_current; } else { - /* The second jump is not taken if the first is (so - jump past it), and all conditional jumps pop their - argument when they're not taken (so change the - first jump to pop its argument when it's taken). */ + /* The second jump is not taken + if the first is (so jump past + it), and all conditional + jumps pop their argument when + they're not taken (so change + the first jump to pop its + argument when it's taken). */ if (JUMPS_ON_TRUE(opcode)) codestr[i] = POP_JUMP_IF_TRUE; else @@ -569,15 +608,19 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, if (opcode == JUMP_FORWARD) /* JMP_ABS can go backwards */ opcode = JUMP_ABSOLUTE; if (!ABSOLUTE_JUMP(opcode)) - tgttgt -= i + 3; /* Calc relative jump addr */ - if (tgttgt < 0) /* No backward relative jumps */ + tgttgt -= i + 3; /* Calc relative jump addr */ + if (tgttgt < 0) /* No backward relative jumps */ continue; codestr[i] = opcode; SETARG(codestr, i, tgttgt); break; case EXTENDED_ARG: - goto exitUnchanged; + if (codestr[i+3] != MAKE_FUNCTION) + goto exitUnchanged; + /* don't visit MAKE_FUNCTION as GETARG will be wrong */ + i += 3; + break; /* Replace RETURN LOAD_CONST None RETURN with just RETURN */ /* Remove unreachable JUMPs after RETURN */ @@ -644,7 +687,7 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, } assert(h + nops == codelen); - code = PyString_FromStringAndSize((char *)codestr, h); + code = PyBytes_FromStringAndSize((char *)codestr, h); PyMem_Free(addrmap); PyMem_Free(codestr); PyMem_Free(blocks); diff --git a/Python/pyarena.c b/Python/pyarena.c index 2d63638856..5a255ae497 100644 --- a/Python/pyarena.c +++ b/Python/pyarena.c @@ -1,5 +1,4 @@ #include "Python.h" -#include "pyarena.h" /* A simple arena block structure. diff --git a/Python/pystate.c b/Python/pystate.c index 56eed88407..42bc3eceaf 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -46,7 +46,9 @@ static int autoTLSkey = 0; static PyInterpreterState *interp_head = NULL; -PyThreadState *_PyThreadState_Current = NULL; +/* Assuming the current thread holds the GIL, this is the + PyThreadState for the current thread. */ +_Py_atomic_address _PyThreadState_Current = {NULL}; PyThreadFrameGetter _PyThreadState_GetFrame = NULL; #ifdef WITH_THREAD @@ -68,12 +70,15 @@ PyInterpreterState_New(void) #endif interp->modules = NULL; interp->modules_reloading = NULL; + interp->modules_by_index = NULL; interp->sysdict = NULL; interp->builtins = NULL; interp->tstate_head = NULL; interp->codec_search_path = NULL; interp->codec_search_cache = NULL; interp->codec_error_registry = NULL; + interp->codecs_initialized = 0; + interp->fscodec_initialized = 0; #ifdef HAVE_DLOPEN #ifdef RTLD_NOW interp->dlopenflags = RTLD_NOW; @@ -107,6 +112,7 @@ PyInterpreterState_Clear(PyInterpreterState *interp) Py_CLEAR(interp->codec_search_cache); Py_CLEAR(interp->codec_error_registry); Py_CLEAR(interp->modules); + Py_CLEAR(interp->modules_by_index); Py_CLEAR(interp->modules_reloading); Py_CLEAR(interp->sysdict); Py_CLEAR(interp->builtins); @@ -143,6 +149,12 @@ PyInterpreterState_Delete(PyInterpreterState *interp) *p = interp->next; HEAD_UNLOCK(); free(interp); +#ifdef WITH_THREAD + if (interp_head == NULL && head_mutex != NULL) { + PyThread_free_lock(head_mutex); + head_mutex = NULL; + } +#endif } @@ -166,6 +178,8 @@ new_threadstate(PyInterpreterState *interp, int init) tstate->frame = NULL; tstate->recursion_depth = 0; + tstate->overflowed = 0; + tstate->recursion_critical = 0; tstate->tracing = 0; tstate->use_tracing = 0; tstate->tick_counter = 0; @@ -227,6 +241,41 @@ _PyThreadState_Init(PyThreadState *tstate) #endif } +PyObject* +PyState_FindModule(struct PyModuleDef* m) +{ + Py_ssize_t index = m->m_base.m_index; + PyInterpreterState *state = PyThreadState_GET()->interp; + PyObject *res; + if (index == 0) + return NULL; + if (state->modules_by_index == NULL) + return NULL; + if (index >= PyList_GET_SIZE(state->modules_by_index)) + return NULL; + res = PyList_GET_ITEM(state->modules_by_index, index); + return res==Py_None ? NULL : res; +} + +int +_PyState_AddModule(PyObject* module, struct PyModuleDef* def) +{ + PyInterpreterState *state = PyThreadState_GET()->interp; + if (!def) + return -1; + if (!state->modules_by_index) { + state->modules_by_index = PyList_New(0); + if (!state->modules_by_index) + return -1; + } + while(PyList_GET_SIZE(state->modules_by_index) <= def->m_base.m_index) + if (PyList_Append(state->modules_by_index, Py_None) < 0) + return -1; + Py_INCREF(module); + return PyList_SetItem(state->modules_by_index, + def->m_base.m_index, module); +} + void PyThreadState_Clear(PyThreadState *tstate) { @@ -296,7 +345,7 @@ tstate_delete_common(PyThreadState *tstate) void PyThreadState_Delete(PyThreadState *tstate) { - if (tstate == _PyThreadState_Current) + if (tstate == _Py_atomic_load_relaxed(&_PyThreadState_Current)) Py_FatalError("PyThreadState_Delete: tstate is still current"); tstate_delete_common(tstate); #ifdef WITH_THREAD @@ -310,11 +359,12 @@ PyThreadState_Delete(PyThreadState *tstate) void PyThreadState_DeleteCurrent() { - PyThreadState *tstate = _PyThreadState_Current; + PyThreadState *tstate = (PyThreadState*)_Py_atomic_load_relaxed( + &_PyThreadState_Current); if (tstate == NULL) Py_FatalError( "PyThreadState_DeleteCurrent: no current tstate"); - _PyThreadState_Current = NULL; + _Py_atomic_store_relaxed(&_PyThreadState_Current, NULL); tstate_delete_common(tstate); if (autoInterpreterState && PyThread_get_key_value(autoTLSkey) == tstate) PyThread_delete_key_value(autoTLSkey); @@ -326,19 +376,22 @@ PyThreadState_DeleteCurrent() PyThreadState * PyThreadState_Get(void) { - if (_PyThreadState_Current == NULL) + PyThreadState *tstate = (PyThreadState*)_Py_atomic_load_relaxed( + &_PyThreadState_Current); + if (tstate == NULL) Py_FatalError("PyThreadState_Get: no current thread"); - return _PyThreadState_Current; + return tstate; } PyThreadState * PyThreadState_Swap(PyThreadState *newts) { - PyThreadState *oldts = _PyThreadState_Current; + PyThreadState *oldts = (PyThreadState*)_Py_atomic_load_relaxed( + &_PyThreadState_Current); - _PyThreadState_Current = newts; + _Py_atomic_store_relaxed(&_PyThreadState_Current, newts); /* It should not be possible for more than one thread state to be used for a thread. Check this the best we can in debug builds. @@ -367,16 +420,18 @@ PyThreadState_Swap(PyThreadState *newts) PyObject * PyThreadState_GetDict(void) { - if (_PyThreadState_Current == NULL) + PyThreadState *tstate = (PyThreadState*)_Py_atomic_load_relaxed( + &_PyThreadState_Current); + if (tstate == NULL) return NULL; - if (_PyThreadState_Current->dict == NULL) { + if (tstate->dict == NULL) { PyObject *d; - _PyThreadState_Current->dict = d = PyDict_New(); + tstate->dict = d = PyDict_New(); if (d == NULL) PyErr_Clear(); } - return _PyThreadState_Current->dict; + return tstate->dict; } @@ -415,6 +470,7 @@ PyThreadState_SetAsyncExc(long id, PyObject *exc) { p->async_exc = exc; HEAD_UNLOCK(); Py_XDECREF(old_exc); + _PyEval_SignalAsyncExc(); return 1; } } @@ -477,7 +533,7 @@ _PyThread_CurrentFrames(void) struct _frame *frame = t->frame; if (frame == NULL) continue; - id = PyInt_FromLong(t->thread_id); + id = PyLong_FromLong(t->thread_id); if (id == NULL) goto Fail; stat = PyDict_SetItem(result, id, (PyObject *)frame); @@ -511,10 +567,7 @@ PyThreadState_IsCurrent(PyThreadState *tstate) { /* Must be the tstate for this thread */ assert(PyGILState_GetThisThreadState()==tstate); - /* On Windows at least, simple reads and writes to 32 bit values - are atomic. - */ - return tstate == _PyThreadState_Current; + return tstate == _Py_atomic_load_relaxed(&_PyThreadState_Current); } /* Internal initialization/finalization functions called by @@ -525,6 +578,8 @@ _PyGILState_Init(PyInterpreterState *i, PyThreadState *t) { assert(i && t); /* must init with valid states */ autoTLSkey = PyThread_create_key(); + if (autoTLSkey == -1) + Py_FatalError("Could not allocate TLS entry"); autoInterpreterState = i; assert(PyThread_get_key_value(autoTLSkey) == NULL); assert(t->gilstate_counter == 0); @@ -539,6 +594,24 @@ _PyGILState_Fini(void) autoInterpreterState = NULL; } +/* Reset the TLS key - called by PyOS_AfterFork(). + * This should not be necessary, but some - buggy - pthread implementations + * don't reset TLS upon fork(), see issue #10517. + */ +void +_PyGILState_Reinit(void) +{ + PyThreadState *tstate = PyGILState_GetThisThreadState(); + PyThread_delete_key(autoTLSkey); + if ((autoTLSkey = PyThread_create_key()) == -1) + Py_FatalError("Could not allocate TLS entry"); + + /* If the thread had an associated auto thread state, reassociate it with + * the new key. */ + if (tstate && PyThread_set_key_value(autoTLSkey, (void *)tstate) < 0) + Py_FatalError("Couldn't create autoTLSkey mapping"); +} + /* When a thread state is created for a thread by some mechanism other than PyGILState_Ensure, it's important that the GILState machinery knows about it so it doesn't try to create another thread state for the thread (this is diff --git a/Python/pystrtod.c b/Python/pystrtod.c index 859af435f9..75e3032fb7 100644 --- a/Python/pystrtod.c +++ b/Python/pystrtod.c @@ -58,7 +58,7 @@ _Py_parse_inf_or_nan(const char *p, char **endptr) } /** - * PyOS_ascii_strtod: + * _PyOS_ascii_strtod: * @nptr: the string to convert to a numeric value. * @endptr: if non-%NULL, it returns the character after * the last character used in the conversion. @@ -88,7 +88,7 @@ _Py_parse_inf_or_nan(const char *p, char **endptr) #ifndef PY_NO_SHORT_FLOAT_REPR -double +static double _PyOS_ascii_strtod(const char *nptr, char **endptr) { double result; @@ -121,7 +121,7 @@ _PyOS_ascii_strtod(const char *nptr, char **endptr) correctly rounded results. */ -double +static double _PyOS_ascii_strtod(const char *nptr, char **endptr) { char *fail_pos; @@ -270,48 +270,10 @@ _PyOS_ascii_strtod(const char *nptr, char **endptr) #endif -/* PyOS_ascii_strtod is DEPRECATED in Python 2.7 and 3.1 */ - -double -PyOS_ascii_strtod(const char *nptr, char **endptr) -{ - char *fail_pos; - const char *p; - double x; - - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "PyOS_ascii_strtod and PyOS_ascii_atof are " - "deprecated. Use PyOS_string_to_double " - "instead.", 1) < 0) - return -1.0; - - /* _PyOS_ascii_strtod already does everything that we want, - except that it doesn't parse leading whitespace */ - p = nptr; - while (Py_ISSPACE(*p)) - p++; - x = _PyOS_ascii_strtod(p, &fail_pos); - if (fail_pos == p) - fail_pos = (char *)nptr; - if (endptr) - *endptr = (char *)fail_pos; - return x; -} - -/* PyOS_ascii_strtod is DEPRECATED in Python 2.7 and 3.1 */ - -double -PyOS_ascii_atof(const char *nptr) -{ - return PyOS_ascii_strtod(nptr, NULL); -} - -/* PyOS_string_to_double is the recommended replacement for the deprecated - PyOS_ascii_strtod and PyOS_ascii_atof functions. It converts a - null-terminated byte string s (interpreted as a string of ASCII characters) - to a float. The string should not have leading or trailing whitespace (in - contrast, PyOS_ascii_strtod allows leading whitespace but not trailing - whitespace). The conversion is independent of the current locale. +/* PyOS_string_to_double converts a null-terminated byte string s (interpreted + as a string of ASCII characters) to a float. The string should not have + leading or trailing whitespace. The conversion is independent of the + current locale. If endptr is NULL, try to convert the whole string. Raise ValueError and return -1.0 if the string is not a valid representation of a floating-point @@ -369,6 +331,8 @@ PyOS_string_to_double(const char *s, return result; } +#ifdef PY_NO_SHORT_FLOAT_REPR + /* Given a string that may have a decimal point in the current locale, change it back to a dot. Since the string cannot get longer, no need for a maximum buffer size parameter. */ @@ -618,12 +582,13 @@ ensure_decimal_point(char* buffer, size_t buf_size, int precision) #define FLOAT_FORMATBUFLEN 120 /** - * PyOS_ascii_formatd: + * _PyOS_ascii_formatd: * @buffer: A buffer to place the resulting string in * @buf_size: The length of the buffer. * @format: The printf()-style format to use for the * code to use for converting. * @d: The #gdouble to convert + * @precision: The precision to use when formatting. * * Converts a #gdouble to a string, using the '.' as * decimal point. To format the number you pass in @@ -636,7 +601,7 @@ ensure_decimal_point(char* buffer, size_t buf_size, int precision) * Return value: The pointer to the buffer with the converted string. * On failure returns NULL but does not set any Python exception. **/ -char * +static char * _PyOS_ascii_formatd(char *buffer, size_t buf_size, const char *format, @@ -716,22 +681,6 @@ _PyOS_ascii_formatd(char *buffer, return buffer; } -char * -PyOS_ascii_formatd(char *buffer, - size_t buf_size, - const char *format, - double d) -{ - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "PyOS_ascii_formatd is deprecated, " - "use PyOS_double_to_string instead", 1) < 0) - return NULL; - - return _PyOS_ascii_formatd(buffer, buf_size, format, d, -1); -} - -#ifdef PY_NO_SHORT_FLOAT_REPR - /* The fallback code to use if _Py_dg_dtoa is not available. */ PyAPI_FUNC(char *) PyOS_double_to_string(double val, diff --git a/Python/pythonrun.c b/Python/pythonrun.c index a2e1e74589..3639fa7e22 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -11,13 +11,10 @@ #include "parsetok.h" #include "errcode.h" #include "code.h" -#include "compile.h" #include "symtable.h" -#include "pyarena.h" #include "ast.h" -#include "eval.h" #include "marshal.h" -#include "abstract.h" +#include "osdefs.h" #ifdef HAVE_SIGNAL_H #include <signal.h> @@ -35,6 +32,7 @@ #ifdef MS_WINDOWS #undef BYTE #include "windows.h" +#define PATH_MAX MAXPATHLEN #endif #ifndef Py_REF_DEBUG @@ -49,24 +47,29 @@ extern "C" { #endif -extern char *Py_GetPath(void); +extern wchar_t *Py_GetPath(void); extern grammar _PyParser_Grammar; /* From graminit.c */ /* Forward */ static void initmain(void); +static int initfsencoding(PyInterpreterState *interp); static void initsite(void); +static int initstdio(void); +static void flush_io(void); static PyObject *run_mod(mod_ty, const char *, PyObject *, PyObject *, PyCompilerFlags *, PyArena *); static PyObject *run_pyc_file(FILE *, const char *, PyObject *, PyObject *, PyCompilerFlags *); static void err_input(perrdetail *); static void initsigs(void); +static void call_py_exitfuncs(void); static void wait_for_thread_shutdown(void); -static void call_sys_exitfunc(void); static void call_ll_exitfuncs(void); extern void _PyUnicode_Init(void); extern void _PyUnicode_Fini(void); +extern int _PyLong_Init(void); +extern void PyLong_Fini(void); #ifdef WITH_THREAD extern void _PyGILState_Init(PyInterpreterState *, PyThreadState *); @@ -75,6 +78,7 @@ extern void _PyGILState_Fini(void); int Py_DebugFlag; /* Needed by parser.c */ int Py_VerboseFlag; /* Needed by import.c */ +int Py_QuietFlag; /* Needed by sysmodule.c */ int Py_InteractiveFlag; /* Needed by Py_FdIsInteractive() below */ int Py_InspectFlag; /* Needed to determine whether to exit at SystemExit */ int Py_NoSiteFlag; /* Suppress 'import site' */ @@ -82,15 +86,13 @@ int Py_BytesWarningFlag; /* Warn on str(bytes) and str(buffer) */ int Py_DontWriteBytecodeFlag; /* Suppress writing bytecode files (*.py[co]) */ int Py_UseClassExceptionsFlag = 1; /* Needed by bltinmodule.c: deprecated */ int Py_FrozenFlag; /* Needed by getpath.c */ -int Py_UnicodeFlag = 0; /* Needed by compile.c */ int Py_IgnoreEnvironmentFlag; /* e.g. PYTHONPATH, PYTHONHOME */ -/* _XXX Py_QnewFlag should go away in 2.3. It's true iff -Qnew is passed, - on the command line, and is used in 2.2 by ceval.c to make all "/" divisions - true divisions (which they will be in 2.3). */ -int _Py_QnewFlag = 0; int Py_NoUserSiteDirectory = 0; /* for -s and site.py */ +int Py_UnbufferedStdioFlag = 0; /* Unbuffered binary std{in,out,err} */ int Py_HashRandomizationFlag = 0; /* for -R and PYTHONHASHSEED */ +PyThreadState *_Py_Finalizing = NULL; + /* PyModule_GetWarningsModule is no longer necessary as of 2.6 since _warnings is builtin. This API should not be used. */ PyObject * @@ -132,32 +134,71 @@ add_flag(int flag, const char *envs) return flag; } +static char* +get_codec_name(const char *encoding) +{ + char *name_utf8, *name_str; + PyObject *codec, *name = NULL; + + codec = _PyCodec_Lookup(encoding); + if (!codec) + goto error; + + name = PyObject_GetAttrString(codec, "name"); + Py_CLEAR(codec); + if (!name) + goto error; + + name_utf8 = _PyUnicode_AsString(name); + if (name_utf8 == NULL) + goto error; + name_str = strdup(name_utf8); + Py_DECREF(name); + if (name_str == NULL) { + PyErr_NoMemory(); + return NULL; + } + return name_str; + +error: + Py_XDECREF(codec); + Py_XDECREF(name); + return NULL; +} + +#if defined(HAVE_LANGINFO_H) && defined(CODESET) +static char* +get_codeset(void) +{ + char* codeset = nl_langinfo(CODESET); + if (!codeset || codeset[0] == '\0') { + PyErr_SetString(PyExc_ValueError, "CODESET is not set or empty"); + return NULL; + } + return get_codec_name(codeset); +} +#endif + void Py_InitializeEx(int install_sigs) { PyInterpreterState *interp; PyThreadState *tstate; - PyObject *bimod, *sysmod; + PyObject *bimod, *sysmod, *pstderr; char *p; - char *icodeset = NULL; /* On Windows, input codeset may theoretically - differ from output codeset. */ - char *codeset = NULL; - char *errors = NULL; - int free_codeset = 0; - int overridden = 0; - PyObject *sys_stream, *sys_isatty; -#if defined(Py_USING_UNICODE) && defined(HAVE_LANGINFO_H) && defined(CODESET) - char *saved_locale, *loc_codeset; -#endif -#ifdef MS_WINDOWS - char ibuf[128]; - char buf[128]; -#endif extern void _Py_ReadyTypes(void); if (initialized) return; initialized = 1; + _Py_Finalizing = NULL; + +#if defined(HAVE_LANGINFO_H) && defined(HAVE_SETLOCALE) + /* Set up the LC_CTYPE locale, so we can obtain + the locale's charset without having to switch + locales. */ + setlocale(LC_CTYPE, ""); +#endif if ((p = Py_GETENV("PYTHONDEBUG")) && *p != '\0') Py_DebugFlag = add_flag(Py_DebugFlag, p); @@ -183,14 +224,23 @@ Py_InitializeEx(int install_sigs) Py_FatalError("Py_Initialize: can't make first thread"); (void) PyThreadState_Swap(tstate); +#ifdef WITH_THREAD + /* We can't call _PyEval_FiniThreads() in Py_Finalize because + destroying the GIL might fail when it is being referenced from + another running thread (see issue #9901). + Instead we destroy the previously created GIL here, which ensures + that we can call Py_Initialize / Py_Finalize multiple times. */ + _PyEval_FiniThreads(); + + /* Auto-thread-state API */ + _PyGILState_Init(interp, tstate); +#endif /* WITH_THREAD */ + _Py_ReadyTypes(); if (!_PyFrame_Init()) Py_FatalError("Py_Initialize: can't init frames"); - if (!_PyInt_Init()) - Py_FatalError("Py_Initialize: can't init ints"); - if (!_PyLong_Init()) Py_FatalError("Py_Initialize: can't init longs"); @@ -206,19 +256,21 @@ Py_InitializeEx(int install_sigs) if (interp->modules_reloading == NULL) Py_FatalError("Py_Initialize: can't make modules_reloading dictionary"); -#ifdef Py_USING_UNICODE /* Init Unicode implementation; relies on the codec registry */ _PyUnicode_Init(); -#endif bimod = _PyBuiltin_Init(); if (bimod == NULL) - Py_FatalError("Py_Initialize: can't initialize __builtin__"); + Py_FatalError("Py_Initialize: can't initialize builtins modules"); + _PyImport_FixupBuiltin(bimod, "builtins"); interp->builtins = PyModule_GetDict(bimod); if (interp->builtins == NULL) Py_FatalError("Py_Initialize: can't initialize builtins dict"); Py_INCREF(interp->builtins); + /* initialize builtin exceptions */ + _PyExc_Init(); + sysmod = _PySys_Init(); if (sysmod == NULL) Py_FatalError("Py_Initialize: can't initialize sys"); @@ -226,149 +278,52 @@ Py_InitializeEx(int install_sigs) if (interp->sysdict == NULL) Py_FatalError("Py_Initialize: can't initialize sys dict"); Py_INCREF(interp->sysdict); - _PyImport_FixupExtension("sys", "sys"); + _PyImport_FixupBuiltin(sysmod, "sys"); PySys_SetPath(Py_GetPath()); PyDict_SetItemString(interp->sysdict, "modules", interp->modules); + /* Set up a preliminary stderr printer until we have enough + infrastructure for the io module in place. */ + pstderr = PyFile_NewStdPrinter(fileno(stderr)); + if (pstderr == NULL) + Py_FatalError("Py_Initialize: can't set preliminary stderr"); + PySys_SetObject("stderr", pstderr); + PySys_SetObject("__stderr__", pstderr); + Py_DECREF(pstderr); + _PyImport_Init(); - /* initialize builtin exceptions */ - _PyExc_Init(); - _PyImport_FixupExtension("exceptions", "exceptions"); + _PyImportHooks_Init(); + + /* Initialize _warnings. */ + _PyWarnings_Init(); - /* phase 2 of builtins */ - _PyImport_FixupExtension("__builtin__", "__builtin__"); + _PyTime_Init(); - _PyImportHooks_Init(); + if (initfsencoding(interp) < 0) + Py_FatalError("Py_Initialize: unable to load the file system codec"); if (install_sigs) initsigs(); /* Signal handling stuff, including initintr() */ + initmain(); /* Module __main__ */ + if (initstdio() < 0) + Py_FatalError( + "Py_Initialize: can't initialize sys standard streams"); + /* Initialize warnings. */ - _PyWarnings_Init(); if (PySys_HasWarnOptions()) { PyObject *warnings_module = PyImport_ImportModule("warnings"); - if (!warnings_module) - PyErr_Clear(); + if (warnings_module == NULL) { + fprintf(stderr, "'import warnings' failed; traceback:\n"); + PyErr_Print(); + } Py_XDECREF(warnings_module); } - initmain(); /* Module __main__ */ - - /* auto-thread-state API, if available */ -#ifdef WITH_THREAD - _PyGILState_Init(interp, tstate); -#endif /* WITH_THREAD */ - if (!Py_NoSiteFlag) initsite(); /* Module site */ - - if ((p = Py_GETENV("PYTHONIOENCODING")) && *p != '\0') { - p = icodeset = codeset = strdup(p); - free_codeset = 1; - errors = strchr(p, ':'); - if (errors) { - *errors = '\0'; - errors++; - } - overridden = 1; - } - -#if defined(Py_USING_UNICODE) && defined(HAVE_LANGINFO_H) && defined(CODESET) - /* On Unix, set the file system encoding according to the - user's preference, if the CODESET names a well-known - Python codec, and Py_FileSystemDefaultEncoding isn't - initialized by other means. Also set the encoding of - stdin and stdout if these are terminals, unless overridden. */ - - if (!overridden || !Py_FileSystemDefaultEncoding) { - saved_locale = strdup(setlocale(LC_CTYPE, NULL)); - setlocale(LC_CTYPE, ""); - loc_codeset = nl_langinfo(CODESET); - if (loc_codeset && *loc_codeset) { - PyObject *enc = PyCodec_Encoder(loc_codeset); - if (enc) { - loc_codeset = strdup(loc_codeset); - Py_DECREF(enc); - } else { - if (PyErr_ExceptionMatches(PyExc_LookupError)) { - PyErr_Clear(); - loc_codeset = NULL; - } else { - PyErr_Print(); - exit(1); - } - } - } else - loc_codeset = NULL; - setlocale(LC_CTYPE, saved_locale); - free(saved_locale); - - if (!overridden) { - codeset = icodeset = loc_codeset; - free_codeset = 1; - } - - /* Initialize Py_FileSystemDefaultEncoding from - locale even if PYTHONIOENCODING is set. */ - if (!Py_FileSystemDefaultEncoding) { - Py_FileSystemDefaultEncoding = loc_codeset; - if (!overridden) - free_codeset = 0; - } - } -#endif - -#ifdef MS_WINDOWS - if (!overridden) { - icodeset = ibuf; - codeset = buf; - sprintf(ibuf, "cp%d", GetConsoleCP()); - sprintf(buf, "cp%d", GetConsoleOutputCP()); - } -#endif - - if (codeset) { - sys_stream = PySys_GetObject("stdin"); - sys_isatty = PyObject_CallMethod(sys_stream, "isatty", ""); - if (!sys_isatty) - PyErr_Clear(); - if ((overridden || - (sys_isatty && PyObject_IsTrue(sys_isatty))) && - PyFile_Check(sys_stream)) { - if (!PyFile_SetEncodingAndErrors(sys_stream, icodeset, errors)) - Py_FatalError("Cannot set codeset of stdin"); - } - Py_XDECREF(sys_isatty); - - sys_stream = PySys_GetObject("stdout"); - sys_isatty = PyObject_CallMethod(sys_stream, "isatty", ""); - if (!sys_isatty) - PyErr_Clear(); - if ((overridden || - (sys_isatty && PyObject_IsTrue(sys_isatty))) && - PyFile_Check(sys_stream)) { - if (!PyFile_SetEncodingAndErrors(sys_stream, codeset, errors)) - Py_FatalError("Cannot set codeset of stdout"); - } - Py_XDECREF(sys_isatty); - - sys_stream = PySys_GetObject("stderr"); - sys_isatty = PyObject_CallMethod(sys_stream, "isatty", ""); - if (!sys_isatty) - PyErr_Clear(); - if((overridden || - (sys_isatty && PyObject_IsTrue(sys_isatty))) && - PyFile_Check(sys_stream)) { - if (!PyFile_SetEncodingAndErrors(sys_stream, codeset, errors)) - Py_FatalError("Cannot set codeset of stderr"); - } - Py_XDECREF(sys_isatty); - - if (free_codeset) - free(codeset); - } } void @@ -382,6 +337,48 @@ Py_Initialize(void) extern void dump_counts(FILE*); #endif +/* Flush stdout and stderr */ + +static int +file_is_closed(PyObject *fobj) +{ + int r; + PyObject *tmp = PyObject_GetAttrString(fobj, "closed"); + if (tmp == NULL) { + PyErr_Clear(); + return 0; + } + r = PyObject_IsTrue(tmp); + Py_DECREF(tmp); + if (r < 0) + PyErr_Clear(); + return r > 0; +} + +static void +flush_std_files(void) +{ + PyObject *fout = PySys_GetObject("stdout"); + PyObject *ferr = PySys_GetObject("stderr"); + PyObject *tmp; + + if (fout != NULL && fout != Py_None && !file_is_closed(fout)) { + tmp = PyObject_CallMethod(fout, "flush", ""); + if (tmp == NULL) + PyErr_WriteUnraisable(fout); + else + Py_DECREF(tmp); + } + + if (ferr != NULL && ferr != Py_None && !file_is_closed(ferr)) { + tmp = PyObject_CallMethod(ferr, "flush", ""); + if (tmp == NULL) + PyErr_Clear(); + else + Py_DECREF(tmp); + } +} + /* Undo the effect of Py_Initialize(). Beware: if multiple interpreter and/or thread states exist, these @@ -416,13 +413,20 @@ Py_Finalize(void) * threads created thru it, so this also protects pending imports in * the threads created via Threading. */ - call_sys_exitfunc(); - initialized = 0; + call_py_exitfuncs(); /* Get current thread state and interpreter pointer */ tstate = PyThreadState_GET(); interp = tstate->interp; + /* Remaining threads (e.g. daemon threads) will automatically exit + after taking the GIL (in PyEval_RestoreThread()). */ + _Py_Finalizing = tstate; + initialized = 0; + + /* Flush stdout+stderr */ + flush_std_files(); + /* Disable signal handling */ PyOS_FiniInterrupts(); @@ -449,10 +453,16 @@ Py_Finalize(void) while (PyGC_Collect() > 0) /* nothing */; #endif + /* We run this while most interpreter state is still alive, so that + debug information can be printed out */ + _PyGC_Fini(); /* Destroy all modules */ PyImport_Cleanup(); + /* Flush stdout+stderr (again, in case more was printed) */ + flush_std_files(); + /* Collect final garbage. This disposes of cycles created by * new-style class definitions, for example. * XXX This is disabled because it caused too many problems. If @@ -519,16 +529,20 @@ Py_Finalize(void) PyTuple_Fini(); PyList_Fini(); PySet_Fini(); - PyString_Fini(); + PyBytes_Fini(); PyByteArray_Fini(); - PyInt_Fini(); + PyLong_Fini(); PyFloat_Fini(); PyDict_Fini(); -#ifdef Py_USING_UNICODE /* Cleanup Unicode implementation */ _PyUnicode_Fini(); -#endif + + /* reset file system default encoding */ + if (!Py_HasFileSystemDefaultEncoding && Py_FileSystemDefaultEncoding) { + free((char*)Py_FileSystemDefaultEncoding); + Py_FileSystemDefaultEncoding = NULL; + } /* XXX Still allocated: - various static ad-hoc pointers to interned strings @@ -594,15 +608,20 @@ Py_NewInterpreter(void) interp->modules = PyDict_New(); interp->modules_reloading = PyDict_New(); - bimod = _PyImport_FindExtension("__builtin__", "__builtin__"); + bimod = _PyImport_FindBuiltin("builtins"); if (bimod != NULL) { interp->builtins = PyModule_GetDict(bimod); if (interp->builtins == NULL) goto handle_error; Py_INCREF(interp->builtins); } - sysmod = _PyImport_FindExtension("sys", "sys"); + + /* initialize builtin exceptions */ + _PyExc_Init(); + + sysmod = _PyImport_FindBuiltin("sys"); if (bimod != NULL && sysmod != NULL) { + PyObject *pstderr; interp->sysdict = PyModule_GetDict(sysmod); if (interp->sysdict == NULL) goto handle_error; @@ -610,7 +629,23 @@ Py_NewInterpreter(void) PySys_SetPath(Py_GetPath()); PyDict_SetItemString(interp->sysdict, "modules", interp->modules); + /* Set up a preliminary stderr printer until we have enough + infrastructure for the io module in place. */ + pstderr = PyFile_NewStdPrinter(fileno(stderr)); + if (pstderr == NULL) + Py_FatalError("Py_Initialize: can't set preliminary stderr"); + PySys_SetObject("stderr", pstderr); + PySys_SetObject("__stderr__", pstderr); + Py_DECREF(pstderr); + _PyImportHooks_Init(); + + if (initfsencoding(interp) < 0) + goto handle_error; + + if (initstdio() < 0) + Py_FatalError( + "Py_Initialize: can't initialize sys standard streams"); initmain(); if (!Py_NoSiteFlag) initsite(); @@ -622,7 +657,7 @@ Py_NewInterpreter(void) handle_error: /* Oops, it didn't work. Undo it all. */ - PyErr_Print(); + PyErr_PrintEx(0); PyThreadState_Clear(tstate); PyThreadState_Swap(save_tstate); PyThreadState_Delete(tstate); @@ -661,35 +696,47 @@ Py_EndInterpreter(PyThreadState *tstate) PyInterpreterState_Delete(interp); } -static char *progname = "python"; +#ifdef MS_WINDOWS +static wchar_t *progname = L"python"; +#else +static wchar_t *progname = L"python3"; +#endif void -Py_SetProgramName(char *pn) +Py_SetProgramName(wchar_t *pn) { if (pn && *pn) progname = pn; } -char * +wchar_t * Py_GetProgramName(void) { return progname; } -static char *default_home = NULL; +static wchar_t *default_home = NULL; +static wchar_t env_home[PATH_MAX+1]; void -Py_SetPythonHome(char *home) +Py_SetPythonHome(wchar_t *home) { default_home = home; } -char * +wchar_t * Py_GetPythonHome(void) { - char *home = default_home; - if (home == NULL && !Py_IgnoreEnvironmentFlag) - home = Py_GETENV("PYTHONHOME"); + wchar_t *home = default_home; + if (home == NULL && !Py_IgnoreEnvironmentFlag) { + char* chome = Py_GETENV("PYTHONHOME"); + if (chome) { + size_t r = mbstowcs(env_home, chome, PATH_MAX+1); + if (r != (size_t)-1 && r <= PATH_MAX) + home = env_home; + } + + } return home; } @@ -704,14 +751,50 @@ initmain(void) Py_FatalError("can't create __main__ module"); d = PyModule_GetDict(m); if (PyDict_GetItemString(d, "__builtins__") == NULL) { - PyObject *bimod = PyImport_ImportModule("__builtin__"); + PyObject *bimod = PyImport_ImportModule("builtins"); if (bimod == NULL || PyDict_SetItemString(d, "__builtins__", bimod) != 0) Py_FatalError("can't add __builtins__ to __main__"); - Py_XDECREF(bimod); + Py_DECREF(bimod); } } +static int +initfsencoding(PyInterpreterState *interp) +{ + PyObject *codec; +#if defined(HAVE_LANGINFO_H) && defined(CODESET) + char *codeset = NULL; + + if (Py_FileSystemDefaultEncoding == NULL) { + /* On Unix, set the file system encoding according to the + user's preference, if the CODESET names a well-known + Python codec, and Py_FileSystemDefaultEncoding isn't + initialized by other means. */ + codeset = get_codeset(); + if (codeset == NULL) + Py_FatalError("Py_Initialize: Unable to get the locale encoding"); + + Py_FileSystemDefaultEncoding = codeset; + Py_HasFileSystemDefaultEncoding = 0; + interp->fscodec_initialized = 1; + return 0; + } +#endif + + /* the encoding is mbcs, utf-8 or ascii */ + codec = _PyCodec_Lookup(Py_FileSystemDefaultEncoding); + if (!codec) { + /* Such error can only occurs in critical situations: no more + * memory, import a module of the standard library failed, + * etc. */ + return -1; + } + Py_DECREF(codec); + interp->fscodec_initialized = 1; + return 0; +} + /* Import the site module (not into __main__ though) */ static void @@ -729,6 +812,244 @@ initsite(void) } } +static PyObject* +create_stdio(PyObject* io, + int fd, int write_mode, char* name, + char* encoding, char* errors) +{ + PyObject *buf = NULL, *stream = NULL, *text = NULL, *raw = NULL, *res; + const char* mode; + const char* newline; + PyObject *line_buffering; + int buffering, isatty; + + /* stdin is always opened in buffered mode, first because it shouldn't + make a difference in common use cases, second because TextIOWrapper + depends on the presence of a read1() method which only exists on + buffered streams. + */ + if (Py_UnbufferedStdioFlag && write_mode) + buffering = 0; + else + buffering = -1; + if (write_mode) + mode = "wb"; + else + mode = "rb"; + buf = PyObject_CallMethod(io, "open", "isiOOOi", + fd, mode, buffering, + Py_None, Py_None, Py_None, 0); + if (buf == NULL) + goto error; + + if (buffering) { + raw = PyObject_GetAttrString(buf, "raw"); + if (raw == NULL) + goto error; + } + else { + raw = buf; + Py_INCREF(raw); + } + + text = PyUnicode_FromString(name); + if (text == NULL || PyObject_SetAttrString(raw, "name", text) < 0) + goto error; + res = PyObject_CallMethod(raw, "isatty", ""); + if (res == NULL) + goto error; + isatty = PyObject_IsTrue(res); + Py_DECREF(res); + if (isatty == -1) + goto error; + if (isatty || Py_UnbufferedStdioFlag) + line_buffering = Py_True; + else + line_buffering = Py_False; + + Py_CLEAR(raw); + Py_CLEAR(text); + +#ifdef MS_WINDOWS + /* sys.stdin: enable universal newline mode, translate "\r\n" and "\r" + newlines to "\n". + sys.stdout and sys.stderr: translate "\n" to "\r\n". */ + newline = NULL; +#else + /* sys.stdin: split lines at "\n". + sys.stdout and sys.stderr: don't translate newlines (use "\n"). */ + newline = "\n"; +#endif + + stream = PyObject_CallMethod(io, "TextIOWrapper", "OsssO", + buf, encoding, errors, + newline, line_buffering); + Py_CLEAR(buf); + if (stream == NULL) + goto error; + + if (write_mode) + mode = "w"; + else + mode = "r"; + text = PyUnicode_FromString(mode); + if (!text || PyObject_SetAttrString(stream, "mode", text) < 0) + goto error; + Py_CLEAR(text); + return stream; + +error: + Py_XDECREF(buf); + Py_XDECREF(stream); + Py_XDECREF(text); + Py_XDECREF(raw); + return NULL; +} + +static int +is_valid_fd(int fd) +{ + int dummy_fd; + if (fd < 0 || !_PyVerify_fd(fd)) + return 0; + dummy_fd = dup(fd); + if (dummy_fd < 0) + return 0; + close(dummy_fd); + return 1; +} + +/* Initialize sys.stdin, stdout, stderr and builtins.open */ +static int +initstdio(void) +{ + PyObject *iomod = NULL, *wrapper; + PyObject *bimod = NULL; + PyObject *m; + PyObject *std = NULL; + int status = 0, fd; + PyObject * encoding_attr; + char *encoding = NULL, *errors; + + /* Hack to avoid a nasty recursion issue when Python is invoked + in verbose mode: pre-import the Latin-1 and UTF-8 codecs */ + if ((m = PyImport_ImportModule("encodings.utf_8")) == NULL) { + goto error; + } + Py_DECREF(m); + + if (!(m = PyImport_ImportModule("encodings.latin_1"))) { + goto error; + } + Py_DECREF(m); + + if (!(bimod = PyImport_ImportModule("builtins"))) { + goto error; + } + + if (!(iomod = PyImport_ImportModule("io"))) { + goto error; + } + if (!(wrapper = PyObject_GetAttrString(iomod, "OpenWrapper"))) { + goto error; + } + + /* Set builtins.open */ + if (PyObject_SetAttrString(bimod, "open", wrapper) == -1) { + Py_DECREF(wrapper); + goto error; + } + Py_DECREF(wrapper); + + encoding = Py_GETENV("PYTHONIOENCODING"); + errors = NULL; + if (encoding) { + encoding = strdup(encoding); + errors = strchr(encoding, ':'); + if (errors) { + *errors = '\0'; + errors++; + } + } + + /* Set sys.stdin */ + fd = fileno(stdin); + /* Under some conditions stdin, stdout and stderr may not be connected + * and fileno() may point to an invalid file descriptor. For example + * GUI apps don't have valid standard streams by default. + */ + if (!is_valid_fd(fd)) { + std = Py_None; + Py_INCREF(std); + } + else { + std = create_stdio(iomod, fd, 0, "<stdin>", encoding, errors); + if (std == NULL) + goto error; + } /* if (fd < 0) */ + PySys_SetObject("__stdin__", std); + PySys_SetObject("stdin", std); + Py_DECREF(std); + + /* Set sys.stdout */ + fd = fileno(stdout); + if (!is_valid_fd(fd)) { + std = Py_None; + Py_INCREF(std); + } + else { + std = create_stdio(iomod, fd, 1, "<stdout>", encoding, errors); + if (std == NULL) + goto error; + } /* if (fd < 0) */ + PySys_SetObject("__stdout__", std); + PySys_SetObject("stdout", std); + Py_DECREF(std); + +#if 1 /* Disable this if you have trouble debugging bootstrap stuff */ + /* Set sys.stderr, replaces the preliminary stderr */ + fd = fileno(stderr); + if (!is_valid_fd(fd)) { + std = Py_None; + Py_INCREF(std); + } + else { + std = create_stdio(iomod, fd, 1, "<stderr>", encoding, "backslashreplace"); + if (std == NULL) + goto error; + } /* if (fd < 0) */ + + /* Same as hack above, pre-import stderr's codec to avoid recursion + when import.c tries to write to stderr in verbose mode. */ + encoding_attr = PyObject_GetAttrString(std, "encoding"); + if (encoding_attr != NULL) { + const char * encoding; + encoding = _PyUnicode_AsString(encoding_attr); + if (encoding != NULL) { + PyObject *codec_info = _PyCodec_Lookup(encoding); + Py_XDECREF(codec_info); + } + Py_DECREF(encoding_attr); + } + PyErr_Clear(); /* Not a fatal error if codec isn't available */ + + PySys_SetObject("__stderr__", std); + PySys_SetObject("stderr", std); + Py_DECREF(std); +#endif + + if (0) { + error: + status = -1; + } + + if (encoding) + free(encoding); + Py_XDECREF(bimod); + Py_XDECREF(iomod); + return status; +} + /* Parse input from a file and execute it */ int @@ -760,12 +1081,12 @@ PyRun_InteractiveLoopFlags(FILE *fp, const char *filename, PyCompilerFlags *flag } v = PySys_GetObject("ps1"); if (v == NULL) { - PySys_SetObject("ps1", v = PyString_FromString(">>> ")); + PySys_SetObject("ps1", v = PyUnicode_FromString(">>> ")); Py_XDECREF(v); } v = PySys_GetObject("ps2"); if (v == NULL) { - PySys_SetObject("ps2", v = PyString_FromString("... ")); + PySys_SetObject("ps2", v = PyUnicode_FromString("... ")); Py_XDECREF(v); } for (;;) { @@ -780,60 +1101,90 @@ PyRun_InteractiveLoopFlags(FILE *fp, const char *filename, PyCompilerFlags *flag } } -#if 0 /* compute parser flags based on compiler flags */ -#define PARSER_FLAGS(flags) \ - ((flags) ? ((((flags)->cf_flags & PyCF_DONT_IMPLY_DEDENT) ? \ - PyPARSE_DONT_IMPLY_DEDENT : 0)) : 0) -#endif -#if 1 +static int PARSER_FLAGS(PyCompilerFlags *flags) +{ + int parser_flags = 0; + if (!flags) + return 0; + if (flags->cf_flags & PyCF_DONT_IMPLY_DEDENT) + parser_flags |= PyPARSE_DONT_IMPLY_DEDENT; + if (flags->cf_flags & PyCF_IGNORE_COOKIE) + parser_flags |= PyPARSE_IGNORE_COOKIE; + if (flags->cf_flags & CO_FUTURE_BARRY_AS_BDFL) + parser_flags |= PyPARSE_BARRY_AS_BDFL; + return parser_flags; +} + +#if 0 /* Keep an example of flags with future keyword support. */ #define PARSER_FLAGS(flags) \ ((flags) ? ((((flags)->cf_flags & PyCF_DONT_IMPLY_DEDENT) ? \ PyPARSE_DONT_IMPLY_DEDENT : 0) \ - | (((flags)->cf_flags & CO_FUTURE_PRINT_FUNCTION) ? \ - PyPARSE_PRINT_IS_FUNCTION : 0) \ - | (((flags)->cf_flags & CO_FUTURE_UNICODE_LITERALS) ? \ - PyPARSE_UNICODE_LITERALS : 0) \ - ) : 0) + | ((flags)->cf_flags & CO_FUTURE_WITH_STATEMENT ? \ + PyPARSE_WITH_IS_KEYWORD : 0)) : 0) #endif int PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags) { - PyObject *m, *d, *v, *w; + PyObject *m, *d, *v, *w, *oenc = NULL; mod_ty mod; PyArena *arena; - char *ps1 = "", *ps2 = ""; + char *ps1 = "", *ps2 = "", *enc = NULL; int errcode = 0; + if (fp == stdin) { + /* Fetch encoding from sys.stdin */ + v = PySys_GetObject("stdin"); + if (v == NULL || v == Py_None) + return -1; + oenc = PyObject_GetAttrString(v, "encoding"); + if (!oenc) + return -1; + enc = _PyUnicode_AsString(oenc); + if (enc == NULL) + return -1; + } v = PySys_GetObject("ps1"); if (v != NULL) { v = PyObject_Str(v); if (v == NULL) PyErr_Clear(); - else if (PyString_Check(v)) - ps1 = PyString_AsString(v); + else if (PyUnicode_Check(v)) { + ps1 = _PyUnicode_AsString(v); + if (ps1 == NULL) { + PyErr_Clear(); + ps1 = ""; + } + } } w = PySys_GetObject("ps2"); if (w != NULL) { w = PyObject_Str(w); if (w == NULL) PyErr_Clear(); - else if (PyString_Check(w)) - ps2 = PyString_AsString(w); + else if (PyUnicode_Check(w)) { + ps2 = _PyUnicode_AsString(w); + if (ps2 == NULL) { + PyErr_Clear(); + ps2 = ""; + } + } } arena = PyArena_New(); if (arena == NULL) { Py_XDECREF(v); Py_XDECREF(w); + Py_XDECREF(oenc); return -1; } - mod = PyParser_ASTFromFile(fp, filename, + mod = PyParser_ASTFromFile(fp, filename, enc, Py_single_input, ps1, ps2, flags, &errcode, arena); Py_XDECREF(v); Py_XDECREF(w); + Py_XDECREF(oenc); if (mod == NULL) { PyArena_Free(arena); if (errcode == E_EOF) { @@ -851,13 +1202,12 @@ PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags d = PyModule_GetDict(m); v = run_mod(mod, filename, d, d, flags, arena); PyArena_Free(arena); + flush_io(); if (v == NULL) { PyErr_Print(); return -1; } Py_DECREF(v); - if (Py_FlushLine()) - PyErr_Clear(); return 0; } @@ -907,7 +1257,8 @@ PyRun_SimpleFileExFlags(FILE *fp, const char *filename, int closeit, { PyObject *m, *d, *v; const char *ext; - int set_file_name = 0, len, ret = -1; + int set_file_name = 0, ret = -1; + size_t len; m = PyImport_AddModule("__main__"); if (m == NULL) @@ -915,13 +1266,18 @@ PyRun_SimpleFileExFlags(FILE *fp, const char *filename, int closeit, Py_INCREF(m); d = PyModule_GetDict(m); if (PyDict_GetItemString(d, "__file__") == NULL) { - PyObject *f = PyString_FromString(filename); + PyObject *f; + f = PyUnicode_DecodeFSDefault(filename); if (f == NULL) goto done; if (PyDict_SetItemString(d, "__file__", f) < 0) { Py_DECREF(f); goto done; } + if (PyDict_SetItemString(d, "__cached__", Py_None) < 0) { + Py_DECREF(f); + goto done; + } set_file_name = 1; Py_DECREF(f); } @@ -943,13 +1299,12 @@ PyRun_SimpleFileExFlags(FILE *fp, const char *filename, int closeit, v = PyRun_FileExFlags(fp, filename, Py_file_input, d, d, closeit, flags); } + flush_io(); if (v == NULL) { PyErr_Print(); goto done; } Py_DECREF(v); - if (Py_FlushLine()) - PyErr_Clear(); ret = 0; done: if (set_file_name && PyDict_DelItemString(d, "__file__")) @@ -972,8 +1327,6 @@ PyRun_SimpleStringFlags(const char *command, PyCompilerFlags *flags) return -1; } Py_DECREF(v); - if (Py_FlushLine()) - PyErr_Clear(); return 0; } @@ -1004,7 +1357,7 @@ parse_syntax_error(PyObject *err, PyObject **message, const char **filename, *filename = NULL; } else { - *filename = PyString_AsString(v); + *filename = _PyUnicode_AsString(v); Py_DECREF(v); if (!*filename) goto finally; @@ -1013,7 +1366,7 @@ parse_syntax_error(PyObject *err, PyObject **message, const char **filename, v = PyObject_GetAttrString(err, "lineno"); if (!v) goto finally; - hold = PyInt_AsLong(v); + hold = PyLong_AsLong(v); Py_DECREF(v); if (hold < 0 && PyErr_Occurred()) goto finally; @@ -1026,7 +1379,7 @@ parse_syntax_error(PyObject *err, PyObject **message, const char **filename, *offset = -1; Py_DECREF(v); } else { - hold = PyInt_AsLong(v); + hold = PyLong_AsLong(v); Py_DECREF(v); if (hold < 0 && PyErr_Occurred()) goto finally; @@ -1041,7 +1394,7 @@ parse_syntax_error(PyObject *err, PyObject **message, const char **filename, *text = NULL; } else { - *text = PyString_AsString(v); + *text = _PyUnicode_AsString(v); Py_DECREF(v); if (!*text) goto finally; @@ -1085,11 +1438,8 @@ print_error_text(PyObject *f, int offset, const char *text) if (offset == -1) return; PyFile_WriteString(" ", f); - offset--; - while (offset > 0) { + while (--offset > 0) PyFile_WriteString(" ", f); - offset--; - } PyFile_WriteString("^\n", f); } @@ -1105,8 +1455,6 @@ handle_system_exit(void) return; PyErr_Fetch(&exception, &value, &tb); - if (Py_FlushLine()) - PyErr_Clear(); fflush(stdout); if (value == NULL || value == Py_None) goto done; @@ -1122,8 +1470,8 @@ handle_system_exit(void) /* If we failed to dig out the 'code' attribute, just let the else clause below print the error. */ } - if (PyInt_Check(value)) - exitcode = (int)PyInt_AsLong(value); + if (PyLong_Check(value)) + exitcode = (int)PyLong_AsLong(value); else { PyObject *sys_stderr = PySys_GetObject("stderr"); if (sys_stderr != NULL && sys_stderr != Py_None) { @@ -1159,6 +1507,11 @@ PyErr_PrintEx(int set_sys_last_vars) if (exception == NULL) return; PyErr_NormalizeException(&exception, &v, &tb); + if (tb == NULL) { + tb = Py_None; + Py_INCREF(tb); + } + PyException_SetTraceback(v, tb); if (exception == NULL) return; /* Now we know v != NULL too */ @@ -1168,9 +1521,8 @@ PyErr_PrintEx(int set_sys_last_vars) PySys_SetObject("last_traceback", tb); } hook = PySys_GetObject("excepthook"); - if (hook && hook != Py_None) { - PyObject *args = PyTuple_Pack(3, - exception, v, tb ? tb : Py_None); + if (hook) { + PyObject *args = PyTuple_Pack(3, exception, v, tb); PyObject *result = PyEval_CallObject(hook, args); if (result == NULL) { PyObject *exception2, *v2, *tb2; @@ -1190,8 +1542,6 @@ PyErr_PrintEx(int set_sys_last_vars) v2 = Py_None; Py_INCREF(v2); } - if (Py_FlushLine()) - PyErr_Clear(); fflush(stdout); PySys_WriteStderr("Error in sys.excepthook:\n"); PyErr_Display(exception2, v2, tb2); @@ -1212,100 +1562,108 @@ PyErr_PrintEx(int set_sys_last_vars) Py_XDECREF(tb); } -void -PyErr_Display(PyObject *exception, PyObject *value, PyObject *tb) +static void +print_exception(PyObject *f, PyObject *value) { int err = 0; - PyObject *f = PySys_GetObject("stderr"); + PyObject *type, *tb; + + if (!PyExceptionInstance_Check(value)) { + PyFile_WriteString("TypeError: print_exception(): Exception expected for value, ", f); + PyFile_WriteString(Py_TYPE(value)->tp_name, f); + PyFile_WriteString(" found\n", f); + return; + } + Py_INCREF(value); - if (f == NULL || f == Py_None) - fprintf(stderr, "lost sys.stderr\n"); - else { - if (Py_FlushLine()) + fflush(stdout); + type = (PyObject *) Py_TYPE(value); + tb = PyException_GetTraceback(value); + if (tb && tb != Py_None) + err = PyTraceBack_Print(tb, f); + if (err == 0 && + PyObject_HasAttrString(value, "print_file_and_line")) + { + PyObject *message; + const char *filename, *text; + int lineno, offset; + if (!parse_syntax_error(value, &message, &filename, + &lineno, &offset, &text)) PyErr_Clear(); - fflush(stdout); - if (tb && tb != Py_None) - err = PyTraceBack_Print(tb, f); - if (err == 0 && - PyObject_HasAttrString(value, "print_file_and_line")) - { - PyObject *message; - const char *filename, *text; - int lineno, offset; - if (!parse_syntax_error(value, &message, &filename, - &lineno, &offset, &text)) - PyErr_Clear(); - else { - char buf[10]; - PyFile_WriteString(" File \"", f); - if (filename == NULL) - PyFile_WriteString("<string>", f); - else - PyFile_WriteString(filename, f); - PyFile_WriteString("\", line ", f); - PyOS_snprintf(buf, sizeof(buf), "%d", lineno); - PyFile_WriteString(buf, f); - PyFile_WriteString("\n", f); - if (text != NULL) - print_error_text(f, offset, text); - Py_DECREF(value); - value = message; - /* Can't be bothered to check all those - PyFile_WriteString() calls */ - if (PyErr_Occurred()) - err = -1; - } + else { + char buf[10]; + PyFile_WriteString(" File \"", f); + if (filename == NULL) + PyFile_WriteString("<string>", f); + else + PyFile_WriteString(filename, f); + PyFile_WriteString("\", line ", f); + PyOS_snprintf(buf, sizeof(buf), "%d", lineno); + PyFile_WriteString(buf, f); + PyFile_WriteString("\n", f); + if (text != NULL) + print_error_text(f, offset, text); + Py_DECREF(value); + value = message; + /* Can't be bothered to check all those + PyFile_WriteString() calls */ + if (PyErr_Occurred()) + err = -1; } - if (err) { - /* Don't do anything else */ + } + if (err) { + /* Don't do anything else */ + } + else { + PyObject* moduleName; + char* className; + assert(PyExceptionClass_Check(type)); + className = PyExceptionClass_Name(type); + if (className != NULL) { + char *dot = strrchr(className, '.'); + if (dot != NULL) + className = dot+1; } - else if (PyExceptionClass_Check(exception)) { - PyObject* moduleName; - char* className = PyExceptionClass_Name(exception); - if (className != NULL) { - char *dot = strrchr(className, '.'); - if (dot != NULL) - className = dot+1; - } - moduleName = PyObject_GetAttrString(exception, "__module__"); - if (moduleName == NULL) - err = PyFile_WriteString("<unknown>", f); - else { - char* modstr = PyString_AsString(moduleName); - if (modstr && strcmp(modstr, "exceptions")) - { - err = PyFile_WriteString(modstr, f); - err += PyFile_WriteString(".", f); - } - Py_DECREF(moduleName); + moduleName = PyObject_GetAttrString(type, "__module__"); + if (moduleName == NULL || !PyUnicode_Check(moduleName)) + { + Py_DECREF(moduleName); + err = PyFile_WriteString("<unknown>", f); + } + else { + char* modstr = _PyUnicode_AsString(moduleName); + if (modstr && strcmp(modstr, "builtins")) + { + err = PyFile_WriteString(modstr, f); + err += PyFile_WriteString(".", f); } - if (err == 0) { - if (className == NULL) + Py_DECREF(moduleName); + } + if (err == 0) { + if (className == NULL) err = PyFile_WriteString("<unknown>", f); - else + else err = PyFile_WriteString(className, f); - } - } - else - err = PyFile_WriteObject(exception, f, Py_PRINT_RAW); - if (err == 0 && (value != Py_None)) { - PyObject *s = PyObject_Str(value); - /* only print colon if the str() of the - object is not the empty string - */ - if (s == NULL) - err = -1; - else if (!PyString_Check(s) || - PyString_GET_SIZE(s) != 0) - err = PyFile_WriteString(": ", f); - if (err == 0) - err = PyFile_WriteObject(s, f, Py_PRINT_RAW); - Py_XDECREF(s); } - /* try to write a newline in any case */ - err += PyFile_WriteString("\n", f); } + if (err == 0 && (value != Py_None)) { + PyObject *s = PyObject_Str(value); + /* only print colon if the str() of the + object is not the empty string + */ + if (s == NULL) + err = -1; + else if (!PyUnicode_Check(s) || + PyUnicode_GetSize(s) != 0) + err = PyFile_WriteString(": ", f); + if (err == 0) + err = PyFile_WriteObject(s, f, Py_PRINT_RAW); + Py_XDECREF(s); + } + /* try to write a newline in any case */ + err += PyFile_WriteString("\n", f); + Py_XDECREF(tb); Py_DECREF(value); /* If an error happened here, don't show it. XXX This is wrong, but too many callers rely on this behavior. */ @@ -1313,6 +1671,82 @@ PyErr_Display(PyObject *exception, PyObject *value, PyObject *tb) PyErr_Clear(); } +static const char *cause_message = + "\nThe above exception was the direct cause " + "of the following exception:\n\n"; + +static const char *context_message = + "\nDuring handling of the above exception, " + "another exception occurred:\n\n"; + +static void +print_exception_recursive(PyObject *f, PyObject *value, PyObject *seen) +{ + int err = 0, res; + PyObject *cause, *context; + + if (seen != NULL) { + /* Exception chaining */ + if (PySet_Add(seen, value) == -1) + PyErr_Clear(); + else if (PyExceptionInstance_Check(value)) { + cause = PyException_GetCause(value); + context = PyException_GetContext(value); + if (cause) { + res = PySet_Contains(seen, cause); + if (res == -1) + PyErr_Clear(); + if (res == 0) { + print_exception_recursive( + f, cause, seen); + err |= PyFile_WriteString( + cause_message, f); + } + } + else if (context) { + res = PySet_Contains(seen, context); + if (res == -1) + PyErr_Clear(); + if (res == 0) { + print_exception_recursive( + f, context, seen); + err |= PyFile_WriteString( + context_message, f); + } + } + Py_XDECREF(context); + Py_XDECREF(cause); + } + } + print_exception(f, value); + if (err != 0) + PyErr_Clear(); +} + +void +PyErr_Display(PyObject *exception, PyObject *value, PyObject *tb) +{ + PyObject *seen; + PyObject *f = PySys_GetObject("stderr"); + if (f == Py_None) { + /* pass */ + } + else if (f == NULL) { + _PyObject_Dump(value); + fprintf(stderr, "lost sys.stderr\n"); + } + else { + /* We choose to ignore seen being possibly NULL, and report + at least the main exception (it could be a MemoryError). + */ + seen = PySet_New(NULL); + if (seen == NULL) + PyErr_Clear(); + print_exception_recursive(f, value, seen); + Py_XDECREF(seen); + } +} + PyObject * PyRun_StringFlags(const char *str, int start, PyObject *globals, PyObject *locals, PyCompilerFlags *flags) @@ -1340,7 +1774,7 @@ PyRun_FileExFlags(FILE *fp, const char *filename, int start, PyObject *globals, if (arena == NULL) return NULL; - mod = PyParser_ASTFromFile(fp, filename, start, 0, 0, + mod = PyParser_ASTFromFile(fp, filename, NULL, start, 0, 0, flags, NULL, arena); if (closeit) fclose(fp); @@ -1353,6 +1787,35 @@ PyRun_FileExFlags(FILE *fp, const char *filename, int start, PyObject *globals, return ret; } +static void +flush_io(void) +{ + PyObject *f, *r; + PyObject *type, *value, *traceback; + + /* Save the current exception */ + PyErr_Fetch(&type, &value, &traceback); + + f = PySys_GetObject("stderr"); + if (f != NULL) { + r = PyObject_CallMethod(f, "flush", ""); + if (r) + Py_DECREF(r); + else + PyErr_Clear(); + } + f = PySys_GetObject("stdout"); + if (f != NULL) { + r = PyObject_CallMethod(f, "flush", ""); + if (r) + Py_DECREF(r); + else + PyErr_Clear(); + } + + PyErr_Restore(type, value, traceback); +} + static PyObject * run_mod(mod_ty mod, const char *filename, PyObject *globals, PyObject *locals, PyCompilerFlags *flags, PyArena *arena) @@ -1362,7 +1825,7 @@ run_mod(mod_ty mod, const char *filename, PyObject *globals, PyObject *locals, co = PyAST_Compile(mod, filename, flags, arena); if (co == NULL) return NULL; - v = PyEval_EvalCode(co, globals, locals); + v = PyEval_EvalCode((PyObject*)co, globals, locals); Py_DECREF(co); return v; } @@ -1392,7 +1855,7 @@ run_pyc_file(FILE *fp, const char *filename, PyObject *globals, return NULL; } co = (PyCodeObject *)v; - v = PyEval_EvalCode(co, globals, locals); + v = PyEval_EvalCode((PyObject*)co, globals, locals); if (v && flags) flags->cf_flags |= (co->co_flags & PyCF_MASK); Py_DECREF(co); @@ -1400,8 +1863,8 @@ run_pyc_file(FILE *fp, const char *filename, PyObject *globals, } PyObject * -Py_CompileStringFlags(const char *str, const char *filename, int start, - PyCompilerFlags *flags) +Py_CompileStringExFlags(const char *str, const char *filename, int start, + PyCompilerFlags *flags, int optimize) { PyCodeObject *co; mod_ty mod; @@ -1419,11 +1882,19 @@ Py_CompileStringFlags(const char *str, const char *filename, int start, PyArena_Free(arena); return result; } - co = PyAST_Compile(mod, filename, flags, arena); + co = PyAST_CompileEx(mod, filename, flags, optimize, arena); PyArena_Free(arena); return (PyObject *)co; } +/* For use in Py_LIMITED_API */ +#undef Py_CompileString +PyObject * +PyCompileString(const char *str, const char *filename, int start) +{ + return Py_CompileStringFlags(str, filename, start, NULL); +} + struct symtable * Py_SymtableString(const char *str, const char *filename, int start) { @@ -1435,7 +1906,6 @@ Py_SymtableString(const char *str, const char *filename, int start) return NULL; flags.cf_flags = 0; - mod = PyParser_ASTFromString(str, filename, start, &flags, arena); if (mod == NULL) { PyArena_Free(arena); @@ -1476,7 +1946,8 @@ PyParser_ASTFromString(const char *s, const char *filename, int start, } mod_ty -PyParser_ASTFromFile(FILE *fp, const char *filename, int start, char *ps1, +PyParser_ASTFromFile(FILE *fp, const char *filename, const char* enc, + int start, char *ps1, char *ps2, PyCompilerFlags *flags, int *errcode, PyArena *arena) { @@ -1485,7 +1956,8 @@ PyParser_ASTFromFile(FILE *fp, const char *filename, int start, char *ps1, perrdetail err; int iflags = PARSER_FLAGS(flags); - node *n = PyParser_ParseFileFlagsEx(fp, filename, &_PyParser_Grammar, + node *n = PyParser_ParseFileFlagsEx(fp, filename, enc, + &_PyParser_Grammar, start, ps1, ps2, &err, &iflags); if (flags == NULL) { localflags.cf_flags = 0; @@ -1511,7 +1983,8 @@ node * PyParser_SimpleParseFileFlags(FILE *fp, const char *filename, int start, int flags) { perrdetail err; - node *n = PyParser_ParseFileFlags(fp, filename, &_PyParser_Grammar, + node *n = PyParser_ParseFileFlags(fp, filename, NULL, + &_PyParser_Grammar, start, NULL, NULL, &err, flags); if (n == NULL) err_input(&err); @@ -1564,9 +2037,11 @@ PyParser_SetError(perrdetail *err) static void err_input(perrdetail *err) { - PyObject *v, *w, *errtype; - PyObject* u = NULL; + PyObject *v, *w, *errtype, *errtext; + PyObject *msg_obj = NULL; + PyObject *filename; char *msg = NULL; + errtype = PyExc_SyntaxError; switch (err->error) { case E_ERROR: @@ -1621,14 +2096,9 @@ err_input(perrdetail *err) case E_DECODE: { PyObject *type, *value, *tb; PyErr_Fetch(&type, &value, &tb); - if (value != NULL) { - u = PyObject_Str(value); - if (u != NULL) { - msg = PyString_AsString(u); - } - } - if (msg == NULL) - msg = "unknown decode error"; + msg = "unknown decode error"; + if (value != NULL) + msg_obj = PyObject_Str(value); Py_XDECREF(type); Py_XDECREF(value); Py_XDECREF(tb); @@ -1637,21 +2107,47 @@ err_input(perrdetail *err) case E_LINECONT: msg = "unexpected character after line continuation character"; break; + + case E_IDENTIFIER: + msg = "invalid character in identifier"; + break; default: fprintf(stderr, "error=%d\n", err->error); msg = "unknown parsing error"; break; } - v = Py_BuildValue("(ziiz)", err->filename, - err->lineno, err->offset, err->text); - w = NULL; - if (v != NULL) - w = Py_BuildValue("(sO)", msg, v); - Py_XDECREF(u); + /* err->text may not be UTF-8 in case of decoding errors. + Explicitly convert to an object. */ + if (!err->text) { + errtext = Py_None; + Py_INCREF(Py_None); + } else { + errtext = PyUnicode_DecodeUTF8(err->text, strlen(err->text), + "replace"); + } + if (err->filename != NULL) + filename = PyUnicode_DecodeFSDefault(err->filename); + else { + Py_INCREF(Py_None); + filename = Py_None; + } + if (filename != NULL) + v = Py_BuildValue("(NiiN)", filename, + err->lineno, err->offset, errtext); + else + v = NULL; + if (v != NULL) { + if (msg_obj) + w = Py_BuildValue("(OO)", msg_obj, v); + else + w = Py_BuildValue("(sO)", msg, v); + } else + w = NULL; Py_XDECREF(v); PyErr_SetObject(errtype, w); Py_XDECREF(w); cleanup: + Py_XDECREF(msg_obj); if (err->text != NULL) { PyObject_FREE(err->text); err->text = NULL; @@ -1665,7 +2161,9 @@ Py_FatalError(const char *msg) { fprintf(stderr, "Fatal Python error: %s\n", msg); fflush(stderr); /* it helps in Windows debug build */ - + if (PyErr_Occurred()) { + PyErr_PrintEx(0); + } #ifdef MS_WINDOWS { size_t len = strlen(msg); @@ -1695,6 +2193,23 @@ Py_FatalError(const char *msg) #include "pythread.h" #endif +static void (*pyexitfunc)(void) = NULL; +/* For the atexit module. */ +void _Py_PyAtExit(void (*func)(void)) +{ + pyexitfunc = func; +} + +static void +call_py_exitfuncs(void) +{ + if (pyexitfunc == NULL) + return; + + (*pyexitfunc)(); + PyErr_Clear(); +} + /* Wait until threading._shutdown completes, provided the threading module was imported in the first place. The shutdown routine will wait until all non-daemon @@ -1713,10 +2228,12 @@ wait_for_thread_shutdown(void) return; } result = PyObject_CallMethod(threading, "_shutdown", ""); - if (result == NULL) + if (result == NULL) { PyErr_WriteUnraisable(threading); - else + } + else { Py_DECREF(result); + } Py_DECREF(threading); #endif } @@ -1734,29 +2251,6 @@ int Py_AtExit(void (*func)(void)) } static void -call_sys_exitfunc(void) -{ - PyObject *exitfunc = PySys_GetObject("exitfunc"); - - if (exitfunc) { - PyObject *res; - Py_INCREF(exitfunc); - PySys_SetObject("exitfunc", (PyObject *)NULL); - res = PyEval_CallObject(exitfunc, (PyObject *)NULL); - if (res == NULL) { - if (!PyErr_ExceptionMatches(PyExc_SystemExit)) { - PySys_WriteStderr("Error in sys.exitfunc:\n"); - } - PyErr_Print(); - } - Py_DECREF(exitfunc); - } - - if (Py_FlushLine()) - PyErr_Clear(); -} - -static void call_ll_exitfuncs(void) { while (nexitfuncs > 0) @@ -1790,6 +2284,27 @@ initsigs(void) } +/* Restore signals that the interpreter has called SIG_IGN on to SIG_DFL. + * + * All of the code in this function must only use async-signal-safe functions, + * listed at `man 7 signal` or + * http://www.opengroup.org/onlinepubs/009695399/functions/xsh_chap02_04.html. + */ +void +_Py_RestoreSignals(void) +{ +#ifdef SIGPIPE + PyOS_setsig(SIGPIPE, SIG_DFL); +#endif +#ifdef SIGXFZ + PyOS_setsig(SIGXFZ, SIG_DFL); +#endif +#ifdef SIGXFSZ + PyOS_setsig(SIGXFSZ, SIG_DFL); +#endif +} + + /* * The file descriptor fd is considered ``interactive'' if either * a) isatty(fd) is TRUE, or @@ -1883,6 +2398,11 @@ PyOS_getsig(int sig) #endif } +/* + * All of the code in this function must only use async-signal-safe functions, + * listed at `man 7 signal` or + * http://www.opengroup.org/onlinepubs/009695399/functions/xsh_chap02_04.html. + */ PyOS_sighandler_t PyOS_setsig(int sig, PyOS_sighandler_t handler) { @@ -2000,7 +2520,15 @@ PyRun_SimpleString(const char *s) PyAPI_FUNC(PyObject *) Py_CompileString(const char *str, const char *p, int s) { - return Py_CompileStringFlags(str, p, s, NULL); + return Py_CompileStringExFlags(str, p, s, NULL, -1); +} + +#undef Py_CompileStringFlags +PyAPI_FUNC(PyObject *) +Py_CompileStringFlags(const char *str, const char *p, int s, + PyCompilerFlags *flags) +{ + return Py_CompileStringExFlags(str, p, s, flags, -1); } #undef PyRun_InteractiveOne @@ -2020,4 +2548,3 @@ PyRun_InteractiveLoop(FILE *f, const char *p) #ifdef __cplusplus } #endif - diff --git a/Python/pytime.c b/Python/pytime.c new file mode 100644 index 0000000000..6fb7695911 --- /dev/null +++ b/Python/pytime.c @@ -0,0 +1,60 @@ +#include "Python.h" + +#ifdef __APPLE__ +#if defined(HAVE_GETTIMEOFDAY) && defined(HAVE_FTIME) + /* + * _PyTime_gettimeofday falls back to ftime when getttimeofday fails because the latter + * might fail on some platforms. This fallback is unwanted on MacOSX because + * that makes it impossible to use a binary build on OSX 10.4 on earlier + * releases of the OS. Therefore claim we don't support ftime. + */ +# undef HAVE_FTIME +#endif +#endif + +#ifdef HAVE_FTIME +#include <sys/timeb.h> +#if !defined(MS_WINDOWS) && !defined(PYOS_OS2) +extern int ftime(struct timeb *); +#endif /* MS_WINDOWS */ +#endif /* HAVE_FTIME */ + +void +_PyTime_gettimeofday(_PyTime_timeval *tp) +{ + /* There are three ways to get the time: + (1) gettimeofday() -- resolution in microseconds + (2) ftime() -- resolution in milliseconds + (3) time() -- resolution in seconds + In all cases the return value in a timeval struct. + Since on some systems (e.g. SCO ODT 3.0) gettimeofday() may + fail, so we fall back on ftime() or time(). + Note: clock resolution does not imply clock accuracy! */ +#ifdef HAVE_GETTIMEOFDAY +#ifdef GETTIMEOFDAY_NO_TZ + if (gettimeofday(tp) == 0) + return; +#else /* !GETTIMEOFDAY_NO_TZ */ + if (gettimeofday(tp, (struct timezone *)NULL) == 0) + return; +#endif /* !GETTIMEOFDAY_NO_TZ */ +#endif /* !HAVE_GETTIMEOFDAY */ +#if defined(HAVE_FTIME) + { + struct timeb t; + ftime(&t); + tp->tv_sec = t.time; + tp->tv_usec = t.millitm * 1000; + } +#else /* !HAVE_FTIME */ + tp->tv_sec = time(NULL); + tp->tv_usec = 0; +#endif /* !HAVE_FTIME */ + return; +} + +void +_PyTime_Init() +{ + /* Do nothing. Needed to force linking. */ +} diff --git a/Python/strtod.c b/Python/strtod.c deleted file mode 100644 index ee558982d5..0000000000 --- a/Python/strtod.c +++ /dev/null @@ -1,159 +0,0 @@ -#include <stdio.h> -#include <string.h> - -#include "pyconfig.h" - -/* comp.sources.misc strtod(), as posted in comp.lang.tcl, - with bugfix for "123000.0" and acceptance of space after 'e' sign nuked. - - ************************************************************ - * YOU MUST EDIT THE MACHINE-DEPENDENT DEFINITIONS BELOW!!! * - ************************************************************ -*/ - -/* File : stdtod.c (Modified version of str2dbl.c) - Author : Richard A. O'Keefe @ Quintus Computer Systems, Inc. - Updated: Tuesday August 2nd, 1988 - Defines: double strtod (char *str, char**ptr) -*/ - -/* This is an implementation of the strtod() function described in the - System V manuals, with a different name to avoid linker problems. - All that str2dbl() does itself is check that the argument is well-formed - and is in range. It leaves the work of conversion to atof(), which is - assumed to exist and deliver correct results (if they can be represented). - - There are two reasons why this should be provided to the net: - (a) some UNIX systems do not yet have strtod(), or do not have it - available in the BSD "universe" (but they do have atof()). - (b) some of the UNIX systems that *do* have it get it wrong. - (some crash with large arguments, some assign the wrong *ptr value). - There is a reason why *we* are providing it: we need a correct version - of strtod(), and if we give this one away maybe someone will look for - mistakes in it and fix them for us (:-). -*/ - -/* The following constants are machine-specific. MD{MIN,MAX}EXPT are - integers and MD{MIN,MAX}FRAC are strings such that - 0.${MDMAXFRAC}e${MDMAXEXPT} is the largest representable double, - 0.${MDMINFRAC}e${MDMINEXPT} is the smallest representable +ve double - MD{MIN,MAX}FRAC must not have any trailing zeros. - The values here are for IEEE-754 64-bit floats. - It is not perfectly clear to me whether an IEEE infinity should be - returned for overflow, nor what a portable way of writing one is, - so HUGE is just 0.MAXFRAC*10**MAXEXPT (this seems still to be the - UNIX convention). - - I do know about <values.h>, but the whole point of this file is that - we can't always trust that stuff to be there or to be correct. -*/ -static int MDMINEXPT = -323; -static char MDMINFRAC[] = "494065645841246544"; -static double ZERO = 0.0; - -static int MDMAXEXPT = 309; -static char MDMAXFRAC[] = "17976931348623157"; -static double HUGE = 1.7976931348623157e308; - -extern double atof(const char *); /* Only called when result known to be ok */ - -#ifdef HAVE_ERRNO_H -#include <errno.h> -#endif -extern int errno; - -double strtod(char *str, char **ptr) -{ - int sign, scale, dotseen; - int esign, expt; - char *save; - register char *sp, *dp; - register int c; - char *buforg, *buflim; - char buffer[64]; /* 45-digit significant + */ - /* 13-digit exponent */ - sp = str; - while (*sp == ' ') sp++; - sign = 1; - if (*sp == '-') sign -= 2, sp++; - dotseen = 0, scale = 0; - dp = buffer; - *dp++ = '0'; *dp++ = '.'; - buforg = dp, buflim = buffer+48; - for (save = sp; (c = *sp); sp++) - if (c == '.') { - if (dotseen) break; - dotseen++; - } else - if ((unsigned)(c-'0') > (unsigned)('9'-'0')) { - break; - } else - if (c == '0') { - if (dp != buforg) { - /* This is not the first digit, so we want to keep it */ - if (dp < buflim) *dp++ = c; - if (!dotseen) scale++; - } else { - /* No non-zero digits seen yet */ - /* If a . has been seen, scale must be adjusted */ - if (dotseen) scale--; - } - } else { - /* This is a nonzero digit, so we want to keep it */ - if (dp < buflim) *dp++ = c; - /* If it precedes a ., scale must be adjusted */ - if (!dotseen) scale++; - } - if (sp == save) { - if (ptr) *ptr = str; - errno = EDOM; /* what should this be? */ - return ZERO; - } - - while (dp > buforg && dp[-1] == '0') --dp; - if (dp == buforg) *dp++ = '0'; - *dp = '\0'; - /* Now the contents of buffer are - +--+--------+-+--------+ - |0.|fraction|\|leftover| - +--+--------+-+--------+ - ^dp points here - where fraction begins with 0 iff it is "0", and has at most - 45 digits in it, and leftover is at least 16 characters. - */ - save = sp, expt = 0, esign = 1; - do { - c = *sp++; - if (c != 'e' && c != 'E') break; - c = *sp++; - if (c == '-') esign -= 2, c = *sp++; else - if (c == '+' /* || c == ' ' */ ) c = *sp++; - if ((unsigned)(c-'0') > (unsigned)('9'-'0')) break; - while (c == '0') c = *sp++; - for (; (unsigned)(c-'0') <= (unsigned)('9'-'0'); c = *sp++) - expt = expt*10 + c-'0'; - if (esign < 0) expt = -expt; - save = sp-1; - } while (0); - if (ptr) *ptr = save; - expt += scale; - /* Now the number is sign*0.fraction*10**expt */ - errno = ERANGE; - if (expt > MDMAXEXPT) { - return HUGE*sign; - } else - if (expt == MDMAXEXPT) { - if (strcmp(buforg, MDMAXFRAC) > 0) return HUGE*sign; - } else - if (expt < MDMINEXPT) { - return ZERO*sign; - } else - if (expt == MDMINEXPT) { - if (strcmp(buforg, MDMINFRAC) < 0) return ZERO*sign; - } - /* We have now established that the number can be */ - /* represented without overflow or underflow */ - (void) sprintf(dp, "E%d", expt); - errno = 0; - return atof(buffer)*sign; -} diff --git a/Python/structmember.c b/Python/structmember.c index e347b51aac..af0296d802 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -5,91 +5,42 @@ #include "structmember.h" -static PyObject * -listmembers(struct memberlist *mlist) -{ - int i, n; - PyObject *v; - for (n = 0; mlist[n].name != NULL; n++) - ; - v = PyList_New(n); - if (v != NULL) { - for (i = 0; i < n; i++) - PyList_SetItem(v, i, - PyString_FromString(mlist[i].name)); - if (PyErr_Occurred()) { - Py_DECREF(v); - v = NULL; - } - else { - PyList_Sort(v); - } - } - return v; -} - -PyObject * -PyMember_Get(const char *addr, struct memberlist *mlist, const char *name) -{ - struct memberlist *l; - - if (strcmp(name, "__members__") == 0) - return listmembers(mlist); - for (l = mlist; l->name != NULL; l++) { - if (strcmp(l->name, name) == 0) { - PyMemberDef copy; - copy.name = l->name; - copy.type = l->type; - copy.offset = l->offset; - copy.flags = l->flags; - copy.doc = NULL; - return PyMember_GetOne(addr, ©); - } - } - PyErr_SetString(PyExc_AttributeError, name); - return NULL; -} - PyObject * PyMember_GetOne(const char *addr, PyMemberDef *l) { PyObject *v; - if ((l->flags & READ_RESTRICTED) && - PyEval_GetRestricted()) { - PyErr_SetString(PyExc_RuntimeError, "restricted attribute"); - return NULL; - } + addr += l->offset; switch (l->type) { case T_BOOL: v = PyBool_FromLong(*(char*)addr); break; case T_BYTE: - v = PyInt_FromLong(*(char*)addr); + v = PyLong_FromLong(*(char*)addr); break; case T_UBYTE: v = PyLong_FromUnsignedLong(*(unsigned char*)addr); break; case T_SHORT: - v = PyInt_FromLong(*(short*)addr); + v = PyLong_FromLong(*(short*)addr); break; case T_USHORT: v = PyLong_FromUnsignedLong(*(unsigned short*)addr); break; case T_INT: - v = PyInt_FromLong(*(int*)addr); + v = PyLong_FromLong(*(int*)addr); break; case T_UINT: v = PyLong_FromUnsignedLong(*(unsigned int*)addr); break; case T_LONG: - v = PyInt_FromLong(*(long*)addr); + v = PyLong_FromLong(*(long*)addr); break; case T_ULONG: v = PyLong_FromUnsignedLong(*(unsigned long*)addr); break; case T_PYSSIZET: - v = PyInt_FromSsize_t(*(Py_ssize_t*)addr); + v = PyLong_FromSsize_t(*(Py_ssize_t*)addr); break; case T_FLOAT: v = PyFloat_FromDouble((double)*(float*)addr); @@ -103,13 +54,13 @@ PyMember_GetOne(const char *addr, PyMemberDef *l) v = Py_None; } else - v = PyString_FromString(*(char**)addr); + v = PyUnicode_FromString(*(char**)addr); break; case T_STRING_INPLACE: - v = PyString_FromString((char*)addr); + v = PyUnicode_FromString((char*)addr); break; case T_CHAR: - v = PyString_FromStringAndSize((char*)addr, 1); + v = PyUnicode_FromStringAndSize((char*)addr, 1); break; case T_OBJECT: v = *(PyObject **)addr; @@ -131,6 +82,10 @@ PyMember_GetOne(const char *addr, PyMemberDef *l) v = PyLong_FromUnsignedLongLong(*(unsigned PY_LONG_LONG *)addr); break; #endif /* HAVE_LONG_LONG */ + case T_NONE: + v = Py_None; + Py_INCREF(v); + break; default: PyErr_SetString(PyExc_SystemError, "bad memberdescr type"); v = NULL; @@ -138,31 +93,10 @@ PyMember_GetOne(const char *addr, PyMemberDef *l) return v; } -int -PyMember_Set(char *addr, struct memberlist *mlist, const char *name, PyObject *v) -{ - struct memberlist *l; - - for (l = mlist; l->name != NULL; l++) { - if (strcmp(l->name, name) == 0) { - PyMemberDef copy; - copy.name = l->name; - copy.type = l->type; - copy.offset = l->offset; - copy.flags = l->flags; - copy.doc = NULL; - return PyMember_SetOne(addr, ©, v); - } - } - - PyErr_SetString(PyExc_AttributeError, name); - return -1; -} - -#define WARN(msg) \ - do { \ - if (PyErr_Warn(PyExc_RuntimeWarning, msg) < 0) \ - return -1; \ +#define WARN(msg) \ + do { \ + if (PyErr_WarnEx(PyExc_RuntimeWarning, msg, 1) < 0) \ + return -1; \ } while (0) int @@ -174,11 +108,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if ((l->flags & READONLY)) { - PyErr_SetString(PyExc_TypeError, "readonly attribute"); - return -1; - } - if ((l->flags & PY_WRITE_RESTRICTED) && PyEval_GetRestricted()) { - PyErr_SetString(PyExc_RuntimeError, "restricted attribute"); + PyErr_SetString(PyExc_AttributeError, "readonly attribute"); return -1; } if (v == NULL) { @@ -209,7 +139,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) break; } case T_BYTE:{ - long long_val = PyInt_AsLong(v); + long long_val = PyLong_AsLong(v); if ((long_val == -1) && PyErr_Occurred()) return -1; *(char*)addr = (char)long_val; @@ -220,7 +150,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) break; } case T_UBYTE:{ - long long_val = PyInt_AsLong(v); + long long_val = PyLong_AsLong(v); if ((long_val == -1) && PyErr_Occurred()) return -1; *(unsigned char*)addr = (unsigned char)long_val; @@ -229,7 +159,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) break; } case T_SHORT:{ - long long_val = PyInt_AsLong(v); + long long_val = PyLong_AsLong(v); if ((long_val == -1) && PyErr_Occurred()) return -1; *(short*)addr = (short)long_val; @@ -238,7 +168,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) break; } case T_USHORT:{ - long long_val = PyInt_AsLong(v); + long long_val = PyLong_AsLong(v); if ((long_val == -1) && PyErr_Occurred()) return -1; *(unsigned short*)addr = (unsigned short)long_val; @@ -247,7 +177,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) break; } case T_INT:{ - long long_val = PyInt_AsLong(v); + long long_val = PyLong_AsLong(v); if ((long_val == -1) && PyErr_Occurred()) return -1; *(int *)addr = (int)long_val; @@ -295,7 +225,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) break; } case T_PYSSIZET:{ - *(Py_ssize_t*)addr = PyInt_AsSsize_t(v); + *(Py_ssize_t*)addr = PyLong_AsSsize_t(v); if ((*(Py_ssize_t*)addr == (Py_ssize_t)-1) && PyErr_Occurred()) return -1; @@ -320,15 +250,18 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) *(PyObject **)addr = v; Py_XDECREF(oldv); break; - case T_CHAR: - if (PyString_Check(v) && PyString_Size(v) == 1) { - *(char*)addr = PyString_AsString(v)[0]; - } - else { + case T_CHAR: { + char *string; + Py_ssize_t len; + + string = _PyUnicode_AsStringAndSize(v, &len); + if (string == NULL || len != 1) { PyErr_BadArgument(); return -1; } + *(char*)addr = string[0]; break; + } case T_STRING: case T_STRING_INPLACE: PyErr_SetString(PyExc_TypeError, "readonly attribute"); @@ -348,7 +281,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if (PyLong_Check(v)) *(unsigned PY_LONG_LONG*)addr = value = PyLong_AsUnsignedLongLong(v); else - *(unsigned PY_LONG_LONG*)addr = value = PyInt_AsLong(v); + *(unsigned PY_LONG_LONG*)addr = value = PyLong_AsLong(v); if ((value == (unsigned PY_LONG_LONG)-1) && PyErr_Occurred()) return -1; break; diff --git a/Python/symtable.c b/Python/symtable.c index 644d9c5083..35fc6e1954 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -8,9 +8,15 @@ #define GLOBAL_AFTER_ASSIGN \ "name '%.400s' is assigned to before global declaration" +#define NONLOCAL_AFTER_ASSIGN \ +"name '%.400s' is assigned to before nonlocal declaration" + #define GLOBAL_AFTER_USE \ "name '%.400s' is used prior to global declaration" +#define NONLOCAL_AFTER_USE \ +"name '%.400s' is used prior to nonlocal declaration" + #define IMPORT_STAR_WARNING "import * only allowed at module level" #define RETURN_VAL_IN_GENERATOR \ @@ -19,7 +25,7 @@ static PySTEntryObject * ste_new(struct symtable *st, identifier name, _Py_block_ty block, - void *key, int lineno) + void *key, int lineno, int col_offset) { PySTEntryObject *ste = NULL; PyObject *k = NULL; @@ -61,8 +67,10 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block, ste->ste_varargs = 0; ste->ste_varkeywords = 0; ste->ste_opt_lineno = 0; + ste->ste_opt_col_offset = 0; ste->ste_tmpname = 0; ste->ste_lineno = lineno; + ste->ste_col_offset = col_offset; if (st->st_cur != NULL && (st->st_cur->ste_nested || @@ -72,7 +80,7 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block, ste->ste_generator = 0; ste->ste_returns_value = 0; - if (PyDict_SetItem(st->st_symbols, ste->ste_id, (PyObject *)ste) < 0) + if (PyDict_SetItem(st->st_blocks, ste->ste_id, (PyObject *)ste) < 0) goto fail; return ste; @@ -84,13 +92,9 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block, static PyObject * ste_repr(PySTEntryObject *ste) { - char buf[256]; - - PyOS_snprintf(buf, sizeof(buf), - "<symtable entry %.100s(%ld), line %d>", - PyString_AS_STRING(ste->ste_name), - PyInt_AS_LONG(ste->ste_id), ste->ste_lineno); - return PyString_FromString(buf); + return PyUnicode_FromFormat("<symtable entry %U(%ld), line %d>", + ste->ste_name, + PyLong_AS_LONG(ste->ste_id), ste->ste_lineno); } static void @@ -129,7 +133,7 @@ PyTypeObject PySTEntry_Type = { 0, /* tp_print */ 0, /* tp_getattr */ 0, /* tp_setattr */ - 0, /* tp_compare */ + 0, /* tp_reserved */ (reprfunc)ste_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ @@ -164,32 +168,36 @@ PyTypeObject PySTEntry_Type = { static int symtable_analyze(struct symtable *st); static int symtable_warn(struct symtable *st, char *msg, int lineno); static int symtable_enter_block(struct symtable *st, identifier name, - _Py_block_ty block, void *ast, int lineno); + _Py_block_ty block, void *ast, int lineno, + int col_offset); static int symtable_exit_block(struct symtable *st, void *ast); static int symtable_visit_stmt(struct symtable *st, stmt_ty s); static int symtable_visit_expr(struct symtable *st, expr_ty s); static int symtable_visit_genexp(struct symtable *st, expr_ty s); -static int symtable_visit_setcomp(struct symtable *st, expr_ty e); -static int symtable_visit_dictcomp(struct symtable *st, expr_ty e); +static int symtable_visit_listcomp(struct symtable *st, expr_ty s); +static int symtable_visit_setcomp(struct symtable *st, expr_ty s); +static int symtable_visit_dictcomp(struct symtable *st, expr_ty s); static int symtable_visit_arguments(struct symtable *st, arguments_ty); static int symtable_visit_excepthandler(struct symtable *st, excepthandler_ty); static int symtable_visit_alias(struct symtable *st, alias_ty); static int symtable_visit_comprehension(struct symtable *st, comprehension_ty); static int symtable_visit_keyword(struct symtable *st, keyword_ty); static int symtable_visit_slice(struct symtable *st, slice_ty); -static int symtable_visit_params(struct symtable *st, asdl_seq *args, int top); -static int symtable_visit_params_nested(struct symtable *st, asdl_seq *args); +static int symtable_visit_params(struct symtable *st, asdl_seq *args); +static int symtable_visit_argannotations(struct symtable *st, asdl_seq *args); static int symtable_implicit_arg(struct symtable *st, int pos); +static int symtable_visit_annotations(struct symtable *st, stmt_ty s); -static identifier top = NULL, lambda = NULL, genexpr = NULL, setcomp = NULL, - dictcomp = NULL; +static identifier top = NULL, lambda = NULL, genexpr = NULL, + listcomp = NULL, setcomp = NULL, dictcomp = NULL, + __class__ = NULL, __locals__ = NULL; #define GET_IDENTIFIER(VAR) \ - ((VAR) ? (VAR) : ((VAR) = PyString_InternFromString(# VAR))) + ((VAR) ? (VAR) : ((VAR) = PyUnicode_InternFromString(# VAR))) #define DUPLICATE_ARGUMENT \ -"duplicate argument '%s' in function definition" +"duplicate argument '%U' in function definition" static struct symtable * symtable_new(void) @@ -201,11 +209,11 @@ symtable_new(void) return NULL; st->st_filename = NULL; - st->st_symbols = NULL; + st->st_blocks = NULL; if ((st->st_stack = PyList_New(0)) == NULL) goto fail; - if ((st->st_symbols = PyDict_New()) == NULL) + if ((st->st_blocks = PyDict_New()) == NULL) goto fail; st->st_cur = NULL; st->st_private = NULL; @@ -226,15 +234,15 @@ PySymtable_Build(mod_ty mod, const char *filename, PyFutureFeatures *future) return st; st->st_filename = filename; st->st_future = future; + /* Make the initial symbol information gathering pass */ if (!GET_IDENTIFIER(top) || - !symtable_enter_block(st, top, ModuleBlock, (void *)mod, 0)) { + !symtable_enter_block(st, top, ModuleBlock, (void *)mod, 0, 0)) { PySymtable_Free(st); return NULL; } st->st_top = st->st_cur; st->st_cur->ste_unoptimized = OPT_TOPLEVEL; - /* Any other top-level initialization? */ switch (mod->kind) { case Module_kind: seq = mod->v.Module.body; @@ -263,6 +271,7 @@ PySymtable_Build(mod_ty mod, const char *filename, PyFutureFeatures *future) PySymtable_Free(st); return NULL; } + /* Make the second symbol analysis pass */ if (symtable_analyze(st)) return st; PySymtable_Free(st); @@ -276,7 +285,7 @@ PySymtable_Build(mod_ty mod, const char *filename, PyFutureFeatures *future) void PySymtable_Free(struct symtable *st) { - Py_XDECREF(st->st_symbols); + Py_XDECREF(st->st_blocks); Py_XDECREF(st->st_stack); PyMem_Free((void *)st); } @@ -289,7 +298,7 @@ PySymtable_Lookup(struct symtable *st, void *key) k = PyLong_FromVoidPtr(key); if (k == NULL) return NULL; - v = PyDict_GetItem(st->st_symbols, k); + v = PyDict_GetItem(st->st_blocks, k); if (v) { assert(PySTEntry_Check(v)); Py_INCREF(v); @@ -309,19 +318,19 @@ PyST_GetScope(PySTEntryObject *ste, PyObject *name) PyObject *v = PyDict_GetItem(ste->ste_symbols, name); if (!v) return 0; - assert(PyInt_Check(v)); - return (PyInt_AS_LONG(v) >> SCOPE_OFF) & SCOPE_MASK; + assert(PyLong_Check(v)); + return (PyLong_AS_LONG(v) >> SCOPE_OFFSET) & SCOPE_MASK; } /* Analyze raw symbol information to determine scope of each name. - The next several functions are helpers for PySymtable_Analyze(), + The next several functions are helpers for symtable_analyze(), which determines whether a name is local, global, or free. In addition, it determines which local variables are cell variables; they provide bindings that are used for free variables in enclosed blocks. - There are also two kinds of free variables, implicit and explicit. An + There are also two kinds of global variables, implicit and explicit. An explicit global is declared with the global statement. An implicit global is a free variable for which the compiler has found no binding in an enclosing function scope. The implicit global is either a global @@ -331,28 +340,36 @@ PyST_GetScope(PySTEntryObject *ste, PyObject *name) is treated as a local. The symbol table requires two passes to determine the scope of each name. - The first pass collects raw facts from the AST: the name is a parameter - here, the name is used by not defined here, etc. The second pass analyzes - these facts during a pass over the PySTEntryObjects created during pass 1. + The first pass collects raw facts from the AST via the symtable_visit_* + functions: the name is a parameter here, the name is used but not defined + here, etc. The second pass analyzes these facts during a pass over the + PySTEntryObjects created during pass 1. When a function is entered during the second pass, the parent passes the set of all name bindings visible to its children. These bindings - are used to determine if the variable is free or an implicit global. - After doing the local analysis, it analyzes each of its child blocks - using an updated set of name bindings. - - The children update the free variable set. If a local variable is free - in a child, the variable is marked as a cell. The current function must - provide runtime storage for the variable that may outlive the function's - frame. Cell variables are removed from the free set before the analyze - function returns to its parent. - - The sets of bound and free variables are implemented as dictionaries - mapping strings to None. + are used to determine if non-local variables are free or implicit globals. + Names which are explicitly declared nonlocal must exist in this set of + visible names - if they do not, a syntax error is raised. After doing + the local analysis, it analyzes each of its child blocks using an + updated set of name bindings. + + The children update the free variable set. If a local variable is added to + the free variable set by the child, the variable is marked as a cell. The + function object being defined must provide runtime storage for the variable + that may outlive the function's frame. Cell variables are removed from the + free set before the analyze function returns to its parent. + + During analysis, the names are: + symbols: dict mapping from symbol names to flag values (including offset scope values) + scopes: dict mapping from symbol names to scope values (no offset) + local: set of all symbol names local to the current scope + bound: set of all symbol names local to a containing function scope + free: set of all symbol names referenced but not bound in child scopes + global: set of all symbol names explicitly declared as global */ #define SET_SCOPE(DICT, NAME, I) { \ - PyObject *o = PyInt_FromLong(I); \ + PyObject *o = PyLong_FromLong(I); \ if (!o) \ return 0; \ if (PyDict_SetItem((DICT), (NAME), o) < 0) { \ @@ -370,37 +387,62 @@ PyST_GetScope(PySTEntryObject *ste, PyObject *name) */ static int -analyze_name(PySTEntryObject *ste, PyObject *dict, PyObject *name, long flags, +analyze_name(PySTEntryObject *ste, PyObject *scopes, PyObject *name, long flags, PyObject *bound, PyObject *local, PyObject *free, PyObject *global) { if (flags & DEF_GLOBAL) { if (flags & DEF_PARAM) { PyErr_Format(PyExc_SyntaxError, - "name '%s' is local and global", - PyString_AS_STRING(name)); - PyErr_SyntaxLocation(ste->ste_table->st_filename, - ste->ste_lineno); + "name '%U' is parameter and global", + name); + PyErr_SyntaxLocationEx(ste->ste_table->st_filename, + ste->ste_lineno, ste->ste_col_offset); return 0; } - SET_SCOPE(dict, name, GLOBAL_EXPLICIT); - if (PyDict_SetItem(global, name, Py_None) < 0) + if (flags & DEF_NONLOCAL) { + PyErr_Format(PyExc_SyntaxError, + "name '%U' is nonlocal and global", + name); return 0; - if (bound && PyDict_GetItem(bound, name)) { - if (PyDict_DelItem(bound, name) < 0) - return 0; } + SET_SCOPE(scopes, name, GLOBAL_EXPLICIT); + if (PySet_Add(global, name) < 0) + return 0; + if (bound && (PySet_Discard(bound, name) < 0)) + return 0; return 1; } - if (flags & DEF_BOUND) { - SET_SCOPE(dict, name, LOCAL); - if (PyDict_SetItem(local, name, Py_None) < 0) + if (flags & DEF_NONLOCAL) { + if (flags & DEF_PARAM) { + PyErr_Format(PyExc_SyntaxError, + "name '%U' is parameter and nonlocal", + name); return 0; - if (PyDict_GetItem(global, name)) { - if (PyDict_DelItem(global, name) < 0) - return 0; } + if (!bound) { + PyErr_Format(PyExc_SyntaxError, + "nonlocal declaration not allowed at module level"); + return 0; + } + if (!PySet_Contains(bound, name)) { + PyErr_Format(PyExc_SyntaxError, + "no binding for nonlocal '%U' found", + name); + + return 0; + } + SET_SCOPE(scopes, name, FREE); + ste->ste_free = 1; + return PySet_Add(free, name) >= 0; + } + if (flags & DEF_BOUND) { + SET_SCOPE(scopes, name, LOCAL); + if (PySet_Add(local, name) < 0) + return 0; + if (PySet_Discard(global, name) < 0) + return 0; return 1; } /* If an enclosing block has a binding for this name, it @@ -408,30 +450,22 @@ analyze_name(PySTEntryObject *ste, PyObject *dict, PyObject *name, long flags, Note that having a non-NULL bound implies that the block is nested. */ - if (bound && PyDict_GetItem(bound, name)) { - SET_SCOPE(dict, name, FREE); + if (bound && PySet_Contains(bound, name)) { + SET_SCOPE(scopes, name, FREE); ste->ste_free = 1; - if (PyDict_SetItem(free, name, Py_None) < 0) - return 0; - return 1; + return PySet_Add(free, name) >= 0; } /* If a parent has a global statement, then call it global explicit? It could also be global implicit. */ - else if (global && PyDict_GetItem(global, name)) { - SET_SCOPE(dict, name, GLOBAL_IMPLICIT); - return 1; - } - else { - if (ste->ste_nested) - ste->ste_free = 1; - SET_SCOPE(dict, name, GLOBAL_IMPLICIT); + if (global && PySet_Contains(global, name)) { + SET_SCOPE(scopes, name, GLOBAL_IMPLICIT); return 1; } - /* Should never get here. */ - PyErr_Format(PyExc_SystemError, "failed to set scope for %s", - PyString_AS_STRING(name)); - return 0; + if (ste->ste_nested) + ste->ste_free = 1; + SET_SCOPE(scopes, name, GLOBAL_IMPLICIT); + return 1; } #undef SET_SCOPE @@ -442,45 +476,50 @@ analyze_name(PySTEntryObject *ste, PyObject *dict, PyObject *name, long flags, Note that the current block's free variables are included in free. That's safe because no name can be free and local in the same scope. + + The 'restricted' argument may be set to a string to restrict the analysis + to the one variable whose name equals that string (e.g. "__class__"). */ static int -analyze_cells(PyObject *scope, PyObject *free) +analyze_cells(PyObject *scopes, PyObject *free, const char *restricted) { - PyObject *name, *v, *w; + PyObject *name, *v, *v_cell; int success = 0; Py_ssize_t pos = 0; - w = PyInt_FromLong(CELL); - if (!w) + v_cell = PyLong_FromLong(CELL); + if (!v_cell) return 0; - while (PyDict_Next(scope, &pos, &name, &v)) { - long flags; - assert(PyInt_Check(v)); - flags = PyInt_AS_LONG(v); - if (flags != LOCAL) + while (PyDict_Next(scopes, &pos, &name, &v)) { + long scope; + assert(PyLong_Check(v)); + scope = PyLong_AS_LONG(v); + if (scope != LOCAL) + continue; + if (!PySet_Contains(free, name)) continue; - if (!PyDict_GetItem(free, name)) + if (restricted != NULL && + PyUnicode_CompareWithASCIIString(name, restricted)) continue; /* Replace LOCAL with CELL for this name, and remove from free. It is safe to replace the value of name in the dict, because it will not cause a resize. */ - if (PyDict_SetItem(scope, name, w) < 0) + if (PyDict_SetItem(scopes, name, v_cell) < 0) goto error; - if (!PyDict_DelItem(free, name) < 0) + if (PySet_Discard(free, name) < 0) goto error; } success = 1; error: - Py_DECREF(w); + Py_DECREF(v_cell); return success; } /* Check for illegal statements in unoptimized namespaces */ static int check_unoptimized(const PySTEntryObject* ste) { - char buf[300]; const char* trailer; if (ste->ste_type != FunctionBlock || !ste->ste_unoptimized @@ -492,106 +531,105 @@ check_unoptimized(const PySTEntryObject* ste) { "is a nested function"); switch (ste->ste_unoptimized) { - case OPT_TOPLEVEL: /* exec / import * at top-level is fine */ - case OPT_EXEC: /* qualified exec is fine */ + case OPT_TOPLEVEL: /* import * at top-level is fine */ return 1; case OPT_IMPORT_STAR: - PyOS_snprintf(buf, sizeof(buf), - "import * is not allowed in function '%.100s' " - "because it %s", - PyString_AS_STRING(ste->ste_name), trailer); - break; - case OPT_BARE_EXEC: - PyOS_snprintf(buf, sizeof(buf), - "unqualified exec is not allowed in function " - "'%.100s' it %s", - PyString_AS_STRING(ste->ste_name), trailer); - break; - default: - PyOS_snprintf(buf, sizeof(buf), - "function '%.100s' uses import * and bare exec, " - "which are illegal because it %s", - PyString_AS_STRING(ste->ste_name), trailer); + PyErr_Format(PyExc_SyntaxError, + "import * is not allowed in function '%U' because it %s", + ste->ste_name, trailer); break; } - PyErr_SetString(PyExc_SyntaxError, buf); - PyErr_SyntaxLocation(ste->ste_table->st_filename, - ste->ste_opt_lineno); + PyErr_SyntaxLocationEx(ste->ste_table->st_filename, ste->ste_opt_lineno, + ste->ste_opt_col_offset); return 0; } -/* Enter the final scope information into the st_symbols dict. +/* Enter the final scope information into the ste_symbols dict. * * All arguments are dicts. Modifies symbols, others are read-only. */ static int -update_symbols(PyObject *symbols, PyObject *scope, +update_symbols(PyObject *symbols, PyObject *scopes, PyObject *bound, PyObject *free, int classflag) { - PyObject *name, *v, *u, *w, *free_value = NULL; + PyObject *name = NULL, *itr = NULL; + PyObject *v = NULL, *v_scope = NULL, *v_new = NULL, *v_free = NULL; Py_ssize_t pos = 0; + /* Update scope information for all symbols in this scope */ while (PyDict_Next(symbols, &pos, &name, &v)) { - long i, flags; - assert(PyInt_Check(v)); - flags = PyInt_AS_LONG(v); - w = PyDict_GetItem(scope, name); - assert(w && PyInt_Check(w)); - i = PyInt_AS_LONG(w); - flags |= (i << SCOPE_OFF); - u = PyInt_FromLong(flags); - if (!u) + long scope, flags; + assert(PyLong_Check(v)); + flags = PyLong_AS_LONG(v); + v_scope = PyDict_GetItem(scopes, name); + assert(v_scope && PyLong_Check(v_scope)); + scope = PyLong_AS_LONG(v_scope); + flags |= (scope << SCOPE_OFFSET); + v_new = PyLong_FromLong(flags); + if (!v_new) return 0; - if (PyDict_SetItem(symbols, name, u) < 0) { - Py_DECREF(u); + if (PyDict_SetItem(symbols, name, v_new) < 0) { + Py_DECREF(v_new); return 0; } - Py_DECREF(u); + Py_DECREF(v_new); } - free_value = PyInt_FromLong(FREE << SCOPE_OFF); - if (!free_value) + /* Record not yet resolved free variables from children (if any) */ + v_free = PyLong_FromLong(FREE << SCOPE_OFFSET); + if (!v_free) return 0; - /* add a free variable when it's only use is for creating a closure */ - pos = 0; - while (PyDict_Next(free, &pos, &name, &v)) { - PyObject *o = PyDict_GetItem(symbols, name); + itr = PyObject_GetIter(free); + if (!itr) + goto error; - if (o) { - /* It could be a free variable in a method of + while ((name = PyIter_Next(itr))) { + v = PyDict_GetItem(symbols, name); + + /* Handle symbol that already exists in this scope */ + if (v) { + /* Handle a free variable in a method of the class that has the same name as a local or global in the class scope. */ if (classflag && - PyInt_AS_LONG(o) & (DEF_BOUND | DEF_GLOBAL)) { - long i = PyInt_AS_LONG(o) | DEF_FREE_CLASS; - o = PyInt_FromLong(i); - if (!o) { - Py_DECREF(free_value); - return 0; + PyLong_AS_LONG(v) & (DEF_BOUND | DEF_GLOBAL)) { + long flags = PyLong_AS_LONG(v) | DEF_FREE_CLASS; + v_new = PyLong_FromLong(flags); + if (!v_new) { + goto error; } - if (PyDict_SetItem(symbols, name, o) < 0) { - Py_DECREF(o); - Py_DECREF(free_value); - return 0; + if (PyDict_SetItem(symbols, name, v_new) < 0) { + Py_DECREF(v_new); + goto error; } - Py_DECREF(o); + Py_DECREF(v_new); } - /* else it's not free, probably a cell */ + /* It's a cell, or already free in this scope */ + Py_DECREF(name); continue; } - if (!PyDict_GetItem(bound, name)) + /* Handle global symbol */ + if (!PySet_Contains(bound, name)) { + Py_DECREF(name); continue; /* it's a global */ - - if (PyDict_SetItem(symbols, name, free_value) < 0) { - Py_DECREF(free_value); - return 0; } + /* Propagate new free symbol up the lexical stack */ + if (PyDict_SetItem(symbols, name, v_free) < 0) { + goto error; + } + Py_DECREF(name); } - Py_DECREF(free_value); + Py_DECREF(itr); + Py_DECREF(v_free); return 1; +error: + Py_XDECREF(v_free); + Py_XDECREF(itr); + Py_XDECREF(name); + return 0; } /* Make final symbol table decisions for block of ste. @@ -622,17 +660,17 @@ static int analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, PyObject *global) { - PyObject *name, *v, *local = NULL, *scope = NULL; - PyObject *newbound = NULL, *newglobal = NULL; - PyObject *newfree = NULL, *allfree = NULL; + PyObject *name, *v, *local = NULL, *scopes = NULL, *newbound = NULL; + PyObject *newglobal = NULL, *newfree = NULL, *allfree = NULL; + PyObject *temp; int i, success = 0; Py_ssize_t pos = 0; - local = PyDict_New(); /* collect new names bound in block */ + local = PySet_New(NULL); /* collect new names bound in block */ if (!local) goto error; - scope = PyDict_New(); /* collect scopes defined for each name */ - if (!scope) + scopes = PyDict_New(); /* collect scopes defined for each name */ + if (!scopes) goto error; /* Allocate new global and bound variable dictionaries. These @@ -646,41 +684,71 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, /* TODO(jhylton): Package these dicts in a struct so that we can write reasonable helper functions? */ - newglobal = PyDict_New(); + newglobal = PySet_New(NULL); if (!newglobal) goto error; - newbound = PyDict_New(); - if (!newbound) - goto error; - newfree = PyDict_New(); + newfree = PySet_New(NULL); if (!newfree) goto error; + newbound = PySet_New(NULL); + if (!newbound) + goto error; + /* Class namespace has no effect on names visible in + nested functions, so populate the global and bound + sets to be passed to child blocks before analyzing + this one. + */ if (ste->ste_type == ClassBlock) { - if (PyDict_Update(newglobal, global) < 0) + /* Pass down known globals */ + temp = PyNumber_InPlaceOr(newglobal, global); + if (!temp) goto error; - if (bound) - if (PyDict_Update(newbound, bound) < 0) + Py_DECREF(temp); + /* Pass down previously bound symbols */ + if (bound) { + temp = PyNumber_InPlaceOr(newbound, bound); + if (!temp) goto error; + Py_DECREF(temp); + } } while (PyDict_Next(ste->ste_symbols, &pos, &name, &v)) { - long flags = PyInt_AS_LONG(v); - if (!analyze_name(ste, scope, name, flags, + long flags = PyLong_AS_LONG(v); + if (!analyze_name(ste, scopes, name, flags, bound, local, free, global)) goto error; } + /* Populate global and bound sets to be passed to children. */ if (ste->ste_type != ClassBlock) { + /* Add function locals to bound set */ if (ste->ste_type == FunctionBlock) { - if (PyDict_Update(newbound, local) < 0) + temp = PyNumber_InPlaceOr(newbound, local); + if (!temp) goto error; + Py_DECREF(temp); } + /* Pass down previously bound symbols */ if (bound) { - if (PyDict_Update(newbound, bound) < 0) + temp = PyNumber_InPlaceOr(newbound, bound); + if (!temp) goto error; + Py_DECREF(temp); } - if (PyDict_Update(newglobal, global) < 0) + /* Pass down known globals */ + temp = PyNumber_InPlaceOr(newglobal, global); + if (!temp) + goto error; + Py_DECREF(temp); + } + else { + /* Special-case __class__ */ + if (!GET_IDENTIFIER(__class__)) + goto error; + assert(PySet_Contains(local, __class__) == 1); + if (PySet_Add(newbound, __class__) < 0) goto error; } @@ -690,7 +758,7 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, nested blocks. The free variables in the children will be collected in allfree. */ - allfree = PyDict_New(); + allfree = PySet_New(NULL); if (!allfree) goto error; for (i = 0; i < PyList_GET_SIZE(ste->ste_children); ++i) { @@ -701,26 +769,38 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, if (!analyze_child_block(entry, newbound, newfree, newglobal, allfree)) goto error; + /* Check if any children have free variables */ if (entry->ste_free || entry->ste_child_free) ste->ste_child_free = 1; } - if (PyDict_Update(newfree, allfree) < 0) + temp = PyNumber_InPlaceOr(newfree, allfree); + if (!temp) + goto error; + Py_DECREF(temp); + + /* Check if any local variables must be converted to cell variables */ + if (ste->ste_type == FunctionBlock && !analyze_cells(scopes, newfree, + NULL)) goto error; - if (ste->ste_type == FunctionBlock && !analyze_cells(scope, newfree)) + else if (ste->ste_type == ClassBlock && !analyze_cells(scopes, newfree, + "__class__")) goto error; - if (!update_symbols(ste->ste_symbols, scope, bound, newfree, + /* Records the results of the analysis in the symbol table entry */ + if (!update_symbols(ste->ste_symbols, scopes, bound, newfree, ste->ste_type == ClassBlock)) goto error; if (!check_unoptimized(ste)) goto error; - if (PyDict_Update(free, newfree) < 0) + temp = PyNumber_InPlaceOr(free, newfree); + if (!temp) goto error; + Py_DECREF(temp); success = 1; error: + Py_XDECREF(scopes); Py_XDECREF(local); - Py_XDECREF(scope); Py_XDECREF(newbound); Py_XDECREF(newglobal); Py_XDECREF(newfree); @@ -735,6 +815,7 @@ analyze_child_block(PySTEntryObject *entry, PyObject *bound, PyObject *free, PyObject *global, PyObject* child_free) { PyObject *temp_bound = NULL, *temp_global = NULL, *temp_free = NULL; + PyObject *temp; /* Copy the bound and global dictionaries. @@ -743,26 +824,22 @@ analyze_child_block(PySTEntryObject *entry, PyObject *bound, PyObject *free, dictionaries. */ - temp_bound = PyDict_New(); + temp_bound = PySet_New(bound); if (!temp_bound) goto error; - if (PyDict_Update(temp_bound, bound) < 0) - goto error; - temp_free = PyDict_New(); + temp_free = PySet_New(free); if (!temp_free) goto error; - if (PyDict_Update(temp_free, free) < 0) - goto error; - temp_global = PyDict_New(); + temp_global = PySet_New(global); if (!temp_global) goto error; - if (PyDict_Update(temp_global, global) < 0) - goto error; if (!analyze_block(entry, temp_bound, temp_free, temp_global)) goto error; - if (PyDict_Update(child_free, temp_free) < 0) + temp = PyNumber_InPlaceOr(child_free, temp_free); + if (!temp) goto error; + Py_DECREF(temp); Py_DECREF(temp_bound); Py_DECREF(temp_free); Py_DECREF(temp_global); @@ -780,10 +857,10 @@ symtable_analyze(struct symtable *st) PyObject *free, *global; int r; - free = PyDict_New(); + free = PySet_New(NULL); if (!free) return 0; - global = PyDict_New(); + global = PySet_New(NULL); if (!global) { Py_DECREF(free); return 0; @@ -802,8 +879,8 @@ symtable_warn(struct symtable *st, char *msg, int lineno) lineno, NULL, NULL) < 0) { if (PyErr_ExceptionMatches(PyExc_SyntaxWarning)) { PyErr_SetString(PyExc_SyntaxError, msg); - PyErr_SyntaxLocation(st->st_filename, - st->st_cur->ste_lineno); + PyErr_SyntaxLocationEx(st->st_filename, st->st_cur->ste_lineno, + st->st_cur->ste_col_offset); } return 0; } @@ -836,7 +913,7 @@ symtable_exit_block(struct symtable *st, void *ast) static int symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block, - void *ast, int lineno) + void *ast, int lineno, int col_offset) { PySTEntryObject *prev = NULL; @@ -847,7 +924,7 @@ symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block, } Py_DECREF(st->st_cur); } - st->st_cur = ste_new(st, name, block, ast, lineno); + st->st_cur = ste_new(st, name, block, ast, lineno, col_offset); if (st->st_cur == NULL) return 0; if (block == ModuleBlock) @@ -872,7 +949,7 @@ symtable_lookup(struct symtable *st, PyObject *name) Py_DECREF(mangled); if (!o) return 0; - return PyInt_AsLong(o); + return PyLong_AsLong(o); } static int @@ -883,23 +960,24 @@ symtable_add_def(struct symtable *st, PyObject *name, int flag) long val; PyObject *mangled = _Py_Mangle(st->st_private, name); + if (!mangled) return 0; dict = st->st_cur->ste_symbols; if ((o = PyDict_GetItem(dict, mangled))) { - val = PyInt_AS_LONG(o); + val = PyLong_AS_LONG(o); if ((flag & DEF_PARAM) && (val & DEF_PARAM)) { /* Is it better to use 'mangled' or 'name' here? */ - PyErr_Format(PyExc_SyntaxError, DUPLICATE_ARGUMENT, - PyString_AsString(name)); - PyErr_SyntaxLocation(st->st_filename, - st->st_cur->ste_lineno); + PyErr_Format(PyExc_SyntaxError, DUPLICATE_ARGUMENT, name); + PyErr_SyntaxLocationEx(st->st_filename, + st->st_cur->ste_lineno, + st->st_cur->ste_col_offset); goto error; } val |= flag; } else val = flag; - o = PyInt_FromLong(val); + o = PyLong_FromLong(val); if (o == NULL) goto error; if (PyDict_SetItem(dict, mangled, o) < 0) { @@ -916,9 +994,9 @@ symtable_add_def(struct symtable *st, PyObject *name, int flag) perhaps only DEF_FREE_GLOBAL */ val = flag; if ((o = PyDict_GetItem(st->st_global, mangled))) { - val |= PyInt_AS_LONG(o); + val |= PyLong_AS_LONG(o); } - o = PyInt_FromLong(val); + o = PyLong_FromLong(val); if (o == NULL) goto error; if (PyDict_SetItem(st->st_global, mangled, o) < 0) { @@ -997,6 +1075,35 @@ error: } \ } +#define VISIT_KWONLYDEFAULTS(ST, KW_DEFAULTS) { \ + int i = 0; \ + asdl_seq *seq = (KW_DEFAULTS); /* avoid variable capture */ \ + for (i = 0; i < asdl_seq_LEN(seq); i++) { \ + expr_ty elt = (expr_ty)asdl_seq_GET(seq, i); \ + if (!elt) continue; /* can be NULL */ \ + if (!symtable_visit_expr((ST), elt)) \ + return 0; \ + } \ +} + +static int +symtable_new_tmpname(struct symtable *st) +{ + char tmpname[256]; + identifier tmp; + + PyOS_snprintf(tmpname, sizeof(tmpname), "_[%d]", + ++st->st_cur->ste_tmpname); + tmp = PyUnicode_InternFromString(tmpname); + if (!tmp) + return 0; + if (!symtable_add_def(st, tmp, DEF_LOCAL)) + return 0; + Py_DECREF(tmp); + return 1; +} + + static int symtable_visit_stmt(struct symtable *st, stmt_ty s) { @@ -1006,10 +1113,16 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) return 0; if (s->v.FunctionDef.args->defaults) VISIT_SEQ(st, expr, s->v.FunctionDef.args->defaults); + if (s->v.FunctionDef.args->kw_defaults) + VISIT_KWONLYDEFAULTS(st, + s->v.FunctionDef.args->kw_defaults); + if (!symtable_visit_annotations(st, s)) + return 0; if (s->v.FunctionDef.decorator_list) VISIT_SEQ(st, expr, s->v.FunctionDef.decorator_list); if (!symtable_enter_block(st, s->v.FunctionDef.name, - FunctionBlock, (void *)s, s->lineno)) + FunctionBlock, (void *)s, s->lineno, + s->col_offset)) return 0; VISIT_IN_BLOCK(st, arguments, s->v.FunctionDef.args, s); VISIT_SEQ_IN_BLOCK(st, stmt, s->v.FunctionDef.body, s); @@ -1021,11 +1134,23 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) if (!symtable_add_def(st, s->v.ClassDef.name, DEF_LOCAL)) return 0; VISIT_SEQ(st, expr, s->v.ClassDef.bases); + VISIT_SEQ(st, keyword, s->v.ClassDef.keywords); + if (s->v.ClassDef.starargs) + VISIT(st, expr, s->v.ClassDef.starargs); + if (s->v.ClassDef.kwargs) + VISIT(st, expr, s->v.ClassDef.kwargs); if (s->v.ClassDef.decorator_list) VISIT_SEQ(st, expr, s->v.ClassDef.decorator_list); if (!symtable_enter_block(st, s->v.ClassDef.name, ClassBlock, - (void *)s, s->lineno)) + (void *)s, s->lineno, s->col_offset)) + return 0; + if (!GET_IDENTIFIER(__class__) || + !symtable_add_def(st, __class__, DEF_LOCAL) || + !GET_IDENTIFIER(__locals__) || + !symtable_add_def(st, __locals__, DEF_PARAM)) { + symtable_exit_block(st, s); return 0; + } tmp = st->st_private; st->st_private = s->v.ClassDef.name; VISIT_SEQ_IN_BLOCK(st, stmt, s->v.ClassDef.body, s); @@ -1041,8 +1166,9 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) if (st->st_cur->ste_generator) { PyErr_SetString(PyExc_SyntaxError, RETURN_VAL_IN_GENERATOR); - PyErr_SyntaxLocation(st->st_filename, - s->lineno); + PyErr_SyntaxLocationEx(st->st_filename, + s->lineno, + s->col_offset); return 0; } } @@ -1058,11 +1184,6 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT(st, expr, s->v.AugAssign.target); VISIT(st, expr, s->v.AugAssign.value); break; - case Print_kind: - if (s->v.Print.dest) - VISIT(st, expr, s->v.Print.dest); - VISIT_SEQ(st, expr, s->v.Print.values); - break; case For_kind: VISIT(st, expr, s->v.For.target); VISIT(st, expr, s->v.For.iter); @@ -1084,13 +1205,11 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT_SEQ(st, stmt, s->v.If.orelse); break; case Raise_kind: - if (s->v.Raise.type) { - VISIT(st, expr, s->v.Raise.type); - if (s->v.Raise.inst) { - VISIT(st, expr, s->v.Raise.inst); - if (s->v.Raise.tback) - VISIT(st, expr, s->v.Raise.tback); - } + if (s->v.Raise.exc) { + VISIT(st, expr, s->v.Raise.exc); + if (s->v.Raise.cause) { + VISIT(st, expr, s->v.Raise.cause); + } } break; case TryExcept_kind: @@ -1111,27 +1230,18 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT_SEQ(st, alias, s->v.Import.names); /* XXX Don't have the lineno available inside visit_alias */ - if (st->st_cur->ste_unoptimized && !st->st_cur->ste_opt_lineno) + if (st->st_cur->ste_unoptimized && !st->st_cur->ste_opt_lineno) { st->st_cur->ste_opt_lineno = s->lineno; + st->st_cur->ste_opt_col_offset = s->col_offset; + } break; case ImportFrom_kind: VISIT_SEQ(st, alias, s->v.ImportFrom.names); /* XXX Don't have the lineno available inside visit_alias */ - if (st->st_cur->ste_unoptimized && !st->st_cur->ste_opt_lineno) + if (st->st_cur->ste_unoptimized && !st->st_cur->ste_opt_lineno) { st->st_cur->ste_opt_lineno = s->lineno; - break; - case Exec_kind: - VISIT(st, expr, s->v.Exec.body); - if (!st->st_cur->ste_opt_lineno) - st->st_cur->ste_opt_lineno = s->lineno; - if (s->v.Exec.globals) { - st->st_cur->ste_unoptimized |= OPT_EXEC; - VISIT(st, expr, s->v.Exec.globals); - if (s->v.Exec.locals) - VISIT(st, expr, s->v.Exec.locals); - } else { - st->st_cur->ste_unoptimized |= OPT_BARE_EXEC; + st->st_cur->ste_opt_col_offset = s->col_offset; } break; case Global_kind: { @@ -1139,7 +1249,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) asdl_seq *seq = s->v.Global.names; for (i = 0; i < asdl_seq_LEN(seq); i++) { identifier name = (identifier)asdl_seq_GET(seq, i); - char *c_name = PyString_AS_STRING(name); + char *c_name = _PyUnicode_AsString(name); long cur = symtable_lookup(st, name); if (cur < 0) return 0; @@ -1161,6 +1271,33 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) } break; } + case Nonlocal_kind: { + int i; + asdl_seq *seq = s->v.Nonlocal.names; + for (i = 0; i < asdl_seq_LEN(seq); i++) { + identifier name = (identifier)asdl_seq_GET(seq, i); + char *c_name = _PyUnicode_AsString(name); + long cur = symtable_lookup(st, name); + if (cur < 0) + return 0; + if (cur & (DEF_LOCAL | USE)) { + char buf[256]; + if (cur & DEF_LOCAL) + PyOS_snprintf(buf, sizeof(buf), + NONLOCAL_AFTER_ASSIGN, + c_name); + else + PyOS_snprintf(buf, sizeof(buf), + NONLOCAL_AFTER_USE, + c_name); + if (!symtable_warn(st, buf, s->lineno)) + return 0; + } + if (!symtable_add_def(st, name, DEF_NONLOCAL)) + return 0; + } + break; + } case Expr_kind: VISIT(st, expr, s->v.Expr.value); break; @@ -1199,8 +1336,12 @@ symtable_visit_expr(struct symtable *st, expr_ty e) return 0; if (e->v.Lambda.args->defaults) VISIT_SEQ(st, expr, e->v.Lambda.args->defaults); + if (e->v.Lambda.args->kw_defaults) + VISIT_KWONLYDEFAULTS(st, + e->v.Lambda.args->kw_defaults); if (!symtable_enter_block(st, lambda, - FunctionBlock, (void *)e, e->lineno)) + FunctionBlock, (void *)e, e->lineno, + e->col_offset)) return 0; VISIT_IN_BLOCK(st, arguments, e->v.Lambda.args, (void*)e); VISIT_IN_BLOCK(st, expr, e->v.Lambda.body, (void*)e); @@ -1220,14 +1361,14 @@ symtable_visit_expr(struct symtable *st, expr_ty e) case Set_kind: VISIT_SEQ(st, expr, e->v.Set.elts); break; - case ListComp_kind: - VISIT(st, expr, e->v.ListComp.elt); - VISIT_SEQ(st, comprehension, e->v.ListComp.generators); - break; case GeneratorExp_kind: if (!symtable_visit_genexp(st, e)) return 0; break; + case ListComp_kind: + if (!symtable_visit_listcomp(st, e)) + return 0; + break; case SetComp_kind: if (!symtable_visit_setcomp(st, e)) return 0; @@ -1243,8 +1384,8 @@ symtable_visit_expr(struct symtable *st, expr_ty e) if (st->st_cur->ste_returns_value) { PyErr_SetString(PyExc_SyntaxError, RETURN_VAL_IN_GENERATOR); - PyErr_SyntaxLocation(st->st_filename, - e->lineno); + PyErr_SyntaxLocationEx(st->st_filename, + e->lineno, e->col_offset); return 0; } break; @@ -1261,11 +1402,10 @@ symtable_visit_expr(struct symtable *st, expr_ty e) if (e->v.Call.kwargs) VISIT(st, expr, e->v.Call.kwargs); break; - case Repr_kind: - VISIT(st, expr, e->v.Repr.value); - break; case Num_kind: case Str_kind: + case Bytes_kind: + case Ellipsis_kind: /* Nothing to do here. */ break; /* The following exprs can be assignment targets. */ @@ -1276,10 +1416,21 @@ symtable_visit_expr(struct symtable *st, expr_ty e) VISIT(st, expr, e->v.Subscript.value); VISIT(st, slice, e->v.Subscript.slice); break; + case Starred_kind: + VISIT(st, expr, e->v.Starred.value); + break; case Name_kind: if (!symtable_add_def(st, e->v.Name.id, e->v.Name.ctx == Load ? USE : DEF_LOCAL)) return 0; + /* Special-case super: it counts as a use of __class__ */ + if (e->v.Name.ctx == Load && + st->st_cur->ste_type == FunctionBlock && + !PyUnicode_CompareWithASCIIString(e->v.Name.id, "super")) { + if (!GET_IDENTIFIER(__class__) || + !symtable_add_def(st, __class__, USE)) + return 0; + } break; /* child nodes of List and Tuple will have expr_context set */ case List_kind: @@ -1295,7 +1446,7 @@ symtable_visit_expr(struct symtable *st, expr_ty e) static int symtable_implicit_arg(struct symtable *st, int pos) { - PyObject *id = PyString_FromFormat(".%d", pos); + PyObject *id = PyUnicode_FromFormat(".%d", pos); if (id == NULL) return 0; if (!symtable_add_def(st, id, DEF_PARAM)) { @@ -1307,37 +1458,16 @@ symtable_implicit_arg(struct symtable *st, int pos) } static int -symtable_visit_params(struct symtable *st, asdl_seq *args, int toplevel) +symtable_visit_params(struct symtable *st, asdl_seq *args) { int i; - /* go through all the toplevel arguments first */ - for (i = 0; i < asdl_seq_LEN(args); i++) { - expr_ty arg = (expr_ty)asdl_seq_GET(args, i); - if (arg->kind == Name_kind) { - assert(arg->v.Name.ctx == Param || - (arg->v.Name.ctx == Store && !toplevel)); - if (!symtable_add_def(st, arg->v.Name.id, DEF_PARAM)) - return 0; - } - else if (arg->kind == Tuple_kind) { - assert(arg->v.Tuple.ctx == Store); - if (toplevel) { - if (!symtable_implicit_arg(st, i)) - return 0; - } - } - else { - PyErr_SetString(PyExc_SyntaxError, - "invalid expression in parameter list"); - PyErr_SyntaxLocation(st->st_filename, - st->st_cur->ste_lineno); - return 0; - } - } + if (!args) + return -1; - if (!toplevel) { - if (!symtable_visit_params_nested(st, args)) + for (i = 0; i < asdl_seq_LEN(args); i++) { + arg_ty arg = (arg_ty)asdl_seq_GET(args, i); + if (!symtable_add_def(st, arg->arg, DEF_PARAM)) return 0; } @@ -1345,26 +1475,49 @@ symtable_visit_params(struct symtable *st, asdl_seq *args, int toplevel) } static int -symtable_visit_params_nested(struct symtable *st, asdl_seq *args) +symtable_visit_argannotations(struct symtable *st, asdl_seq *args) { int i; + + if (!args) + return -1; + for (i = 0; i < asdl_seq_LEN(args); i++) { - expr_ty arg = (expr_ty)asdl_seq_GET(args, i); - if (arg->kind == Tuple_kind && - !symtable_visit_params(st, arg->v.Tuple.elts, 0)) - return 0; + arg_ty arg = (arg_ty)asdl_seq_GET(args, i); + if (arg->annotation) + VISIT(st, expr, arg->annotation); } return 1; } static int +symtable_visit_annotations(struct symtable *st, stmt_ty s) +{ + arguments_ty a = s->v.FunctionDef.args; + + if (a->args && !symtable_visit_argannotations(st, a->args)) + return 0; + if (a->varargannotation) + VISIT(st, expr, a->varargannotation); + if (a->kwargannotation) + VISIT(st, expr, a->kwargannotation); + if (a->kwonlyargs && !symtable_visit_argannotations(st, a->kwonlyargs)) + return 0; + if (s->v.FunctionDef.returns) + VISIT(st, expr, s->v.FunctionDef.returns); + return 1; +} + +static int symtable_visit_arguments(struct symtable *st, arguments_ty a) { /* skip default arguments inside function block XXX should ast be different? */ - if (a->args && !symtable_visit_params(st, a->args, 1)) + if (a->args && !symtable_visit_params(st, a->args)) + return 0; + if (a->kwonlyargs && !symtable_visit_params(st, a->kwonlyargs)) return 0; if (a->vararg) { if (!symtable_add_def(st, a->vararg, DEF_PARAM)) @@ -1376,8 +1529,6 @@ symtable_visit_arguments(struct symtable *st, arguments_ty a) return 0; st->st_cur->ste_varkeywords = 1; } - if (a->args && !symtable_visit_params_nested(st, a->args)) - return 0; return 1; } @@ -1388,7 +1539,8 @@ symtable_visit_excepthandler(struct symtable *st, excepthandler_ty eh) if (eh->v.ExceptHandler.type) VISIT(st, expr, eh->v.ExceptHandler.type); if (eh->v.ExceptHandler.name) - VISIT(st, expr, eh->v.ExceptHandler.name); + if (!symtable_add_def(st, eh->v.ExceptHandler.name, DEF_LOCAL)) + return 0; VISIT_SEQ(st, stmt, eh->v.ExceptHandler.body); return 1; } @@ -1403,10 +1555,10 @@ symtable_visit_alias(struct symtable *st, alias_ty a) */ PyObject *store_name; PyObject *name = (a->asname == NULL) ? a->name : a->asname; - const char *base = PyString_AS_STRING(name); - char *dot = strchr(base, '.'); + const Py_UNICODE *base = PyUnicode_AS_UNICODE(name); + Py_UNICODE *dot = Py_UNICODE_strchr(base, '.'); if (dot) { - store_name = PyString_FromStringAndSize(base, dot - base); + store_name = PyUnicode_FromUnicode(base, dot - base); if (!store_name) return 0; } @@ -1414,19 +1566,20 @@ symtable_visit_alias(struct symtable *st, alias_ty a) store_name = name; Py_INCREF(store_name); } - if (strcmp(PyString_AS_STRING(name), "*")) { + if (PyUnicode_CompareWithASCIIString(name, "*")) { int r = symtable_add_def(st, store_name, DEF_IMPORT); Py_DECREF(store_name); return r; } else { if (st->st_cur->ste_type != ModuleBlock) { - int lineno = st->st_cur->ste_lineno; - if (!symtable_warn(st, IMPORT_STAR_WARNING, lineno)) { + int lineno = st->st_cur->ste_lineno; + int col_offset = st->st_cur->ste_col_offset; + PyErr_SetString(PyExc_SyntaxError, IMPORT_STAR_WARNING); + PyErr_SyntaxLocationEx(st->st_filename, lineno, col_offset); Py_DECREF(store_name); return 0; } - } st->st_cur->ste_unoptimized |= OPT_IMPORT_STAR; Py_DECREF(store_name); return 1; @@ -1470,30 +1623,11 @@ symtable_visit_slice(struct symtable *st, slice_ty s) case Index_kind: VISIT(st, expr, s->v.Index.value) break; - case Ellipsis_kind: - break; } return 1; } static int -symtable_new_tmpname(struct symtable *st) -{ - char tmpname[256]; - identifier tmp; - - PyOS_snprintf(tmpname, sizeof(tmpname), "_[%d]", - ++st->st_cur->ste_tmpname); - tmp = PyString_InternFromString(tmpname); - if (!tmp) - return 0; - if (!symtable_add_def(st, tmp, DEF_LOCAL)) - return 0; - Py_DECREF(tmp); - return 1; -} - -static int symtable_handle_comprehension(struct symtable *st, expr_ty e, identifier scope_name, asdl_seq *generators, expr_ty elt, expr_ty value) @@ -1506,7 +1640,8 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e, VISIT(st, expr, outermost->iter); /* Create comprehension scope for the rest */ if (!scope_name || - !symtable_enter_block(st, scope_name, FunctionBlock, (void *)e, 0)) { + !symtable_enter_block(st, scope_name, FunctionBlock, (void *)e, + e->lineno, e->col_offset)) { return 0; } st->st_cur->ste_generator = is_generator; @@ -1539,6 +1674,14 @@ symtable_visit_genexp(struct symtable *st, expr_ty e) } static int +symtable_visit_listcomp(struct symtable *st, expr_ty e) +{ + return symtable_handle_comprehension(st, e, GET_IDENTIFIER(listcomp), + e->v.ListComp.generators, + e->v.ListComp.elt, NULL); +} + +static int symtable_visit_setcomp(struct symtable *st, expr_ty e) { return symtable_handle_comprehension(st, e, GET_IDENTIFIER(setcomp), diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 814eccb157..c4f27d0868 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -15,16 +15,14 @@ Data members: */ #include "Python.h" -#include "structseq.h" #include "code.h" #include "frameobject.h" -#include "eval.h" #include "osdefs.h" #ifdef MS_WINDOWS #define WIN32_LEAN_AND_MEAN -#include "windows.h" +#include <windows.h> #endif /* MS_WINDOWS */ #ifdef MS_COREDLL @@ -37,17 +35,13 @@ extern const char *PyWin_DLLVersionString; #include <unixlib.h> #endif -#ifdef MS_WINDOWS -#include <windows.h> -#endif - #ifdef HAVE_LANGINFO_H #include <locale.h> #include <langinfo.h> #endif PyObject * -PySys_GetObject(char *name) +PySys_GetObject(const char *name) { PyThreadState *tstate = PyThreadState_GET(); PyObject *sd = tstate->interp->sysdict; @@ -56,20 +50,8 @@ PySys_GetObject(char *name) return PyDict_GetItemString(sd, name); } -FILE * -PySys_GetFile(char *name, FILE *def) -{ - FILE *fp = NULL; - PyObject *v = PySys_GetObject(name); - if (v != NULL && PyFile_Check(v)) - fp = PyFile_AsFile(v); - if (fp == NULL) - fp = def; - return fp; -} - int -PySys_SetObject(char *name, PyObject *v) +PySys_SetObject(const char *name, PyObject *v) { PyThreadState *tstate = PyThreadState_GET(); PyObject *sd = tstate->interp->sysdict; @@ -83,16 +65,79 @@ PySys_SetObject(char *name, PyObject *v) return PyDict_SetItemString(sd, name, v); } +/* Write repr(o) to sys.stdout using sys.stdout.encoding and 'backslashreplace' + error handler. If sys.stdout has a buffer attribute, use + sys.stdout.buffer.write(encoded), otherwise redecode the string and use + sys.stdout.write(redecoded). + + Helper function for sys_displayhook(). */ +static int +sys_displayhook_unencodable(PyObject *outf, PyObject *o) +{ + PyObject *stdout_encoding = NULL; + PyObject *encoded, *escaped_str, *repr_str, *buffer, *result; + char *stdout_encoding_str; + int ret; + + stdout_encoding = PyObject_GetAttrString(outf, "encoding"); + if (stdout_encoding == NULL) + goto error; + stdout_encoding_str = _PyUnicode_AsString(stdout_encoding); + if (stdout_encoding_str == NULL) + goto error; + + repr_str = PyObject_Repr(o); + if (repr_str == NULL) + goto error; + encoded = PyUnicode_AsEncodedString(repr_str, + stdout_encoding_str, + "backslashreplace"); + Py_DECREF(repr_str); + if (encoded == NULL) + goto error; + + buffer = PyObject_GetAttrString(outf, "buffer"); + if (buffer) { + result = PyObject_CallMethod(buffer, "write", "(O)", encoded); + Py_DECREF(buffer); + Py_DECREF(encoded); + if (result == NULL) + goto error; + Py_DECREF(result); + } + else { + PyErr_Clear(); + escaped_str = PyUnicode_FromEncodedObject(encoded, + stdout_encoding_str, + "strict"); + Py_DECREF(encoded); + if (PyFile_WriteObject(escaped_str, outf, Py_PRINT_RAW) != 0) { + Py_DECREF(escaped_str); + goto error; + } + Py_DECREF(escaped_str); + } + ret = 0; + goto finally; + +error: + ret = -1; +finally: + Py_XDECREF(stdout_encoding); + return ret; +} + static PyObject * sys_displayhook(PyObject *self, PyObject *o) { PyObject *outf; PyInterpreterState *interp = PyThreadState_GET()->interp; PyObject *modules = interp->modules; - PyObject *builtins = PyDict_GetItemString(modules, "__builtin__"); + PyObject *builtins = PyDict_GetItemString(modules, "builtins"); + int err; if (builtins == NULL) { - PyErr_SetString(PyExc_RuntimeError, "lost __builtin__"); + PyErr_SetString(PyExc_RuntimeError, "lost builtins module"); return NULL; } @@ -105,17 +150,25 @@ sys_displayhook(PyObject *self, PyObject *o) } if (PyObject_SetAttrString(builtins, "_", Py_None) != 0) return NULL; - if (Py_FlushLine() != 0) - return NULL; outf = PySys_GetObject("stdout"); - if (outf == NULL) { + if (outf == NULL || outf == Py_None) { PyErr_SetString(PyExc_RuntimeError, "lost sys.stdout"); return NULL; } - if (PyFile_WriteObject(o, outf, 0) != 0) - return NULL; - PyFile_SoftSpace(outf, 1); - if (Py_FlushLine() != 0) + if (PyFile_WriteObject(o, outf, 0) != 0) { + if (PyErr_ExceptionMatches(PyExc_UnicodeEncodeError)) { + /* repr(o) is not encodable to sys.stdout.encoding with + * sys.stdout.errors error handler (which is probably 'strict') */ + PyErr_Clear(); + err = sys_displayhook_unencodable(outf, o); + if (err) + return NULL; + } + else { + return NULL; + } + } + if (PyFile_WriteString("\n", outf) != 0) return NULL; if (PyObject_SetAttrString(builtins, "_", o) != 0) return NULL; @@ -126,7 +179,7 @@ sys_displayhook(PyObject *self, PyObject *o) PyDoc_STRVAR(displayhook_doc, "displayhook(object) -> None\n" "\n" -"Print an object to sys.stdout and also save it in __builtin__._\n" +"Print an object to sys.stdout and also save it in builtins._\n" ); static PyObject * @@ -167,43 +220,6 @@ clause in the current stack frame or in an older stack frame." ); static PyObject * -sys_exc_clear(PyObject *self, PyObject *noargs) -{ - PyThreadState *tstate; - PyObject *tmp_type, *tmp_value, *tmp_tb; - - if (PyErr_WarnPy3k("sys.exc_clear() not supported in 3.x; " - "use except clauses", 1) < 0) - return NULL; - - tstate = PyThreadState_GET(); - tmp_type = tstate->exc_type; - tmp_value = tstate->exc_value; - tmp_tb = tstate->exc_traceback; - tstate->exc_type = NULL; - tstate->exc_value = NULL; - tstate->exc_traceback = NULL; - Py_XDECREF(tmp_type); - Py_XDECREF(tmp_value); - Py_XDECREF(tmp_tb); - /* For b/w compatibility */ - PySys_SetObject("exc_type", Py_None); - PySys_SetObject("exc_value", Py_None); - PySys_SetObject("exc_traceback", Py_None); - Py_INCREF(Py_None); - return Py_None; -} - -PyDoc_STRVAR(exc_clear_doc, -"exc_clear() -> None\n\ -\n\ -Clear global information on the current exception. Subsequent calls to\n\ -exc_info() will return (None,None,None) until another exception is raised\n\ -in the current thread or the execution stack returns to a frame where\n\ -another exception is being handled." -); - -static PyObject * sys_exit(PyObject *self, PyObject *args) { PyObject *exit_code = 0; @@ -224,12 +240,11 @@ If it is another kind of object, it will be printed and the system\n\ exit status will be one (i.e., failure)." ); -#ifdef Py_USING_UNICODE static PyObject * sys_getdefaultencoding(PyObject *self) { - return PyString_FromString(PyUnicode_GetDefaultEncoding()); + return PyUnicode_FromString(PyUnicode_GetDefaultEncoding()); } PyDoc_STRVAR(getdefaultencoding_doc, @@ -240,30 +255,13 @@ implementation." ); static PyObject * -sys_setdefaultencoding(PyObject *self, PyObject *args) -{ - char *encoding; - if (!PyArg_ParseTuple(args, "s:setdefaultencoding", &encoding)) - return NULL; - if (PyUnicode_SetDefaultEncoding(encoding)) - return NULL; - Py_INCREF(Py_None); - return Py_None; -} - -PyDoc_STRVAR(setdefaultencoding_doc, -"setdefaultencoding(encoding)\n\ -\n\ -Set the current default string encoding used by the Unicode implementation." -); - -static PyObject * sys_getfilesystemencoding(PyObject *self) { if (Py_FileSystemDefaultEncoding) - return PyString_FromString(Py_FileSystemDefaultEncoding); - Py_INCREF(Py_None); - return Py_None; + return PyUnicode_FromString(Py_FileSystemDefaultEncoding); + PyErr_SetString(PyExc_RuntimeError, + "filesystem encoding is not initialized"); + return NULL; } PyDoc_STRVAR(getfilesystemencoding_doc, @@ -273,7 +271,32 @@ Return the encoding used to convert Unicode filenames in\n\ operating system filenames." ); -#endif +static PyObject * +sys_intern(PyObject *self, PyObject *args) +{ + PyObject *s; + if (!PyArg_ParseTuple(args, "U:intern", &s)) + return NULL; + if (PyUnicode_CheckExact(s)) { + Py_INCREF(s); + PyUnicode_InternInPlace(&s); + return s; + } + else { + PyErr_Format(PyExc_TypeError, + "can't intern %.400s", s->ob_type->tp_name); + return NULL; + } +} + +PyDoc_STRVAR(intern_doc, +"intern(string) -> string\n\ +\n\ +``Intern'' the given string. This enters the string in the (global)\n\ +table of interned strings whose purpose is to speed up dictionary lookups.\n\ +Return the string itself or the previously interned string object with the\n\ +same value."); + /* * Cached interned string objects used for calling the profile and @@ -290,7 +313,7 @@ trace_init(void) int i; for (i = 0; i < 7; ++i) { if (whatstrings[i] == NULL) { - name = PyString_InternFromString(whatnames[i]); + name = PyUnicode_InternFromString(whatnames[i]); if (name == NULL) return -1; whatstrings[i] = name; @@ -461,12 +484,18 @@ Return the profiling function set with sys.setprofile.\n\ See the profiler chapter in the library manual." ); +static int _check_interval = 100; + static PyObject * sys_setcheckinterval(PyObject *self, PyObject *args) { - if (!PyArg_ParseTuple(args, "i:setcheckinterval", &_Py_CheckInterval)) + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "sys.getcheckinterval() and sys.setcheckinterval() " + "are deprecated. Use sys.setswitchinterval() " + "instead.", 1) < 0) + return NULL; + if (!PyArg_ParseTuple(args, "i:setcheckinterval", &_check_interval)) return NULL; - _Py_Ticker = _Py_CheckInterval; Py_INCREF(Py_None); return Py_None; } @@ -481,13 +510,59 @@ n instructions. This also affects how often thread switches occur." static PyObject * sys_getcheckinterval(PyObject *self, PyObject *args) { - return PyInt_FromLong(_Py_CheckInterval); + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "sys.getcheckinterval() and sys.setcheckinterval() " + "are deprecated. Use sys.getswitchinterval() " + "instead.", 1) < 0) + return NULL; + return PyLong_FromLong(_check_interval); } PyDoc_STRVAR(getcheckinterval_doc, "getcheckinterval() -> current check interval; see setcheckinterval()." ); +#ifdef WITH_THREAD +static PyObject * +sys_setswitchinterval(PyObject *self, PyObject *args) +{ + double d; + if (!PyArg_ParseTuple(args, "d:setswitchinterval", &d)) + return NULL; + if (d <= 0.0) { + PyErr_SetString(PyExc_ValueError, + "switch interval must be strictly positive"); + return NULL; + } + _PyEval_SetSwitchInterval((unsigned long) (1e6 * d)); + Py_INCREF(Py_None); + return Py_None; +} + +PyDoc_STRVAR(setswitchinterval_doc, +"setswitchinterval(n)\n\ +\n\ +Set the ideal thread switching delay inside the Python interpreter\n\ +The actual frequency of switching threads can be lower if the\n\ +interpreter executes long sequences of uninterruptible code\n\ +(this is implementation-specific and workload-dependent).\n\ +\n\ +The parameter must represent the desired switching delay in seconds\n\ +A typical value is 0.005 (5 milliseconds)." +); + +static PyObject * +sys_getswitchinterval(PyObject *self, PyObject *args) +{ + return PyFloat_FromDouble(1e-6 * _PyEval_GetSwitchInterval()); +} + +PyDoc_STRVAR(getswitchinterval_doc, +"getswitchinterval() -> current thread switch interval; see setswitchinterval()." +); + +#endif /* WITH_THREAD */ + #ifdef WITH_TSC static PyObject * sys_settscdump(PyObject *self, PyObject *args) @@ -531,6 +606,57 @@ sys_setrecursionlimit(PyObject *self, PyObject *args) return Py_None; } +static PyTypeObject Hash_InfoType; + +PyDoc_STRVAR(hash_info_doc, +"hash_info\n\ +\n\ +A struct sequence providing parameters used for computing\n\ +numeric hashes. The attributes are read only."); + +static PyStructSequence_Field hash_info_fields[] = { + {"width", "width of the type used for hashing, in bits"}, + {"modulus", "prime number giving the modulus on which the hash " + "function is based"}, + {"inf", "value to be used for hash of a positive infinity"}, + {"nan", "value to be used for hash of a nan"}, + {"imag", "multiplier used for the imaginary part of a complex number"}, + {NULL, NULL} +}; + +static PyStructSequence_Desc hash_info_desc = { + "sys.hash_info", + hash_info_doc, + hash_info_fields, + 5, +}; + +static PyObject * +get_hash_info(void) +{ + PyObject *hash_info; + int field = 0; + hash_info = PyStructSequence_New(&Hash_InfoType); + if (hash_info == NULL) + return NULL; + PyStructSequence_SET_ITEM(hash_info, field++, + PyLong_FromLong(8*sizeof(Py_hash_t))); + PyStructSequence_SET_ITEM(hash_info, field++, + PyLong_FromSsize_t(_PyHASH_MODULUS)); + PyStructSequence_SET_ITEM(hash_info, field++, + PyLong_FromLong(_PyHASH_INF)); + PyStructSequence_SET_ITEM(hash_info, field++, + PyLong_FromLong(_PyHASH_NAN)); + PyStructSequence_SET_ITEM(hash_info, field++, + PyLong_FromLong(_PyHASH_IMAG)); + if (PyErr_Occurred()) { + Py_CLEAR(hash_info); + return NULL; + } + return hash_info; +} + + PyDoc_STRVAR(setrecursionlimit_doc, "setrecursionlimit(n)\n\ \n\ @@ -543,7 +669,7 @@ dependent." static PyObject * sys_getrecursionlimit(PyObject *self) { - return PyInt_FromLong(Py_GetRecursionLimit()); + return PyLong_FromLong(Py_GetRecursionLimit()); } PyDoc_STRVAR(getrecursionlimit_doc, @@ -606,15 +732,15 @@ sys_getwindowsversion(PyObject *self) if (version == NULL) return NULL; - PyStructSequence_SET_ITEM(version, pos++, PyInt_FromLong(ver.dwMajorVersion)); - PyStructSequence_SET_ITEM(version, pos++, PyInt_FromLong(ver.dwMinorVersion)); - PyStructSequence_SET_ITEM(version, pos++, PyInt_FromLong(ver.dwBuildNumber)); - PyStructSequence_SET_ITEM(version, pos++, PyInt_FromLong(ver.dwPlatformId)); - PyStructSequence_SET_ITEM(version, pos++, PyString_FromString(ver.szCSDVersion)); - PyStructSequence_SET_ITEM(version, pos++, PyInt_FromLong(ver.wServicePackMajor)); - PyStructSequence_SET_ITEM(version, pos++, PyInt_FromLong(ver.wServicePackMinor)); - PyStructSequence_SET_ITEM(version, pos++, PyInt_FromLong(ver.wSuiteMask)); - PyStructSequence_SET_ITEM(version, pos++, PyInt_FromLong(ver.wProductType)); + PyStructSequence_SET_ITEM(version, pos++, PyLong_FromLong(ver.dwMajorVersion)); + PyStructSequence_SET_ITEM(version, pos++, PyLong_FromLong(ver.dwMinorVersion)); + PyStructSequence_SET_ITEM(version, pos++, PyLong_FromLong(ver.dwBuildNumber)); + PyStructSequence_SET_ITEM(version, pos++, PyLong_FromLong(ver.dwPlatformId)); + PyStructSequence_SET_ITEM(version, pos++, PyUnicode_FromString(ver.szCSDVersion)); + PyStructSequence_SET_ITEM(version, pos++, PyLong_FromLong(ver.wServicePackMajor)); + PyStructSequence_SET_ITEM(version, pos++, PyLong_FromLong(ver.wServicePackMinor)); + PyStructSequence_SET_ITEM(version, pos++, PyLong_FromLong(ver.wSuiteMask)); + PyStructSequence_SET_ITEM(version, pos++, PyLong_FromLong(ver.wProductType)); return version; } @@ -654,7 +780,7 @@ sys_getdlopenflags(PyObject *self, PyObject *args) PyThreadState *tstate = PyThreadState_GET(); if (!tstate) return NULL; - return PyInt_FromLong(tstate->interp->dlopenflags); + return PyLong_FromLong(tstate->interp->dlopenflags); } PyDoc_STRVAR(getdlopenflags_doc, @@ -688,6 +814,7 @@ sys_getsizeof(PyObject *self, PyObject *args, PyObject *kwds) static PyObject *str__sizeof__ = NULL, *gc_head_size = NULL; static char *kwlist[] = {"object", "default", 0}; PyObject *o, *dflt = NULL; + PyObject *method; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|O:getsizeof", kwlist, &o, &dflt)) @@ -695,7 +822,7 @@ sys_getsizeof(PyObject *self, PyObject *args, PyObject *kwds) /* Initialize static variable for GC head size */ if (gc_head_size == NULL) { - gc_head_size = PyInt_FromSsize_t(sizeof(PyGC_Head)); + gc_head_size = PyLong_FromSsize_t(sizeof(PyGC_Head)); if (gc_head_size == NULL) return NULL; } @@ -704,26 +831,20 @@ sys_getsizeof(PyObject *self, PyObject *args, PyObject *kwds) if (PyType_Ready(Py_TYPE(o)) < 0) return NULL; - /* Instance of old-style class */ - if (PyInstance_Check(o)) - res = PyInt_FromSsize_t(PyInstance_Type.tp_basicsize); - /* all other objects */ + method = _PyObject_LookupSpecial(o, "__sizeof__", + &str__sizeof__); + if (method == NULL) { + if (!PyErr_Occurred()) + PyErr_Format(PyExc_TypeError, + "Type %.100s doesn't define __sizeof__", + Py_TYPE(o)->tp_name); + } else { - PyObject *method = _PyObject_LookupSpecial(o, "__sizeof__", - &str__sizeof__); - if (method == NULL) { - if (!PyErr_Occurred()) - PyErr_Format(PyExc_TypeError, - "Type %.100s doesn't define __sizeof__", - Py_TYPE(o)->tp_name); - } - else { - res = PyObject_CallFunctionObjArgs(method, NULL); - Py_DECREF(method); - } + res = PyObject_CallFunctionObjArgs(method, NULL); + Py_DECREF(method); } - /* Has a default value been given? */ + /* Has a default value been given */ if ((res == NULL) && (dflt != NULL) && PyErr_ExceptionMatches(PyExc_TypeError)) { @@ -751,14 +872,14 @@ Return the size of object in bytes."); static PyObject * sys_getrefcount(PyObject *self, PyObject *arg) { - return PyInt_FromSsize_t(arg->ob_refcnt); + return PyLong_FromSsize_t(arg->ob_refcnt); } #ifdef Py_REF_DEBUG static PyObject * sys_gettotalrefcount(PyObject *self) { - return PyInt_FromSsize_t(_Py_GetRefTotal()); + return PyLong_FromSsize_t(_Py_GetRefTotal()); } #endif /* Py_REF_DEBUG */ @@ -909,13 +1030,10 @@ static PyMethodDef sys_methods[] = { current_frames_doc}, {"displayhook", sys_displayhook, METH_O, displayhook_doc}, {"exc_info", sys_exc_info, METH_NOARGS, exc_info_doc}, - {"exc_clear", sys_exc_clear, METH_NOARGS, exc_clear_doc}, {"excepthook", sys_excepthook, METH_VARARGS, excepthook_doc}, {"exit", sys_exit, METH_VARARGS, exit_doc}, -#ifdef Py_USING_UNICODE {"getdefaultencoding", (PyCFunction)sys_getdefaultencoding, METH_NOARGS, getdefaultencoding_doc}, -#endif #ifdef HAVE_DLOPEN {"getdlopenflags", (PyCFunction)sys_getdlopenflags, METH_NOARGS, getdlopenflags_doc}, @@ -926,10 +1044,8 @@ static PyMethodDef sys_methods[] = { #ifdef DYNAMIC_EXECUTION_PROFILE {"getdxp", _Py_GetDXProfile, METH_VARARGS}, #endif -#ifdef Py_USING_UNICODE {"getfilesystemencoding", (PyCFunction)sys_getfilesystemencoding, METH_NOARGS, getfilesystemencoding_doc}, -#endif #ifdef Py_TRACE_REFS {"getobjects", _Py_GetObjects, METH_VARARGS}, #endif @@ -946,17 +1062,20 @@ static PyMethodDef sys_methods[] = { {"getwindowsversion", (PyCFunction)sys_getwindowsversion, METH_NOARGS, getwindowsversion_doc}, #endif /* MS_WINDOWS */ + {"intern", sys_intern, METH_VARARGS, intern_doc}, #ifdef USE_MALLOPT {"mdebug", sys_mdebug, METH_VARARGS}, #endif -#ifdef Py_USING_UNICODE - {"setdefaultencoding", sys_setdefaultencoding, METH_VARARGS, - setdefaultencoding_doc}, -#endif {"setcheckinterval", sys_setcheckinterval, METH_VARARGS, setcheckinterval_doc}, {"getcheckinterval", sys_getcheckinterval, METH_NOARGS, getcheckinterval_doc}, +#ifdef WITH_THREAD + {"setswitchinterval", sys_setswitchinterval, METH_VARARGS, + setswitchinterval_doc}, + {"getswitchinterval", sys_getswitchinterval, METH_NOARGS, + getswitchinterval_doc}, +#endif #ifdef HAVE_DLOPEN {"setdlopenflags", sys_setdlopenflags, METH_VARARGS, setdlopenflags_doc}, @@ -982,7 +1101,7 @@ list_builtin_module_names(void) if (list == NULL) return NULL; for (i = 0; PyImport_Inittab[i].name != NULL; i++) { - PyObject *name = PyString_FromString( + PyObject *name = PyUnicode_FromString( PyImport_Inittab[i].name); if (name == NULL) break; @@ -1012,21 +1131,26 @@ PySys_ResetWarnOptions(void) } void -PySys_AddWarnOption(char *s) +PySys_AddWarnOptionUnicode(PyObject *unicode) { - PyObject *str; - if (warnoptions == NULL || !PyList_Check(warnoptions)) { Py_XDECREF(warnoptions); warnoptions = PyList_New(0); if (warnoptions == NULL) return; } - str = PyString_FromString(s); - if (str != NULL) { - PyList_Append(warnoptions, str); - Py_DECREF(str); - } + PyList_Append(warnoptions, unicode); +} + +void +PySys_AddWarnOption(const wchar_t *s) +{ + PyObject *unicode; + unicode = PyUnicode_FromWideChar(s, -1); + if (unicode == NULL) + return; + PySys_AddWarnOptionUnicode(unicode); + Py_DECREF(unicode); } int @@ -1035,6 +1159,61 @@ PySys_HasWarnOptions(void) return (warnoptions != NULL && (PyList_Size(warnoptions) > 0)) ? 1 : 0; } +static PyObject *xoptions = NULL; + +static PyObject * +get_xoptions(void) +{ + if (xoptions == NULL || !PyDict_Check(xoptions)) { + Py_XDECREF(xoptions); + xoptions = PyDict_New(); + } + return xoptions; +} + +void +PySys_AddXOption(const wchar_t *s) +{ + PyObject *opts; + PyObject *name = NULL, *value = NULL; + const wchar_t *name_end; + int r; + + opts = get_xoptions(); + if (opts == NULL) + goto error; + + name_end = wcschr(s, L'='); + if (!name_end) { + name = PyUnicode_FromWideChar(s, -1); + value = Py_True; + Py_INCREF(value); + } + else { + name = PyUnicode_FromWideChar(s, name_end - s); + value = PyUnicode_FromWideChar(name_end + 1, -1); + } + if (name == NULL || value == NULL) + goto error; + r = PyDict_SetItem(opts, name, value); + Py_DECREF(name); + Py_DECREF(value); + return; + +error: + Py_XDECREF(name); + Py_XDECREF(value); + /* No return value, therefore clear error state if possible */ + if (_Py_atomic_load_relaxed(&_PyThreadState_Current)) + PyErr_Clear(); +} + +PyObject * +PySys_GetXOptions(void) +{ + return get_xoptions(); +} + /* XXX This doc string is too long to be a single string literal in VC++ 5.0. Two literals concatenated works just fine. If you have a K&R compiler or other abomination that however *does* understand longer strings, @@ -1055,11 +1234,8 @@ excepthook -- called to handle any uncaught exception other than SystemExit\n\ To customize printing in an interactive session or to install a custom\n\ top-level exception handler, assign other functions to replace these.\n\ \n\ -exitfunc -- if sys.exitfunc exists, this routine is called when Python exits\n\ - Assigning to sys.exitfunc is deprecated; use the atexit module instead.\n\ -\n\ -stdin -- standard input file object; used by raw_input() and input()\n\ -stdout -- standard output file object; used by the print statement\n\ +stdin -- standard input file object; used by input()\n\ +stdout -- standard output file object; used by print()\n\ stderr -- standard error object; used for error messages\n\ By assigning other file objects (or objects that behave like files)\n\ to these, it is possible to redirect all of the interpreter's I/O.\n\ @@ -1069,12 +1245,6 @@ last_value -- value of last uncaught exception\n\ last_traceback -- traceback of last uncaught exception\n\ These three are only available in an interactive session after a\n\ traceback has been printed.\n\ -\n\ -exc_type -- type of exception currently being handled\n\ -exc_value -- value of exception currently being handled\n\ -exc_traceback -- traceback of exception currently being handled\n\ - The function exc_info() should be used instead of these three,\n\ - because it is thread-safe.\n\ " ) /* concatenating string here */ @@ -1082,12 +1252,12 @@ PyDoc_STR( "\n\ Static objects:\n\ \n\ -float_info -- a dict with information about the float inplementation.\n\ -long_info -- a struct sequence with information about the long implementation.\n\ -maxint -- the largest supported integer (the smallest is -maxint-1)\n\ +float_info -- a dict with information about the float implementation.\n\ +int_info -- a struct sequence with information about the int implementation.\n\ maxsize -- the largest supported length of containers.\n\ maxunicode -- the largest supported character\n\ builtin_module_names -- tuple of module names built into this interpreter\n\ +subversion -- subversion information of the build as tuple\n\ version -- the version of this interpreter as a string\n\ version_info -- version information as a named tuple\n\ hexversion -- version information encoded as a single integer\n\ @@ -1116,10 +1286,9 @@ __excepthook__ -- the original excepthook; don't touch!\n\ \n\ Functions:\n\ \n\ -displayhook() -- print an object to the screen, and save it in __builtin__._\n\ +displayhook() -- print an object to the screen, and save it in builtins._\n\ excepthook() -- print an exception and its traceback to sys.stderr\n\ exc_info() -- return thread-safe information about the current exception\n\ -exc_clear() -- clear the exception state for the current thread\n\ exit() -- exit the interpreter by raising SystemExit\n\ getdlopenflags() -- returns flags to be used for dlopen() calls\n\ getprofile() -- get the global profiling function\n\ @@ -1136,13 +1305,6 @@ settrace() -- set the global debug tracing function\n\ ) /* end of sys_doc */ ; -static int -_check_and_flush (FILE *stream) -{ - int prev_fail = ferror (stream); - return fflush (stream) || prev_fail ? EOF : 0; -} - /* Subversion branch and revision management */ static int svn_initialized; static char patchlevel_revision[50]; /* Just the number */ @@ -1155,12 +1317,12 @@ svnversion_init(void) { if (svn_initialized) return; + svn_initialized = 1; *patchlevel_revision = '\0'; strcpy(branch, ""); strcpy(shortbranch, "unknown"); svn_revision = ""; - return; } /* Return svnversion output if available. @@ -1186,13 +1348,11 @@ PyDoc_STRVAR(flags__doc__, \n\ Flags provided through command line arguments or environment vars."); -static PyTypeObject FlagsType = {0, 0, 0, 0, 0, 0}; +static PyTypeObject FlagsType; static PyStructSequence_Field flags_fields[] = { {"debug", "-d"}, - {"py3k_warning", "-3"}, {"division_warning", "-Q"}, - {"division_new", "-Qnew"}, {"inspect", "-i"}, {"interactive", "-i"}, {"optimize", "-O or -OO"}, @@ -1200,16 +1360,15 @@ static PyStructSequence_Field flags_fields[] = { {"no_user_site", "-s"}, {"no_site", "-S"}, {"ignore_environment", "-E"}, - {"tabcheck", "-t or -tt"}, {"verbose", "-v"}, #ifdef RISCOS {"riscos_wimp", "???"}, #endif /* {"unbuffered", "-u"}, */ - {"unicode", "-U"}, /* {"skip_first", "-x"}, */ - {"bytes_warning", "-b"}, - {"hash_randomization", "-R"}, + {"bytes_warning", "-b"}, + {"quiet", "-q"}, + {"hash_randomization", "-R"}, {0} }; @@ -1218,9 +1377,9 @@ static PyStructSequence_Desc flags_desc = { flags__doc__, /* doc */ flags_fields, /* fields */ #ifdef RISCOS - 17 + 14 #else - 16 + 13 #endif }; @@ -1235,12 +1394,10 @@ make_flags(void) return NULL; #define SetFlag(flag) \ - PyStructSequence_SET_ITEM(seq, pos++, PyInt_FromLong(flag)) + PyStructSequence_SET_ITEM(seq, pos++, PyLong_FromLong(flag)) SetFlag(Py_DebugFlag); - SetFlag(Py_Py3kWarningFlag); SetFlag(Py_DivisionWarningFlag); - SetFlag(_Py_QnewFlag); SetFlag(Py_InspectFlag); SetFlag(Py_InteractiveFlag); SetFlag(Py_OptimizeFlag); @@ -1248,15 +1405,14 @@ make_flags(void) SetFlag(Py_NoUserSiteDirectory); SetFlag(Py_NoSiteFlag); SetFlag(Py_IgnoreEnvironmentFlag); - SetFlag(Py_TabcheckFlag); SetFlag(Py_VerboseFlag); #ifdef RISCOS SetFlag(Py_RISCOSWimpFlag); #endif /* SetFlag(saw_unbuffered_flag); */ - SetFlag(Py_UnicodeFlag); /* SetFlag(skipfirstline); */ SetFlag(Py_BytesWarningFlag); + SetFlag(Py_QuietFlag); SetFlag(Py_HashRandomizationFlag); #undef SetFlag @@ -1271,7 +1427,7 @@ PyDoc_STRVAR(version_info__doc__, \n\ Version information as a named tuple."); -static PyTypeObject VersionInfoType = {0, 0, 0, 0, 0, 0}; +static PyTypeObject VersionInfoType; static PyStructSequence_Field version_info_fields[] = { {"major", "Major release number"}, @@ -1316,9 +1472,9 @@ make_version_info(void) #endif #define SetIntItem(flag) \ - PyStructSequence_SET_ITEM(version_info, pos++, PyInt_FromLong(flag)) + PyStructSequence_SET_ITEM(version_info, pos++, PyLong_FromLong(flag)) #define SetStrItem(flag) \ - PyStructSequence_SET_ITEM(version_info, pos++, PyString_FromString(flag)) + PyStructSequence_SET_ITEM(version_info, pos++, PyUnicode_FromString(flag)) SetIntItem(PY_MAJOR_VERSION); SetIntItem(PY_MINOR_VERSION); @@ -1335,14 +1491,25 @@ make_version_info(void) return version_info; } +static struct PyModuleDef sysmodule = { + PyModuleDef_HEAD_INIT, + "sys", + sys_doc, + -1, /* multiple "initialization" just copies the module dict. */ + sys_methods, + NULL, + NULL, + NULL, + NULL +}; + PyObject * _PySys_Init(void) { PyObject *m, *v, *sysdict; - PyObject *sysin, *sysout, *syserr; char *s; - m = Py_InitModule3("sys", sys_methods, sys_doc); + m = PyModule_Create(&sysmodule); if (m == NULL) return NULL; sysdict = PyModule_GetDict(m); @@ -1370,43 +1537,19 @@ _PySys_Init(void) } #endif - /* Closing the standard FILE* if sys.std* goes aways causes problems - * for embedded Python usages. Closing them when somebody explicitly - * invokes .close() might be possible, but the FAQ promises they get - * never closed. However, we still need to get write errors when - * writing fails (e.g. because stdout is redirected), so we flush the - * streams and check for errors before the file objects are deleted. - * On OS X, fflush()ing stdin causes an error, so we exempt stdin - * from that procedure. - */ - sysin = PyFile_FromFile(stdin, "<stdin>", "r", NULL); - sysout = PyFile_FromFile(stdout, "<stdout>", "w", _check_and_flush); - syserr = PyFile_FromFile(stderr, "<stderr>", "w", _check_and_flush); - if (PyErr_Occurred()) - return NULL; + /* stdin/stdout/stderr are now set by pythonrun.c */ - PyDict_SetItemString(sysdict, "stdin", sysin); - PyDict_SetItemString(sysdict, "stdout", sysout); - PyDict_SetItemString(sysdict, "stderr", syserr); - /* Make backup copies for cleanup */ - PyDict_SetItemString(sysdict, "__stdin__", sysin); - PyDict_SetItemString(sysdict, "__stdout__", sysout); - PyDict_SetItemString(sysdict, "__stderr__", syserr); PyDict_SetItemString(sysdict, "__displayhook__", PyDict_GetItemString(sysdict, "displayhook")); PyDict_SetItemString(sysdict, "__excepthook__", PyDict_GetItemString(sysdict, "excepthook")); - Py_XDECREF(sysin); - Py_XDECREF(sysout); - Py_XDECREF(syserr); - SET_SYS_FROM_STRING("version", - PyString_FromString(Py_GetVersion())); + PyUnicode_FromString(Py_GetVersion())); SET_SYS_FROM_STRING("hexversion", - PyInt_FromLong(PY_VERSION_HEX)); + PyLong_FromLong(PY_VERSION_HEX)); svnversion_init(); SET_SYS_FROM_STRING("subversion", - Py_BuildValue("(ssz)", "CPython", branch, + Py_BuildValue("(sss)", "CPython", branch, svn_revision)); SET_SYS_FROM_STRING("_mercurial", Py_BuildValue("(szz)", "CPython", _Py_hgidentifier(), @@ -1414,31 +1557,31 @@ _PySys_Init(void) SET_SYS_FROM_STRING("dont_write_bytecode", PyBool_FromLong(Py_DontWriteBytecodeFlag)); SET_SYS_FROM_STRING("api_version", - PyInt_FromLong(PYTHON_API_VERSION)); + PyLong_FromLong(PYTHON_API_VERSION)); SET_SYS_FROM_STRING("copyright", - PyString_FromString(Py_GetCopyright())); + PyUnicode_FromString(Py_GetCopyright())); SET_SYS_FROM_STRING("platform", - PyString_FromString(Py_GetPlatform())); + PyUnicode_FromString(Py_GetPlatform())); SET_SYS_FROM_STRING("executable", - PyString_FromString(Py_GetProgramFullPath())); + PyUnicode_FromWideChar( + Py_GetProgramFullPath(), -1)); SET_SYS_FROM_STRING("prefix", - PyString_FromString(Py_GetPrefix())); + PyUnicode_FromWideChar(Py_GetPrefix(), -1)); SET_SYS_FROM_STRING("exec_prefix", - PyString_FromString(Py_GetExecPrefix())); + PyUnicode_FromWideChar(Py_GetExecPrefix(), -1)); SET_SYS_FROM_STRING("maxsize", - PyInt_FromSsize_t(PY_SSIZE_T_MAX)); - SET_SYS_FROM_STRING("maxint", - PyInt_FromLong(PyInt_GetMax())); - SET_SYS_FROM_STRING("py3kwarning", - PyBool_FromLong(Py_Py3kWarningFlag)); + PyLong_FromSsize_t(PY_SSIZE_T_MAX)); SET_SYS_FROM_STRING("float_info", PyFloat_GetInfo()); - SET_SYS_FROM_STRING("long_info", + SET_SYS_FROM_STRING("int_info", PyLong_GetInfo()); -#ifdef Py_USING_UNICODE + /* initialize hash_info */ + if (Hash_InfoType.tp_name == 0) + PyStructSequence_InitType(&Hash_InfoType, &hash_info_desc); + SET_SYS_FROM_STRING("hash_info", + get_hash_info()); SET_SYS_FROM_STRING("maxunicode", - PyInt_FromLong(PyUnicode_GetMax())); -#endif + PyLong_FromLong(PyUnicode_GetMax())); SET_SYS_FROM_STRING("builtin_module_names", list_builtin_module_names()); { @@ -1453,13 +1596,17 @@ _PySys_Init(void) else value = "little"; SET_SYS_FROM_STRING("byteorder", - PyString_FromString(value)); + PyUnicode_FromString(value)); } #ifdef MS_COREDLL SET_SYS_FROM_STRING("dllhandle", PyLong_FromVoidPtr(PyWin_DLLhModule)); SET_SYS_FROM_STRING("winver", - PyString_FromString(PyWin_DLLVersionString)); + PyUnicode_FromString(PyWin_DLLVersionString)); +#endif +#ifdef ABIFLAGS + SET_SYS_FROM_STRING("abiflags", + PyUnicode_FromString(ABIFLAGS)); #endif if (warnoptions == NULL) { warnoptions = PyList_New(0); @@ -1471,6 +1618,11 @@ _PySys_Init(void) PyDict_SetItemString(sysdict, "warnoptions", warnoptions); } + v = get_xoptions(); + if (v != NULL) { + PyDict_SetItemString(sysdict, "_xoptions", v); + } + /* version_info */ if (VersionInfoType.tp_name == 0) PyStructSequence_InitType(&VersionInfoType, &version_info_desc); @@ -1500,10 +1652,10 @@ _PySys_Init(void) /* float repr style: 0.03 (short) vs 0.029999999999999999 (legacy) */ #ifndef PY_NO_SHORT_FLOAT_REPR SET_SYS_FROM_STRING("float_repr_style", - PyString_FromString("short")); + PyUnicode_FromString("short")); #else SET_SYS_FROM_STRING("float_repr_style", - PyString_FromString("legacy")); + PyUnicode_FromString("legacy")); #endif #undef SET_SYS_FROM_STRING @@ -1513,15 +1665,15 @@ _PySys_Init(void) } static PyObject * -makepathobject(char *path, int delim) +makepathobject(const wchar_t *path, wchar_t delim) { int i, n; - char *p; + const wchar_t *p; PyObject *v, *w; n = 1; p = path; - while ((p = strchr(p, delim)) != NULL) { + while ((p = wcschr(p, delim)) != NULL) { n++; p++; } @@ -1529,10 +1681,10 @@ makepathobject(char *path, int delim) if (v == NULL) return NULL; for (i = 0; ; i++) { - p = strchr(path, delim); + p = wcschr(path, delim); if (p == NULL) - p = strchr(path, '\0'); /* End of string */ - w = PyString_FromStringAndSize(path, (Py_ssize_t) (p - path)); + p = path + wcslen(path); /* End of string */ + w = PyUnicode_FromWideChar(path, (Py_ssize_t)(p - path)); if (w == NULL) { Py_DECREF(v); return NULL; @@ -1546,7 +1698,7 @@ makepathobject(char *path, int delim) } void -PySys_SetPath(char *path) +PySys_SetPath(const wchar_t *path) { PyObject *v; if ((v = makepathobject(path, DELIM)) == NULL) @@ -1557,12 +1709,12 @@ PySys_SetPath(char *path) } static PyObject * -makeargvobject(int argc, char **argv) +makeargvobject(int argc, wchar_t **argv) { PyObject *av; if (argc <= 0 || argv == NULL) { /* Ensure at least one (empty) argument is seen */ - static char *empty_argv[1] = {""}; + static wchar_t *empty_argv[1] = {L""}; argv = empty_argv; argc = 1; } @@ -1577,14 +1729,14 @@ makeargvobject(int argc, char **argv) if (i == 0) { char* fn = decc$translate_vms(argv[0]); if ((fn == (char *)0) || fn == (char *)-1) - v = PyString_FromString(argv[0]); + v = PyUnicode_FromString(argv[0]); else - v = PyString_FromString( + v = PyUnicode_FromString( decc$translate_vms(argv[0])); } else - v = PyString_FromString(argv[i]); + v = PyUnicode_FromString(argv[i]); #else - PyObject *v = PyString_FromString(argv[i]); + PyObject *v = PyUnicode_FromWideChar(argv[i], -1); #endif if (v == NULL) { Py_DECREF(av); @@ -1597,116 +1749,184 @@ makeargvobject(int argc, char **argv) return av; } -void -PySys_SetArgvEx(int argc, char **argv, int updatepath) +#define _HAVE_SCRIPT_ARGUMENT(argc, argv) \ + (argc > 0 && argv0 != NULL && \ + wcscmp(argv0, L"-c") != 0 && wcscmp(argv0, L"-m") != 0) + +static void +sys_update_path(int argc, wchar_t **argv) { + wchar_t *argv0; + wchar_t *p = NULL; + Py_ssize_t n = 0; + PyObject *a; + PyObject *path; +#ifdef HAVE_READLINK + wchar_t link[MAXPATHLEN+1]; + wchar_t argv0copy[2*MAXPATHLEN+1]; + int nr = 0; +#endif #if defined(HAVE_REALPATH) - char fullpath[MAXPATHLEN]; + wchar_t fullpath[MAXPATHLEN]; #elif defined(MS_WINDOWS) && !defined(MS_WINCE) - char fullpath[MAX_PATH]; + wchar_t fullpath[MAX_PATH]; #endif - PyObject *av = makeargvobject(argc, argv); - PyObject *path = PySys_GetObject("path"); - if (av == NULL) - Py_FatalError("no mem for sys.argv"); - if (PySys_SetObject("argv", av) != 0) - Py_FatalError("can't assign sys.argv"); - if (updatepath && path != NULL) { - char *argv0 = argv[0]; - char *p = NULL; - Py_ssize_t n = 0; - PyObject *a; + + path = PySys_GetObject("path"); + if (path == NULL) + return; + + argv0 = argv[0]; + #ifdef HAVE_READLINK - char link[MAXPATHLEN+1]; - char argv0copy[2*MAXPATHLEN+1]; - int nr = 0; - if (argc > 0 && argv0 != NULL && strcmp(argv0, "-c") != 0) - nr = readlink(argv0, link, MAXPATHLEN); - if (nr > 0) { - /* It's a symlink */ - link[nr] = '\0'; - if (link[0] == SEP) - argv0 = link; /* Link to absolute path */ - else if (strchr(link, SEP) == NULL) - ; /* Link without path */ + if (_HAVE_SCRIPT_ARGUMENT(argc, argv)) + nr = _Py_wreadlink(argv0, link, MAXPATHLEN); + if (nr > 0) { + /* It's a symlink */ + link[nr] = '\0'; + if (link[0] == SEP) + argv0 = link; /* Link to absolute path */ + else if (wcschr(link, SEP) == NULL) + ; /* Link without path */ + else { + /* Must join(dirname(argv0), link) */ + wchar_t *q = wcsrchr(argv0, SEP); + if (q == NULL) + argv0 = link; /* argv0 without path */ else { - /* Must join(dirname(argv0), link) */ - char *q = strrchr(argv0, SEP); - if (q == NULL) - argv0 = link; /* argv0 without path */ - else { - /* Must make a copy */ - strcpy(argv0copy, argv0); - q = strrchr(argv0copy, SEP); - strcpy(q+1, link); - argv0 = argv0copy; - } + /* Must make a copy */ + wcscpy(argv0copy, argv0); + q = wcsrchr(argv0copy, SEP); + wcscpy(q+1, link); + argv0 = argv0copy; } } + } #endif /* HAVE_READLINK */ #if SEP == '\\' /* Special case for MS filename syntax */ - if (argc > 0 && argv0 != NULL && strcmp(argv0, "-c") != 0) { - char *q; + if (_HAVE_SCRIPT_ARGUMENT(argc, argv)) { + wchar_t *q; #if defined(MS_WINDOWS) && !defined(MS_WINCE) - /* This code here replaces the first element in argv with the full - path that it represents. Under CE, there are no relative paths so - the argument must be the full path anyway. */ - char *ptemp; - if (GetFullPathName(argv0, - sizeof(fullpath), - fullpath, - &ptemp)) { - argv0 = fullpath; - } + /* This code here replaces the first element in argv with the full + path that it represents. Under CE, there are no relative paths so + the argument must be the full path anyway. */ + wchar_t *ptemp; + if (GetFullPathNameW(argv0, + sizeof(fullpath)/sizeof(fullpath[0]), + fullpath, + &ptemp)) { + argv0 = fullpath; + } #endif - p = strrchr(argv0, SEP); - /* Test for alternate separator */ - q = strrchr(p ? p : argv0, '/'); - if (q != NULL) - p = q; - if (p != NULL) { - n = p + 1 - argv0; - if (n > 1 && p[-1] != ':') - n--; /* Drop trailing separator */ - } + p = wcsrchr(argv0, SEP); + /* Test for alternate separator */ + q = wcsrchr(p ? p : argv0, '/'); + if (q != NULL) + p = q; + if (p != NULL) { + n = p + 1 - argv0; + if (n > 1 && p[-1] != ':') + n--; /* Drop trailing separator */ } + } #else /* All other filename syntaxes */ - if (argc > 0 && argv0 != NULL && strcmp(argv0, "-c") != 0) { + if (_HAVE_SCRIPT_ARGUMENT(argc, argv)) { #if defined(HAVE_REALPATH) - if (realpath(argv0, fullpath)) { - argv0 = fullpath; - } -#endif - p = strrchr(argv0, SEP); + if (_Py_wrealpath(argv0, fullpath, PATH_MAX)) { + argv0 = fullpath; } - if (p != NULL) { -#ifndef RISCOS - n = p + 1 - argv0; -#else /* don't include trailing separator */ - n = p - argv0; -#endif /* RISCOS */ +#endif + p = wcsrchr(argv0, SEP); + } + if (p != NULL) { + n = p + 1 - argv0; #if SEP == '/' /* Special case for Unix filename syntax */ - if (n > 1) - n--; /* Drop trailing separator */ + if (n > 1) + n--; /* Drop trailing separator */ #endif /* Unix */ - } -#endif /* All others */ - a = PyString_FromStringAndSize(argv0, n); - if (a == NULL) - Py_FatalError("no mem for sys.path insertion"); - if (PyList_Insert(path, 0, a) < 0) - Py_FatalError("sys.path.insert(0) failed"); - Py_DECREF(a); } +#endif /* All others */ + a = PyUnicode_FromWideChar(argv0, n); + if (a == NULL) + Py_FatalError("no mem for sys.path insertion"); + if (PyList_Insert(path, 0, a) < 0) + Py_FatalError("sys.path.insert(0) failed"); + Py_DECREF(a); +} + +void +PySys_SetArgvEx(int argc, wchar_t **argv, int updatepath) +{ + PyObject *av = makeargvobject(argc, argv); + if (av == NULL) + Py_FatalError("no mem for sys.argv"); + if (PySys_SetObject("argv", av) != 0) + Py_FatalError("can't assign sys.argv"); Py_DECREF(av); + if (updatepath) + sys_update_path(argc, argv); } void -PySys_SetArgv(int argc, char **argv) +PySys_SetArgv(int argc, wchar_t **argv) { PySys_SetArgvEx(argc, argv, 1); } +/* Reimplementation of PyFile_WriteString() no calling indirectly + PyErr_CheckSignals(): avoid the call to PyObject_Str(). */ + +static int +sys_pyfile_write_unicode(PyObject *unicode, PyObject *file) +{ + PyObject *writer = NULL, *args = NULL, *result = NULL; + int err; + + if (file == NULL) + return -1; + + writer = PyObject_GetAttrString(file, "write"); + if (writer == NULL) + goto error; + + args = PyTuple_Pack(1, unicode); + if (args == NULL) + goto error; + + result = PyEval_CallObject(writer, args); + if (result == NULL) { + goto error; + } else { + err = 0; + goto finally; + } + +error: + err = -1; +finally: + Py_XDECREF(writer); + Py_XDECREF(args); + Py_XDECREF(result); + return err; +} + +static int +sys_pyfile_write(const char *text, PyObject *file) +{ + PyObject *unicode = NULL; + int err; + + if (file == NULL) + return -1; + + unicode = PyUnicode_FromString(text); + if (unicode == NULL) + return -1; + + err = sys_pyfile_write_unicode(unicode, file); + Py_DECREF(unicode); + return err; +} /* APIs to write to sys.stdout or sys.stderr using a printf-like interface. Adapted from code submitted by Just van Rossum. @@ -1718,6 +1938,10 @@ PySys_SetArgv(int argc, char **argv) there is a problem, they write to the real (C level) stdout or stderr; no exceptions are raised. + PyErr_CheckSignals() is not called to avoid the execution of the Python + signal handlers: they may raise a new exception whereas sys_write() + ignores all exceptions. + Both take a printf-style format string as their first argument followed by a variable length argument list determined by the format string. @@ -1733,30 +1957,24 @@ PySys_SetArgv(int argc, char **argv) */ static void -mywrite(char *name, FILE *fp, const char *format, va_list va) +sys_write(char *name, FILE *fp, const char *format, va_list va) { PyObject *file; PyObject *error_type, *error_value, *error_traceback; + char buffer[1001]; + int written; PyErr_Fetch(&error_type, &error_value, &error_traceback); file = PySys_GetObject(name); - if (file == NULL || PyFile_AsFile(file) == fp) - vfprintf(fp, format, va); - else { - char buffer[1001]; - const int written = PyOS_vsnprintf(buffer, sizeof(buffer), - format, va); - if (PyFile_WriteString(buffer, file) != 0) { - PyErr_Clear(); - fputs(buffer, fp); - } - if (written < 0 || (size_t)written >= sizeof(buffer)) { - const char *truncated = "... truncated"; - if (PyFile_WriteString(truncated, file) != 0) { - PyErr_Clear(); - fputs(truncated, fp); - } - } + written = PyOS_vsnprintf(buffer, sizeof(buffer), format, va); + if (sys_pyfile_write(buffer, file) != 0) { + PyErr_Clear(); + fputs(buffer, fp); + } + if (written < 0 || (size_t)written >= sizeof(buffer)) { + const char *truncated = "... truncated"; + if (sys_pyfile_write(truncated, file) != 0) + fputs(truncated, fp); } PyErr_Restore(error_type, error_value, error_traceback); } @@ -1767,7 +1985,7 @@ PySys_WriteStdout(const char *format, ...) va_list va; va_start(va, format); - mywrite("stdout", stdout, format, va); + sys_write("stdout", stdout, format, va); va_end(va); } @@ -1777,6 +1995,48 @@ PySys_WriteStderr(const char *format, ...) va_list va; va_start(va, format); - mywrite("stderr", stderr, format, va); + sys_write("stderr", stderr, format, va); + va_end(va); +} + +static void +sys_format(char *name, FILE *fp, const char *format, va_list va) +{ + PyObject *file, *message; + PyObject *error_type, *error_value, *error_traceback; + char *utf8; + + PyErr_Fetch(&error_type, &error_value, &error_traceback); + file = PySys_GetObject(name); + message = PyUnicode_FromFormatV(format, va); + if (message != NULL) { + if (sys_pyfile_write_unicode(message, file) != 0) { + PyErr_Clear(); + utf8 = _PyUnicode_AsString(message); + if (utf8 != NULL) + fputs(utf8, fp); + } + Py_DECREF(message); + } + PyErr_Restore(error_type, error_value, error_traceback); +} + +void +PySys_FormatStdout(const char *format, ...) +{ + va_list va; + + va_start(va, format); + sys_format("stdout", stdout, format, va); + va_end(va); +} + +void +PySys_FormatStderr(const char *format, ...) +{ + va_list va; + + va_start(va, format); + sys_format("stderr", stderr, format, va); va_end(va); } diff --git a/Python/thread.c b/Python/thread.c index dd333e8f94..d224046e64 100644 --- a/Python/thread.c +++ b/Python/thread.c @@ -23,12 +23,6 @@ #include <stdlib.h> -#ifdef __sgi -#ifndef HAVE_PTHREAD_H /* XXX Need to check in configure.ac */ -#undef _POSIX_THREADS -#endif -#endif - #include "pythread.h" #ifndef _POSIX_THREADS @@ -101,6 +95,7 @@ PyThread_init_thread(void) static size_t _pythread_stacksize = 0; #ifdef SGI_THREADS +#error SGI Irix threads are now unsupported, and code will be removed in 3.3. #include "thread_sgi.h" #endif @@ -109,10 +104,12 @@ static size_t _pythread_stacksize = 0; #endif #ifdef SUN_LWP +#error SunOS lightweight processes are now unsupported, and code will be removed in 3.3. #include "thread_lwp.h" #endif #ifdef HAVE_PTH +#error GNU pth threads are now unsupported, and code will be removed in 3.3. #include "thread_pth.h" #undef _POSIX_THREADS #endif @@ -122,6 +119,7 @@ static size_t _pythread_stacksize = 0; #endif #ifdef C_THREADS +#error Mach C Threads are now unsupported, and code will be removed in 3.3. #include "thread_cthread.h" #endif @@ -133,18 +131,10 @@ static size_t _pythread_stacksize = 0; #include "thread_os2.h" #endif -#ifdef BEOS_THREADS -#include "thread_beos.h" -#endif - #ifdef PLAN9_THREADS #include "thread_plan9.h" #endif -#ifdef ATHEOS_THREADS -#include "thread_atheos.h" -#endif - /* #ifdef FOOBAR_THREADS #include "thread_foobar.h" diff --git a/Python/thread_atheos.h b/Python/thread_atheos.h deleted file mode 100644 index 230594f144..0000000000 --- a/Python/thread_atheos.h +++ /dev/null @@ -1,259 +0,0 @@ -/* Threading for AtheOS. - Based on thread_beos.h. */ - -#include <atheos/threads.h> -#include <atheos/semaphore.h> -#include <atheos/atomic.h> -#include <errno.h> -#include <string.h> - -/* Missing decl from threads.h */ -extern int exit_thread(int); - - -/* Undefine FASTLOCK to play with simple semaphores. */ -#define FASTLOCK - - -#ifdef FASTLOCK - -/* Use an atomic counter and a semaphore for maximum speed. */ -typedef struct fastmutex { - sem_id sem; - atomic_t count; -} fastmutex_t; - - -static int fastmutex_create(const char *name, fastmutex_t * mutex); -static int fastmutex_destroy(fastmutex_t * mutex); -static int fastmutex_lock(fastmutex_t * mutex); -static int fastmutex_timedlock(fastmutex_t * mutex, bigtime_t timeout); -static int fastmutex_unlock(fastmutex_t * mutex); - - -static int fastmutex_create(const char *name, fastmutex_t * mutex) -{ - mutex->count = 0; - mutex->sem = create_semaphore(name, 0, 0); - return (mutex->sem < 0) ? -1 : 0; -} - - -static int fastmutex_destroy(fastmutex_t * mutex) -{ - if (fastmutex_timedlock(mutex, 0) == 0 || errno == EWOULDBLOCK) { - return delete_semaphore(mutex->sem); - } - return 0; -} - - -static int fastmutex_lock(fastmutex_t * mutex) -{ - atomic_t prev = atomic_add(&mutex->count, 1); - if (prev > 0) - return lock_semaphore(mutex->sem); - return 0; -} - - -static int fastmutex_timedlock(fastmutex_t * mutex, bigtime_t timeout) -{ - atomic_t prev = atomic_add(&mutex->count, 1); - if (prev > 0) - return lock_semaphore_x(mutex->sem, 1, 0, timeout); - return 0; -} - - -static int fastmutex_unlock(fastmutex_t * mutex) -{ - atomic_t prev = atomic_add(&mutex->count, -1); - if (prev > 1) - return unlock_semaphore(mutex->sem); - return 0; -} - - -#endif /* FASTLOCK */ - - -/* - * Initialization. - * - */ -static void PyThread__init_thread(void) -{ - /* Do nothing. */ - return; -} - - -/* - * Thread support. - * - */ - -static atomic_t thread_count = 0; - -long PyThread_start_new_thread(void (*func) (void *), void *arg) -{ - status_t success = -1; - thread_id tid; - char name[OS_NAME_LENGTH]; - atomic_t this_thread; - - dprintf(("PyThread_start_new_thread called\n")); - - this_thread = atomic_add(&thread_count, 1); - PyOS_snprintf(name, sizeof(name), "python thread (%d)", this_thread); - - tid = spawn_thread(name, func, NORMAL_PRIORITY, 0, arg); - if (tid < 0) { - dprintf(("PyThread_start_new_thread spawn_thread failed: %s\n", strerror(errno))); - } else { - success = resume_thread(tid); - if (success < 0) { - dprintf(("PyThread_start_new_thread resume_thread failed: %s\n", strerror(errno))); - } - } - - return (success < 0 ? -1 : tid); -} - - -long PyThread_get_thread_ident(void) -{ - return get_thread_id(NULL); -} - - -void PyThread_exit_thread(void) -{ - dprintf(("PyThread_exit_thread called\n")); - - /* Thread-safe way to read a variable without a mutex: */ - if (atomic_add(&thread_count, 0) == 0) { - /* No threads around, so exit main(). */ - exit(0); - } else { - /* We're a thread */ - exit_thread(0); - } -} - - -/* - * Lock support. - * - */ - -static atomic_t lock_count = 0; - -PyThread_type_lock PyThread_allocate_lock(void) -{ -#ifdef FASTLOCK - fastmutex_t *lock; -#else - sem_id sema; -#endif - char name[OS_NAME_LENGTH]; - atomic_t this_lock; - - dprintf(("PyThread_allocate_lock called\n")); - -#ifdef FASTLOCK - lock = (fastmutex_t *) malloc(sizeof(fastmutex_t)); - if (lock == NULL) { - dprintf(("PyThread_allocate_lock failed: out of memory\n")); - return (PyThread_type_lock) NULL; - } -#endif - this_lock = atomic_add(&lock_count, 1); - PyOS_snprintf(name, sizeof(name), "python lock (%d)", this_lock); - -#ifdef FASTLOCK - if (fastmutex_create(name, lock) < 0) { - dprintf(("PyThread_allocate_lock failed: %s\n", - strerror(errno))); - free(lock); - lock = NULL; - } - dprintf(("PyThread_allocate_lock()-> %p\n", lock)); - return (PyThread_type_lock) lock; -#else - sema = create_semaphore(name, 1, 0); - if (sema < 0) { - dprintf(("PyThread_allocate_lock failed: %s\n", - strerror(errno))); - sema = 0; - } - dprintf(("PyThread_allocate_lock()-> %p\n", sema)); - return (PyThread_type_lock) sema; -#endif -} - - -void PyThread_free_lock(PyThread_type_lock lock) -{ - dprintf(("PyThread_free_lock(%p) called\n", lock)); - -#ifdef FASTLOCK - if (fastmutex_destroy((fastmutex_t *) lock) < 0) { - dprintf(("PyThread_free_lock(%p) failed: %s\n", lock, - strerror(errno))); - } - free(lock); -#else - if (delete_semaphore((sem_id) lock) < 0) { - dprintf(("PyThread_free_lock(%p) failed: %s\n", lock, - strerror(errno))); - } -#endif -} - - -int PyThread_acquire_lock(PyThread_type_lock lock, int waitflag) -{ - int retval; - - dprintf(("PyThread_acquire_lock(%p, %d) called\n", lock, - waitflag)); - -#ifdef FASTLOCK - if (waitflag) - retval = fastmutex_lock((fastmutex_t *) lock); - else - retval = fastmutex_timedlock((fastmutex_t *) lock, 0); -#else - if (waitflag) - retval = lock_semaphore((sem_id) lock); - else - retval = lock_semaphore_x((sem_id) lock, 1, 0, 0); -#endif - if (retval < 0) { - dprintf(("PyThread_acquire_lock(%p, %d) failed: %s\n", - lock, waitflag, strerror(errno))); - } - dprintf(("PyThread_acquire_lock(%p, %d)-> %d\n", lock, waitflag, - retval)); - return retval < 0 ? 0 : 1; -} - - -void PyThread_release_lock(PyThread_type_lock lock) -{ - dprintf(("PyThread_release_lock(%p) called\n", lock)); - -#ifdef FASTLOCK - if (fastmutex_unlock((fastmutex_t *) lock) < 0) { - dprintf(("PyThread_release_lock(%p) failed: %s\n", lock, - strerror(errno))); - } -#else - if (unlock_semaphore((sem_id) lock) < 0) { - dprintf(("PyThread_release_lock(%p) failed: %s\n", lock, - strerror(errno))); - } -#endif -} diff --git a/Python/thread_beos.h b/Python/thread_beos.h deleted file mode 100644 index 65dc470bcd..0000000000 --- a/Python/thread_beos.h +++ /dev/null @@ -1,248 +0,0 @@ -#include <kernel/OS.h> -#include <support/SupportDefs.h> -#include <errno.h> - -/* ---------------------------------------------------------------------- - * Fast locking mechanism described by Benoit Schillings (benoit@be.com) - * in the Be Developer's Newsletter, Issue #26 (http://www.be.com/). - */ -typedef struct benaphore { - sem_id _sem; - int32 _atom; -} benaphore_t; - -static status_t benaphore_create( const char *name, benaphore_t *ben ); -static status_t benaphore_destroy( benaphore_t *ben ); -static status_t benaphore_lock( benaphore_t *ben ); -static status_t benaphore_timedlock( benaphore_t *ben, bigtime_t micros ); -static status_t benaphore_unlock( benaphore_t *ben ); - -static status_t benaphore_create( const char *name, benaphore_t *ben ) -{ - if( ben != NULL ) { - ben->_atom = 0; - ben->_sem = create_sem( 0, name ); - - if( ben->_sem < B_NO_ERROR ) { - return B_BAD_SEM_ID; - } - } else { - return EFAULT; - } - - return EOK; -} - -static status_t benaphore_destroy( benaphore_t *ben ) -{ - if( ben->_sem >= B_NO_ERROR ) { - status_t retval = benaphore_timedlock( ben, 0 ); - - if( retval == EOK || retval == EWOULDBLOCK ) { - status_t del_retval = delete_sem( ben->_sem ); - - return del_retval; - } - } - - return B_BAD_SEM_ID; -} - -static status_t benaphore_lock( benaphore_t *ben ) -{ - int32 prev = atomic_add( &(ben->_atom), 1 ); - - if( prev > 0 ) { - return acquire_sem( ben->_sem ); - } - - return EOK; -} - -static status_t benaphore_timedlock( benaphore_t *ben, bigtime_t micros ) -{ - int32 prev = atomic_add( &(ben->_atom), 1 ); - - if( prev > 0 ) { - status_t retval = acquire_sem_etc( ben->_sem, 1, B_TIMEOUT, micros ); - - switch( retval ) { - case B_WOULD_BLOCK: /* Fall through... */ - case B_TIMED_OUT: - return EWOULDBLOCK; - break; - case B_OK: - return EOK; - break; - default: - return retval; - break; - } - } - - return EOK; -} - -static status_t benaphore_unlock( benaphore_t *ben ) -{ - int32 prev = atomic_add( &(ben->_atom), -1 ); - - if( prev > 1 ) { - return release_sem( ben->_sem ); - } - - return EOK; -} - -/* ---------------------------------------------------------------------- - * Initialization. - */ -static void PyThread__init_thread( void ) -{ - /* Do nothing. */ - return; -} - -/* ---------------------------------------------------------------------- - * Thread support. - * - * Only ANSI C, renamed functions here; you can't use K&R on BeOS, - * and there's no legacy thread module to support. - */ - -static int32 thread_count = 0; - -long PyThread_start_new_thread( void (*func)(void *), void *arg ) -{ - status_t success = 0; - thread_id tid; - char name[B_OS_NAME_LENGTH]; - int32 this_thread; - - dprintf(("PyThread_start_new_thread called\n")); - - /* We are so very thread-safe... */ - this_thread = atomic_add( &thread_count, 1 ); - PyOS_snprintf(name, sizeof(name), - "python thread (%d)", this_thread ); - - tid = spawn_thread( (thread_func)func, name, - B_NORMAL_PRIORITY, arg ); - if( tid > B_NO_ERROR ) { - success = resume_thread( tid ); - } - - return ( success == B_NO_ERROR ? tid : -1 ); -} - -long PyThread_get_thread_ident( void ) -{ - /* Presumed to return the current thread's ID... */ - thread_id tid; - tid = find_thread( NULL ); - - return ( tid != B_NAME_NOT_FOUND ? tid : -1 ); -} - -void PyThread_exit_thread( void ) -{ - int32 threads; - - dprintf(("PyThread_exit_thread called\n")); - - /* Thread-safe way to read a variable without a mutex: */ - threads = atomic_add( &thread_count, 0 ); - - if( threads == 0 ) { - /* No threads around, so exit main(). */ - exit(0); - } else { - /* Oh, we're a thread, let's try to exit gracefully... */ - exit_thread( B_NO_ERROR ); - } -} - -/* ---------------------------------------------------------------------- - * Lock support. - */ - -static int32 lock_count = 0; - -PyThread_type_lock PyThread_allocate_lock( void ) -{ - benaphore_t *lock; - status_t retval; - char name[B_OS_NAME_LENGTH]; - int32 this_lock; - - dprintf(("PyThread_allocate_lock called\n")); - - lock = (benaphore_t *)malloc( sizeof( benaphore_t ) ); - if( lock == NULL ) { - /* TODO: that's bad, raise MemoryError */ - return (PyThread_type_lock)NULL; - } - - this_lock = atomic_add( &lock_count, 1 ); - PyOS_snprintf(name, sizeof(name), "python lock (%d)", this_lock); - - retval = benaphore_create( name, lock ); - if( retval != EOK ) { - /* TODO: that's bad, raise an exception */ - return (PyThread_type_lock)NULL; - } - - dprintf(("PyThread_allocate_lock() -> %p\n", lock)); - return (PyThread_type_lock) lock; -} - -void PyThread_free_lock( PyThread_type_lock lock ) -{ - status_t retval; - - dprintf(("PyThread_free_lock(%p) called\n", lock)); - - retval = benaphore_destroy( (benaphore_t *)lock ); - if( retval != EOK ) { - /* TODO: that's bad, raise an exception */ - return; - } -} - -int PyThread_acquire_lock( PyThread_type_lock lock, int waitflag ) -{ - int success; - status_t retval; - - dprintf(("PyThread_acquire_lock(%p, %d) called\n", lock, waitflag)); - - if( waitflag ) { - retval = benaphore_lock( (benaphore_t *)lock ); - } else { - retval = benaphore_timedlock( (benaphore_t *)lock, 0 ); - } - - if( retval == EOK ) { - success = 1; - } else { - success = 0; - - /* TODO: that's bad, raise an exception */ - } - - dprintf(("PyThread_acquire_lock(%p, %d) -> %d\n", lock, waitflag, success)); - return success; -} - -void PyThread_release_lock( PyThread_type_lock lock ) -{ - status_t retval; - - dprintf(("PyThread_release_lock(%p) called\n", lock)); - - retval = benaphore_unlock( (benaphore_t *)lock ); - if( retval != EOK ) { - /* TODO: that's bad, raise an exception */ - return; - } -} diff --git a/Python/thread_nt.h b/Python/thread_nt.h index 9796a34141..d1bb0e5673 100644 --- a/Python/thread_nt.h +++ b/Python/thread_nt.h @@ -9,82 +9,31 @@ #include <process.h> #endif -typedef struct NRMUTEX { - LONG owned ; - DWORD thread_id ; - HANDLE hevent ; -} NRMUTEX, *PNRMUTEX ; +#define PNRMUTEX HANDLE - -BOOL -InitializeNonRecursiveMutex(PNRMUTEX mutex) +PNRMUTEX +AllocNonRecursiveMutex() { - mutex->owned = -1 ; /* No threads have entered NonRecursiveMutex */ - mutex->thread_id = 0 ; - mutex->hevent = CreateEvent(NULL, FALSE, FALSE, NULL) ; - return mutex->hevent != NULL ; /* TRUE if the mutex is created */ + return CreateSemaphore(NULL, 1, 1, NULL); } VOID -DeleteNonRecursiveMutex(PNRMUTEX mutex) +FreeNonRecursiveMutex(PNRMUTEX mutex) { /* No in-use check */ - CloseHandle(mutex->hevent) ; - mutex->hevent = NULL ; /* Just in case */ + CloseHandle(mutex); } DWORD -EnterNonRecursiveMutex(PNRMUTEX mutex, BOOL wait) +EnterNonRecursiveMutex(PNRMUTEX mutex, DWORD milliseconds) { - /* Assume that the thread waits successfully */ - DWORD ret ; - - /* InterlockedIncrement(&mutex->owned) == 0 means that no thread currently owns the mutex */ - if (!wait) - { - if (InterlockedCompareExchange(&mutex->owned, 0, -1) != -1) - return WAIT_TIMEOUT ; - ret = WAIT_OBJECT_0 ; - } - else - ret = InterlockedIncrement(&mutex->owned) ? - /* Some thread owns the mutex, let's wait... */ - WaitForSingleObject(mutex->hevent, INFINITE) : WAIT_OBJECT_0 ; - - mutex->thread_id = GetCurrentThreadId() ; /* We own it */ - return ret ; + return WaitForSingleObject(mutex, milliseconds); } BOOL LeaveNonRecursiveMutex(PNRMUTEX mutex) { - /* We don't own the mutex */ - mutex->thread_id = 0 ; - return - InterlockedDecrement(&mutex->owned) < 0 || - SetEvent(mutex->hevent) ; /* Other threads are waiting, wake one on them up */ -} - -PNRMUTEX -AllocNonRecursiveMutex(void) -{ - PNRMUTEX mutex = (PNRMUTEX)malloc(sizeof(NRMUTEX)) ; - if (mutex && !InitializeNonRecursiveMutex(mutex)) - { - free(mutex) ; - mutex = NULL ; - } - return mutex ; -} - -void -FreeNonRecursiveMutex(PNRMUTEX mutex) -{ - if (mutex) - { - DeleteNonRecursiveMutex(mutex) ; - free(mutex) ; - } + return ReleaseSemaphore(mutex, 1, NULL); } long PyThread_get_thread_ident(void); @@ -238,19 +187,47 @@ PyThread_free_lock(PyThread_type_lock aLock) * and 0 if the lock was not acquired. This means a 0 is returned * if the lock has already been acquired by this thread! */ -int -PyThread_acquire_lock(PyThread_type_lock aLock, int waitflag) +PyLockStatus +PyThread_acquire_lock_timed(PyThread_type_lock aLock, + PY_TIMEOUT_T microseconds, int intr_flag) { - int success ; + /* Fow now, intr_flag does nothing on Windows, and lock acquires are + * uninterruptible. */ + PyLockStatus success; + PY_TIMEOUT_T milliseconds; + + if (microseconds >= 0) { + milliseconds = microseconds / 1000; + if (microseconds % 1000 > 0) + ++milliseconds; + if ((DWORD) milliseconds != milliseconds) + Py_FatalError("Timeout too large for a DWORD, " + "please check PY_TIMEOUT_MAX"); + } + else + milliseconds = INFINITE; - dprintf(("%ld: PyThread_acquire_lock(%p, %d) called\n", PyThread_get_thread_ident(),aLock, waitflag)); + dprintf(("%ld: PyThread_acquire_lock_timed(%p, %lld) called\n", + PyThread_get_thread_ident(), aLock, microseconds)); - success = aLock && EnterNonRecursiveMutex((PNRMUTEX) aLock, (waitflag ? INFINITE : 0)) == WAIT_OBJECT_0 ; + if (aLock && EnterNonRecursiveMutex((PNRMUTEX)aLock, + (DWORD)milliseconds) == WAIT_OBJECT_0) { + success = PY_LOCK_ACQUIRED; + } + else { + success = PY_LOCK_FAILURE; + } - dprintf(("%ld: PyThread_acquire_lock(%p, %d) -> %d\n", PyThread_get_thread_ident(),aLock, waitflag, success)); + dprintf(("%ld: PyThread_acquire_lock(%p, %lld) -> %d\n", + PyThread_get_thread_ident(), aLock, microseconds, success)); return success; } +int +PyThread_acquire_lock(PyThread_type_lock aLock, int waitflag) +{ + return PyThread_acquire_lock_timed(aLock, waitflag ? -1 : 0, 0); +} void PyThread_release_lock(PyThread_type_lock aLock) @@ -296,7 +273,10 @@ _pythread_nt_set_stacksize(size_t size) int PyThread_create_key(void) { - return (int) TlsAlloc(); + DWORD result= TlsAlloc(); + if (result == TLS_OUT_OF_INDEXES) + return -1; + return (int)result; } void diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index c1c92d1a15..de42f1a165 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -19,14 +19,18 @@ #define THREAD_STACK_SIZE 0 /* use default stack size */ #endif -#if (defined(__APPLE__) || defined(__FreeBSD__)) && defined(THREAD_STACK_SIZE) && THREAD_STACK_SIZE == 0 - /* The default stack size for new threads on OSX is small enough that - * we'll get hard crashes instead of 'maximum recursion depth exceeded' - * exceptions. - * - * The default stack size below is the minimal stack size where a - * simple recursive function doesn't cause a hard crash. - */ +/* The default stack size for new threads on OSX and BSD is small enough that + * we'll get hard crashes instead of 'maximum recursion depth exceeded' + * exceptions. + * + * The default stack sizes below are the empirically determined minimal stack + * sizes where a simple recursive function doesn't cause a hard crash. + */ +#if defined(__APPLE__) && defined(THREAD_STACK_SIZE) && THREAD_STACK_SIZE == 0 +#undef THREAD_STACK_SIZE +#define THREAD_STACK_SIZE 0x500000 +#endif +#if defined(__FreeBSD__) && defined(THREAD_STACK_SIZE) && THREAD_STACK_SIZE == 0 #undef THREAD_STACK_SIZE #define THREAD_STACK_SIZE 0x400000 #endif @@ -76,7 +80,8 @@ /* Whether or not to use semaphores directly rather than emulating them with * mutexes and condition variables: */ -#if defined(_POSIX_SEMAPHORES) && !defined(HAVE_BROKEN_POSIX_SEMAPHORES) +#if (defined(_POSIX_SEMAPHORES) && !defined(HAVE_BROKEN_POSIX_SEMAPHORES) && \ + defined(HAVE_SEM_TIMEDWAIT)) # define USE_SEMAPHORES #else # undef USE_SEMAPHORES @@ -95,6 +100,26 @@ #endif +/* We assume all modern POSIX systems have gettimeofday() */ +#ifdef GETTIMEOFDAY_NO_TZ +#define GETTIMEOFDAY(ptv) gettimeofday(ptv) +#else +#define GETTIMEOFDAY(ptv) gettimeofday(ptv, (struct timezone *)NULL) +#endif + +#define MICROSECONDS_TO_TIMESPEC(microseconds, ts) \ +do { \ + struct timeval tv; \ + GETTIMEOFDAY(&tv); \ + tv.tv_usec += microseconds % 1000000; \ + tv.tv_sec += microseconds / 1000000; \ + tv.tv_sec += tv.tv_usec / 1000000; \ + tv.tv_usec %= 1000000; \ + ts.tv_sec = tv.tv_sec; \ + ts.tv_nsec = tv.tv_usec * 1000; \ +} while(0) + + /* A pthread mutex isn't sufficient to model the Python lock type * because, according to Draft 5 of the docs (P1003.4a/D5), both of the * following are undefined: @@ -241,9 +266,9 @@ void PyThread_exit_thread(void) { dprintf(("PyThread_exit_thread called\n")); - if (!initialized) { + if (!initialized) exit(0); - } + pthread_exit(0); } #ifdef USE_SEMAPHORES @@ -308,32 +333,57 @@ fix_status(int status) return (status == -1) ? errno : status; } -int -PyThread_acquire_lock(PyThread_type_lock lock, int waitflag) +PyLockStatus +PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds, + int intr_flag) { - int success; + PyLockStatus success; sem_t *thelock = (sem_t *)lock; int status, error = 0; + struct timespec ts; (void) error; /* silence unused-but-set-variable warning */ - dprintf(("PyThread_acquire_lock(%p, %d) called\n", lock, waitflag)); + dprintf(("PyThread_acquire_lock_timed(%p, %lld, %d) called\n", + lock, microseconds, intr_flag)); + if (microseconds > 0) + MICROSECONDS_TO_TIMESPEC(microseconds, ts); do { - if (waitflag) - status = fix_status(sem_wait(thelock)); - else + if (microseconds > 0) + status = fix_status(sem_timedwait(thelock, &ts)); + else if (microseconds == 0) status = fix_status(sem_trywait(thelock)); - } while (status == EINTR); /* Retry if interrupted by a signal */ - - if (waitflag) { - CHECK_STATUS("sem_wait"); - } else if (status != EAGAIN) { - CHECK_STATUS("sem_trywait"); + else + status = fix_status(sem_wait(thelock)); + /* Retry if interrupted by a signal, unless the caller wants to be + notified. */ + } while (!intr_flag && status == EINTR); + + /* Don't check the status if we're stopping because of an interrupt. */ + if (!(intr_flag && status == EINTR)) { + if (microseconds > 0) { + if (status != ETIMEDOUT) + CHECK_STATUS("sem_timedwait"); + } + else if (microseconds == 0) { + if (status != EAGAIN) + CHECK_STATUS("sem_trywait"); + } + else { + CHECK_STATUS("sem_wait"); + } } - success = (status == 0) ? 1 : 0; + if (status == 0) { + success = PY_LOCK_ACQUIRED; + } else if (intr_flag && status == EINTR) { + success = PY_LOCK_INTR; + } else { + success = PY_LOCK_FAILURE; + } - dprintf(("PyThread_acquire_lock(%p, %d) -> %d\n", lock, waitflag, success)); + dprintf(("PyThread_acquire_lock_timed(%p, %lld, %d) -> %d\n", + lock, microseconds, intr_flag, success)); return success; } @@ -373,6 +423,12 @@ PyThread_allocate_lock(void) status = pthread_mutex_init(&lock->mut, pthread_mutexattr_default); CHECK_STATUS("pthread_mutex_init"); + /* Mark the pthread mutex underlying a Python mutex as + pure happens-before. We can't simply mark the + Python-level mutex as a mutex because it can be + acquired and released in different threads, which + will cause errors. */ + _Py_ANNOTATE_PURE_HAPPENS_BEFORE_MUTEX(&lock->mut); status = pthread_cond_init(&lock->lock_released, pthread_condattr_default); @@ -405,37 +461,69 @@ PyThread_free_lock(PyThread_type_lock lock) free((void *)thelock); } -int -PyThread_acquire_lock(PyThread_type_lock lock, int waitflag) +PyLockStatus +PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds, + int intr_flag) { - int success; + PyLockStatus success; pthread_lock *thelock = (pthread_lock *)lock; int status, error = 0; - dprintf(("PyThread_acquire_lock(%p, %d) called\n", lock, waitflag)); + dprintf(("PyThread_acquire_lock_timed(%p, %lld, %d) called\n", + lock, microseconds, intr_flag)); status = pthread_mutex_lock( &thelock->mut ); CHECK_STATUS("pthread_mutex_lock[1]"); - success = thelock->locked == 0; - if ( !success && waitflag ) { + if (thelock->locked == 0) { + success = PY_LOCK_ACQUIRED; + } else if (microseconds == 0) { + success = PY_LOCK_FAILURE; + } else { + struct timespec ts; + if (microseconds > 0) + MICROSECONDS_TO_TIMESPEC(microseconds, ts); /* continue trying until we get the lock */ /* mut must be locked by me -- part of the condition * protocol */ - while ( thelock->locked ) { - status = pthread_cond_wait(&thelock->lock_released, - &thelock->mut); - CHECK_STATUS("pthread_cond_wait"); + success = PY_LOCK_FAILURE; + while (success == PY_LOCK_FAILURE) { + if (microseconds > 0) { + status = pthread_cond_timedwait( + &thelock->lock_released, + &thelock->mut, &ts); + if (status == ETIMEDOUT) + break; + CHECK_STATUS("pthread_cond_timed_wait"); + } + else { + status = pthread_cond_wait( + &thelock->lock_released, + &thelock->mut); + CHECK_STATUS("pthread_cond_wait"); + } + + if (intr_flag && status == 0 && thelock->locked) { + /* We were woken up, but didn't get the lock. We probably received + * a signal. Return PY_LOCK_INTR to allow the caller to handle + * it and retry. */ + success = PY_LOCK_INTR; + break; + } else if (status == 0 && !thelock->locked) { + success = PY_LOCK_ACQUIRED; + } else { + success = PY_LOCK_FAILURE; + } } - success = 1; } - if (success) thelock->locked = 1; + if (success == PY_LOCK_ACQUIRED) thelock->locked = 1; status = pthread_mutex_unlock( &thelock->mut ); CHECK_STATUS("pthread_mutex_unlock[1]"); - if (error) success = 0; - dprintf(("PyThread_acquire_lock(%p, %d) -> %d\n", lock, waitflag, success)); + if (error) success = PY_LOCK_FAILURE; + dprintf(("PyThread_acquire_lock_timed(%p, %lld, %d) -> %d\n", + lock, microseconds, intr_flag, success)); return success; } @@ -462,6 +550,12 @@ PyThread_release_lock(PyThread_type_lock lock) #endif /* USE_SEMAPHORES */ +int +PyThread_acquire_lock(PyThread_type_lock lock, int waitflag) +{ + return PyThread_acquire_lock_timed(lock, waitflag ? -1 : 0, /*intr_flag=*/0); +} + /* set the thread stack size. * Return 0 if size is valid, -1 if size is invalid, * -2 if setting stack size is not supported. @@ -506,3 +600,46 @@ _pythread_pthread_set_stacksize(size_t size) } #define THREAD_SET_STACKSIZE(x) _pythread_pthread_set_stacksize(x) + +#define Py_HAVE_NATIVE_TLS + +int +PyThread_create_key(void) +{ + pthread_key_t key; + int fail = pthread_key_create(&key, NULL); + return fail ? -1 : key; +} + +void +PyThread_delete_key(int key) +{ + pthread_key_delete(key); +} + +void +PyThread_delete_key_value(int key) +{ + pthread_setspecific(key, NULL); +} + +int +PyThread_set_key_value(int key, void *value) +{ + int fail; + void *oldValue = pthread_getspecific(key); + if (oldValue != NULL) + return 0; + fail = pthread_setspecific(key, value); + return fail ? -1 : 0; +} + +void * +PyThread_get_key_value(int key) +{ + return pthread_getspecific(key); +} + +void +PyThread_ReInitTLS(void) +{} diff --git a/Python/traceback.c b/Python/traceback.c index adfd66c0de..59bb3f0d15 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -7,14 +7,31 @@ #include "frameobject.h" #include "structmember.h" #include "osdefs.h" -#include "traceback.h" +#ifdef HAVE_FCNTL_H +#include <fcntl.h> +#endif #define OFF(x) offsetof(PyTracebackObject, x) +/* Method from Parser/tokenizer.c */ +extern char * PyTokenizer_FindEncoding(int); + +static PyObject * +tb_dir(PyTracebackObject *self) +{ + return Py_BuildValue("[ssss]", "tb_frame", "tb_next", + "tb_lasti", "tb_lineno"); +} + +static PyMethodDef tb_methods[] = { + {"__dir__", (PyCFunction)tb_dir, METH_NOARGS}, + {NULL, NULL, 0, NULL}, +}; + static PyMemberDef tb_memberlist[] = { - {"tb_next", T_OBJECT, OFF(tb_next), READONLY}, - {"tb_frame", T_OBJECT, OFF(tb_frame), READONLY}, - {"tb_lasti", T_INT, OFF(tb_lasti), READONLY}, + {"tb_next", T_OBJECT, OFF(tb_next), READONLY}, + {"tb_frame", T_OBJECT, OFF(tb_frame), READONLY}, + {"tb_lasti", T_INT, OFF(tb_lasti), READONLY}, {"tb_lineno", T_INT, OFF(tb_lineno), READONLY}, {NULL} /* Sentinel */ }; @@ -52,9 +69,9 @@ PyTypeObject PyTraceBack_Type = { 0, (destructor)tb_dealloc, /*tp_dealloc*/ 0, /*tp_print*/ - 0, /*tp_getattr*/ + 0, /*tp_getattr*/ 0, /*tp_setattr*/ - 0, /*tp_compare*/ + 0, /*tp_reserved*/ 0, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ @@ -62,7 +79,7 @@ PyTypeObject PyTraceBack_Type = { 0, /* tp_hash */ 0, /* tp_call */ 0, /* tp_str */ - 0, /* tp_getattro */ + PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */ @@ -73,8 +90,8 @@ PyTypeObject PyTraceBack_Type = { 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ - 0, /* tp_methods */ - tb_memberlist, /* tp_members */ + tb_methods, /* tp_methods */ + tb_memberlist, /* tp_members */ 0, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ @@ -115,118 +132,204 @@ PyTraceBack_Here(PyFrameObject *frame) return 0; } +static PyObject * +_Py_FindSourceFile(PyObject *filename, char* namebuf, size_t namelen, PyObject *io) +{ + Py_ssize_t i; + PyObject *binary; + PyObject *v; + Py_ssize_t npath; + size_t taillen; + PyObject *syspath; + PyObject *path; + const char* tail; + PyObject *filebytes; + const char* filepath; + Py_ssize_t len; + PyObject* result; + + filebytes = PyUnicode_EncodeFSDefault(filename); + if (filebytes == NULL) { + PyErr_Clear(); + return NULL; + } + filepath = PyBytes_AS_STRING(filebytes); + + /* Search tail of filename in sys.path before giving up */ + tail = strrchr(filepath, SEP); + if (tail == NULL) + tail = filepath; + else + tail++; + taillen = strlen(tail); + + syspath = PySys_GetObject("path"); + if (syspath == NULL || !PyList_Check(syspath)) + goto error; + npath = PyList_Size(syspath); + + for (i = 0; i < npath; i++) { + v = PyList_GetItem(syspath, i); + if (v == NULL) { + PyErr_Clear(); + break; + } + if (!PyUnicode_Check(v)) + continue; + path = PyUnicode_EncodeFSDefault(v); + if (path == NULL) { + PyErr_Clear(); + continue; + } + len = PyBytes_GET_SIZE(path); + if (len + 1 + (Py_ssize_t)taillen >= (Py_ssize_t)namelen - 1) { + Py_DECREF(path); + continue; /* Too long */ + } + strcpy(namebuf, PyBytes_AS_STRING(path)); + Py_DECREF(path); + if (strlen(namebuf) != len) + continue; /* v contains '\0' */ + if (len > 0 && namebuf[len-1] != SEP) + namebuf[len++] = SEP; + strcpy(namebuf+len, tail); + + binary = PyObject_CallMethod(io, "open", "ss", namebuf, "rb"); + if (binary != NULL) { + result = binary; + goto finally; + } + PyErr_Clear(); + } + goto error; + +error: + result = NULL; +finally: + Py_DECREF(filebytes); + return result; +} + int -_Py_DisplaySourceLine(PyObject *f, const char *filename, int lineno, int indent) +_Py_DisplaySourceLine(PyObject *f, PyObject *filename, int lineno, int indent) { int err = 0; - FILE *xfp = NULL; - char linebuf[2000]; + int fd; int i; - char namebuf[MAXPATHLEN+1]; + char *found_encoding; + char *encoding; + PyObject *io; + PyObject *binary; + PyObject *fob = NULL; + PyObject *lineobj = NULL; + PyObject *res; + char buf[MAXPATHLEN+1]; + Py_UNICODE *u, *p; + Py_ssize_t len; + /* open the file */ if (filename == NULL) + return 0; + + io = PyImport_ImportModuleNoBlock("io"); + if (io == NULL) return -1; - /* This is needed by Emacs' compile command */ -#define FMT " File \"%.500s\", line %d, in %.500s\n" - xfp = fopen(filename, "r" PY_STDIOTEXTMODE); - if (xfp == NULL) { - /* Search tail of filename in sys.path before giving up */ - PyObject *path; - const char *tail = strrchr(filename, SEP); - if (tail == NULL) - tail = filename; - else - tail++; - path = PySys_GetObject("path"); - if (path != NULL && PyList_Check(path)) { - Py_ssize_t _npath = PyList_Size(path); - int npath = Py_SAFE_DOWNCAST(_npath, Py_ssize_t, int); - size_t taillen = strlen(tail); - for (i = 0; i < npath; i++) { - PyObject *v = PyList_GetItem(path, i); - if (v == NULL) { - PyErr_Clear(); - break; - } - if (PyString_Check(v)) { - size_t len; - len = PyString_GET_SIZE(v); - if (len + 1 + taillen >= MAXPATHLEN) - continue; /* Too long */ - strcpy(namebuf, PyString_AsString(v)); - if (strlen(namebuf) != len) - continue; /* v contains '\0' */ - if (len > 0 && namebuf[len-1] != SEP) - namebuf[len++] = SEP; - strcpy(namebuf+len, tail); - xfp = fopen(namebuf, "r" PY_STDIOTEXTMODE); - if (xfp != NULL) { - break; - } - } - } + binary = PyObject_CallMethod(io, "open", "Os", filename, "rb"); + + if (binary == NULL) { + binary = _Py_FindSourceFile(filename, buf, sizeof(buf), io); + if (binary == NULL) { + Py_DECREF(io); + return 0; } } - if (xfp == NULL) - return err; - if (err != 0) { - fclose(xfp); - return err; + /* use the right encoding to decode the file as unicode */ + fd = PyObject_AsFileDescriptor(binary); + found_encoding = PyTokenizer_FindEncoding(fd); + encoding = (found_encoding != NULL) ? found_encoding : "utf-8"; + lseek(fd, 0, 0); /* Reset position */ + fob = PyObject_CallMethod(io, "TextIOWrapper", "Os", binary, encoding); + Py_DECREF(io); + Py_DECREF(binary); + PyMem_FREE(found_encoding); + + if (fob == NULL) { + PyErr_Clear(); + return 0; } + /* get the line number lineno */ for (i = 0; i < lineno; i++) { - char* pLastChar = &linebuf[sizeof(linebuf)-2]; - do { - *pLastChar = '\0'; - if (Py_UniversalNewlineFgets(linebuf, sizeof linebuf, xfp, NULL) == NULL) - break; - /* fgets read *something*; if it didn't get as - far as pLastChar, it must have found a newline - or hit the end of the file; if pLastChar is \n, - it obviously found a newline; else we haven't - yet seen a newline, so must continue */ - } while (*pLastChar != '\0' && *pLastChar != '\n'); + Py_XDECREF(lineobj); + lineobj = PyFile_GetLine(fob, -1); + if (!lineobj) { + err = -1; + break; + } + } + res = PyObject_CallMethod(fob, "close", ""); + if (res) + Py_DECREF(res); + else + PyErr_Clear(); + Py_DECREF(fob); + if (!lineobj || !PyUnicode_Check(lineobj)) { + Py_XDECREF(lineobj); + return err; } - if (i == lineno) { - char buf[11]; - char *p = linebuf; - while (*p == ' ' || *p == '\t' || *p == '\014') - p++; - - /* Write some spaces before the line */ - strcpy(buf, " "); - assert (strlen(buf) == 10); - while (indent > 0) { - if(indent < 10) - buf[indent] = '\0'; - err = PyFile_WriteString(buf, f); - if (err != 0) - break; - indent -= 10; + + /* remove the indentation of the line */ + u = PyUnicode_AS_UNICODE(lineobj); + len = PyUnicode_GET_SIZE(lineobj); + for (p=u; *p == ' ' || *p == '\t' || *p == '\014'; p++) + len--; + if (u != p) { + PyObject *truncated; + truncated = PyUnicode_FromUnicode(p, len); + if (truncated) { + Py_DECREF(lineobj); + lineobj = truncated; + } else { + PyErr_Clear(); } + } - if (err == 0) - err = PyFile_WriteString(p, f); - if (err == 0 && strchr(p, '\n') == NULL) - err = PyFile_WriteString("\n", f); + /* Write some spaces before the line */ + strcpy(buf, " "); + assert (strlen(buf) == 10); + while (indent > 0) { + if(indent < 10) + buf[indent] = '\0'; + err = PyFile_WriteString(buf, f); + if (err != 0) + break; + indent -= 10; } - fclose(xfp); + + /* finally display the line */ + if (err == 0) + err = PyFile_WriteObject(lineobj, f, Py_PRINT_RAW); + Py_DECREF(lineobj); + if (err == 0) + err = PyFile_WriteString("\n", f); return err; } static int -tb_displayline(PyObject *f, const char *filename, int lineno, const char *name) +tb_displayline(PyObject *f, PyObject *filename, int lineno, PyObject *name) { - int err = 0; - char linebuf[2000]; + int err; + PyObject *line; if (filename == NULL || name == NULL) return -1; - /* This is needed by Emacs' compile command */ -#define FMT " File \"%.500s\", line %d, in %.500s\n" - PyOS_snprintf(linebuf, sizeof(linebuf), FMT, filename, lineno, name); - err = PyFile_WriteString(linebuf, f); + line = PyUnicode_FromFormat(" File \"%U\", line %d, in %U\n", + filename, lineno, name); + if (line == NULL) + return -1; + err = PyFile_WriteObject(line, f, Py_PRINT_RAW); + Py_DECREF(line); if (err != 0) return err; return _Py_DisplaySourceLine(f, filename, lineno, 4); @@ -245,10 +348,9 @@ tb_printinternal(PyTracebackObject *tb, PyObject *f, long limit) while (tb != NULL && err == 0) { if (depth <= limit) { err = tb_displayline(f, - PyString_AsString( - tb->tb_frame->f_code->co_filename), - tb->tb_lineno, - PyString_AsString(tb->tb_frame->f_code->co_name)); + tb->tb_frame->f_code->co_filename, + tb->tb_lineno, + tb->tb_frame->f_code->co_name); } depth--; tb = tb->tb_next; @@ -258,12 +360,15 @@ tb_printinternal(PyTracebackObject *tb, PyObject *f, long limit) return err; } +#define PyTraceBack_LIMIT 1000 + int PyTraceBack_Print(PyObject *v, PyObject *f) { int err; PyObject *limitv; - long limit = 1000; + long limit = PyTraceBack_LIMIT; + if (v == NULL) return 0; if (!PyTraceBack_Check(v)) { @@ -271,10 +376,26 @@ PyTraceBack_Print(PyObject *v, PyObject *f) return -1; } limitv = PySys_GetObject("tracebacklimit"); - if (limitv && PyInt_Check(limitv)) { - limit = PyInt_AsLong(limitv); - if (limit <= 0) - return 0; + if (limitv) { + PyObject *exc_type, *exc_value, *exc_tb; + + PyErr_Fetch(&exc_type, &exc_value, &exc_tb); + limit = PyLong_AsLong(limitv); + if (limit == -1 && PyErr_Occurred()) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + limit = PyTraceBack_LIMIT; + } + else { + Py_XDECREF(exc_type); + Py_XDECREF(exc_value); + Py_XDECREF(exc_tb); + return 0; + } + } + else if (limit <= 0) { + limit = PyTraceBack_LIMIT; + } + PyErr_Restore(exc_type, exc_value, exc_tb); } err = PyFile_WriteString("Traceback (most recent call last):\n", f); if (!err) |