diff options
Diffstat (limited to 'Python/compile.c')
| -rw-r--r-- | Python/compile.c | 725 |
1 files changed, 477 insertions, 248 deletions
diff --git a/Python/compile.c b/Python/compile.c index 93f47e0220..45e4262b40 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -29,6 +29,7 @@ #include "code.h" #include "symtable.h" #include "opcode.h" +#include "wordcode_helpers.h" #define DEFAULT_BLOCK_SIZE 16 #define DEFAULT_BLOCKS 8 @@ -43,7 +44,6 @@ struct instr { unsigned i_jabs : 1; unsigned i_jrel : 1; - unsigned i_hasarg : 1; unsigned char i_opcode; int i_oparg; struct basicblock_ *i_target; /* target block (if jump instruction) */ @@ -171,7 +171,6 @@ static int compiler_addop(struct compiler *, int); static int compiler_addop_o(struct compiler *, int, PyObject *, PyObject *); static int compiler_addop_i(struct compiler *, int, Py_ssize_t); static int compiler_addop_j(struct compiler *, int, basicblock *, int); -static basicblock *compiler_use_new_block(struct compiler *); static int compiler_error(struct compiler *, const char *); static int compiler_nameop(struct compiler *, identifier, expr_context_ty); @@ -196,7 +195,7 @@ static int expr_constant(struct compiler *, expr_ty); static int compiler_with(struct compiler *, stmt_ty, int); static int compiler_async_with(struct compiler *, stmt_ty, int); static int compiler_async_for(struct compiler *, stmt_ty); -static int compiler_call_helper(struct compiler *c, Py_ssize_t n, +static int compiler_call_helper(struct compiler *c, int n, asdl_seq *args, asdl_seq *keywords); static int compiler_try_except(struct compiler *, stmt_ty); @@ -477,9 +476,9 @@ compiler_unit_check(struct compiler_unit *u) { basicblock *block; for (block = u->u_blocks; block != NULL; block = block->b_list) { - assert((void *)block != (void *)0xcbcbcbcb); - assert((void *)block != (void *)0xfbfbfbfb); - assert((void *)block != (void *)0xdbdbdbdb); + assert((uintptr_t)block != 0xcbcbcbcbU); + assert((uintptr_t)block != 0xfbfbfbfbU); + assert((uintptr_t)block != 0xdbdbdbdbU); if (block->b_instr != NULL) { assert(block->b_ialloc > 0); assert(block->b_iused > 0); @@ -523,6 +522,7 @@ compiler_enter_scope(struct compiler *c, identifier name, int scope_type, void *key, int lineno) { struct compiler_unit *u; + basicblock *block; u = (struct compiler_unit *)PyObject_Malloc(sizeof( struct compiler_unit)); @@ -620,8 +620,11 @@ compiler_enter_scope(struct compiler *c, identifier name, c->u = u; c->c_nestlevel++; - if (compiler_use_new_block(c) == NULL) + + block = compiler_new_block(c); + if (block == NULL) return 0; + c->u->u_curblock = block; if (u->u_scope_type != COMPILER_SCOPE_MODULE) { if (!compiler_set_qualname(c)) @@ -731,6 +734,7 @@ compiler_set_qualname(struct compiler *c) return 1; } + /* Allocate a new block and return a pointer to it. Returns NULL on error. */ @@ -755,16 +759,6 @@ compiler_new_block(struct compiler *c) } static basicblock * -compiler_use_new_block(struct compiler *c) -{ - basicblock *block = compiler_new_block(c); - if (block == NULL) - return NULL; - c->u->u_curblock = block; - return block; -} - -static basicblock * compiler_next_block(struct compiler *c) { basicblock *block = compiler_new_block(c); @@ -810,7 +804,7 @@ compiler_next_instr(struct compiler *c, basicblock *b) oldsize = b->b_ialloc * sizeof(struct instr); newsize = oldsize << 1; - if (oldsize > (PY_SIZE_MAX >> 1)) { + if (oldsize > (SIZE_MAX >> 1)) { PyErr_NoMemory(); return -1; } @@ -976,6 +970,7 @@ PyCompile_OpcodeStackEffect(int opcode, int oparg) case BUILD_TUPLE: case BUILD_LIST: case BUILD_SET: + case BUILD_STRING: return 1-oparg; case BUILD_LIST_UNPACK: case BUILD_TUPLE_UNPACK: @@ -986,6 +981,8 @@ PyCompile_OpcodeStackEffect(int opcode, int oparg) return 1 - (oparg & 0xFF); case BUILD_MAP: return 1 - 2*oparg; + case BUILD_CONST_KEY_MAP: + return -oparg; case LOAD_ATTR: return 0; case COMPARE_OP: @@ -1034,11 +1031,10 @@ PyCompile_OpcodeStackEffect(int opcode, int oparg) return -NARGS(oparg)-1; case CALL_FUNCTION_VAR_KW: return -NARGS(oparg)-2; - case MAKE_FUNCTION: - return -1 -NARGS(oparg) - ((oparg >> 16) & 0xffff); - case MAKE_CLOSURE: - return -2 - NARGS(oparg) - ((oparg >> 16) & 0xffff); #undef NARGS + case MAKE_FUNCTION: + return -1 - ((oparg & 0x01) != 0) - ((oparg & 0x02) != 0) - + ((oparg & 0x04) != 0) - ((oparg & 0x08) != 0); case BUILD_SLICE: if (oparg == 3) return -2; @@ -1066,6 +1062,10 @@ PyCompile_OpcodeStackEffect(int opcode, int oparg) return 1; case GET_YIELD_FROM_ITER: return 0; + case FORMAT_VALUE: + /* If there's a fmt_spec on the stack, we go from 2->1, + else 1->1. */ + return (oparg & FVS_MASK) == FVS_HAVE_SPEC ? -1 : 0; default: return PY_INVALID_STACK_EFFECT; } @@ -1082,13 +1082,14 @@ compiler_addop(struct compiler *c, int opcode) basicblock *b; struct instr *i; int off; + assert(!HAS_ARG(opcode)); off = compiler_next_instr(c, c->u->u_curblock); if (off < 0) return 0; b = c->u->u_curblock; i = &b->b_instr[off]; i->i_opcode = opcode; - i->i_hasarg = 0; + i->i_oparg = 0; if (opcode == RETURN_VALUE) b->b_return = 1; compiler_set_lineno(c, off); @@ -1165,10 +1166,15 @@ compiler_addop_i(struct compiler *c, int opcode, Py_ssize_t oparg) struct instr *i; int off; - /* Integer arguments are limit to 16-bit. There is an extension for 32-bit - integer arguments. */ - assert((-2147483647-1) <= oparg); - assert(oparg <= 2147483647); + /* oparg value is unsigned, but a signed C int is usually used to store + it in the C code (like Python/ceval.c). + + Limit to 32-bit signed C int (rather than INT_MAX) for portability. + + The argument of a concrete bytecode instruction is limited to 8-bit. + EXTENDED_ARG is used for 16, 24, and 32-bit arguments. */ + assert(HAS_ARG(opcode)); + assert(0 <= oparg && oparg <= 2147483647); off = compiler_next_instr(c, c->u->u_curblock); if (off < 0) @@ -1176,7 +1182,6 @@ compiler_addop_i(struct compiler *c, int opcode, Py_ssize_t oparg) i = &c->u->u_curblock->b_instr[off]; i->i_opcode = opcode; i->i_oparg = Py_SAFE_DOWNCAST(oparg, Py_ssize_t, int); - i->i_hasarg = 1; compiler_set_lineno(c, off); return 1; } @@ -1187,6 +1192,7 @@ compiler_addop_j(struct compiler *c, int opcode, basicblock *b, int absolute) struct instr *i; int off; + assert(HAS_ARG(opcode)); assert(b != NULL); off = compiler_next_instr(c, c->u->u_curblock); if (off < 0) @@ -1194,7 +1200,6 @@ compiler_addop_j(struct compiler *c, int opcode, basicblock *b, int absolute) i = &c->u->u_curblock->b_instr[off]; i->i_opcode = opcode; i->i_target = b; - i->i_hasarg = 1; if (absolute) i->i_jabs = 1; else @@ -1203,22 +1208,12 @@ compiler_addop_j(struct compiler *c, int opcode, basicblock *b, int absolute) return 1; } -/* The distinction between NEW_BLOCK and NEXT_BLOCK is subtle. (I'd - like to find better names.) NEW_BLOCK() creates a new block and sets - it as the current block. NEXT_BLOCK() also creates an implicit jump - from the current block to the new block. -*/ +/* NEXT_BLOCK() creates an implicit jump from the current block + to the new block. -/* The returns inside these macros make it impossible to decref objects - created in the local function. Local objects should use the arena. + The returns inside this macro make it impossible to decref objects + created in the local function. Local objects should use the arena. */ - - -#define NEW_BLOCK(C) { \ - if (compiler_use_new_block((C)) == NULL) \ - return 0; \ -} - #define NEXT_BLOCK(C) { \ if (compiler_next_block((C)) == NULL) \ return 0; \ @@ -1241,6 +1236,15 @@ compiler_addop_j(struct compiler *c, int opcode, basicblock *b, int absolute) return 0; \ } +/* Same as ADDOP_O, but steals a reference. */ +#define ADDOP_N(C, OP, O, TYPE) { \ + if (!compiler_addop_o((C), (OP), (C)->u->u_ ## TYPE, (O))) { \ + Py_DECREF((O)); \ + return 0; \ + } \ + Py_DECREF((O)); \ +} + #define ADDOP_NAME(C, OP, O, TYPE) { \ if (!compiler_addop_name((C), (OP), (C)->u->u_ ## TYPE, (O))) \ return 0; \ @@ -1309,7 +1313,49 @@ compiler_isdocstring(stmt_ty s) { if (s->kind != Expr_kind) return 0; - return s->v.Expr.value->kind == Str_kind; + if (s->v.Expr.value->kind == Str_kind) + return 1; + if (s->v.Expr.value->kind == Constant_kind) + return PyUnicode_CheckExact(s->v.Expr.value->v.Constant.value); + return 0; +} + +static int +is_const(expr_ty e) +{ + switch (e->kind) { + case Constant_kind: + case Num_kind: + case Str_kind: + case Bytes_kind: + case Ellipsis_kind: + case NameConstant_kind: + return 1; + default: + return 0; + } +} + +static PyObject * +get_const_value(expr_ty e) +{ + switch (e->kind) { + case Constant_kind: + return e->v.Constant.value; + case Num_kind: + return e->v.Num.n; + case Str_kind: + return e->v.Str.s; + case Bytes_kind: + return e->v.Bytes.s; + case Ellipsis_kind: + return Py_Ellipsis; + case NameConstant_kind: + return e->v.NameConstant.value; + default: + assert(!is_const(e)); + return NULL; + } } /* Compile a sequence of statements, checking for a docstring. */ @@ -1426,53 +1472,50 @@ compiler_lookup_arg(PyObject *dict, PyObject *name) } static int -compiler_make_closure(struct compiler *c, PyCodeObject *co, Py_ssize_t args, PyObject *qualname) +compiler_make_closure(struct compiler *c, PyCodeObject *co, Py_ssize_t flags, PyObject *qualname) { Py_ssize_t i, free = PyCode_GetNumFree(co); if (qualname == NULL) qualname = co->co_name; - if (free == 0) { - ADDOP_O(c, LOAD_CONST, (PyObject*)co, consts); - ADDOP_O(c, LOAD_CONST, qualname, consts); - ADDOP_I(c, MAKE_FUNCTION, args); - return 1; - } - for (i = 0; i < free; ++i) { - /* Bypass com_addop_varname because it will generate - LOAD_DEREF but LOAD_CLOSURE is needed. - */ - PyObject *name = PyTuple_GET_ITEM(co->co_freevars, i); - int arg, reftype; - - /* Special case: If a class contains a method with a - free variable that has the same name as a method, - the name will be considered free *and* local in the - class. It should be handled by the closure, as - well as by the normal name loookup logic. - */ - reftype = get_ref_type(c, name); - if (reftype == CELL) - arg = compiler_lookup_arg(c->u->u_cellvars, name); - else /* (reftype == FREE) */ - arg = compiler_lookup_arg(c->u->u_freevars, name); - if (arg == -1) { - fprintf(stderr, - "lookup %s in %s %d %d\n" - "freevars of %s: %s\n", - PyUnicode_AsUTF8(PyObject_Repr(name)), - PyUnicode_AsUTF8(c->u->u_name), - reftype, arg, - PyUnicode_AsUTF8(co->co_name), - PyUnicode_AsUTF8(PyObject_Repr(co->co_freevars))); - Py_FatalError("compiler_make_closure()"); + if (free) { + for (i = 0; i < free; ++i) { + /* Bypass com_addop_varname because it will generate + LOAD_DEREF but LOAD_CLOSURE is needed. + */ + PyObject *name = PyTuple_GET_ITEM(co->co_freevars, i); + int arg, reftype; + + /* Special case: If a class contains a method with a + free variable that has the same name as a method, + the name will be considered free *and* local in the + class. It should be handled by the closure, as + well as by the normal name loookup logic. + */ + reftype = get_ref_type(c, name); + if (reftype == CELL) + arg = compiler_lookup_arg(c->u->u_cellvars, name); + else /* (reftype == FREE) */ + arg = compiler_lookup_arg(c->u->u_freevars, name); + if (arg == -1) { + fprintf(stderr, + "lookup %s in %s %d %d\n" + "freevars of %s: %s\n", + PyUnicode_AsUTF8(PyObject_Repr(name)), + PyUnicode_AsUTF8(c->u->u_name), + reftype, arg, + PyUnicode_AsUTF8(co->co_name), + PyUnicode_AsUTF8(PyObject_Repr(co->co_freevars))); + Py_FatalError("compiler_make_closure()"); + } + ADDOP_I(c, LOAD_CLOSURE, arg); } - ADDOP_I(c, LOAD_CLOSURE, arg); + flags |= 0x08; + ADDOP_I(c, BUILD_TUPLE, free); } - ADDOP_I(c, BUILD_TUPLE, free); ADDOP_O(c, LOAD_CONST, (PyObject*)co, consts); ADDOP_O(c, LOAD_CONST, qualname, consts); - ADDOP_I(c, MAKE_CLOSURE, args); + ADDOP_I(c, MAKE_FUNCTION, flags); return 1; } @@ -1494,26 +1537,60 @@ static int compiler_visit_kwonlydefaults(struct compiler *c, asdl_seq *kwonlyargs, asdl_seq *kw_defaults) { - /* Return the number of defaults + 1. - Returns 0 on error. - */ - int i, default_count = 0; + /* Push a dict of keyword-only default values. + + Return 0 on error, -1 if no dict pushed, 1 if a dict is pushed. + */ + int i; + PyObject *keys = NULL; + for (i = 0; i < asdl_seq_LEN(kwonlyargs); i++) { arg_ty arg = asdl_seq_GET(kwonlyargs, i); expr_ty default_ = asdl_seq_GET(kw_defaults, i); if (default_) { PyObject *mangled = _Py_Mangle(c->u->u_private, arg->arg); - if (!mangled) - return 0; - ADDOP_O(c, LOAD_CONST, mangled, consts); - Py_DECREF(mangled); + if (!mangled) { + goto error; + } + if (keys == NULL) { + keys = PyList_New(1); + if (keys == NULL) { + Py_DECREF(mangled); + return 0; + } + PyList_SET_ITEM(keys, 0, mangled); + } + else { + int res = PyList_Append(keys, mangled); + Py_DECREF(mangled); + if (res == -1) { + goto error; + } + } if (!compiler_visit_expr(c, default_)) { - return 0; + goto error; } - default_count++; } } - return default_count + 1; + if (keys != NULL) { + Py_ssize_t default_count = PyList_GET_SIZE(keys); + PyObject *keys_tuple = PyList_AsTuple(keys); + Py_DECREF(keys); + if (keys_tuple == NULL) { + return 0; + } + ADDOP_N(c, LOAD_CONST, keys_tuple, consts); + ADDOP_I(c, BUILD_CONST_KEY_MAP, default_count); + assert(default_count > 0); + return 1; + } + else { + return -1; + } + +error: + Py_XDECREF(keys); + return 0; } static int @@ -1556,11 +1633,10 @@ static int compiler_visit_annotations(struct compiler *c, arguments_ty args, expr_ty returns) { - /* Push arg annotations and a list of the argument names. Return the # - of items pushed + 1. The expressions are evaluated out-of-order wrt the - source code. + /* Push arg annotation dict. + The expressions are evaluated out-of-order wrt the source code. - More than 2^16-1 annotations is a SyntaxError. Returns 0 on error. + Return 0 on error, -1 if no dict pushed, 1 if a dict is pushed. */ static identifier return_str; PyObject *names; @@ -1592,32 +1668,20 @@ compiler_visit_annotations(struct compiler *c, arguments_ty args, } len = PyList_GET_SIZE(names); - if (len > 65534) { - /* len must fit in 16 bits, and len is incremented below */ - PyErr_SetString(PyExc_SyntaxError, - "too many annotations"); - goto error; - } if (len) { - /* convert names to a tuple and place on stack */ - PyObject *elt; - Py_ssize_t i; - PyObject *s = PyTuple_New(len); - if (!s) - goto error; - for (i = 0; i < len; i++) { - elt = PyList_GET_ITEM(names, i); - Py_INCREF(elt); - PyTuple_SET_ITEM(s, i, elt); + PyObject *keytuple = PyList_AsTuple(names); + Py_DECREF(names); + if (keytuple == NULL) { + return 0; } - ADDOP_O(c, LOAD_CONST, s, consts); - Py_DECREF(s); - len++; /* include the just-pushed tuple */ + ADDOP_N(c, LOAD_CONST, keytuple, consts); + ADDOP_I(c, BUILD_CONST_KEY_MAP, len); + return 1; + } + else { + Py_DECREF(names); + return -1; } - Py_DECREF(names); - - /* We just checked that len <= 65535, see above */ - return Py_SAFE_DOWNCAST(len + 1, Py_ssize_t, int); error: Py_DECREF(names); @@ -1625,6 +1689,36 @@ error: } static int +compiler_visit_defaults(struct compiler *c, arguments_ty args) +{ + VISIT_SEQ(c, expr, args->defaults); + ADDOP_I(c, BUILD_TUPLE, asdl_seq_LEN(args->defaults)); + return 1; +} + +static Py_ssize_t +compiler_default_arguments(struct compiler *c, arguments_ty args) +{ + Py_ssize_t funcflags = 0; + if (args->defaults && asdl_seq_LEN(args->defaults) > 0) { + if (!compiler_visit_defaults(c, args)) + return -1; + funcflags |= 0x01; + } + if (args->kwonlyargs) { + int res = compiler_visit_kwonlydefaults(c, args->kwonlyargs, + args->kw_defaults); + if (res == 0) { + return -1; + } + else if (res > 0) { + funcflags |= 0x02; + } + } + return funcflags; +} + +static int compiler_function(struct compiler *c, stmt_ty s, int is_async) { PyCodeObject *co; @@ -1635,12 +1729,11 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) asdl_seq* decos; asdl_seq *body; stmt_ty st; - Py_ssize_t i, n, arglength; - int docstring, kw_default_count = 0; - int num_annotations; + Py_ssize_t i, n, funcflags; + int docstring; + int annotations; int scope_type; - if (is_async) { assert(s->kind == AsyncFunctionDef_kind); @@ -1665,30 +1758,32 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) if (!compiler_decorators(c, decos)) return 0; - if (args->defaults) - VISIT_SEQ(c, expr, args->defaults); - if (args->kwonlyargs) { - int res = compiler_visit_kwonlydefaults(c, args->kwonlyargs, - args->kw_defaults); - if (res == 0) - return 0; - kw_default_count = res - 1; + + funcflags = compiler_default_arguments(c, args); + if (funcflags == -1) { + return 0; } - num_annotations = compiler_visit_annotations(c, args, returns); - if (num_annotations == 0) + + annotations = compiler_visit_annotations(c, args, returns); + if (annotations == 0) { return 0; - num_annotations--; - assert((num_annotations & 0xFFFF) == num_annotations); + } + else if (annotations > 0) { + funcflags |= 0x04; + } - if (!compiler_enter_scope(c, name, - scope_type, (void *)s, - s->lineno)) + if (!compiler_enter_scope(c, name, scope_type, (void *)s, s->lineno)) { return 0; + } st = (stmt_ty)asdl_seq_GET(body, 0); docstring = compiler_isdocstring(st); - if (docstring && c->c_optimize < 2) - first_const = st->v.Expr.value->v.Str.s; + if (docstring && c->c_optimize < 2) { + if (st->v.Expr.value->kind == Constant_kind) + first_const = st->v.Expr.value->v.Constant.value; + else + first_const = st->v.Expr.value->v.Str.s; + } if (compiler_add_o(c, c->u->u_consts, first_const) < 0) { compiler_exit_scope(c); return 0; @@ -1712,12 +1807,9 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) return 0; } - arglength = asdl_seq_LEN(args->defaults); - arglength |= kw_default_count << 8; - arglength |= num_annotations << 16; if (is_async) co->co_flags |= CO_COROUTINE; - compiler_make_closure(c, co, arglength, qualname); + compiler_make_closure(c, co, funcflags, qualname); Py_DECREF(qualname); Py_DECREF(co); @@ -1877,8 +1969,7 @@ compiler_lambda(struct compiler *c, expr_ty e) PyCodeObject *co; PyObject *qualname; static identifier name; - int kw_default_count = 0; - Py_ssize_t arglength; + Py_ssize_t funcflags; arguments_ty args = e->v.Lambda.args; assert(e->kind == Lambda_kind); @@ -1888,14 +1979,11 @@ compiler_lambda(struct compiler *c, expr_ty e) return 0; } - if (args->defaults) - VISIT_SEQ(c, expr, args->defaults); - if (args->kwonlyargs) { - int res = compiler_visit_kwonlydefaults(c, args->kwonlyargs, - args->kw_defaults); - if (res == 0) return 0; - kw_default_count = res - 1; + funcflags = compiler_default_arguments(c, args); + if (funcflags == -1) { + return 0; } + if (!compiler_enter_scope(c, name, COMPILER_SCOPE_LAMBDA, (void *)e, e->lineno)) return 0; @@ -1921,9 +2009,7 @@ compiler_lambda(struct compiler *c, expr_ty e) if (co == NULL) return 0; - arglength = asdl_seq_LEN(args->defaults); - arglength |= kw_default_count << 8; - compiler_make_closure(c, co, arglength, qualname); + compiler_make_closure(c, co, funcflags, qualname); Py_DECREF(qualname); Py_DECREF(co); @@ -2599,6 +2685,25 @@ compiler_assert(struct compiler *c, stmt_ty s) } static int +compiler_visit_stmt_expr(struct compiler *c, expr_ty value) +{ + if (c->c_interactive && c->c_nestlevel <= 1) { + VISIT(c, expr, value); + ADDOP(c, PRINT_EXPR); + return 1; + } + + if (is_const(value)) { + /* ignore constant statement */ + return 1; + } + + VISIT(c, expr, value); + ADDOP(c, POP_TOP); + return 1; +} + +static int compiler_visit_stmt(struct compiler *c, stmt_ty s) { Py_ssize_t i, n; @@ -2668,16 +2773,7 @@ compiler_visit_stmt(struct compiler *c, stmt_ty s) case Nonlocal_kind: break; case Expr_kind: - if (c->c_interactive && c->c_nestlevel <= 1) { - VISIT(c, expr, s->v.Expr.value); - ADDOP(c, PRINT_EXPR); - } - else if (s->v.Expr.value->kind != Str_kind && - s->v.Expr.value->kind != Num_kind) { - VISIT(c, expr, s->v.Expr.value); - ADDOP(c, POP_TOP); - } - break; + return compiler_visit_stmt_expr(c, s->v.Expr.value); case Pass_kind: break; case Break_kind: @@ -3079,9 +3175,53 @@ compiler_set(struct compiler *c, expr_ty e) } static int +are_all_items_const(asdl_seq *seq, Py_ssize_t begin, Py_ssize_t end) +{ + Py_ssize_t i; + for (i = begin; i < end; i++) { + expr_ty key = (expr_ty)asdl_seq_GET(seq, i); + if (key == NULL || !is_const(key)) + return 0; + } + return 1; +} + +static int +compiler_subdict(struct compiler *c, expr_ty e, Py_ssize_t begin, Py_ssize_t end) +{ + Py_ssize_t i, n = end - begin; + PyObject *keys, *key; + if (n > 1 && are_all_items_const(e->v.Dict.keys, begin, end)) { + for (i = begin; i < end; i++) { + VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.values, i)); + } + keys = PyTuple_New(n); + if (keys == NULL) { + return 0; + } + for (i = begin; i < end; i++) { + key = get_const_value((expr_ty)asdl_seq_GET(e->v.Dict.keys, i)); + Py_INCREF(key); + PyTuple_SET_ITEM(keys, i - begin, key); + } + ADDOP_N(c, LOAD_CONST, keys, consts); + ADDOP_I(c, BUILD_CONST_KEY_MAP, n); + } + else { + for (i = begin; i < end; i++) { + VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.keys, i)); + VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.values, i)); + } + ADDOP_I(c, BUILD_MAP, n); + } + return 1; +} + +static int compiler_dict(struct compiler *c, expr_ty e) { - Py_ssize_t i, n, containers, elements; + Py_ssize_t i, n, elements; + int containers; int is_unpacking = 0; n = asdl_seq_LEN(e->v.Dict.values); containers = 0; @@ -3089,7 +3229,8 @@ compiler_dict(struct compiler *c, expr_ty e) for (i = 0; i < n; i++) { is_unpacking = (expr_ty)asdl_seq_GET(e->v.Dict.keys, i) == NULL; if (elements == 0xFFFF || (elements && is_unpacking)) { - ADDOP_I(c, BUILD_MAP, elements); + if (!compiler_subdict(c, e, i - elements, i)) + return 0; containers++; elements = 0; } @@ -3098,13 +3239,12 @@ compiler_dict(struct compiler *c, expr_ty e) containers++; } else { - VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.keys, i)); - VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.values, i)); elements++; } } if (elements || containers == 0) { - ADDOP_I(c, BUILD_MAP, elements); + if (!compiler_subdict(c, e, n - elements, n)) + return 0; containers++; } /* If there is more than one dict, they need to be merged into a new @@ -3173,15 +3313,104 @@ compiler_call(struct compiler *c, expr_ty e) e->v.Call.keywords); } +static int +compiler_joined_str(struct compiler *c, expr_ty e) +{ + VISIT_SEQ(c, expr, e->v.JoinedStr.values); + ADDOP_I(c, BUILD_STRING, asdl_seq_LEN(e->v.JoinedStr.values)); + return 1; +} + +/* Used to implement f-strings. Format a single value. */ +static int +compiler_formatted_value(struct compiler *c, expr_ty e) +{ + /* Our oparg encodes 2 pieces of information: the conversion + character, and whether or not a format_spec was provided. + + Convert the conversion char to 2 bits: + None: 000 0x0 FVC_NONE + !s : 001 0x1 FVC_STR + !r : 010 0x2 FVC_REPR + !a : 011 0x3 FVC_ASCII + + next bit is whether or not we have a format spec: + yes : 100 0x4 + no : 000 0x0 + */ + + int oparg; + + /* Evaluate the expression to be formatted. */ + VISIT(c, expr, e->v.FormattedValue.value); + + switch (e->v.FormattedValue.conversion) { + case 's': oparg = FVC_STR; break; + case 'r': oparg = FVC_REPR; break; + case 'a': oparg = FVC_ASCII; break; + case -1: oparg = FVC_NONE; break; + default: + PyErr_SetString(PyExc_SystemError, + "Unrecognized conversion character"); + return 0; + } + if (e->v.FormattedValue.format_spec) { + /* Evaluate the format spec, and update our opcode arg. */ + VISIT(c, expr, e->v.FormattedValue.format_spec); + oparg |= FVS_HAVE_SPEC; + } + + /* And push our opcode and oparg */ + ADDOP_I(c, FORMAT_VALUE, oparg); + return 1; +} + +static int +compiler_subkwargs(struct compiler *c, asdl_seq *keywords, Py_ssize_t begin, Py_ssize_t end) +{ + Py_ssize_t i, n = end - begin; + keyword_ty kw; + PyObject *keys, *key; + assert(n > 0); + if (n > 1) { + for (i = begin; i < end; i++) { + kw = asdl_seq_GET(keywords, i); + VISIT(c, expr, kw->value); + } + keys = PyTuple_New(n); + if (keys == NULL) { + return 0; + } + for (i = begin; i < end; i++) { + key = ((keyword_ty) asdl_seq_GET(keywords, i))->arg; + Py_INCREF(key); + PyTuple_SET_ITEM(keys, i - begin, key); + } + ADDOP_N(c, LOAD_CONST, keys, consts); + ADDOP_I(c, BUILD_CONST_KEY_MAP, n); + } + else { + /* a for loop only executes once */ + for (i = begin; i < end; i++) { + kw = asdl_seq_GET(keywords, i); + ADDOP_O(c, LOAD_CONST, kw->arg, consts); + VISIT(c, expr, kw->value); + } + ADDOP_I(c, BUILD_MAP, n); + } + return 1; +} + /* shared code between compiler_call and compiler_class */ static int compiler_call_helper(struct compiler *c, - Py_ssize_t n, /* Args already pushed */ + int n, /* Args already pushed */ asdl_seq *args, asdl_seq *keywords) { int code = 0; - Py_ssize_t nelts, i, nseen, nkw; + Py_ssize_t nelts, i, nseen; + int nkw; /* the number of tuples and dictionaries on the stack */ Py_ssize_t nsubargs = 0, nsubkwargs = 0; @@ -3238,29 +3467,38 @@ compiler_call_helper(struct compiler *c, if (kw->arg == NULL) { /* A keyword argument unpacking. */ if (nseen) { - ADDOP_I(c, BUILD_MAP, nseen); + if (nsubkwargs) { + if (!compiler_subkwargs(c, keywords, i - nseen, i)) + return 0; + nsubkwargs++; + } + else { + Py_ssize_t j; + for (j = 0; j < nseen; j++) { + VISIT(c, keyword, asdl_seq_GET(keywords, j)); + } + nkw = nseen; + } nseen = 0; - nsubkwargs++; } VISIT(c, expr, kw->value); nsubkwargs++; } - else if (nsubkwargs) { - /* A keyword argument and we already have a dict. */ - ADDOP_O(c, LOAD_CONST, kw->arg, consts); - VISIT(c, expr, kw->value); - nseen++; - } else { - /* keyword argument */ - VISIT(c, keyword, kw) - nkw++; + nseen++; } } if (nseen) { - /* Pack up any trailing keyword arguments. */ - ADDOP_I(c, BUILD_MAP, nseen); - nsubkwargs++; + if (nsubkwargs) { + /* Pack up any trailing keyword arguments. */ + if (!compiler_subkwargs(c, keywords, nelts - nseen, nelts)) + return 0; + nsubkwargs++; + } + else { + VISIT_SEQ(c, keyword, keywords); + nkw = nseen; + } } if (nsubkwargs) { code |= 2; @@ -3549,6 +3787,8 @@ expr_constant(struct compiler *c, expr_ty e) switch (e->kind) { case Ellipsis_kind: return 1; + case Constant_kind: + return PyObject_IsTrue(e->v.Constant.value); case Num_kind: return PyObject_IsTrue(e->v.Num.n); case Str_kind: @@ -3592,9 +3832,9 @@ expr_constant(struct compiler *c, expr_ty e) BLOCK finally: if an exception was raised: - exc = copy of (exception, instance, traceback) + exc = copy of (exception, instance, traceback) else: - exc = (None, None, None) + exc = (None, None, None) if not (await exit(*exc)): raise */ @@ -3836,12 +4076,19 @@ compiler_visit_expr(struct compiler *c, expr_ty e) return compiler_compare(c, e); case Call_kind: return compiler_call(c, e); + case Constant_kind: + ADDOP_O(c, LOAD_CONST, e->v.Constant.value, consts); + break; case Num_kind: ADDOP_O(c, LOAD_CONST, e->v.Num.n, consts); break; case Str_kind: ADDOP_O(c, LOAD_CONST, e->v.Str.s, consts); break; + case JoinedStr_kind: + return compiler_joined_str(c, e); + case FormattedValue_kind: + return compiler_formatted_value(c, e); case Bytes_kind: ADDOP_O(c, LOAD_CONST, e->v.Bytes.s, consts); break; @@ -4273,7 +4520,7 @@ assemble_init(struct assembler *a, int nblocks, int firstlineno) a->a_lnotab = PyBytes_FromStringAndSize(NULL, DEFAULT_LNOTAB_SIZE); if (!a->a_lnotab) return 0; - if ((size_t)nblocks > PY_SIZE_MAX / sizeof(basicblock *)) { + if ((size_t)nblocks > SIZE_MAX / sizeof(basicblock *)) { PyErr_NoMemory(); return 0; } @@ -4295,18 +4542,6 @@ assemble_free(struct assembler *a) PyObject_Free(a->a_postorder); } -/* Return the size of a basic block in bytes. */ - -static int -instrsize(struct instr *instr) -{ - if (!instr->i_hasarg) - return 1; /* 1 byte for the opcode*/ - if (instr->i_oparg > 0xffff) - return 6; /* 1 (opcode) + 1 (EXTENDED_ARG opcode) + 2 (oparg) + 2(oparg extended) */ - return 3; /* 1 (opcode) + 2 (oparg) */ -} - static int blocksize(basicblock *b) { @@ -4314,7 +4549,7 @@ blocksize(basicblock *b) int size = 0; for (i = 0; i < b->b_iused; i++) - size += instrsize(&b->b_instr[i]); + size += instrsize(b->b_instr[i].i_oparg); return size; } @@ -4333,7 +4568,6 @@ assemble_lnotab(struct assembler *a, struct instr *i) d_lineno = i->i_lineno - a->a_lineno; assert(d_bytecode >= 0); - assert(d_lineno >= 0); if(d_bytecode == 0 && d_lineno == 0) return 1; @@ -4363,9 +4597,21 @@ assemble_lnotab(struct assembler *a, struct instr *i) d_bytecode -= ncodes * 255; a->a_lnotab_off += ncodes * 2; } - assert(d_bytecode <= 255); - if (d_lineno > 255) { - int j, nbytes, ncodes = d_lineno / 255; + assert(0 <= d_bytecode && d_bytecode <= 255); + + if (d_lineno < -128 || 127 < d_lineno) { + int j, nbytes, ncodes, k; + if (d_lineno < 0) { + k = -128; + /* use division on positive numbers */ + ncodes = (-d_lineno) / 128; + } + else { + k = 127; + ncodes = d_lineno / 127; + } + d_lineno -= ncodes * k; + assert(ncodes >= 1); nbytes = a->a_lnotab_off + 2 * ncodes; len = PyBytes_GET_SIZE(a->a_lnotab); if (nbytes >= len) { @@ -4383,15 +4629,15 @@ assemble_lnotab(struct assembler *a, struct instr *i) lnotab = (unsigned char *) PyBytes_AS_STRING(a->a_lnotab) + a->a_lnotab_off; *lnotab++ = d_bytecode; - *lnotab++ = 255; + *lnotab++ = k; d_bytecode = 0; for (j = 1; j < ncodes; j++) { *lnotab++ = 0; - *lnotab++ = 255; + *lnotab++ = k; } - d_lineno -= ncodes * 255; a->a_lnotab_off += ncodes * 2; } + assert(-128 <= d_lineno && d_lineno <= 127); len = PyBytes_GET_SIZE(a->a_lnotab); if (a->a_lnotab_off + 2 >= len) { @@ -4423,15 +4669,12 @@ assemble_lnotab(struct assembler *a, struct instr *i) static int assemble_emit(struct assembler *a, struct instr *i) { - int size, arg = 0, ext = 0; + int size, arg = 0; Py_ssize_t len = PyBytes_GET_SIZE(a->a_bytecode); char *code; - size = instrsize(i); - if (i->i_hasarg) { - arg = i->i_oparg; - ext = arg >> 16; - } + arg = i->i_oparg; + size = instrsize(arg); if (i->i_lineno && !assemble_lnotab(a, i)) return 0; if (a->a_offset + size >= len) { @@ -4442,19 +4685,7 @@ assemble_emit(struct assembler *a, struct instr *i) } code = PyBytes_AS_STRING(a->a_bytecode) + a->a_offset; a->a_offset += size; - if (size == 6) { - assert(i->i_hasarg); - *code++ = (char)EXTENDED_ARG; - *code++ = ext & 0xff; - *code++ = ext >> 8; - arg &= 0xffff; - } - *code++ = i->i_opcode; - if (i->i_hasarg) { - assert(size == 3 || size == 6); - *code++ = arg & 0xff; - *code++ = arg >> 8; - } + write_op_arg((unsigned char*)code, i->i_opcode, arg, size); return 1; } @@ -4462,7 +4693,7 @@ static void assemble_jump_offsets(struct assembler *a, struct compiler *c) { basicblock *b; - int bsize, totsize, extended_arg_count = 0, last_extended_arg_count; + int bsize, totsize, extended_arg_recompile; int i; /* Compute the size of each block and fixup jump args. @@ -4475,27 +4706,26 @@ assemble_jump_offsets(struct assembler *a, struct compiler *c) b->b_offset = totsize; totsize += bsize; } - last_extended_arg_count = extended_arg_count; - extended_arg_count = 0; + extended_arg_recompile = 0; for (b = c->u->u_blocks; b != NULL; b = b->b_list) { bsize = b->b_offset; for (i = 0; i < b->b_iused; i++) { struct instr *instr = &b->b_instr[i]; + int isize = instrsize(instr->i_oparg); /* Relative jumps are computed relative to the instruction pointer after fetching the jump instruction. */ - bsize += instrsize(instr); - if (instr->i_jabs) + bsize += isize; + if (instr->i_jabs || instr->i_jrel) { instr->i_oparg = instr->i_target->b_offset; - else if (instr->i_jrel) { - int delta = instr->i_target->b_offset - bsize; - instr->i_oparg = delta; + if (instr->i_jrel) { + instr->i_oparg -= bsize; + } + if (instrsize(instr->i_oparg) != isize) { + extended_arg_recompile = 1; + } } - else - continue; - if (instr->i_oparg > 0xffff) - extended_arg_count++; } } @@ -4505,7 +4735,7 @@ assemble_jump_offsets(struct assembler *a, struct compiler *c) The issue is that in the first loop blocksize() is called which calls instrsize() which requires i_oparg be set - appropriately. There is a bootstrap problem because + appropriately. There is a bootstrap problem because i_oparg is calculated in the second loop above. So we loop until we stop seeing new EXTENDED_ARGs. @@ -4513,7 +4743,7 @@ assemble_jump_offsets(struct assembler *a, struct compiler *c) ones in jump instructions. So this should converge fairly quickly. */ - } while (last_extended_arg_count != extended_arg_count); + } while (extended_arg_recompile); } static PyObject * @@ -4659,9 +4889,9 @@ dump_instr(const struct instr *i) char arg[128]; *arg = '\0'; - if (i->i_hasarg) + if (HAS_ARG(i->i_opcode)) { sprintf(arg, "arg: %d ", i->i_oparg); - + } fprintf(stderr, "line: %d, opcode: %d %s%s%s\n", i->i_lineno, i->i_opcode, arg, jabs, jrel); } @@ -4711,7 +4941,7 @@ assemble(struct compiler *c, int addNone) /* Set firstlineno if it wasn't explicitly set. */ if (!c->u->u_firstlineno) { - if (entryblock && entryblock->b_instr) + if (entryblock && entryblock->b_instr && entryblock->b_instr->i_lineno) c->u->u_firstlineno = entryblock->b_instr->i_lineno; else c->u->u_firstlineno = 1; @@ -4749,4 +4979,3 @@ PyAST_Compile(mod_ty mod, const char *filename, PyCompilerFlags *flags, { return PyAST_CompileEx(mod, filename, flags, -1, arena); } - |