diff options
Diffstat (limited to 'Python/ceval.c')
| -rw-r--r-- | Python/ceval.c | 1947 |
1 files changed, 1034 insertions, 913 deletions
diff --git a/Python/ceval.c b/Python/ceval.c index 7b405188d3..d5172b9631 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -15,87 +15,12 @@ #include "dictobject.h" #include "frameobject.h" #include "opcode.h" +#include "pydtrace.h" #include "setobject.h" #include "structmember.h" #include <ctype.h> -#ifndef WITH_TSC - -#define READ_TIMESTAMP(var) - -#else - -typedef unsigned long long uint64; - -/* PowerPC support. - "__ppc__" appears to be the preprocessor definition to detect on OS X, whereas - "__powerpc__" appears to be the correct one for Linux with GCC -*/ -#if defined(__ppc__) || defined (__powerpc__) - -#define READ_TIMESTAMP(var) ppc_getcounter(&var) - -static void -ppc_getcounter(uint64 *v) -{ - unsigned long tbu, tb, tbu2; - - loop: - asm volatile ("mftbu %0" : "=r" (tbu) ); - asm volatile ("mftb %0" : "=r" (tb) ); - asm volatile ("mftbu %0" : "=r" (tbu2)); - if (__builtin_expect(tbu != tbu2, 0)) goto loop; - - /* The slightly peculiar way of writing the next lines is - compiled better by GCC than any other way I tried. */ - ((long*)(v))[0] = tbu; - ((long*)(v))[1] = tb; -} - -#elif defined(__i386__) - -/* this is for linux/x86 (and probably any other GCC/x86 combo) */ - -#define READ_TIMESTAMP(val) \ - __asm__ __volatile__("rdtsc" : "=A" (val)) - -#elif defined(__x86_64__) - -/* for gcc/x86_64, the "A" constraint in DI mode means *either* rax *or* rdx; - not edx:eax as it does for i386. Since rdtsc puts its result in edx:eax - even in 64-bit mode, we need to use "a" and "d" for the lower and upper - 32-bit pieces of the result. */ - -#define READ_TIMESTAMP(val) do { \ - unsigned int h, l; \ - __asm__ __volatile__("rdtsc" : "=a" (l), "=d" (h)); \ - (val) = ((uint64)l) | (((uint64)h) << 32); \ - } while(0) - - -#else - -#error "Don't know how to implement timestamp counter for this architecture" - -#endif - -void dump_tsc(int opcode, int ticked, uint64 inst0, uint64 inst1, - uint64 loop0, uint64 loop1, uint64 intr0, uint64 intr1) -{ - uint64 intr, inst, loop; - PyThreadState *tstate = PyThreadState_Get(); - if (!tstate->interp->tscdump) - return; - intr = intr1 - intr0; - inst = inst1 - inst0 - intr; - loop = loop1 - loop0 - intr; - fprintf(stderr, "opcode=%03d t=%d inst=%06lld loop=%06lld\n", - opcode, ticked, inst, loop); -} - -#endif - /* Turn this on if your compiler chokes on the big switch: */ /* #define CASE_TOO_BIG 1 */ @@ -108,24 +33,13 @@ void dump_tsc(int opcode, int ticked, uint64 inst0, uint64 inst1, typedef PyObject *(*callproc)(PyObject *, PyObject *, PyObject *); /* Forward declarations */ -#ifdef WITH_TSC -static PyObject * call_function(PyObject ***, int, uint64*, uint64*); -#else -static PyObject * call_function(PyObject ***, int); -#endif -static PyObject * fast_function(PyObject *, PyObject ***, int, int, int); -static PyObject * do_call(PyObject *, PyObject ***, int, int); -static PyObject * ext_do_call(PyObject *, PyObject ***, int, int, int); -static PyObject * update_keyword_args(PyObject *, int, PyObject ***, - PyObject *); -static PyObject * update_star_args(int, int, PyObject *, PyObject ***); -static PyObject * load_args(PyObject ***, int); -#define CALL_FLAG_VAR 1 -#define CALL_FLAG_KW 2 +static PyObject * call_function(PyObject ***, Py_ssize_t, PyObject *); +static PyObject * fast_function(PyObject *, PyObject **, Py_ssize_t, PyObject *); +static PyObject * do_call_core(PyObject *, PyObject *, PyObject *); #ifdef LLTRACE static int lltrace; -static int prtrace(PyObject *, char *); +static int prtrace(PyObject *, const char *); #endif static int call_trace(Py_tracefunc, PyObject *, PyThreadState *, PyFrameObject *, @@ -137,14 +51,18 @@ static void call_exc_trace(Py_tracefunc, PyObject *, PyThreadState *, PyFrameObject *); static int maybe_call_line_trace(Py_tracefunc, PyObject *, PyThreadState *, PyFrameObject *, int *, int *, int *); +static void maybe_dtrace_line(PyFrameObject *, int *, int *, int *); +static void dtrace_function_entry(PyFrameObject *); +static void dtrace_function_return(PyFrameObject *); static PyObject * cmp_outcome(int, PyObject *, PyObject *); +static PyObject * import_name(PyFrameObject *, PyObject *, PyObject *, PyObject *); static PyObject * import_from(PyObject *, PyObject *); static int import_all_from(PyObject *, PyObject *); static void format_exc_check_arg(PyObject *, const char *, PyObject *); static void format_exc_unbound(PyCodeObject *co, int oparg); static PyObject * unicode_concatenate(PyObject *, PyObject *, - PyFrameObject *, unsigned char *); + PyFrameObject *, const _Py_CODEUNIT *); static PyObject * special_lookup(PyObject *, _Py_Identifier *); #define NAME_ERROR_MSG \ @@ -796,11 +714,18 @@ PyEval_EvalFrame(PyFrameObject *f) { PyObject * PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) { + PyThreadState *tstate = PyThreadState_GET(); + return tstate->interp->eval_frame(f, throwflag); +} + +PyObject * +_PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) +{ #ifdef DXPAIRS int lastopcode = 0; #endif PyObject **stack_pointer; /* Next free slot in value stack */ - unsigned char *next_instr; + const _Py_CODEUNIT *next_instr; int opcode; /* Current opcode */ int oparg; /* Current opcode argument, if any */ enum why_code why; /* Reason for block stack unwind */ @@ -818,7 +743,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) time it is tested. */ int instr_ub = -1, instr_lb = 0, instr_prev = -1; - unsigned char *first_instr; + const _Py_CODEUNIT *first_instr; PyObject *names; PyObject *consts; @@ -886,24 +811,10 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) /* Import the static jump table */ #include "opcode_targets.h" -/* This macro is used when several opcodes defer to the same implementation - (e.g. SETUP_LOOP, SETUP_FINALLY) */ -#define TARGET_WITH_IMPL(op, impl) \ - TARGET_##op: \ - opcode = op; \ - if (HAS_ARG(op)) \ - oparg = NEXTARG(); \ - case op: \ - goto impl; \ - #define TARGET(op) \ TARGET_##op: \ - opcode = op; \ - if (HAS_ARG(op)) \ - oparg = NEXTARG(); \ case op: - #define DISPATCH() \ { \ if (!_Py_atomic_load_relaxed(&eval_breaker)) { \ @@ -915,18 +826,20 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) #ifdef LLTRACE #define FAST_DISPATCH() \ { \ - if (!lltrace && !_Py_TracingPossible) { \ + if (!lltrace && !_Py_TracingPossible && !PyDTrace_LINE_ENABLED()) { \ f->f_lasti = INSTR_OFFSET(); \ - goto *opcode_targets[*next_instr++]; \ + NEXTOPARG(); \ + goto *opcode_targets[opcode]; \ } \ goto fast_next_opcode; \ } #else #define FAST_DISPATCH() \ { \ - if (!_Py_TracingPossible) { \ + if (!_Py_TracingPossible && !PyDTrace_LINE_ENABLED()) { \ f->f_lasti = INSTR_OFFSET(); \ - goto *opcode_targets[*next_instr++]; \ + NEXTOPARG(); \ + goto *opcode_targets[opcode]; \ } \ goto fast_next_opcode; \ } @@ -935,10 +848,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) #else #define TARGET(op) \ case op: -#define TARGET_WITH_IMPL(op, impl) \ - /* silence compiler warnings about `impl` unused */ \ - if (0) goto impl; \ - case op: + #define DISPATCH() continue #define FAST_DISPATCH() goto fast_next_opcode #endif @@ -952,70 +862,33 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) #define GETITEM(v, i) PyTuple_GetItem((v), (i)) #endif -#ifdef WITH_TSC -/* Use Pentium timestamp counter to mark certain events: - inst0 -- beginning of switch statement for opcode dispatch - inst1 -- end of switch statement (may be skipped) - loop0 -- the top of the mainloop - loop1 -- place where control returns again to top of mainloop - (may be skipped) - intr1 -- beginning of long interruption - intr2 -- end of long interruption - - Many opcodes call out to helper C functions. In some cases, the - time in those functions should be counted towards the time for the - opcode, but not in all cases. For example, a CALL_FUNCTION opcode - calls another Python function; there's no point in charge all the - bytecode executed by the called function to the caller. - - It's hard to make a useful judgement statically. In the presence - of operator overloading, it's impossible to tell if a call will - execute new Python code or not. - - It's a case-by-case judgement. I'll use intr1 for the following - cases: - - IMPORT_STAR - IMPORT_FROM - CALL_FUNCTION (and friends) - - */ - uint64 inst0, inst1, loop0, loop1, intr0 = 0, intr1 = 0; - int ticked = 0; - - READ_TIMESTAMP(inst0); - READ_TIMESTAMP(inst1); - READ_TIMESTAMP(loop0); - READ_TIMESTAMP(loop1); - - /* shut up the compiler */ - opcode = 0; -#endif - /* Code access macros */ -#define INSTR_OFFSET() ((int)(next_instr - first_instr)) -#define NEXTOP() (*next_instr++) -#define NEXTARG() (next_instr += 2, (next_instr[-1]<<8) + next_instr[-2]) -#define PEEKARG() ((next_instr[2]<<8) + next_instr[1]) -#define JUMPTO(x) (next_instr = first_instr + (x)) -#define JUMPBY(x) (next_instr += (x)) +/* The integer overflow is checked by an assertion below. */ +#define INSTR_OFFSET() (sizeof(_Py_CODEUNIT) * (int)(next_instr - first_instr)) +#define NEXTOPARG() do { \ + _Py_CODEUNIT word = *next_instr; \ + opcode = _Py_OPCODE(word); \ + oparg = _Py_OPARG(word); \ + next_instr++; \ + } while (0) +#define JUMPTO(x) (next_instr = first_instr + (x) / sizeof(_Py_CODEUNIT)) +#define JUMPBY(x) (next_instr += (x) / sizeof(_Py_CODEUNIT)) /* OpCode prediction macros Some opcodes tend to come in pairs thus making it possible to predict the second code when the first is run. For example, - COMPARE_OP is often followed by JUMP_IF_FALSE or JUMP_IF_TRUE. And, - those opcodes are often followed by a POP_TOP. + COMPARE_OP is often followed by POP_JUMP_IF_FALSE or POP_JUMP_IF_TRUE. Verifying the prediction costs a single high-speed test of a register variable against a constant. If the pairing was good, then the processor's own internal branch predication has a high likelihood of success, resulting in a nearly zero-overhead transition to the next opcode. A successful prediction saves a trip through the eval-loop - including its two unpredictable branches, the HAS_ARG test and the - switch-case. Combined with the processor's internal branch prediction, - a successful PREDICT has the effect of making the two opcodes run as if - they were a single new opcode with the bodies combined. + including its unpredictable switch-case branch. Combined with the + processor's internal branch prediction, a successful PREDICT has the + effect of making the two opcodes run as if they were a single new opcode + with the bodies combined. If collecting opcode statistics, your choices are to either keep the predictions turned-on and interpret the results as if some opcodes @@ -1030,13 +903,19 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) #if defined(DYNAMIC_EXECUTION_PROFILE) || USE_COMPUTED_GOTOS #define PREDICT(op) if (0) goto PRED_##op -#define PREDICTED(op) PRED_##op: -#define PREDICTED_WITH_ARG(op) PRED_##op: #else -#define PREDICT(op) if (*next_instr == op) goto PRED_##op -#define PREDICTED(op) PRED_##op: next_instr++ -#define PREDICTED_WITH_ARG(op) PRED_##op: oparg = PEEKARG(); next_instr += 3 +#define PREDICT(op) \ + do{ \ + _Py_CODEUNIT word = *next_instr; \ + opcode = _Py_OPCODE(word); \ + if (opcode == op){ \ + oparg = _Py_OPARG(word); \ + next_instr++; \ + goto PRED_##op; \ + } \ + } while(0) #endif +#define PREDICTED(op) PRED_##op: /* Stack manipulation macros */ @@ -1100,7 +979,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) } #define UNWIND_EXCEPT_HANDLER(b) \ - { \ + do { \ PyObject *type, *value, *traceback; \ assert(STACK_LEVEL() >= (b)->b_level + 3); \ while (STACK_LEVEL() > (b)->b_level + 3) { \ @@ -1116,7 +995,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) Py_XDECREF(type); \ Py_XDECREF(value); \ Py_XDECREF(traceback); \ - } + } while(0) /* Start of code */ @@ -1160,21 +1039,24 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) } } + if (PyDTrace_FUNCTION_ENTRY_ENABLED()) + dtrace_function_entry(f); + co = f->f_code; names = co->co_names; consts = co->co_consts; fastlocals = f->f_localsplus; freevars = f->f_localsplus + co->co_nlocals; - first_instr = (unsigned char*) PyBytes_AS_STRING(co->co_code); - /* An explanation is in order for the next line. - - f->f_lasti now refers to the index of the last instruction - executed. You might think this was obvious from the name, but - this wasn't always true before 2.3! PyFrame_New now sets - f->f_lasti to -1 (i.e. the index *before* the first instruction) - and YIELD_VALUE doesn't fiddle with f_lasti any more. So this - does work. Promise. - YIELD_FROM sets f_lasti to itself, in order to repeated yield + assert(PyBytes_Check(co->co_code)); + assert(PyBytes_GET_SIZE(co->co_code) <= INT_MAX); + assert(PyBytes_GET_SIZE(co->co_code) % sizeof(_Py_CODEUNIT) == 0); + assert(_Py_IS_ALIGNED(PyBytes_AS_STRING(co->co_code), sizeof(_Py_CODEUNIT))); + first_instr = (_Py_CODEUNIT *) PyBytes_AS_STRING(co->co_code); + /* + f->f_lasti refers to the index of the last instruction, + unless it's -1 in which case next_instr should be first_instr. + + YIELD_FROM sets f_lasti to itself, in order to repeatedly yield multiple values. When the PREDICT() macros are enabled, some opcode pairs follow in @@ -1183,15 +1065,20 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) were a single new opcode; accordingly,f->f_lasti will point to the first code in the pair (for instance, GET_ITER followed by FOR_ITER is effectively a single opcode and f->f_lasti will point - at to the beginning of the combined pair.) + to the beginning of the combined pair.) */ - next_instr = first_instr + f->f_lasti + 1; + assert(f->f_lasti >= -1); + next_instr = first_instr; + if (f->f_lasti >= 0) { + assert(f->f_lasti % sizeof(_Py_CODEUNIT) == 0); + next_instr += f->f_lasti / sizeof(_Py_CODEUNIT) + 1; + } stack_pointer = f->f_stacktop; assert(stack_pointer != NULL); f->f_stacktop = NULL; /* remains NULL unless yield suspends frame */ f->f_executing = 1; - if (co->co_flags & (CO_GENERATOR | CO_COROUTINE)) { + if (co->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) { if (!throwflag && f->f_exc_type != NULL && f->f_exc_type != Py_None) { /* We were in an except handler when we left, restore the exception state which was put aside @@ -1219,23 +1106,6 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) #endif for (;;) { -#ifdef WITH_TSC - if (inst1 == 0) { - /* Almost surely, the opcode executed a break - or a continue, preventing inst1 from being set - on the way out of the loop. - */ - READ_TIMESTAMP(inst1); - loop1 = inst1; - } - dump_tsc(opcode, ticked, inst0, inst1, loop0, loop1, - intr0, intr1); - ticked = 0; - inst1 = 0; - intr0 = 0; - intr1 = 0; - READ_TIMESTAMP(loop0); -#endif assert(stack_pointer >= f->f_valuestack); /* else underflow */ assert(STACK_LEVEL() <= co->co_stacksize); /* else overflow */ assert(!PyErr_Occurred()); @@ -1249,14 +1119,11 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) Py_MakePendingCalls() above. */ if (_Py_atomic_load_relaxed(&eval_breaker)) { - if (*next_instr == SETUP_FINALLY) { + if (_Py_OPCODE(*next_instr) == SETUP_FINALLY) { /* Make the last opcode before a try: finally: block uninterruptible. */ goto fast_next_opcode; } -#ifdef WITH_TSC - ticked = 1; -#endif if (_Py_atomic_load_relaxed(&pendingcalls_to_do)) { if (Py_MakePendingCalls() < 0) goto error; @@ -1296,6 +1163,9 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) fast_next_opcode: f->f_lasti = INSTR_OFFSET(); + if (PyDTrace_LINE_ENABLED()) + maybe_dtrace_line(f, &instr_lb, &instr_ub, &instr_prev); + /* line-by-line tracing support */ if (_Py_TracingPossible && @@ -1322,11 +1192,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) /* Extract opcode and argument */ - opcode = NEXTOP(); - oparg = 0; /* allows oparg to be stored in a register because - it doesn't have to be remembered across a full loop */ - if (HAS_ARG(opcode)) - oparg = NEXTARG(); + NEXTOPARG(); dispatch_opcode: #ifdef DYNAMIC_EXECUTION_PROFILE #ifdef DXPAIRS @@ -1351,9 +1217,6 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) } #endif - /* Main switch on opcode */ - READ_TIMESTAMP(inst0); - switch (opcode) { /* BEWARE! @@ -1377,6 +1240,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) FAST_DISPATCH(); } + PREDICTED(LOAD_CONST); TARGET(LOAD_CONST) { PyObject *value = GETITEM(consts, oparg); Py_INCREF(value); @@ -1384,7 +1248,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) FAST_DISPATCH(); } - PREDICTED_WITH_ARG(STORE_FAST); + PREDICTED(STORE_FAST); TARGET(STORE_FAST) { PyObject *value = POP(); SETLOCAL(oparg, value); @@ -1863,6 +1727,61 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) DISPATCH(); } + TARGET(STORE_ANNOTATION) { + _Py_IDENTIFIER(__annotations__); + PyObject *ann_dict; + PyObject *ann = POP(); + PyObject *name = GETITEM(names, oparg); + int err; + if (f->f_locals == NULL) { + PyErr_Format(PyExc_SystemError, + "no locals found when storing annotation"); + Py_DECREF(ann); + goto error; + } + /* first try to get __annotations__ from locals... */ + if (PyDict_CheckExact(f->f_locals)) { + ann_dict = _PyDict_GetItemId(f->f_locals, + &PyId___annotations__); + if (ann_dict == NULL) { + PyErr_SetString(PyExc_NameError, + "__annotations__ not found"); + Py_DECREF(ann); + goto error; + } + Py_INCREF(ann_dict); + } + else { + PyObject *ann_str = _PyUnicode_FromId(&PyId___annotations__); + if (ann_str == NULL) { + Py_DECREF(ann); + goto error; + } + ann_dict = PyObject_GetItem(f->f_locals, ann_str); + if (ann_dict == NULL) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_SetString(PyExc_NameError, + "__annotations__ not found"); + } + Py_DECREF(ann); + goto error; + } + } + /* ...if succeeded, __annotations__[name] = ann */ + if (PyDict_CheckExact(ann_dict)) { + err = PyDict_SetItem(ann_dict, name, ann); + } + else { + err = PyObject_SetItem(ann_dict, name, ann); + } + Py_DECREF(ann_dict); + Py_DECREF(ann); + if (err != 0) { + goto error; + } + DISPATCH(); + } + TARGET(DELETE_SUBSCR) { PyObject *sub = TOP(); PyObject *container = SECOND(); @@ -1992,7 +1911,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) Py_DECREF(iter); if (PyErr_WarnFormat( - PyExc_PendingDeprecationWarning, 1, + PyExc_DeprecationWarning, 1, "'%.100s' implements legacy __aiter__ protocol; " "__aiter__ should return an asynchronous " "iterator, not awaitable", @@ -2006,6 +1925,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) } SET_TOP(awaitable); + PREDICT(LOAD_CONST); DISPATCH(); } @@ -2016,41 +1936,52 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) PyObject *aiter = TOP(); PyTypeObject *type = Py_TYPE(aiter); - if (type->tp_as_async != NULL) - getter = type->tp_as_async->am_anext; + if (PyAsyncGen_CheckExact(aiter)) { + awaitable = type->tp_as_async->am_anext(aiter); + if (awaitable == NULL) { + goto error; + } + } else { + if (type->tp_as_async != NULL){ + getter = type->tp_as_async->am_anext; + } - if (getter != NULL) { - next_iter = (*getter)(aiter); - if (next_iter == NULL) { + if (getter != NULL) { + next_iter = (*getter)(aiter); + if (next_iter == NULL) { + goto error; + } + } + else { + PyErr_Format( + PyExc_TypeError, + "'async for' requires an iterator with " + "__anext__ method, got %.100s", + type->tp_name); goto error; } - } - else { - PyErr_Format( - PyExc_TypeError, - "'async for' requires an iterator with " - "__anext__ method, got %.100s", - type->tp_name); - goto error; - } - awaitable = _PyCoro_GetAwaitableIter(next_iter); - if (awaitable == NULL) { - PyErr_Format( - PyExc_TypeError, - "'async for' received an invalid object " - "from __anext__: %.100s", - Py_TYPE(next_iter)->tp_name); + awaitable = _PyCoro_GetAwaitableIter(next_iter); + if (awaitable == NULL) { + PyErr_Format( + PyExc_TypeError, + "'async for' received an invalid object " + "from __anext__: %.100s", + Py_TYPE(next_iter)->tp_name); - Py_DECREF(next_iter); - goto error; - } else - Py_DECREF(next_iter); + Py_DECREF(next_iter); + goto error; + } else { + Py_DECREF(next_iter); + } + } PUSH(awaitable); + PREDICT(LOAD_CONST); DISPATCH(); } + PREDICTED(GET_AWAITABLE); TARGET(GET_AWAITABLE) { PyObject *iterable = TOP(); PyObject *iter = _PyCoro_GetAwaitableIter(iterable); @@ -2078,21 +2009,22 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) goto error; } + PREDICT(LOAD_CONST); DISPATCH(); } TARGET(YIELD_FROM) { PyObject *v = POP(); - PyObject *reciever = TOP(); + PyObject *receiver = TOP(); int err; - if (PyGen_CheckExact(reciever) || PyCoro_CheckExact(reciever)) { - retval = _PyGen_Send((PyGenObject *)reciever, v); + if (PyGen_CheckExact(receiver) || PyCoro_CheckExact(receiver)) { + retval = _PyGen_Send((PyGenObject *)receiver, v); } else { _Py_IDENTIFIER(send); if (v == Py_None) - retval = Py_TYPE(reciever)->tp_iternext(reciever); + retval = Py_TYPE(receiver)->tp_iternext(receiver); else - retval = _PyObject_CallMethodIdObjArgs(reciever, &PyId_send, v, NULL); + retval = _PyObject_CallMethodIdObjArgs(receiver, &PyId_send, v, NULL); } Py_DECREF(v); if (retval == NULL) { @@ -2103,7 +2035,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) err = _PyGen_FetchStopIterationValue(&val); if (err < 0) goto error; - Py_DECREF(reciever); + Py_DECREF(receiver); SET_TOP(val); DISPATCH(); } @@ -2111,12 +2043,24 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) f->f_stacktop = stack_pointer; why = WHY_YIELD; /* and repeat... */ - f->f_lasti--; + assert(f->f_lasti >= (int)sizeof(_Py_CODEUNIT)); + f->f_lasti -= sizeof(_Py_CODEUNIT); goto fast_yield; } TARGET(YIELD_VALUE) { retval = POP(); + + if (co->co_flags & CO_ASYNC_GENERATOR) { + PyObject *w = _PyAsyncGenValueWrapperNew(retval); + Py_DECREF(retval); + if (w == NULL) { + retval = NULL; + goto error; + } + retval = w; + } + f->f_stacktop = stack_pointer; why = WHY_YIELD; goto fast_yield; @@ -2133,6 +2077,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) DISPATCH(); } + PREDICTED(POP_BLOCK); TARGET(POP_BLOCK) { PyTryBlock *b = PyFrame_BlockPop(f); UNWIND_BLOCK(b); @@ -2249,7 +2194,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) DISPATCH(); } - PREDICTED_WITH_ARG(UNPACK_SEQUENCE); + PREDICTED(UNPACK_SEQUENCE); TARGET(UNPACK_SEQUENCE) { PyObject *seq = POP(), *item, **items; if (PyTuple_CheckExact(seq) && @@ -2358,7 +2303,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) } else { v = PyObject_GetItem(locals, name); - if (v == NULL && _PyErr_OCCURRED()) { + if (v == NULL) { if (!PyErr_ExceptionMatches(PyExc_KeyError)) goto error; PyErr_Clear(); @@ -2398,26 +2343,33 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) PyObject *name = GETITEM(names, oparg); PyObject *v; if (PyDict_CheckExact(f->f_globals) - && PyDict_CheckExact(f->f_builtins)) { + && PyDict_CheckExact(f->f_builtins)) + { v = _PyDict_LoadGlobal((PyDictObject *)f->f_globals, (PyDictObject *)f->f_builtins, name); if (v == NULL) { - if (!_PyErr_OCCURRED()) + if (!_PyErr_OCCURRED()) { + /* _PyDict_LoadGlobal() returns NULL without raising + * an exception if the key doesn't exist */ format_exc_check_arg(PyExc_NameError, NAME_ERROR_MSG, name); + } goto error; } Py_INCREF(v); } else { /* Slow-path if globals or builtins is not a dict */ + + /* namespace 1: globals */ v = PyObject_GetItem(f->f_globals, name); if (v == NULL) { if (!PyErr_ExceptionMatches(PyExc_KeyError)) goto error; PyErr_Clear(); + /* namespace 2: builtins */ v = PyObject_GetItem(f->f_builtins, name); if (v == NULL) { if (PyErr_ExceptionMatches(PyExc_KeyError)) @@ -2477,7 +2429,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) } else { value = PyObject_GetItem(locals, name); - if (value == NULL && PyErr_Occurred()) { + if (value == NULL) { if (!PyErr_ExceptionMatches(PyExc_KeyError)) goto error; PyErr_Clear(); @@ -2511,8 +2463,27 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) TARGET(STORE_DEREF) { PyObject *v = POP(); PyObject *cell = freevars[oparg]; - PyCell_Set(cell, v); - Py_DECREF(v); + PyObject *oldobj = PyCell_GET(cell); + PyCell_SET(cell, v); + Py_XDECREF(oldobj); + DISPATCH(); + } + + TARGET(BUILD_STRING) { + PyObject *str; + PyObject *empty = PyUnicode_New(0, 0); + if (empty == NULL) { + goto error; + } + str = _PyUnicode_JoinArray(empty, stack_pointer - oparg, oparg); + Py_DECREF(empty); + if (str == NULL) + goto error; + while (--oparg >= 0) { + PyObject *item = POP(); + Py_DECREF(item); + } + PUSH(str); DISPATCH(); } @@ -2540,13 +2511,14 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) DISPATCH(); } - TARGET_WITH_IMPL(BUILD_TUPLE_UNPACK, _build_list_unpack) - TARGET(BUILD_LIST_UNPACK) - _build_list_unpack: { - int convert_to_tuple = opcode == BUILD_TUPLE_UNPACK; - int i; + TARGET(BUILD_TUPLE_UNPACK_WITH_CALL) + TARGET(BUILD_TUPLE_UNPACK) + TARGET(BUILD_LIST_UNPACK) { + int convert_to_tuple = opcode != BUILD_LIST_UNPACK; + Py_ssize_t i; PyObject *sum = PyList_New(0); PyObject *return_value; + if (sum == NULL) goto error; @@ -2555,6 +2527,16 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) none_val = _PyList_Extend((PyListObject *)sum, PEEK(i)); if (none_val == NULL) { + if (opcode == BUILD_TUPLE_UNPACK_WITH_CALL && + PyErr_ExceptionMatches(PyExc_TypeError)) { + PyObject *func = PEEK(1 + oparg); + PyErr_Format(PyExc_TypeError, + "%.200s%.200s argument after * " + "must be an iterable, not %.200s", + PyEval_GetFuncName(func), + PyEval_GetFuncDesc(func), + PEEK(i)->ob_type->tp_name); + } Py_DECREF(sum); goto error; } @@ -2599,7 +2581,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) } TARGET(BUILD_SET_UNPACK) { - int i; + Py_ssize_t i; PyObject *sum = PySet_New(NULL); if (sum == NULL) goto error; @@ -2618,7 +2600,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) } TARGET(BUILD_MAP) { - int i; + Py_ssize_t i; PyObject *map = _PyDict_NewPresized((Py_ssize_t)oparg); if (map == NULL) goto error; @@ -2641,87 +2623,162 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) DISPATCH(); } - TARGET_WITH_IMPL(BUILD_MAP_UNPACK_WITH_CALL, _build_map_unpack) - TARGET(BUILD_MAP_UNPACK) - _build_map_unpack: { - int with_call = opcode == BUILD_MAP_UNPACK_WITH_CALL; - int num_maps; - int i; - PyObject *sum = PyDict_New(); - if (sum == NULL) + TARGET(SETUP_ANNOTATIONS) { + _Py_IDENTIFIER(__annotations__); + int err; + PyObject *ann_dict; + if (f->f_locals == NULL) { + PyErr_Format(PyExc_SystemError, + "no locals found when setting up annotations"); goto error; - if (with_call) { - num_maps = oparg & 0xff; } - else { - num_maps = oparg; + /* check if __annotations__ in locals()... */ + if (PyDict_CheckExact(f->f_locals)) { + ann_dict = _PyDict_GetItemId(f->f_locals, + &PyId___annotations__); + if (ann_dict == NULL) { + /* ...if not, create a new one */ + ann_dict = PyDict_New(); + if (ann_dict == NULL) { + goto error; + } + err = _PyDict_SetItemId(f->f_locals, + &PyId___annotations__, ann_dict); + Py_DECREF(ann_dict); + if (err != 0) { + goto error; + } + } } - - for (i = num_maps; i > 0; i--) { - PyObject *arg = PEEK(i); - if (with_call) { - PyObject *intersection = _PyDictView_Intersect(sum, arg); - - if (intersection == NULL) { - if (PyErr_ExceptionMatches(PyExc_AttributeError) || - !PyMapping_Check(arg)) { - int function_location = (oparg>>8) & 0xff; - PyObject *func = ( - PEEK(function_location + num_maps)); - PyErr_Format(PyExc_TypeError, - "%.200s%.200s argument after ** " - "must be a mapping, not %.200s", - PyEval_GetFuncName(func), - PyEval_GetFuncDesc(func), - arg->ob_type->tp_name); - } - Py_DECREF(sum); + else { + /* do the same if locals() is not a dict */ + PyObject *ann_str = _PyUnicode_FromId(&PyId___annotations__); + if (ann_str == NULL) { + goto error; + } + ann_dict = PyObject_GetItem(f->f_locals, ann_str); + if (ann_dict == NULL) { + if (!PyErr_ExceptionMatches(PyExc_KeyError)) { goto error; } - - if (PySet_GET_SIZE(intersection)) { - Py_ssize_t idx = 0; - PyObject *key; - int function_location = (oparg>>8) & 0xff; - PyObject *func = PEEK(function_location + num_maps); - Py_hash_t hash; - _PySet_NextEntry(intersection, &idx, &key, &hash); - if (!PyUnicode_Check(key)) { - PyErr_Format(PyExc_TypeError, - "%.200s%.200s keywords must be strings", - PyEval_GetFuncName(func), - PyEval_GetFuncDesc(func)); - } else { - PyErr_Format(PyExc_TypeError, - "%.200s%.200s got multiple " - "values for keyword argument '%U'", - PyEval_GetFuncName(func), - PyEval_GetFuncDesc(func), - key); - } - Py_DECREF(intersection); - Py_DECREF(sum); + PyErr_Clear(); + ann_dict = PyDict_New(); + if (ann_dict == NULL) { + goto error; + } + err = PyObject_SetItem(f->f_locals, ann_str, ann_dict); + Py_DECREF(ann_dict); + if (err != 0) { goto error; } - Py_DECREF(intersection); } + else { + Py_DECREF(ann_dict); + } + } + DISPATCH(); + } + TARGET(BUILD_CONST_KEY_MAP) { + Py_ssize_t i; + PyObject *map; + PyObject *keys = TOP(); + if (!PyTuple_CheckExact(keys) || + PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { + PyErr_SetString(PyExc_SystemError, + "bad BUILD_CONST_KEY_MAP keys argument"); + goto error; + } + map = _PyDict_NewPresized((Py_ssize_t)oparg); + if (map == NULL) { + goto error; + } + for (i = oparg; i > 0; i--) { + int err; + PyObject *key = PyTuple_GET_ITEM(keys, oparg - i); + PyObject *value = PEEK(i + 1); + err = PyDict_SetItem(map, key, value); + if (err != 0) { + Py_DECREF(map); + goto error; + } + } + + Py_DECREF(POP()); + while (oparg--) { + Py_DECREF(POP()); + } + PUSH(map); + DISPATCH(); + } + + TARGET(BUILD_MAP_UNPACK) { + Py_ssize_t i; + PyObject *sum = PyDict_New(); + if (sum == NULL) + goto error; + + for (i = oparg; i > 0; i--) { + PyObject *arg = PEEK(i); if (PyDict_Update(sum, arg) < 0) { if (PyErr_ExceptionMatches(PyExc_AttributeError)) { - if (with_call) { - int function_location = (oparg>>8) & 0xff; - PyObject *func = PEEK(function_location + num_maps); - PyErr_Format(PyExc_TypeError, - "%.200s%.200s argument after ** " - "must be a mapping, not %.200s", - PyEval_GetFuncName(func), - PyEval_GetFuncDesc(func), - arg->ob_type->tp_name); + PyErr_Format(PyExc_TypeError, + "'%.200s' object is not a mapping", + arg->ob_type->tp_name); + } + Py_DECREF(sum); + goto error; + } + } + + while (oparg--) + Py_DECREF(POP()); + PUSH(sum); + DISPATCH(); + } + + TARGET(BUILD_MAP_UNPACK_WITH_CALL) { + Py_ssize_t i; + PyObject *sum = PyDict_New(); + if (sum == NULL) + goto error; + + for (i = oparg; i > 0; i--) { + PyObject *arg = PEEK(i); + if (_PyDict_MergeEx(sum, arg, 2) < 0) { + PyObject *func = PEEK(2 + oparg); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Format(PyExc_TypeError, + "%.200s%.200s argument after ** " + "must be a mapping, not %.200s", + PyEval_GetFuncName(func), + PyEval_GetFuncDesc(func), + arg->ob_type->tp_name); + } + else if (PyErr_ExceptionMatches(PyExc_KeyError)) { + PyObject *exc, *val, *tb; + PyErr_Fetch(&exc, &val, &tb); + if (val && PyTuple_Check(val) && PyTuple_GET_SIZE(val) == 1) { + PyObject *key = PyTuple_GET_ITEM(val, 0); + if (!PyUnicode_Check(key)) { + PyErr_Format(PyExc_TypeError, + "%.200s%.200s keywords must be strings", + PyEval_GetFuncName(func), + PyEval_GetFuncDesc(func)); + } else { + PyErr_Format(PyExc_TypeError, + "%.200s%.200s got multiple " + "values for keyword argument '%U'", + PyEval_GetFuncName(func), + PyEval_GetFuncDesc(func), + key); + } + Py_XDECREF(exc); + Py_XDECREF(val); + Py_XDECREF(tb); } else { - PyErr_Format(PyExc_TypeError, - "'%.200s' object is not a mapping", - arg->ob_type->tp_name); + PyErr_Restore(exc, val, tb); } } Py_DECREF(sum); @@ -2729,7 +2786,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) } } - while (num_maps--) + while (oparg--) Py_DECREF(POP()); PUSH(sum); DISPATCH(); @@ -2778,45 +2835,13 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) } TARGET(IMPORT_NAME) { - _Py_IDENTIFIER(__import__); PyObject *name = GETITEM(names, oparg); - PyObject *func = _PyDict_GetItemId(f->f_builtins, &PyId___import__); - PyObject *from, *level, *args, *res; - if (func == NULL) { - PyErr_SetString(PyExc_ImportError, - "__import__ not found"); - goto error; - } - Py_INCREF(func); - from = POP(); - level = TOP(); - if (PyLong_AsLong(level) != -1 || PyErr_Occurred()) - args = PyTuple_Pack(5, - name, - f->f_globals, - f->f_locals == NULL ? - Py_None : f->f_locals, - from, - level); - else - args = PyTuple_Pack(4, - name, - f->f_globals, - f->f_locals == NULL ? - Py_None : f->f_locals, - from); + PyObject *fromlist = POP(); + PyObject *level = TOP(); + PyObject *res; + res = import_name(f, name, fromlist, level); Py_DECREF(level); - Py_DECREF(from); - if (args == NULL) { - Py_DECREF(func); - STACKADJ(-1); - goto error; - } - READ_TIMESTAMP(intr0); - res = PyEval_CallObject(func, args); - READ_TIMESTAMP(intr1); - Py_DECREF(args); - Py_DECREF(func); + Py_DECREF(fromlist); SET_TOP(res); if (res == NULL) goto error; @@ -2835,9 +2860,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) "no locals found during 'import *'"); goto error; } - READ_TIMESTAMP(intr0); err = import_all_from(locals, from); - READ_TIMESTAMP(intr1); PyFrame_LocalsToFast(f, 0); Py_DECREF(from); if (err != 0) @@ -2849,9 +2872,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) PyObject *name = GETITEM(names, oparg); PyObject *from = TOP(); PyObject *res; - READ_TIMESTAMP(intr0); res = import_from(from, name); - READ_TIMESTAMP(intr1); PUSH(res); if (res == NULL) goto error; @@ -2863,7 +2884,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) FAST_DISPATCH(); } - PREDICTED_WITH_ARG(POP_JUMP_IF_FALSE); + PREDICTED(POP_JUMP_IF_FALSE); TARGET(POP_JUMP_IF_FALSE) { PyObject *cond = POP(); int err; @@ -2887,7 +2908,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) DISPATCH(); } - PREDICTED_WITH_ARG(POP_JUMP_IF_TRUE); + PREDICTED(POP_JUMP_IF_TRUE); TARGET(POP_JUMP_IF_TRUE) { PyObject *cond = POP(); int err; @@ -2964,7 +2985,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) DISPATCH(); } - PREDICTED_WITH_ARG(JUMP_ABSOLUTE); + PREDICTED(JUMP_ABSOLUTE); TARGET(JUMP_ABSOLUTE) { JUMPTO(oparg); #if FAST_LOOPS @@ -2990,6 +3011,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) if (iter == NULL) goto error; PREDICT(FOR_ITER); + PREDICT(CALL_FUNCTION); DISPATCH(); } @@ -3018,10 +3040,11 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) if (iter == NULL) goto error; } + PREDICT(LOAD_CONST); DISPATCH(); } - PREDICTED_WITH_ARG(FOR_ITER); + PREDICTED(FOR_ITER); TARGET(FOR_ITER) { /* before: [iter]; after: [iter, iter()] *or* [] */ PyObject *iter = TOP(); @@ -3043,6 +3066,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) STACKADJ(-1); Py_DECREF(iter); JUMPBY(oparg); + PREDICT(POP_BLOCK); DISPATCH(); } @@ -3059,10 +3083,9 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) goto fast_block_end; } - TARGET_WITH_IMPL(SETUP_LOOP, _setup_finally) - TARGET_WITH_IMPL(SETUP_EXCEPT, _setup_finally) - TARGET(SETUP_FINALLY) - _setup_finally: { + TARGET(SETUP_LOOP) + TARGET(SETUP_EXCEPT) + TARGET(SETUP_FINALLY) { /* NOTE: If you add any new block-setup opcodes that are not try/except/finally handlers, you may need to update the PyGen_NeedsFinalizing() function. @@ -3093,6 +3116,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) if (res == NULL) goto error; PUSH(res); + PREDICT(GET_AWAITABLE); DISPATCH(); } @@ -3110,15 +3134,17 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) _Py_IDENTIFIER(__exit__); _Py_IDENTIFIER(__enter__); PyObject *mgr = TOP(); - PyObject *exit = special_lookup(mgr, &PyId___exit__), *enter; + PyObject *enter = special_lookup(mgr, &PyId___enter__), *exit; PyObject *res; - if (exit == NULL) + if (enter == NULL) goto error; + exit = special_lookup(mgr, &PyId___exit__); + if (exit == NULL) { + Py_DECREF(enter); + goto error; + } SET_TOP(exit); - enter = special_lookup(mgr, &PyId___enter__); Py_DECREF(mgr); - if (enter == NULL) - goto error; res = PyObject_CallFunctionObjArgs(enter, NULL); Py_DECREF(enter); if (res == NULL) @@ -3241,178 +3267,135 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) DISPATCH(); } + PREDICTED(CALL_FUNCTION); TARGET(CALL_FUNCTION) { PyObject **sp, *res; PCALL(PCALL_ALL); sp = stack_pointer; -#ifdef WITH_TSC - res = call_function(&sp, oparg, &intr0, &intr1); -#else - res = call_function(&sp, oparg); -#endif + res = call_function(&sp, oparg, NULL); stack_pointer = sp; PUSH(res); - if (res == NULL) + if (res == NULL) { goto error; + } DISPATCH(); } - TARGET_WITH_IMPL(CALL_FUNCTION_VAR, _call_function_var_kw) - TARGET_WITH_IMPL(CALL_FUNCTION_KW, _call_function_var_kw) - TARGET(CALL_FUNCTION_VAR_KW) - _call_function_var_kw: { - int na = oparg & 0xff; - int nk = (oparg>>8) & 0xff; - int flags = (opcode - CALL_FUNCTION) & 3; - int n = na + 2 * nk; - PyObject **pfunc, *func, **sp, *res; + TARGET(CALL_FUNCTION_KW) { + PyObject **sp, *res, *names; + + names = POP(); + assert(PyTuple_CheckExact(names) && PyTuple_GET_SIZE(names) <= oparg); PCALL(PCALL_ALL); - if (flags & CALL_FLAG_VAR) - n++; - if (flags & CALL_FLAG_KW) - n++; - pfunc = stack_pointer - n - 1; - func = *pfunc; - - if (PyMethod_Check(func) - && PyMethod_GET_SELF(func) != NULL) { - PyObject *self = PyMethod_GET_SELF(func); - Py_INCREF(self); - func = PyMethod_GET_FUNCTION(func); - Py_INCREF(func); - Py_SETREF(*pfunc, self); - na++; - /* n++; */ - } else - Py_INCREF(func); sp = stack_pointer; - READ_TIMESTAMP(intr0); - res = ext_do_call(func, &sp, flags, na, nk); - READ_TIMESTAMP(intr1); + res = call_function(&sp, oparg, names); stack_pointer = sp; - Py_DECREF(func); - - while (stack_pointer > pfunc) { - PyObject *o = POP(); - Py_DECREF(o); - } PUSH(res); - if (res == NULL) - goto error; - DISPATCH(); - } + Py_DECREF(names); - TARGET_WITH_IMPL(MAKE_CLOSURE, _make_function) - TARGET(MAKE_FUNCTION) - _make_function: { - int posdefaults = oparg & 0xff; - int kwdefaults = (oparg>>8) & 0xff; - int num_annotations = (oparg >> 16) & 0x7fff; - - PyObject *qualname = POP(); /* qualname */ - PyObject *code = POP(); /* code object */ - PyObject *func = PyFunction_NewWithQualName(code, f->f_globals, qualname); - Py_DECREF(code); - Py_DECREF(qualname); - - if (func == NULL) + if (res == NULL) { goto error; - - if (opcode == MAKE_CLOSURE) { - PyObject *closure = POP(); - if (PyFunction_SetClosure(func, closure) != 0) { - /* Can't happen unless bytecode is corrupt. */ - Py_DECREF(func); - Py_DECREF(closure); - goto error; - } - Py_DECREF(closure); } + DISPATCH(); + } - if (num_annotations > 0) { - Py_ssize_t name_ix; - PyObject *names = POP(); /* names of args with annotations */ - PyObject *anns = PyDict_New(); - if (anns == NULL) { - Py_DECREF(func); - Py_DECREF(names); - goto error; - } - name_ix = PyTuple_Size(names); - assert(num_annotations == name_ix+1); - while (name_ix > 0) { - PyObject *name, *value; - int err; - --name_ix; - name = PyTuple_GET_ITEM(names, name_ix); - value = POP(); - err = PyDict_SetItem(anns, name, value); - Py_DECREF(value); - if (err != 0) { - Py_DECREF(anns); - Py_DECREF(func); - Py_DECREF(names); + TARGET(CALL_FUNCTION_EX) { + PyObject *func, *callargs, *kwargs = NULL, *result; + PCALL(PCALL_ALL); + if (oparg & 0x01) { + kwargs = POP(); + if (!PyDict_CheckExact(kwargs)) { + PyObject *d = PyDict_New(); + if (d == NULL) goto error; - } - } - Py_DECREF(names); - - if (PyFunction_SetAnnotations(func, anns) != 0) { - /* Can't happen unless - PyFunction_SetAnnotations changes. */ - Py_DECREF(anns); - Py_DECREF(func); - goto error; - } - Py_DECREF(anns); - } - - /* XXX Maybe this should be a separate opcode? */ - if (kwdefaults > 0) { - PyObject *defs = PyDict_New(); - if (defs == NULL) { - Py_DECREF(func); - goto error; - } - while (--kwdefaults >= 0) { - PyObject *v = POP(); /* default value */ - PyObject *key = POP(); /* kw only arg name */ - int err = PyDict_SetItem(defs, key, v); - Py_DECREF(v); - Py_DECREF(key); - if (err != 0) { - Py_DECREF(defs); - Py_DECREF(func); + if (PyDict_Update(d, kwargs) != 0) { + Py_DECREF(d); + /* PyDict_Update raises attribute + * error (percolated from an attempt + * to get 'keys' attribute) instead of + * a type error if its second argument + * is not a mapping. + */ + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + func = SECOND(); + PyErr_Format(PyExc_TypeError, + "%.200s%.200s argument after ** " + "must be a mapping, not %.200s", + PyEval_GetFuncName(func), + PyEval_GetFuncDesc(func), + kwargs->ob_type->tp_name); + } + Py_DECREF(kwargs); goto error; } + Py_DECREF(kwargs); + kwargs = d; } - if (PyFunction_SetKwDefaults(func, defs) != 0) { - /* Can't happen unless - PyFunction_SetKwDefaults changes. */ - Py_DECREF(func); - Py_DECREF(defs); - goto error; - } - Py_DECREF(defs); + assert(PyDict_CheckExact(kwargs)); } - if (posdefaults > 0) { - PyObject *defs = PyTuple_New(posdefaults); - if (defs == NULL) { - Py_DECREF(func); + callargs = POP(); + func = TOP(); + if (!PyTuple_CheckExact(callargs)) { + if (Py_TYPE(callargs)->tp_iter == NULL && + !PySequence_Check(callargs)) { + PyErr_Format(PyExc_TypeError, + "%.200s%.200s argument after * " + "must be an iterable, not %.200s", + PyEval_GetFuncName(func), + PyEval_GetFuncDesc(func), + callargs->ob_type->tp_name); + Py_DECREF(callargs); goto error; } - while (--posdefaults >= 0) - PyTuple_SET_ITEM(defs, posdefaults, POP()); - if (PyFunction_SetDefaults(func, defs) != 0) { - /* Can't happen unless - PyFunction_SetDefaults changes. */ - Py_DECREF(defs); - Py_DECREF(func); + Py_SETREF(callargs, PySequence_Tuple(callargs)); + if (callargs == NULL) { goto error; } - Py_DECREF(defs); } - PUSH(func); + assert(PyTuple_CheckExact(callargs)); + + result = do_call_core(func, callargs, kwargs); + Py_DECREF(func); + Py_DECREF(callargs); + Py_XDECREF(kwargs); + + SET_TOP(result); + if (result == NULL) { + goto error; + } + DISPATCH(); + } + + TARGET(MAKE_FUNCTION) { + PyObject *qualname = POP(); + PyObject *codeobj = POP(); + PyFunctionObject *func = (PyFunctionObject *) + PyFunction_NewWithQualName(codeobj, f->f_globals, qualname); + + Py_DECREF(codeobj); + Py_DECREF(qualname); + if (func == NULL) { + goto error; + } + + if (oparg & 0x08) { + assert(PyTuple_CheckExact(TOP())); + func ->func_closure = POP(); + } + if (oparg & 0x04) { + assert(PyDict_CheckExact(TOP())); + func->func_annotations = POP(); + } + if (oparg & 0x02) { + assert(PyDict_CheckExact(TOP())); + func->func_kwdefaults = POP(); + } + if (oparg & 0x01) { + assert(PyTuple_CheckExact(TOP())); + func->func_defaults = POP(); + } + + PUSH((PyObject *)func); DISPATCH(); } @@ -3434,9 +3417,68 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) DISPATCH(); } + TARGET(FORMAT_VALUE) { + /* Handles f-string value formatting. */ + PyObject *result; + PyObject *fmt_spec; + PyObject *value; + PyObject *(*conv_fn)(PyObject *); + int which_conversion = oparg & FVC_MASK; + int have_fmt_spec = (oparg & FVS_MASK) == FVS_HAVE_SPEC; + + fmt_spec = have_fmt_spec ? POP() : NULL; + value = POP(); + + /* See if any conversion is specified. */ + switch (which_conversion) { + case FVC_STR: conv_fn = PyObject_Str; break; + case FVC_REPR: conv_fn = PyObject_Repr; break; + case FVC_ASCII: conv_fn = PyObject_ASCII; break; + + /* Must be 0 (meaning no conversion), since only four + values are allowed by (oparg & FVC_MASK). */ + default: conv_fn = NULL; break; + } + + /* If there's a conversion function, call it and replace + value with that result. Otherwise, just use value, + without conversion. */ + if (conv_fn != NULL) { + result = conv_fn(value); + Py_DECREF(value); + if (result == NULL) { + Py_XDECREF(fmt_spec); + goto error; + } + value = result; + } + + /* If value is a unicode object, and there's no fmt_spec, + then we know the result of format(value) is value + itself. In that case, skip calling format(). I plan to + move this optimization in to PyObject_Format() + itself. */ + if (PyUnicode_CheckExact(value) && fmt_spec == NULL) { + /* Do nothing, just transfer ownership to result. */ + result = value; + } else { + /* Actually call format(). */ + result = PyObject_Format(value, fmt_spec); + Py_DECREF(value); + Py_XDECREF(fmt_spec); + if (result == NULL) { + goto error; + } + } + + PUSH(result); + DISPATCH(); + } + TARGET(EXTENDED_ARG) { - opcode = NEXTOP(); - oparg = oparg<<16 | NEXTARG(); + int oldoparg = oparg; + NEXTOPARG(); + oparg |= oldoparg << 8; goto dispatch_opcode; } @@ -3463,7 +3505,6 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) assert(0); error: - READ_TIMESTAMP(inst1); assert(why == WHY_NOT); why = WHY_EXCEPTION; @@ -3567,7 +3608,6 @@ fast_block_end: if (why != WHY_NOT) break; - READ_TIMESTAMP(loop1); assert(!PyErr_Occurred()); @@ -3586,7 +3626,7 @@ fast_block_end: assert((retval != NULL) ^ (PyErr_Occurred() != NULL)); fast_yield: - if (co->co_flags & (CO_GENERATOR | CO_COROUTINE)) { + if (co->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) { /* The purpose of this block is to put aside the generator's exception state and restore that of the calling frame. If the current @@ -3597,9 +3637,11 @@ fast_yield: state came into existence in this frame. (An uncaught exception would have why == WHY_EXCEPTION, and we wouldn't be here). */ int i; - for (i = 0; i < f->f_iblock; i++) - if (f->f_blockstack[i].b_type == EXCEPT_HANDLER) + for (i = 0; i < f->f_iblock; i++) { + if (f->f_blockstack[i].b_type == EXCEPT_HANDLER) { break; + } + } if (i == f->f_iblock) /* We did not create this exception. */ restore_and_clear_exc_state(tstate, f); @@ -3640,6 +3682,8 @@ fast_yield: /* pop frame */ exit_eval_frame: + if (PyDTrace_FUNCTION_RETURN_ENABLED()) + dtrace_function_return(f); Py_LeaveRecursiveCall(); f->f_executing = 0; tstate->frame = f->f_back; @@ -3710,12 +3754,12 @@ format_missing(const char *kind, PyCodeObject *co, PyObject *names) } static void -missing_arguments(PyCodeObject *co, int missing, int defcount, +missing_arguments(PyCodeObject *co, Py_ssize_t missing, Py_ssize_t defcount, PyObject **fastlocals) { - int i, j = 0; - int start, end; - int positional = defcount != -1; + Py_ssize_t i, j = 0; + Py_ssize_t start, end; + int positional = (defcount != -1); const char *kind = positional ? "positional" : "keyword-only"; PyObject *missing_names; @@ -3748,33 +3792,39 @@ missing_arguments(PyCodeObject *co, int missing, int defcount, } static void -too_many_positional(PyCodeObject *co, int given, int defcount, PyObject **fastlocals) +too_many_positional(PyCodeObject *co, Py_ssize_t given, Py_ssize_t defcount, + PyObject **fastlocals) { int plural; - int kwonly_given = 0; - int i; + Py_ssize_t kwonly_given = 0; + Py_ssize_t i; PyObject *sig, *kwonly_sig; + Py_ssize_t co_argcount = co->co_argcount; assert((co->co_flags & CO_VARARGS) == 0); /* Count missing keyword-only args. */ - for (i = co->co_argcount; i < co->co_argcount + co->co_kwonlyargcount; i++) - if (GETLOCAL(i) != NULL) + for (i = co_argcount; i < co_argcount + co->co_kwonlyargcount; i++) { + if (GETLOCAL(i) != NULL) { kwonly_given++; + } + } if (defcount) { - int atleast = co->co_argcount - defcount; + Py_ssize_t atleast = co_argcount - defcount; plural = 1; - sig = PyUnicode_FromFormat("from %d to %d", atleast, co->co_argcount); + sig = PyUnicode_FromFormat("from %zd to %zd", atleast, co_argcount); } else { - plural = co->co_argcount != 1; - sig = PyUnicode_FromFormat("%d", co->co_argcount); + plural = (co_argcount != 1); + sig = PyUnicode_FromFormat("%zd", co_argcount); } if (sig == NULL) return; if (kwonly_given) { - const char *format = " positional argument%s (and %d keyword-only argument%s)"; - kwonly_sig = PyUnicode_FromFormat(format, given != 1 ? "s" : "", kwonly_given, - kwonly_given != 1 ? "s" : ""); + const char *format = " positional argument%s (and %zd keyword-only argument%s)"; + kwonly_sig = PyUnicode_FromFormat(format, + given != 1 ? "s" : "", + kwonly_given, + kwonly_given != 1 ? "s" : ""); if (kwonly_sig == NULL) { Py_DECREF(sig); return; @@ -3786,7 +3836,7 @@ too_many_positional(PyCodeObject *co, int given, int defcount, PyObject **fastlo assert(kwonly_sig != NULL); } PyErr_Format(PyExc_TypeError, - "%U() takes %U positional argument%s but %d%U %s given", + "%U() takes %U positional argument%s but %zd%U %s given", co->co_name, sig, plural ? "s" : "", @@ -3803,20 +3853,22 @@ too_many_positional(PyCodeObject *co, int given, int defcount, PyObject **fastlo static PyObject * _PyEval_EvalCodeWithName(PyObject *_co, PyObject *globals, PyObject *locals, - PyObject **args, int argcount, PyObject **kws, int kwcount, - PyObject **defs, int defcount, PyObject *kwdefs, PyObject *closure, + PyObject **args, Py_ssize_t argcount, + PyObject **kwnames, PyObject **kwargs, + Py_ssize_t kwcount, int kwstep, + PyObject **defs, Py_ssize_t defcount, + PyObject *kwdefs, PyObject *closure, PyObject *name, PyObject *qualname) { PyCodeObject* co = (PyCodeObject*)_co; PyFrameObject *f; PyObject *retval = NULL; PyObject **fastlocals, **freevars; - PyThreadState *tstate = PyThreadState_GET(); + PyThreadState *tstate; PyObject *x, *u; - int total_args = co->co_argcount + co->co_kwonlyargcount; - int i; - int n = argcount; - PyObject *kwdict = NULL; + const Py_ssize_t total_args = co->co_argcount + co->co_kwonlyargcount; + Py_ssize_t i, n; + PyObject *kwdict; if (globals == NULL) { PyErr_SetString(PyExc_SystemError, @@ -3824,36 +3876,50 @@ _PyEval_EvalCodeWithName(PyObject *_co, PyObject *globals, PyObject *locals, return NULL; } + /* Create the frame */ + tstate = PyThreadState_GET(); assert(tstate != NULL); - assert(globals != NULL); f = PyFrame_New(tstate, co, globals, locals); - if (f == NULL) + if (f == NULL) { return NULL; - + } fastlocals = f->f_localsplus; freevars = f->f_localsplus + co->co_nlocals; - /* Parse arguments. */ + /* Create a dictionary for keyword parameters (**kwags) */ if (co->co_flags & CO_VARKEYWORDS) { kwdict = PyDict_New(); if (kwdict == NULL) goto fail; i = total_args; - if (co->co_flags & CO_VARARGS) + if (co->co_flags & CO_VARARGS) { i++; + } SETLOCAL(i, kwdict); } - if (argcount > co->co_argcount) + else { + kwdict = NULL; + } + + /* Copy positional arguments into local variables */ + if (argcount > co->co_argcount) { n = co->co_argcount; + } + else { + n = argcount; + } for (i = 0; i < n; i++) { x = args[i]; Py_INCREF(x); SETLOCAL(i, x); } + + /* Pack other positional arguments into the *args argument */ if (co->co_flags & CO_VARARGS) { u = PyTuple_New(argcount - n); - if (u == NULL) + if (u == NULL) { goto fail; + } SETLOCAL(total_args, u); for (i = n; i < argcount; i++) { x = args[i]; @@ -3861,69 +3927,82 @@ _PyEval_EvalCodeWithName(PyObject *_co, PyObject *globals, PyObject *locals, PyTuple_SET_ITEM(u, i-n, x); } } - for (i = 0; i < kwcount; i++) { + + /* Handle keyword arguments passed as two strided arrays */ + kwcount *= kwstep; + for (i = 0; i < kwcount; i += kwstep) { PyObject **co_varnames; - PyObject *keyword = kws[2*i]; - PyObject *value = kws[2*i + 1]; - int j; + PyObject *keyword = kwnames[i]; + PyObject *value = kwargs[i]; + Py_ssize_t j; + if (keyword == NULL || !PyUnicode_Check(keyword)) { PyErr_Format(PyExc_TypeError, "%U() keywords must be strings", co->co_name); goto fail; } + /* Speed hack: do raw pointer compares. As names are normally interned this should almost always hit. */ co_varnames = ((PyTupleObject *)(co->co_varnames))->ob_item; for (j = 0; j < total_args; j++) { - PyObject *nm = co_varnames[j]; - if (nm == keyword) + PyObject *name = co_varnames[j]; + if (name == keyword) { goto kw_found; + } } + /* Slow fallback, just in case */ for (j = 0; j < total_args; j++) { - PyObject *nm = co_varnames[j]; - int cmp = PyObject_RichCompareBool( - keyword, nm, Py_EQ); - if (cmp > 0) + PyObject *name = co_varnames[j]; + int cmp = PyObject_RichCompareBool( keyword, name, Py_EQ); + if (cmp > 0) { goto kw_found; - else if (cmp < 0) + } + else if (cmp < 0) { goto fail; + } } + if (j >= total_args && kwdict == NULL) { PyErr_Format(PyExc_TypeError, - "%U() got an unexpected " - "keyword argument '%S'", - co->co_name, - keyword); + "%U() got an unexpected keyword argument '%S'", + co->co_name, keyword); goto fail; } + if (PyDict_SetItem(kwdict, keyword, value) == -1) { goto fail; } continue; + kw_found: if (GETLOCAL(j) != NULL) { PyErr_Format(PyExc_TypeError, - "%U() got multiple " - "values for argument '%S'", - co->co_name, - keyword); + "%U() got multiple values for argument '%S'", + co->co_name, keyword); goto fail; } Py_INCREF(value); SETLOCAL(j, value); } + + /* Check the number of positional arguments */ if (argcount > co->co_argcount && !(co->co_flags & CO_VARARGS)) { too_many_positional(co, argcount, defcount, fastlocals); goto fail; } + + /* Add missing positional arguments (copy default values from defs) */ if (argcount < co->co_argcount) { - int m = co->co_argcount - defcount; - int missing = 0; - for (i = argcount; i < m; i++) - if (GETLOCAL(i) == NULL) + Py_ssize_t m = co->co_argcount - defcount; + Py_ssize_t missing = 0; + for (i = argcount; i < m; i++) { + if (GETLOCAL(i) == NULL) { missing++; + } + } if (missing) { missing_arguments(co, missing, defcount, fastlocals); goto fail; @@ -3940,8 +4019,10 @@ _PyEval_EvalCodeWithName(PyObject *_co, PyObject *globals, PyObject *locals, } } } + + /* Add missing keyword arguments (copy default values from kwdefs) */ if (co->co_kwonlyargcount > 0) { - int missing = 0; + Py_ssize_t missing = 0; for (i = co->co_argcount; i < total_args; i++) { PyObject *name; if (GETLOCAL(i) != NULL) @@ -3982,13 +4063,16 @@ _PyEval_EvalCodeWithName(PyObject *_co, PyObject *globals, PyObject *locals, goto fail; SETLOCAL(co->co_nlocals + i, c); } + + /* Copy closure variables to free variables */ for (i = 0; i < PyTuple_GET_SIZE(co->co_freevars); ++i) { PyObject *o = PyTuple_GET_ITEM(closure, i); Py_INCREF(o); freevars[PyTuple_GET_SIZE(co->co_cellvars) + i] = o; } - if (co->co_flags & (CO_GENERATOR | CO_COROUTINE)) { + /* Handle generator/coroutine/asynchronous generator */ + if (co->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) { PyObject *gen; PyObject *coro_wrapper = tstate->coroutine_wrapper; int is_coro = co->co_flags & CO_COROUTINE; @@ -4013,6 +4097,8 @@ _PyEval_EvalCodeWithName(PyObject *_co, PyObject *globals, PyObject *locals, * and return that as the value. */ if (is_coro) { gen = PyCoro_New(f, name, qualname); + } else if (co->co_flags & CO_ASYNC_GENERATOR) { + gen = PyAsyncGen_New(f, name, qualname); } else { gen = PyGen_NewWithQualName(f, name, qualname); } @@ -4052,8 +4138,10 @@ PyEval_EvalCodeEx(PyObject *_co, PyObject *globals, PyObject *locals, PyObject **defs, int defcount, PyObject *kwdefs, PyObject *closure) { return _PyEval_EvalCodeWithName(_co, globals, locals, - args, argcount, kws, kwcount, - defs, defcount, kwdefs, closure, + args, argcount, + kws, kws + 1, kwcount, 2, + defs, defcount, + kwdefs, closure, NULL, NULL); } @@ -4312,7 +4400,7 @@ Error: #ifdef LLTRACE static int -prtrace(PyObject *v, char *str) +prtrace(PyObject *v, const char *str) { printf("%s ", str); if (PyObject_Print(v, stdout, 0) != 0) @@ -4489,6 +4577,38 @@ _PyEval_GetCoroutineWrapper(void) return tstate->coroutine_wrapper; } +void +_PyEval_SetAsyncGenFirstiter(PyObject *firstiter) +{ + PyThreadState *tstate = PyThreadState_GET(); + + Py_XINCREF(firstiter); + Py_XSETREF(tstate->async_gen_firstiter, firstiter); +} + +PyObject * +_PyEval_GetAsyncGenFirstiter(void) +{ + PyThreadState *tstate = PyThreadState_GET(); + return tstate->async_gen_firstiter; +} + +void +_PyEval_SetAsyncGenFinalizer(PyObject *finalizer) +{ + PyThreadState *tstate = PyThreadState_GET(); + + Py_XINCREF(finalizer); + Py_XSETREF(tstate->async_gen_finalizer, finalizer); +} + +PyObject * +_PyEval_GetAsyncGenFinalizer(void) +{ + PyThreadState *tstate = PyThreadState_GET(); + return tstate->async_gen_finalizer; +} + PyObject * PyEval_GetBuiltins(void) { @@ -4561,10 +4681,8 @@ PyEval_MergeCompilerFlags(PyCompilerFlags *cf) The arg must be a tuple or NULL. The kw must be a dict or NULL. */ PyObject * -PyEval_CallObjectWithKeywords(PyObject *func, PyObject *arg, PyObject *kw) +PyEval_CallObjectWithKeywords(PyObject *func, PyObject *args, PyObject *kwargs) { - PyObject *result; - #ifdef Py_DEBUG /* PyEval_CallObjectWithKeywords() must not be called with an exception set. It raises a new exception if parameters are invalid or if @@ -4572,30 +4690,23 @@ PyEval_CallObjectWithKeywords(PyObject *func, PyObject *arg, PyObject *kw) assert(!PyErr_Occurred()); #endif - if (arg == NULL) { - arg = PyTuple_New(0); - if (arg == NULL) - return NULL; + if (args == NULL) { + return _PyObject_FastCallDict(func, NULL, 0, kwargs); } - else if (!PyTuple_Check(arg)) { + + if (!PyTuple_Check(args)) { PyErr_SetString(PyExc_TypeError, "argument list must be a tuple"); return NULL; } - else - Py_INCREF(arg); - if (kw != NULL && !PyDict_Check(kw)) { + if (kwargs != NULL && !PyDict_Check(kwargs)) { PyErr_SetString(PyExc_TypeError, "keyword list must be a dictionary"); - Py_DECREF(arg); return NULL; } - result = PyObject_Call(func, arg, kw); - Py_DECREF(arg); - - return result; + return PyObject_Call(func, args, kwargs); } const char * @@ -4604,7 +4715,7 @@ PyEval_GetFuncName(PyObject *func) if (PyMethod_Check(func)) return PyEval_GetFuncName(PyMethod_GET_FUNCTION(func)); else if (PyFunction_Check(func)) - return _PyUnicode_AsString(((PyFunctionObject*)func)->func_name); + return PyUnicode_AsUTF8(((PyFunctionObject*)func)->func_name); else if (PyCFunction_Check(func)) return ((PyCFunctionObject*)func)->m_ml->ml_name; else @@ -4624,21 +4735,6 @@ PyEval_GetFuncDesc(PyObject *func) return " object"; } -static void -err_args(PyObject *func, int flags, int nargs) -{ - if (flags & METH_NOARGS) - PyErr_Format(PyExc_TypeError, - "%.200s() takes no arguments (%d given)", - ((PyCFunctionObject *)func)->m_ml->ml_name, - nargs); - else - PyErr_Format(PyExc_TypeError, - "%.200s() takes exactly one argument (%d given)", - ((PyCFunctionObject *)func)->m_ml->ml_name, - nargs); -} - #define C_TRACE(x, call) \ if (tstate->use_tracing && tstate->c_profilefunc) { \ if (call_trace(tstate->c_profilefunc, tstate->c_profileobj, \ @@ -4671,60 +4767,25 @@ if (tstate->use_tracing && tstate->c_profilefunc) { \ } static PyObject * -call_function(PyObject ***pp_stack, int oparg -#ifdef WITH_TSC - , uint64* pintr0, uint64* pintr1 -#endif - ) +call_function(PyObject ***pp_stack, Py_ssize_t oparg, PyObject *kwnames) { - int na = oparg & 0xff; - int nk = (oparg>>8) & 0xff; - int n = na + 2 * nk; - PyObject **pfunc = (*pp_stack) - n - 1; + PyObject **pfunc = (*pp_stack) - oparg - 1; PyObject *func = *pfunc; PyObject *x, *w; + Py_ssize_t nkwargs = (kwnames == NULL) ? 0 : PyTuple_GET_SIZE(kwnames); + Py_ssize_t nargs = oparg - nkwargs; + PyObject **stack; /* Always dispatch PyCFunction first, because these are presumed to be the most frequent callable object. */ - if (PyCFunction_Check(func) && nk == 0) { - int flags = PyCFunction_GET_FLAGS(func); + if (PyCFunction_Check(func)) { PyThreadState *tstate = PyThreadState_GET(); PCALL(PCALL_CFUNCTION); - if (flags & (METH_NOARGS | METH_O)) { - PyCFunction meth = PyCFunction_GET_FUNCTION(func); - PyObject *self = PyCFunction_GET_SELF(func); - if (flags & METH_NOARGS && na == 0) { - C_TRACE(x, (*meth)(self,NULL)); - x = _Py_CheckFunctionResult(func, x, NULL); - } - else if (flags & METH_O && na == 1) { - PyObject *arg = EXT_POP(*pp_stack); - C_TRACE(x, (*meth)(self,arg)); - Py_DECREF(arg); - - x = _Py_CheckFunctionResult(func, x, NULL); - } - else { - err_args(func, flags, na); - x = NULL; - } - } - else { - PyObject *callargs; - callargs = load_args(pp_stack, na); - if (callargs != NULL) { - READ_TIMESTAMP(*pintr0); - C_TRACE(x, PyCFunction_Call(func,callargs,NULL)); - READ_TIMESTAMP(*pintr1); - Py_XDECREF(callargs); - } - else { - x = NULL; - } - } + stack = (*pp_stack) - nargs - nkwargs; + C_TRACE(x, _PyCFunction_FastCallKeywords(func, stack, nargs, kwnames)); } else { if (PyMethod_Check(func) && PyMethod_GET_SELF(func) != NULL) { @@ -4736,21 +4797,26 @@ call_function(PyObject ***pp_stack, int oparg func = PyMethod_GET_FUNCTION(func); Py_INCREF(func); Py_SETREF(*pfunc, self); - na++; - n++; - } else + nargs++; + } + else { Py_INCREF(func); - READ_TIMESTAMP(*pintr0); - if (PyFunction_Check(func)) - x = fast_function(func, pp_stack, n, na, nk); - else - x = do_call(func, pp_stack, na, nk); - READ_TIMESTAMP(*pintr1); - Py_DECREF(func); + } + + stack = (*pp_stack) - nargs - nkwargs; - assert((x != NULL) ^ (PyErr_Occurred() != NULL)); + if (PyFunction_Check(func)) { + x = fast_function(func, stack, nargs, kwnames); + } + else { + x = _PyObject_FastCallKeywords(func, stack, nargs, kwnames); + } + + Py_DECREF(func); } + assert((x != NULL) ^ (PyErr_Occurred() != NULL)); + /* Clear the stack of the function object. Also removes the arguments in case they weren't consumed already (fast_function() and err_args() leave them on the stack). @@ -4761,7 +4827,6 @@ call_function(PyObject ***pp_stack, int oparg PCALL(PCALL_POP); } - assert((x != NULL) ^ (PyErr_Occurred() != NULL)); return x; } @@ -4774,257 +4839,197 @@ call_function(PyObject ***pp_stack, int oparg done before evaluating the frame. */ +static PyObject* +_PyFunction_FastCall(PyCodeObject *co, PyObject **args, Py_ssize_t nargs, + PyObject *globals) +{ + PyFrameObject *f; + PyThreadState *tstate = PyThreadState_GET(); + PyObject **fastlocals; + Py_ssize_t i; + PyObject *result; + + PCALL(PCALL_FASTER_FUNCTION); + assert(globals != NULL); + /* XXX Perhaps we should create a specialized + PyFrame_New() that doesn't take locals, but does + take builtins without sanity checking them. + */ + assert(tstate != NULL); + f = PyFrame_New(tstate, co, globals, NULL); + if (f == NULL) { + return NULL; + } + + fastlocals = f->f_localsplus; + + for (i = 0; i < nargs; i++) { + Py_INCREF(*args); + fastlocals[i] = *args++; + } + result = PyEval_EvalFrameEx(f,0); + + ++tstate->recursion_depth; + Py_DECREF(f); + --tstate->recursion_depth; + + return result; +} + static PyObject * -fast_function(PyObject *func, PyObject ***pp_stack, int n, int na, int nk) +fast_function(PyObject *func, PyObject **stack, + Py_ssize_t nargs, PyObject *kwnames) { PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); PyObject *globals = PyFunction_GET_GLOBALS(func); PyObject *argdefs = PyFunction_GET_DEFAULTS(func); - PyObject *kwdefs = PyFunction_GET_KW_DEFAULTS(func); - PyObject *name = ((PyFunctionObject *)func) -> func_name; - PyObject *qualname = ((PyFunctionObject *)func) -> func_qualname; - PyObject **d = NULL; - int nd = 0; + PyObject *kwdefs, *closure, *name, *qualname; + PyObject **d; + Py_ssize_t nkwargs = (kwnames == NULL) ? 0 : PyTuple_GET_SIZE(kwnames); + Py_ssize_t nd; + + assert(PyFunction_Check(func)); + assert(nargs >= 0); + assert(kwnames == NULL || PyTuple_CheckExact(kwnames)); + assert((nargs == 0 && nkwargs == 0) || stack != NULL); + /* kwnames must only contains str strings, no subclass, and all keys must + be unique */ PCALL(PCALL_FUNCTION); PCALL(PCALL_FAST_FUNCTION); - if (argdefs == NULL && co->co_argcount == n && - co->co_kwonlyargcount == 0 && nk==0 && - co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) { - PyFrameObject *f; - PyObject *retval = NULL; - PyThreadState *tstate = PyThreadState_GET(); - PyObject **fastlocals, **stack; - int i; - - PCALL(PCALL_FASTER_FUNCTION); - assert(globals != NULL); - /* XXX Perhaps we should create a specialized - PyFrame_New() that doesn't take locals, but does - take builtins without sanity checking them. - */ - assert(tstate != NULL); - f = PyFrame_New(tstate, co, globals, NULL); - if (f == NULL) - return NULL; - fastlocals = f->f_localsplus; - stack = (*pp_stack) - n; - - for (i = 0; i < n; i++) { - Py_INCREF(*stack); - fastlocals[i] = *stack++; + if (co->co_kwonlyargcount == 0 && nkwargs == 0 && + co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) + { + if (argdefs == NULL && co->co_argcount == nargs) { + return _PyFunction_FastCall(co, stack, nargs, globals); + } + else if (nargs == 0 && argdefs != NULL + && co->co_argcount == Py_SIZE(argdefs)) { + /* function called with no arguments, but all parameters have + a default value: use default values as arguments .*/ + stack = &PyTuple_GET_ITEM(argdefs, 0); + return _PyFunction_FastCall(co, stack, Py_SIZE(argdefs), globals); } - retval = PyEval_EvalFrameEx(f,0); - ++tstate->recursion_depth; - Py_DECREF(f); - --tstate->recursion_depth; - return retval; } + + kwdefs = PyFunction_GET_KW_DEFAULTS(func); + closure = PyFunction_GET_CLOSURE(func); + name = ((PyFunctionObject *)func) -> func_name; + qualname = ((PyFunctionObject *)func) -> func_qualname; + if (argdefs != NULL) { d = &PyTuple_GET_ITEM(argdefs, 0); nd = Py_SIZE(argdefs); } - return _PyEval_EvalCodeWithName((PyObject*)co, globals, - (PyObject *)NULL, (*pp_stack)-n, na, - (*pp_stack)-2*nk, nk, d, nd, kwdefs, - PyFunction_GET_CLOSURE(func), - name, qualname); + else { + d = NULL; + nd = 0; + } + return _PyEval_EvalCodeWithName((PyObject*)co, globals, (PyObject *)NULL, + stack, nargs, + nkwargs ? &PyTuple_GET_ITEM(kwnames, 0) : NULL, + stack + nargs, + nkwargs, 1, + d, (int)nd, kwdefs, + closure, name, qualname); } -static PyObject * -update_keyword_args(PyObject *orig_kwdict, int nk, PyObject ***pp_stack, - PyObject *func) +PyObject * +_PyFunction_FastCallKeywords(PyObject *func, PyObject **stack, + Py_ssize_t nargs, PyObject *kwnames) { - PyObject *kwdict = NULL; - if (orig_kwdict == NULL) - kwdict = PyDict_New(); - else { - kwdict = PyDict_Copy(orig_kwdict); - Py_DECREF(orig_kwdict); - } - if (kwdict == NULL) - return NULL; - while (--nk >= 0) { - int err; - PyObject *value = EXT_POP(*pp_stack); - PyObject *key = EXT_POP(*pp_stack); - if (PyDict_GetItem(kwdict, key) != NULL) { - PyErr_Format(PyExc_TypeError, - "%.200s%s got multiple values " - "for keyword argument '%U'", - PyEval_GetFuncName(func), - PyEval_GetFuncDesc(func), - key); - Py_DECREF(key); - Py_DECREF(value); - Py_DECREF(kwdict); - return NULL; - } - err = PyDict_SetItem(kwdict, key, value); - Py_DECREF(key); - Py_DECREF(value); - if (err) { - Py_DECREF(kwdict); - return NULL; - } - } - return kwdict; + return fast_function(func, stack, nargs, kwnames); } -static PyObject * -update_star_args(int nstack, int nstar, PyObject *stararg, - PyObject ***pp_stack) +PyObject * +_PyFunction_FastCallDict(PyObject *func, PyObject **args, Py_ssize_t nargs, + PyObject *kwargs) { - PyObject *callargs, *w; + PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); + PyObject *globals = PyFunction_GET_GLOBALS(func); + PyObject *argdefs = PyFunction_GET_DEFAULTS(func); + PyObject *kwdefs, *closure, *name, *qualname; + PyObject *kwtuple, **k; + PyObject **d; + Py_ssize_t nd, nk; + PyObject *result; - callargs = PyTuple_New(nstack + nstar); - if (callargs == NULL) { - return NULL; - } - if (nstar) { - int i; - for (i = 0; i < nstar; i++) { - PyObject *a = PyTuple_GET_ITEM(stararg, i); - Py_INCREF(a); - PyTuple_SET_ITEM(callargs, nstack + i, a); + assert(func != NULL); + assert(nargs >= 0); + assert(nargs == 0 || args != NULL); + assert(kwargs == NULL || PyDict_Check(kwargs)); + + PCALL(PCALL_FUNCTION); + PCALL(PCALL_FAST_FUNCTION); + + if (co->co_kwonlyargcount == 0 && + (kwargs == NULL || PyDict_Size(kwargs) == 0) && + co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) + { + /* Fast paths */ + if (argdefs == NULL && co->co_argcount == nargs) { + return _PyFunction_FastCall(co, args, nargs, globals); + } + else if (nargs == 0 && argdefs != NULL + && co->co_argcount == Py_SIZE(argdefs)) { + /* function called with no arguments, but all parameters have + a default value: use default values as arguments .*/ + args = &PyTuple_GET_ITEM(argdefs, 0); + return _PyFunction_FastCall(co, args, Py_SIZE(argdefs), globals); } } - while (--nstack >= 0) { - w = EXT_POP(*pp_stack); - PyTuple_SET_ITEM(callargs, nstack, w); - } - return callargs; -} -static PyObject * -load_args(PyObject ***pp_stack, int na) -{ - PyObject *args = PyTuple_New(na); - PyObject *w; + if (kwargs != NULL) { + Py_ssize_t pos, i; + nk = PyDict_Size(kwargs); - if (args == NULL) - return NULL; - while (--na >= 0) { - w = EXT_POP(*pp_stack); - PyTuple_SET_ITEM(args, na, w); + kwtuple = PyTuple_New(2 * nk); + if (kwtuple == NULL) { + return NULL; + } + + k = &PyTuple_GET_ITEM(kwtuple, 0); + pos = i = 0; + while (PyDict_Next(kwargs, &pos, &k[i], &k[i+1])) { + Py_INCREF(k[i]); + Py_INCREF(k[i+1]); + i += 2; + } + nk = i / 2; + } + else { + kwtuple = NULL; + k = NULL; + nk = 0; } - return args; -} -static PyObject * -do_call(PyObject *func, PyObject ***pp_stack, int na, int nk) -{ - PyObject *callargs = NULL; - PyObject *kwdict = NULL; - PyObject *result = NULL; + kwdefs = PyFunction_GET_KW_DEFAULTS(func); + closure = PyFunction_GET_CLOSURE(func); + name = ((PyFunctionObject *)func) -> func_name; + qualname = ((PyFunctionObject *)func) -> func_qualname; - if (nk > 0) { - kwdict = update_keyword_args(NULL, nk, pp_stack, func); - if (kwdict == NULL) - goto call_fail; + if (argdefs != NULL) { + d = &PyTuple_GET_ITEM(argdefs, 0); + nd = Py_SIZE(argdefs); } - callargs = load_args(pp_stack, na); - if (callargs == NULL) - goto call_fail; -#ifdef CALL_PROFILE - /* At this point, we have to look at the type of func to - update the call stats properly. Do it here so as to avoid - exposing the call stats machinery outside ceval.c - */ - if (PyFunction_Check(func)) - PCALL(PCALL_FUNCTION); - else if (PyMethod_Check(func)) - PCALL(PCALL_METHOD); - else if (PyType_Check(func)) - PCALL(PCALL_TYPE); - else if (PyCFunction_Check(func)) - PCALL(PCALL_CFUNCTION); - else - PCALL(PCALL_OTHER); -#endif - if (PyCFunction_Check(func)) { - PyThreadState *tstate = PyThreadState_GET(); - C_TRACE(result, PyCFunction_Call(func, callargs, kwdict)); + else { + d = NULL; + nd = 0; } - else - result = PyObject_Call(func, callargs, kwdict); -call_fail: - Py_XDECREF(callargs); - Py_XDECREF(kwdict); + + result = _PyEval_EvalCodeWithName((PyObject*)co, globals, (PyObject *)NULL, + args, nargs, + k, k + 1, nk, 2, + d, nd, kwdefs, + closure, name, qualname); + Py_XDECREF(kwtuple); return result; } static PyObject * -ext_do_call(PyObject *func, PyObject ***pp_stack, int flags, int na, int nk) +do_call_core(PyObject *func, PyObject *callargs, PyObject *kwdict) { - int nstar = 0; - PyObject *callargs = NULL; - PyObject *stararg = NULL; - PyObject *kwdict = NULL; - PyObject *result = NULL; - - if (flags & CALL_FLAG_KW) { - kwdict = EXT_POP(*pp_stack); - if (!PyDict_Check(kwdict)) { - PyObject *d; - d = PyDict_New(); - if (d == NULL) - goto ext_call_fail; - if (PyDict_Update(d, kwdict) != 0) { - Py_DECREF(d); - /* PyDict_Update raises attribute - * error (percolated from an attempt - * to get 'keys' attribute) instead of - * a type error if its second argument - * is not a mapping. - */ - if (PyErr_ExceptionMatches(PyExc_AttributeError)) { - PyErr_Format(PyExc_TypeError, - "%.200s%.200s argument after ** " - "must be a mapping, not %.200s", - PyEval_GetFuncName(func), - PyEval_GetFuncDesc(func), - kwdict->ob_type->tp_name); - } - goto ext_call_fail; - } - Py_DECREF(kwdict); - kwdict = d; - } - } - if (nk > 0) { - kwdict = update_keyword_args(kwdict, nk, pp_stack, func); - if (kwdict == NULL) - goto ext_call_fail; - } - - if (flags & CALL_FLAG_VAR) { - stararg = EXT_POP(*pp_stack); - if (!PyTuple_Check(stararg)) { - PyObject *t = NULL; - if (Py_TYPE(stararg)->tp_iter == NULL && - !PySequence_Check(stararg)) { - PyErr_Format(PyExc_TypeError, - "%.200s%.200s argument after * " - "must be an iterable, not %.200s", - PyEval_GetFuncName(func), - PyEval_GetFuncDesc(func), - stararg->ob_type->tp_name); - goto ext_call_fail; - } - t = PySequence_Tuple(stararg); - if (t == NULL) { - goto ext_call_fail; - } - Py_DECREF(stararg); - stararg = t; - } - nstar = PyTuple_GET_SIZE(stararg); - } - callargs = update_star_args(na, nstar, stararg, pp_stack); - if (callargs == NULL) - goto ext_call_fail; #ifdef CALL_PROFILE /* At this point, we have to look at the type of func to update the call stats properly. Do it here so as to avoid @@ -5041,17 +5046,16 @@ ext_do_call(PyObject *func, PyObject ***pp_stack, int flags, int na, int nk) else PCALL(PCALL_OTHER); #endif + if (PyCFunction_Check(func)) { + PyObject *result; PyThreadState *tstate = PyThreadState_GET(); C_TRACE(result, PyCFunction_Call(func, callargs, kwdict)); + return result; + } + else { + return PyObject_Call(func, callargs, kwdict); } - else - result = PyObject_Call(func, callargs, kwdict); -ext_call_fail: - Py_XDECREF(callargs); - Py_XDECREF(kwdict); - Py_XDECREF(stararg); - return result; } /* Extract a slice index from a PyLong or an object with the @@ -5141,6 +5145,46 @@ cmp_outcome(int op, PyObject *v, PyObject *w) } static PyObject * +import_name(PyFrameObject *f, PyObject *name, PyObject *fromlist, PyObject *level) +{ + _Py_IDENTIFIER(__import__); + PyObject *import_func, *res; + PyObject* stack[5]; + + import_func = _PyDict_GetItemId(f->f_builtins, &PyId___import__); + if (import_func == NULL) { + PyErr_SetString(PyExc_ImportError, "__import__ not found"); + return NULL; + } + + /* Fast path for not overloaded __import__. */ + if (import_func == PyThreadState_GET()->interp->import_func) { + int ilevel = _PyLong_AsInt(level); + if (ilevel == -1 && PyErr_Occurred()) { + return NULL; + } + res = PyImport_ImportModuleLevelObject( + name, + f->f_globals, + f->f_locals == NULL ? Py_None : f->f_locals, + fromlist, + ilevel); + return res; + } + + Py_INCREF(import_func); + + stack[0] = name; + stack[1] = f->f_globals; + stack[2] = f->f_locals == NULL ? Py_None : f->f_locals; + stack[3] = fromlist; + stack[4] = level; + res = _PyObject_FastCall(import_func, stack, 5); + Py_DECREF(import_func); + return res; +} + +static PyObject * import_from(PyObject *v, PyObject *name) { PyObject *x; @@ -5245,7 +5289,7 @@ format_exc_check_arg(PyObject *exc, const char *format_str, PyObject *obj) if (!obj) return; - obj_str = _PyUnicode_AsString(obj); + obj_str = PyUnicode_AsUTF8(obj); if (!obj_str) return; @@ -5276,7 +5320,7 @@ format_exc_unbound(PyCodeObject *co, int oparg) static PyObject * unicode_concatenate(PyObject *v, PyObject *w, - PyFrameObject *f, unsigned char *next_instr) + PyFrameObject *f, const _Py_CODEUNIT *next_instr) { PyObject *res; if (Py_REFCNT(v) == 2) { @@ -5286,10 +5330,11 @@ unicode_concatenate(PyObject *v, PyObject *w, * 'variable'. We try to delete the variable now to reduce * the refcnt to 1. */ - switch (*next_instr) { + int opcode, oparg; + NEXTOPARG(); + switch (opcode) { case STORE_FAST: { - int oparg = PEEKARG(); PyObject **fastlocals = f->f_localsplus; if (GETLOCAL(oparg) == v) SETLOCAL(oparg, NULL); @@ -5299,7 +5344,7 @@ unicode_concatenate(PyObject *v, PyObject *w, { PyObject **freevars = (f->f_localsplus + f->f_code->co_nlocals); - PyObject *c = freevars[PEEKARG()]; + PyObject *c = freevars[oparg]; if (PyCell_GET(c) == v) PyCell_Set(c, NULL); break; @@ -5307,7 +5352,7 @@ unicode_concatenate(PyObject *v, PyObject *w, case STORE_NAME: { PyObject *names = f->f_code->co_names; - PyObject *name = GETITEM(names, PEEKARG()); + PyObject *name = GETITEM(names, oparg); PyObject *locals = f->f_locals; if (PyDict_CheckExact(locals) && PyDict_GetItem(locals, name) == v) { @@ -5367,3 +5412,79 @@ _Py_GetDXProfile(PyObject *self, PyObject *args) } #endif + +Py_ssize_t +_PyEval_RequestCodeExtraIndex(freefunc free) +{ + PyThreadState *tstate = PyThreadState_Get(); + Py_ssize_t new_index; + + if (tstate->co_extra_user_count == MAX_CO_EXTRA_USERS - 1) { + return -1; + } + new_index = tstate->co_extra_user_count++; + tstate->co_extra_freefuncs[new_index] = free; + return new_index; +} + +static void +dtrace_function_entry(PyFrameObject *f) +{ + char* filename; + char* funcname; + int lineno; + + filename = PyUnicode_AsUTF8(f->f_code->co_filename); + funcname = PyUnicode_AsUTF8(f->f_code->co_name); + lineno = PyCode_Addr2Line(f->f_code, f->f_lasti); + + PyDTrace_FUNCTION_ENTRY(filename, funcname, lineno); +} + +static void +dtrace_function_return(PyFrameObject *f) +{ + char* filename; + char* funcname; + int lineno; + + filename = PyUnicode_AsUTF8(f->f_code->co_filename); + funcname = PyUnicode_AsUTF8(f->f_code->co_name); + lineno = PyCode_Addr2Line(f->f_code, f->f_lasti); + + PyDTrace_FUNCTION_RETURN(filename, funcname, lineno); +} + +/* DTrace equivalent of maybe_call_line_trace. */ +static void +maybe_dtrace_line(PyFrameObject *frame, + int *instr_lb, int *instr_ub, int *instr_prev) +{ + int line = frame->f_lineno; + char *co_filename, *co_name; + + /* If the last instruction executed isn't in the current + instruction window, reset the window. + */ + if (frame->f_lasti < *instr_lb || frame->f_lasti >= *instr_ub) { + PyAddrPair bounds; + line = _PyCode_CheckLineNumber(frame->f_code, frame->f_lasti, + &bounds); + *instr_lb = bounds.ap_lower; + *instr_ub = bounds.ap_upper; + } + /* If the last instruction falls at the start of a line or if + it represents a jump backwards, update the frame's line + number and call the trace function. */ + if (frame->f_lasti == *instr_lb || frame->f_lasti < *instr_prev) { + frame->f_lineno = line; + co_filename = PyUnicode_AsUTF8(frame->f_code->co_filename); + if (!co_filename) + co_filename = "?"; + co_name = PyUnicode_AsUTF8(frame->f_code->co_name); + if (!co_name) + co_name = "?"; + PyDTrace_LINE(co_filename, co_name, line); + } + *instr_prev = frame->f_lasti; +} |