diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index ed3ee090ae53db..3efec47f3f4fcf 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -191,6 +191,8 @@ struct _ts { PyObject *previous_executor; uint64_t dict_global_version; + int sp_cached; /* Only used in debug builds */ + }; #ifdef Py_DEBUG diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 994900c007f4bd..81590dde93ac06 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -63,7 +63,7 @@ typedef struct _PyInterpreterFrame { PyObject *f_locals; /* Strong reference, may be NULL. Only valid if not on C stack */ PyFrameObject *frame_obj; /* Strong reference, may be NULL. Only valid if not on C stack */ _Py_CODEUNIT *instr_ptr; /* Instruction currently executing (or about to begin) */ - int stacktop; /* Offset of TOS from localsplus */ + PyObject **stackpointer; uint16_t return_offset; /* Only relevant during a function call */ char owner; /* Locals and stack */ @@ -83,20 +83,20 @@ static inline PyObject **_PyFrame_Stackbase(_PyInterpreterFrame *f) { } static inline PyObject *_PyFrame_StackPeek(_PyInterpreterFrame *f) { - assert(f->stacktop > _PyFrame_GetCode(f)->co_nlocalsplus); - assert(f->localsplus[f->stacktop-1] != NULL); - return f->localsplus[f->stacktop-1]; + assert(f->stackpointer > f->localsplus + _PyFrame_GetCode(f)->co_nlocalsplus); + assert(f->stackpointer[-1] != NULL); + return f->stackpointer[-1]; } static inline PyObject *_PyFrame_StackPop(_PyInterpreterFrame *f) { - assert(f->stacktop > _PyFrame_GetCode(f)->co_nlocalsplus); - f->stacktop--; - return f->localsplus[f->stacktop]; + assert(f->stackpointer > f->localsplus + _PyFrame_GetCode(f)->co_nlocalsplus); + f->stackpointer--; + return *f->stackpointer; } static inline void _PyFrame_StackPush(_PyInterpreterFrame *f, PyObject *value) { - f->localsplus[f->stacktop] = value; - f->stacktop++; + *f->stackpointer = value; + f->stackpointer++; } #define FRAME_SPECIALS_SIZE ((int)((sizeof(_PyInterpreterFrame)-1)/sizeof(PyObject *))) @@ -112,9 +112,12 @@ _PyFrame_NumSlotsForCodeObject(PyCodeObject *code) static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame *dest) { - assert(src->stacktop >= _PyFrame_GetCode(src)->co_nlocalsplus); *dest = *src; - for (int i = 1; i < src->stacktop; i++) { + assert(src->stackpointer != NULL); + int stacktop = (int)(src->stackpointer - src->localsplus); + assert(stacktop >= _PyFrame_GetCode(src)->co_nlocalsplus); + dest->stackpointer = dest->localsplus + stacktop; + for (int i = 1; i < stacktop; i++) { dest->localsplus[i] = src->localsplus[i]; } // Don't leave a dangling pointer to the old frame when creating generators @@ -136,7 +139,7 @@ _PyFrame_Initialize( frame->f_builtins = func->func_builtins; frame->f_globals = func->func_globals; frame->f_locals = locals; - frame->stacktop = code->co_nlocalsplus; + frame->stackpointer = frame->localsplus + code->co_nlocalsplus; frame->frame_obj = NULL; frame->instr_ptr = _PyCode_CODE(code); frame->return_offset = 0; @@ -156,22 +159,29 @@ _PyFrame_GetLocalsArray(_PyInterpreterFrame *frame) return frame->localsplus; } -/* Fetches the stack pointer, and sets stacktop to -1. - Having stacktop <= 0 ensures that invalid - values are not visible to the cycle GC. - We choose -1 rather than 0 to assist debugging. */ +/* Fetches the stack pointer, and sets stackpointer to NULL. + Having stackpointer == NULL ensures that invalid + values are not visible to the cycle GC. */ static inline PyObject** _PyFrame_GetStackPointer(_PyInterpreterFrame *frame) { - PyObject **sp = frame->localsplus + frame->stacktop; - frame->stacktop = -1; +#ifdef Py_DEBUG + PyThreadState_GET()->sp_cached++; +#endif + assert(frame->stackpointer != NULL); + PyObject **sp = frame->stackpointer; + frame->stackpointer = NULL; return sp; } static inline void _PyFrame_SetStackPointer(_PyInterpreterFrame *frame, PyObject **stack_pointer) { - frame->stacktop = (int)(stack_pointer - frame->localsplus); +#ifdef Py_DEBUG + PyThreadState_GET()->sp_cached--; +#endif + assert(frame->stackpointer == NULL); + frame->stackpointer = stack_pointer; } /* Determine whether a frame is incomplete. @@ -299,7 +309,8 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int frame->f_globals = NULL; #endif frame->f_locals = NULL; - frame->stacktop = code->co_nlocalsplus + stackdepth; + assert(stackdepth <= code->co_stacksize); + frame->stackpointer = frame->localsplus + code->co_nlocalsplus + stackdepth; frame->frame_obj = NULL; frame->instr_ptr = _PyCode_CODE(code); frame->owner = FRAME_OWNED_BY_THREAD; diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index d3535800139a66..c39c4591ac5b98 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1149,7 +1149,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [MATCH_MAPPING] = { true, INSTR_FMT_IX, 0 }, [MATCH_SEQUENCE] = { true, INSTR_FMT_IX, 0 }, [NOP] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, - [POP_EXCEPT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [POP_EXCEPT] = { true, INSTR_FMT_IX, 0 }, [POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 78f0eafaa32042..f73a581b4d25b3 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -92,7 +92,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GET_ANEXT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GET_AWAITABLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_YIELD_VALUE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, - [_POP_EXCEPT] = HAS_ESCAPES_FLAG, + [_POP_EXCEPT] = 0, [_LOAD_COMMON_CONSTANT] = HAS_ARG_FLAG, [_LOAD_BUILD_CLASS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 8fe1d77756866a..0562be2a9a55bb 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1561,7 +1561,7 @@ class C(object): pass def func(): return sys._getframe() x = func() - check(x, size('3Pi2cP7P2ic??2P')) + check(x, size('3Pi2cP9Phc2P')) # function def func(): pass check(func, size('16Pi')) @@ -1578,7 +1578,7 @@ def bar(cls): check(bar, size('PP')) # generator def get_gen(): yield 1 - check(get_gen(), size('PP4P4c7P2ic??2P')) + check(get_gen(), size('PP4P4cP9PhcP')) # iterator check(iter('abc'), size('lP')) # callable-iterator diff --git a/Objects/frameobject.c b/Objects/frameobject.c index fc8d6c7a7aee89..1d8727167f68e7 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1619,8 +1619,10 @@ frame_dealloc(PyFrameObject *f) Py_CLEAR(frame->f_funcobj); Py_CLEAR(frame->f_locals); PyObject **locals = _PyFrame_GetLocalsArray(frame); - for (int i = 0; i < frame->stacktop; i++) { - Py_CLEAR(locals[i]); + PyObject **sp = frame->stackpointer; + while (sp > locals) { + sp--; + Py_CLEAR(*sp); } } Py_CLEAR(f->f_back); @@ -1652,11 +1654,13 @@ frame_tp_clear(PyFrameObject *f) /* locals and stack */ PyObject **locals = _PyFrame_GetLocalsArray(f->f_frame); - assert(f->f_frame->stacktop >= 0); - for (int i = 0; i < f->f_frame->stacktop; i++) { - Py_CLEAR(locals[i]); + PyObject **sp = f->f_frame->stackpointer; + assert(sp >= locals); + while (sp > locals) { + sp--; + Py_CLEAR(*sp); } - f->f_frame->stacktop = 0; + f->f_frame->stackpointer = locals; Py_CLEAR(f->f_frame->f_locals); return 0; } @@ -1874,7 +1878,7 @@ frame_get_var(_PyInterpreterFrame *frame, PyCodeObject *co, int i, } PyObject *value = frame->localsplus[i]; - if (frame->stacktop) { + if (frame->stackpointer > frame->localsplus) { if (kind & CO_FAST_FREE) { // The cell was set by COPY_FREE_VARS. assert(value != NULL && PyCell_Check(value)); diff --git a/Objects/object.c b/Objects/object.c index d4fe14c5b3d1aa..a9edc72deca91d 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2831,6 +2831,9 @@ _Py_Dealloc(PyObject *op) destructor dealloc = type->tp_dealloc; #ifdef Py_DEBUG PyThreadState *tstate = _PyThreadState_GET(); +#ifndef Py_GIL_DISABLED + assert(tstate->sp_cached == 0); +#endif PyObject *old_exc = tstate != NULL ? tstate->current_exception : NULL; // Keep the old exception type alive to prevent undefined behavior // on (tstate->curexc_type != old_exc_type) below diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 9a8198515dea5e..d2a38056ae5ef6 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -292,7 +292,10 @@ dummy_func( /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ if (PyGen_Check(receiver)) { - if (monitor_stop_iteration(tstate, frame, this_instr, value)) { + SAVE_SP(); + int err = monitor_stop_iteration(tstate, frame, this_instr, value); + LOAD_SP(); + if (err) { ERROR_NO_POP(); } } @@ -305,7 +308,10 @@ dummy_func( tier1 inst(INSTRUMENTED_END_SEND, (receiver, value -- value)) { if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { - if (monitor_stop_iteration(tstate, frame, this_instr, value)) { + SAVE_SP(); + int err = monitor_stop_iteration(tstate, frame, this_instr, value); + LOAD_SP(); + if (err) { ERROR_NO_POP(); } } @@ -313,7 +319,9 @@ dummy_func( } inst(UNARY_NEGATIVE, (value -- res)) { + SAVE_SP(); res = PyNumber_Negative(value); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(res == NULL, error); } @@ -336,7 +344,9 @@ dummy_func( #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_ToBool(value, next_instr); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(TO_BOOL, deferred); @@ -345,7 +355,9 @@ dummy_func( } op(_TO_BOOL, (value -- res)) { + SAVE_SP(); int err = PyObject_IsTrue(value); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(err < 0, error); res = err ? Py_True : Py_False; @@ -410,7 +422,9 @@ dummy_func( _REPLACE_WITH_TRUE; inst(UNARY_INVERT, (value -- res)) { + SAVE_SP(); res = PyNumber_Invert(value); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(res == NULL, error); } @@ -554,7 +568,9 @@ dummy_func( */ assert(Py_REFCNT(left) >= 2); _Py_DECREF_NO_DEALLOC(left); + SAVE_SP(); PyUnicode_Append(target_local, right); + LOAD_SP(); _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); ERROR_IF(*target_local == NULL, error); // The STORE_FAST is already done. @@ -577,7 +593,9 @@ dummy_func( #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_BinarySubscr(container, sub, next_instr); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_SUBSCR, deferred); @@ -586,7 +604,9 @@ dummy_func( } op(_BINARY_SUBSCR, (container, sub -- res)) { + SAVE_SP(); res = PyObject_GetItem(container, sub); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(res == NULL, error); } @@ -594,14 +614,18 @@ dummy_func( macro(BINARY_SUBSCR) = _SPECIALIZE_BINARY_SUBSCR + _BINARY_SUBSCR; inst(BINARY_SLICE, (container, start, stop -- res)) { + SAVE_SP(); PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); + LOAD_SP(); // Can't use ERROR_IF() here, because we haven't // DECREF'ed container yet, and we still own slice. if (slice == NULL) { res = NULL; } else { + SAVE_SP(); res = PyObject_GetItem(container, slice); + LOAD_SP(); Py_DECREF(slice); } Py_DECREF(container); @@ -615,7 +639,9 @@ dummy_func( err = 1; } else { + SAVE_SP(); err = PyObject_SetItem(container, slice, v); + LOAD_SP(); Py_DECREF(slice); } Py_DECREF(v); @@ -673,7 +699,9 @@ dummy_func( inst(BINARY_SUBSCR_DICT, (unused/1, dict, sub -- res)) { DEOPT_IF(!PyDict_CheckExact(dict)); STAT_INC(BINARY_SUBSCR, hit); + SAVE_SP(); int rc = PyDict_GetItemRef(dict, sub, &res); + LOAD_SP(); if (rc == 0) { _PyErr_SetKeyError(sub); } @@ -710,7 +738,9 @@ dummy_func( } inst(SET_ADD, (set, unused[oparg-1], v -- set, unused[oparg-1])) { + SAVE_SP(); int err = PySet_Add(set, v); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(err, error); } @@ -724,7 +754,9 @@ dummy_func( #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_StoreSubscr(container, sub, next_instr); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_SUBSCR, deferred); @@ -734,7 +766,9 @@ dummy_func( op(_STORE_SUBSCR, (v, container, sub -- )) { /* container[sub] = v */ + SAVE_SP(); int err = PyObject_SetItem(container, sub, v); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(err, error); } @@ -763,34 +797,43 @@ dummy_func( inst(STORE_SUBSCR_DICT, (unused/1, value, dict, sub -- )) { DEOPT_IF(!PyDict_CheckExact(dict)); STAT_INC(STORE_SUBSCR, hit); + SAVE_SP(); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); + LOAD_SP(); Py_DECREF(dict); ERROR_IF(err, error); } inst(DELETE_SUBSCR, (container, sub --)) { /* del container[sub] */ + SAVE_SP(); int err = PyObject_DelItem(container, sub); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(err, error); } inst(CALL_INTRINSIC_1, (value -- res)) { assert(oparg <= MAX_INTRINSIC_1); + SAVE_SP(); res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(res == NULL, error); } inst(CALL_INTRINSIC_2, (value2, value1 -- res)) { assert(oparg <= MAX_INTRINSIC_2); + SAVE_SP(); res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(res == NULL, error); } tier1 inst(RAISE_VARARGS, (args[oparg] -- )) { PyObject *cause = NULL, *exc = NULL; + SAVE_SP(); switch (oparg) { case 2: cause = args[1]; @@ -802,6 +845,7 @@ dummy_func( if (do_raise(tstate, exc, cause)) { assert(oparg == 0); monitor_reraise(tstate, frame, this_instr); + LOAD_SP(); goto exception_unwind; } break; @@ -810,6 +854,7 @@ dummy_func( "bad RAISE_VARARGS oparg"); break; } + LOAD_SP(); ERROR_IF(true, error); } @@ -817,6 +862,7 @@ dummy_func( assert(frame == &entry_frame); assert(_PyFrame_IsIncomplete(frame)); /* Restore previous frame and return. */ + SAVE_SP(); tstate->current_frame = frame->previous; assert(!_PyErr_Occurred(tstate)); tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; @@ -849,9 +895,11 @@ dummy_func( _POP_FRAME; inst(INSTRUMENTED_RETURN_VALUE, (retval --)) { + SAVE_SP(); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, this_instr, retval); + LOAD_SP(); if (err) ERROR_NO_POP(); STACK_SHRINK(1); assert(EMPTY()); @@ -873,9 +921,11 @@ dummy_func( inst(INSTRUMENTED_RETURN_CONST, (--)) { PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); + SAVE_SP(); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, this_instr, retval); + LOAD_SP(); if (err) ERROR_NO_POP(); Py_INCREF(retval); assert(EMPTY()); @@ -900,15 +950,19 @@ dummy_func( } if (getter == NULL) { + SAVE_SP(); _PyErr_Format(tstate, PyExc_TypeError, "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(true, error); } + SAVE_SP(); iter = (*getter)(obj); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(iter == NULL, error); @@ -930,7 +984,9 @@ dummy_func( PyTypeObject *type = Py_TYPE(aiter); if (PyAsyncGen_CheckExact(aiter)) { + SAVE_SP(); awaitable = type->tp_as_async->am_anext(aiter); + LOAD_SP(); if (awaitable == NULL) { ERROR_NO_POP(); } @@ -940,27 +996,34 @@ dummy_func( } if (getter != NULL) { + SAVE_SP(); next_iter = (*getter)(aiter); + LOAD_SP(); if (next_iter == NULL) { ERROR_NO_POP(); } } else { + SAVE_SP(); _PyErr_Format(tstate, PyExc_TypeError, "'async for' requires an iterator with " "__anext__ method, got %.100s", type->tp_name); + LOAD_SP(); ERROR_NO_POP(); } + SAVE_SP(); awaitable = _PyCoro_GetAwaitableIter(next_iter); + LOAD_SP(); if (awaitable == NULL) { + SAVE_SP(); _PyErr_FormatFromCause( PyExc_TypeError, "'async for' received an invalid object " "from __anext__: %.100s", Py_TYPE(next_iter)->tp_name); - + LOAD_SP(); Py_DECREF(next_iter); ERROR_NO_POP(); } else { @@ -970,10 +1033,14 @@ dummy_func( } inst(GET_AWAITABLE, (iterable -- iter)) { + SAVE_SP(); iter = _PyCoro_GetAwaitableIter(iterable); + LOAD_SP(); if (iter == NULL) { + SAVE_SP(); _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg); + LOAD_SP(); } DECREF_INPUTS(); @@ -1003,7 +1070,9 @@ dummy_func( #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_Send(receiver, next_instr); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(SEND, deferred); @@ -1028,18 +1097,25 @@ dummy_func( frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); DISPATCH_INLINED(gen_frame); } + SAVE_SP(); if (Py_IsNone(v) && PyIter_Check(receiver)) { retval = Py_TYPE(receiver)->tp_iternext(receiver); } else { retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); } + LOAD_SP(); if (retval == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) ) { + SAVE_SP(); monitor_raise(tstate, frame, this_instr); + LOAD_SP(); } - if (_PyGen_FetchStopIterationValue(&retval) == 0) { + SAVE_SP(); + int err = _PyGen_FetchStopIterationValue(&retval); + LOAD_SP(); + if (err == 0) { assert(retval != NULL); JUMPBY(oparg); } @@ -1132,7 +1208,11 @@ dummy_func( inst(POP_EXCEPT, (exc_value -- )) { _PyErr_StackItem *exc_info = tstate->exc_info; - Py_XSETREF(exc_info->exc_value, exc_value == Py_None ? NULL : exc_value); + PyObject *tmp = exc_info->exc_value; + exc_info->exc_value = exc_value == Py_None ? NULL : exc_value; + if (tmp != NULL) { + INTERPRETER_DECREF(tmp); + } } tier1 inst(RERAISE, (values[oparg], exc -- values[oparg])) { @@ -1152,7 +1232,9 @@ dummy_func( assert(exc && PyExceptionInstance_Check(exc)); Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); + SAVE_SP(); monitor_reraise(tstate, frame, this_instr); + LOAD_SP(); goto exception_unwind; } @@ -1164,7 +1246,9 @@ dummy_func( else { Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); + SAVE_SP(); monitor_reraise(tstate, frame, this_instr); + LOAD_SP(); goto exception_unwind; } } @@ -1179,7 +1263,9 @@ dummy_func( } else { _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); + SAVE_SP(); monitor_reraise(tstate, frame, this_instr); + LOAD_SP(); goto exception_unwind; } } @@ -1199,7 +1285,10 @@ dummy_func( } inst(LOAD_BUILD_CLASS, ( -- bc)) { - ERROR_IF(PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0, error); + SAVE_SP(); + int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc); + LOAD_SP(); + ERROR_IF(err < 0, error); if (bc == NULL) { _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); @@ -1217,10 +1306,12 @@ dummy_func( DECREF_INPUTS(); ERROR_IF(true, error); } + SAVE_SP(); if (PyDict_CheckExact(ns)) err = PyDict_SetItem(ns, name, v); else err = PyObject_SetItem(ns, name, v); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(err, error); } @@ -1230,16 +1321,22 @@ dummy_func( PyObject *ns = LOCALS(); int err; if (ns == NULL) { + SAVE_SP(); _PyErr_Format(tstate, PyExc_SystemError, "no locals when deleting %R", name); + LOAD_SP(); ERROR_NO_POP(); } + SAVE_SP(); err = PyObject_DelItem(ns, name); + LOAD_SP(); // Can't use ERROR_IF here. if (err != 0) { + SAVE_SP(); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); + LOAD_SP(); ERROR_NO_POP(); } } @@ -1254,7 +1351,9 @@ dummy_func( #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_UnpackSequence(seq, next_instr, oparg); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(UNPACK_SEQUENCE, deferred); @@ -1266,7 +1365,9 @@ dummy_func( op(_UNPACK_SEQUENCE, (seq -- unused[oparg])) { PyObject **top = stack_pointer + oparg - 1; + SAVE_SP(); int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(res == 0, error); } @@ -1308,7 +1409,9 @@ dummy_func( inst(UNPACK_EX, (seq -- unused[oparg & 0xFF], unused, unused[oparg >> 8])) { int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); PyObject **top = stack_pointer + totalargs - 1; + SAVE_SP(); int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(res == 0, error); } @@ -1324,7 +1427,9 @@ dummy_func( if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); next_instr = this_instr; + SAVE_SP(); _Py_Specialize_StoreAttr(owner, next_instr, name); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_ATTR, deferred); @@ -1334,7 +1439,9 @@ dummy_func( op(_STORE_ATTR, (v, owner --)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + SAVE_SP(); int err = PyObject_SetAttr(owner, name, v); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(err, error); } @@ -1343,28 +1450,36 @@ dummy_func( inst(DELETE_ATTR, (owner --)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + SAVE_SP(); int err = PyObject_DelAttr(owner, name); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(err, error); } inst(STORE_GLOBAL, (v --)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + SAVE_SP(); int err = PyDict_SetItem(GLOBALS(), name, v); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(err, error); } inst(DELETE_GLOBAL, (--)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + SAVE_SP(); int err = PyDict_Pop(GLOBALS(), name, NULL); + LOAD_SP(); // Can't use ERROR_IF here. if (err < 0) { ERROR_NO_POP(); } if (err == 0) { + SAVE_SP(); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); + LOAD_SP(); ERROR_NO_POP(); } } @@ -1372,8 +1487,10 @@ dummy_func( inst(LOAD_LOCALS, ( -- locals)) { locals = LOCALS(); if (locals == NULL) { + SAVE_SP(); _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); + LOAD_SP(); ERROR_IF(true, error); } Py_INCREF(locals); @@ -1381,21 +1498,32 @@ dummy_func( inst(LOAD_FROM_DICT_OR_GLOBALS, (mod_or_class_dict -- v)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { + SAVE_SP(); + int err = PyMapping_GetOptionalItem(mod_or_class_dict, name, &v); + LOAD_SP(); + if (err < 0) { ERROR_NO_POP(); } if (v == NULL) { - if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { + SAVE_SP(); + err = PyDict_GetItemRef(GLOBALS(), name, &v); + LOAD_SP(); + if (err < 0) { ERROR_NO_POP(); } if (v == NULL) { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { + SAVE_SP(); + err = PyMapping_GetOptionalItem(BUILTINS(), name, &v); + LOAD_SP(); + if (err < 0) { ERROR_NO_POP(); } if (v == NULL) { + SAVE_SP(); _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); + LOAD_SP(); ERROR_NO_POP(); } } @@ -1406,26 +1534,39 @@ dummy_func( inst(LOAD_NAME, (-- v)) { PyObject *mod_or_class_dict = LOCALS(); if (mod_or_class_dict == NULL) { + SAVE_SP(); _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); + LOAD_SP(); ERROR_IF(true, error); } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { + SAVE_SP(); + int err = PyMapping_GetOptionalItem(mod_or_class_dict, name, &v); + LOAD_SP(); + if (err < 0) { ERROR_NO_POP(); } if (v == NULL) { - if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { + SAVE_SP(); + err = PyDict_GetItemRef(GLOBALS(), name, &v); + LOAD_SP(); + if (err < 0) { ERROR_NO_POP(); } if (v == NULL) { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { + SAVE_SP(); + err = PyMapping_GetOptionalItem(BUILTINS(), name, &v); + LOAD_SP(); + if (err < 0) { ERROR_NO_POP(); } if (v == NULL) { + SAVE_SP(); _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); + LOAD_SP(); ERROR_NO_POP(); } } @@ -1442,7 +1583,9 @@ dummy_func( if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; + SAVE_SP(); _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_GLOBAL, deferred); @@ -1455,15 +1598,19 @@ dummy_func( if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) { + SAVE_SP(); res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), (PyDictObject *)BUILTINS(), name); + LOAD_SP(); if (res == NULL) { if (!_PyErr_Occurred(tstate)) { /* _PyDict_LoadGlobal() returns NULL without raising * an exception if the key doesn't exist */ + SAVE_SP(); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); + LOAD_SP(); } ERROR_IF(true, error); } @@ -1471,14 +1618,22 @@ dummy_func( else { /* Slow-path if globals or builtins is not a dict */ /* namespace 1: globals */ - ERROR_IF(PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0, error); + SAVE_SP(); + int err = PyMapping_GetOptionalItem(GLOBALS(), name, &res); + LOAD_SP(); + ERROR_IF(err < 0, error); if (res == NULL) { /* namespace 2: builtins */ - ERROR_IF(PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0, error); + SAVE_SP(); + err = PyMapping_GetOptionalItem(BUILTINS(), name, &res); + LOAD_SP(); + ERROR_IF(err < 0, error); if (res == NULL) { + SAVE_SP(); _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); + LOAD_SP(); ERROR_IF(true, error); } } @@ -1542,10 +1697,12 @@ dummy_func( inst(DELETE_FAST, (--)) { PyObject *v = GETLOCAL(oparg); if (v == NULL) { + SAVE_SP(); _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); + LOAD_SP(); ERROR_IF(1, error); } SETLOCAL(oparg, NULL); @@ -1568,7 +1725,9 @@ dummy_func( // Fortunately we don't need its superpower. PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); if (oldobj == NULL) { + SAVE_SP(); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + LOAD_SP(); ERROR_NO_POP(); } Py_DECREF(oldobj); @@ -1579,14 +1738,19 @@ dummy_func( assert(class_dict); assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); - if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) { + SAVE_SP(); + int err = PyMapping_GetOptionalItem(class_dict, name, &value); + LOAD_SP(); + if (err < 0) { ERROR_NO_POP(); } if (!value) { PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); value = PyCell_GetRef(cell); if (value == NULL) { + SAVE_SP(); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + LOAD_SP(); ERROR_NO_POP(); } } @@ -1597,14 +1761,18 @@ dummy_func( PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); value = PyCell_GetRef(cell); if (value == NULL) { + SAVE_SP(); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + LOAD_SP(); ERROR_IF(true, error); } } inst(STORE_DEREF, (v --)) { PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); + SAVE_SP(); PyCell_SetTakeRef(cell, v); + LOAD_SP(); } inst(COPY_FREE_VARS, (--)) { @@ -1637,8 +1805,11 @@ dummy_func( } inst(LIST_EXTEND, (list, unused[oparg-1], iterable -- list, unused[oparg-1])) { + SAVE_SP(); PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); + LOAD_SP(); if (none_val == NULL) { + SAVE_SP(); if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) { @@ -1648,6 +1819,7 @@ dummy_func( Py_TYPE(iterable)->tp_name); } DECREF_INPUTS(); + LOAD_SP(); ERROR_IF(true, error); } assert(Py_IsNone(none_val)); @@ -1655,7 +1827,9 @@ dummy_func( } inst(SET_UPDATE, (set, unused[oparg-1], iterable -- set, unused[oparg-1])) { + SAVE_SP(); int err = _PySet_Update(set, iterable); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(err < 0, error); } @@ -1667,8 +1841,11 @@ dummy_func( int err = 0; for (int i = 0; i < oparg; i++) { PyObject *item = values[i]; - if (err == 0) + if (err == 0) { + SAVE_SP(); err = PySet_Add(set, item); + LOAD_SP(); + } Py_DECREF(item); } if (err != 0) { @@ -1678,10 +1855,12 @@ dummy_func( } inst(BUILD_MAP, (values[oparg*2] -- map)) { + SAVE_SP(); map = _PyDict_FromItems( values, 2, values+1, 2, oparg); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(map == NULL, error); } @@ -1690,12 +1869,17 @@ dummy_func( int err; PyObject *ann_dict; if (LOCALS() == NULL) { + SAVE_SP(); _PyErr_Format(tstate, PyExc_SystemError, "no locals found when setting up annotations"); + LOAD_SP(); ERROR_IF(true, error); } /* check if __annotations__ in locals()... */ - ERROR_IF(PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0, error); + SAVE_SP(); + err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); + LOAD_SP(); + ERROR_IF(err < 0, error); if (ann_dict == NULL) { ann_dict = PyDict_New(); ERROR_IF(ann_dict == NULL, error); @@ -1712,20 +1896,27 @@ dummy_func( inst(BUILD_CONST_KEY_MAP, (values[oparg], keys -- map)) { assert(PyTuple_CheckExact(keys)); assert(PyTuple_GET_SIZE(keys) == (Py_ssize_t)oparg); + SAVE_SP(); map = _PyDict_FromItems( &PyTuple_GET_ITEM(keys, 0), 1, values, 1, oparg); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(map == NULL, error); } inst(DICT_UPDATE, (dict, unused[oparg - 1], update -- dict, unused[oparg - 1])) { - if (PyDict_Update(dict, update) < 0) { + SAVE_SP(); + int err = PyDict_Update(dict, update); + LOAD_SP(); + if (err < 0) { + SAVE_SP(); if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { _PyErr_Format(tstate, PyExc_TypeError, "'%.200s' object is not a mapping", Py_TYPE(update)->tp_name); } + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(true, error); } @@ -1733,8 +1924,13 @@ dummy_func( } inst(DICT_MERGE, (callable, unused, unused, dict, unused[oparg - 1], update -- callable, unused, unused, dict, unused[oparg - 1])) { - if (_PyDict_MergeEx(dict, update, 2) < 0) { + SAVE_SP(); + int err = _PyDict_MergeEx(dict, update, 2); + LOAD_SP(); + if (err < 0) { + SAVE_SP(); _PyEval_FormatKwargsError(tstate, callable, update); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(true, error); } @@ -1745,7 +1941,10 @@ dummy_func( assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references - ERROR_IF(_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0, error); + SAVE_SP(); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, key, value); + LOAD_SP(); + ERROR_IF(err != 0, error); } inst(INSTRUMENTED_LOAD_SUPER_ATTR, (unused/1, unused, unused, unused -- unused, unused if (oparg & 1))) { @@ -1765,7 +1964,9 @@ dummy_func( int load_method = oparg & 1; if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_SUPER_ATTR, deferred); @@ -1775,15 +1976,18 @@ dummy_func( tier1 op(_LOAD_SUPER_ATTR, (global_super, class, self -- attr, null if (oparg & 1))) { if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + SAVE_SP(); PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, global_super, arg); + LOAD_SP(); ERROR_IF(err, error); } // we make no attempt to optimize here; specializations should // handle any case whose performance we care about PyObject *stack[] = {class, self}; + SAVE_SP(); PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; @@ -1801,10 +2005,13 @@ dummy_func( } } } + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(super == NULL, error); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + SAVE_SP(); attr = PyObject_GetAttr(super, name); + LOAD_SP(); Py_DECREF(super); ERROR_IF(attr == NULL, error); null = NULL; @@ -1818,7 +2025,9 @@ dummy_func( DEOPT_IF(!PyType_Check(class)); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + SAVE_SP(); attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(attr == NULL, error); } @@ -1831,8 +2040,10 @@ dummy_func( PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; int method_found = 0; + SAVE_SP(); attr = _PySuper_Lookup(cls, self, name, Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); + LOAD_SP(); Py_DECREF(global_super); Py_DECREF(class); if (attr == NULL) { @@ -1867,7 +2078,9 @@ dummy_func( if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; + SAVE_SP(); _Py_Specialize_LoadAttr(owner, next_instr, name); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_ATTR, deferred); @@ -1880,7 +2093,10 @@ dummy_func( if (oparg & 1) { /* Designed to work in tandem with CALL, pushes two values. */ attr = NULL; - if (_PyObject_GetMethod(owner, name, &attr)) { + SAVE_SP(); + int err = _PyObject_GetMethod(owner, name, &attr); + LOAD_SP(); + if (err) { /* We can bypass temporary bound method object. meth is unbound method and obj is self. meth | self | arg1 | ... | argN @@ -1902,7 +2118,9 @@ dummy_func( } else { /* Classic, pushes one value. */ + SAVE_SP(); attr = PyObject_GetAttr(owner, name); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(attr == NULL, error); } @@ -2165,7 +2383,9 @@ dummy_func( STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; *(PyObject **)addr = value; - Py_XDECREF(old_value); + if (old_value != NULL) { + Py_DECREF(old_value); + } Py_DECREF(owner); } @@ -2184,7 +2404,9 @@ dummy_func( #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_CompareOp(left, right, next_instr, oparg); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(COMPARE_OP, deferred); @@ -2194,7 +2416,9 @@ dummy_func( op(_COMPARE_OP, (left, right -- res)) { assert((oparg >> 5) <= Py_GE); + SAVE_SP(); res = PyObject_RichCompare(left, right, oparg >> 5); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(res == NULL, error); if (oparg & 16) { @@ -2271,7 +2495,9 @@ dummy_func( }; op(_CONTAINS_OP, (left, right -- b)) { + SAVE_SP(); int res = PySequence_Contains(right, left); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(res < 0, error); b = (res ^ oparg) ? Py_True : Py_False; @@ -2281,7 +2507,9 @@ dummy_func( #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_ContainsOp(right, next_instr); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(CONTAINS_OP, deferred); @@ -2295,7 +2523,9 @@ dummy_func( DEOPT_IF(!(PySet_CheckExact(right) || PyFrozenSet_CheckExact(right))); STAT_INC(CONTAINS_OP, hit); // Note: both set and frozenset use the same seq_contains method! + SAVE_SP(); int res = _PySet_Contains((PySetObject *)right, left); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(res < 0, error); b = (res ^ oparg) ? Py_True : Py_False; @@ -2304,7 +2534,9 @@ dummy_func( inst(CONTAINS_OP_DICT, (unused/1, left, right -- b)) { DEOPT_IF(!PyDict_CheckExact(right)); STAT_INC(CONTAINS_OP, hit); + SAVE_SP(); int res = PyDict_Contains(right, left); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(res < 0, error); b = (res ^ oparg) ? Py_True : Py_False; @@ -2318,8 +2550,10 @@ dummy_func( match = NULL; rest = NULL; + SAVE_SP(); int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, &match, &rest); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(res < 0, error); @@ -2333,26 +2567,32 @@ dummy_func( inst(CHECK_EXC_MATCH, (left, right -- left, b)) { assert(PyExceptionInstance_Check(left)); + SAVE_SP(); if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(true, error); } - int res = PyErr_GivenExceptionMatches(left, right); + LOAD_SP(); DECREF_INPUTS(); b = res ? Py_True : Py_False; } tier1 inst(IMPORT_NAME, (level, fromlist -- res)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + SAVE_SP(); res = import_name(tstate, frame, name, fromlist, level); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(res == NULL, error); } tier1 inst(IMPORT_FROM, (from -- from, res)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + SAVE_SP(); res = import_from(tstate, from, name); + LOAD_SP(); ERROR_IF(res == NULL, error); } @@ -2488,7 +2728,9 @@ dummy_func( // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(names)); + SAVE_SP(); attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names); + LOAD_SP(); DECREF_INPUTS(); if (attrs) { assert(PyTuple_CheckExact(attrs)); // Success! @@ -2511,13 +2753,17 @@ dummy_func( inst(MATCH_KEYS, (subject, keys -- subject, keys, values_or_none)) { // On successful match, PUSH(values). Otherwise, PUSH(None). + SAVE_SP(); values_or_none = _PyEval_MatchKeys(tstate, subject, keys); + LOAD_SP(); ERROR_IF(values_or_none == NULL, error); } inst(GET_ITER, (iterable -- iter)) { /* before: [obj]; after [getiter(obj)] */ + SAVE_SP(); iter = PyObject_GetIter(iterable); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(iter == NULL, error); } @@ -2541,7 +2787,9 @@ dummy_func( } else { /* `iterable` is not a generator. */ + SAVE_SP(); iter = PyObject_GetIter(iterable); + LOAD_SP(); if (iter == NULL) { ERROR_NO_POP(); } @@ -2566,7 +2814,9 @@ dummy_func( #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_ForIter(iter, next_instr, oparg); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(FOR_ITER, deferred); @@ -2576,14 +2826,18 @@ dummy_func( replaced op(_FOR_ITER, (iter -- iter, next)) { /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ + SAVE_SP(); next = (*Py_TYPE(iter)->tp_iternext)(iter); + LOAD_SP(); if (next == NULL) { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { ERROR_NO_POP(); } + SAVE_SP(); monitor_raise(tstate, frame, this_instr); _PyErr_Clear(tstate); + LOAD_SP(); } /* iterator ended normally */ assert(next_instr[oparg].op.code == END_FOR || @@ -2599,13 +2853,17 @@ dummy_func( op(_FOR_ITER_TIER_TWO, (iter -- iter, next)) { /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ + SAVE_SP(); next = (*Py_TYPE(iter)->tp_iternext)(iter); + LOAD_SP(); if (next == NULL) { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { ERROR_NO_POP(); } + SAVE_SP(); _PyErr_Clear(tstate); + LOAD_SP(); } /* iterator ended normally */ /* The translator sets the deopt target just past the matching END_FOR */ @@ -2619,7 +2877,9 @@ dummy_func( inst(INSTRUMENTED_FOR_ITER, (unused/1 -- )) { _Py_CODEUNIT *target; PyObject *iter = TOP(); + SAVE_SP(); PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); + LOAD_SP(); if (next != NULL) { PUSH(next); target = next_instr; @@ -2629,8 +2889,10 @@ dummy_func( if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { ERROR_NO_POP(); } + SAVE_SP(); monitor_raise(tstate, frame, this_instr); _PyErr_Clear(tstate); + LOAD_SP(); } /* iterator ended normally */ assert(next_instr[oparg].op.code == END_FOR || @@ -2811,7 +3073,9 @@ dummy_func( } ERROR_NO_POP(); } + SAVE_SP(); exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__)); + LOAD_SP(); if (exit == NULL) { if (!_PyErr_Occurred(tstate)) { _PyErr_Format(tstate, PyExc_TypeError, @@ -2824,7 +3088,9 @@ dummy_func( ERROR_NO_POP(); } DECREF_INPUTS(); + SAVE_SP(); res = PyObject_CallNoArgs(enter); + LOAD_SP(); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); @@ -2836,7 +3102,9 @@ dummy_func( /* pop the context manager, push its __exit__ and the * value returned from calling its __enter__ */ + SAVE_SP(); PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__enter__)); + LOAD_SP(); if (enter == NULL) { if (!_PyErr_Occurred(tstate)) { _PyErr_Format(tstate, PyExc_TypeError, @@ -2846,7 +3114,9 @@ dummy_func( } ERROR_NO_POP(); } + SAVE_SP(); exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__exit__)); + LOAD_SP(); if (exit == NULL) { if (!_PyErr_Occurred(tstate)) { _PyErr_Format(tstate, PyExc_TypeError, @@ -2859,7 +3129,9 @@ dummy_func( ERROR_NO_POP(); } DECREF_INPUTS(); + SAVE_SP(); res = PyObject_CallNoArgs(enter); + LOAD_SP(); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); @@ -2890,8 +3162,10 @@ dummy_func( assert(PyLong_Check(lasti)); (void)lasti; // Shut up compiler warning if asserts are off PyObject *stack[4] = {NULL, exc, val, tb}; + SAVE_SP(); res = PyObject_Vectorcall(exit_func, stack + 1, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + LOAD_SP(); ERROR_IF(res == NULL, error); } @@ -3034,9 +3308,11 @@ dummy_func( PyObject *function = PEEK(oparg + 2); PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING : PEEK(total_args); + SAVE_SP(); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, function, arg); + LOAD_SP(); ERROR_IF(err, error); PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); GO_TO_INSTRUCTION(CALL); @@ -3071,7 +3347,9 @@ dummy_func( #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_Call(callable, next_instr, oparg + (self_or_null != NULL)); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(CALL, deferred); @@ -3104,10 +3382,12 @@ dummy_func( { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + SAVE_SP(); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)callable, locals, args, total_args, NULL ); + LOAD_SP(); // Manipulate stack directly since we leave using DISPATCH_INLINED(). STACK_SHRINK(oparg + 2); // The frame has stolen all the arguments from the stack, @@ -3119,13 +3399,16 @@ dummy_func( DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ + SAVE_SP(); res = PyObject_Vectorcall( callable, args, total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + LOAD_SP(); if (opcode == INSTRUMENTED_CALL) { PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING : args[0]; + SAVE_SP(); if (res == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, @@ -3139,6 +3422,7 @@ dummy_func( Py_CLEAR(res); } } + LOAD_SP(); } assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(callable); @@ -3164,10 +3448,12 @@ dummy_func( assert(Py_TYPE(callable) == &PyFunction_Type); int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + SAVE_SP(); new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)callable, locals, args, total_args, NULL ); + LOAD_SP(); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. SYNC_SP(); @@ -3234,10 +3520,12 @@ dummy_func( total_args++; } /* Callable is not a normal Python function */ + SAVE_SP(); res = PyObject_Vectorcall( callable, args, total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + LOAD_SP(); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(callable); for (int i = 0; i < total_args; i++) { @@ -3302,7 +3590,7 @@ dummy_func( // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); SYNC_SP(); - _PyFrame_SetStackPointer(frame, stack_pointer); + SAVE_SP(); new_frame->previous = frame; CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; @@ -3339,7 +3627,9 @@ dummy_func( DEOPT_IF(null != NULL); DEOPT_IF(callable != (PyObject *)&PyType_Type); STAT_INC(CALL, hit); + SAVE_SP(); res = Py_NewRef(Py_TYPE(arg)); + LOAD_SP(); Py_DECREF(arg); } @@ -3348,7 +3638,9 @@ dummy_func( DEOPT_IF(null != NULL); DEOPT_IF(callable != (PyObject *)&PyUnicode_Type); STAT_INC(CALL, hit); + SAVE_SP(); res = PyObject_Str(arg); + LOAD_SP(); Py_DECREF(arg); ERROR_IF(res == NULL, error); } @@ -3364,7 +3656,9 @@ dummy_func( DEOPT_IF(null != NULL); DEOPT_IF(callable != (PyObject *)&PyTuple_Type); STAT_INC(CALL, hit); + SAVE_SP(); res = PySequence_Tuple(arg); + LOAD_SP(); Py_DECREF(arg); ERROR_IF(res == NULL, error); } @@ -3446,7 +3740,9 @@ dummy_func( PyTypeObject *tp = (PyTypeObject *)callable; DEOPT_IF(tp->tp_vectorcall == NULL); STAT_INC(CALL, hit); + SAVE_SP(); res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); + LOAD_SP(); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { Py_DECREF(args[i]); @@ -3477,7 +3773,9 @@ dummy_func( PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); PyObject *arg = args[0]; _Py_EnterRecursiveCallTstateUnchecked(tstate); + SAVE_SP(); res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); + LOAD_SP(); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -3504,10 +3802,12 @@ dummy_func( STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); /* res = func(self, args, nargs) */ + SAVE_SP(); res = ((PyCFunctionFast)(void(*)(void))cfunc)( PyCFunction_GET_SELF(callable), args, total_args); + LOAD_SP(); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ @@ -3538,7 +3838,9 @@ dummy_func( PyCFunctionFastWithKeywords cfunc = (PyCFunctionFastWithKeywords)(void(*)(void)) PyCFunction_GET_FUNCTION(callable); + SAVE_SP(); res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); + LOAD_SP(); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ @@ -3567,7 +3869,9 @@ dummy_func( DEOPT_IF(callable != interp->callable_cache.len); STAT_INC(CALL, hit); PyObject *arg = args[0]; + SAVE_SP(); Py_ssize_t len_i = PyObject_Length(arg); + LOAD_SP(); if (len_i < 0) { ERROR_NO_POP(); } @@ -3593,7 +3897,9 @@ dummy_func( STAT_INC(CALL, hit); PyObject *cls = args[1]; PyObject *inst = args[0]; + SAVE_SP(); int retval = PyObject_IsInstance(inst, cls); + LOAD_SP(); if (retval < 0) { ERROR_NO_POP(); } @@ -3646,7 +3952,9 @@ dummy_func( STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); + SAVE_SP(); res = _PyCFunction_TrampolineCall(cfunc, self, arg); + LOAD_SP(); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(self); @@ -3678,7 +3986,9 @@ dummy_func( int nargs = total_args - 1; PyCFunctionFastWithKeywords cfunc = (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; + SAVE_SP(); res = cfunc(self, args + 1, nargs, NULL); + LOAD_SP(); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ @@ -3714,7 +4024,9 @@ dummy_func( STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); + SAVE_SP(); res = _PyCFunction_TrampolineCall(cfunc, self, NULL); + LOAD_SP(); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(self); @@ -3745,7 +4057,9 @@ dummy_func( PyCFunctionFast cfunc = (PyCFunctionFast)(void(*)(void))meth->ml_meth; int nargs = total_args - 1; + SAVE_SP(); res = cfunc(self, args + 1, nargs); + LOAD_SP(); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Clear the stack of the arguments. */ for (int i = 0; i < total_args; i++) { @@ -3767,9 +4081,11 @@ dummy_func( PyObject *function = PEEK(oparg + 3); PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING : PEEK(total_args + 1); + SAVE_SP(); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, function, arg); + LOAD_SP(); ERROR_IF(err, error); GO_TO_INSTRUCTION(CALL_KW); } @@ -3799,10 +4115,12 @@ dummy_func( { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + SAVE_SP(); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)callable, locals, args, positional_args, kwnames ); + LOAD_SP(); Py_DECREF(kwnames); // Manipulate stack directly since we leave using DISPATCH_INLINED(). STACK_SHRINK(oparg + 3); @@ -3816,13 +4134,16 @@ dummy_func( DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ + SAVE_SP(); res = PyObject_Vectorcall( callable, args, positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames); + LOAD_SP(); if (opcode == INSTRUMENTED_CALL_KW) { PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING : args[0]; + SAVE_SP(); if (res == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, @@ -3836,6 +4157,7 @@ dummy_func( Py_CLEAR(res); } } + LOAD_SP(); } Py_DECREF(kwnames); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -3859,33 +4181,43 @@ dummy_func( if (check_args_iterable(tstate, func, callargs) < 0) { ERROR_NO_POP(); } + SAVE_SP(); PyObject *tuple = PySequence_Tuple(callargs); + LOAD_SP(); if (tuple == NULL) { ERROR_NO_POP(); } - Py_SETREF(callargs, tuple); + Py_DECREF(callargs); + callargs = tuple; } assert(PyTuple_CheckExact(callargs)); EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; + SAVE_SP(); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, func, arg); + LOAD_SP(); if (err) ERROR_NO_POP(); + SAVE_SP(); result = PyObject_Call(func, callargs, kwargs); - + LOAD_SP(); if (!PyFunction_Check(func) && !PyMethod_Check(func)) { if (result == NULL) { + SAVE_SP(); _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, frame, this_instr, func, arg); + LOAD_SP(); } else { + SAVE_SP(); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, frame, this_instr, func, arg); + LOAD_SP(); if (err < 0) { Py_CLEAR(result); } @@ -3900,10 +4232,11 @@ dummy_func( Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - + SAVE_SP(); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, (PyFunctionObject *)func, locals, nargs, callargs, kwargs); + LOAD_SP(); // Need to manually shrink the stack since we exit with DISPATCH_INLINED. STACK_SHRINK(oparg + 3); if (new_frame == NULL) { @@ -3913,7 +4246,9 @@ dummy_func( frame->return_offset = 1; DISPATCH_INLINED(new_frame); } + SAVE_SP(); result = PyObject_Call(func, callargs, kwargs); + LOAD_SP(); } DECREF_INPUTS(); assert(PEEK(2 + (oparg & 1)) == NULL); @@ -3922,10 +4257,10 @@ dummy_func( } inst(MAKE_FUNCTION, (codeobj -- func)) { - + SAVE_SP(); PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); - + LOAD_SP(); Py_DECREF(codeobj); if (func_obj == NULL) { ERROR_NO_POP(); @@ -3998,7 +4333,9 @@ dummy_func( conversion_func conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); conv_fn = _PyEval_ConversionFuncs[oparg]; + SAVE_SP(); result = conv_fn(value); + LOAD_SP(); Py_DECREF(value); ERROR_IF(result == NULL, error); } @@ -4007,7 +4344,9 @@ dummy_func( /* If value is a unicode object, then we know the result * of format(value) is value itself. */ if (!PyUnicode_CheckExact(value)) { + SAVE_SP(); res = PyObject_Format(value, NULL); + LOAD_SP(); Py_DECREF(value); ERROR_IF(res == NULL, error); } @@ -4017,7 +4356,9 @@ dummy_func( } inst(FORMAT_WITH_SPEC, (value, fmt_spec -- res)) { + SAVE_SP(); res = PyObject_Format(value, fmt_spec); + LOAD_SP(); Py_DECREF(value); Py_DECREF(fmt_spec); ERROR_IF(res == NULL, error); @@ -4032,7 +4373,9 @@ dummy_func( #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_OP, deferred); @@ -4044,7 +4387,9 @@ dummy_func( op(_BINARY_OP, (lhs, rhs -- res)) { assert(_PyEval_BinaryOps[oparg]); + SAVE_SP(); res = _PyEval_BinaryOps[oparg](lhs, rhs); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(res == NULL, error); } @@ -4057,8 +4402,10 @@ dummy_func( } inst(INSTRUMENTED_INSTRUCTION, ( -- )) { + SAVE_SP(); int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, this_instr); + LOAD_SP(); ERROR_IF(next_opcode < 0, error); next_instr = this_instr; if (_PyOpcode_Caches[next_opcode]) { diff --git a/Python/ceval.c b/Python/ceval.c index 324d062fe9bb43..b16ebff4ee6e9a 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -51,16 +51,34 @@ # error "ceval.c must be build with Py_BUILD_CORE define for best performance" #endif -#if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_GIL_DISABLED) -// GH-89279: The MSVC compiler does not inline these static inline functions -// in PGO build in _PyEval_EvalFrameDefault(), because this function is over -// the limit of PGO, and that limit cannot be configured. -// Define them as macros to make sure that they are always inlined by the -// preprocessor. -// TODO: implement Py_DECREF macro for Py_GIL_DISABLED - -#undef Py_DECREF -#define Py_DECREF(arg) \ +#ifdef Py_GIL_DISABLED + +#define INTERPRETER_DECREF Py_DECREF + +#else + +#ifdef Py_REF_DEBUG +static inline void interpreter_decref(_PyInterpreterFrame *frame, PyObject **sp, const char *filename, int lineno, PyObject *op) +{ + if (op->ob_refcnt <= 0) { + _Py_NegativeRefcount(filename, lineno, op); + } + if (_Py_IsImmortal(op)) { + return; + } + _Py_DECREF_STAT_INC(); + _Py_DECREF_DecRefTotal(); + if (--op->ob_refcnt == 0) { + _PyFrame_SetStackPointer(frame, sp); + _Py_Dealloc(op); + _PyFrame_GetStackPointer(frame); + } +} +#define INTERPRETER_DECREF(op) interpreter_decref(frame, stack_pointer, __FILE__, __LINE__, _PyObject_CAST(op)) + +#else + +#define INTERPRETER_DECREF(arg) \ do { \ PyObject *op = _PyObject_CAST(arg); \ if (_Py_IsImmortal(op)) { \ @@ -68,19 +86,14 @@ } \ _Py_DECREF_STAT_INC(); \ if (--op->ob_refcnt == 0) { \ - destructor dealloc = Py_TYPE(op)->tp_dealloc; \ - (*dealloc)(op); \ + _PyFrame_SetStackPointer(frame, stack_pointer) \ + _Py_Dealloc(op); \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ } \ } while (0) +#endif // Py_REF_DEBUG -#undef Py_XDECREF -#define Py_XDECREF(arg) \ - do { \ - PyObject *xop = _PyObject_CAST(arg); \ - if (xop != NULL) { \ - Py_DECREF(xop); \ - } \ - } while (0) +#endif // Py_GIL_DISABLED #undef Py_IS_TYPE #define Py_IS_TYPE(ob, type) \ @@ -99,7 +112,6 @@ d(op); \ } \ } while (0) -#endif #ifdef LLTRACE @@ -719,7 +731,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #endif entry_frame.f_executable = Py_None; entry_frame.instr_ptr = (_Py_CODEUNIT *)_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS + 1; - entry_frame.stacktop = 0; + entry_frame.stackpointer = entry_frame.localsplus; entry_frame.owner = FRAME_OWNED_BY_CSTACK; entry_frame.return_offset = 0; /* Push frame */ @@ -882,7 +894,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int PyTraceBack_Here(f); } } + SAVE_SP(); monitor_raise(tstate, frame, next_instr-1); + LOAD_SP(); exception_unwind: { /* We can't use frame->instr_ptr here, as RERAISE may have set it */ @@ -896,7 +910,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int PyObject **stackbase = _PyFrame_Stackbase(frame); while (stack_pointer > stackbase) { PyObject *o = POP(); - Py_XDECREF(o); + if (o != NULL) { + INTERPRETER_DECREF(o); + } } assert(STACK_LEVEL() == 0); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -908,7 +924,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int PyObject **new_top = _PyFrame_Stackbase(frame) + level; while (stack_pointer > new_top) { PyObject *v = POP(); - Py_XDECREF(v); + if (v != NULL) { + INTERPRETER_DECREF(v); + } } if (lasti) { int frame_lasti = _PyInterpreterFrame_LASTI(frame); @@ -927,7 +945,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int PUSH(exc); next_instr = _PyCode_CODE(_PyFrame_GetCode(frame)) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + SAVE_SP(); + int err = monitor_handled(tstate, frame, next_instr, exc); + LOAD_SP(); + if (err < 0) { goto exception_unwind; } /* Resume normal execution */ diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 50941e4ec473e8..9828162f6ada36 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -137,7 +137,10 @@ do { \ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); \ QSBR_QUIESCENT_STATE(tstate); \ if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { \ - if (_Py_HandlePending(tstate) != 0) { \ + SAVE_SP(); \ + int err = _Py_HandlePending(tstate); \ + LOAD_SP(); \ + if (err != 0) { \ GOTO_ERROR(error); \ } \ } @@ -262,9 +265,14 @@ GETITEM(PyObject *v, Py_ssize_t i) { This is because it is possible that during the DECREF the frame is accessed by other code (e.g. a __del__ method or gc.collect()) and the variable would be pointing to already-freed memory. */ -#define SETLOCAL(i, value) do { PyObject *tmp = GETLOCAL(i); \ - GETLOCAL(i) = value; \ - Py_XDECREF(tmp); } while (0) +#define SETLOCAL(i, value) \ +do { \ + PyObject *tmp = frame->localsplus[i]; \ + frame->localsplus[i] = value; \ + if (tmp != NULL) { \ + INTERPRETER_DECREF(tmp); \ + } \ +} while (0) #define GO_TO_INSTRUCTION(op) goto PREDICT_ID(op) @@ -400,7 +408,10 @@ static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) { /* There's no STORE_IP(), it's inlined by the code generator. */ #define LOAD_SP() \ -stack_pointer = _PyFrame_GetStackPointer(frame); +stack_pointer = _PyFrame_GetStackPointer(frame) + +#define SAVE_SP() \ +_PyFrame_SetStackPointer(frame, stack_pointer) /* Tier-switching macros. */ @@ -447,3 +458,4 @@ do { \ #define EXIT_TO_TRACE() goto exit_to_trace #define EXIT_TO_TIER1() goto exit_to_tier1 #define EXIT_TO_TIER1_DYNAMIC() goto exit_to_tier1_dynamic; + diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index e862364cb23e7a..4961fc02b7ff2e 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -269,7 +269,7 @@ case _POP_TOP: { PyObject *value; value = stack_pointer[-1]; - Py_DECREF(value); + INTERPRETER_DECREF(value); stack_pointer += -1; break; } @@ -287,7 +287,7 @@ PyObject *receiver; value = stack_pointer[-1]; receiver = stack_pointer[-2]; - Py_DECREF(receiver); + INTERPRETER_DECREF(receiver); stack_pointer[-2] = value; stack_pointer += -1; break; @@ -297,8 +297,10 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; + SAVE_SP(); res = PyNumber_Negative(value); - Py_DECREF(value); + LOAD_SP(); + INTERPRETER_DECREF(value); if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-1] = res; break; @@ -318,8 +320,10 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; + SAVE_SP(); int err = PyObject_IsTrue(value); - Py_DECREF(value); + LOAD_SP(); + INTERPRETER_DECREF(value); if (err < 0) JUMP_TO_ERROR(); res = err ? Py_True : Py_False; stack_pointer[-1] = res; @@ -351,7 +355,7 @@ res = Py_False; } else { - Py_DECREF(value); + INTERPRETER_DECREF(value); res = Py_True; } stack_pointer[-1] = res; @@ -368,7 +372,7 @@ } STAT_INC(TO_BOOL, hit); res = Py_SIZE(value) ? Py_True : Py_False; - Py_DECREF(value); + INTERPRETER_DECREF(value); stack_pointer[-1] = res; break; } @@ -403,7 +407,7 @@ } else { assert(Py_SIZE(value)); - Py_DECREF(value); + INTERPRETER_DECREF(value); res = Py_True; } stack_pointer[-1] = res; @@ -414,7 +418,7 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; - Py_DECREF(value); + INTERPRETER_DECREF(value); res = Py_True; stack_pointer[-1] = res; break; @@ -424,8 +428,10 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; + SAVE_SP(); res = PyNumber_Invert(value); - Py_DECREF(value); + LOAD_SP(); + INTERPRETER_DECREF(value); if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-1] = res; break; @@ -637,9 +643,11 @@ PyObject *res; sub = stack_pointer[-1]; container = stack_pointer[-2]; + SAVE_SP(); res = PyObject_GetItem(container, sub); - Py_DECREF(container); - Py_DECREF(sub); + LOAD_SP(); + INTERPRETER_DECREF(container); + INTERPRETER_DECREF(sub); if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2] = res; stack_pointer += -1; @@ -654,17 +662,21 @@ stop = stack_pointer[-1]; start = stack_pointer[-2]; container = stack_pointer[-3]; + SAVE_SP(); PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); + LOAD_SP(); // Can't use ERROR_IF() here, because we haven't // DECREF'ed container yet, and we still own slice. if (slice == NULL) { res = NULL; } else { + SAVE_SP(); res = PyObject_GetItem(container, slice); - Py_DECREF(slice); + LOAD_SP(); + INTERPRETER_DECREF(slice); } - Py_DECREF(container); + INTERPRETER_DECREF(container); if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-3] = res; stack_pointer += -2; @@ -686,11 +698,13 @@ err = 1; } else { + SAVE_SP(); err = PyObject_SetItem(container, slice, v); - Py_DECREF(slice); + LOAD_SP(); + INTERPRETER_DECREF(slice); } - Py_DECREF(v); - Py_DECREF(container); + INTERPRETER_DECREF(v); + INTERPRETER_DECREF(container); if (err) JUMP_TO_ERROR(); stack_pointer += -4; break; @@ -725,7 +739,7 @@ assert(res != NULL); Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(list); + INTERPRETER_DECREF(list); stack_pointer[-2] = res; stack_pointer += -1; break; @@ -763,7 +777,7 @@ STAT_INC(BINARY_SUBSCR, hit); res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(str); + INTERPRETER_DECREF(str); stack_pointer[-2] = res; stack_pointer += -1; break; @@ -798,7 +812,7 @@ assert(res != NULL); Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(tuple); + INTERPRETER_DECREF(tuple); stack_pointer[-2] = res; stack_pointer += -1; break; @@ -815,12 +829,14 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(BINARY_SUBSCR, hit); + SAVE_SP(); int rc = PyDict_GetItemRef(dict, sub, &res); + LOAD_SP(); if (rc == 0) { _PyErr_SetKeyError(sub); } - Py_DECREF(dict); - Py_DECREF(sub); + INTERPRETER_DECREF(dict); + INTERPRETER_DECREF(sub); if (rc <= 0) JUMP_TO_ERROR(); // not found or error stack_pointer[-2] = res; @@ -847,8 +863,10 @@ oparg = CURRENT_OPARG(); v = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; + SAVE_SP(); int err = PySet_Add(set, v); - Py_DECREF(v); + LOAD_SP(); + INTERPRETER_DECREF(v); if (err) JUMP_TO_ERROR(); stack_pointer += -1; break; @@ -862,10 +880,12 @@ container = stack_pointer[-2]; v = stack_pointer[-3]; /* container[sub] = v */ + SAVE_SP(); int err = PyObject_SetItem(container, sub, v); - Py_DECREF(v); - Py_DECREF(container); - Py_DECREF(sub); + LOAD_SP(); + INTERPRETER_DECREF(v); + INTERPRETER_DECREF(container); + INTERPRETER_DECREF(sub); if (err) JUMP_TO_ERROR(); stack_pointer += -3; break; @@ -901,9 +921,9 @@ PyObject *old_value = PyList_GET_ITEM(list, index); PyList_SET_ITEM(list, index, value); assert(old_value != NULL); - Py_DECREF(old_value); + INTERPRETER_DECREF(old_value); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(list); + INTERPRETER_DECREF(list); stack_pointer += -3; break; } @@ -920,8 +940,10 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(STORE_SUBSCR, hit); + SAVE_SP(); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); - Py_DECREF(dict); + LOAD_SP(); + INTERPRETER_DECREF(dict); if (err) JUMP_TO_ERROR(); stack_pointer += -3; break; @@ -933,9 +955,11 @@ sub = stack_pointer[-1]; container = stack_pointer[-2]; /* del container[sub] */ + SAVE_SP(); int err = PyObject_DelItem(container, sub); - Py_DECREF(container); - Py_DECREF(sub); + LOAD_SP(); + INTERPRETER_DECREF(container); + INTERPRETER_DECREF(sub); if (err) JUMP_TO_ERROR(); stack_pointer += -2; break; @@ -947,8 +971,10 @@ oparg = CURRENT_OPARG(); value = stack_pointer[-1]; assert(oparg <= MAX_INTRINSIC_1); + SAVE_SP(); res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value); - Py_DECREF(value); + LOAD_SP(); + INTERPRETER_DECREF(value); if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-1] = res; break; @@ -962,9 +988,11 @@ value1 = stack_pointer[-1]; value2 = stack_pointer[-2]; assert(oparg <= MAX_INTRINSIC_2); + SAVE_SP(); res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); - Py_DECREF(value2); - Py_DECREF(value1); + LOAD_SP(); + INTERPRETER_DECREF(value2); + INTERPRETER_DECREF(value1); if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2] = res; stack_pointer += -1; @@ -1006,15 +1034,19 @@ getter = type->tp_as_async->am_aiter; } if (getter == NULL) { + SAVE_SP(); _PyErr_Format(tstate, PyExc_TypeError, "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); - Py_DECREF(obj); + LOAD_SP(); + INTERPRETER_DECREF(obj); if (true) JUMP_TO_ERROR(); } + SAVE_SP(); iter = (*getter)(obj); - Py_DECREF(obj); + LOAD_SP(); + INTERPRETER_DECREF(obj); if (iter == NULL) JUMP_TO_ERROR(); if (Py_TYPE(iter)->tp_as_async == NULL || Py_TYPE(iter)->tp_as_async->am_anext == NULL) { @@ -1022,7 +1054,7 @@ "'async for' received an object from __aiter__ " "that does not implement __anext__: %.100s", Py_TYPE(iter)->tp_name); - Py_DECREF(iter); + INTERPRETER_DECREF(iter); if (true) JUMP_TO_ERROR(); } stack_pointer[-1] = iter; @@ -1037,7 +1069,9 @@ PyObject *next_iter = NULL; PyTypeObject *type = Py_TYPE(aiter); if (PyAsyncGen_CheckExact(aiter)) { + SAVE_SP(); awaitable = type->tp_as_async->am_anext(aiter); + LOAD_SP(); if (awaitable == NULL) { JUMP_TO_ERROR(); } @@ -1046,29 +1080,37 @@ getter = type->tp_as_async->am_anext; } if (getter != NULL) { + SAVE_SP(); next_iter = (*getter)(aiter); + LOAD_SP(); if (next_iter == NULL) { JUMP_TO_ERROR(); } } else { + SAVE_SP(); _PyErr_Format(tstate, PyExc_TypeError, "'async for' requires an iterator with " "__anext__ method, got %.100s", type->tp_name); + LOAD_SP(); JUMP_TO_ERROR(); } + SAVE_SP(); awaitable = _PyCoro_GetAwaitableIter(next_iter); + LOAD_SP(); if (awaitable == NULL) { + SAVE_SP(); _PyErr_FormatFromCause( PyExc_TypeError, "'async for' received an invalid object " "from __anext__: %.100s", Py_TYPE(next_iter)->tp_name); - Py_DECREF(next_iter); + LOAD_SP(); + INTERPRETER_DECREF(next_iter); JUMP_TO_ERROR(); } else { - Py_DECREF(next_iter); + INTERPRETER_DECREF(next_iter); } } stack_pointer[0] = awaitable; @@ -1081,18 +1123,22 @@ PyObject *iter; oparg = CURRENT_OPARG(); iterable = stack_pointer[-1]; + SAVE_SP(); iter = _PyCoro_GetAwaitableIter(iterable); + LOAD_SP(); if (iter == NULL) { + SAVE_SP(); _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg); + LOAD_SP(); } - Py_DECREF(iterable); + INTERPRETER_DECREF(iterable); if (iter != NULL && PyCoro_CheckExact(iter)) { PyObject *yf = _PyGen_yf((PyGenObject*)iter); if (yf != NULL) { /* `iter` is a coroutine object that is being awaited, `yf` is a pointer to the current awaitable being awaited on. */ - Py_DECREF(yf); + INTERPRETER_DECREF(yf); Py_CLEAR(iter); _PyErr_SetString(tstate, PyExc_RuntimeError, "coroutine is being awaited already"); @@ -1157,7 +1203,11 @@ PyObject *exc_value; exc_value = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; - Py_XSETREF(exc_info->exc_value, exc_value == Py_None ? NULL : exc_value); + PyObject *tmp = exc_info->exc_value; + exc_info->exc_value = exc_value == Py_None ? NULL : exc_value; + if (tmp != NULL) { + INTERPRETER_DECREF(tmp); + } stack_pointer += -1; break; } @@ -1183,7 +1233,10 @@ case _LOAD_BUILD_CLASS: { PyObject *bc; - if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) JUMP_TO_ERROR(); + SAVE_SP(); + int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc); + LOAD_SP(); + if (err < 0) JUMP_TO_ERROR(); if (bc == NULL) { _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); @@ -1204,14 +1257,16 @@ if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); - Py_DECREF(v); + INTERPRETER_DECREF(v); if (true) JUMP_TO_ERROR(); } + SAVE_SP(); if (PyDict_CheckExact(ns)) err = PyDict_SetItem(ns, name, v); else err = PyObject_SetItem(ns, name, v); - Py_DECREF(v); + LOAD_SP(); + INTERPRETER_DECREF(v); if (err) JUMP_TO_ERROR(); stack_pointer += -1; break; @@ -1223,16 +1278,22 @@ PyObject *ns = LOCALS(); int err; if (ns == NULL) { + SAVE_SP(); _PyErr_Format(tstate, PyExc_SystemError, "no locals when deleting %R", name); + LOAD_SP(); JUMP_TO_ERROR(); } + SAVE_SP(); err = PyObject_DelItem(ns, name); + LOAD_SP(); // Can't use ERROR_IF here. if (err != 0) { + SAVE_SP(); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); + LOAD_SP(); JUMP_TO_ERROR(); } break; @@ -1243,8 +1304,10 @@ oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; PyObject **top = stack_pointer + oparg - 1; + SAVE_SP(); int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); - Py_DECREF(seq); + LOAD_SP(); + INTERPRETER_DECREF(seq); if (res == 0) JUMP_TO_ERROR(); stack_pointer += -1 + oparg; break; @@ -1268,7 +1331,7 @@ STAT_INC(UNPACK_SEQUENCE, hit); val0 = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); val1 = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); - Py_DECREF(seq); + INTERPRETER_DECREF(seq); stack_pointer[-1] = val1; stack_pointer[0] = val0; stack_pointer += 1; @@ -1294,7 +1357,7 @@ for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } - Py_DECREF(seq); + INTERPRETER_DECREF(seq); stack_pointer += -1 + oparg; break; } @@ -1318,7 +1381,7 @@ for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } - Py_DECREF(seq); + INTERPRETER_DECREF(seq); stack_pointer += -1 + oparg; break; } @@ -1329,8 +1392,10 @@ seq = stack_pointer[-1]; int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); PyObject **top = stack_pointer + totalargs - 1; + SAVE_SP(); int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); - Py_DECREF(seq); + LOAD_SP(); + INTERPRETER_DECREF(seq); if (res == 0) JUMP_TO_ERROR(); stack_pointer += (oparg >> 8) + (oparg & 0xFF); break; @@ -1343,9 +1408,11 @@ owner = stack_pointer[-1]; v = stack_pointer[-2]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + SAVE_SP(); int err = PyObject_SetAttr(owner, name, v); - Py_DECREF(v); - Py_DECREF(owner); + LOAD_SP(); + INTERPRETER_DECREF(v); + INTERPRETER_DECREF(owner); if (err) JUMP_TO_ERROR(); stack_pointer += -2; break; @@ -1356,8 +1423,10 @@ oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + SAVE_SP(); int err = PyObject_DelAttr(owner, name); - Py_DECREF(owner); + LOAD_SP(); + INTERPRETER_DECREF(owner); if (err) JUMP_TO_ERROR(); stack_pointer += -1; break; @@ -1368,8 +1437,10 @@ oparg = CURRENT_OPARG(); v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + SAVE_SP(); int err = PyDict_SetItem(GLOBALS(), name, v); - Py_DECREF(v); + LOAD_SP(); + INTERPRETER_DECREF(v); if (err) JUMP_TO_ERROR(); stack_pointer += -1; break; @@ -1378,14 +1449,18 @@ case _DELETE_GLOBAL: { oparg = CURRENT_OPARG(); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + SAVE_SP(); int err = PyDict_Pop(GLOBALS(), name, NULL); + LOAD_SP(); // Can't use ERROR_IF here. if (err < 0) { JUMP_TO_ERROR(); } if (err == 0) { + SAVE_SP(); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); + LOAD_SP(); JUMP_TO_ERROR(); } break; @@ -1395,8 +1470,10 @@ PyObject *locals; locals = LOCALS(); if (locals == NULL) { + SAVE_SP(); _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); + LOAD_SP(); if (true) JUMP_TO_ERROR(); } Py_INCREF(locals); @@ -1411,26 +1488,37 @@ oparg = CURRENT_OPARG(); mod_or_class_dict = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { + SAVE_SP(); + int err = PyMapping_GetOptionalItem(mod_or_class_dict, name, &v); + LOAD_SP(); + if (err < 0) { JUMP_TO_ERROR(); } if (v == NULL) { - if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { + SAVE_SP(); + err = PyDict_GetItemRef(GLOBALS(), name, &v); + LOAD_SP(); + if (err < 0) { JUMP_TO_ERROR(); } if (v == NULL) { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { + SAVE_SP(); + err = PyMapping_GetOptionalItem(BUILTINS(), name, &v); + LOAD_SP(); + if (err < 0) { JUMP_TO_ERROR(); } if (v == NULL) { + SAVE_SP(); _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); + LOAD_SP(); JUMP_TO_ERROR(); } } } - Py_DECREF(mod_or_class_dict); + INTERPRETER_DECREF(mod_or_class_dict); stack_pointer[-1] = v; break; } @@ -1445,15 +1533,19 @@ if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) { + SAVE_SP(); res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), (PyDictObject *)BUILTINS(), name); + LOAD_SP(); if (res == NULL) { if (!_PyErr_Occurred(tstate)) { /* _PyDict_LoadGlobal() returns NULL without raising * an exception if the key doesn't exist */ + SAVE_SP(); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); + LOAD_SP(); } if (true) JUMP_TO_ERROR(); } @@ -1461,14 +1553,22 @@ else { /* Slow-path if globals or builtins is not a dict */ /* namespace 1: globals */ - if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) JUMP_TO_ERROR(); + SAVE_SP(); + int err = PyMapping_GetOptionalItem(GLOBALS(), name, &res); + LOAD_SP(); + if (err < 0) JUMP_TO_ERROR(); if (res == NULL) { /* namespace 2: builtins */ - if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) JUMP_TO_ERROR(); + SAVE_SP(); + err = PyMapping_GetOptionalItem(BUILTINS(), name, &res); + LOAD_SP(); + if (err < 0) JUMP_TO_ERROR(); if (res == NULL) { + SAVE_SP(); _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); + LOAD_SP(); if (true) JUMP_TO_ERROR(); } } @@ -1556,10 +1656,12 @@ oparg = CURRENT_OPARG(); PyObject *v = GETLOCAL(oparg); if (v == NULL) { + SAVE_SP(); _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); + LOAD_SP(); if (1) JUMP_TO_ERROR(); } SETLOCAL(oparg, NULL); @@ -1586,10 +1688,12 @@ // Fortunately we don't need its superpower. PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); if (oldobj == NULL) { + SAVE_SP(); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + LOAD_SP(); JUMP_TO_ERROR(); } - Py_DECREF(oldobj); + INTERPRETER_DECREF(oldobj); break; } @@ -1602,18 +1706,23 @@ assert(class_dict); assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); - if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) { + SAVE_SP(); + int err = PyMapping_GetOptionalItem(class_dict, name, &value); + LOAD_SP(); + if (err < 0) { JUMP_TO_ERROR(); } if (!value) { PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); value = PyCell_GetRef(cell); if (value == NULL) { + SAVE_SP(); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + LOAD_SP(); JUMP_TO_ERROR(); } } - Py_DECREF(class_dict); + INTERPRETER_DECREF(class_dict); stack_pointer[-1] = value; break; } @@ -1624,7 +1733,9 @@ PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); value = PyCell_GetRef(cell); if (value == NULL) { + SAVE_SP(); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + LOAD_SP(); if (true) JUMP_TO_ERROR(); } stack_pointer[0] = value; @@ -1637,7 +1748,9 @@ oparg = CURRENT_OPARG(); v = stack_pointer[-1]; PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); + SAVE_SP(); PyCell_SetTakeRef(cell, v); + LOAD_SP(); stack_pointer += -1; break; } @@ -1664,7 +1777,7 @@ pieces = &stack_pointer[-oparg]; str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); for (int _i = oparg; --_i >= 0;) { - Py_DECREF(pieces[_i]); + INTERPRETER_DECREF(pieces[_i]); } if (str == NULL) JUMP_TO_ERROR(); stack_pointer[-oparg] = str; @@ -1702,8 +1815,11 @@ oparg = CURRENT_OPARG(); iterable = stack_pointer[-1]; list = stack_pointer[-2 - (oparg-1)]; + SAVE_SP(); PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); + LOAD_SP(); if (none_val == NULL) { + SAVE_SP(); if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) { @@ -1712,11 +1828,12 @@ "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); } - Py_DECREF(iterable); + INTERPRETER_DECREF(iterable); + LOAD_SP(); if (true) JUMP_TO_ERROR(); } assert(Py_IsNone(none_val)); - Py_DECREF(iterable); + INTERPRETER_DECREF(iterable); stack_pointer += -1; break; } @@ -1727,8 +1844,10 @@ oparg = CURRENT_OPARG(); iterable = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; + SAVE_SP(); int err = _PySet_Update(set, iterable); - Py_DECREF(iterable); + LOAD_SP(); + INTERPRETER_DECREF(iterable); if (err < 0) JUMP_TO_ERROR(); stack_pointer += -1; break; @@ -1741,12 +1860,14 @@ PyObject *map; oparg = CURRENT_OPARG(); values = &stack_pointer[-oparg*2]; + SAVE_SP(); map = _PyDict_FromItems( values, 2, values+1, 2, oparg); + LOAD_SP(); for (int _i = oparg*2; --_i >= 0;) { - Py_DECREF(values[_i]); + INTERPRETER_DECREF(values[_i]); } if (map == NULL) JUMP_TO_ERROR(); stack_pointer[-oparg*2] = map; @@ -1758,22 +1879,27 @@ int err; PyObject *ann_dict; if (LOCALS() == NULL) { + SAVE_SP(); _PyErr_Format(tstate, PyExc_SystemError, "no locals found when setting up annotations"); + LOAD_SP(); if (true) JUMP_TO_ERROR(); } /* check if __annotations__ in locals()... */ - if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) JUMP_TO_ERROR(); + SAVE_SP(); + err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); + LOAD_SP(); + if (err < 0) JUMP_TO_ERROR(); if (ann_dict == NULL) { ann_dict = PyDict_New(); if (ann_dict == NULL) JUMP_TO_ERROR(); err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), ann_dict); - Py_DECREF(ann_dict); + INTERPRETER_DECREF(ann_dict); if (err) JUMP_TO_ERROR(); } else { - Py_DECREF(ann_dict); + INTERPRETER_DECREF(ann_dict); } break; } @@ -1787,13 +1913,15 @@ values = &stack_pointer[-1 - oparg]; assert(PyTuple_CheckExact(keys)); assert(PyTuple_GET_SIZE(keys) == (Py_ssize_t)oparg); + SAVE_SP(); map = _PyDict_FromItems( &PyTuple_GET_ITEM(keys, 0), 1, values, 1, oparg); + LOAD_SP(); for (int _i = oparg; --_i >= 0;) { - Py_DECREF(values[_i]); + INTERPRETER_DECREF(values[_i]); } - Py_DECREF(keys); + INTERPRETER_DECREF(keys); if (map == NULL) JUMP_TO_ERROR(); stack_pointer[-1 - oparg] = map; stack_pointer += -oparg; @@ -1806,16 +1934,21 @@ oparg = CURRENT_OPARG(); update = stack_pointer[-1]; dict = stack_pointer[-2 - (oparg - 1)]; - if (PyDict_Update(dict, update) < 0) { + SAVE_SP(); + int err = PyDict_Update(dict, update); + LOAD_SP(); + if (err < 0) { + SAVE_SP(); if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { _PyErr_Format(tstate, PyExc_TypeError, "'%.200s' object is not a mapping", Py_TYPE(update)->tp_name); } - Py_DECREF(update); + LOAD_SP(); + INTERPRETER_DECREF(update); if (true) JUMP_TO_ERROR(); } - Py_DECREF(update); + INTERPRETER_DECREF(update); stack_pointer += -1; break; } @@ -1828,12 +1961,17 @@ update = stack_pointer[-1]; dict = stack_pointer[-2 - (oparg - 1)]; callable = stack_pointer[-5 - (oparg - 1)]; - if (_PyDict_MergeEx(dict, update, 2) < 0) { + SAVE_SP(); + int err = _PyDict_MergeEx(dict, update, 2); + LOAD_SP(); + if (err < 0) { + SAVE_SP(); _PyEval_FormatKwargsError(tstate, callable, update); - Py_DECREF(update); + LOAD_SP(); + INTERPRETER_DECREF(update); if (true) JUMP_TO_ERROR(); } - Py_DECREF(update); + INTERPRETER_DECREF(update); stack_pointer += -1; break; } @@ -1849,7 +1987,10 @@ assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references - if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) JUMP_TO_ERROR(); + SAVE_SP(); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, key, value); + LOAD_SP(); + if (err != 0) JUMP_TO_ERROR(); stack_pointer += -2; break; } @@ -1876,10 +2017,12 @@ } STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + SAVE_SP(); attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); - Py_DECREF(global_super); - Py_DECREF(class); - Py_DECREF(self); + LOAD_SP(); + INTERPRETER_DECREF(global_super); + INTERPRETER_DECREF(class); + INTERPRETER_DECREF(self); if (attr == NULL) JUMP_TO_ERROR(); stack_pointer[-3] = attr; stack_pointer += -2; @@ -1909,18 +2052,20 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; int method_found = 0; + SAVE_SP(); attr = _PySuper_Lookup(cls, self, name, Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); - Py_DECREF(global_super); - Py_DECREF(class); + LOAD_SP(); + INTERPRETER_DECREF(global_super); + INTERPRETER_DECREF(class); if (attr == NULL) { - Py_DECREF(self); + INTERPRETER_DECREF(self); if (true) JUMP_TO_ERROR(); } if (method_found) { self_or_null = self; // transfer ownership } else { - Py_DECREF(self); + INTERPRETER_DECREF(self); self_or_null = NULL; } stack_pointer[-3] = attr; @@ -1939,7 +2084,10 @@ if (oparg & 1) { /* Designed to work in tandem with CALL, pushes two values. */ attr = NULL; - if (_PyObject_GetMethod(owner, name, &attr)) { + SAVE_SP(); + int err = _PyObject_GetMethod(owner, name, &attr); + LOAD_SP(); + if (err) { /* We can bypass temporary bound method object. meth is unbound method and obj is self. meth | self | arg1 | ... | argN @@ -1954,15 +2102,17 @@ CALL that it's not a method call. meth | NULL | arg1 | ... | argN */ - Py_DECREF(owner); + INTERPRETER_DECREF(owner); if (attr == NULL) JUMP_TO_ERROR(); self_or_null = NULL; } } else { /* Classic, pushes one value. */ + SAVE_SP(); attr = PyObject_GetAttr(owner, name); - Py_DECREF(owner); + LOAD_SP(); + INTERPRETER_DECREF(owner); if (attr == NULL) JUMP_TO_ERROR(); } stack_pointer[-1] = attr; @@ -2011,7 +2161,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; - Py_DECREF(owner); + INTERPRETER_DECREF(owner); stack_pointer[-1] = attr; break; } @@ -2031,7 +2181,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; - Py_DECREF(owner); + INTERPRETER_DECREF(owner); stack_pointer[-1] = attr; stack_pointer[0] = null; stack_pointer += 1; @@ -2076,7 +2226,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; - Py_DECREF(owner); + INTERPRETER_DECREF(owner); stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); @@ -2132,7 +2282,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; - Py_DECREF(owner); + INTERPRETER_DECREF(owner); stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); @@ -2155,7 +2305,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; - Py_DECREF(owner); + INTERPRETER_DECREF(owner); stack_pointer[-1] = attr; break; } @@ -2176,7 +2326,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; - Py_DECREF(owner); + INTERPRETER_DECREF(owner); stack_pointer[-1] = attr; stack_pointer[0] = null; stack_pointer += 1; @@ -2212,7 +2362,7 @@ assert(descr != NULL); attr = Py_NewRef(descr); null = NULL; - Py_DECREF(owner); + INTERPRETER_DECREF(owner); stack_pointer[-1] = attr; break; } @@ -2228,7 +2378,7 @@ assert(descr != NULL); attr = Py_NewRef(descr); null = NULL; - Py_DECREF(owner); + INTERPRETER_DECREF(owner); stack_pointer[-1] = attr; stack_pointer[0] = null; stack_pointer += 1; @@ -2272,9 +2422,9 @@ _PyDictValues_AddToInsertionOrder(values, index); } else { - Py_DECREF(old_value); + INTERPRETER_DECREF(old_value); } - Py_DECREF(owner); + INTERPRETER_DECREF(owner); stack_pointer += -2; break; } @@ -2328,7 +2478,7 @@ new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); ep->me_value = value; } - Py_DECREF(old_value); + INTERPRETER_DECREF(old_value); STAT_INC(STORE_ATTR, hit); /* Ensure dict is GC tracked if it needs to be */ if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(value)) { @@ -2336,7 +2486,7 @@ } /* PEP 509 */ dict->ma_version_tag = new_version; - Py_DECREF(owner); + INTERPRETER_DECREF(owner); stack_pointer += -2; break; } @@ -2351,8 +2501,10 @@ STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; *(PyObject **)addr = value; - Py_XDECREF(old_value); - Py_DECREF(owner); + if (old_value != NULL) { + INTERPRETER_DECREF(old_value); + } + INTERPRETER_DECREF(owner); stack_pointer += -2; break; } @@ -2365,13 +2517,15 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; assert((oparg >> 5) <= Py_GE); + SAVE_SP(); res = PyObject_RichCompare(left, right, oparg >> 5); - Py_DECREF(left); - Py_DECREF(right); + LOAD_SP(); + INTERPRETER_DECREF(left); + INTERPRETER_DECREF(right); if (res == NULL) JUMP_TO_ERROR(); if (oparg & 16) { int res_bool = PyObject_IsTrue(res); - Py_DECREF(res); + INTERPRETER_DECREF(res); if (res_bool < 0) JUMP_TO_ERROR(); res = res_bool ? Py_True : Py_False; } @@ -2462,8 +2616,8 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; int res = Py_Is(left, right) ^ oparg; - Py_DECREF(left); - Py_DECREF(right); + INTERPRETER_DECREF(left); + INTERPRETER_DECREF(right); b = res ? Py_True : Py_False; stack_pointer[-2] = b; stack_pointer += -1; @@ -2477,9 +2631,11 @@ oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; + SAVE_SP(); int res = PySequence_Contains(right, left); - Py_DECREF(left); - Py_DECREF(right); + LOAD_SP(); + INTERPRETER_DECREF(left); + INTERPRETER_DECREF(right); if (res < 0) JUMP_TO_ERROR(); b = (res ^ oparg) ? Py_True : Py_False; stack_pointer[-2] = b; @@ -2500,9 +2656,11 @@ } STAT_INC(CONTAINS_OP, hit); // Note: both set and frozenset use the same seq_contains method! + SAVE_SP(); int res = _PySet_Contains((PySetObject *)right, left); - Py_DECREF(left); - Py_DECREF(right); + LOAD_SP(); + INTERPRETER_DECREF(left); + INTERPRETER_DECREF(right); if (res < 0) JUMP_TO_ERROR(); b = (res ^ oparg) ? Py_True : Py_False; stack_pointer[-2] = b; @@ -2522,9 +2680,11 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(CONTAINS_OP, hit); + SAVE_SP(); int res = PyDict_Contains(right, left); - Py_DECREF(left); - Py_DECREF(right); + LOAD_SP(); + INTERPRETER_DECREF(left); + INTERPRETER_DECREF(right); if (res < 0) JUMP_TO_ERROR(); b = (res ^ oparg) ? Py_True : Py_False; stack_pointer[-2] = b; @@ -2540,16 +2700,18 @@ match_type = stack_pointer[-1]; exc_value = stack_pointer[-2]; if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { - Py_DECREF(exc_value); - Py_DECREF(match_type); + INTERPRETER_DECREF(exc_value); + INTERPRETER_DECREF(match_type); if (true) JUMP_TO_ERROR(); } match = NULL; rest = NULL; + SAVE_SP(); int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, &match, &rest); - Py_DECREF(exc_value); - Py_DECREF(match_type); + LOAD_SP(); + INTERPRETER_DECREF(exc_value); + INTERPRETER_DECREF(match_type); if (res < 0) JUMP_TO_ERROR(); assert((match == NULL) == (rest == NULL)); if (match == NULL) JUMP_TO_ERROR(); @@ -2568,12 +2730,15 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; assert(PyExceptionInstance_Check(left)); + SAVE_SP(); if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { - Py_DECREF(right); + LOAD_SP(); + INTERPRETER_DECREF(right); if (true) JUMP_TO_ERROR(); } int res = PyErr_GivenExceptionMatches(left, right); - Py_DECREF(right); + LOAD_SP(); + INTERPRETER_DECREF(right); b = res ? Py_True : Py_False; stack_pointer[-1] = b; break; @@ -2592,7 +2757,7 @@ } else { b = Py_False; - Py_DECREF(value); + INTERPRETER_DECREF(value); } stack_pointer[-1] = b; break; @@ -2624,10 +2789,12 @@ // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(names)); + SAVE_SP(); attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names); - Py_DECREF(subject); - Py_DECREF(type); - Py_DECREF(names); + LOAD_SP(); + INTERPRETER_DECREF(subject); + INTERPRETER_DECREF(type); + INTERPRETER_DECREF(names); if (attrs) { assert(PyTuple_CheckExact(attrs)); // Success! } @@ -2670,7 +2837,9 @@ keys = stack_pointer[-1]; subject = stack_pointer[-2]; // On successful match, PUSH(values). Otherwise, PUSH(None). + SAVE_SP(); values_or_none = _PyEval_MatchKeys(tstate, subject, keys); + LOAD_SP(); if (values_or_none == NULL) JUMP_TO_ERROR(); stack_pointer[0] = values_or_none; stack_pointer += 1; @@ -2682,8 +2851,10 @@ PyObject *iter; iterable = stack_pointer[-1]; /* before: [obj]; after [getiter(obj)] */ + SAVE_SP(); iter = PyObject_GetIter(iterable); - Py_DECREF(iterable); + LOAD_SP(); + INTERPRETER_DECREF(iterable); if (iter == NULL) JUMP_TO_ERROR(); stack_pointer[-1] = iter; break; @@ -2711,11 +2882,13 @@ } else { /* `iterable` is not a generator. */ + SAVE_SP(); iter = PyObject_GetIter(iterable); + LOAD_SP(); if (iter == NULL) { JUMP_TO_ERROR(); } - Py_DECREF(iterable); + INTERPRETER_DECREF(iterable); } stack_pointer[-1] = iter; break; @@ -2728,13 +2901,17 @@ PyObject *next; iter = stack_pointer[-1]; /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ + SAVE_SP(); next = (*Py_TYPE(iter)->tp_iternext)(iter); + LOAD_SP(); if (next == NULL) { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { JUMP_TO_ERROR(); } + SAVE_SP(); _PyErr_Clear(tstate); + LOAD_SP(); } /* iterator ended normally */ /* The translator sets the deopt target just past the matching END_FOR */ @@ -2936,13 +3113,15 @@ tb = Py_None; } else { - Py_DECREF(tb); + INTERPRETER_DECREF(tb); } assert(PyLong_Check(lasti)); (void)lasti; // Shut up compiler warning if asserts are off PyObject *stack[4] = {NULL, exc, val, tb}; + SAVE_SP(); res = PyObject_Vectorcall(exit_func, stack + 1, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + LOAD_SP(); if (res == NULL) JUMP_TO_ERROR(); stack_pointer[0] = res; stack_pointer += 1; @@ -3041,7 +3220,7 @@ assert((oparg & 1) == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - Py_DECREF(owner); + INTERPRETER_DECREF(owner); attr = Py_NewRef(descr); stack_pointer[-1] = attr; break; @@ -3057,7 +3236,7 @@ assert(Py_TYPE(owner)->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - Py_DECREF(owner); + INTERPRETER_DECREF(owner); attr = Py_NewRef(descr); stack_pointer[-1] = attr; break; @@ -3123,10 +3302,12 @@ assert(Py_TYPE(callable) == &PyFunction_Type); int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + SAVE_SP(); new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)callable, locals, args, total_args, NULL ); + LOAD_SP(); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. stack_pointer += -2 - oparg; @@ -3198,7 +3379,7 @@ method = ((PyMethodObject *)callable)->im_func; assert(PyFunction_Check(method)); Py_INCREF(method); - Py_DECREF(callable); + INTERPRETER_DECREF(callable); stack_pointer[-2 - oparg] = method; stack_pointer[-1 - oparg] = self; break; @@ -3237,14 +3418,16 @@ total_args++; } /* Callable is not a normal Python function */ + SAVE_SP(); res = PyObject_Vectorcall( callable, args, total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + LOAD_SP(); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); + INTERPRETER_DECREF(callable); for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + INTERPRETER_DECREF(args[i]); } if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2 - oparg] = res; @@ -3280,7 +3463,7 @@ stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS func = Py_NewRef(((PyMethodObject *)callable)->im_func); stack_pointer[-2 - oparg] = func; // This is used by CALL, upon deoptimization - Py_DECREF(callable); + INTERPRETER_DECREF(callable); stack_pointer[-2 - oparg] = func; stack_pointer[-1 - oparg] = self; break; @@ -3477,7 +3660,7 @@ // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); stack_pointer += -1; - _PyFrame_SetStackPointer(frame, stack_pointer); + SAVE_SP(); new_frame->previous = frame; CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; @@ -3507,8 +3690,10 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); + SAVE_SP(); res = Py_NewRef(Py_TYPE(arg)); - Py_DECREF(arg); + LOAD_SP(); + INTERPRETER_DECREF(arg); stack_pointer[-3] = res; stack_pointer += -2; break; @@ -3533,8 +3718,10 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); + SAVE_SP(); res = PyObject_Str(arg); - Py_DECREF(arg); + LOAD_SP(); + INTERPRETER_DECREF(arg); if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-3] = res; stack_pointer += -2; @@ -3560,8 +3747,10 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); + SAVE_SP(); res = PySequence_Tuple(arg); - Py_DECREF(arg); + LOAD_SP(); + INTERPRETER_DECREF(arg); if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-3] = res; stack_pointer += -2; @@ -3608,12 +3797,14 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); + SAVE_SP(); res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); + LOAD_SP(); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + INTERPRETER_DECREF(args[i]); } - Py_DECREF(tp); + INTERPRETER_DECREF(tp); if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -3656,11 +3847,13 @@ PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); PyObject *arg = args[0]; _Py_EnterRecursiveCallTstateUnchecked(tstate); + SAVE_SP(); res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); + LOAD_SP(); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(arg); - Py_DECREF(callable); + INTERPRETER_DECREF(arg); + INTERPRETER_DECREF(callable); if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -3693,16 +3886,18 @@ STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); /* res = func(self, args, nargs) */ + SAVE_SP(); res = ((PyCFunctionFast)(void(*)(void))cfunc)( PyCFunction_GET_SELF(callable), args, total_args); + LOAD_SP(); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + INTERPRETER_DECREF(args[i]); } - Py_DECREF(callable); + INTERPRETER_DECREF(callable); if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -3737,13 +3932,15 @@ PyCFunctionFastWithKeywords cfunc = (PyCFunctionFastWithKeywords)(void(*)(void)) PyCFunction_GET_FUNCTION(callable); + SAVE_SP(); res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); + LOAD_SP(); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + INTERPRETER_DECREF(args[i]); } - Py_DECREF(callable); + INTERPRETER_DECREF(callable); if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -3776,7 +3973,9 @@ } STAT_INC(CALL, hit); PyObject *arg = args[0]; + SAVE_SP(); Py_ssize_t len_i = PyObject_Length(arg); + LOAD_SP(); if (len_i < 0) { JUMP_TO_ERROR(); } @@ -3785,8 +3984,8 @@ if (res == NULL) { GOTO_ERROR(error); } - Py_DECREF(callable); - Py_DECREF(arg); + INTERPRETER_DECREF(callable); + INTERPRETER_DECREF(arg); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; break; @@ -3819,7 +4018,9 @@ STAT_INC(CALL, hit); PyObject *cls = args[1]; PyObject *inst = args[0]; + SAVE_SP(); int retval = PyObject_IsInstance(inst, cls); + LOAD_SP(); if (retval < 0) { JUMP_TO_ERROR(); } @@ -3828,9 +4029,9 @@ if (res == NULL) { GOTO_ERROR(error); } - Py_DECREF(inst); - Py_DECREF(cls); - Py_DECREF(callable); + INTERPRETER_DECREF(inst); + INTERPRETER_DECREF(cls); + INTERPRETER_DECREF(callable); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; break; @@ -3878,12 +4079,14 @@ STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); + SAVE_SP(); res = _PyCFunction_TrampolineCall(cfunc, self, arg); + LOAD_SP(); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(arg); - Py_DECREF(callable); + INTERPRETER_DECREF(self); + INTERPRETER_DECREF(arg); + INTERPRETER_DECREF(callable); if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -3924,13 +4127,15 @@ int nargs = total_args - 1; PyCFunctionFastWithKeywords cfunc = (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; + SAVE_SP(); res = cfunc(self, args + 1, nargs, NULL); + LOAD_SP(); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + INTERPRETER_DECREF(args[i]); } - Py_DECREF(callable); + INTERPRETER_DECREF(callable); if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -3979,11 +4184,13 @@ STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); + SAVE_SP(); res = _PyCFunction_TrampolineCall(cfunc, self, NULL); + LOAD_SP(); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(callable); + INTERPRETER_DECREF(self); + INTERPRETER_DECREF(callable); if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -4024,13 +4231,15 @@ PyCFunctionFast cfunc = (PyCFunctionFast)(void(*)(void))meth->ml_meth; int nargs = total_args - 1; + SAVE_SP(); res = cfunc(self, args + 1, nargs); + LOAD_SP(); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Clear the stack of the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + INTERPRETER_DECREF(args[i]); } - Py_DECREF(callable); + INTERPRETER_DECREF(callable); if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -4049,9 +4258,11 @@ PyObject *codeobj; PyObject *func; codeobj = stack_pointer[-1]; + SAVE_SP(); PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); - Py_DECREF(codeobj); + LOAD_SP(); + INTERPRETER_DECREF(codeobj); if (func_obj == NULL) { JUMP_TO_ERROR(); } @@ -4136,9 +4347,9 @@ stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)]; start = stack_pointer[-2 - ((oparg == 3) ? 1 : 0)]; slice = PySlice_New(start, stop, step); - Py_DECREF(start); - Py_DECREF(stop); - Py_XDECREF(step); + INTERPRETER_DECREF(start); + INTERPRETER_DECREF(stop); + if (step != NULL) { INTERPRETER_DECREF(step); } if (slice == NULL) JUMP_TO_ERROR(); stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; stack_pointer += -1 - ((oparg == 3) ? 1 : 0); @@ -4153,8 +4364,10 @@ conversion_func conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); conv_fn = _PyEval_ConversionFuncs[oparg]; + SAVE_SP(); result = conv_fn(value); - Py_DECREF(value); + LOAD_SP(); + INTERPRETER_DECREF(value); if (result == NULL) JUMP_TO_ERROR(); stack_pointer[-1] = result; break; @@ -4167,8 +4380,10 @@ /* If value is a unicode object, then we know the result * of format(value) is value itself. */ if (!PyUnicode_CheckExact(value)) { + SAVE_SP(); res = PyObject_Format(value, NULL); - Py_DECREF(value); + LOAD_SP(); + INTERPRETER_DECREF(value); if (res == NULL) JUMP_TO_ERROR(); } else { @@ -4184,9 +4399,11 @@ PyObject *res; fmt_spec = stack_pointer[-1]; value = stack_pointer[-2]; + SAVE_SP(); res = PyObject_Format(value, fmt_spec); - Py_DECREF(value); - Py_DECREF(fmt_spec); + LOAD_SP(); + INTERPRETER_DECREF(value); + INTERPRETER_DECREF(fmt_spec); if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2] = res; stack_pointer += -1; @@ -4213,9 +4430,11 @@ rhs = stack_pointer[-1]; lhs = stack_pointer[-2]; assert(_PyEval_BinaryOps[oparg]); + SAVE_SP(); res = _PyEval_BinaryOps[oparg](lhs, rhs); - Py_DECREF(lhs); - Py_DECREF(rhs); + LOAD_SP(); + INTERPRETER_DECREF(lhs); + INTERPRETER_DECREF(rhs); if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2] = res; stack_pointer += -1; @@ -4277,7 +4496,7 @@ val = stack_pointer[-1]; stack_pointer += -1; if (!Py_IsNone(val)) { - Py_DECREF(val); + INTERPRETER_DECREF(val); if (1) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -4294,7 +4513,7 @@ UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - Py_DECREF(val); + INTERPRETER_DECREF(val); break; } @@ -4372,7 +4591,7 @@ PyObject *value; pop = stack_pointer[-1]; PyObject *ptr = (PyObject *)CURRENT_OPERAND(); - Py_DECREF(pop); + INTERPRETER_DECREF(pop); value = ptr; stack_pointer[-1] = value; break; @@ -4442,7 +4661,7 @@ if (optimized <= 0) { exit->temperature = restart_backoff_counter(temperature); if (optimized < 0) { - Py_DECREF(previous); + INTERPRETER_DECREF(previous); tstate->previous_executor = Py_None; GOTO_UNWIND(); } @@ -4477,7 +4696,7 @@ if (optimized <= 0) { exit->temperature = restart_backoff_counter(exit->temperature); if (optimized < 0) { - Py_DECREF(current_executor); + INTERPRETER_DECREF(current_executor); tstate->previous_executor = Py_None; GOTO_UNWIND(); } @@ -4493,7 +4712,7 @@ case _START_EXECUTOR: { PyObject *executor = (PyObject *)CURRENT_OPERAND(); - Py_DECREF(tstate->previous_executor); + INTERPRETER_DECREF(tstate->previous_executor); tstate->previous_executor = NULL; #ifndef _Py_JIT current_executor = (_PyExecutorObject*)executor; diff --git a/Python/frame.c b/Python/frame.c index 2bb12823572028..32b4bf8ecc9c04 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -17,10 +17,11 @@ _PyFrame_Traverse(_PyInterpreterFrame *frame, visitproc visit, void *arg) Py_VISIT(_PyFrame_GetCode(frame)); /* locals */ PyObject **locals = _PyFrame_GetLocalsArray(frame); - int i = 0; + PyObject **sp = frame->stackpointer; /* locals and stack */ - for (; i stacktop; i++) { - Py_VISIT(locals[i]); + while (sp > locals) { + sp--; + Py_VISIT(*sp); } return 0; } @@ -59,10 +60,11 @@ take_ownership(PyFrameObject *f, _PyInterpreterFrame *frame) assert(frame->owner != FRAME_OWNED_BY_CSTACK); assert(frame->owner != FRAME_OWNED_BY_FRAME_OBJECT); assert(frame->owner != FRAME_CLEARED); - Py_ssize_t size = ((char*)&frame->localsplus[frame->stacktop]) - (char *)frame; + Py_ssize_t size = ((char*)frame->stackpointer) - (char *)frame; Py_INCREF(_PyFrame_GetCode(frame)); memcpy((_PyInterpreterFrame *)f->_f_frame_data, frame, size); frame = (_PyInterpreterFrame *)f->_f_frame_data; + frame->stackpointer = (PyObject **)(((char *)frame) + size); f->f_frame = frame; frame->owner = FRAME_OWNED_BY_FRAME_OBJECT; if (_PyFrame_IsIncomplete(frame)) { @@ -97,11 +99,13 @@ take_ownership(PyFrameObject *f, _PyInterpreterFrame *frame) void _PyFrame_ClearLocals(_PyInterpreterFrame *frame) { - assert(frame->stacktop >= 0); - int stacktop = frame->stacktop; - frame->stacktop = 0; - for (int i = 0; i < stacktop; i++) { - Py_XDECREF(frame->localsplus[i]); + assert(frame->stackpointer != NULL); + PyObject **sp = frame->stackpointer; + PyObject **locals = frame->localsplus; + frame->stackpointer = locals; + while (sp > locals) { + sp--; + Py_XDECREF(*sp); } Py_CLEAR(frame->f_locals); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 4402787d96f12e..ab3d21a5989f5f 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -27,7 +27,9 @@ } goto error; } + SAVE_SP(); exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__)); + LOAD_SP(); if (exit == NULL) { if (!_PyErr_Occurred(tstate)) { _PyErr_Format(tstate, PyExc_TypeError, @@ -36,14 +38,16 @@ "(missed __aexit__ method)", Py_TYPE(mgr)->tp_name); } - Py_DECREF(enter); + INTERPRETER_DECREF(enter); goto error; } - Py_DECREF(mgr); + INTERPRETER_DECREF(mgr); + SAVE_SP(); res = PyObject_CallNoArgs(enter); - Py_DECREF(enter); + LOAD_SP(); + INTERPRETER_DECREF(enter); if (res == NULL) { - Py_DECREF(exit); + INTERPRETER_DECREF(exit); if (true) goto pop_1_error; } stack_pointer[-1] = exit; @@ -63,7 +67,9 @@ /* pop the context manager, push its __exit__ and the * value returned from calling its __enter__ */ + SAVE_SP(); PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__enter__)); + LOAD_SP(); if (enter == NULL) { if (!_PyErr_Occurred(tstate)) { _PyErr_Format(tstate, PyExc_TypeError, @@ -73,7 +79,9 @@ } goto error; } + SAVE_SP(); exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__exit__)); + LOAD_SP(); if (exit == NULL) { if (!_PyErr_Occurred(tstate)) { _PyErr_Format(tstate, PyExc_TypeError, @@ -82,14 +90,16 @@ "(missed __exit__ method)", Py_TYPE(mgr)->tp_name); } - Py_DECREF(enter); + INTERPRETER_DECREF(enter); goto error; } - Py_DECREF(mgr); + INTERPRETER_DECREF(mgr); + SAVE_SP(); res = PyObject_CallNoArgs(enter); - Py_DECREF(enter); + LOAD_SP(); + INTERPRETER_DECREF(enter); if (res == NULL) { - Py_DECREF(exit); + INTERPRETER_DECREF(exit); if (true) goto pop_1_error; } stack_pointer[-1] = exit; @@ -117,7 +127,9 @@ #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_OP, deferred); @@ -129,9 +141,11 @@ // _BINARY_OP { assert(_PyEval_BinaryOps[oparg]); + SAVE_SP(); res = _PyEval_BinaryOps[oparg](lhs, rhs); - Py_DECREF(lhs); - Py_DECREF(rhs); + LOAD_SP(); + INTERPRETER_DECREF(lhs); + INTERPRETER_DECREF(rhs); if (res == NULL) goto pop_2_error; } stack_pointer[-2] = res; @@ -260,7 +274,9 @@ */ assert(Py_REFCNT(left) >= 2); _Py_DECREF_NO_DEALLOC(left); + SAVE_SP(); PyUnicode_Append(target_local, right); + LOAD_SP(); _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); if (*target_local == NULL) goto pop_2_error; // The STORE_FAST is already done. @@ -398,17 +414,21 @@ stop = stack_pointer[-1]; start = stack_pointer[-2]; container = stack_pointer[-3]; + SAVE_SP(); PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); + LOAD_SP(); // Can't use ERROR_IF() here, because we haven't // DECREF'ed container yet, and we still own slice. if (slice == NULL) { res = NULL; } else { + SAVE_SP(); res = PyObject_GetItem(container, slice); - Py_DECREF(slice); + LOAD_SP(); + INTERPRETER_DECREF(slice); } - Py_DECREF(container); + INTERPRETER_DECREF(container); if (res == NULL) goto pop_3_error; stack_pointer[-3] = res; stack_pointer += -2; @@ -434,7 +454,9 @@ #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_BinarySubscr(container, sub, next_instr); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_SUBSCR, deferred); @@ -443,9 +465,11 @@ } // _BINARY_SUBSCR { + SAVE_SP(); res = PyObject_GetItem(container, sub); - Py_DECREF(container); - Py_DECREF(sub); + LOAD_SP(); + INTERPRETER_DECREF(container); + INTERPRETER_DECREF(sub); if (res == NULL) goto pop_2_error; } stack_pointer[-2] = res; @@ -466,12 +490,14 @@ dict = stack_pointer[-2]; DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); + SAVE_SP(); int rc = PyDict_GetItemRef(dict, sub, &res); + LOAD_SP(); if (rc == 0) { _PyErr_SetKeyError(sub); } - Py_DECREF(dict); - Py_DECREF(sub); + INTERPRETER_DECREF(dict); + INTERPRETER_DECREF(sub); if (rc <= 0) goto pop_2_error; // not found or error stack_pointer[-2] = res; @@ -534,7 +560,7 @@ assert(res != NULL); Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(list); + INTERPRETER_DECREF(list); stack_pointer[-2] = res; stack_pointer += -1; DISPATCH(); @@ -562,7 +588,7 @@ STAT_INC(BINARY_SUBSCR, hit); res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(str); + INTERPRETER_DECREF(str); stack_pointer[-2] = res; stack_pointer += -1; DISPATCH(); @@ -590,7 +616,7 @@ assert(res != NULL); Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(tuple); + INTERPRETER_DECREF(tuple); stack_pointer[-2] = res; stack_pointer += -1; DISPATCH(); @@ -607,13 +633,15 @@ values = &stack_pointer[-1 - oparg]; assert(PyTuple_CheckExact(keys)); assert(PyTuple_GET_SIZE(keys) == (Py_ssize_t)oparg); + SAVE_SP(); map = _PyDict_FromItems( &PyTuple_GET_ITEM(keys, 0), 1, values, 1, oparg); + LOAD_SP(); for (int _i = oparg; --_i >= 0;) { - Py_DECREF(values[_i]); + INTERPRETER_DECREF(values[_i]); } - Py_DECREF(keys); + INTERPRETER_DECREF(keys); if (map == NULL) { stack_pointer += -1 - oparg; goto error; } stack_pointer[-1 - oparg] = map; stack_pointer += -oparg; @@ -641,12 +669,14 @@ PyObject **values; PyObject *map; values = &stack_pointer[-oparg*2]; + SAVE_SP(); map = _PyDict_FromItems( values, 2, values+1, 2, oparg); + LOAD_SP(); for (int _i = oparg*2; --_i >= 0;) { - Py_DECREF(values[_i]); + INTERPRETER_DECREF(values[_i]); } if (map == NULL) { stack_pointer += -oparg*2; goto error; } stack_pointer[-oparg*2] = map; @@ -667,12 +697,15 @@ int err = 0; for (int i = 0; i < oparg; i++) { PyObject *item = values[i]; - if (err == 0) - err = PySet_Add(set, item); - Py_DECREF(item); + if (err == 0) { + SAVE_SP(); + err = PySet_Add(set, item); + LOAD_SP(); + } + INTERPRETER_DECREF(item); } if (err != 0) { - Py_DECREF(set); + INTERPRETER_DECREF(set); if (true) { stack_pointer += -oparg; goto error; } } stack_pointer[-oparg] = set; @@ -692,9 +725,9 @@ stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)]; start = stack_pointer[-2 - ((oparg == 3) ? 1 : 0)]; slice = PySlice_New(start, stop, step); - Py_DECREF(start); - Py_DECREF(stop); - Py_XDECREF(step); + INTERPRETER_DECREF(start); + INTERPRETER_DECREF(stop); + if (step != NULL) { INTERPRETER_DECREF(step); } if (slice == NULL) { stack_pointer += -2 - ((oparg == 3) ? 1 : 0); goto error; } stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; stack_pointer += -1 - ((oparg == 3) ? 1 : 0); @@ -710,7 +743,7 @@ pieces = &stack_pointer[-oparg]; str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); for (int _i = oparg; --_i >= 0;) { - Py_DECREF(pieces[_i]); + INTERPRETER_DECREF(pieces[_i]); } if (str == NULL) { stack_pointer += -oparg; goto error; } stack_pointer[-oparg] = str; @@ -762,7 +795,9 @@ #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_Call(callable, next_instr, oparg + (self_or_null != NULL)); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(CALL, deferred); @@ -785,7 +820,7 @@ args[0] = Py_NewRef(self); PyObject *method = ((PyMethodObject *)callable)->im_func; args[-1] = Py_NewRef(method); - Py_DECREF(callable); + INTERPRETER_DECREF(callable); callable = method; } // Check if the call can be inlined or not @@ -795,10 +830,12 @@ { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + SAVE_SP(); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)callable, locals, args, total_args, NULL ); + LOAD_SP(); // Manipulate stack directly since we leave using DISPATCH_INLINED(). STACK_SHRINK(oparg + 2); // The frame has stolen all the arguments from the stack, @@ -810,13 +847,16 @@ DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ + SAVE_SP(); res = PyObject_Vectorcall( callable, args, total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + LOAD_SP(); if (opcode == INSTRUMENTED_CALL) { PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING : args[0]; + SAVE_SP(); if (res == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, @@ -830,11 +870,12 @@ Py_CLEAR(res); } } + LOAD_SP(); } assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); + INTERPRETER_DECREF(callable); for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + INTERPRETER_DECREF(args[i]); } if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } @@ -881,7 +922,7 @@ if (self == NULL) { goto error; } - Py_DECREF(tp); + INTERPRETER_DECREF(tp); _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, (PyCodeObject *)&_Py_InitCleanup, 1); assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK); @@ -941,7 +982,7 @@ stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS func = Py_NewRef(((PyMethodObject *)callable)->im_func); stack_pointer[-2 - oparg] = func; // This is used by CALL, upon deoptimization - Py_DECREF(callable); + INTERPRETER_DECREF(callable); } // _CHECK_FUNCTION_VERSION callable = func; @@ -995,7 +1036,7 @@ // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); stack_pointer += -2 - oparg; - _PyFrame_SetStackPointer(frame, stack_pointer); + SAVE_SP(); new_frame->previous = frame; CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; @@ -1045,7 +1086,7 @@ method = ((PyMethodObject *)callable)->im_func; assert(PyFunction_Check(method)); Py_INCREF(method); - Py_DECREF(callable); + INTERPRETER_DECREF(callable); } // _PY_FRAME_GENERAL args = &stack_pointer[-oparg]; @@ -1061,10 +1102,12 @@ assert(Py_TYPE(callable) == &PyFunction_Type); int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + SAVE_SP(); new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)callable, locals, args, total_args, NULL ); + LOAD_SP(); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. stack_pointer += -2 - oparg; @@ -1086,7 +1129,7 @@ // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); - _PyFrame_SetStackPointer(frame, stack_pointer); + SAVE_SP(); new_frame->previous = frame; CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; @@ -1123,12 +1166,14 @@ PyTypeObject *tp = (PyTypeObject *)callable; DEOPT_IF(tp->tp_vectorcall == NULL, CALL); STAT_INC(CALL, hit); + SAVE_SP(); res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); + LOAD_SP(); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + INTERPRETER_DECREF(args[i]); } - Py_DECREF(tp); + INTERPRETER_DECREF(tp); if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } // _CHECK_PERIODIC @@ -1167,16 +1212,18 @@ STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); /* res = func(self, args, nargs) */ + SAVE_SP(); res = ((PyCFunctionFast)(void(*)(void))cfunc)( PyCFunction_GET_SELF(callable), args, total_args); + LOAD_SP(); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + INTERPRETER_DECREF(args[i]); } - Py_DECREF(callable); + INTERPRETER_DECREF(callable); if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } // _CHECK_PERIODIC @@ -1217,13 +1264,15 @@ PyCFunctionFastWithKeywords cfunc = (PyCFunctionFastWithKeywords)(void(*)(void)) PyCFunction_GET_FUNCTION(callable); + SAVE_SP(); res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); + LOAD_SP(); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + INTERPRETER_DECREF(args[i]); } - Py_DECREF(callable); + INTERPRETER_DECREF(callable); if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } // _CHECK_PERIODIC @@ -1266,11 +1315,13 @@ PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); PyObject *arg = args[0]; _Py_EnterRecursiveCallTstateUnchecked(tstate); + SAVE_SP(); res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); + LOAD_SP(); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(arg); - Py_DECREF(callable); + INTERPRETER_DECREF(arg); + INTERPRETER_DECREF(callable); if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } // _CHECK_PERIODIC @@ -1303,32 +1354,43 @@ if (check_args_iterable(tstate, func, callargs) < 0) { goto error; } + SAVE_SP(); PyObject *tuple = PySequence_Tuple(callargs); + LOAD_SP(); if (tuple == NULL) { goto error; } - Py_SETREF(callargs, tuple); + INTERPRETER_DECREF(callargs); + callargs = tuple; } assert(PyTuple_CheckExact(callargs)); EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; + SAVE_SP(); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, func, arg); + LOAD_SP(); if (err) goto error; + SAVE_SP(); result = PyObject_Call(func, callargs, kwargs); + LOAD_SP(); if (!PyFunction_Check(func) && !PyMethod_Check(func)) { if (result == NULL) { + SAVE_SP(); _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, frame, this_instr, func, arg); + LOAD_SP(); } else { + SAVE_SP(); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, frame, this_instr, func, arg); + LOAD_SP(); if (err < 0) { Py_CLEAR(result); } @@ -1343,9 +1405,11 @@ Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + SAVE_SP(); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, (PyFunctionObject *)func, locals, nargs, callargs, kwargs); + LOAD_SP(); // Need to manually shrink the stack since we exit with DISPATCH_INLINED. STACK_SHRINK(oparg + 3); if (new_frame == NULL) { @@ -1355,11 +1419,13 @@ frame->return_offset = 1; DISPATCH_INLINED(new_frame); } + SAVE_SP(); result = PyObject_Call(func, callargs, kwargs); + LOAD_SP(); } - Py_DECREF(func); - Py_DECREF(callargs); - Py_XDECREF(kwargs); + INTERPRETER_DECREF(func); + INTERPRETER_DECREF(callargs); + if (kwargs != NULL) { INTERPRETER_DECREF(kwargs); } assert(PEEK(2 + (oparg & 1)) == NULL); if (result == NULL) { stack_pointer += -3 - (oparg & 1); goto error; } stack_pointer[-3 - (oparg & 1)] = result; @@ -1376,8 +1442,10 @@ PyObject *res; value = stack_pointer[-1]; assert(oparg <= MAX_INTRINSIC_1); + SAVE_SP(); res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value); - Py_DECREF(value); + LOAD_SP(); + INTERPRETER_DECREF(value); if (res == NULL) goto pop_1_error; stack_pointer[-1] = res; DISPATCH(); @@ -1393,9 +1461,11 @@ value1 = stack_pointer[-1]; value2 = stack_pointer[-2]; assert(oparg <= MAX_INTRINSIC_2); + SAVE_SP(); res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); - Py_DECREF(value2); - Py_DECREF(value1); + LOAD_SP(); + INTERPRETER_DECREF(value2); + INTERPRETER_DECREF(value1); if (res == NULL) goto pop_2_error; stack_pointer[-2] = res; stack_pointer += -1; @@ -1428,7 +1498,9 @@ STAT_INC(CALL, hit); PyObject *cls = args[1]; PyObject *inst = args[0]; + SAVE_SP(); int retval = PyObject_IsInstance(inst, cls); + LOAD_SP(); if (retval < 0) { goto error; } @@ -1437,9 +1509,9 @@ if (res == NULL) { GOTO_ERROR(error); } - Py_DECREF(inst); - Py_DECREF(cls); - Py_DECREF(callable); + INTERPRETER_DECREF(inst); + INTERPRETER_DECREF(cls); + INTERPRETER_DECREF(callable); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; DISPATCH(); @@ -1474,7 +1546,7 @@ args[0] = Py_NewRef(self); PyObject *method = ((PyMethodObject *)callable)->im_func; args[-1] = Py_NewRef(method); - Py_DECREF(callable); + INTERPRETER_DECREF(callable); callable = method; } int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames); @@ -1485,11 +1557,13 @@ { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + SAVE_SP(); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)callable, locals, args, positional_args, kwnames ); - Py_DECREF(kwnames); + LOAD_SP(); + INTERPRETER_DECREF(kwnames); // Manipulate stack directly since we leave using DISPATCH_INLINED(). STACK_SHRINK(oparg + 3); // The frame has stolen all the arguments from the stack, @@ -1502,13 +1576,16 @@ DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ + SAVE_SP(); res = PyObject_Vectorcall( callable, args, positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames); + LOAD_SP(); if (opcode == INSTRUMENTED_CALL_KW) { PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING : args[0]; + SAVE_SP(); if (res == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, @@ -1522,12 +1599,13 @@ Py_CLEAR(res); } } + LOAD_SP(); } - Py_DECREF(kwnames); + INTERPRETER_DECREF(kwnames); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); + INTERPRETER_DECREF(callable); for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + INTERPRETER_DECREF(args[i]); } if (res == NULL) { stack_pointer += -3 - oparg; goto error; } stack_pointer[-3 - oparg] = res; @@ -1561,7 +1639,9 @@ DEOPT_IF(callable != interp->callable_cache.len, CALL); STAT_INC(CALL, hit); PyObject *arg = args[0]; + SAVE_SP(); Py_ssize_t len_i = PyObject_Length(arg); + LOAD_SP(); if (len_i < 0) { goto error; } @@ -1570,8 +1650,8 @@ if (res == NULL) { GOTO_ERROR(error); } - Py_DECREF(callable); - Py_DECREF(arg); + INTERPRETER_DECREF(callable); + INTERPRETER_DECREF(arg); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; DISPATCH(); @@ -1599,8 +1679,8 @@ if (_PyList_AppendTakeRef((PyListObject *)self, arg) < 0) { goto pop_1_error; // Since arg is DECREF'ed already } - Py_DECREF(self); - Py_DECREF(callable); + INTERPRETER_DECREF(self); + INTERPRETER_DECREF(callable); STACK_SHRINK(3); // Skip POP_TOP assert(next_instr->op.code == POP_TOP); @@ -1640,13 +1720,15 @@ PyCFunctionFast cfunc = (PyCFunctionFast)(void(*)(void))meth->ml_meth; int nargs = total_args - 1; + SAVE_SP(); res = cfunc(self, args + 1, nargs); + LOAD_SP(); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Clear the stack of the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + INTERPRETER_DECREF(args[i]); } - Py_DECREF(callable); + INTERPRETER_DECREF(callable); if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } // _CHECK_PERIODIC @@ -1690,13 +1772,15 @@ int nargs = total_args - 1; PyCFunctionFastWithKeywords cfunc = (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; + SAVE_SP(); res = cfunc(self, args + 1, nargs, NULL); + LOAD_SP(); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + INTERPRETER_DECREF(args[i]); } - Py_DECREF(callable); + INTERPRETER_DECREF(callable); if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } // _CHECK_PERIODIC @@ -1742,11 +1826,13 @@ STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); + SAVE_SP(); res = _PyCFunction_TrampolineCall(cfunc, self, NULL); + LOAD_SP(); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(callable); + INTERPRETER_DECREF(self); + INTERPRETER_DECREF(callable); if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } // _CHECK_PERIODIC @@ -1792,12 +1878,14 @@ STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); + SAVE_SP(); res = _PyCFunction_TrampolineCall(cfunc, self, arg); + LOAD_SP(); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(arg); - Py_DECREF(callable); + INTERPRETER_DECREF(self); + INTERPRETER_DECREF(arg); + INTERPRETER_DECREF(callable); if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } // _CHECK_PERIODIC @@ -1839,14 +1927,16 @@ total_args++; } /* Callable is not a normal Python function */ + SAVE_SP(); res = PyObject_Vectorcall( callable, args, total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + LOAD_SP(); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); + INTERPRETER_DECREF(callable); for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + INTERPRETER_DECREF(args[i]); } if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } @@ -1925,7 +2015,7 @@ // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); stack_pointer += -2 - oparg; - _PyFrame_SetStackPointer(frame, stack_pointer); + SAVE_SP(); new_frame->previous = frame; CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; @@ -1972,10 +2062,12 @@ assert(Py_TYPE(callable) == &PyFunction_Type); int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + SAVE_SP(); new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)callable, locals, args, total_args, NULL ); + LOAD_SP(); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. stack_pointer += -2 - oparg; @@ -1997,7 +2089,7 @@ // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); - _PyFrame_SetStackPointer(frame, stack_pointer); + SAVE_SP(); new_frame->previous = frame; CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; @@ -2029,8 +2121,10 @@ DEOPT_IF(null != NULL, CALL); DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); STAT_INC(CALL, hit); + SAVE_SP(); res = PyObject_Str(arg); - Py_DECREF(arg); + LOAD_SP(); + INTERPRETER_DECREF(arg); if (res == NULL) goto pop_3_error; } // _CHECK_PERIODIC @@ -2062,8 +2156,10 @@ DEOPT_IF(null != NULL, CALL); DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); STAT_INC(CALL, hit); + SAVE_SP(); res = PySequence_Tuple(arg); - Py_DECREF(arg); + LOAD_SP(); + INTERPRETER_DECREF(arg); if (res == NULL) goto pop_3_error; } // _CHECK_PERIODIC @@ -2093,8 +2189,10 @@ DEOPT_IF(null != NULL, CALL); DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL); STAT_INC(CALL, hit); + SAVE_SP(); res = Py_NewRef(Py_TYPE(arg)); - Py_DECREF(arg); + LOAD_SP(); + INTERPRETER_DECREF(arg); stack_pointer[-3] = res; stack_pointer += -2; DISPATCH(); @@ -2111,16 +2209,18 @@ match_type = stack_pointer[-1]; exc_value = stack_pointer[-2]; if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { - Py_DECREF(exc_value); - Py_DECREF(match_type); + INTERPRETER_DECREF(exc_value); + INTERPRETER_DECREF(match_type); if (true) goto pop_2_error; } match = NULL; rest = NULL; + SAVE_SP(); int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, &match, &rest); - Py_DECREF(exc_value); - Py_DECREF(match_type); + LOAD_SP(); + INTERPRETER_DECREF(exc_value); + INTERPRETER_DECREF(match_type); if (res < 0) goto pop_2_error; assert((match == NULL) == (rest == NULL)); if (match == NULL) goto pop_2_error; @@ -2142,12 +2242,15 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; assert(PyExceptionInstance_Check(left)); + SAVE_SP(); if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { - Py_DECREF(right); + LOAD_SP(); + INTERPRETER_DECREF(right); if (true) goto pop_1_error; } int res = PyErr_GivenExceptionMatches(left, right); - Py_DECREF(right); + LOAD_SP(); + INTERPRETER_DECREF(right); b = res ? Py_True : Py_False; stack_pointer[-1] = b; DISPATCH(); @@ -2170,14 +2273,16 @@ assert(exc_value && PyExceptionInstance_Check(exc_value)); if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { value = Py_NewRef(((PyStopIterationObject *)exc_value)->value); - Py_DECREF(sub_iter); - Py_DECREF(last_sent_val); - Py_DECREF(exc_value); + INTERPRETER_DECREF(sub_iter); + INTERPRETER_DECREF(last_sent_val); + INTERPRETER_DECREF(exc_value); none = Py_None; } else { _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); + SAVE_SP(); monitor_reraise(tstate, frame, this_instr); + LOAD_SP(); goto exception_unwind; } stack_pointer[-3] = none; @@ -2205,7 +2310,9 @@ #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_CompareOp(left, right, next_instr, oparg); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(COMPARE_OP, deferred); @@ -2215,13 +2322,15 @@ // _COMPARE_OP { assert((oparg >> 5) <= Py_GE); + SAVE_SP(); res = PyObject_RichCompare(left, right, oparg >> 5); - Py_DECREF(left); - Py_DECREF(right); + LOAD_SP(); + INTERPRETER_DECREF(left); + INTERPRETER_DECREF(right); if (res == NULL) goto pop_2_error; if (oparg & 16) { int res_bool = PyObject_IsTrue(res); - Py_DECREF(res); + INTERPRETER_DECREF(res); if (res_bool < 0) goto pop_2_error; res = res_bool ? Py_True : Py_False; } @@ -2354,7 +2463,9 @@ #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_ContainsOp(right, next_instr); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(CONTAINS_OP, deferred); @@ -2363,9 +2474,11 @@ } // _CONTAINS_OP { + SAVE_SP(); int res = PySequence_Contains(right, left); - Py_DECREF(left); - Py_DECREF(right); + LOAD_SP(); + INTERPRETER_DECREF(left); + INTERPRETER_DECREF(right); if (res < 0) goto pop_2_error; b = (res ^ oparg) ? Py_True : Py_False; } @@ -2387,9 +2500,11 @@ left = stack_pointer[-2]; DEOPT_IF(!PyDict_CheckExact(right), CONTAINS_OP); STAT_INC(CONTAINS_OP, hit); + SAVE_SP(); int res = PyDict_Contains(right, left); - Py_DECREF(left); - Py_DECREF(right); + LOAD_SP(); + INTERPRETER_DECREF(left); + INTERPRETER_DECREF(right); if (res < 0) goto pop_2_error; b = (res ^ oparg) ? Py_True : Py_False; stack_pointer[-2] = b; @@ -2411,9 +2526,11 @@ DEOPT_IF(!(PySet_CheckExact(right) || PyFrozenSet_CheckExact(right)), CONTAINS_OP); STAT_INC(CONTAINS_OP, hit); // Note: both set and frozenset use the same seq_contains method! + SAVE_SP(); int res = _PySet_Contains((PySetObject *)right, left); - Py_DECREF(left); - Py_DECREF(right); + LOAD_SP(); + INTERPRETER_DECREF(left); + INTERPRETER_DECREF(right); if (res < 0) goto pop_2_error; b = (res ^ oparg) ? Py_True : Py_False; stack_pointer[-2] = b; @@ -2431,8 +2548,10 @@ conversion_func conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); conv_fn = _PyEval_ConversionFuncs[oparg]; + SAVE_SP(); result = conv_fn(value); - Py_DECREF(value); + LOAD_SP(); + INTERPRETER_DECREF(value); if (result == NULL) goto pop_1_error; stack_pointer[-1] = result; DISPATCH(); @@ -2476,8 +2595,10 @@ PyObject *owner; owner = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + SAVE_SP(); int err = PyObject_DelAttr(owner, name); - Py_DECREF(owner); + LOAD_SP(); + INTERPRETER_DECREF(owner); if (err) goto pop_1_error; stack_pointer += -1; DISPATCH(); @@ -2492,10 +2613,12 @@ // Fortunately we don't need its superpower. PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); if (oldobj == NULL) { + SAVE_SP(); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + LOAD_SP(); goto error; } - Py_DECREF(oldobj); + INTERPRETER_DECREF(oldobj); DISPATCH(); } @@ -2505,10 +2628,12 @@ INSTRUCTION_STATS(DELETE_FAST); PyObject *v = GETLOCAL(oparg); if (v == NULL) { + SAVE_SP(); _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); + LOAD_SP(); if (1) goto error; } SETLOCAL(oparg, NULL); @@ -2520,14 +2645,18 @@ next_instr += 1; INSTRUCTION_STATS(DELETE_GLOBAL); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + SAVE_SP(); int err = PyDict_Pop(GLOBALS(), name, NULL); + LOAD_SP(); // Can't use ERROR_IF here. if (err < 0) { goto error; } if (err == 0) { + SAVE_SP(); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); + LOAD_SP(); goto error; } DISPATCH(); @@ -2541,16 +2670,22 @@ PyObject *ns = LOCALS(); int err; if (ns == NULL) { + SAVE_SP(); _PyErr_Format(tstate, PyExc_SystemError, "no locals when deleting %R", name); + LOAD_SP(); goto error; } + SAVE_SP(); err = PyObject_DelItem(ns, name); + LOAD_SP(); // Can't use ERROR_IF here. if (err != 0) { + SAVE_SP(); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); + LOAD_SP(); goto error; } DISPATCH(); @@ -2565,9 +2700,11 @@ sub = stack_pointer[-1]; container = stack_pointer[-2]; /* del container[sub] */ + SAVE_SP(); int err = PyObject_DelItem(container, sub); - Py_DECREF(container); - Py_DECREF(sub); + LOAD_SP(); + INTERPRETER_DECREF(container); + INTERPRETER_DECREF(sub); if (err) goto pop_2_error; stack_pointer += -2; DISPATCH(); @@ -2583,12 +2720,17 @@ update = stack_pointer[-1]; dict = stack_pointer[-2 - (oparg - 1)]; callable = stack_pointer[-5 - (oparg - 1)]; - if (_PyDict_MergeEx(dict, update, 2) < 0) { + SAVE_SP(); + int err = _PyDict_MergeEx(dict, update, 2); + LOAD_SP(); + if (err < 0) { + SAVE_SP(); _PyEval_FormatKwargsError(tstate, callable, update); - Py_DECREF(update); + LOAD_SP(); + INTERPRETER_DECREF(update); if (true) goto pop_1_error; } - Py_DECREF(update); + INTERPRETER_DECREF(update); stack_pointer += -1; DISPATCH(); } @@ -2601,16 +2743,21 @@ PyObject *dict; update = stack_pointer[-1]; dict = stack_pointer[-2 - (oparg - 1)]; - if (PyDict_Update(dict, update) < 0) { + SAVE_SP(); + int err = PyDict_Update(dict, update); + LOAD_SP(); + if (err < 0) { + SAVE_SP(); if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { _PyErr_Format(tstate, PyExc_TypeError, "'%.200s' object is not a mapping", Py_TYPE(update)->tp_name); } - Py_DECREF(update); + LOAD_SP(); + INTERPRETER_DECREF(update); if (true) goto pop_1_error; } - Py_DECREF(update); + INTERPRETER_DECREF(update); stack_pointer += -1; DISPATCH(); } @@ -2626,13 +2773,15 @@ awaitable = stack_pointer[-2]; assert(exc && PyExceptionInstance_Check(exc)); if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { - Py_DECREF(awaitable); - Py_DECREF(exc); + INTERPRETER_DECREF(awaitable); + INTERPRETER_DECREF(exc); } else { Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); + SAVE_SP(); monitor_reraise(tstate, frame, this_instr); + LOAD_SP(); goto exception_unwind; } stack_pointer += -2; @@ -2645,7 +2794,7 @@ INSTRUCTION_STATS(END_FOR); PyObject *value; value = stack_pointer[-1]; - Py_DECREF(value); + INTERPRETER_DECREF(value); stack_pointer += -1; DISPATCH(); } @@ -2658,7 +2807,7 @@ PyObject *receiver; value = stack_pointer[-1]; receiver = stack_pointer[-2]; - Py_DECREF(receiver); + INTERPRETER_DECREF(receiver); stack_pointer[-2] = value; stack_pointer += -1; DISPATCH(); @@ -2735,8 +2884,10 @@ /* If value is a unicode object, then we know the result * of format(value) is value itself. */ if (!PyUnicode_CheckExact(value)) { + SAVE_SP(); res = PyObject_Format(value, NULL); - Py_DECREF(value); + LOAD_SP(); + INTERPRETER_DECREF(value); if (res == NULL) goto pop_1_error; } else { @@ -2755,9 +2906,11 @@ PyObject *res; fmt_spec = stack_pointer[-1]; value = stack_pointer[-2]; + SAVE_SP(); res = PyObject_Format(value, fmt_spec); - Py_DECREF(value); - Py_DECREF(fmt_spec); + LOAD_SP(); + INTERPRETER_DECREF(value); + INTERPRETER_DECREF(fmt_spec); if (res == NULL) goto pop_2_error; stack_pointer[-2] = res; stack_pointer += -1; @@ -2781,7 +2934,9 @@ #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_ForIter(iter, next_instr, oparg); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(FOR_ITER, deferred); @@ -2791,19 +2946,23 @@ // _FOR_ITER { /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ + SAVE_SP(); next = (*Py_TYPE(iter)->tp_iternext)(iter); + LOAD_SP(); if (next == NULL) { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { goto error; } + SAVE_SP(); monitor_raise(tstate, frame, this_instr); _PyErr_Clear(tstate); + LOAD_SP(); } /* iterator ended normally */ assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - Py_DECREF(iter); + INTERPRETER_DECREF(iter); STACK_SHRINK(1); /* Jump forward oparg, then skip following END_FOR and POP_TOP instruction */ JUMPBY(oparg + 2); @@ -2850,7 +3009,7 @@ // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); - _PyFrame_SetStackPointer(frame, stack_pointer); + SAVE_SP(); new_frame->previous = frame; CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; @@ -2886,10 +3045,10 @@ #ifndef Py_GIL_DISABLED if (seq != NULL) { it->it_seq = NULL; - Py_DECREF(seq); + INTERPRETER_DECREF(seq); } #endif - Py_DECREF(iter); + INTERPRETER_DECREF(iter); STACK_SHRINK(1); /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ JUMPBY(oparg + 2); @@ -2931,7 +3090,7 @@ STAT_INC(FOR_ITER, hit); if (r->len <= 0) { STACK_SHRINK(1); - Py_DECREF(r); + INTERPRETER_DECREF(r); // Jump over END_FOR and POP_TOP instructions. JUMPBY(oparg + 2); DISPATCH(); @@ -2975,9 +3134,9 @@ if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) { if (seq != NULL) { it->it_seq = NULL; - Py_DECREF(seq); + INTERPRETER_DECREF(seq); } - Py_DECREF(iter); + INTERPRETER_DECREF(iter); STACK_SHRINK(1); /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ JUMPBY(oparg + 2); @@ -3011,15 +3170,19 @@ getter = type->tp_as_async->am_aiter; } if (getter == NULL) { + SAVE_SP(); _PyErr_Format(tstate, PyExc_TypeError, "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); - Py_DECREF(obj); + LOAD_SP(); + INTERPRETER_DECREF(obj); if (true) goto pop_1_error; } + SAVE_SP(); iter = (*getter)(obj); - Py_DECREF(obj); + LOAD_SP(); + INTERPRETER_DECREF(obj); if (iter == NULL) goto pop_1_error; if (Py_TYPE(iter)->tp_as_async == NULL || Py_TYPE(iter)->tp_as_async->am_anext == NULL) { @@ -3027,7 +3190,7 @@ "'async for' received an object from __aiter__ " "that does not implement __anext__: %.100s", Py_TYPE(iter)->tp_name); - Py_DECREF(iter); + INTERPRETER_DECREF(iter); if (true) goto pop_1_error; } stack_pointer[-1] = iter; @@ -3045,7 +3208,9 @@ PyObject *next_iter = NULL; PyTypeObject *type = Py_TYPE(aiter); if (PyAsyncGen_CheckExact(aiter)) { + SAVE_SP(); awaitable = type->tp_as_async->am_anext(aiter); + LOAD_SP(); if (awaitable == NULL) { goto error; } @@ -3054,29 +3219,37 @@ getter = type->tp_as_async->am_anext; } if (getter != NULL) { + SAVE_SP(); next_iter = (*getter)(aiter); + LOAD_SP(); if (next_iter == NULL) { goto error; } } else { + SAVE_SP(); _PyErr_Format(tstate, PyExc_TypeError, "'async for' requires an iterator with " "__anext__ method, got %.100s", type->tp_name); + LOAD_SP(); goto error; } + SAVE_SP(); awaitable = _PyCoro_GetAwaitableIter(next_iter); + LOAD_SP(); if (awaitable == NULL) { + SAVE_SP(); _PyErr_FormatFromCause( PyExc_TypeError, "'async for' received an invalid object " "from __anext__: %.100s", Py_TYPE(next_iter)->tp_name); - Py_DECREF(next_iter); + LOAD_SP(); + INTERPRETER_DECREF(next_iter); goto error; } else { - Py_DECREF(next_iter); + INTERPRETER_DECREF(next_iter); } } stack_pointer[0] = awaitable; @@ -3091,18 +3264,22 @@ PyObject *iterable; PyObject *iter; iterable = stack_pointer[-1]; + SAVE_SP(); iter = _PyCoro_GetAwaitableIter(iterable); + LOAD_SP(); if (iter == NULL) { + SAVE_SP(); _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg); + LOAD_SP(); } - Py_DECREF(iterable); + INTERPRETER_DECREF(iterable); if (iter != NULL && PyCoro_CheckExact(iter)) { PyObject *yf = _PyGen_yf((PyGenObject*)iter); if (yf != NULL) { /* `iter` is a coroutine object that is being awaited, `yf` is a pointer to the current awaitable being awaited on. */ - Py_DECREF(yf); + INTERPRETER_DECREF(yf); Py_CLEAR(iter); _PyErr_SetString(tstate, PyExc_RuntimeError, "coroutine is being awaited already"); @@ -3122,8 +3299,10 @@ PyObject *iter; iterable = stack_pointer[-1]; /* before: [obj]; after [getiter(obj)] */ + SAVE_SP(); iter = PyObject_GetIter(iterable); - Py_DECREF(iterable); + LOAD_SP(); + INTERPRETER_DECREF(iterable); if (iter == NULL) goto pop_1_error; stack_pointer[-1] = iter; DISPATCH(); @@ -3171,11 +3350,13 @@ } else { /* `iterable` is not a generator. */ + SAVE_SP(); iter = PyObject_GetIter(iterable); + LOAD_SP(); if (iter == NULL) { goto error; } - Py_DECREF(iterable); + INTERPRETER_DECREF(iterable); } stack_pointer[-1] = iter; DISPATCH(); @@ -3189,7 +3370,9 @@ PyObject *res; from = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + SAVE_SP(); res = import_from(tstate, from, name); + LOAD_SP(); if (res == NULL) goto error; stack_pointer[0] = res; stack_pointer += 1; @@ -3206,9 +3389,11 @@ fromlist = stack_pointer[-1]; level = stack_pointer[-2]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + SAVE_SP(); res = import_name(tstate, frame, name, fromlist, level); - Py_DECREF(level); - Py_DECREF(fromlist); + LOAD_SP(); + INTERPRETER_DECREF(level); + INTERPRETER_DECREF(fromlist); if (res == NULL) goto pop_2_error; stack_pointer[-2] = res; stack_pointer += -1; @@ -3226,9 +3411,11 @@ PyObject *function = PEEK(oparg + 2); PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING : PEEK(total_args); + SAVE_SP(); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, function, arg); + LOAD_SP(); if (err) goto error; PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); GO_TO_INSTRUCTION(CALL); @@ -3251,9 +3438,11 @@ PyObject *function = PEEK(oparg + 3); PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING : PEEK(total_args + 1); + SAVE_SP(); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, function, arg); + LOAD_SP(); if (err) goto error; GO_TO_INSTRUCTION(CALL_KW); } @@ -3270,11 +3459,14 @@ /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ if (PyGen_Check(receiver)) { - if (monitor_stop_iteration(tstate, frame, this_instr, value)) { + SAVE_SP(); + int err = monitor_stop_iteration(tstate, frame, this_instr, value); + LOAD_SP(); + if (err) { goto error; } } - Py_DECREF(value); + INTERPRETER_DECREF(value); stack_pointer += -1; DISPATCH(); } @@ -3289,11 +3481,14 @@ value = stack_pointer[-1]; receiver = stack_pointer[-2]; if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { - if (monitor_stop_iteration(tstate, frame, this_instr, value)) { + SAVE_SP(); + int err = monitor_stop_iteration(tstate, frame, this_instr, value); + LOAD_SP(); + if (err) { goto error; } } - Py_DECREF(receiver); + INTERPRETER_DECREF(receiver); stack_pointer[-2] = value; stack_pointer += -1; DISPATCH(); @@ -3307,7 +3502,9 @@ /* Skip 1 cache entry */ _Py_CODEUNIT *target; PyObject *iter = TOP(); + SAVE_SP(); PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); + LOAD_SP(); if (next != NULL) { PUSH(next); target = next_instr; @@ -3317,14 +3514,16 @@ if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { goto error; } + SAVE_SP(); monitor_raise(tstate, frame, this_instr); _PyErr_Clear(tstate); + LOAD_SP(); } /* iterator ended normally */ assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); STACK_SHRINK(1); - Py_DECREF(iter); + INTERPRETER_DECREF(iter); /* Skip END_FOR and POP_TOP */ target = next_instr + oparg + 2; } @@ -3337,8 +3536,10 @@ (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); + SAVE_SP(); int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, this_instr); + LOAD_SP(); if (next_opcode < 0) goto error; next_instr = this_instr; if (_PyOpcode_Caches[next_opcode]) { @@ -3411,7 +3612,7 @@ offset = oparg; } else { - Py_DECREF(value); + INTERPRETER_DECREF(value); offset = 0; } #if ENABLE_SPECIALIZATION @@ -3434,7 +3635,7 @@ offset = 0; } else { - Py_DECREF(value); + INTERPRETER_DECREF(value); offset = oparg; } #if ENABLE_SPECIALIZATION @@ -3498,9 +3699,11 @@ next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_RETURN_CONST); PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); + SAVE_SP(); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, this_instr, retval); + LOAD_SP(); if (err) goto error; Py_INCREF(retval); assert(EMPTY()); @@ -3523,9 +3726,11 @@ INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); PyObject *retval; retval = stack_pointer[-1]; + SAVE_SP(); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, this_instr, retval); + LOAD_SP(); if (err) goto error; STACK_SHRINK(1); assert(EMPTY()); @@ -3581,6 +3786,7 @@ assert(frame == &entry_frame); assert(_PyFrame_IsIncomplete(frame)); /* Restore previous frame and return. */ + SAVE_SP(); tstate->current_frame = frame->previous; assert(!_PyErr_Occurred(tstate)); tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; @@ -3597,8 +3803,8 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; int res = Py_Is(left, right) ^ oparg; - Py_DECREF(left); - Py_DECREF(right); + INTERPRETER_DECREF(left); + INTERPRETER_DECREF(right); b = res ? Py_True : Py_False; stack_pointer[-2] = b; stack_pointer += -1; @@ -3686,8 +3892,11 @@ PyObject *list; iterable = stack_pointer[-1]; list = stack_pointer[-2 - (oparg-1)]; + SAVE_SP(); PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); + LOAD_SP(); if (none_val == NULL) { + SAVE_SP(); if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) { @@ -3696,11 +3905,12 @@ "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); } - Py_DECREF(iterable); + INTERPRETER_DECREF(iterable); + LOAD_SP(); if (true) goto pop_1_error; } assert(Py_IsNone(none_val)); - Py_DECREF(iterable); + INTERPRETER_DECREF(iterable); stack_pointer += -1; DISPATCH(); } @@ -3724,7 +3934,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; + SAVE_SP(); _Py_Specialize_LoadAttr(owner, next_instr, name); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_ATTR, deferred); @@ -3738,7 +3950,10 @@ if (oparg & 1) { /* Designed to work in tandem with CALL, pushes two values. */ attr = NULL; - if (_PyObject_GetMethod(owner, name, &attr)) { + SAVE_SP(); + int err = _PyObject_GetMethod(owner, name, &attr); + LOAD_SP(); + if (err) { /* We can bypass temporary bound method object. meth is unbound method and obj is self. meth | self | arg1 | ... | argN @@ -3753,15 +3968,17 @@ CALL that it's not a method call. meth | NULL | arg1 | ... | argN */ - Py_DECREF(owner); + INTERPRETER_DECREF(owner); if (attr == NULL) goto pop_1_error; self_or_null = NULL; } } else { /* Classic, pushes one value. */ + SAVE_SP(); attr = PyObject_GetAttr(owner, name); - Py_DECREF(owner); + LOAD_SP(); + INTERPRETER_DECREF(owner); if (attr == NULL) goto pop_1_error; } } @@ -3796,7 +4013,7 @@ assert(descr != NULL); attr = Py_NewRef(descr); null = NULL; - Py_DECREF(owner); + INTERPRETER_DECREF(owner); } stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; @@ -3870,7 +4087,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; - Py_DECREF(owner); + INTERPRETER_DECREF(owner); } /* Skip 5 cache entries */ stack_pointer[-1] = attr; @@ -4032,7 +4249,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; - Py_DECREF(owner); + INTERPRETER_DECREF(owner); } /* Skip 5 cache entries */ stack_pointer[-1] = attr; @@ -4065,7 +4282,7 @@ assert(Py_TYPE(owner)->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - Py_DECREF(owner); + INTERPRETER_DECREF(owner); attr = Py_NewRef(descr); } stack_pointer[-1] = attr; @@ -4106,7 +4323,7 @@ assert((oparg & 1) == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - Py_DECREF(owner); + INTERPRETER_DECREF(owner); attr = Py_NewRef(descr); } stack_pointer[-1] = attr; @@ -4172,7 +4389,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; - Py_DECREF(owner); + INTERPRETER_DECREF(owner); } /* Skip 5 cache entries */ stack_pointer[-1] = attr; @@ -4225,7 +4442,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; - Py_DECREF(owner); + INTERPRETER_DECREF(owner); } /* Skip 5 cache entries */ stack_pointer[-1] = attr; @@ -4239,7 +4456,10 @@ next_instr += 1; INSTRUCTION_STATS(LOAD_BUILD_CLASS); PyObject *bc; - if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) goto error; + SAVE_SP(); + int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc); + LOAD_SP(); + if (err < 0) goto error; if (bc == NULL) { _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); @@ -4291,7 +4511,9 @@ PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); value = PyCell_GetRef(cell); if (value == NULL) { + SAVE_SP(); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + LOAD_SP(); if (true) goto error; } stack_pointer[0] = value; @@ -4373,18 +4595,23 @@ assert(class_dict); assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); - if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) { + SAVE_SP(); + int err = PyMapping_GetOptionalItem(class_dict, name, &value); + LOAD_SP(); + if (err < 0) { goto error; } if (!value) { PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); value = PyCell_GetRef(cell); if (value == NULL) { + SAVE_SP(); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + LOAD_SP(); goto error; } } - Py_DECREF(class_dict); + INTERPRETER_DECREF(class_dict); stack_pointer[-1] = value; DISPATCH(); } @@ -4397,26 +4624,37 @@ PyObject *v; mod_or_class_dict = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { + SAVE_SP(); + int err = PyMapping_GetOptionalItem(mod_or_class_dict, name, &v); + LOAD_SP(); + if (err < 0) { goto error; } if (v == NULL) { - if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { + SAVE_SP(); + err = PyDict_GetItemRef(GLOBALS(), name, &v); + LOAD_SP(); + if (err < 0) { goto error; } if (v == NULL) { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { + SAVE_SP(); + err = PyMapping_GetOptionalItem(BUILTINS(), name, &v); + LOAD_SP(); + if (err < 0) { goto error; } if (v == NULL) { + SAVE_SP(); _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); + LOAD_SP(); goto error; } } } - Py_DECREF(mod_or_class_dict); + INTERPRETER_DECREF(mod_or_class_dict); stack_pointer[-1] = v; DISPATCH(); } @@ -4438,7 +4676,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; + SAVE_SP(); _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_GLOBAL, deferred); @@ -4454,15 +4694,19 @@ if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) { + SAVE_SP(); res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), (PyDictObject *)BUILTINS(), name); + LOAD_SP(); if (res == NULL) { if (!_PyErr_Occurred(tstate)) { /* _PyDict_LoadGlobal() returns NULL without raising * an exception if the key doesn't exist */ + SAVE_SP(); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); + LOAD_SP(); } if (true) goto error; } @@ -4470,14 +4714,22 @@ else { /* Slow-path if globals or builtins is not a dict */ /* namespace 1: globals */ - if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) goto error; + SAVE_SP(); + int err = PyMapping_GetOptionalItem(GLOBALS(), name, &res); + LOAD_SP(); + if (err < 0) goto error; if (res == NULL) { /* namespace 2: builtins */ - if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) goto error; + SAVE_SP(); + err = PyMapping_GetOptionalItem(BUILTINS(), name, &res); + LOAD_SP(); + if (err < 0) goto error; if (res == NULL) { + SAVE_SP(); _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); + LOAD_SP(); if (true) goto error; } } @@ -4572,8 +4824,10 @@ PyObject *locals; locals = LOCALS(); if (locals == NULL) { + SAVE_SP(); _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); + LOAD_SP(); if (true) goto error; } Py_INCREF(locals); @@ -4589,26 +4843,39 @@ PyObject *v; PyObject *mod_or_class_dict = LOCALS(); if (mod_or_class_dict == NULL) { + SAVE_SP(); _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); + LOAD_SP(); if (true) goto error; } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { + SAVE_SP(); + int err = PyMapping_GetOptionalItem(mod_or_class_dict, name, &v); + LOAD_SP(); + if (err < 0) { goto error; } if (v == NULL) { - if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { + SAVE_SP(); + err = PyDict_GetItemRef(GLOBALS(), name, &v); + LOAD_SP(); + if (err < 0) { goto error; } if (v == NULL) { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { + SAVE_SP(); + err = PyMapping_GetOptionalItem(BUILTINS(), name, &v); + LOAD_SP(); + if (err < 0) { goto error; } if (v == NULL) { + SAVE_SP(); _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); + LOAD_SP(); goto error; } } @@ -4640,7 +4907,9 @@ int load_method = oparg & 1; if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_SUPER_ATTR, deferred); @@ -4651,15 +4920,18 @@ self = stack_pointer[-1]; { if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + SAVE_SP(); PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, global_super, arg); + LOAD_SP(); if (err) goto pop_3_error; } // we make no attempt to optimize here; specializations should // handle any case whose performance we care about PyObject *stack[] = {class, self}; + SAVE_SP(); PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; @@ -4677,13 +4949,16 @@ } } } - Py_DECREF(global_super); - Py_DECREF(class); - Py_DECREF(self); + LOAD_SP(); + INTERPRETER_DECREF(global_super); + INTERPRETER_DECREF(class); + INTERPRETER_DECREF(self); if (super == NULL) goto pop_3_error; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + SAVE_SP(); attr = PyObject_GetAttr(super, name); - Py_DECREF(super); + LOAD_SP(); + INTERPRETER_DECREF(super); if (attr == NULL) goto pop_3_error; null = NULL; } @@ -4711,10 +4986,12 @@ DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + SAVE_SP(); attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); - Py_DECREF(global_super); - Py_DECREF(class); - Py_DECREF(self); + LOAD_SP(); + INTERPRETER_DECREF(global_super); + INTERPRETER_DECREF(class); + INTERPRETER_DECREF(self); if (attr == NULL) goto pop_3_error; stack_pointer[-3] = attr; stack_pointer += -2; @@ -4742,18 +5019,20 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; int method_found = 0; + SAVE_SP(); attr = _PySuper_Lookup(cls, self, name, Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); - Py_DECREF(global_super); - Py_DECREF(class); + LOAD_SP(); + INTERPRETER_DECREF(global_super); + INTERPRETER_DECREF(class); if (attr == NULL) { - Py_DECREF(self); + INTERPRETER_DECREF(self); if (true) goto pop_3_error; } if (method_found) { self_or_null = self; // transfer ownership } else { - Py_DECREF(self); + INTERPRETER_DECREF(self); self_or_null = NULL; } stack_pointer[-3] = attr; @@ -4784,9 +5063,11 @@ PyObject *codeobj; PyObject *func; codeobj = stack_pointer[-1]; + SAVE_SP(); PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); - Py_DECREF(codeobj); + LOAD_SP(); + INTERPRETER_DECREF(codeobj); if (func_obj == NULL) { goto error; } @@ -4810,7 +5091,10 @@ assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references - if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; + SAVE_SP(); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, key, value); + LOAD_SP(); + if (err != 0) goto pop_2_error; stack_pointer += -2; DISPATCH(); } @@ -4829,10 +5113,12 @@ // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(names)); + SAVE_SP(); attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names); - Py_DECREF(subject); - Py_DECREF(type); - Py_DECREF(names); + LOAD_SP(); + INTERPRETER_DECREF(subject); + INTERPRETER_DECREF(type); + INTERPRETER_DECREF(names); if (attrs) { assert(PyTuple_CheckExact(attrs)); // Success! } @@ -4856,7 +5142,9 @@ keys = stack_pointer[-1]; subject = stack_pointer[-2]; // On successful match, PUSH(values). Otherwise, PUSH(None). + SAVE_SP(); values_or_none = _PyEval_MatchKeys(tstate, subject, keys); + LOAD_SP(); if (values_or_none == NULL) goto error; stack_pointer[0] = values_or_none; stack_pointer += 1; @@ -4905,7 +5193,11 @@ PyObject *exc_value; exc_value = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; - Py_XSETREF(exc_info->exc_value, exc_value == Py_None ? NULL : exc_value); + PyObject *tmp = exc_info->exc_value; + exc_info->exc_value = exc_value == Py_None ? NULL : exc_value; + if (tmp != NULL) { + INTERPRETER_DECREF(tmp); + } stack_pointer += -1; DISPATCH(); } @@ -4945,7 +5237,7 @@ } else { b = Py_False; - Py_DECREF(value); + INTERPRETER_DECREF(value); } } // _POP_JUMP_IF_TRUE @@ -4979,7 +5271,7 @@ } else { b = Py_False; - Py_DECREF(value); + INTERPRETER_DECREF(value); } } // _POP_JUMP_IF_FALSE @@ -5020,7 +5312,7 @@ INSTRUCTION_STATS(POP_TOP); PyObject *value; value = stack_pointer[-1]; - Py_DECREF(value); + INTERPRETER_DECREF(value); stack_pointer += -1; DISPATCH(); } @@ -5066,6 +5358,7 @@ PyObject **args; args = &stack_pointer[-oparg]; PyObject *cause = NULL, *exc = NULL; + SAVE_SP(); switch (oparg) { case 2: cause = args[1]; @@ -5077,6 +5370,7 @@ if (do_raise(tstate, exc, cause)) { assert(oparg == 0); monitor_reraise(tstate, frame, this_instr); + LOAD_SP(); goto exception_unwind; } break; @@ -5085,6 +5379,7 @@ "bad RAISE_VARARGS oparg"); break; } + LOAD_SP(); if (true) { stack_pointer += -oparg; goto error; } } @@ -5113,7 +5408,9 @@ assert(exc && PyExceptionInstance_Check(exc)); Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); + SAVE_SP(); monitor_reraise(tstate, frame, this_instr); + LOAD_SP(); goto exception_unwind; } @@ -5285,7 +5582,9 @@ #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_Send(receiver, next_instr); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(SEND, deferred); @@ -5311,18 +5610,25 @@ frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); DISPATCH_INLINED(gen_frame); } + SAVE_SP(); if (Py_IsNone(v) && PyIter_Check(receiver)) { retval = Py_TYPE(receiver)->tp_iternext(receiver); } else { retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); } + LOAD_SP(); if (retval == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) ) { + SAVE_SP(); monitor_raise(tstate, frame, this_instr); + LOAD_SP(); } - if (_PyGen_FetchStopIterationValue(&retval) == 0) { + SAVE_SP(); + int err = _PyGen_FetchStopIterationValue(&retval); + LOAD_SP(); + if (err == 0) { assert(retval != NULL); JUMPBY(oparg); } @@ -5330,7 +5636,7 @@ goto error; } } - Py_DECREF(v); + INTERPRETER_DECREF(v); } stack_pointer[-1] = retval; DISPATCH(); @@ -5369,22 +5675,27 @@ int err; PyObject *ann_dict; if (LOCALS() == NULL) { + SAVE_SP(); _PyErr_Format(tstate, PyExc_SystemError, "no locals found when setting up annotations"); + LOAD_SP(); if (true) goto error; } /* check if __annotations__ in locals()... */ - if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) goto error; + SAVE_SP(); + err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); + LOAD_SP(); + if (err < 0) goto error; if (ann_dict == NULL) { ann_dict = PyDict_New(); if (ann_dict == NULL) goto error; err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), ann_dict); - Py_DECREF(ann_dict); + INTERPRETER_DECREF(ann_dict); if (err) goto error; } else { - Py_DECREF(ann_dict); + INTERPRETER_DECREF(ann_dict); } DISPATCH(); } @@ -5397,8 +5708,10 @@ PyObject *set; v = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; + SAVE_SP(); int err = PySet_Add(set, v); - Py_DECREF(v); + LOAD_SP(); + INTERPRETER_DECREF(v); if (err) goto pop_1_error; stack_pointer += -1; DISPATCH(); @@ -5449,8 +5762,10 @@ PyObject *set; iterable = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; + SAVE_SP(); int err = _PySet_Update(set, iterable); - Py_DECREF(iterable); + LOAD_SP(); + INTERPRETER_DECREF(iterable); if (err < 0) goto pop_1_error; stack_pointer += -1; DISPATCH(); @@ -5474,7 +5789,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); next_instr = this_instr; + SAVE_SP(); _Py_Specialize_StoreAttr(owner, next_instr, name); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_ATTR, deferred); @@ -5486,9 +5803,11 @@ v = stack_pointer[-2]; { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + SAVE_SP(); int err = PyObject_SetAttr(owner, name, v); - Py_DECREF(v); - Py_DECREF(owner); + LOAD_SP(); + INTERPRETER_DECREF(v); + INTERPRETER_DECREF(owner); if (err) goto pop_2_error; } stack_pointer += -2; @@ -5531,9 +5850,9 @@ _PyDictValues_AddToInsertionOrder(values, index); } else { - Py_DECREF(old_value); + INTERPRETER_DECREF(old_value); } - Py_DECREF(owner); + INTERPRETER_DECREF(owner); } stack_pointer += -2; DISPATCH(); @@ -5563,8 +5882,10 @@ STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; *(PyObject **)addr = value; - Py_XDECREF(old_value); - Py_DECREF(owner); + if (old_value != NULL) { + INTERPRETER_DECREF(old_value); + } + INTERPRETER_DECREF(owner); } stack_pointer += -2; DISPATCH(); @@ -5614,7 +5935,7 @@ new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); ep->me_value = value; } - Py_DECREF(old_value); + INTERPRETER_DECREF(old_value); STAT_INC(STORE_ATTR, hit); /* Ensure dict is GC tracked if it needs to be */ if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(value)) { @@ -5622,7 +5943,7 @@ } /* PEP 509 */ dict->ma_version_tag = new_version; - Py_DECREF(owner); + INTERPRETER_DECREF(owner); } stack_pointer += -2; DISPATCH(); @@ -5635,7 +5956,9 @@ PyObject *v; v = stack_pointer[-1]; PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); + SAVE_SP(); PyCell_SetTakeRef(cell, v); + LOAD_SP(); stack_pointer += -1; DISPATCH(); } @@ -5690,8 +6013,10 @@ PyObject *v; v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + SAVE_SP(); int err = PyDict_SetItem(GLOBALS(), name, v); - Py_DECREF(v); + LOAD_SP(); + INTERPRETER_DECREF(v); if (err) goto pop_1_error; stack_pointer += -1; DISPATCH(); @@ -5709,14 +6034,16 @@ if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); - Py_DECREF(v); + INTERPRETER_DECREF(v); if (true) goto pop_1_error; } + SAVE_SP(); if (PyDict_CheckExact(ns)) err = PyDict_SetItem(ns, name, v); else err = PyObject_SetItem(ns, name, v); - Py_DECREF(v); + LOAD_SP(); + INTERPRETER_DECREF(v); if (err) goto pop_1_error; stack_pointer += -1; DISPATCH(); @@ -5740,11 +6067,13 @@ err = 1; } else { + SAVE_SP(); err = PyObject_SetItem(container, slice, v); - Py_DECREF(slice); + LOAD_SP(); + INTERPRETER_DECREF(slice); } - Py_DECREF(v); - Py_DECREF(container); + INTERPRETER_DECREF(v); + INTERPRETER_DECREF(container); if (err) goto pop_4_error; stack_pointer += -4; DISPATCH(); @@ -5769,7 +6098,9 @@ #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_StoreSubscr(container, sub, next_instr); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_SUBSCR, deferred); @@ -5780,10 +6111,12 @@ v = stack_pointer[-3]; { /* container[sub] = v */ + SAVE_SP(); int err = PyObject_SetItem(container, sub, v); - Py_DECREF(v); - Py_DECREF(container); - Py_DECREF(sub); + LOAD_SP(); + INTERPRETER_DECREF(v); + INTERPRETER_DECREF(container); + INTERPRETER_DECREF(sub); if (err) goto pop_3_error; } stack_pointer += -3; @@ -5804,8 +6137,10 @@ value = stack_pointer[-3]; DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); + SAVE_SP(); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); - Py_DECREF(dict); + LOAD_SP(); + INTERPRETER_DECREF(dict); if (err) goto pop_3_error; stack_pointer += -3; DISPATCH(); @@ -5834,9 +6169,9 @@ PyObject *old_value = PyList_GET_ITEM(list, index); PyList_SET_ITEM(list, index, value); assert(old_value != NULL); - Py_DECREF(old_value); + INTERPRETER_DECREF(old_value); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(list); + INTERPRETER_DECREF(list); stack_pointer += -3; DISPATCH(); } @@ -5872,7 +6207,9 @@ #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_ToBool(value, next_instr); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(TO_BOOL, deferred); @@ -5882,8 +6219,10 @@ /* Skip 2 cache entries */ // _TO_BOOL { + SAVE_SP(); int err = PyObject_IsTrue(value); - Py_DECREF(value); + LOAD_SP(); + INTERPRETER_DECREF(value); if (err < 0) goto pop_1_error; res = err ? Py_True : Py_False; } @@ -5911,7 +6250,7 @@ // _REPLACE_WITH_TRUE value = owner; { - Py_DECREF(value); + INTERPRETER_DECREF(value); res = Py_True; } stack_pointer[-1] = res; @@ -5949,7 +6288,7 @@ res = Py_False; } else { - Py_DECREF(value); + INTERPRETER_DECREF(value); res = Py_True; } stack_pointer[-1] = res; @@ -5969,7 +6308,7 @@ DEOPT_IF(!PyList_CheckExact(value), TO_BOOL); STAT_INC(TO_BOOL, hit); res = Py_SIZE(value) ? Py_True : Py_False; - Py_DECREF(value); + INTERPRETER_DECREF(value); stack_pointer[-1] = res; DISPATCH(); } @@ -6010,7 +6349,7 @@ } else { assert(Py_SIZE(value)); - Py_DECREF(value); + INTERPRETER_DECREF(value); res = Py_True; } stack_pointer[-1] = res; @@ -6024,8 +6363,10 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; + SAVE_SP(); res = PyNumber_Invert(value); - Py_DECREF(value); + LOAD_SP(); + INTERPRETER_DECREF(value); if (res == NULL) goto pop_1_error; stack_pointer[-1] = res; DISPATCH(); @@ -6038,8 +6379,10 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; + SAVE_SP(); res = PyNumber_Negative(value); - Py_DECREF(value); + LOAD_SP(); + INTERPRETER_DECREF(value); if (res == NULL) goto pop_1_error; stack_pointer[-1] = res; DISPATCH(); @@ -6066,8 +6409,10 @@ seq = stack_pointer[-1]; int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); PyObject **top = stack_pointer + totalargs - 1; + SAVE_SP(); int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); - Py_DECREF(seq); + LOAD_SP(); + INTERPRETER_DECREF(seq); if (res == 0) goto pop_1_error; stack_pointer += (oparg >> 8) + (oparg & 0xFF); DISPATCH(); @@ -6089,7 +6434,9 @@ #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_UnpackSequence(seq, next_instr, oparg); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(UNPACK_SEQUENCE, deferred); @@ -6101,8 +6448,10 @@ // _UNPACK_SEQUENCE { PyObject **top = stack_pointer + oparg - 1; + SAVE_SP(); int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); - Py_DECREF(seq); + LOAD_SP(); + INTERPRETER_DECREF(seq); if (res == 0) goto pop_1_error; } stack_pointer += -1 + oparg; @@ -6126,7 +6475,7 @@ for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } - Py_DECREF(seq); + INTERPRETER_DECREF(seq); stack_pointer += -1 + oparg; DISPATCH(); } @@ -6148,7 +6497,7 @@ for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } - Py_DECREF(seq); + INTERPRETER_DECREF(seq); stack_pointer += -1 + oparg; DISPATCH(); } @@ -6169,7 +6518,7 @@ STAT_INC(UNPACK_SEQUENCE, hit); val0 = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); val1 = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); - Py_DECREF(seq); + INTERPRETER_DECREF(seq); stack_pointer[-1] = val1; stack_pointer[0] = val0; stack_pointer += 1; @@ -6203,13 +6552,15 @@ tb = Py_None; } else { - Py_DECREF(tb); + INTERPRETER_DECREF(tb); } assert(PyLong_Check(lasti)); (void)lasti; // Shut up compiler warning if asserts are off PyObject *stack[4] = {NULL, exc, val, tb}; + SAVE_SP(); res = PyObject_Vectorcall(exit_func, stack + 1, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + LOAD_SP(); if (res == NULL) goto error; stack_pointer[0] = res; stack_pointer += 1; diff --git a/Python/pystate.c b/Python/pystate.c index 1ea1ad982a0ec9..d047bf8f6dcc5b 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1498,6 +1498,7 @@ init_threadstate(_PyThreadStateImpl *_tstate, tstate->what_event = -1; tstate->previous_executor = NULL; tstate->dict_global_version = 0; + tstate->sp_cached = 0; tstate->delete_later = NULL; diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index cc9eb8a0e90eeb..988c27d9582dc3 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -130,15 +130,39 @@ def replace_decrefs( continue if var.size != "1": out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") - out.emit(f"Py_DECREF({var.name}[_i]);\n") + out.emit(f"INTERPRETER_DECREF({var.name}[_i]);\n") out.emit("}\n") elif var.condition: if var.condition == "1": - out.emit(f"Py_DECREF({var.name});\n") + out.emit(f"INTERPRETER_DECREF({var.name});\n") elif var.condition != "0": - out.emit(f"Py_XDECREF({var.name});\n") + out.emit(f"if ({var.name} != NULL) {{ INTERPRETER_DECREF({var.name}); }}\n") else: - out.emit(f"Py_DECREF({var.name});\n") + out.emit(f"INTERPRETER_DECREF({var.name});\n") + +def replace_decref( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + unused: Stack, + inst: Instruction | None, +) -> None: + out.emit_at("INTERPRETER_DECREF", tkn) + out.emit(next(tkn_iter)) + emit_to(out, tkn_iter, "RPAREN") + next(tkn_iter) # Semi colon + out.emit(");\n") + +def replace_xdecref( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + unused: Stack, + inst: Instruction | None, +) -> None: + raise Exception("XDECREF not supported") def replace_sync_sp( @@ -175,6 +199,8 @@ def replace_check_eval_breaker( "DEOPT_IF": replace_deopt, "ERROR_IF": replace_error, "ERROR_NO_POP": replace_error_no_pop, + "Py_DECREF": replace_decref, + "Py_XDECREF": replace_xdecref, "DECREF_INPUTS": replace_decrefs, "CHECK_EVAL_BREAKER": replace_check_eval_breaker, "SYNC_SP": replace_sync_sp,