#include "Python.h"

#include "pycore_abstract.h" // _PyIndex_Check()

#include "pycore_audit.h" // _PySys_Audit()

#include "pycore_backoff.h"

#include "pycore_cell.h" // PyCell_GetRef()

#include "pycore_ceval.h" // _PyEval_LazyImportName(), _PyEval_LazyImportFrom()

#include "pycore_code.h"

#include "pycore_emscripten_signal.h" // _Py_CHECK_EMSCRIPTEN_SIGNALS

#include "pycore_function.h"

#include "pycore_import.h" // _PyImport_LoadLazyImportTstate()

#include "pycore_instruments.h"

#include "pycore_interpolation.h" // _PyInterpolation_Build()

#include "pycore_intrinsics.h"

#include "pycore_lazyimportobject.h" // PyLazyImport_CheckExact()

#include "pycore_long.h" // _PyLong_ExactDealloc(), _PyLong_GetZero()

#include "pycore_moduleobject.h" // PyModuleObject

#include "pycore_object.h" // _PyObject_GC_TRACK()

#include "pycore_opcode_metadata.h" // uop names

#include "pycore_opcode_utils.h" // MAKE_FUNCTION_*

#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_*

#include "pycore_pyerrors.h" // _PyErr_GetRaisedException()

#include "pycore_pystate.h" // _PyInterpreterState_GET()

#include "pycore_range.h" // _PyRangeIterObject

#include "pycore_setobject.h" // _PySet_NextEntry()

#include "pycore_sliceobject.h" // _PyBuildSlice_ConsumeRefs

#include "pycore_stackref.h"

#include "pycore_template.h" // _PyTemplate_Build()

#include "pycore_tuple.h" // _PyStolenTuple_Free(), _PyTuple_ITEMS()

#include "pycore_typeobject.h" // _PySuper_Lookup()

#include "pycore_dict.h"

#include "dictobject.h"

#include "pycore_frame.h"

#include "opcode.h"

#include "optimizer.h"

#include "pydtrace.h"

#include "setobject.h"

#define USE_COMPUTED_GOTOS 0

#include "ceval_macros.h"

#define inst(name, ...) case name:

#define op(name, ...) /* NAME is ignored */

#define macro(name) static int MACRO_##name

#define super(name) static int SUPER_##name

#define family(name, ...) static int family_##name

#define pseudo(name) static int pseudo_##name

#define label(name) name:

#define guard

#define override

#define specializing

#define replicate(TIMES)

#define tier1

#define no_save_ip

static PyObject *value, *value1, *value2, *left, *right, *res, *sum, *prod, *sub;

static PyObject *container, *start, *stop, *v, *lhs, *rhs, *res2;

static PyObject *list, *tuple, *dict, *owner, *set, *str, *tup, *map, *keys;

static PyObject *exit_func, *lasti, *val, *retval, *obj, *iter, *exhausted;

static PyObject *aiter, *awaitable, *iterable, *w, *exc_value, *bc, *locals;

static PyObject *orig, *excs, *update, *b, *fromlist, *level, *from;

static PyObject **pieces, **values;

static size_t jump;

static uint16_t invert, counter, index, hint;

#define unused 0 // Used in a macro def, can't be static

static uint32_t type_version;

static _PyExecutorObject *current_executor;

static PyObject *

dummy_func(

)

{

// Dummy labels.

pop_1_error:

// Dummy locals.

PyObject *dummy;

_Py_CODEUNIT *this_instr;

PyObject *attr;

PyObject *attrs;

PyObject *bottom;

PyObject *callable;

PyObject *callargs;

PyObject *codeobj;

PyObject *cond;

PyObject *descr;

PyObject *exc;

PyObject *exit;

PyObject *fget;

PyObject *fmt_spec;

PyObject *func;

uint32_t func_version;

PyObject *getattribute;

PyObject *kwargs;

PyObject *kwdefaults;

PyObject *len_o;

PyObject *match;

PyObject *match_type;

PyObject *method;

PyObject *mgr;

Py_ssize_t min_args;

PyObject *names;

PyObject *new_exc;

PyObject *next;

PyObject *none;

PyObject *null;

PyObject *prev_exc;

PyObject *receiver;

PyObject *rest;

int result;

PyObject *self;

PyObject *seq;

PyObject *slice;

PyObject *step;

PyObject *subject;

PyObject *top;

PyObject *type;

PyObject *typevars;

PyObject *val0;

PyObject *val1;

int values_or_none;

switch (opcode) {

pure inst(NOP, (--)) {

}

family(RESUME, 1) = {

RESUME_CHECK,

RESUME_CHECK_JIT,

};

macro(NOT_TAKEN) = NOP;

op(_CHECK_PERIODIC, (--)) {

int err = check_periodics(tstate);

ERROR_IF(err != 0);

}

replaced op(_CHECK_PERIODIC_AT_END, (--)) {

int err = check_periodics(tstate);

ERROR_IF(err != 0);

}

op(_CHECK_PERIODIC_IF_NOT_YIELD_FROM, (--)) {

if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) {

int err = check_periodics(tstate);

ERROR_IF(err != 0);

}

}

tier1 op(_QUICKEN_RESUME, (counter/1 --)) {

_Py_Specialize_Resume(this_instr, tstate, frame);

}

tier1 op(_MAYBE_INSTRUMENT, (--)) {

#ifdef Py_GIL_DISABLED

// For thread-safety, we need to check instrumentation version

// even when tracing. Otherwise, another thread may concurrently

// re-write the bytecode while we are executing this function.

int check_instrumentation = 1;

#else

int check_instrumentation = (tstate->tracing == 0);

#endif

if (check_instrumentation) {

uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK;

uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version);

if (code_version != global_version) {

int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp);

if (err) {

ERROR_NO_POP();

}

next_instr = this_instr;

DISPATCH();

}

}

}

replaced op(_LOAD_BYTECODE, (--)) {

#ifdef Py_GIL_DISABLED

if (frame->tlbc_index !=

((_PyThreadStateImpl *)tstate)->tlbc_index) {

_Py_CODEUNIT *bytecode =

_PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame));

ERROR_IF(bytecode == NULL);

ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame);

frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index;

frame->instr_ptr = bytecode + off;

// Make sure this_instr gets reset correctly for any uops that

// follow

next_instr = frame->instr_ptr;

DISPATCH();

}

#endif

}

macro(RESUME) =

_LOAD_BYTECODE +

_MAYBE_INSTRUMENT +

_QUICKEN_RESUME +

_CHECK_PERIODIC_IF_NOT_YIELD_FROM;

macro(RESUME_CHECK) =

unused/1 +

_RESUME_CHECK;

op(_RESUME_CHECK, (--)) {

#if defined(__EMSCRIPTEN__)

DEOPT_IF(_Py_emscripten_signal_clock == 0);

_Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING;

uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker);

uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version);

assert((version & _PY_EVAL_EVENTS_MASK) == 0);

DEOPT_IF(eval_breaker != version);

#ifdef Py_GIL_DISABLED

DEOPT_IF(frame->tlbc_index !=

((_PyThreadStateImpl *)tstate)->tlbc_index);

#endif

}

macro(RESUME_CHECK_JIT) =

unused/1 +

_RESUME_CHECK +

_JIT;

op(_MONITOR_RESUME, (--)) {

int err = _Py_call_instrumentation(

tstate, oparg == 0 ? PY_MONITORING_EVENT_PY_START : PY_MONITORING_EVENT_PY_RESUME, frame, this_instr);

ERROR_IF(err);

if (frame->instr_ptr != this_instr) {

/* Instrumentation has jumped */

next_instr = frame->instr_ptr;

}

}

macro(INSTRUMENTED_RESUME) =

unused/1 +

_LOAD_BYTECODE +

_MAYBE_INSTRUMENT +

_CHECK_PERIODIC_IF_NOT_YIELD_FROM +

_MONITOR_RESUME;

pseudo(LOAD_CLOSURE, (-- unused)) = {

LOAD_FAST,

};

inst(LOAD_FAST_CHECK, (-- value)) {

_PyStackRef value_s = GETLOCAL(oparg);

if (PyStackRef_IsNull(value_s)) {

_PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError,

UNBOUNDLOCAL_ERROR_MSG,

PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)

);

ERROR_IF(true);

}

value = PyStackRef_DUP(value_s);

}

replicate(8) pure inst(LOAD_FAST, (-- value)) {

assert(!PyStackRef_IsNull(GETLOCAL(oparg)));

value = PyStackRef_DUP(GETLOCAL(oparg));

}

replicate(8) pure inst (LOAD_FAST_BORROW, (-- value)) {

assert(!PyStackRef_IsNull(GETLOCAL(oparg)));

value = PyStackRef_Borrow(GETLOCAL(oparg));

}

inst(LOAD_FAST_AND_CLEAR, (-- value)) {

value = GETLOCAL(oparg);

GETLOCAL(oparg) = PyStackRef_NULL;

}

inst(LOAD_FAST_LOAD_FAST, ( -- value1, value2)) {

uint32_t oparg1 = oparg >> 4;

uint32_t oparg2 = oparg & 15;

value1 = PyStackRef_DUP(GETLOCAL(oparg1));

value2 = PyStackRef_DUP(GETLOCAL(oparg2));

}

inst(LOAD_FAST_BORROW_LOAD_FAST_BORROW, ( -- value1, value2)) {

uint32_t oparg1 = oparg >> 4;

uint32_t oparg2 = oparg & 15;

value1 = PyStackRef_Borrow(GETLOCAL(oparg1));

value2 = PyStackRef_Borrow(GETLOCAL(oparg2));

}

inst(LOAD_CONST, (-- value)) {

PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg);

value = PyStackRef_FromPyObjectBorrow(obj);

}

replicate(4) inst(LOAD_SMALL_INT, (-- value)) {

assert(oparg < _PY_NSMALLPOSINTS);

PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg];

value = PyStackRef_FromPyObjectBorrow(obj);

}

macro(STORE_FAST) = _SWAP_FAST + POP_TOP;

replicate(8) op(_SWAP_FAST, (value -- trash)) {

_PyStackRef tmp = GETLOCAL(oparg);

GETLOCAL(oparg) = value;

DEAD(value);

trash = tmp;

}

pseudo(STORE_FAST_MAYBE_NULL, (unused --)) = {

STORE_FAST,

};

inst(STORE_FAST_LOAD_FAST, (value1 -- value2)) {

uint32_t oparg1 = oparg >> 4;

uint32_t oparg2 = oparg & 15;

_PyStackRef tmp = GETLOCAL(oparg1);

GETLOCAL(oparg1) = value1;

DEAD(value1);

value2 = PyStackRef_DUP(GETLOCAL(oparg2));

PyStackRef_XCLOSE(tmp);

}

inst(STORE_FAST_STORE_FAST, (value2, value1 --)) {

uint32_t oparg1 = oparg >> 4;

uint32_t oparg2 = oparg & 15;

_PyStackRef tmp = GETLOCAL(oparg1);

GETLOCAL(oparg1) = value1;

DEAD(value1);

PyStackRef_XCLOSE(tmp);

tmp = GETLOCAL(oparg2);

GETLOCAL(oparg2) = value2;

DEAD(value2);

PyStackRef_XCLOSE(tmp);

}

pure inst(POP_TOP, (value --)) {

PyStackRef_XCLOSE(value);

}

op(_POP_TOP_NOP, (value --)) {

assert(PyStackRef_IsNull(value) || (!PyStackRef_RefcountOnObject(value)) ||

_Py_IsImmortal((PyStackRef_AsPyObjectBorrow(value))));

DEAD(value);

}

op(_POP_TOP_INT, (value --)) {

assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value)));

PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc);

}

op(_POP_TOP_FLOAT, (value --)) {

assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value)));

PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc);

}

op(_POP_TOP_UNICODE, (value --)) {

assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(value)));

PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc);

}

op(_POP_TOP_OPARG, (args[oparg] -- )) {

_PyStackRef_CloseStack(args, oparg);

DEAD(args);

}

pure inst(PUSH_NULL, (-- res)) {

res = PyStackRef_NULL;

}

no_save_ip inst(END_FOR, (value -- )) {

/* Don't update instr_ptr, so that POP_ITER sees

PyStackRef_CLOSE(value);

}

inst(POP_ITER, (iter, index_or_null -- )) {

(void)index_or_null;

DEAD(index_or_null);

PyStackRef_CLOSE(iter);

}

no_save_ip tier1 inst(INSTRUMENTED_END_FOR, (receiver, index_or_null, value -- receiver, index_or_null)) {

/* Need to create a fake StopIteration error here,

if (PyStackRef_GenCheck(receiver)) {

int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value));

if (err) {

ERROR_NO_POP();

}

}

PyStackRef_CLOSE(value);

}

tier1 inst(INSTRUMENTED_POP_ITER, (iter, index_or_null -- )) {

(void)index_or_null;

DEAD(index_or_null);

INSTRUMENTED_JUMP(prev_instr, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT);

PyStackRef_CLOSE(iter);

}

pure inst(END_SEND, (receiver, index_or_null, value -- val)) {

val = value;

(void)index_or_null;

DEAD(value);

DEAD(index_or_null);

PyStackRef_CLOSE(receiver);

}

tier1 inst(INSTRUMENTED_END_SEND, (receiver, index_or_null, value -- val)) {

PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver);

if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) {

int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value));

if (err) {

ERROR_NO_POP();

}

}

val = value;

(void)index_or_null;

DEAD(index_or_null);

DEAD(value);

PyStackRef_CLOSE(receiver);

}

macro(UNARY_NEGATIVE) = _UNARY_NEGATIVE + POP_TOP;

op(_UNARY_NEGATIVE, (value -- res, v)) {

PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value));

if (res_o == NULL) {

ERROR_NO_POP();

}

res = PyStackRef_FromPyObjectSteal(res_o);

v = value;

DEAD(value);

}

// Inplace negation: negate a uniquely-referenced float in place.

// Tier 2 only.

tier2 op(_UNARY_NEGATIVE_FLOAT_INPLACE, (value -- res, v)) {

PyObject *val_o = PyStackRef_AsPyObjectBorrow(value);

assert(PyFloat_CheckExact(val_o));

assert(_PyObject_IsUniquelyReferenced(val_o));

STAT_INC(UNARY_NEGATIVE, hit);

double dres = -((PyFloatObject *)val_o)->ob_fval;

((PyFloatObject *)val_o)->ob_fval = dres;

res = value;

v = PyStackRef_NULL;

INPUTS_DEAD();

}

inst(UNARY_NOT, (value -- res)) {

assert(PyStackRef_BoolCheck(value));

res = PyStackRef_IsFalse(value)

? PyStackRef_True : PyStackRef_False;

DEAD(value);

}

family(TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL) = {

TO_BOOL_ALWAYS_TRUE,

TO_BOOL_BOOL,

TO_BOOL_INT,

TO_BOOL_LIST,

TO_BOOL_NONE,

TO_BOOL_STR,

};

specializing op(_SPECIALIZE_TO_BOOL, (counter/1, value -- value)) {

#if ENABLE_SPECIALIZATION

if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {

next_instr = this_instr;

_Py_Specialize_ToBool(value, next_instr);

DISPATCH_SAME_OPARG();

}

OPCODE_DEFERRED_INC(TO_BOOL);

ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);

#endif /* ENABLE_SPECIALIZATION */

}

op(_TO_BOOL, (value -- res)) {

int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value));

PyStackRef_CLOSE(value);

ERROR_IF(err < 0);

res = err ? PyStackRef_True : PyStackRef_False;

}

macro(TO_BOOL) = _SPECIALIZE_TO_BOOL + unused/2 + _TO_BOOL;

inst(TO_BOOL_BOOL, (unused/1, unused/2, value -- value)) {

EXIT_IF(!PyStackRef_BoolCheck(value));

STAT_INC(TO_BOOL, hit);

}

op(_TO_BOOL_INT, (value -- res, v)) {

STAT_INC(TO_BOOL, hit);

PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);

res = (_PyLong_IsZero((PyLongObject *)value_o)) ? PyStackRef_False : PyStackRef_True;

v = value;

DEAD(value);

}

macro(TO_BOOL_INT) =

_GUARD_TOS_INT + unused/1 + unused/2 + _TO_BOOL_INT + _POP_TOP_INT;

op(_GUARD_NOS_LIST, (nos, unused -- nos, unused)) {

PyObject *o = PyStackRef_AsPyObjectBorrow(nos);

EXIT_IF(!PyList_CheckExact(o));

}

op(_GUARD_TOS_LIST, (tos -- tos)) {

PyObject *o = PyStackRef_AsPyObjectBorrow(tos);

EXIT_IF(!PyList_CheckExact(o));

}

op(_GUARD_TOS_SLICE, (tos -- tos)) {

PyObject *o = PyStackRef_AsPyObjectBorrow(tos);

EXIT_IF(!PySlice_Check(o));

}

macro(TO_BOOL_LIST) = _GUARD_TOS_LIST + unused/1 + unused/2 + _TO_BOOL_LIST + POP_TOP;

op(_TO_BOOL_LIST, (value -- res, v)) {

PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);

assert(PyList_CheckExact(value_o));

STAT_INC(TO_BOOL, hit);

res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False;

v = value;

DEAD(value);

}

inst(TO_BOOL_NONE, (unused/1, unused/2, value -- res)) {

// This one is a bit weird, because we expect *some* failures:

EXIT_IF(!PyStackRef_IsNone(value));

DEAD(value);

STAT_INC(TO_BOOL, hit);

res = PyStackRef_False;

}

op(_GUARD_NOS_COMPACT_ASCII, (nos, unused -- nos, unused)) {

PyObject *o = PyStackRef_AsPyObjectBorrow(nos);

EXIT_IF(!PyUnicode_CheckExact(o));

EXIT_IF(!PyUnicode_IS_COMPACT_ASCII(o));

}

op(_GUARD_NOS_UNICODE, (nos, unused -- nos, unused)) {

PyObject *o = PyStackRef_AsPyObjectBorrow(nos);

EXIT_IF(!PyUnicode_CheckExact(o));

}

op(_GUARD_TOS_UNICODE, (value -- value)) {

PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);

EXIT_IF(!PyUnicode_CheckExact(value_o));

}

op(_TO_BOOL_STR, (value -- res, v)) {

STAT_INC(TO_BOOL, hit);

PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);

res = value_o == &_Py_STR(empty) ? PyStackRef_False : PyStackRef_True;

v = value;

DEAD(value);

}

macro(TO_BOOL_STR) =

_GUARD_TOS_UNICODE + unused/1 + unused/2 + _TO_BOOL_STR + _POP_TOP_UNICODE;

op(_REPLACE_WITH_TRUE, (value -- res, v)) {

res = PyStackRef_True;

v = value;

DEAD(value);

}

macro(TO_BOOL_ALWAYS_TRUE) =

unused/1 +

_RECORD_TOS_TYPE +

_GUARD_TYPE_VERSION +

_REPLACE_WITH_TRUE +

POP_TOP;

macro(UNARY_INVERT) = _UNARY_INVERT + POP_TOP;

op(_UNARY_INVERT, (value -- res, v)) {

PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value));

if (res_o == NULL) {

ERROR_NO_POP();

}

res = PyStackRef_FromPyObjectSteal(res_o);

v = value;

DEAD(value);

}

family(BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP) = {

BINARY_OP_MULTIPLY_INT,

BINARY_OP_ADD_INT,

BINARY_OP_SUBTRACT_INT,

BINARY_OP_MULTIPLY_FLOAT,

BINARY_OP_ADD_FLOAT,

BINARY_OP_SUBTRACT_FLOAT,

BINARY_OP_ADD_UNICODE,

BINARY_OP_SUBSCR_LIST_INT,

BINARY_OP_SUBSCR_LIST_SLICE,

BINARY_OP_SUBSCR_TUPLE_INT,

BINARY_OP_SUBSCR_STR_INT,

BINARY_OP_SUBSCR_USTR_INT,

BINARY_OP_SUBSCR_DICT,

BINARY_OP_SUBSCR_GETITEM,

BINARY_OP_INPLACE_ADD_UNICODE,

BINARY_OP_EXTEND,

};

op(_GUARD_NOS_INT, (left, unused -- left, unused)) {

PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);

EXIT_IF(!_PyLong_CheckExactAndCompact(left_o));

}

op(_GUARD_TOS_INT, (value -- value)) {

PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);

EXIT_IF(!_PyLong_CheckExactAndCompact(value_o));

}

op(_GUARD_NOS_OVERFLOWED, (left, unused -- left, unused)) {

PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);

assert(Py_TYPE(left_o) == &PyLong_Type);

EXIT_IF(!_PyLong_IsCompact((PyLongObject *)left_o));

}

op(_GUARD_TOS_OVERFLOWED, (value -- value)) {

PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);

assert(Py_TYPE(value_o) == &PyLong_Type);

EXIT_IF(!_PyLong_IsCompact((PyLongObject *)value_o));

}

pure op(_BINARY_OP_MULTIPLY_INT, (left, right -- res, l, r)) {

PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);

PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);

assert(PyLong_CheckExact(left_o));

assert(PyLong_CheckExact(right_o));

assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o));

STAT_INC(BINARY_OP, hit);

res = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o);

EXIT_IF(PyStackRef_IsNull(res));

l = left;

r = right;

INPUTS_DEAD();

}

pure op(_BINARY_OP_ADD_INT, (left, right -- res, l, r)) {

PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);

PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);

assert(PyLong_CheckExact(left_o));

assert(PyLong_CheckExact(right_o));

assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o));

STAT_INC(BINARY_OP, hit);

res = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o);

EXIT_IF(PyStackRef_IsNull(res));

l = left;

r = right;

INPUTS_DEAD();

}

pure op(_BINARY_OP_SUBTRACT_INT, (left, right -- res, l, r)) {

PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);

PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);

assert(PyLong_CheckExact(left_o));

assert(PyLong_CheckExact(right_o));

assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o));

STAT_INC(BINARY_OP, hit);

res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o);

EXIT_IF(PyStackRef_IsNull(res));

l = left;

r = right;

INPUTS_DEAD();

}

macro(BINARY_OP_MULTIPLY_INT) =

_GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_MULTIPLY_INT + _POP_TOP_INT + _POP_TOP_INT;

macro(BINARY_OP_ADD_INT) =

_GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_ADD_INT + _POP_TOP_INT + _POP_TOP_INT;

macro(BINARY_OP_SUBTRACT_INT) =

_GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_SUBTRACT_INT + _POP_TOP_INT + _POP_TOP_INT;

// Inplace compact int ops: mutate the uniquely-referenced operand

// when possible. The op handles decref of TARGET internally so

// the following _POP_TOP_INT becomes _POP_TOP_NOP. Tier 2 only.

tier2 op(_BINARY_OP_ADD_INT_INPLACE, (left, right -- res, l, r)) {

INT_INPLACE_OP(left, right, left, +, _PyCompactLong_Add);

EXIT_IF(PyStackRef_IsNull(_int_inplace_res));

res = _int_inplace_res;

l = left;

r = right;

INPUTS_DEAD();

}

tier2 op(_BINARY_OP_SUBTRACT_INT_INPLACE, (left, right -- res, l, r)) {

INT_INPLACE_OP(left, right, left, -, _PyCompactLong_Subtract);

EXIT_IF(PyStackRef_IsNull(_int_inplace_res));

res = _int_inplace_res;

l = left;

r = right;

INPUTS_DEAD();

}

tier2 op(_BINARY_OP_MULTIPLY_INT_INPLACE, (left, right -- res, l, r)) {

INT_INPLACE_OP(left, right, left, *, _PyCompactLong_Multiply);

EXIT_IF(PyStackRef_IsNull(_int_inplace_res));

res = _int_inplace_res;

l = left;

r = right;

INPUTS_DEAD();

}

tier2 op(_BINARY_OP_ADD_INT_INPLACE_RIGHT, (left, right -- res, l, r)) {

INT_INPLACE_OP(left, right, right, +, _PyCompactLong_Add);

EXIT_IF(PyStackRef_IsNull(_int_inplace_res));

res = _int_inplace_res;

l = left;

r = right;

INPUTS_DEAD();

}

tier2 op(_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT, (left, right -- res, l, r)) {

INT_INPLACE_OP(left, right, right, -, _PyCompactLong_Subtract);

EXIT_IF(PyStackRef_IsNull(_int_inplace_res));

res = _int_inplace_res;

l = left;

r = right;

INPUTS_DEAD();

}

tier2 op(_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT, (left, right -- res, l, r)) {

INT_INPLACE_OP(left, right, right, *, _PyCompactLong_Multiply);

EXIT_IF(PyStackRef_IsNull(_int_inplace_res));

res = _int_inplace_res;

l = left;

r = right;

INPUTS_DEAD();

}

op(_GUARD_NOS_FLOAT, (left, unused -- left, unused)) {

PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);

EXIT_IF(!PyFloat_CheckExact(left_o));

}

op(_GUARD_TOS_FLOAT, (value -- value)) {

PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);

EXIT_IF(!PyFloat_CheckExact(value_o));

}

pure op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res, l, r)) {

PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);

PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);

assert(PyFloat_CheckExact(left_o));

assert(PyFloat_CheckExact(right_o));

STAT_INC(BINARY_OP, hit);

double dres =

((PyFloatObject *)left_o)->ob_fval *

((PyFloatObject *)right_o)->ob_fval;

PyObject *d = PyFloat_FromDouble(dres);

if (d == NULL) {

ERROR_NO_POP();

}

res = PyStackRef_FromPyObjectSteal(d);

l = left;

r = right;

INPUTS_DEAD();

}

pure op(_BINARY_OP_ADD_FLOAT, (left, right -- res, l, r)) {

PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);

PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);

assert(PyFloat_CheckExact(left_o));

assert(PyFloat_CheckExact(right_o));

STAT_INC(BINARY_OP, hit);

double dres =

((PyFloatObject *)left_o)->ob_fval +

((PyFloatObject *)right_o)->ob_fval;

PyObject *d = PyFloat_FromDouble(dres);

if (d == NULL) {

ERROR_NO_POP();

}

res = PyStackRef_FromPyObjectSteal(d);

l = left;

r = right;

INPUTS_DEAD();

}

pure op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res, l, r)) {

PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);

PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);

assert(PyFloat_CheckExact(left_o));

assert(PyFloat_CheckExact(right_o));

STAT_INC(BINARY_OP, hit);

double dres =

((PyFloatObject *)left_o)->ob_fval -

((PyFloatObject *)right_o)->ob_fval;

PyObject *d = PyFloat_FromDouble(dres);

if (d == NULL) {

ERROR_NO_POP();

}

res = PyStackRef_FromPyObjectSteal(d);

l = left;

r = right;

INPUTS_DEAD();

}

macro(BINARY_OP_MULTIPLY_FLOAT) =

_GUARD_TOS_FLOAT + _GUARD_NOS_FLOAT + unused/5 + _BINARY_OP_MULTIPLY_FLOAT + _POP_TOP_FLOAT + _POP_TOP_FLOAT;

macro(BINARY_OP_ADD_FLOAT) =

_GUARD_TOS_FLOAT + _GUARD_NOS_FLOAT + unused/5 + _BINARY_OP_ADD_FLOAT + _POP_TOP_FLOAT + _POP_TOP_FLOAT;

macro(BINARY_OP_SUBTRACT_FLOAT) =

_GUARD_TOS_FLOAT + _GUARD_NOS_FLOAT + unused/5 + _BINARY_OP_SUBTRACT_FLOAT + _POP_TOP_FLOAT + _POP_TOP_FLOAT;

// Inplace float ops: mutate the uniquely-referenced left operand

// instead of allocating a new float. Tier 2 only.

// The optimizer sets l to a borrowed value so the following _POP_TOP_FLOAT

// becomes _POP_TOP_NOP.

tier2 op(_BINARY_OP_ADD_FLOAT_INPLACE, (left, right -- res, l, r)) {

FLOAT_INPLACE_OP(left, right, left, +);

res = left;

l = PyStackRef_NULL;

r = right;

INPUTS_DEAD();

}

tier2 op(_BINARY_OP_SUBTRACT_FLOAT_INPLACE, (left, right -- res, l, r)) {

FLOAT_INPLACE_OP(left, right, left, -);

res = left;

l = PyStackRef_NULL;

r = right;

INPUTS_DEAD();

}

tier2 op(_BINARY_OP_MULTIPLY_FLOAT_INPLACE, (left, right -- res, l, r)) {

FLOAT_INPLACE_OP(left, right, left, *);

res = left;

l = PyStackRef_NULL;

r = right;

INPUTS_DEAD();

}

// Inplace RIGHT variants: mutate the uniquely-referenced right operand.

tier2 op(_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT, (left, right -- res, l, r)) {

FLOAT_INPLACE_OP(left, right, right, +);

res = right;

l = left;

r = PyStackRef_NULL;

INPUTS_DEAD();

}

tier2 op(_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT, (left, right -- res, l, r)) {

FLOAT_INPLACE_OP(left, right, right, *);

res = right;

l = left;

r = PyStackRef_NULL;

INPUTS_DEAD();

}

tier2 op(_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT, (left, right -- res, l, r)) {

FLOAT_INPLACE_OP(left, right, right, -);

res = right;

l = left;

r = PyStackRef_NULL;

INPUTS_DEAD();

}

// Float true division --- not specialized at tier 1, emitted by the

// tier 2 optimizer when both operands are known floats.

tier2 op(_BINARY_OP_TRUEDIV_FLOAT, (left, right -- res, l, r)) {

PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);

PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);

assert(PyFloat_CheckExact(left_o));

assert(PyFloat_CheckExact(right_o));

STAT_INC(BINARY_OP, hit);

double divisor = ((PyFloatObject *)right_o)->ob_fval;

if (divisor == 0.0) {

PyErr_SetString(PyExc_ZeroDivisionError,

"float division by zero");

ERROR_NO_POP();

}

double dres = ((PyFloatObject *)left_o)->ob_fval / divisor;

PyObject *d = PyFloat_FromDouble(dres);

if (d == NULL) {

ERROR_NO_POP();

}

res = PyStackRef_FromPyObjectSteal(d);

l = left;

r = right;

INPUTS_DEAD();

}

tier2 op(_BINARY_OP_TRUEDIV_FLOAT_INPLACE, (left, right -- res, l, r)) {

FLOAT_INPLACE_DIVOP(left, right, left);

if (_divop_err) {

ERROR_NO_POP();

}

res = left;

l = PyStackRef_NULL;

r = right;

INPUTS_DEAD();

}

tier2 op(_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT, (left, right -- res, l, r)) {

FLOAT_INPLACE_DIVOP(left, right, right);

if (_divop_err) {

ERROR_NO_POP();

}

res = right;

l = left;

r = PyStackRef_NULL;

INPUTS_DEAD();

}

pure op(_BINARY_OP_ADD_UNICODE, (left, right -- res, l, r)) {

PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);

PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);

assert(PyUnicode_CheckExact(left_o));

assert(PyUnicode_CheckExact(right_o));

STAT_INC(BINARY_OP, hit);

PyObject *res_o = PyUnicode_Concat(left_o, right_o);

if (res_o == NULL) {

ERROR_NO_POP();

}

res = PyStackRef_FromPyObjectSteal(res_o);

l = left;

r = right;

INPUTS_DEAD();

}

macro(BINARY_OP_ADD_UNICODE) =

_GUARD_TOS_UNICODE + _GUARD_NOS_UNICODE + unused/5 + _BINARY_OP_ADD_UNICODE + _POP_TOP_UNICODE + _POP_TOP_UNICODE;

// This is a subtle one. We write NULL to the local

// of the following STORE_FAST and leave the result for STORE_FAST

// later to store.

op(_BINARY_OP_INPLACE_ADD_UNICODE, (left, right -- res)) {

PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);

assert(PyUnicode_CheckExact(left_o));

assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(right)));

int next_oparg;

#if TIER_ONE

assert(next_instr->op.code == STORE_FAST);

next_oparg = next_instr->op.arg;

#else

next_oparg = (int)CURRENT_OPERAND0_16();

#endif

_PyStackRef *target_local = &GETLOCAL(next_oparg);

assert(PyUnicode_CheckExact(left_o));

EXIT_IF(PyStackRef_AsPyObjectBorrow(*target_local) != left_o);

STAT_INC(BINARY_OP, hit);

/* Handle `left = left + right` or `left += right` for str.

assert(Py_REFCNT(left_o) >= 2 || !PyStackRef_IsHeapSafe(left));

PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local);

PyObject *right_o = PyStackRef_AsPyObjectSteal(right);

/* gh-143403: It's critical to close this reference *before*