#include "Python.h"

#include "pycore_ceval.h" // _PyEval_SignalReceived()

#include "pycore_gc.h" // _Py_RunGC()

#include "pycore_initconfig.h" // _PyStatus_OK()

#include "pycore_optimizer.h" // _Py_Executors_InvalidateCold()

#include "pycore_pyerrors.h" // _PyErr_GetRaisedException()

#include "pycore_pylifecycle.h" // _PyErr_Print()

#include "pycore_pystats.h" // _Py_PrintSpecializationStats()

#include "pycore_runtime.h" // _PyRuntime

static inline void

copy_eval_breaker_bits(uintptr_t *from, uintptr_t *to, uintptr_t mask)

{

uintptr_t from_bits = _Py_atomic_load_uintptr_relaxed(from) & mask;

uintptr_t old_value = _Py_atomic_load_uintptr_relaxed(to);

uintptr_t to_bits = old_value & mask;

if (from_bits == to_bits) {

return;

}

uintptr_t new_value;

do {

new_value = (old_value & ~mask) | from_bits;

} while (!_Py_atomic_compare_exchange_uintptr(to, &old_value, new_value));

}

static inline void

update_eval_breaker_for_thread(PyInterpreterState *interp, PyThreadState *tstate)

{

#ifdef Py_GIL_DISABLED

// Free-threaded builds eagerly update the eval_breaker on *all* threads as

// needed, so this function doesn't apply.

return;

int32_t npending = _Py_atomic_load_int32_relaxed(

&interp->ceval.pending.npending);

if (npending) {

_Py_set_eval_breaker_bit(tstate, _PY_CALLS_TO_DO_BIT);

}

else if (_Py_IsMainThread()) {

npending = _Py_atomic_load_int32_relaxed(

&_PyRuntime.ceval.pending_mainthread.npending);

if (npending) {

_Py_set_eval_breaker_bit(tstate, _PY_CALLS_TO_DO_BIT);

}

}

// _PY_CALLS_TO_DO_BIT was derived from other state above, so the only bits

// we copy from our interpreter's state are the instrumentation version.

copy_eval_breaker_bits(&interp->ceval.instrumentation_version,

&tstate->eval_breaker,

~_PY_EVAL_EVENTS_MASK);

}

#include <stdlib.h>

#include <errno.h>

#include "condvar.h"

#define MUTEX_INIT(mut) \

if (PyMUTEX_INIT(&(mut))) { \

Py_FatalError("PyMUTEX_INIT(" #mut ") failed"); };

#define MUTEX_FINI(mut) \

if (PyMUTEX_FINI(&(mut))) { \

Py_FatalError("PyMUTEX_FINI(" #mut ") failed"); };

#define MUTEX_LOCK(mut) \

if (PyMUTEX_LOCK(&(mut))) { \

Py_FatalError("PyMUTEX_LOCK(" #mut ") failed"); };

#define MUTEX_UNLOCK(mut) \

if (PyMUTEX_UNLOCK(&(mut))) { \

Py_FatalError("PyMUTEX_UNLOCK(" #mut ") failed"); };

#define COND_INIT(cond) \

if (PyCOND_INIT(&(cond))) { \

Py_FatalError("PyCOND_INIT(" #cond ") failed"); };

#define COND_FINI(cond) \

if (PyCOND_FINI(&(cond))) { \

Py_FatalError("PyCOND_FINI(" #cond ") failed"); };

#define COND_SIGNAL(cond) \

if (PyCOND_SIGNAL(&(cond))) { \

Py_FatalError("PyCOND_SIGNAL(" #cond ") failed"); };

#define COND_WAIT(cond, mut) \

if (PyCOND_WAIT(&(cond), &(mut))) { \

Py_FatalError("PyCOND_WAIT(" #cond ") failed"); };

#define COND_TIMED_WAIT(cond, mut, microseconds, timeout_result) \

{ \

int r = PyCOND_TIMEDWAIT(&(cond), &(mut), (microseconds)); \

if (r < 0) \

Py_FatalError("PyCOND_WAIT(" #cond ") failed"); \

if (r) /* 1 == timeout, 2 == impl. can't say, so assume timeout */ \

timeout_result = 1; \

else \

timeout_result = 0; \

} \

#define DEFAULT_INTERVAL 5000

static void _gil_initialize(struct _gil_runtime_state *gil)

{

gil->locked = -1;

gil->interval = DEFAULT_INTERVAL;

}

static int gil_created(struct _gil_runtime_state *gil)

{

if (gil == NULL) {

return 0;

}

return (_Py_atomic_load_int_acquire(&gil->locked) >= 0);

}

static void create_gil(struct _gil_runtime_state *gil)

{

MUTEX_INIT(gil->mutex);

#ifdef FORCE_SWITCHING

MUTEX_INIT(gil->switch_mutex);

COND_INIT(gil->cond);

#ifdef FORCE_SWITCHING

COND_INIT(gil->switch_cond);

_Py_atomic_store_ptr_relaxed(&gil->last_holder, 0);

_Py_ANNOTATE_RWLOCK_CREATE(&gil->locked);

_Py_atomic_store_int_release(&gil->locked, 0);

}

static void destroy_gil(struct _gil_runtime_state *gil)

{

/* some pthread-like implementations tie the mutex to the cond

COND_FINI(gil->cond);

MUTEX_FINI(gil->mutex);

#ifdef FORCE_SWITCHING

COND_FINI(gil->switch_cond);

MUTEX_FINI(gil->switch_mutex);

_Py_atomic_store_int_release(&gil->locked, -1);

_Py_ANNOTATE_RWLOCK_DESTROY(&gil->locked);

}

#ifdef HAVE_FORK

static void recreate_gil(struct _gil_runtime_state *gil)

{

_Py_ANNOTATE_RWLOCK_DESTROY(&gil->locked);

/* XXX should we destroy the old OS resources here? */

create_gil(gil);

}

static inline void

drop_gil_impl(PyThreadState *tstate, struct _gil_runtime_state *gil)

{

MUTEX_LOCK(gil->mutex);

_Py_ANNOTATE_RWLOCK_RELEASED(&gil->locked, /*is_write=*/1);

_Py_atomic_store_int_relaxed(&gil->locked, 0);

if (tstate != NULL) {

tstate->holds_gil = 0;

tstate->gil_requested = 0;

}

COND_SIGNAL(gil->cond);

MUTEX_UNLOCK(gil->mutex);

}

static void

drop_gil(PyInterpreterState *interp, PyThreadState *tstate, int final_release)

{

struct _ceval_state *ceval = &interp->ceval;

/* If final_release is true, the caller is indicating that we're releasing

// XXX It may be more correct to check tstate->_status.finalizing.

// XXX assert(final_release || !tstate->_status.cleared);

assert(final_release || tstate != NULL);

struct _gil_runtime_state *gil = ceval->gil;

#ifdef Py_GIL_DISABLED

// Check if we have the GIL before dropping it. tstate will be NULL if

// take_gil() detected that this thread has been destroyed, in which case

// we know we have the GIL.

if (tstate != NULL && !tstate->holds_gil) {

return;

}

if (!_Py_atomic_load_int_relaxed(&gil->locked)) {

Py_FatalError("drop_gil: GIL is not locked");

}

if (!final_release) {

/* Sub-interpreter support: threads might have been switched

_Py_atomic_store_ptr_relaxed(&gil->last_holder, tstate);

}

drop_gil_impl(tstate, gil);

#ifdef FORCE_SWITCHING

/* We might be releasing the GIL for the last time in this thread. In that

if (!final_release &&

_Py_eval_breaker_bit_is_set(tstate, _PY_GIL_DROP_REQUEST_BIT)) {

MUTEX_LOCK(gil->switch_mutex);

/* Not switched yet => wait */

if (((PyThreadState*)_Py_atomic_load_ptr_relaxed(&gil->last_holder)) == tstate)

{

assert(_PyThreadState_CheckConsistency(tstate));

_Py_unset_eval_breaker_bit(tstate, _PY_GIL_DROP_REQUEST_BIT);

/* NOTE: if COND_WAIT does not atomically start waiting when

COND_WAIT(gil->switch_cond, gil->switch_mutex);

}

MUTEX_UNLOCK(gil->switch_mutex);

}

}

static void

take_gil(PyThreadState *tstate)

{

int err = errno;

assert(tstate != NULL);

/* We shouldn't be using a thread state that isn't viable any more. */

// XXX It may be more correct to check tstate->_status.finalizing.

// XXX assert(!tstate->_status.cleared);

if (_PyThreadState_MustExit(tstate)) {

/* bpo-39877: If Py_Finalize() has been called and tstate is not the

_PyThreadState_HangThread(tstate);

}

assert(_PyThreadState_CheckConsistency(tstate));

PyInterpreterState *interp = tstate->interp;

struct _gil_runtime_state *gil = interp->ceval.gil;

#ifdef Py_GIL_DISABLED

if (!_Py_atomic_load_int_relaxed(&gil->enabled)) {

return;

}

/* Check that _PyEval_InitThreads() was called to create the lock */

assert(gil_created(gil));

MUTEX_LOCK(gil->mutex);

tstate->gil_requested = 1;

int drop_requested = 0;

while (_Py_atomic_load_int_relaxed(&gil->locked)) {

unsigned long saved_switchnum = gil->switch_number;

unsigned long interval = _Py_atomic_load_ulong_relaxed(&gil->interval);

if (interval < 1) {

interval = 1;

}

int timed_out = 0;

COND_TIMED_WAIT(gil->cond, gil->mutex, interval, timed_out);

/* If we timed out and no switch occurred in the meantime, it is time

if (timed_out &&

_Py_atomic_load_int_relaxed(&gil->locked) &&

gil->switch_number == saved_switchnum)

{

PyThreadState *holder_tstate =

(PyThreadState*)_Py_atomic_load_ptr_relaxed(&gil->last_holder);

if (_PyThreadState_MustExit(tstate)) {

MUTEX_UNLOCK(gil->mutex);

// gh-96387: If the loop requested a drop request in a previous

// iteration, reset the request. Otherwise, drop_gil() can

// block forever waiting for the thread which exited. Drop

// requests made by other threads are also reset: these threads

// may have to request again a drop request (iterate one more

// time).

if (drop_requested) {

_Py_unset_eval_breaker_bit(holder_tstate, _PY_GIL_DROP_REQUEST_BIT);

}

// gh-87135: hang the thread as *thread_exit() is not a safe

// API. It lacks stack unwind and local variable destruction.

_PyThreadState_HangThread(tstate);

}

assert(_PyThreadState_CheckConsistency(tstate));

_Py_set_eval_breaker_bit(holder_tstate, _PY_GIL_DROP_REQUEST_BIT);

drop_requested = 1;

}

}

#ifdef Py_GIL_DISABLED

if (!_Py_atomic_load_int_relaxed(&gil->enabled)) {

// Another thread disabled the GIL between our check above and

// now. Don't take the GIL, signal any other waiting threads, and

// return.

COND_SIGNAL(gil->cond);

MUTEX_UNLOCK(gil->mutex);

return;

}

#ifdef FORCE_SWITCHING

/* This mutex must be taken before modifying gil->last_holder:

MUTEX_LOCK(gil->switch_mutex);

/* We now hold the GIL */

_Py_atomic_store_int_relaxed(&gil->locked, 1);

_Py_ANNOTATE_RWLOCK_ACQUIRED(&gil->locked, /*is_write=*/1);

if (tstate != (PyThreadState*)_Py_atomic_load_ptr_relaxed(&gil->last_holder)) {

_Py_atomic_store_ptr_relaxed(&gil->last_holder, tstate);

++gil->switch_number;

}

#ifdef FORCE_SWITCHING

COND_SIGNAL(gil->switch_cond);

MUTEX_UNLOCK(gil->switch_mutex);

if (_PyThreadState_MustExit(tstate)) {

/* bpo-36475: If Py_Finalize() has been called and tstate is not

MUTEX_UNLOCK(gil->mutex);

/* tstate could be a dangling pointer, so don't pass it to

drop_gil(interp, NULL, 1);

_PyThreadState_HangThread(tstate);

}

assert(_PyThreadState_CheckConsistency(tstate));

tstate->gil_requested = 0;

tstate->holds_gil = 1;

_Py_unset_eval_breaker_bit(tstate, _PY_GIL_DROP_REQUEST_BIT);

update_eval_breaker_for_thread(interp, tstate);

MUTEX_UNLOCK(gil->mutex);

errno = err;

return;

}

void _PyEval_SetSwitchInterval(unsigned long microseconds)

{

PyInterpreterState *interp = _PyInterpreterState_GET();

struct _gil_runtime_state *gil = interp->ceval.gil;

assert(gil != NULL);

_Py_atomic_store_ulong_relaxed(&gil->interval, microseconds);

}

unsigned long _PyEval_GetSwitchInterval(void)

{

PyInterpreterState *interp = _PyInterpreterState_GET();

struct _gil_runtime_state *gil = interp->ceval.gil;

assert(gil != NULL);

return _Py_atomic_load_ulong_relaxed(&gil->interval);

}

_PyEval_ThreadsInitialized(void)

{

/* XXX This is only needed for an assert in PyGILState_Ensure(),

PyInterpreterState *interp = _PyInterpreterState_Main();

if (interp == NULL) {

return 0;

}

struct _gil_runtime_state *gil = interp->ceval.gil;

return gil_created(gil);

}

PyAPI_FUNC(int)

PyEval_ThreadsInitialized(void)

{

return _PyEval_ThreadsInitialized();

}

#ifndef NDEBUG

static inline int

current_thread_holds_gil(struct _gil_runtime_state *gil, PyThreadState *tstate)

{

int holds_gil = tstate->holds_gil;

// holds_gil is the source of truth; check that last_holder and gil->locked

// are consistent with it.

int locked = _Py_atomic_load_int_relaxed(&gil->locked);

int is_last_holder =

((PyThreadState*)_Py_atomic_load_ptr_relaxed(&gil->last_holder)) == tstate;

assert(!holds_gil || locked);

assert(!holds_gil || is_last_holder);

return holds_gil;

}

static void

init_shared_gil(PyInterpreterState *interp, struct _gil_runtime_state *gil)

{

assert(gil_created(gil));

interp->ceval.gil = gil;

interp->ceval.own_gil = 0;

}

static void

init_own_gil(PyInterpreterState *interp, struct _gil_runtime_state *gil)

{

assert(!gil_created(gil));

#ifdef Py_GIL_DISABLED

const PyConfig *config = _PyInterpreterState_GetConfig(interp);

gil->enabled = config->enable_gil == _PyConfig_GIL_ENABLE ? INT_MAX : 0;

create_gil(gil);

assert(gil_created(gil));

interp->ceval.gil = gil;

interp->ceval.own_gil = 1;

}

_PyEval_InitGIL(PyThreadState *tstate, int own_gil)

{

assert(tstate->interp->ceval.gil == NULL);

if (!own_gil) {

/* The interpreter will share the main interpreter's instead. */

PyInterpreterState *main_interp = _PyInterpreterState_Main();

assert(tstate->interp != main_interp);

struct _gil_runtime_state *gil = main_interp->ceval.gil;

init_shared_gil(tstate->interp, gil);

assert(!current_thread_holds_gil(gil, tstate));

}

else {

PyThread_init_thread();

init_own_gil(tstate->interp, &tstate->interp->_gil);

}

// Lock the GIL and mark the current thread as attached.

_PyThreadState_Attach(tstate);

}

_PyEval_FiniGIL(PyInterpreterState *interp)

{

struct _gil_runtime_state *gil = interp->ceval.gil;

if (gil == NULL) {

/* It was already finalized (or hasn't been initialized yet). */

assert(!interp->ceval.own_gil);

return;

}

else if (!interp->ceval.own_gil) {

#ifdef Py_DEBUG

PyInterpreterState *main_interp = _PyInterpreterState_Main();

assert(main_interp != NULL && interp != main_interp);

assert(interp->ceval.gil == main_interp->ceval.gil);

interp->ceval.gil = NULL;

return;

}

if (!gil_created(gil)) {

/* First Py_InitializeFromConfig() call: the GIL doesn't exist

return;

}

destroy_gil(gil);

assert(!gil_created(gil));

interp->ceval.gil = NULL;

}

PyAPI_FUNC(void)

PyEval_InitThreads(void)

{

/* Do nothing: kept for backward compatibility */

}

_PyEval_Fini(void)

{

#ifdef Py_STATS

_Py_PrintSpecializationStats(1);

}

PyAPI_FUNC(void)

PyEval_AcquireLock(void)

{

PyThreadState *tstate = _PyThreadState_GET();

_Py_EnsureTstateNotNULL(tstate);

take_gil(tstate);

}

PyAPI_FUNC(void)

PyEval_ReleaseLock(void)

{

PyThreadState *tstate = _PyThreadState_GET();

/* This function must succeed when the current thread state is NULL.

drop_gil(tstate->interp, tstate, 0);

}

_PyEval_AcquireLock(PyThreadState *tstate)

{

_Py_EnsureTstateNotNULL(tstate);

take_gil(tstate);

}

_PyEval_ReleaseLock(PyInterpreterState *interp,

PyThreadState *tstate,

int final_release)

{

assert(tstate != NULL);

assert(tstate->interp == interp);

drop_gil(interp, tstate, final_release);

}

PyEval_AcquireThread(PyThreadState *tstate)

{

_Py_EnsureTstateNotNULL(tstate);

_PyThreadState_Attach(tstate);

}

PyEval_ReleaseThread(PyThreadState *tstate)

{

assert(_PyThreadState_CheckConsistency(tstate));

_PyThreadState_Detach(tstate);

}

#ifdef HAVE_FORK

_PyEval_ReInitThreads(PyThreadState *tstate)

{

assert(tstate->interp == _PyInterpreterState_Main());

struct _gil_runtime_state *gil = tstate->interp->ceval.gil;

if (!gil_created(gil)) {

return _PyStatus_OK();

}

recreate_gil(gil);

take_gil(tstate);

struct _pending_calls *pending = &tstate->interp->ceval.pending;

_PyMutex_at_fork_reinit(&pending->mutex);

return _PyStatus_OK();

}

PyThreadState *

PyEval_SaveThread(void)

{

PyThreadState *tstate = _PyThreadState_GET();

_PyThreadState_Detach(tstate);

return tstate;

}

PyEval_RestoreThread(PyThreadState *tstate)

{

#ifdef MS_WINDOWS

int err = GetLastError();

_Py_EnsureTstateNotNULL(tstate);

_PyThreadState_Attach(tstate);

#ifdef MS_WINDOWS

SetLastError(err);

}

_PyEval_SignalReceived(void)

{

_Py_set_eval_breaker_bit(_PyRuntime.main_tstate, _PY_SIGNALS_PENDING_BIT);

}

#ifndef Py_GIL_DISABLED

static void

signal_active_thread(PyInterpreterState *interp, uintptr_t bit)

{

struct _gil_runtime_state *gil = interp->ceval.gil;

// If a thread from the targeted interpreter is holding the GIL, signal

// that thread. Otherwise, the next thread to run from the targeted

// interpreter will have its bit set as part of taking the GIL.

MUTEX_LOCK(gil->mutex);

if (_Py_atomic_load_int_relaxed(&gil->locked)) {

PyThreadState *holder = (PyThreadState*)_Py_atomic_load_ptr_relaxed(&gil->last_holder);

if (holder->interp == interp) {

_Py_set_eval_breaker_bit(holder, bit);

}

}

MUTEX_UNLOCK(gil->mutex);

}

static int

_push_pending_call(struct _pending_calls *pending,

_Py_pending_call_func func, void *arg, int flags)

{

if (pending->npending == pending->max) {

return _Py_ADD_PENDING_FULL;

}

assert(pending->npending < pending->max);

int i = pending->next;

assert(pending->calls[i].func == NULL);

pending->calls[i].func = func;

pending->calls[i].arg = arg;

pending->calls[i].flags = flags;

assert(pending->npending < PENDINGCALLSARRAYSIZE);

_Py_atomic_add_int32(&pending->npending, 1);

pending->next = (i + 1) % PENDINGCALLSARRAYSIZE;

assert(pending->next != pending->first

|| pending->npending == pending->max);

return _Py_ADD_PENDING_SUCCESS;

}

static int

_next_pending_call(struct _pending_calls *pending,

int (**func)(void *), void **arg, int *flags)

{

int i = pending->first;

if (pending->npending == 0) {

/* Queue empty */

assert(i == pending->next);

assert(pending->calls[i].func == NULL);

return -1;

}

*func = pending->calls[i].func;

*arg = pending->calls[i].arg;

*flags = pending->calls[i].flags;

return i;

}

static void

_pop_pending_call(struct _pending_calls *pending,

int (**func)(void *), void **arg, int *flags)

{

int i = _next_pending_call(pending, func, arg, flags);

if (i >= 0) {

pending->calls[i] = (struct _pending_call){0};

pending->first = (i + 1) % PENDINGCALLSARRAYSIZE;

assert(pending->npending > 0);

_Py_atomic_add_int32(&pending->npending, -1);

}

}

_PyEval_AddPendingCall(PyInterpreterState *interp,

_Py_pending_call_func func, void *arg, int flags)

{

struct _pending_calls *pending = &interp->ceval.pending;

int main_only = (flags & _Py_PENDING_MAINTHREADONLY) != 0;

if (main_only) {

/* The main thread only exists in the main interpreter. */

assert(_Py_IsMainInterpreter(interp));

pending = &_PyRuntime.ceval.pending_mainthread;

}

PyMutex_Lock(&pending->mutex);

_Py_add_pending_call_result result =

_push_pending_call(pending, func, arg, flags);

PyMutex_Unlock(&pending->mutex);

if (main_only) {

_Py_set_eval_breaker_bit(_PyRuntime.main_tstate, _PY_CALLS_TO_DO_BIT);

}

else {

#ifdef Py_GIL_DISABLED

_Py_set_eval_breaker_bit_all(interp, _PY_CALLS_TO_DO_BIT);

signal_active_thread(interp, _PY_CALLS_TO_DO_BIT);

}

return result;

}

Py_AddPendingCall(_Py_pending_call_func func, void *arg)

{

/* Legacy users of this API will continue to target the main thread

PyInterpreterState *interp = _PyInterpreterState_Main();

_Py_add_pending_call_result r =

_PyEval_AddPendingCall(interp, func, arg, _Py_PENDING_MAINTHREADONLY);

if (r == _Py_ADD_PENDING_FULL) {

return -1;

}

else {

assert(r == _Py_ADD_PENDING_SUCCESS);

return 0;

}

}

static int

handle_signals(PyThreadState *tstate)

{

assert(_PyThreadState_CheckConsistency(tstate));

_Py_unset_eval_breaker_bit(tstate, _PY_SIGNALS_PENDING_BIT);

if (!_Py_ThreadCanHandleSignals(tstate->interp)) {

return 0;

}

if (_PyErr_CheckSignalsTstate(tstate) < 0) {

/* On failure, re-schedule a call to handle_signals(). */

_Py_set_eval_breaker_bit(tstate, _PY_SIGNALS_PENDING_BIT);

return -1;

}

return 0;

}

static int

_make_pending_calls(struct _pending_calls *pending, int32_t *p_npending)

{

int res = 0;

int32_t npending = -1;

assert(sizeof(pending->max) <= sizeof(size_t)

&& ((size_t)pending->max) <= Py_ARRAY_LENGTH(pending->calls));

int32_t maxloop = pending->maxloop;

if (maxloop == 0) {

maxloop = pending->max;

}

assert(maxloop > 0 && maxloop <= pending->max);

/* perform a bounded number of calls, in case of recursion */

for (int i=0; i<maxloop; i++) {

_Py_pending_call_func func = NULL;

void *arg = NULL;

int flags = 0;

/* pop one item off the queue while holding the lock */

PyMutex_Lock(&pending->mutex);

_pop_pending_call(pending, &func, &arg, &flags);

npending = pending->npending;

PyMutex_Unlock(&pending->mutex);

/* Check if there are any more pending calls. */

if (func == NULL) {

assert(npending == 0);

break;

}

/* having released the lock, perform the callback */

res = func(arg);

if ((flags & _Py_PENDING_RAWFREE) && arg != NULL) {

PyMem_RawFree(arg);

}

if (res != 0) {

res = -1;

goto finally;

}

}

finally:

*p_npending = npending;

return res;

}

static void

signal_pending_calls(PyThreadState *tstate, PyInterpreterState *interp)

{

#ifdef Py_GIL_DISABLED

_Py_set_eval_breaker_bit_all(interp, _PY_CALLS_TO_DO_BIT);

_Py_set_eval_breaker_bit(tstate, _PY_CALLS_TO_DO_BIT);

}

static void

unsignal_pending_calls(PyThreadState *tstate, PyInterpreterState *interp)

{

#ifdef Py_GIL_DISABLED

_Py_unset_eval_breaker_bit_all(interp, _PY_CALLS_TO_DO_BIT);

_Py_unset_eval_breaker_bit(tstate, _PY_CALLS_TO_DO_BIT);

}

static void

clear_pending_handling_thread(struct _pending_calls *pending)

{

FT_MUTEX_LOCK(&pending->mutex);

pending->handling_thread = NULL;

FT_MUTEX_UNLOCK(&pending->mutex);

}

static int

make_pending_calls(PyThreadState *tstate)

{

PyInterpreterState *interp = tstate->interp;

struct _pending_calls *pending = &interp->ceval.pending;

struct _pending_calls *pending_main = &_PyRuntime.ceval.pending_mainthread;

/* Only one thread (per interpreter) may run the pending calls

PyMutex_Lock(&pending->mutex);

if (pending->handling_thread != NULL) {

/* A pending call was added after another thread was already

_Py_set_eval_breaker_bit(pending->handling_thread, _PY_CALLS_TO_DO_BIT);

_Py_unset_eval_breaker_bit(tstate, _PY_CALLS_TO_DO_BIT);

PyMutex_Unlock(&pending->mutex);

return 0;

}

pending->handling_thread = tstate;

PyMutex_Unlock(&pending->mutex);

/* unsignal before starting to call callbacks, so that any callback

unsignal_pending_calls(tstate, interp);

int32_t npending;

if (_make_pending_calls(pending, &npending) != 0) {

clear_pending_handling_thread(pending);

/* There might not be more calls to make, but we play it safe. */

signal_pending_calls(tstate, interp);

return -1;

}

if (npending > 0) {

/* We hit pending->maxloop. */

signal_pending_calls(tstate, interp);

}

if (_Py_IsMainThread() && _Py_IsMainInterpreter(interp)) {

if (_make_pending_calls(pending_main, &npending) != 0) {

clear_pending_handling_thread(pending);

/* There might not be more calls to make, but we play it safe. */

signal_pending_calls(tstate, interp);

return -1;

}

if (npending > 0) {

/* We hit pending_main->maxloop. */

signal_pending_calls(tstate, interp);

}

}

clear_pending_handling_thread(pending);

return 0;

}

_Py_set_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit)

{

_Py_FOR_EACH_TSTATE_BEGIN(interp, tstate) {

_Py_set_eval_breaker_bit(tstate, bit);

}

_Py_FOR_EACH_TSTATE_END(interp);

}

_Py_unset_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit)

{

_Py_FOR_EACH_TSTATE_BEGIN(interp, tstate) {

_Py_unset_eval_breaker_bit(tstate, bit);

}

_Py_FOR_EACH_TSTATE_END(interp);

}

_Py_FinishPendingCalls(PyThreadState *tstate)

{