Visitar URL original
gh-101659: Isolate "obmalloc" State to Each Interpreter (gh-101660) · python/cpython@df3173d · GitHub
Skip to content

Commit df3173d

Browse files
ericsnowcurrentlyericsnowcurrently
authored
gh-101659: Isolate "obmalloc" State to Each Interpreter (gh-101660)
This is strictly about moving the "obmalloc" runtime state from `_PyRuntimeState` to `PyInterpreterState`. Doing so improves isolation between interpreters, specifically most of the memory (incl. objects) allocated for each interpreter's use. This is important for a per-interpreter GIL, but such isolation is valuable even without it. FWIW, a per-interpreter obmalloc is the proverbial canary-in-the-coalmine when it comes to the isolation of objects between interpreters. Any object that leaks (unintentionally) to another interpreter is highly likely to cause a crash (on debug builds at least). That's a useful thing to know, relative to interpreter isolation.
1 parent 01be52e commit df3173d

20 files changed

Lines changed: 322 additions & 73 deletions

‎Include/cpython/initconfig.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -245,6 +245,8 @@ PyAPI_FUNC(PyStatus) PyConfig_SetWideStringList(PyConfig *config,
245245
/* --- PyInterpreterConfig ------------------------------------ */
246246

247247
typedef struct {
248+
// XXX "allow_object_sharing"? "own_objects"?
249+
int use_main_obmalloc;
248250
int allow_fork;
249251
int allow_exec;
250252
int allow_threads;
@@ -254,6 +256,7 @@ typedef struct {
254256

255257
#define _PyInterpreterConfig_INIT \
256258
{ \
259+
.use_main_obmalloc = 0, \
257260
.allow_fork = 0, \
258261
.allow_exec = 0, \
259262
.allow_threads = 1, \
@@ -263,6 +266,7 @@ typedef struct {
263266

264267
#define _PyInterpreterConfig_LEGACY_INIT \
265268
{ \
269+
.use_main_obmalloc = 1, \
266270
.allow_fork = 1, \
267271
.allow_exec = 1, \
268272
.allow_threads = 1, \

‎Include/cpython/pystate.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,10 @@ is available in a given context. For example, forking the process
1111
might not be allowed in the current interpreter (i.e. os.fork() would fail).
1212
*/
1313

14+
/* Set if the interpreter share obmalloc runtime state
15+
with the main interpreter. */
16+
#define Py_RTFLAGS_USE_MAIN_OBMALLOC (1UL << 5)
17+
1418
/* Set if import should check a module for subinterpreter support. */
1519
#define Py_RTFLAGS_MULTI_INTERP_EXTENSIONS (1UL << 8)
1620

‎Include/internal/pycore_interp.h‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,12 @@ extern "C" {
2323
#include "pycore_function.h" // FUNC_MAX_WATCHERS
2424
#include "pycore_genobject.h" // struct _Py_async_gen_state
2525
#include "pycore_gc.h" // struct _gc_runtime_state
26+
#include "pycore_global_objects.h" // struct _Py_interp_static_objects
2627
#include "pycore_import.h" // struct _import_state
2728
#include "pycore_instruments.h" // PY_MONITORING_EVENTS
2829
#include "pycore_list.h" // struct _Py_list_state
29-
#include "pycore_global_objects.h" // struct _Py_interp_static_objects
3030
#include "pycore_object_state.h" // struct _py_object_state
31+
#include "pycore_obmalloc.h" // struct obmalloc_state
3132
#include "pycore_tuple.h" // struct _Py_tuple_state
3233
#include "pycore_typeobject.h" // struct type_cache
3334
#include "pycore_unicodeobject.h" // struct _Py_unicode_state
@@ -82,6 +83,8 @@ struct _is {
8283
int _initialized;
8384
int finalizing;
8485

86+
struct _obmalloc_state obmalloc;
87+
8588
struct _ceval_state ceval;
8689
struct _gc_runtime_state gc;
8790

‎Include/internal/pycore_obmalloc.h‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -657,8 +657,12 @@ struct _obmalloc_usage {
657657
#endif /* WITH_PYMALLOC_RADIX_TREE */
658658

659659

660-
struct _obmalloc_state {
660+
struct _obmalloc_global_state {
661661
int dump_debug_stats;
662+
Py_ssize_t interpreter_leaks;
663+
};
664+
665+
struct _obmalloc_state {
662666
struct _obmalloc_pools pools;
663667
struct _obmalloc_mgmt mgmt;
664668
struct _obmalloc_usage usage;
@@ -675,7 +679,11 @@ void _PyObject_VirtualFree(void *, size_t size);
675679

676680

677681
/* This function returns the number of allocated memory blocks, regardless of size */
678-
PyAPI_FUNC(Py_ssize_t) _Py_GetAllocatedBlocks(void);
682+
extern Py_ssize_t _Py_GetGlobalAllocatedBlocks(void);
683+
#define _Py_GetAllocatedBlocks() \
684+
_Py_GetGlobalAllocatedBlocks()
685+
extern Py_ssize_t _PyInterpreterState_GetAllocatedBlocks(PyInterpreterState *);
686+
extern void _PyInterpreterState_FinalizeAllocatedBlocks(PyInterpreterState *);
679687

680688

681689
#ifdef WITH_PYMALLOC

‎Include/internal/pycore_obmalloc_init.h‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,9 +54,13 @@ extern "C" {
5454
# error "NB_SMALL_SIZE_CLASSES should be less than 64"
5555
#endif
5656

57-
#define _obmalloc_state_INIT(obmalloc) \
57+
#define _obmalloc_global_state_INIT \
5858
{ \
5959
.dump_debug_stats = -1, \
60+
}
61+
62+
#define _obmalloc_state_INIT(obmalloc) \
63+
{ \
6064
.pools = { \
6165
.used = _obmalloc_pools_INIT(obmalloc.pools), \
6266
}, \

‎Include/internal/pycore_pylifecycle.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ extern void _PyAtExit_Fini(PyInterpreterState *interp);
6464
extern void _PyThread_FiniType(PyInterpreterState *interp);
6565
extern void _Py_Deepfreeze_Fini(void);
6666
extern void _PyArg_Fini(void);
67+
extern void _Py_FinalizeAllocatedBlocks(_PyRuntimeState *);
6768

6869
extern PyStatus _PyGILState_Init(PyInterpreterState *interp);
6970
extern PyStatus _PyGILState_SetTstate(PyThreadState *tstate);

‎Include/internal/pycore_pystate.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,13 @@ _Py_IsMainInterpreter(PyInterpreterState *interp)
3333
return (interp == _PyInterpreterState_Main());
3434
}
3535

36+
static inline int
37+
_Py_IsMainInterpreterFinalizing(PyInterpreterState *interp)
38+
{
39+
return (_PyRuntimeState_GetFinalizing(interp->runtime) != NULL &&
40+
interp == &interp->runtime->_main_interpreter);
41+
}
42+
3643

3744
static inline const PyConfig *
3845
_Py_GetMainConfig(void)

‎Include/internal/pycore_runtime.h‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ extern "C" {
2121
#include "pycore_pymem.h" // struct _pymem_allocators
2222
#include "pycore_pyhash.h" // struct pyhash_runtime_state
2323
#include "pycore_pythread.h" // struct _pythread_runtime_state
24-
#include "pycore_obmalloc.h" // struct obmalloc_state
2524
#include "pycore_signal.h" // struct _signals_runtime_state
2625
#include "pycore_time.h" // struct _time_runtime_state
2726
#include "pycore_tracemalloc.h" // struct _tracemalloc_runtime_state
@@ -88,7 +87,7 @@ typedef struct pyruntimestate {
8887
_Py_atomic_address _finalizing;
8988

9089
struct _pymem_allocators allocators;
91-
struct _obmalloc_state obmalloc;
90+
struct _obmalloc_global_state obmalloc;
9291
struct pyhash_runtime_state pyhash_state;
9392
struct _time_runtime_state time;
9493
struct _pythread_runtime_state threads;

‎Include/internal/pycore_runtime_init.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ extern PyTypeObject _PyExc_MemoryError;
2929
_pymem_allocators_debug_INIT, \
3030
_pymem_allocators_obj_arena_INIT, \
3131
}, \
32-
.obmalloc = _obmalloc_state_INIT(runtime.obmalloc), \
32+
.obmalloc = _obmalloc_global_state_INIT, \
3333
.pyhash_state = pyhash_state_INIT, \
3434
.signals = _signals_RUNTIME_INIT, \
3535
.interpreters = { \
@@ -93,6 +93,7 @@ extern PyTypeObject _PyExc_MemoryError;
9393
{ \
9494
.id_refcount = -1, \
9595
.imports = IMPORTS_INIT, \
96+
.obmalloc = _obmalloc_state_INIT(INTERP.obmalloc), \
9697
.ceval = { \
9798
.recursion_limit = Py_DEFAULT_RECURSION_LIMIT, \
9899
}, \

‎Lib/test/test_capi/test_misc.py‎

Lines changed: 27 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1211,20 +1211,25 @@ def test_configured_settings(self):
12111211
"""
12121212
import json
12131213

1214+
OBMALLOC = 1<<5
12141215
EXTENSIONS = 1<<8
12151216
THREADS = 1<<10
12161217
DAEMON_THREADS = 1<<11
12171218
FORK = 1<<15
12181219
EXEC = 1<<16
12191220

1220-
features = ['fork', 'exec', 'threads', 'daemon_threads', 'extensions']
1221+
features = ['obmalloc', 'fork', 'exec', 'threads', 'daemon_threads',
1222+
'extensions']
12211223
kwlist = [f'allow_{n}' for n in features]
1224+
kwlist[0] = 'use_main_obmalloc'
12221225
kwlist[-1] = 'check_multi_interp_extensions'
1226+
1227+
# expected to work
12231228
for config, expected in {
1224-
(True, True, True, True, True):
1225-
FORK | EXEC | THREADS | DAEMON_THREADS | EXTENSIONS,
1226-
(False, False, False, False, False): 0,
1227-
(False, False, True, False, True): THREADS | EXTENSIONS,
1229+
(True, True, True, True, True, True):
1230+
OBMALLOC | FORK | EXEC | THREADS | DAEMON_THREADS | EXTENSIONS,
1231+
(True, False, False, False, False, False): OBMALLOC,
1232+
(False, False, False, True, False, True): THREADS | EXTENSIONS,
12281233
}.items():
12291234
kwargs = dict(zip(kwlist, config))
12301235
expected = {
@@ -1246,6 +1251,20 @@ def test_configured_settings(self):
12461251

12471252
self.assertEqual(settings, expected)
12481253

1254+
# expected to fail
1255+
for config in [
1256+
(False, False, False, False, False, False),
1257+
]:
1258+
kwargs = dict(zip(kwlist, config))
1259+
with self.subTest(config):
1260+
script = textwrap.dedent(f'''
1261+
import _testinternalcapi
1262+
_testinternalcapi.get_interp_settings()
1263+
raise NotImplementedError('unreachable')
1264+
''')
1265+
with self.assertRaises(RuntimeError):
1266+
support.run_in_subinterp_with_config(script, **kwargs)
1267+
12491268
@unittest.skipIf(_testsinglephase is None, "test requires _testsinglephase module")
12501269
@unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()")
12511270
def test_overridden_setting_extensions_subinterp_check(self):
@@ -1257,13 +1276,15 @@ def test_overridden_setting_extensions_subinterp_check(self):
12571276
"""
12581277
import json
12591278

1279+
OBMALLOC = 1<<5
12601280
EXTENSIONS = 1<<8
12611281
THREADS = 1<<10
12621282
DAEMON_THREADS = 1<<11
12631283
FORK = 1<<15
12641284
EXEC = 1<<16
1265-
BASE_FLAGS = FORK | EXEC | THREADS | DAEMON_THREADS
1285+
BASE_FLAGS = OBMALLOC | FORK | EXEC | THREADS | DAEMON_THREADS
12661286
base_kwargs = {
1287+
'use_main_obmalloc': True,
12671288
'allow_fork': True,
12681289
'allow_exec': True,
12691290
'allow_threads': True,

0 commit comments

Comments
 (0)