vm: keep the interpreter registry usable across bootstrap, drop and fork

Three gaps where the registry did not describe the interpreters that actually exist, each of which hides an interpreter from the collector's stop-the-world. Register before `initialize()`. Registration ran as the last step of `initialize_vm`, so the whole bootstrap — which executes Python bytecode and allocates GC-tracked objects — was invisible to `live_interpreter_states()`. It still cannot run any earlier than this: the init hooks take `PyRc::get_mut` on the state, which fails as soon as the registry holds a weak reference to it. Stop unregistering in `Interpreter::drop`. The handle does not decide the interpreter's lifetime — every `ThreadedVirtualMachine` from `new_thread()` holds its own `PyRc<PyGlobalState>` — so an interpreter with running workers disappeared from the registry while its threads kept mutating the object graph. The entries are weak, so lifetime is already correct without the removal; dead entries are now reaped when registering instead. Interpreters are consequently released rather than unregistered at a fixed point, so the two tests asserting disappearance now wait for it: a collection in progress legitimately holds a reference to every live interpreter. Repair other interpreters after fork. `py_os_after_fork_child` only fixed the forking interpreter, leaving every other one with slots for threads that did not survive (still ATTACHED if they were running bytecode) plus locks and stop-the-world flags held by them. Since a collection stops all interpreters, the child's first collection would wait for threads that no longer exist. Reset their locks, stop-the-world state and thread tables, drop this thread's cached slots for them, and reinit the registry's own locks first, since enumerating interpreters now takes them. Tests: test_gc, test_threading and test_fork1 pass, as do the vm tests in both the threading and default configurations. Assisted-by: Claude Code:claude-opus-4-8
youknowone · Aug 16, 2026 · Aug 13, 2026 · Aug 13, 2026 · Aug 13, 2026 · Aug 13, 2026
commit 007999421df736597e834a757d617ddb1bcb00be
diff --git a/crates/vm/src/stdlib/posix.rs b/crates/vm/src/stdlib/posix.rs
@@ -630,6 +630,13 @@ pub mod module {
     }
 
     fn py_os_after_fork_child(vm: &VirtualMachine) {
+        // The interpreter registry is reachable from every thread, so repair it
+        // before anything enumerates interpreters.
+        #[cfg(all(unix, feature = "threading"))]
+        unsafe {
+            crate::vm::runtime::reinit_after_fork()
+        };
+
         #[cfg(feature = "threading")]
         vm.state.stop_the_world.reset_after_fork();
 
@@ -639,6 +646,12 @@ pub mod module {
         #[cfg(feature = "threading")]
         reinit_locks_after_fork(vm);
 
+        // The collector stops every interpreter, so interpreters other than the
+        // forking one must be repaired too; otherwise the child's first
+        // collection waits for threads that did not survive the fork.
+        #[cfg(all(unix, feature = "threading"))]
+        reinit_other_interpreters_after_fork(vm);
+
         // Reinit per-object IO buffer locks on std streams.
         // BufferedReader/Writer/TextIOWrapper use PyThreadMutex which can be
         // held by dead parent threads, causing deadlocks on any IO in the child.
@@ -727,6 +740,53 @@ pub mod module {
         }
     }
 
+    /// Repair every live interpreter other than the forking one after `fork()`.
+    ///
+    /// Only the forking thread survives, so each other interpreter is left with
+    /// slots for threads that no longer exist (still ATTACHED if they were
+    /// running bytecode) and possibly locks or stop-the-world flags held by
+    /// them. Since a collection stops all interpreters, that state would hang
+    /// the child's first collection.
+    ///
+    /// # Safety
+    /// Must only be called after `fork()` in the child, when no other threads exist.
+    #[cfg(all(unix, feature = "threading"))]
+    fn reinit_other_interpreters_after_fork(vm: &VirtualMachine) {
+        use rustpython_common::lock::reinit_mutex_after_fork;
+
+        for state in crate::vm::runtime::live_interpreter_states() {
+            if state.interpreter_id == vm.state.interpreter_id {
+                continue;
+            }
+
+            unsafe {
+                reinit_mutex_after_fork(&state.before_forkers);
+                reinit_mutex_after_fork(&state.after_forkers_child);
+                reinit_mutex_after_fork(&state.after_forkers_parent);
+                reinit_mutex_after_fork(&state.atexit_funcs);
+                reinit_mutex_after_fork(&state.global_trace_func);
+                reinit_mutex_after_fork(&state.global_profile_func);
+                reinit_mutex_after_fork(&state.type_mutex);
+                reinit_mutex_after_fork(&state.monitoring);
+                reinit_mutex_after_fork(&state.thread_frames);
+                reinit_mutex_after_fork(&state.thread_handles);
+                reinit_mutex_after_fork(&state.shutdown_handles);
+
+                state.codec_registry.reinit_after_fork();
+            }
+
+            state.stop_the_world.reset_after_fork();
+
+            // Every thread registered here belongs to the parent, including any
+            // slot the forking thread itself registered before the fork.
+            state.thread_frames.lock().clear();
+            state.thread_handles.lock().clear();
+            state.shutdown_handles.lock().clear();
+        }
+
+        crate::vm::thread::purge_other_interpreter_slots_after_fork(vm.state.interpreter_id);
+    }
+
     fn py_os_after_fork_parent(vm: &VirtualMachine) {
         #[cfg(feature = "threading")]
         vm.state.stop_the_world.start_the_world(&vm.state);

diff --git a/crates/vm/src/vm/interpreter.rs b/crates/vm/src/vm/interpreter.rs
@@ -200,6 +200,13 @@ where
     // Call custom init function (can mutate vm.state)
     init(&mut vm);
 
+    // Register before `initialize()` runs any Python: it allocates GC-tracked
+    // objects, so a collection on another thread has to be able to stop this
+    // interpreter while that happens. It cannot be registered earlier — the
+    // hooks above take `PyRc::get_mut` on the state, which fails once the
+    // registry holds a weak reference to it.
+    runtime::register_interpreter(&vm.state);
+
     // `initialize()` runs Python bytecode directly (e.g. importing `codecs`
     // and `encodings`) before any `enter_vm` scope exists, so attach this
     // thread for the duration so type cache reads see it as ATTACHED.
@@ -209,7 +216,6 @@ where
 
     // Clone global_state for Interpreter after all initialization is done
     let global_state = vm.state.clone();
-    runtime::register_interpreter(&global_state);
     (vm, global_state)
 }
 
@@ -380,12 +386,6 @@ pub struct Interpreter {
     vm: VirtualMachine,
 }
 
-impl Drop for Interpreter {
-    fn drop(&mut self) {
-        runtime::unregister_interpreter(self.global_state.interpreter_id);
-    }
-}
-
 impl Interpreter {
     /// Create a new interpreter configuration builder.
     ///
@@ -812,7 +812,27 @@ mod tests {
         assert!(ids.contains(&sub2.id()));
     }
 
-    /// Dropping a subinterpreter unregisters it; main remains.
+    /// An interpreter stays looked-up-able until nothing holds its state.
+    ///
+    /// Dropping the handle is not the end of its life: `new_thread()` workers
+    /// hold their own reference, and a collection in progress holds one for
+    /// every live interpreter while the world is stopped. So the registry entry
+    /// goes away eventually rather than at the drop.
+    fn wait_until_unregistered(id: i64) {
+        use core::time::Duration;
+        use std::time::Instant;
+
+        let deadline = Instant::now() + Duration::from_secs(5);
+        while runtime::lookup_interpreter(id).is_some() {
+            assert!(
+                Instant::now() < deadline,
+                "interpreter {id} still registered long after its last reference"
+            );
+            std::thread::yield_now();
+        }
+    }
+
+    /// Dropping a subinterpreter releases it; main remains.
     #[test]
     fn drop_subinterpreter_unregisters() {
         let main = Interpreter::without_stdlib(Default::default());
@@ -822,7 +842,7 @@ mod tests {
             assert!(runtime::lookup_interpreter(id).is_some());
             id
         };
-        assert!(runtime::lookup_interpreter(sub_id).is_none());
+        wait_until_unregistered(sub_id);
         assert!(runtime::lookup_interpreter(main.id()).is_some());
     }
 
@@ -1203,9 +1223,9 @@ mod tests {
         assert!(runtime::lookup_interpreter(id).is_some());
         assert!(runtime::take_owned_interpreter(id).is_none());
 
-        // Dropping the reclaimed handle unregisters it.
+        // Dropping the reclaimed handle releases it.
         drop(reclaimed);
-        assert!(runtime::lookup_interpreter(id).is_none());
+        wait_until_unregistered(id);
     }
 
     /// `create_owned_subinterpreter` stores the sub and returns only its id.

diff --git a/crates/vm/src/vm/runtime.rs b/crates/vm/src/vm/runtime.rs
@@ -150,6 +150,11 @@ pub(crate) fn register_interpreter(state: &PyRc<PyGlobalState>) {
         );
     }
     let mut entries = registry().entries.lock();
+    // Entries are weak and an interpreter's lifetime is decided by its last
+    // `PyRc<PyGlobalState>` — which outlives the `Interpreter` handle whenever
+    // `new_thread()` workers are still running — so nothing removes them at a
+    // fixed point. Reap the dead ones here to bound the table instead.
+    entries.retain(|_, entry| entry.state.strong_count() > 0);
     entries.insert(
         id,
         RegistryEntry {
@@ -159,11 +164,6 @@ pub(crate) fn register_interpreter(state: &PyRc<PyGlobalState>) {
     );
 }
 
-/// Unregister an interpreter (called when its owning `Interpreter` is dropped).
-pub(crate) fn unregister_interpreter(id: i64) {
-    registry().entries.lock().remove(&id);
-}
-
 /// Look up a live interpreter state by id.
 #[must_use]
 pub fn lookup_interpreter(id: i64) -> Option<PyRc<PyGlobalState>> {
@@ -198,6 +198,23 @@ pub fn interpreter_count() -> usize {
     list_interpreters().len()
 }
 
+/// Reset the registry's locks after `fork()`.
+///
+/// The tables are reachable from every thread, so a thread that died in the
+/// fork may have left one locked; the child would then deadlock the first time
+/// it enumerates interpreters (which the collector now does on every stop).
+///
+/// # Safety
+/// Must only be called after `fork()` in the child process, when no other
+/// threads exist and the calling thread holds neither lock.
+#[cfg(all(unix, feature = "threading"))]
+pub unsafe fn reinit_after_fork() {
+    unsafe {
+        crate::common::lock::reinit_mutex_after_fork(&registry().entries);
+        crate::common::lock::reinit_mutex_after_fork(owned_interpreters());
+    }
+}
+
 /// All live interpreter states, ordered by id.
 ///
 /// Used by the cyclic collector, which must stop every interpreter's threads

diff --git a/crates/vm/src/vm/thread.rs b/crates/vm/src/vm/thread.rs
@@ -946,6 +946,19 @@ pub fn reinit_frame_slot_after_fork(vm: &VirtualMachine) {
     });
 }
 
+/// Drop this thread's cached slots for every interpreter except `keep_id`.
+///
+/// After `fork()` only the calling thread survives, and the other
+/// interpreters' registries are cleared; a cached slot would otherwise stay
+/// current for an interpreter that no longer lists it, hiding the thread from
+/// that interpreter's stop-the-world. The next enter builds a fresh slot.
+#[cfg(feature = "threading")]
+pub fn purge_other_interpreter_slots_after_fork(keep_id: i64) {
+    INTERP_THREAD_SLOTS.with(|slots| {
+        slots.borrow_mut().retain(|&id, _| id == keep_id);
+    });
+}
+
 pub fn with_vm<F, R>(obj: &PyObject, f: F) -> Option<R>
 where
     F: Fn(&VirtualMachine) -> R,