Visitar URL original
Keep module cache entries present during import reordering by youknowdot · Pull Request #8948 · RustPython/RustPython · GitHub
Skip to content

Keep module cache entries present during import reordering #8948

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
youknowone merged 3 commits into from
Oct 5, 2026
+447 −2
Merged

Keep module cache entries present during import reordering #8948

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
efe6f37
Reorder the module cache atomically during imports
youknowdot Oct 2, 2026
5dab876
Keep atomic import regression compatible with modern bootstrap
youknowdot Oct 2, 2026
ddace94
Keep the copied bootstrap unchanged in the native relocation PR
youknowdot Oct 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Keep the copied bootstrap unchanged in the native relocation PR 
Restore the exact CPython 3.14 bootstrap and retain only the private exact-dict relocation primitive and its direct native regressions. Remove the PR-added callback integration tests while preserving native helper assertions. This prepares a primitive only; imports do not call it and the import race remains unresolved.

Assisted-by: Codex:model-version-unavailable
  • Loading branch information
@youknowdot
youknowdot committed Oct 5, 2026
commit ddace94901de12c2bc6e951361fb06ca09461d35
25 changes: 8 additions & 17 deletions Lib/importlib/_bootstrap.py
View file
Open in desktop
Comment thread
youknowone marked this conversation as resolved.
Original file line number Diff line number Diff line change
Expand Up @@ -844,19 +844,6 @@ def _module_repr_from_spec(spec):
return f'<module {spec.name!r} ({spec.origin})>'


def _reorder_module(spec):
# RustPython: an exact dict can update shutdown order without exposing a
# temporarily absent module to another importing thread.
modules = sys.modules
if type(modules) is dict and hasattr(_imp, '_dict_move_to_end'):
return _imp._dict_move_to_end(modules, spec.name)
# Preserve custom pop/setitem behavior, including a pop that rebinds
# sys.modules or changes spec.name. CPython also uses this fallback.
module = sys.modules.pop(spec.name)
sys.modules[spec.name] = module
return module


# Used by importlib.reload() and _load_module_shim().
def _exec(spec, module):
"""Execute the spec's specified module in an existing module's namespace."""
Expand All @@ -883,7 +870,8 @@ def _exec(spec, module):
finally:
# Update the order of insertion into sys.modules for module
# clean-up at shutdown.
module = _reorder_module(spec)
module = sys.modules.pop(spec.name)
sys.modules[spec.name] = module
return module


Expand All @@ -894,11 +882,13 @@ def _load_backward_compatible(spec):
spec.loader.load_module(spec.name)
except:
if spec.name in sys.modules:
module = _reorder_module(spec)
module = sys.modules.pop(spec.name)
sys.modules[spec.name] = module
raise
# The module must be in sys.modules at this point!
# Move it to the end of sys.modules.
module = _reorder_module(spec)
module = sys.modules.pop(spec.name)
sys.modules[spec.name] = module
if getattr(module, '__loader__', None) is None:
try:
module.__loader__ = spec.loader
Expand Down Expand Up @@ -956,7 +946,8 @@ def _load_unlocked(spec):
# We don't ensure that the import-related module attributes get
# set in the sys.modules replacement case. Such modules are on
# their own.
module = _reorder_module(spec)
module = sys.modules.pop(spec.name)
sys.modules[spec.name] = module
_verbose_message('import {!r} # {!r}', spec.name, spec.loader)
finally:
spec._initializing = False
Expand Down
2 changes: 1 addition & 1 deletion crates/vm/src/stdlib/_imp.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ mod _imp {

use super::FrozenError;

// Keep sys.modules entries present while importlib updates shutdown order.
// Private exact-dict relocation primitive for future import-cache ordering.
#[pyfunction]
fn _dict_move_to_end(
modules: PyRefExact<PyDict>,
Expand Down
276 changes: 1 addition & 275 deletions extra_tests/snippets/import_atomic_reorder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,7 @@
"""Import-cache ordering must not temporarily remove an exact-dict entry.

The trace-visible presence assertion is a RustPython atomicity invariant, not
an assertion supplied by CPython's old pop/set bootstrap. CPython still checks
the fallback's value, ordering, exception and custom-mapping semantics here.
"""
"""Private exact-dict relocation and callback-reentry regression coverage."""

import _imp
import os
import sys
import types
from importlib import _bootstrap as frozen


def raises(kind, call, expected=None):
Expand All @@ -22,258 +14,6 @@ def raises(kind, call, expected=None):
raise AssertionError(f"{kind.__name__} was not raised")


def source_bootstrap():
path = os.path.join(
os.path.dirname(os.path.dirname(os.path.dirname(__file__))),
"Lib",
"importlib",
"_bootstrap.py",
)
source = types.ModuleType("atomic_reorder_source_bootstrap")
with open(path, encoding="utf-8") as stream:
exec(compile(stream.read(), path, "exec"), source.__dict__)
for name in (
"sys",
"_imp",
"_thread",
"_warnings",
"_weakref",
"_bootstrap_external",
):
setattr(source, name, getattr(frozen, name))
source._blocking_on = source._WeakValueDictionary()
return source


class Loader:
def __init__(self, action):
self.action = action

def create_module(self, spec):
return None

def exec_module(self, module):
self.action(module)


def invoke(bootstrap, action, mode="load"):
name = "_atomic_reorder_test"
spec = bootstrap.ModuleSpec(name, Loader(action))
if mode == "exec":
module = bootstrap.module_from_spec(spec)
sys.modules[name] = module
return bootstrap._exec(spec, module)
return bootstrap._load_unlocked(spec)


def check_trace_presence(bootstrap, mode):
original = sys.modules
old_trace = sys.gettrace()
name = "_atomic_reorder_trace"
expected = types.ModuleType(name)
after = "_atomic_reorder_trace_after"
state = {"armed": False, "events": 0}
failure = RuntimeError("legacy loader failed with its entry still present")
codes = {
bootstrap._exec.__code__,
bootstrap._load_unlocked.__code__,
}
if hasattr(bootstrap, "_load_backward_compatible"):
codes.add(bootstrap._load_backward_compatible.__code__)
if hasattr(bootstrap, "_reorder_module"):
codes.add(bootstrap._reorder_module.__code__)

def trace(frame, event, arg):
if event == "line" and state["armed"] and frame.f_code in codes:
state["events"] += 1
assert name in sys.modules, (mode, frame.f_code.co_name, frame.f_lineno)
assert sys.modules[name] is expected
return trace

def finish(module):
sys.modules[name] = expected
sys.modules[after] = object()
state["armed"] = True

class LegacyLoader:
def load_module(self, fullname):
assert fullname == name
finish(expected)
if mode == "legacy_error":
raise failure
return expected

try:
sys.modules = original.copy()
loader = LegacyLoader() if mode.startswith("legacy") else Loader(finish)
spec = bootstrap.ModuleSpec(name, loader)
if mode == "exec":
initial = bootstrap.module_from_spec(spec)
sys.modules[name] = initial
call = lambda: bootstrap._exec(spec, initial)
elif mode.startswith("legacy"):
call = lambda: bootstrap._load_backward_compatible(spec)
else:
call = lambda: bootstrap._load_unlocked(spec)
sys.settrace(trace)
if mode == "legacy_error":
raises(RuntimeError, call, failure)
else:
assert call() is expected
sys.settrace(old_trace)
assert state["events"] > 0, "the successful loader window was not traced"
assert list(sys.modules)[-2:] == [after, name]
finally:
sys.settrace(old_trace)
sys.modules = original


def check_loader_semantics(bootstrap):
original = sys.modules
name = "_atomic_reorder_test"
after = "_atomic_reorder_after"
try:
for mode in ("load", "exec"):
for replacement in (object(), None):
sys.modules = original.copy()

def replace(module):
sys.modules[name] = replacement
sys.modules[after] = object()

assert invoke(bootstrap, replace, mode) is replacement
assert sys.modules[name] is replacement
assert list(sys.modules)[-2:] == [after, name]

sys.modules = original.copy()

def remove(module):
del sys.modules[name]

exc = raises(KeyError, lambda: invoke(bootstrap, remove, mode))
assert exc.args == (name,)
assert name not in sys.modules

sys.modules = original.copy()
replacement = object()
rebound = {"before": object(), name: replacement, after: object()}

def rebind(module):
sys.modules = rebound

assert invoke(bootstrap, rebind, mode) is replacement
assert sys.modules is rebound
assert list(rebound) == ["before", after, name]

sys.modules = original.copy()
failure = RuntimeError("loader failure after deliberate removal")

def remove_and_fail(module):
del sys.modules[name]
raise failure

raises(RuntimeError, lambda: invoke(bootstrap, remove_and_fail), failure)
assert name not in sys.modules

sys.modules = original.copy()

def fail_with_entry(module):
sys.modules[after] = object()
raise failure

raises(
RuntimeError, lambda: invoke(bootstrap, fail_with_entry, "exec"), failure
)
assert list(sys.modules)[-2:] == [after, name]

if hasattr(bootstrap, "_load_backward_compatible"):

class LegacyLoader:
def load_module(self, fullname):
sys.modules[fullname] = None
del sys.modules[fullname]
raise failure

spec = bootstrap.ModuleSpec(name, LegacyLoader())
sys.modules = original.copy()
raises(
RuntimeError, lambda: bootstrap._load_backward_compatible(spec), failure
)
assert name not in sys.modules
finally:
sys.modules = original


def check_custom_mappings(bootstrap):
original = sys.modules
name = "_atomic_reorder_custom"
value = object()
events = []

class Subclass(dict):
def pop(self, key):
events.append(("pop", key))
return super().pop(key)

def __setitem__(self, key, value):
events.append(("set", key, value))
super().__setitem__(key, value)

class Mapping:
def __init__(self, pairs):
self.data = dict(pairs)

def pop(self, key):
events.append(("pop", key))
return self.data.pop(key)

def __setitem__(self, key, value):
events.append(("set", key, value))
self.data[key] = value

try:
spec = bootstrap.ModuleSpec(name, None)
for cls in (Subclass, Mapping):
events.clear()
modules = cls([(name, value), ("after", None)])
sys.modules = modules
assert bootstrap._reorder_module(spec) is value
assert events == [("pop", name), ("set", name, value)]
assert list(modules if cls is Subclass else modules.data) == ["after", name]

rebound = Subclass()
new_name = "_atomic_reorder_rebound"

class Rebinding(Subclass):
def pop(self, key):
result = super().pop(key)
sys.modules = rebound
spec.name = new_name
return result

events.clear()
modules = Rebinding({name: value})
sys.modules = modules
assert bootstrap._reorder_module(spec) is value
assert events == [("pop", name), ("set", new_name, value)]
assert sys.modules is rebound
assert rebound[new_name] is value
assert name not in modules

failure = KeyError("custom pop exception")

class Raising(Subclass):
def pop(self, key):
raise failure

events.clear()
sys.modules = Raising({new_name: value})
raises(KeyError, lambda: bootstrap._reorder_module(spec), failure)
assert events == []
finally:
sys.modules = original


def check_native_helper():
move = _imp._dict_move_to_end
first, middle, last = object(), object(), object()
Expand Down Expand Up @@ -445,19 +185,5 @@ def __del__(self):
assert list(mapping)[-1] == key


source = source_bootstrap()
for bootstrap in (frozen, source):
if sys.implementation.name == "rustpython":
# Run this before inspecting the new helper so a baseline executable
# fails on the actual pop/set gap, not just a missing private API.
modes = ("load", "exec")
if hasattr(bootstrap, "_load_backward_compatible"):
modes += ("legacy", "legacy_error")
for mode in modes:
check_trace_presence(bootstrap, mode)
check_loader_semantics(bootstrap)
if hasattr(bootstrap, "_reorder_module"):
check_custom_mappings(bootstrap)

if sys.implementation.name == "rustpython":
check_native_helper()
Loading