CPython-compatible marshal format

Unify marshal to a single CPython-compatible format. No separate "cpython_marshal" reader — one format for frozen modules, .pyc files, and the Python-level marshal module. - ComparisonOperator: `(cmp_index << 5) | mask` matching COMPARE_OP - MakeFunctionFlag: bit-position matching SET_FUNCTION_ATTRIBUTE - Exception table varint: big-endian (matching Python/assemble.c) - Linetable varint: little-endian (unchanged) - Integer: TYPE_INT (i32) / TYPE_LONG (base-2^15 digits) - Code objects: CPython field order (argcount, posonlyargcount, ..., co_localsplusnames, co_localspluskinds, ..., co_exceptiontable) - FLAG_REF / TYPE_REF for object deduplication (version >= 3) - allow_code keyword argument on dumps/loads/dump/load - Subclass rejection (int/float/complex/tuple/list/dict/set/frozenset) - Slice serialization (version >= 5) - Buffer protocol fallback for memoryview/array - Recursion depth limit (2000) for both reads and writes - Streaming load (reads one object, seeks file position) - TYPE_INT64, TYPE_FLOAT (text), TYPE_COMPLEX (text) for compat serialize_code writes co_localsplusnames/co_localspluskinds from split varnames/cellvars/freevars. deserialize_code splits them back. Cell variable DEREF indices are translated between flat (wire) and cell-relative (internal) representations in both directions. Replace bitwise trick with match for new ComparisonOperator values. 21 -> 3 expected failures. Remaining: test_bad_reader (IO layer), test_deterministic_sets (PYTHONHASHSEED), testIntern (string interning).
youknowone · Mar 23, 2026 · Mar 20, 2026 · Mar 22, 2026 · Mar 22, 2026 · Mar 23, 2026
commit ce80c08a4c28137e33854c3363d998c9e0254830
diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py
@@ -49,7 +49,6 @@ def test_ints(self):
                 self.helper(expected)
             n = n >> 1
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def test_int64(self):
         # Simulate int marshaling with TYPE_INT64.
         maxint64 = (1 << 63) - 1
@@ -141,7 +140,6 @@ def test_different_filenames(self):
         self.assertEqual(co1.co_filename, "f1")
         self.assertEqual(co2.co_filename, "f2")
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; TypeError: Unexpected keyword argument allow_code
     def test_no_allow_code(self):
         data = {'a': [({0},)]}
         dump = marshal.dumps(data, allow_code=False)
@@ -234,14 +232,12 @@ def test_bytearray(self):
         new = marshal.loads(marshal.dumps(b))
         self.assertEqual(type(new), bytes)
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def test_memoryview(self):
         b = memoryview(b"abc")
         self.helper(b)
         new = marshal.loads(marshal.dumps(b))
         self.assertEqual(type(new), bytes)
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def test_array(self):
         a = array.array('B', b"abc")
         new = marshal.loads(marshal.dumps(a))
@@ -274,7 +270,6 @@ def test_fuzz(self):
             except Exception:
                 pass
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def test_loads_recursion(self):
         def run_tests(N, check):
             # (((...None...),),)
@@ -295,7 +290,7 @@ def check(s):
         run_tests(2**20, check)
 
     @unittest.skipIf(support.is_android, "TODO: RUSTPYTHON; segfault")
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; segfault
+    @unittest.skipIf(os.name == 'nt', "TODO: RUSTPYTHON; write depth limit is 2000 not 1000")
     def test_recursion_limit(self):
         # Create a deeply nested structure.
         head = last = []
@@ -324,7 +319,6 @@ def test_recursion_limit(self):
         last.append([0])
         self.assertRaises(ValueError, marshal.dumps, head)
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def test_exact_type_match(self):
         # Former bug:
         #   >>> class Int(int): pass
@@ -348,7 +342,6 @@ def test_invalid_longs(self):
         invalid_string = b'l\x02\x00\x00\x00\x00\x00\x00\x00'
         self.assertRaises(ValueError, marshal.loads, invalid_string)
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def test_multiple_dumps_and_loads(self):
         # Issue 12291: marshal.load() should be callable multiple times
         # with interleaved data written by non-marshal code
@@ -532,66 +525,56 @@ def helper3(self, rsample, recursive=False, simple=False):
             else:
                 self.assertGreaterEqual(len(s2), len(s3))
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def testInt(self):
         intobj = 123321
         self.helper(intobj)
         self.helper3(intobj, simple=True)
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def testFloat(self):
         floatobj = 1.2345
         self.helper(floatobj)
         self.helper3(floatobj)
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def testStr(self):
         strobj = "abcde"*3
         self.helper(strobj)
         self.helper3(strobj)
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def testBytes(self):
         bytesobj = b"abcde"*3
         self.helper(bytesobj)
         self.helper3(bytesobj)
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def testList(self):
         for obj in self.keys:
             listobj = [obj, obj]
             self.helper(listobj)
             self.helper3(listobj)
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def testTuple(self):
         for obj in self.keys:
             tupleobj = (obj, obj)
             self.helper(tupleobj)
             self.helper3(tupleobj)
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def testSet(self):
         for obj in self.keys:
             setobj = {(obj, 1), (obj, 2)}
             self.helper(setobj)
             self.helper3(setobj)
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def testFrozenSet(self):
         for obj in self.keys:
             frozensetobj = frozenset({(obj, 1), (obj, 2)})
             self.helper(frozensetobj)
             self.helper3(frozensetobj)
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def testDict(self):
         for obj in self.keys:
             dictobj = {"hello": obj, "goodbye": obj, obj: "hello"}
             self.helper(dictobj)
             self.helper3(dictobj)
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def testModule(self):
         with open(__file__, "rb") as f:
             code = f.read()
@@ -651,7 +634,6 @@ def testNoIntern(self):
         self.assertNotEqual(id(s2), id(s))
 
 class SliceTestCase(unittest.TestCase, HelperMixin):
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; NotImplementedError: TODO: not implemented yet or marshal unsupported type
     def test_slice(self):
         for obj in (
             slice(None), slice(1), slice(1, 2), slice(1, 2, 3),

diff --git a/crates/compiler-core/src/bytecode.rs b/crates/compiler-core/src/bytecode.rs
@@ -3,7 +3,7 @@
 
 use crate::{
     marshal::MarshalError,
-    varint::{read_varint, read_varint_with_start, write_varint, write_varint_with_start},
+    varint::{read_varint, read_varint_with_start, write_varint_be, write_varint_with_start},
     {OneIndexed, SourceLocation},
 };
 use alloc::{borrow::ToOwned, boxed::Box, collections::BTreeSet, fmt, string::String, vec::Vec};
@@ -71,9 +71,9 @@ pub fn encode_exception_table(entries: &[ExceptionTableEntry]) -> alloc::boxed::
         let depth_lasti = ((entry.depth as u32) << 1) | (entry.push_lasti as u32);
 
         write_varint_with_start(&mut data, entry.start);
-        write_varint(&mut data, size);
-        write_varint(&mut data, entry.target);
-        write_varint(&mut data, depth_lasti);
+        write_varint_be(&mut data, size);
+        write_varint_be(&mut data, entry.target);
+        write_varint_be(&mut data, depth_lasti);
     }
     data.into_boxed_slice()
 }
@@ -204,7 +204,7 @@ impl PyCodeLocationInfoKind {
     }
 }
 
-pub trait Constant: Sized {
+pub trait Constant: Sized + Clone {
     type Name: AsRef<str>;
 
     /// Transforms the given Constant to a BorrowedConstant
@@ -567,6 +567,14 @@ impl Deref for CodeUnits {
 }
 
 impl CodeUnits {
+    /// Disable adaptive specialization by setting all counters to unreachable.
+    /// Used for CPython-compiled bytecode where specialization may not be safe.
+    pub fn disable_specialization(&self) {
+        for counter in self.adaptive_counters.iter() {
+            counter.store(UNREACHABLE_BACKOFF, Ordering::Relaxed);
+        }
+    }
+
     /// Replace the opcode at `index` in-place without changing the arg byte.
     /// Uses atomic Release store to ensure prior cache writes are visible
     /// to threads that subsequently read the new opcode with Acquire.

diff --git a/crates/compiler-core/src/bytecode/oparg.rs b/crates/compiler-core/src/bytecode/oparg.rs
@@ -382,6 +382,10 @@ oparg_enum!(
 );
 
 bitflagset::bitflag! {
+    /// `SET_FUNCTION_ATTRIBUTE` flags.
+    /// Bitmask: Defaults=0x01, KwOnly=0x02, Annotations=0x04,
+    /// Closure=0x08, TypeParams=0x10, Annotate=0x20.
+    /// Stored as bit position (0-5) by `bitflag!` macro.
     #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
     #[repr(u8)]
     pub enum MakeFunctionFlag {
@@ -426,20 +430,63 @@ impl From<MakeFunctionFlag> for u32 {
 
 impl OpArgType for MakeFunctionFlag {}
 
-oparg_enum!(
-    /// The possible comparison operators.
-    #[derive(Debug, Copy, Clone, PartialEq, Eq)]
-    pub enum ComparisonOperator {
-        // be intentional with bits so that we can do eval_ord with just a bitwise and
-        // bits: | Equal | Greater | Less |
-        Less = 0b001,
-        Greater = 0b010,
-        NotEqual = 0b011,
-        Equal = 0b100,
-        LessOrEqual = 0b101,
-        GreaterOrEqual = 0b110,
+/// `COMPARE_OP` arg is `(cmp_index << 5) | mask`.  Only the upper
+/// 3 bits identify the comparison; the lower 5 bits are an inline
+/// cache mask for adaptive specialization.
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+pub enum ComparisonOperator {
+    Less,
+    LessOrEqual,
+    Equal,
+    NotEqual,
+    Greater,
+    GreaterOrEqual,
+}
+
+impl TryFrom<u8> for ComparisonOperator {
+    type Error = MarshalError;
+    fn try_from(value: u8) -> Result<Self, Self::Error> {
+        Self::try_from(value as u32)
     }
-);
+}
+
+impl TryFrom<u32> for ComparisonOperator {
+    type Error = MarshalError;
+    /// Decode from `COMPARE_OP` arg: `(cmp_index << 5) | mask`.
+    fn try_from(value: u32) -> Result<Self, Self::Error> {
+        match value >> 5 {
+            0 => Ok(Self::Less),
+            1 => Ok(Self::LessOrEqual),
+            2 => Ok(Self::Equal),
+            3 => Ok(Self::NotEqual),
+            4 => Ok(Self::Greater),
+            5 => Ok(Self::GreaterOrEqual),
+            _ => Err(MarshalError::InvalidBytecode),
+        }
+    }
+}
+
+impl From<ComparisonOperator> for u8 {
+    /// Encode as `cmp_index << 5` (mask bits zero).
+    fn from(value: ComparisonOperator) -> Self {
+        match value {
+            ComparisonOperator::Less => 0,
+            ComparisonOperator::LessOrEqual => 1 << 5,
+            ComparisonOperator::Equal => 2 << 5,
+            ComparisonOperator::NotEqual => 3 << 5,
+            ComparisonOperator::Greater => 4 << 5,
+            ComparisonOperator::GreaterOrEqual => 5 << 5,
+        }
+    }
+}
+
+impl From<ComparisonOperator> for u32 {
+    fn from(value: ComparisonOperator) -> Self {
+        Self::from(u8::from(value))
+    }
+}
+
+impl OpArgType for ComparisonOperator {}
 
 oparg_enum!(
     /// The possible Binary operators