Visitar URL original
Stackless issue #268: Add magic number to pickled code objects · akruis/cpython@25bb326 · GitHub
Skip to content

Commit 25bb326

Browse files
akruisakruis
committed
Stackless issue python#268: Add magic number to pickled code objects
Unpickling code objects pickled with an incompatible version of Python now creates a RuntimeWarning. The resulting code object starts with an invalid opcode and renders unpickled frames invalid. Unpickling a function whose code is invalid causes a RuntimeWarning too.
1 parent 1b04eab commit 25bb326

3 files changed

Lines changed: 150 additions & 10 deletions

File tree

‎Stackless/changelog.txt‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,11 @@ What's New in Stackless 3.X.X?
1111

1212
- https://github.com/stackless-dev/stackless/issues/268
1313
Pickled code objects now contain importlib.util.MAGIC_NUMBER. Unpickling
14-
code objects pickled with an incompatible version of Python now raises a
15-
ValueError exception.
14+
code objects pickled with an incompatible version of Python now creates a
15+
RuntimeWarning("Unpickling code object with invalid magic number"). The
16+
resulting code object starts with an invalid opcode and renders unpickled
17+
frames invalid. Unpickling a function whose code is invalid causes a
18+
RuntimeWarning("Unpickling function with invalid code object").
1619

1720
- https://github.com/stackless-dev/stackless/issues/270
1821
Stackless now uses an unmodified PyFrameObject structure. Stackless now

‎Stackless/pickling/prickelpit.c‎

Lines changed: 71 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -611,6 +611,18 @@ slp_from_tuple_with_nulls(PyObject **start, PyObject *tup)
611611
******************************************************/
612612

613613
#define codetuplefmt "liiiiiSOOOSSiSOO"
614+
/* Index of co_code in the tuple given to code_new */
615+
static const Py_ssize_t code_co_code_index = 5;
616+
617+
/*
618+
* An unused (invalid) opcode. See opcode.h for a list of used opcodes.
619+
* If Stackless unpickles a code object with an invalid magic number, it prefixes
620+
* co_code with this opcode.
621+
*
622+
* frame_setstate tests if the first opcode of the code of the frame is CODE_INVALID_OPCODE
623+
* and eventually marks a frame as invalid.
624+
*/
625+
static const char CODE_INVALID_OPCODE = 0;
614626

615627
static struct _typeobject wrap_PyCode_Type;
616628
static long bytecode_magic = 0;
@@ -668,16 +680,42 @@ code_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
668680
return NULL;
669681
}
670682

671-
if (bytecode_magic != magic) {
672-
PyErr_SetString(PyExc_ValueError,
673-
"Wrong magic number. The pickled code object was pickled with a different Python version.");
674-
return NULL;
675-
}
676-
677683
args = PyTuple_GetSlice(args, 1, sizeof(codetuplefmt)-1);
678684
if (NULL == args)
679685
return NULL;
680686

687+
if (bytecode_magic != magic) {
688+
if (PyErr_WarnFormat(PyExc_RuntimeWarning, 1, "Unpickling code object with invalid magic number %ld", magic)) {
689+
Py_DECREF(args);
690+
return NULL;
691+
}
692+
693+
PyObject *code = PyTuple_GET_ITEM(args, code_co_code_index);
694+
if (NULL == code) {
695+
Py_DECREF(args);
696+
return NULL;
697+
}
698+
if (!PyBytes_Check(code)) {
699+
Py_DECREF(args);
700+
PyErr_SetString(PyExc_TypeError,
701+
"Unpickling code object: code is not a bytes object");
702+
return NULL;
703+
}
704+
Py_ssize_t code_len = PyBytes_Size(code);
705+
assert(code_len <= INT_MAX);
706+
assert(code_len % sizeof(_Py_CODEUNIT) == 0);
707+
708+
/* Now prepend an invalid opcode to the code.
709+
*/
710+
PyObject *code2 = PyBytes_FromStringAndSize(NULL, code_len + sizeof(_Py_CODEUNIT));
711+
assert(_Py_IS_ALIGNED(PyBytes_AS_STRING(code), sizeof(_Py_CODEUNIT)));
712+
char *p = PyBytes_AS_STRING(code2);
713+
p[0] = Py_BUILD_ASSERT_EXPR(sizeof(_Py_CODEUNIT) == 2) + CODE_INVALID_OPCODE;
714+
p[1] = 0; /* Argument */
715+
memcpy(p + sizeof(_Py_CODEUNIT), PyBytes_AS_STRING(code), code_len);
716+
PyTuple_SET_ITEM(args, code_co_code_index, code2);
717+
}
718+
681719
PyObject *retval = generic_new(type, args, kwds);
682720
Py_DECREF(args);
683721
return retval;
@@ -822,12 +860,33 @@ func_setstate(PyObject *self, PyObject *args)
822860
{
823861
PyFunctionObject *fu;
824862
PyObject *args2;
863+
char *pcode;
825864

826865
if (is_wrong_type(Py_TYPE(self))) return NULL;
827866
Py_TYPE(self) = Py_TYPE(self)->tp_base;
867+
868+
/* Test for an invalid code object */
869+
args2 = PyTuple_GetItem(args, 0);
870+
if (NULL==args2)
871+
return NULL;
872+
if (! PyCode_Check(args2)) {
873+
PyErr_SetString(PyExc_TypeError, "func_setstate: value for func_code is not a code object");
874+
return NULL;
875+
}
876+
pcode = PyBytes_AsString(((PyCodeObject *) args2)->co_code);
877+
if (NULL == pcode)
878+
return NULL;
879+
if (*pcode == CODE_INVALID_OPCODE) {
880+
/* invalid code object, was pickled with a different version of python */
881+
if (PyErr_WarnFormat(PyExc_RuntimeWarning, 1, "Unpickling function with invalid code object: %V",
882+
PyTuple_GetItem(args, 2), "~ name is missing ~"))
883+
return NULL;
884+
}
885+
828886
args2 = PyTuple_GetSlice(args, 0, 5);
829887
if (args2 == NULL)
830888
return NULL;
889+
831890
fu = (PyFunctionObject *)
832891
Py_TYPE(self)->tp_new(Py_TYPE(self), args2, NULL);
833892
Py_DECREF(args2);
@@ -1001,6 +1060,7 @@ frame_setstate(PyFrameObject *f, PyObject *args)
10011060
int valid, have_locals;
10021061
char f_executing;
10031062
Py_ssize_t tmp;
1063+
char *pcode;
10041064

10051065
if (is_wrong_type(Py_TYPE(f))) return NULL;
10061066

@@ -1025,6 +1085,11 @@ frame_setstate(PyFrameObject *f, PyObject *args)
10251085
"invalid code object for frame_setstate");
10261086
return NULL;
10271087
}
1088+
pcode = PyBytes_AsString(((PyCodeObject *) f_code)->co_code);
1089+
if (NULL == pcode)
1090+
return NULL;
1091+
if (*pcode == CODE_INVALID_OPCODE)
1092+
valid = 0; /* invalid code object, was pickled with a different version of python */
10281093

10291094
if (have_locals) {
10301095
Py_INCREF(f_locals);

‎Stackless/unittests/test_pickle.py‎

Lines changed: 74 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,11 @@
1010
import ctypes
1111
import importlib.util
1212
import struct
13+
import warnings
14+
import subprocess
1315
import stackless
1416

17+
from textwrap import dedent
1518
from stackless import schedule, tasklet
1619

1720
from support import test_main # @UnusedImport
@@ -22,6 +25,7 @@
2225
# we need to make it appear that pickling them is ok, otherwise we will fail when pickling
2326
# closures that refer to test runner instances
2427
import copyreg
28+
from _warnings import warn
2529

2630

2731
def reduce(obj):
@@ -1361,13 +1365,81 @@ def test_reduce_magic(self):
13611365
# see Python C-API documentation for PyImport_GetMagicNumber()
13621366
self.assertEqual(reduced[1][0], struct.unpack("<l", importlib.util.MAGIC_NUMBER)[0])
13631367

1364-
def test_new_with_wrong_magic(self):
1368+
def test_new_with_wrong_magic_error(self):
13651369
code = (lambda :None).__code__
13661370
reduce = stackless._stackless._wrap.code.__reduce__
13671371
reduced = reduce(code)
13681372
args = (reduced[1][0] + 1,) + reduced[1][1:]
13691373
self.assertIsInstance(reduced[0](*reduced[1]), type(code))
1370-
self.assertRaisesRegex(ValueError, "Wrong magic number", reduced[0], *args)
1374+
with self.assertRaisesRegex(RuntimeWarning, "Unpickling code object with invalid magic number"):
1375+
with stackless.atomic():
1376+
with warnings.catch_warnings():
1377+
warnings.simplefilter("error", RuntimeWarning)
1378+
reduced[0](*args)
1379+
1380+
def test_func_with_wrong_magic(self):
1381+
l = (lambda :None)
1382+
code = l.__code__
1383+
reduce = stackless._stackless._wrap.code.__reduce__
1384+
reduced = reduce(code)
1385+
args = (reduced[1][0] + 1,) + reduced[1][1:]
1386+
self.assertIsInstance(reduced[0](*reduced[1]), type(code))
1387+
with stackless.atomic():
1388+
with warnings.catch_warnings():
1389+
warnings.simplefilter("ignore", RuntimeWarning)
1390+
code2 = reduced[0](*args)
1391+
code2.__setstate__(())
1392+
self.assertIs(type(code2), type(code))
1393+
1394+
reduce = stackless._stackless._wrap.function.__reduce__
1395+
reduced_func = reduce(l)
1396+
f = reduced_func[0](*reduced_func[1])
1397+
args = (code2,) + reduced_func[2][1:]
1398+
with self.assertRaisesRegex(RuntimeWarning, "Unpickling function with invalid code object:"):
1399+
with stackless.atomic():
1400+
with warnings.catch_warnings():
1401+
warnings.simplefilter("error", RuntimeWarning)
1402+
f = f.__setstate__(args)
1403+
1404+
def test_run_with_wrong_magic(self):
1405+
# run this test as a subprocess, because it triggers a fprintf(stderr, ...) in ceval.c
1406+
# and I don't like this output in our test suite.
1407+
args = []
1408+
if not stackless.enable_softswitch(None):
1409+
args.append("--hard")
1410+
1411+
rc = subprocess.call([sys.executable, "-s", "-S", "-E", "-c", dedent("""
1412+
import stackless
1413+
import warnings
1414+
import sys
1415+
1416+
sys.stderr = sys.stdout
1417+
if "--hard" in sys.argv:
1418+
stackless.enable_softswitch(False)
1419+
l = (lambda :None)
1420+
code = l.__code__
1421+
reduce = stackless._stackless._wrap.code.__reduce__
1422+
reduced = reduce(code)
1423+
args = (reduced[1][0] + 1,) + reduced[1][1:]
1424+
with stackless.atomic():
1425+
with warnings.catch_warnings():
1426+
warnings.simplefilter("ignore", RuntimeWarning)
1427+
code2 = reduced[0](*args)
1428+
code2.__setstate__(())
1429+
assert(type(code2) is type(code))
1430+
# now execute code 2, first create a function from code2
1431+
f = type(l)(code2, globals())
1432+
# f should raise
1433+
try:
1434+
f()
1435+
except SystemError as e:
1436+
assert(str(e) == 'unknown opcode')
1437+
else:
1438+
assert(0, "expected exception not raised")
1439+
sys.exit(42)
1440+
""")] + args, stderr=subprocess.DEVNULL)
1441+
self.assertEqual(rc, 42)
1442+
13711443

13721444
if __name__ == '__main__':
13731445
if not sys.argv[1:]:

0 commit comments

Comments
 (0)