Visitar URL original
Stackless issue #168: make Stackless compatible with old Cython modul… · akruis/cpython@638e61c · GitHub
Skip to content

Commit 638e61c

Browse files
author
Anselm Kruis
authored
Stackless issue python#168: make Stackless compatible with old Cython modules (python#169)
Many extension modules were created by Cython versions before commit 037bcf0 and were compiled with regular C-Python. These modules call PyEval_EvalFrameEx() with a broken frame object. This commit add code to recover a broken frame in PyEval_EvalFrameEx().
1 parent eb7d66f commit 638e61c

5 files changed

Lines changed: 127 additions & 10 deletions

File tree

‎Objects/frameobject.c‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -512,6 +512,13 @@ frame_dealloc(PyFrameObject *f)
512512

513513
PyObject_GC_UnTrack(f);
514514
Py_TRASHCAN_SAFE_BEGIN(f)
515+
516+
#if defined(STACKLESS) && PY_VERSION_HEX < 0x03080000
517+
/* Clear the magic for the old Cython frame hack.
518+
* See below in PyFrame_New() for a detailed explanation.
519+
*/
520+
f->f_blockstack[0].b_type = 0;
521+
#endif
515522
/* Kill all local variables */
516523
valuestack = f->f_valuestack;
517524
for (p = f->f_localsplus; p < valuestack; p++)
@@ -801,6 +808,25 @@ PyFrame_New(PyThreadState *tstate, PyCodeObject *code, PyObject *globals,
801808

802809
#ifdef STACKLESS
803810
f->f_execute = NULL;
811+
#if PY_VERSION_HEX < 0x03080000
812+
if (code->co_argcount > 0) {
813+
/*
814+
* A hack for binary compatibility with Cython extension modules, which
815+
* were created with an older Cythoncompiled with regular C-Python. These
816+
* modules create frames using PyFrame_New then write to frame->f_localsplus
817+
* to set the arguments. But C-Python f_localsplus is Stackless f_code.
818+
* Therefore we add a copy of f_code and a magic number in the
819+
* uninitialized f_blockstack array.
820+
* If the blockstack is used, the magic is overwritten.
821+
* To make sure, the pointer is aligned correctly, we address it relative to
822+
* f_code.
823+
* See Stackless issue #168
824+
*/
825+
(&(f->f_code))[-1] = code;
826+
/* an arbitrary negative number which is not an opcode */
827+
f->f_blockstack[0].b_type = -31683;
828+
}
829+
#endif
804830
#endif
805831
_PyObject_GC_TRACK(f);
806832
return f;

‎Python/ceval.c‎

Lines changed: 40 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -745,16 +745,51 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
745745
if (f == NULL)
746746
return NULL;
747747

748+
/* The layout of PyFrameObject differs between Stackless and C-Python.
749+
* Stackless f->f_execute is C-Python f->f_code. Stackless f->f_code is at
750+
* the end, just before f_localsplus.
751+
*/
748752
if (PyFrame_Check(f) && f->f_execute == NULL) {
749753
/* A new frame returned from PyFrame_New() has f->f_execute == NULL.
754+
* Set the usual execution function.
750755
*/
751756
f->f_execute = PyEval_EvalFrameEx_slp;
757+
758+
#if PY_VERSION_HEX < 0x03080000
759+
/* Older versions of Cython used to create frames using C-Python layout
760+
* of PyFrameObject. As a consequence f_code is overwritten by the first
761+
* item of f_localsplus[]. To be able to fix it, we have a copy of
762+
* f_code and a signature at the end of the block-stack.
763+
* The Py_BUILD_ASSERT_EXPR checks,that our assumptions about the layout
764+
* of PyFrameObject are true.
765+
* See Stackless issue #168
766+
*/
767+
(void) Py_BUILD_ASSERT_EXPR(offsetof(PyFrameObject, f_code) ==
768+
offsetof(PyFrameObject, f_localsplus) - Py_MEMBER_SIZE(PyFrameObject, f_localsplus[0]));
769+
770+
/* Check for an old Cython frame */
771+
if (f->f_iblock == 0 && f->f_lasti == -1 && /* blockstack is empty */
772+
f->f_blockstack[0].b_type == -31683 && /* magic is present */
773+
/* and f_code has been overwritten */
774+
f->f_code != (&(f->f_code))[-1] &&
775+
/* and (&(f->f_code))[-1] looks like a valid code object */
776+
(&(f->f_code))[-1] && PyCode_Check((&(f->f_code))[-1]) &&
777+
/* and there are arguments */
778+
(&(f->f_code))[-1]->co_argcount > 0 &&
779+
/* the last argument is NULL */
780+
f->f_localsplus[(&(f->f_code))[-1]->co_argcount - 1] == NULL)
781+
{
782+
PyCodeObject * code = (&(f->f_code))[-1];
783+
memmove(f->f_localsplus, f->f_localsplus-1, code->co_argcount * sizeof(f->f_localsplus[0]));
784+
f->f_code = code;
785+
} else
786+
#endif
787+
if (!(f->f_code != NULL && PyCode_Check(f->f_code))) {
788+
PyErr_BadInternalCall();
789+
return NULL;
790+
}
752791
} else {
753-
/* The layout of PyFrameObject differs between Stackless and C-Python.
754-
* Stackless f->f_execute is C-Python f->f_code. Stackless f->f_code is at
755-
* the end, just before f_localsplus.
756-
*
757-
* In order to detect a C-Python frame, we must compare f->f_execute
792+
/* In order to detect a broken C-Python frame, we must compare f->f_execute
758793
* with every valid frame function. Hard to implement completely.
759794
* Therefore I'll check only for relevant functions.
760795
* Amend the list as needed.

‎Stackless/changelog.txt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,10 @@ What's New in Stackless 2.7.XX?
1010

1111
*Release date: XXXX-XX-XX*
1212

13+
- https://github.com/stackless-dev/stackless/issues/168
14+
Make Stackless compatible with old Cython extension modules compiled
15+
for regular C-Python.
16+
1317
- https://github.com/stackless-dev/stackless/issues/167
1418
Replace 'printf(...)' calls by PySys_WriteStderr(...). They are used to emit
1519
an error message, if there is a pending error while entering Stackless

‎Stackless/module/stacklessmodule.c‎

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1138,20 +1138,22 @@ by Stackless Python.\n\
11381138
The function creates a frame from code, globals and args and executes the frame.");
11391139

11401140
static PyObject* test_PyEval_EvalFrameEx(PyObject *self, PyObject *args, PyObject *kwds) {
1141-
static char *kwlist[] = {"code", "globals", "args", "alloca", "throw", NULL};
1141+
static char *kwlist[] = {"code", "globals", "args", "alloca", "throw", "oldcython",
1142+
"code2", NULL};
11421143
PyThreadState *tstate = PyThreadState_GET();
1143-
PyCodeObject *co;
1144+
PyCodeObject *co, *code2 = NULL;
11441145
PyObject *globals, *co_args = NULL;
11451146
Py_ssize_t alloca_size = 0;
11461147
PyObject *exc = NULL;
1148+
PyObject *oldcython = NULL;
11471149
PyFrameObject *f;
11481150
PyObject *result = NULL;
11491151
void *p;
11501152
Py_ssize_t na;
11511153

1152-
if (!PyArg_ParseTupleAndKeywords(args, kwds, "O!O!|O!nO:test_PyEval_EvalFrameEx", kwlist,
1154+
if (!PyArg_ParseTupleAndKeywords(args, kwds, "O!O!|O!nOO!O!:test_PyEval_EvalFrameEx", kwlist,
11531155
&PyCode_Type, &co, &PyDict_Type, &globals, &PyTuple_Type, &co_args, &alloca_size,
1154-
&exc))
1156+
&exc, &PyBool_Type, &oldcython, &PyCode_Type, &code2))
11551157
return NULL;
11561158
if (exc && !PyExceptionInstance_Check(exc)) {
11571159
PyErr_SetString(PyExc_TypeError, "exc must be an exception instance");
@@ -1182,6 +1184,13 @@ static PyObject* test_PyEval_EvalFrameEx(PyObject *self, PyObject *args, PyObjec
11821184
goto exit;
11831185
}
11841186
fastlocals = f->f_localsplus;
1187+
if (oldcython == Py_True) {
1188+
/* Use the f_localsplus offset from regular C-Python. Old versions of cython used to
1189+
* access f_localplus directly. Current versions compute the field offset for
1190+
* f_localsplus at run-time.
1191+
*/
1192+
fastlocals--;
1193+
}
11851194
for (i = 0; i < na; i++) {
11861195
PyObject *arg = PyTuple_GetItem(co_args, i);
11871196
if (arg == NULL) {
@@ -1197,6 +1206,10 @@ static PyObject* test_PyEval_EvalFrameEx(PyObject *self, PyObject *args, PyObjec
11971206
if (exc) {
11981207
PyErr_SetObject(PyExceptionInstance_Class(exc), exc);
11991208
}
1209+
if (code2) {
1210+
Py_INCREF(code2);
1211+
Py_SETREF(f->f_code, code2);
1212+
}
12001213
result = PyEval_EvalFrameEx(f, exc != NULL);
12011214
/* result = Py_None; Py_INCREF(Py_None); */
12021215
exit:

‎Stackless/unittests/test_capi.py‎

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,8 @@
3131
import unittest
3232

3333
from support import test_main # @UnusedImport
34-
from support import StacklessTestCase, withThreads, require_one_thread
34+
from support import (StacklessTestCase, withThreads, require_one_thread,
35+
testcase_leaks_references)
3536

3637
if withThreads:
3738
try:
@@ -161,6 +162,44 @@ def f():
161162
self.assertIn(str(exc), msg)
162163
#print(msg)
163164

165+
def test_oldcython_frame(self):
166+
# A test for Stackless issue #168
167+
self.assertEqual(self.call_PyEval_EvalFrameEx(47110816, oldcython=True), 47110816)
168+
169+
def test_oldcython_frame_code_is_1st_arg_good(self):
170+
# A pathological test for Stackless issue #168
171+
def f(code):
172+
return code
173+
174+
def f2(code):
175+
return code
176+
177+
self.assertIs(stackless.test_PyEval_EvalFrameEx(f.__code__, f.__globals__, (f.__code__,), oldcython=False), f.__code__)
178+
self.assertIs(stackless.test_PyEval_EvalFrameEx(f.__code__, f.__globals__, (f2.__code__,), oldcython=True), f2.__code__)
179+
180+
@testcase_leaks_references("f->f_code get overwritten without Py_DECREF")
181+
def test_oldcython_frame_code_is_1st_arg_bad(self):
182+
# A pathological test for Stackless issue #168
183+
def f(code):
184+
return code
185+
186+
# we can't fix this particular case:
187+
# - running code object is its 1st arg and
188+
# - oldcython=True,
189+
# because a fix would result in a segmentation fault, if the number of
190+
# arguments is to low (test case test_0_args)
191+
self.assertRaises(UnboundLocalError, stackless.test_PyEval_EvalFrameEx, f.__code__, f.__globals__, (f.__code__,), oldcython=True)
192+
193+
def test_other_code_object(self):
194+
# A pathological test for Stackless issue #168
195+
def f(arg):
196+
return arg
197+
198+
def f2(arg):
199+
return arg
200+
201+
self.assertIs(stackless.test_PyEval_EvalFrameEx(f.__code__, f.__globals__, (f2,), code2=f2.__code__), f2)
202+
164203

165204
if __name__ == "__main__":
166205
if not sys.argv[1:]:

0 commit comments

Comments
 (0)