Stackless issue python#186: Make Stackless compatible with subinterpreters.

Previously creating a subinterpreter in one thread and using it in
another thread caused a crash. Now Stackless resets the thread state,
once all C-stacks have been executed.

Author: Anselm Kruis

Python/pystate.c

@@ -598,6 +598,11 @@ tstate_delete_common(PyThreadState *tstate)
     if (tstate->next)
         tstate->next->prev = tstate->prev;
     HEAD_UNLOCK();
+#ifdef STACKLESS
+    if (interp->st.initial_tstate == tstate) {
+        interp->st.initial_tstate = NULL;
+    }
+#endif
     if (tstate->on_delete != NULL) {
         tstate->on_delete(tstate->on_delete_data);
     }

Stackless/changelog.txt

@@ -9,8 +9,12 @@ What's New in Stackless 3.X.X?
 
 *Release date: 20XX-XX-XX*
 
+- https://github.com/stackless-dev/stackless/issues/186
+  Make Stackless compatible with subinterpreters. Previously creating a
+  subinterpreter in one thread and using it in another thread caused a crash.
+
 - https://github.com/stackless-dev/stackless/issues/182
-  Fix "make teststackless" for out of tree builds
+  Fix "make teststackless" for out of tree builds.
 
 - https://github.com/stackless-dev/stackless/issues/180
   Deprecate stackless.pickle_with_tracing_state. It has been replaced by

Stackless/core/stackless_tstate.h

@@ -38,6 +38,11 @@ typedef struct {
     uint8_t pickleflags;                        /* flags for pickling / unpickling */
 } PyStacklessInterpreterState;
 
+#define SLP_INITIAL_TSTATE(tstate) \
+    (assert(tstate), \
+     assert((tstate)->interp->st.initial_tstate), \
+     (tstate)->interp->st.initial_tstate)
+
 #define SPL_INTERPRETERSTATE_NEW(interp)       \
     (interp)->st.cstack_chain = NULL;          \
     (interp)->st.reduce_frame_func = NULL;     \

Stackless/core/stacklesseval.c

@@ -319,13 +319,6 @@ slp_eval_frame(PyFrameObject *f)
     PyObject *retval;
 
     if (fprev == NULL && ts->st.main == NULL) {
-        int returning;
-        if (ts->interp->st.mem_bomb == NULL) {
-            /* allocate the permanent bomb to use for mem errors */
-            if ((ts->interp->st.mem_bomb = slp_new_bomb()) == NULL)
-                return NULL;
-        }
-
         /* this is the initial frame, so mark the stack base */
 
         /*
@@ -337,12 +330,46 @@ slp_eval_frame(PyFrameObject *f)
          * with a __del__ method is destroyed. This __del__
          * will run as a toplevel frame, with f_back == NULL!
          */
+        int returning;
 
+        /* complete the interpreter state */
+        if (ts->interp->st.initial_tstate == NULL) {
+            ts->interp->st.initial_tstate = ts;
+        }
+        if (ts->interp->st.mem_bomb == NULL) {
+            /* allocate the permanent bomb to use for mem errors */
+            if ((ts->interp->st.mem_bomb = slp_new_bomb()) == NULL)
+                return NULL;
+        }
+
+        /* mark the stack base */
         stackref = STACK_REFPLUS + (intptr_t *) &f;
         if (ts->st.cstack_base == NULL)
             ts->st.cstack_base = stackref - CSTACK_GOODGAP;
-        if (stackref > ts->st.cstack_base)
-            return climb_stack_and_eval_frame(f);
+        if (stackref > ts->st.cstack_base) {
+			PyCStackObject *cst;
+            retval = climb_stack_and_eval_frame(f);
+			cst = ts->st.initial_stub;
+			/* might be NULL in OOM conditions */
+			if (cst != NULL) {
+                PyCStackObject *other;
+                register int found = 0;
+                assert(cst->startaddr == ts->st.cstack_base);
+                for (other = cst->next; other != cst; other = other->next) {
+                    if (Py_SIZE(other) != 0 && other->task != NULL && other->tstate == ts) {
+                        assert(other->startaddr == ts->st.cstack_base);
+                        found = 1;
+                        break;
+                    }
+                }
+                if (!found) {
+                    Py_CLEAR(ts->st.initial_stub);
+                    ts->st.cstack_base = NULL;
+                    ts->st.cstack_root = NULL;
+                }
+            }
+            return retval;
+        }
 
         assert(SLP_CURRENT_FRAME(ts) == NULL);  /* else we would change the current frame */
         SLP_STORE_NEXT_FRAME(ts, f);
@@ -358,9 +385,11 @@ slp_eval_frame(PyFrameObject *f)
         if (returning != 1) {
             assert(f == SLP_PEEK_NEXT_FRAME(ts));
         }
+        assert(ts->interp->st.initial_tstate != NULL);
         assert(ts->interp->st.mem_bomb != NULL);
         return slp_run_tasklet();
     }
+    assert(ts->interp->st.initial_tstate != NULL);
     assert(ts->interp->st.mem_bomb != NULL); /* see above */
     Py_INCREF(Py_None);
     Py_XINCREF(fprev);

Stackless/module/scheduling.c

@@ -842,7 +842,7 @@ schedule_task_block(PyObject **result, PyTaskletObject *prev, int stackless, int
 
 cantblock:
     /* cannot block */
-    if (revive_main || (ts == ts->interp->st.initial_tstate && wakeup->next == NULL)) {
+    if (revive_main || (ts == SLP_INITIAL_TSTATE(ts) && wakeup->next == NULL)) {
         /* emulate old revive_main behavior:
          * passing a value only if it is an exception
          */
@@ -1064,7 +1064,7 @@ slp_schedule_task_prepared(PyThreadState *ts, PyObject **result, PyTaskletObject
         prev->cstate = ts->st.initial_stub;
         Py_INCREF(prev->cstate);
     }
-    if (ts != ts->interp->st.initial_tstate) {
+    if (ts != SLP_INITIAL_TSTATE(ts)) {
         /* ensure to get all tasklets into the other thread's chain */
         if (slp_ensure_linkage(prev) || slp_ensure_linkage(next))
             return -1;
@@ -1394,9 +1394,11 @@ slp_tasklet_end(PyObject *retval)
         int handled = 0;
         if (PyErr_ExceptionMatches(PyExc_SystemExit)) {
             /* but if it is truly a SystemExit on the main thread, we want the exit! */
-            if (ts == ts->interp->st.initial_tstate && !PyErr_ExceptionMatches(PyExc_TaskletExit)) {
-                PyStackless_HandleSystemExit();
-                handled = 1; /* handler returned, it wants us to silence it */
+            if (ts == SLP_INITIAL_TSTATE(ts) && !PyErr_ExceptionMatches(PyExc_TaskletExit)) {
+                if (ts->interp == _PyRuntime.interpreters.main) {
+                    PyStackless_HandleSystemExit();
+                    handled = 1; /* handler returned, it wants us to silence it */
+                }
             } else if (!ismain) {
                 /* deal with TaskletExit on a non-main tasklet */
                 handled = 1;

Stackless/module/stacklessmodule.c

@@ -1753,10 +1753,6 @@ static int init_stackless_methods(void)
 int
 _PyStackless_InitTypes(void)
 {
-    /* record the thread state for thread support */
-    PyThreadState * ts = PyThreadState_GET();
-    ts->interp->st.initial_tstate = ts;
-
     if (0
         || slp_init_bombtype()
         || init_stackless_methods()

Stackless/module/taskletobject.c

@@ -453,7 +453,7 @@ tasklet_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
     t->cstate = ts->st.initial_stub;
     t->def_globals = PyEval_GetGlobals();
     Py_XINCREF(t->def_globals);
-    if (ts != ts->interp->st.initial_tstate) {
+    if (ts != SLP_INITIAL_TSTATE(ts)) {
         /* make sure to kill tasklets with their thread */
         if (slp_ensure_linkage(t)) {
             Py_DECREF(t);
@@ -1153,7 +1153,7 @@ bind_tasklet_to_frame(PyTaskletObject *task, PyFrameObject *frame)
         task->cstate = ts->st.initial_stub;
         Py_INCREF(task->cstate);
         Py_DECREF(hold);
-        if (ts != ts->interp->st.initial_tstate)
+        if (ts != SLP_INITIAL_TSTATE(ts))
             if (slp_ensure_linkage(task))
                 return -1;
     }