diff --git a/Doc/library/_interpreters.rst b/Doc/library/_interpreters.rst new file mode 100644 index 00000000000000..bbc945016e2880 --- /dev/null +++ b/Doc/library/_interpreters.rst @@ -0,0 +1,90 @@ +:mod:`_interpreters` --- Low-level interpreters API +=================================================== + +.. module:: _interpreters + :synopsis: Low-level interpreters API. + +.. versionadded:: 3,7 + +-------------- + +This module provides low-level primitives for working with multiple +Python interpreters in the same runtime in the current process. + +More information about (sub)interpreters is found at +:ref:`sub-interpreter-support`, including what data is shared between +interpreters and what is unique. Note particularly that interpreters +aren't inherently threaded, even though they track and manage Python +threads. To run code in an interpreter in a different OS thread, call +:func:`run_string` in a function that you run in a new Python thread. +For example:: + + id = _interpreters.create() + def f(): + _interpreters.run_string(id, 'print("in a thread")') + + t = threading.Thread(target=f) + t.start() + +This module is optional. It is provided by Python implementations which +support multiple interpreters. + +It defines the following functions: + +.. function:: enumerate() + + Return a list of the IDs of every existing interpreter. + + +.. function:: get_current() + + Return the ID of the currently running interpreter. + + +.. function:: get_main() + + Return the ID of the main interpreter. + + +.. function:: is_running(id) + + Return whether or not the identified interpreter is currently + running any code. + + +.. function:: create() + + Initialize a new Python interpreter and return its identifier. The + interpreter will be created in the current thread and will remain + idle until something is run in it. + + +.. function:: destroy(id) + + Finalize and destroy the identified interpreter. + + +.. function:: run_string(id, command) + + A wrapper around :c:func:`PyRun_SimpleString` which runs the provided + Python program in the main thread of the identified interpreter. + Providing an invalid or unknown ID results in a RuntimeError, + likewise if the main interpreter or any other running interpreter + is used. + + Any value returned from the code is thrown away, similar to what + threads do. If the code results in an exception then that exception + is raised in the thread in which run_string() was called, similar to + how :func:`exec` works. This aligns with how interpreters are not + inherently threaded. Note that SystemExit (as raised by sys.exit()) + is not treated any differently and will result in the process ending + if not caught explicitly. + + +.. function:: run_string_unrestricted(id, command, ns=None) + + Like :c:func:`run_string` but returns the dict in which the code + was executed. It also supports providing a namespace that gets + merged into the execution namespace before execution. Note that + this allows objects to leak between interpreters, which may not + be desirable. diff --git a/Doc/library/concurrency.rst b/Doc/library/concurrency.rst index 826bf86d081793..fafbf92c6b0181 100644 --- a/Doc/library/concurrency.rst +++ b/Doc/library/concurrency.rst @@ -29,3 +29,4 @@ The following are support modules for some of the above services: _thread.rst _dummy_thread.rst dummy_threading.rst + _interpreters.rst diff --git a/Include/internal/pystate.h b/Include/internal/pystate.h index b93342120477f3..7816665bb00df5 100644 --- a/Include/internal/pystate.h +++ b/Include/internal/pystate.h @@ -80,6 +80,20 @@ PyAPI_FUNC(_PyInitError) _PyPathConfig_Calculate( PyAPI_FUNC(void) _PyPathConfig_Clear(_PyPathConfig *config); +/* Cross-interpreter data sharing */ + +struct _cid; + +typedef struct _cid { + void *data; + PyObject *(*new_object)(struct _cid *); + void (*free)(void *); + + PyInterpreterState *interp; + PyObject *object; +} _PyCrossInterpreterData; + + /* Full Python runtime state */ typedef struct pyruntimestate { diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py new file mode 100644 index 00000000000000..29d04aa4d00cfb --- /dev/null +++ b/Lib/test/test__interpreters.py @@ -0,0 +1,582 @@ +import contextlib +import os +import os.path +import shutil +import tempfile +from textwrap import dedent, indent +import threading +import unittest + +from test import support +from test.support import script_helper + +interpreters = support.import_module('_interpreters') + + +SCRIPT_THREADED_INTERP = """\ +from textwrap import dedent +import threading +import _interpreters +def f(): + _interpreters.run_string(id, dedent(''' + {} + ''')) + +t = threading.Thread(target=f) +t.start() +""" + + +@contextlib.contextmanager +def _blocked(dirname): + filename = os.path.join(dirname, '.lock') + wait_script = dedent(""" + import os.path + import time + while not os.path.exists('{}'): + time.sleep(0.1) + """).format(filename) + try: + yield wait_script + finally: + support.create_empty_file(filename) + + +class InterpreterTests(unittest.TestCase): + + def setUp(self): + self.dirname = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.dirname) + + def test_still_running_at_exit(self): + subscript = dedent(""" + import time + # Give plenty of time for the main interpreter to finish. + time.sleep(1_000_000) + """) + script = SCRIPT_THREADED_INTERP.format(indent(subscript, ' ')) + filename = script_helper.make_script(self.dirname, 'interp', script) + with script_helper.spawn_python(filename) as proc: + retcode = proc.wait() + + self.assertEqual(retcode, 0) + + +class TestBase(unittest.TestCase): + + def tearDown(self): + for id in interpreters.enumerate(): + if id == 0: # main + continue + try: + interpreters.destroy(id) + except RuntimeError: + pass # already destroyed + + +class EnumerateTests(TestBase): + + def test_multiple(self): + main, = interpreters.enumerate() + id1 = interpreters.create() + id2 = interpreters.create() + ids = interpreters.enumerate() + + self.assertEqual(set(ids), {main, id1, id2}) + + def test_main_only(self): + main, = interpreters.enumerate() + + self.assertEqual(main, 0) + + +class GetCurrentTests(TestBase): + + def test_main(self): + main, = interpreters.enumerate() + id = interpreters.get_current() + + self.assertEqual(id, main) + + def test_sub(self): + id1 = interpreters.create() + ns = interpreters.run_string_unrestricted(id1, dedent(""" + import _interpreters + id = _interpreters.get_current() + """)) + id2 = ns['id'] + + self.assertEqual(id2, id1) + + +class GetMainTests(TestBase): + + def test_main(self): + expected, = interpreters.enumerate() + main = interpreters.get_main() + + self.assertEqual(main, 0) + self.assertEqual(main, expected) + + +class IsRunningTests(TestBase): + + def test_main_running(self): + main, = interpreters.enumerate() + sub = interpreters.create() + main_running = interpreters.is_running(main) + sub_running = interpreters.is_running(sub) + + self.assertTrue(main_running) + self.assertFalse(sub_running) + + def test_sub_running(self): + main, = interpreters.enumerate() + sub1 = interpreters.create() + sub2 = interpreters.create() + ns = interpreters.run_string_unrestricted(sub1, dedent(f""" + import _interpreters + main = _interpreters.is_running({main}) + sub1 = _interpreters.is_running({sub1}) + sub2 = _interpreters.is_running({sub2}) + """)) + main_running = ns['main'] + sub1_running = ns['sub1'] + sub2_running = ns['sub2'] + + self.assertTrue(main_running) + self.assertTrue(sub1_running) + self.assertFalse(sub2_running) + + +class CreateTests(TestBase): + + def test_in_main(self): + id = interpreters.create() + + self.assertIn(id, interpreters.enumerate()) + + @unittest.skip('enable this test when working on pystate.c') + def test_unique_id(self): + seen = set() + for _ in range(100): + id = interpreters.create() + interpreters.destroy(id) + seen.add(id) + + self.assertEqual(len(seen), 100) + + def test_in_thread(self): + lock = threading.Lock() + id = None + def f(): + nonlocal id + id = interpreters.create() + lock.acquire() + lock.release() + + t = threading.Thread(target=f) + with lock: + t.start() + t.join() + self.assertIn(id, interpreters.enumerate()) + + def test_in_subinterpreter(self): + main, = interpreters.enumerate() + id1 = interpreters.create() + ns = interpreters.run_string_unrestricted(id1, dedent(""" + import _interpreters + id = _interpreters.create() + """)) + id2 = ns['id'] + + self.assertEqual(set(interpreters.enumerate()), {main, id1, id2}) + + def test_in_threaded_subinterpreter(self): + main, = interpreters.enumerate() + id1 = interpreters.create() + ns = None + script = dedent(""" + import _interpreters + id = _interpreters.create() + """) + def f(): + nonlocal ns + ns = interpreters.run_string_unrestricted(id1, script) + + t = threading.Thread(target=f) + t.start() + t.join() + id2 = ns['id'] + + self.assertEqual(set(interpreters.enumerate()), {main, id1, id2}) + + + def test_after_destroy_all(self): + before = set(interpreters.enumerate()) + # Create 3 subinterpreters. + ids = [] + for _ in range(3): + id = interpreters.create() + ids.append(id) + # Now destroy them. + for id in ids: + interpreters.destroy(id) + # Finally, create another. + id = interpreters.create() + self.assertEqual(set(interpreters.enumerate()), before | {id}) + + def test_after_destroy_some(self): + before = set(interpreters.enumerate()) + # Create 3 subinterpreters. + id1 = interpreters.create() + id2 = interpreters.create() + id3 = interpreters.create() + # Now destroy 2 of them. + interpreters.destroy(id1) + interpreters.destroy(id3) + # Finally, create another. + id = interpreters.create() + self.assertEqual(set(interpreters.enumerate()), before | {id, id2}) + + +class DestroyTests(TestBase): + + def test_one(self): + id1 = interpreters.create() + id2 = interpreters.create() + id3 = interpreters.create() + self.assertIn(id2, interpreters.enumerate()) + interpreters.destroy(id2) + self.assertNotIn(id2, interpreters.enumerate()) + self.assertIn(id1, interpreters.enumerate()) + self.assertIn(id3, interpreters.enumerate()) + + def test_all(self): + before = set(interpreters.enumerate()) + ids = set() + for _ in range(3): + id = interpreters.create() + ids.add(id) + self.assertEqual(set(interpreters.enumerate()), before | ids) + for id in ids: + interpreters.destroy(id) + self.assertEqual(set(interpreters.enumerate()), before) + + def test_main(self): + main, = interpreters.enumerate() + with self.assertRaises(RuntimeError): + interpreters.destroy(main) + + def f(): + with self.assertRaises(RuntimeError): + interpreters.destroy(main) + + t = threading.Thread(target=f) + t.start() + t.join() + + def test_already_destroyed(self): + id = interpreters.create() + interpreters.destroy(id) + with self.assertRaises(RuntimeError): + interpreters.destroy(id) + + def test_does_not_exist(self): + with self.assertRaises(RuntimeError): + interpreters.destroy(1_000_000) + + def test_bad_id(self): + with self.assertRaises(RuntimeError): + interpreters.destroy(-1) + + def test_from_current(self): + main, = interpreters.enumerate() + id = interpreters.create() + script = dedent(""" + import _interpreters + _interpreters.destroy({}) + """).format(id) + + with self.assertRaises(RuntimeError): + interpreters.run_string(id, script) + self.assertEqual(set(interpreters.enumerate()), {main, id}) + + def test_from_sibling(self): + main, = interpreters.enumerate() + id1 = interpreters.create() + id2 = interpreters.create() + script = dedent(""" + import _interpreters + _interpreters.destroy({}) + """).format(id2) + interpreters.run_string(id1, script) + + self.assertEqual(set(interpreters.enumerate()), {main, id1}) + + def test_from_other_thread(self): + id = interpreters.create() + def f(): + interpreters.destroy(id) + + t = threading.Thread(target=f) + t.start() + t.join() + + def test_still_running(self): + # XXX Rewrite this test without files by using + # run_string_unrestricted(). + main, = interpreters.enumerate() + id = interpreters.create() + def f(): + interpreters.run_string(id, wait_script) + + dirname = tempfile.mkdtemp() + t = threading.Thread(target=f) + with _blocked(dirname) as wait_script: + t.start() + with self.assertRaises(RuntimeError): + interpreters.destroy(id) + + t.join() + self.assertEqual(set(interpreters.enumerate()), {main, id}) + + +class RunStringTests(TestBase): + + SCRIPT = dedent(""" + with open('{}', 'w') as out: + out.write('{}') + """) + FILENAME = 'spam' + + def setUp(self): + self.id = interpreters.create() + self.dirname = None + self.filename = None + + def tearDown(self): + if self.dirname is not None: + try: + shutil.rmtree(self.dirname) + except FileNotFoundError: + pass # already deleted + super().tearDown() + + def _resolve_filename(self, name=None): + if name is None: + name = self.FILENAME + if self.dirname is None: + self.dirname = tempfile.mkdtemp() + return os.path.join(self.dirname, name) + + def _empty_file(self): + self.filename = self._resolve_filename() + support.create_empty_file(self.filename) + return self.filename + + def assert_file_contains(self, expected, filename=None): + if filename is None: + filename = self.filename + self.assertIsNotNone(filename) + with open(filename) as out: + content = out.read() + self.assertEqual(content, expected) + + def test_success(self): + filename = self._empty_file() + expected = 'spam spam spam spam spam' + script = self.SCRIPT.format(filename, expected) + interpreters.run_string(self.id, script) + + self.assert_file_contains(expected) + + def test_in_thread(self): + filename = self._empty_file() + expected = 'spam spam spam spam spam' + script = self.SCRIPT.format(filename, expected) + def f(): + interpreters.run_string(self.id, script) + + t = threading.Thread(target=f) + t.start() + t.join() + + self.assert_file_contains(expected) + + def test_create_thread(self): + filename = self._empty_file() + expected = 'spam spam spam spam spam' + script = dedent(""" + import threading + def f(): + with open('{}', 'w') as out: + out.write('{}') + + t = threading.Thread(target=f) + t.start() + t.join() + """).format(filename, expected) + interpreters.run_string(self.id, script) + + self.assert_file_contains(expected) + + @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") + def test_fork(self): + filename = self._empty_file() + expected = 'spam spam spam spam spam' + script = dedent(""" + import os + r, w = os.pipe() + pid = os.fork() + if pid == 0: # child + import sys + filename = '{}' + with open(filename, 'w') as out: + out.write('{}') + os.write(w, b'done!') + + # Kill the unittest runner in the child process. + os._exit(1) + else: + import select + try: + select.select([r], [], []) + finally: + os.close(r) + os.close(w) + """).format(filename, expected) + interpreters.run_string(self.id, script) + self.assert_file_contains(expected) + + def test_already_running(self): + def f(): + interpreters.run_string(self.id, wait_script) + + t = threading.Thread(target=f) + dirname = tempfile.mkdtemp() + with _blocked(dirname) as wait_script: + t.start() + with self.assertRaises(RuntimeError): + interpreters.run_string(self.id, 'print("spam")') + t.join() + + def test_does_not_exist(self): + id = 0 + while id in interpreters.enumerate(): + id += 1 + with self.assertRaises(RuntimeError): + interpreters.run_string(id, 'print("spam")') + + def test_error_id(self): + with self.assertRaises(RuntimeError): + interpreters.run_string(-1, 'print("spam")') + + def test_bad_id(self): + with self.assertRaises(TypeError): + interpreters.run_string('spam', 'print("spam")') + + def test_bad_code(self): + with self.assertRaises(TypeError): + interpreters.run_string(self.id, 10) + + def test_bytes_for_code(self): + with self.assertRaises(TypeError): + interpreters.run_string(self.id, b'print("spam")') + + def test_invalid_syntax(self): + with self.assertRaises(SyntaxError): + # missing close paren + interpreters.run_string(self.id, 'print("spam"') + + def test_failure(self): + with self.assertRaises(Exception) as caught: + interpreters.run_string(self.id, 'raise Exception("spam")') + self.assertEqual(str(caught.exception), 'spam') + + def test_sys_exit(self): + with self.assertRaises(SystemExit) as cm: + interpreters.run_string(self.id, dedent(""" + import sys + sys.exit() + """)) + self.assertIsNone(cm.exception.code) + + with self.assertRaises(SystemExit) as cm: + interpreters.run_string(self.id, dedent(""" + import sys + sys.exit(42) + """)) + self.assertEqual(cm.exception.code, 42) + + def test_SystemError(self): + with self.assertRaises(SystemExit) as cm: + interpreters.run_string(self.id, 'raise SystemExit(42)') + self.assertEqual(cm.exception.code, 42) + + +class RunStringUnrestrictedTests(TestBase): + + def setUp(self): + self.id = interpreters.create() + + def test_without_ns(self): + script = dedent(""" + spam = 42 + """) + ns = interpreters.run_string_unrestricted(self.id, script) + + self.assertEqual(ns['spam'], 42) + + def test_with_ns(self): + updates = {'spam': 'ham', 'eggs': -1} + script = dedent(""" + spam = 42 + result = spam + eggs + """) + ns = interpreters.run_string_unrestricted(self.id, script, updates) + + self.assertEqual(ns['spam'], 42) + self.assertEqual(ns['eggs'], -1) + self.assertEqual(ns['result'], 41) + + def test_ns_does_not_overwrite(self): + updates = {'__name__': 'not __main__'} + script = dedent(""" + spam = 42 + """) + ns = interpreters.run_string_unrestricted(self.id, script, updates) + + self.assertEqual(ns['__name__'], '__main__') + + def test_main_not_shared(self): + ns1 = interpreters.run_string_unrestricted(self.id, 'spam = True') + ns2 = interpreters.run_string_unrestricted(self.id, 'eggs = False') + + self.assertIn('spam', ns1) + self.assertNotIn('eggs', ns1) + self.assertIn('eggs', ns2) + self.assertNotIn('spam', ns2) + + def test_return_execution_namespace(self): + script = dedent(""" + spam = 42 + """) + ns = interpreters.run_string_unrestricted(self.id, script) + + ns.pop('__builtins__') + ns.pop('__loader__') + self.assertEqual(ns, { + '__name__': '__main__', + '__annotations__': {}, + '__doc__': None, + '__package__': None, + '__spec__': None, + 'spam': 42, + }) + + +if __name__ == '__main__': + unittest.main() diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c new file mode 100644 index 00000000000000..f9b800cb1ace35 --- /dev/null +++ b/Modules/_interpretersmodule.c @@ -0,0 +1,632 @@ + +/* interpreters module */ +/* low-level access to interpreter primitives */ + +#include "Python.h" +#include "frameobject.h" +#include "internal/pystate.h" + + +static PyInterpreterState * +_get_current(void) +{ + PyThreadState *tstate; + + tstate = PyThreadState_Get(); + if (tstate == NULL) + return NULL; + return tstate->interp; +} + +/* sharing-specific functions and structs */ + +static int +_PyObject_CheckShareable(PyObject *obj) +{ + if (PyBytes_CheckExact(obj)) + return 0; + PyErr_SetString(PyExc_ValueError, + "obj is not a cross-interpreter shareable type"); + return 1; +} + +static PyObject * +_new_bytes_object(_PyCrossInterpreterData *data) +{ + return PyBytes_FromString((char *)(data->data)); +} + +static int +_bytes_shared(PyObject *obj, _PyCrossInterpreterData *data) +{ + data->data = (void *)(PyBytes_AS_STRING(obj)); + data->new_object = _new_bytes_object; + data->free = NULL; + return 0; +} + +static int +_PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data) +{ + Py_INCREF(obj); + + if (_PyObject_CheckShareable(obj) != 0) { + Py_DECREF(obj); + return 1; + } + + data->interp = _get_current(); + data->object = obj; + + if (PyBytes_CheckExact(obj)) { + if (_bytes_shared(obj, data) != 0) { + Py_DECREF(obj); + return 1; + } + } + + return 0; +}; + +static void +_PyCrossInterpreterData_Release(_PyCrossInterpreterData *data) +{ + PyThreadState *save_tstate = NULL; + if (data->interp != NULL) { + // Switch to the original interpreter. + PyThreadState *tstate = PyInterpreterState_ThreadHead(data->interp); + save_tstate = PyThreadState_Swap(tstate); + } + + if (data->free != NULL) { + data->free(data->data); + } + Py_XDECREF(data->object); + + // Switch back. + if (save_tstate != NULL) + PyThreadState_Swap(save_tstate); +} + +static PyObject * +_PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data) +{ + return data->new_object(data); +} + +struct _shareditem { + Py_UNICODE *name; + Py_ssize_t namelen; + _PyCrossInterpreterData data; +}; + +void +_sharedns_clear(struct _shareditem *shared) +{ + for (struct _shareditem *item=shared; item->name != NULL; item += 1) { + _PyCrossInterpreterData_Release(&item->data); + } +} + +static struct _shareditem * +_get_shared_ns(PyObject *shareable) +{ + if (shareable == NULL || shareable == Py_None) + return NULL; + Py_ssize_t len = PyDict_Size(shareable); + if (len == 0) + return NULL; + + struct _shareditem *shared = PyMem_NEW(struct _shareditem, len+1); + Py_ssize_t pos = 0; + for (Py_ssize_t i=0; i < len; i++) { + PyObject *key, *value; + if (PyDict_Next(shareable, &pos, &key, &value) == 0) { + break; + } + struct _shareditem *item = shared + i; + + if (_PyObject_GetCrossInterpreterData(value, &item->data) != 0) + break; + item->name = PyUnicode_AsUnicodeAndSize(key, &item->namelen); + if (item->name == NULL) { + _PyCrossInterpreterData_Release(&item->data); + break; + } + (item + 1)->name = NULL; // Mark the next one as the last. + } + if (PyErr_Occurred()) { + _sharedns_clear(shared); + PyMem_Free(shared); + return NULL; + } + return shared; +} + +static int +_shareditem_apply(struct _shareditem *item, PyObject *ns) +{ + PyObject *name = PyUnicode_FromUnicode(item->name, item->namelen); + if (name == NULL) { + return 1; + } + PyObject *value = _PyCrossInterpreterData_NewObject(&item->data); + if (value == NULL) { + Py_DECREF(name); + return 1; + } + int res = PyDict_SetItem(ns, name, value); + Py_DECREF(name); + Py_DECREF(value); + return res; +} + +// XXX This cannot use PyObject fields. + +struct _shared_exception { + PyObject *exc; + PyObject *value; + PyObject *tb; +}; + +static struct _shared_exception * +_get_shared_exception(void) +{ + struct _shared_exception *exc = PyMem_NEW(struct _shared_exception, 1); + // XXX Fatal if NULL? + PyErr_Fetch(&exc->exc, &exc->value, &exc->tb); + return exc; +} + +static void +_apply_shared_exception(struct _shared_exception *exc) +{ + if (PyErr_Occurred()) { + _PyErr_ChainExceptions(exc->exc, exc->value, exc->tb); + } else { + PyErr_Restore(exc->exc, exc->value, exc->tb); + } + +} + +/* interpreter-specific functions */ + +static PyInterpreterState * +_look_up_int64(PY_INT64_T requested_id) +{ + if (requested_id < 0) + goto error; + + PyInterpreterState *interp = PyInterpreterState_Head(); + while (interp != NULL) { + PY_INT64_T id = PyInterpreterState_GetID(interp); + if (id < 0) + return NULL; + if (requested_id == id) + return interp; + interp = PyInterpreterState_Next(interp); + } + +error: + PyErr_Format(PyExc_RuntimeError, + "unrecognized interpreter ID %lld", requested_id); + return NULL; +} + +static PyInterpreterState * +_look_up(PyObject *requested_id) +{ + long long id = PyLong_AsLongLong(requested_id); + if (id == -1 && PyErr_Occurred() != NULL) + return NULL; + assert(id <= INT64_MAX); + return _look_up_int64(id); +} + +static PyObject * +_get_id(PyInterpreterState *interp) +{ + PY_INT64_T id = PyInterpreterState_GetID(interp); + if (id < 0) + return NULL; + return PyLong_FromLongLong(id); +} + +static int +_is_running(PyInterpreterState *interp) +{ + PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); + if (PyThreadState_Next(tstate) != NULL) { + PyErr_SetString(PyExc_RuntimeError, + "interpreter has more than one thread"); + return -1; + } + PyFrameObject *frame = _PyThreadState_GetFrame(tstate); + if (frame == NULL) { + if (PyErr_Occurred() != NULL) + return -1; + return 0; + } + return (int)(frame->f_executing); +} + +static int +_ensure_not_running(PyInterpreterState *interp) +{ + int is_running = _is_running(interp); + if (is_running < 0) + return -1; + if (is_running) { + PyErr_Format(PyExc_RuntimeError, "interpreter already running"); + return -1; + } + return 0; +} + +static int +_run_script(PyInterpreterState *interp, const char *codestr, + struct _shareditem *shared, struct _shared_exception **exc) +{ + PyObject *main_mod = PyMapping_GetItemString(interp->modules, "__main__"); + if (main_mod == NULL) + return -1; + PyObject *ns = PyModule_GetDict(main_mod); // borrowed + Py_INCREF(ns); + Py_DECREF(main_mod); + if (ns == NULL) + return -1; + + // Apply the cross-interpreter data. + if (shared != NULL) { + for (struct _shareditem *item=shared; item->name != NULL; item += 1) { + if (_shareditem_apply(shared, ns) != 0) { + Py_DECREF(ns); + return -1; + } + } + } + + // Run the string (see PyRun_SimpleStringFlags). + PyObject *result = PyRun_StringFlags(codestr, Py_file_input, ns, ns, NULL); + Py_DECREF(ns); + if (result == NULL) { + // Get the exception from the subinterpreter. + *exc = _get_shared_exception(); + // XXX Clear the exception? + } else { + Py_DECREF(result); // We throw away the result. + } + + return 0; +} + +static int +_run_script_in_interpreter(PyInterpreterState *interp, const char *codestr, + PyObject *shareable) +{ + // XXX lock? + if (_ensure_not_running(interp) < 0) + return -1; + + struct _shareditem *shared = _get_shared_ns(shareable); + if (shared == NULL && PyErr_Occurred()) + return -1; + + // Switch to interpreter. + PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); + PyThreadState *save_tstate = PyThreadState_Swap(tstate); + + // Run the script. + struct _shared_exception *exc = NULL; + if (_run_script(interp, codestr, shared, &exc) != 0) { + // XXX What to do if the the result isn't 0? + } + + // Switch back. + if (save_tstate != NULL) + PyThreadState_Swap(save_tstate); + + // Propagate any exception out to the caller. + if (exc != NULL) { + _apply_shared_exception(exc); + } + + if (shared != NULL) { + _sharedns_clear(shared); + PyMem_Free(shared); + } + + return 0; +} + + +/* module level code ********************************************************/ + +static PyObject * +interp_create(PyObject *self, PyObject *args) +{ + if (!PyArg_UnpackTuple(args, "create", 0, 0)) + return NULL; + + // Create and initialize the new interpreter. + PyThreadState *tstate, *save_tstate; + save_tstate = PyThreadState_Swap(NULL); + tstate = Py_NewInterpreter(); + PyThreadState_Swap(save_tstate); + if (tstate == NULL) { + /* Since no new thread state was created, there is no exception to + propagate; raise a fresh one after swapping in the old thread + state. */ + PyErr_SetString(PyExc_RuntimeError, "interpreter creation failed"); + return NULL; + } + return _get_id(tstate->interp); +} + +PyDoc_STRVAR(create_doc, +"create() -> ID\n\ +\n\ +Create a new interpreter and return a unique generated ID."); + + +static PyObject * +interp_destroy(PyObject *self, PyObject *args) +{ + PyObject *id; + if (!PyArg_UnpackTuple(args, "destroy", 1, 1, &id)) + return NULL; + if (!PyLong_Check(id)) { + PyErr_SetString(PyExc_TypeError, "ID must be an int"); + return NULL; + } + + // Look up the interpreter. + PyInterpreterState *interp = _look_up(id); + if (interp == NULL) + return NULL; + + // Ensure we don't try to destroy the current interpreter. + PyInterpreterState *current = _get_current(); + if (current == NULL) + return NULL; + if (interp == current) { + PyErr_SetString(PyExc_RuntimeError, + "cannot destroy the current interpreter"); + return NULL; + } + + // Ensure the interpreter isn't running. + /* XXX We *could* support destroying a running interpreter but + aren't going to worry about it for now. */ + if (_ensure_not_running(interp) < 0) + return NULL; + + // Destroy the interpreter. + //PyInterpreterState_Delete(interp); + PyThreadState *tstate, *save_tstate; + tstate = PyInterpreterState_ThreadHead(interp); + save_tstate = PyThreadState_Swap(tstate); + Py_EndInterpreter(tstate); + PyThreadState_Swap(save_tstate); + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(destroy_doc, +"destroy(ID)\n\ +\n\ +Destroy the identified interpreter.\n\ +\n\ +Attempting to destroy the current interpreter results in a RuntimeError.\n\ +So does an unrecognized ID."); + + +static PyObject * +interp_enumerate(PyObject *self) +{ + PyObject *ids, *id; + PyInterpreterState *interp; + + ids = PyList_New(0); + if (ids == NULL) + return NULL; + + interp = PyInterpreterState_Head(); + while (interp != NULL) { + id = _get_id(interp); + if (id == NULL) + return NULL; + // insert at front of list + if (PyList_Insert(ids, 0, id) < 0) + return NULL; + + interp = PyInterpreterState_Next(interp); + } + + return ids; +} + +PyDoc_STRVAR(enumerate_doc, +"enumerate() -> [ID]\n\ +\n\ +Return a list containing the ID of every existing interpreter."); + + +static PyObject * +interp_get_current(PyObject *self) +{ + PyInterpreterState *interp =_get_current(); + if (interp == NULL) + return NULL; + return _get_id(interp); +} + +PyDoc_STRVAR(get_current_doc, +"get_current() -> ID\n\ +\n\ +Return the ID of current interpreter."); + + +static PyObject * +interp_get_main(PyObject *self) +{ + // Currently, 0 is always the main interpreter. + return PyLong_FromLongLong(0); +} + +PyDoc_STRVAR(get_main_doc, +"get_main() -> ID\n\ +\n\ +Return the ID of main interpreter."); + + +static PyObject * +interp_run_string(PyObject *self, PyObject *args) +{ + PyObject *id, *code; + PyObject *shared = NULL; + if (!PyArg_UnpackTuple(args, "run_string", 2, 3, &id, &code, &shared)) + return NULL; + if (!PyLong_Check(id)) { + PyErr_SetString(PyExc_TypeError, "first arg (ID) must be an int"); + return NULL; + } + if (!PyUnicode_Check(code)) { + PyErr_SetString(PyExc_TypeError, + "second arg (code) must be a string"); + return NULL; + } + + // Look up the interpreter. + PyInterpreterState *interp = _look_up(id); + if (interp == NULL) + return NULL; + + // Extract code. + Py_ssize_t size; + const char *codestr = PyUnicode_AsUTF8AndSize(code, &size); + if (codestr == NULL) + return NULL; + if (strlen(codestr) != (size_t)size) { + PyErr_SetString(PyExc_ValueError, + "source code string cannot contain null bytes"); + return NULL; + } + + // Run the code in the interpreter. + if (_run_script_in_interpreter(interp, codestr, shared) != 0) + return NULL; + Py_RETURN_NONE; +} + +PyDoc_STRVAR(run_string_doc, +"run_string(ID, sourcetext)\n\ +\n\ +Execute the provided string in the identified interpreter.\n\ +\n\ +See PyRun_SimpleStrings."); + + +static PyObject * +object_is_shareable(PyObject *self, PyObject *args) +{ + PyObject *obj; + if (!PyArg_UnpackTuple(args, "is_shareable", 1, 1, &obj)) + return NULL; + if (_PyObject_CheckShareable(obj) == 0) + Py_RETURN_TRUE; + PyErr_Clear(); + Py_RETURN_FALSE; +} + +PyDoc_STRVAR(is_shareable_doc, +"is_shareable(obj) -> bool\n\ +\n\ +Return True if the object's data may be shared between interpreters and\n\ +False otherwise."); + + +static PyObject * +interp_is_running(PyObject *self, PyObject *args) +{ + PyObject *id; + if (!PyArg_UnpackTuple(args, "is_running", 1, 1, &id)) + return NULL; + if (!PyLong_Check(id)) { + PyErr_SetString(PyExc_TypeError, "ID must be an int"); + return NULL; + } + + PyInterpreterState *interp = _look_up(id); + if (interp == NULL) + return NULL; + int is_running = _is_running(interp); + if (is_running < 0) + return NULL; + if (is_running) + Py_RETURN_TRUE; + Py_RETURN_FALSE; +} + +PyDoc_STRVAR(is_running_doc, +"is_running(id) -> bool\n\ +\n\ +Return whether or not the identified interpreter is running."); + + +static PyMethodDef module_functions[] = { + {"create", (PyCFunction)interp_create, + METH_VARARGS, create_doc}, + {"destroy", (PyCFunction)interp_destroy, + METH_VARARGS, destroy_doc}, + + {"enumerate", (PyCFunction)interp_enumerate, + METH_NOARGS, enumerate_doc}, + {"get_current", (PyCFunction)interp_get_current, + METH_NOARGS, get_current_doc}, + {"get_main", (PyCFunction)interp_get_main, + METH_NOARGS, get_main_doc}, + {"is_running", (PyCFunction)interp_is_running, + METH_VARARGS, is_running_doc}, + + {"run_string", (PyCFunction)interp_run_string, + METH_VARARGS, run_string_doc}, + + {"is_shareable", (PyCFunction)object_is_shareable, + METH_VARARGS, is_shareable_doc}, + + {NULL, NULL} /* sentinel */ +}; + + +/* initialization function */ + +PyDoc_STRVAR(module_doc, +"This module provides primitive operations to manage Python interpreters.\n\ +The 'interpreters' module provides a more convenient interface."); + +static struct PyModuleDef interpretersmodule = { + PyModuleDef_HEAD_INIT, + "_interpreters", + module_doc, + -1, + module_functions, + NULL, + NULL, + NULL, + NULL +}; + + +PyMODINIT_FUNC +PyInit__interpreters(void) +{ + PyObject *module; + + module = PyModule_Create(&interpretersmodule); + if (module == NULL) + return NULL; + + + return module; +} diff --git a/setup.py b/setup.py index c22de17f953396..eaf3ba218266c0 100644 --- a/setup.py +++ b/setup.py @@ -740,6 +740,10 @@ def detect_modules(self): ['_xxtestfuzz/_xxtestfuzz.c', '_xxtestfuzz/fuzzer.c']) ) + # Python interface to subinterpreter C-API. + exts.append(Extension('_interpreters', ['_interpretersmodule.c'], + define_macros=[('Py_BUILD_CORE', '')])) + # # Here ends the simple stuff. From here on, modules need certain # libraries, are platform-specific, or present other surprises.