diff --git a/Include/internal/pycore_crossinterp.h b/Include/internal/pycore_crossinterp.h index 64d881dbab7357..2dd165eae74850 100644 --- a/Include/internal/pycore_crossinterp.h +++ b/Include/internal/pycore_crossinterp.h @@ -325,6 +325,7 @@ PyAPI_FUNC(int) _PyXI_HasCapturedException(_PyXI_session *session); // Export for _testinternalcapi shared extension PyAPI_FUNC(PyInterpreterState *) _PyXI_NewInterpreter( PyInterpreterConfig *config, + long *maybe_whence, PyThreadState **p_tstate, PyThreadState **p_save_tstate); PyAPI_FUNC(void) _PyXI_EndInterpreter( diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index c96a9e40e4274a..d38959e3ce4ec5 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -109,7 +109,8 @@ struct _is { #define _PyInterpreterState_WHENCE_LEGACY_CAPI 2 #define _PyInterpreterState_WHENCE_CAPI 3 #define _PyInterpreterState_WHENCE_XI 4 -#define _PyInterpreterState_WHENCE_MAX 4 +#define _PyInterpreterState_WHENCE_STDLIB 5 +#define _PyInterpreterState_WHENCE_MAX 5 long _whence; /* Has been initialized to a safe state. @@ -316,6 +317,8 @@ PyAPI_FUNC(int) _PyInterpreterState_IDInitref(PyInterpreterState *); PyAPI_FUNC(int) _PyInterpreterState_IDIncref(PyInterpreterState *); PyAPI_FUNC(void) _PyInterpreterState_IDDecref(PyInterpreterState *); +PyAPI_FUNC(int) _PyInterpreterState_IsReady(PyInterpreterState *interp); + PyAPI_FUNC(long) _PyInterpreterState_GetWhence(PyInterpreterState *interp); extern void _PyInterpreterState_SetWhence( PyInterpreterState *interp, diff --git a/Lib/test/support/interpreters/__init__.py b/Lib/test/support/interpreters/__init__.py index 60323c9874f9a0..0a5a9259479be4 100644 --- a/Lib/test/support/interpreters/__init__.py +++ b/Lib/test/support/interpreters/__init__.py @@ -74,51 +74,77 @@ def __str__(self): def create(): """Return a new (idle) Python interpreter.""" id = _interpreters.create(reqrefs=True) - return Interpreter(id) + return Interpreter(id, _ownsref=True) def list_all(): """Return all existing interpreters.""" - return [Interpreter(id) - for id, in _interpreters.list_all()] + return [Interpreter(id, _whence=whence) + for id, whence in _interpreters.list_all(require_ready=True)] def get_current(): """Return the currently running interpreter.""" - id, = _interpreters.get_current() - return Interpreter(id) + id, whence = _interpreters.get_current() + return Interpreter(id, _whence=whence) def get_main(): """Return the main interpreter.""" - id, = _interpreters.get_main() - return Interpreter(id) + id, whence = _interpreters.get_main() + assert whence == _interpreters.WHENCE_RUNTIME, repr(whence) + return Interpreter(id, _whence=whence) _known = weakref.WeakValueDictionary() class Interpreter: - """A single Python interpreter.""" + """A single Python interpreter. - def __new__(cls, id, /): + Attributes: + + "id" - the unique process-global ID number for the interpreter + "whence" - indicates where the interpreter was created + + If the interpreter wasn't created by this module + then any method that modifies the interpreter will fail, + i.e. .close(), .prepare_main(), .exec(), and .call() + """ + + _WHENCE_TO_STR = { + _interpreters.WHENCE_UNKNOWN: 'unknown', + _interpreters.WHENCE_RUNTIME: 'runtime init', + _interpreters.WHENCE_LEGACY_CAPI: 'legacy C-API', + _interpreters.WHENCE_CAPI: 'C-API', + _interpreters.WHENCE_XI: 'cross-interpreter C-API', + _interpreters.WHENCE_STDLIB: '_interpreters module', + } + + def __new__(cls, id, /, _whence=None, _ownsref=None): # There is only one instance for any given ID. if not isinstance(id, int): raise TypeError(f'id must be an int, got {id!r}') id = int(id) + if _whence is None: + if _ownsref: + _whence = _interpreters.WHENCE_STDLIB + else: + _whence = _interpreters.whence(id) + assert _whence in cls._WHENCE_TO_STR, repr(_whence) + if _ownsref is None: + _ownsref = (_whence == _interpreters.WHENCE_STDLIB) try: self = _known[id] assert hasattr(self, '_ownsref') except KeyError: - # This may raise InterpreterNotFoundError: - _interpreters.incref(id) - try: - self = super().__new__(cls) - self._id = id - self._ownsref = True - except BaseException: - _interpreters.decref(id) - raise + self = super().__new__(cls) _known[id] = self + self._id = id + self._whence = _whence + self._ownsref = _ownsref + if _ownsref: + # This may raise InterpreterNotFoundError: + _interpreters.incref(id) return self def __repr__(self): @@ -143,7 +169,7 @@ def _decref(self): return self._ownsref = False try: - _interpreters.decref(self.id) + _interpreters.decref(self._id) except InterpreterNotFoundError: pass @@ -151,17 +177,24 @@ def _decref(self): def id(self): return self._id + @property + def whence(self): + return self._WHENCE_TO_STR[self._whence] + def is_running(self): """Return whether or not the identified interpreter is running.""" return _interpreters.is_running(self._id) + # Everything past here is available only to interpreters created by + # interpreters.create(). + def close(self): """Finalize and destroy the interpreter. Attempting to destroy the current interpreter results in an InterpreterError. """ - return _interpreters.destroy(self._id) + return _interpreters.destroy(self._id, restrict=True) def prepare_main(self, ns=None, /, **kwargs): """Bind the given values into the interpreter's __main__. @@ -169,7 +202,7 @@ def prepare_main(self, ns=None, /, **kwargs): The values must be shareable. """ ns = dict(ns, **kwargs) if ns is not None else kwargs - _interpreters.set___main___attrs(self._id, ns) + _interpreters.set___main___attrs(self._id, ns, restrict=True) def exec(self, code, /): """Run the given source code in the interpreter. @@ -189,7 +222,7 @@ def exec(self, code, /): that time, the previous interpreter is allowed to run in other threads. """ - excinfo = _interpreters.exec(self._id, code) + excinfo = _interpreters.exec(self._id, code, restrict=True) if excinfo is not None: raise ExecutionFailed(excinfo) @@ -209,7 +242,7 @@ def call(self, callable, /): # XXX Support args and kwargs. # XXX Support arbitrary callables. # XXX Support returning the return value (e.g. via pickle). - excinfo = _interpreters.call(self._id, callable) + excinfo = _interpreters.call(self._id, callable, restrict=True) if excinfo is not None: raise ExecutionFailed(excinfo) diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index 769bb7bfdb5a32..9a7c7f2f92ef21 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -21,6 +21,14 @@ ) +WHENCE_STR_UNKNOWN = 'unknown' +WHENCE_STR_RUNTIME = 'runtime init' +WHENCE_STR_LEGACY_CAPI = 'legacy C-API' +WHENCE_STR_CAPI = 'C-API' +WHENCE_STR_XI = 'cross-interpreter C-API' +WHENCE_STR_STDLIB = '_interpreters module' + + class ModuleTests(TestBase): def test_queue_aliases(self): @@ -173,10 +181,11 @@ def test_created_with_capi(self): text = self.run_temp_from_capi(f""" import {interpreters.__name__} as interpreters interp = interpreters.get_current() - print(interp.id) + print((interp.id, interp.whence)) """) - interpid = eval(text) + interpid, whence = eval(text) self.assertEqual(interpid, expected) + self.assertEqual(whence, WHENCE_STR_CAPI) class ListAllTests(TestBase): @@ -228,22 +237,22 @@ def test_created_with_capi(self): interpid4 = interpid3 + 1 interpid5 = interpid4 + 1 expected = [ - (mainid,), - (interpid1,), - (interpid2,), - (interpid3,), - (interpid4,), - (interpid5,), + (mainid, WHENCE_STR_RUNTIME), + (interpid1, WHENCE_STR_STDLIB), + (interpid2, WHENCE_STR_STDLIB), + (interpid3, WHENCE_STR_STDLIB), + (interpid4, WHENCE_STR_CAPI), + (interpid5, WHENCE_STR_STDLIB), ] expected2 = expected[:-2] text = self.run_temp_from_capi(f""" import {interpreters.__name__} as interpreters interp = interpreters.create() print( - [(i.id,) for i in interpreters.list_all()]) + [(i.id, i.whence) for i in interpreters.list_all()]) """) res = eval(text) - res2 = [(i.id,) for i in interpreters.list_all()] + res2 = [(i.id, i.whence) for i in interpreters.list_all()] self.assertEqual(res, expected) self.assertEqual(res2, expected2) @@ -299,6 +308,38 @@ def test_id_readonly(self): with self.assertRaises(AttributeError): interp.id = 1_000_000 + def test_whence(self): + main = interpreters.get_main() + interp = interpreters.create() + + with self.subTest('main'): + self.assertEqual(main.whence, WHENCE_STR_RUNTIME) + + with self.subTest('from _interpreters'): + self.assertEqual(interp.whence, WHENCE_STR_STDLIB) + + with self.subTest('from C-API'): + text = self.run_temp_from_capi(f""" + import {interpreters.__name__} as interpreters + interp = interpreters.get_current() + print(repr(interp.whence)) + """) + whence = eval(text) + self.assertEqual(whence, WHENCE_STR_CAPI) + + with self.subTest('readonly'): + for value in [ + None, + WHENCE_STR_UNKNOWN, + WHENCE_STR_RUNTIME, + WHENCE_STR_STDLIB, + WHENCE_STR_CAPI, + ]: + with self.assertRaises(AttributeError): + interp.whence = value + with self.assertRaises(AttributeError): + main.whence = value + def test_hashable(self): interp = interpreters.create() expected = hash(interp.id) @@ -568,41 +609,42 @@ def test_created_with_capi(self): with self.subTest('running __main__ (from self)'): with self.interpreter_from_capi() as interpid: with self.assertRaisesRegex(ExecutionFailed, - 'InterpreterError.*current'): + 'InterpreterError.*unrecognized'): self.run_from_capi(interpid, script, main=True) with self.subTest('running, but not __main__ (from self)'): with self.assertRaisesRegex(ExecutionFailed, - 'InterpreterError.*current'): + 'InterpreterError.*unrecognized'): self.run_temp_from_capi(script) with self.subTest('running __main__ (from other)'): with self.interpreter_obj_from_capi() as (interp, interpid): with self.running_from_capi(interpid, main=True): - with self.assertRaisesRegex(InterpreterError, 'running'): + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): interp.close() # Make sure it wssn't closed. self.assertTrue( - interp.is_running()) + self.interp_exists(interpid)) - # The rest must be skipped until we deal with running threads when - # interp.close() is called. - return + # The rest would be skipped until we deal with running threads when + # interp.close() is called. However, the "whence" restrictions + # trigger first. with self.subTest('running, but not __main__ (from other)'): with self.interpreter_obj_from_capi() as (interp, interpid): with self.running_from_capi(interpid, main=False): - with self.assertRaisesRegex(InterpreterError, 'not managed'): + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): interp.close() # Make sure it wssn't closed. - self.assertFalse(interp.is_running()) + self.assertTrue( + self.interp_exists(interpid)) with self.subTest('not running (from other)'): - with self.interpreter_obj_from_capi() as (interp, _): - with self.assertRaisesRegex(InterpreterError, 'not managed'): + with self.interpreter_obj_from_capi() as (interp, interpid): + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): interp.close() - # Make sure it wssn't closed. - self.assertFalse(interp.is_running()) + self.assertTrue( + self.interp_exists(interpid)) class TestInterpreterPrepareMain(TestBase): @@ -671,12 +713,11 @@ def test_running(self): @requires_test_modules def test_created_with_capi(self): - with self.interpreter_from_capi() as interpid: - interp = interpreters.Interpreter(interpid) - interp.prepare_main({'spam': True}) - rc = _testinternalcapi.exec_interpreter(interpid, - 'assert spam is True') - assert rc == 0, rc + with self.interpreter_obj_from_capi() as (interp, interpid): + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + interp.prepare_main({'spam': True}) + with self.assertRaisesRegex(ExecutionFailed, 'NameError'): + self.run_from_capi(interpid, 'assert spam is True') class TestInterpreterExec(TestBase): @@ -835,7 +876,7 @@ def task(): def test_created_with_capi(self): with self.interpreter_obj_from_capi() as (interp, _): - with self.assertRaisesRegex(ExecutionFailed, 'it worked'): + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): interp.exec('raise Exception("it worked!")') # test_xxsubinterpreters covers the remaining @@ -1114,14 +1155,6 @@ class LowLevelTests(TestBase): # encountered by the high-level module, thus they # mostly shouldn't matter as much. - def interp_exists(self, interpid): - try: - _interpreters.is_running(interpid) - except InterpreterNotFoundError: - return False - else: - return True - def test_new_config(self): # This test overlaps with # test.test_capi.test_misc.InterpreterConfigTests. @@ -1245,32 +1278,35 @@ def test_new_config(self): _interpreters.new_config(gil=value) def test_get_main(self): - interpid, = _interpreters.get_main() + interpid, whence = _interpreters.get_main() self.assertEqual(interpid, 0) + self.assertEqual(whence, _interpreters.WHENCE_RUNTIME) + self.assertEqual( + _interpreters.whence(interpid), + _interpreters.WHENCE_RUNTIME) def test_get_current(self): with self.subTest('main'): main, *_ = _interpreters.get_main() - interpid, = _interpreters.get_current() + interpid, whence = _interpreters.get_current() self.assertEqual(interpid, main) + self.assertEqual(whence, _interpreters.WHENCE_RUNTIME) script = f""" import {_interpreters.__name__} as _interpreters - interpid, = _interpreters.get_current() - print(interpid) + interpid, whence = _interpreters.get_current() + print((interpid, whence)) """ def parse_stdout(text): - parts = text.split() - assert len(parts) == 1, parts - interpid, = parts - interpid = int(interpid) - return interpid, + interpid, whence = eval(text) + return interpid, whence with self.subTest('from _interpreters'): orig = _interpreters.create() text = self.run_and_capture(orig, script) - interpid, = parse_stdout(text) + interpid, whence = parse_stdout(text) self.assertEqual(interpid, orig) + self.assertEqual(whence, _interpreters.WHENCE_STDLIB) with self.subTest('from C-API'): last = 0 @@ -1278,8 +1314,9 @@ def parse_stdout(text): last = max(last, id) expected = last + 1 text = self.run_temp_from_capi(script) - interpid, = parse_stdout(text) + interpid, whence = parse_stdout(text) self.assertEqual(interpid, expected) + self.assertEqual(whence, _interpreters.WHENCE_CAPI) def test_list_all(self): mainid, *_ = _interpreters.get_main() @@ -1287,17 +1324,17 @@ def test_list_all(self): interpid2 = _interpreters.create() interpid3 = _interpreters.create() expected = [ - (mainid,), - (interpid1,), - (interpid2,), - (interpid3,), + (mainid, _interpreters.WHENCE_RUNTIME), + (interpid1, _interpreters.WHENCE_STDLIB), + (interpid2, _interpreters.WHENCE_STDLIB), + (interpid3, _interpreters.WHENCE_STDLIB), ] with self.subTest('main'): res = _interpreters.list_all() self.assertEqual(res, expected) - with self.subTest('from _interpreters'): + with self.subTest('via interp from _interpreters'): text = self.run_and_capture(interpid2, f""" import {_interpreters.__name__} as _interpreters print( @@ -1307,15 +1344,15 @@ def test_list_all(self): res = eval(text) self.assertEqual(res, expected) - with self.subTest('from C-API'): + with self.subTest('via interp from C-API'): interpid4 = interpid3 + 1 interpid5 = interpid4 + 1 expected2 = expected + [ - (interpid4,), - (interpid5,), + (interpid4, _interpreters.WHENCE_CAPI), + (interpid5, _interpreters.WHENCE_STDLIB), ] expected3 = expected + [ - (interpid5,), + (interpid5, _interpreters.WHENCE_STDLIB), ] text = self.run_temp_from_capi(f""" import {_interpreters.__name__} as _interpreters @@ -1380,6 +1417,12 @@ def test_create(self): with self.assertRaises(ValueError): _interpreters.create(orig) + with self.subTest('whence'): + interpid = _interpreters.create() + self.assertEqual( + _interpreters.whence(interpid), + _interpreters.WHENCE_STDLIB) + @requires_test_modules def test_destroy(self): with self.subTest('from _interpreters'): @@ -1401,6 +1444,10 @@ def test_destroy(self): with self.subTest('from C-API'): interpid = _testinternalcapi.create_interpreter() + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + _interpreters.destroy(interpid, restrict=True) + self.assertTrue( + self.interp_exists(interpid)) _interpreters.destroy(interpid) self.assertFalse( self.interp_exists(interpid)) @@ -1433,6 +1480,8 @@ def test_get_config(self): with self.subTest('from C-API'): orig = _interpreters.new_config('isolated') with self.interpreter_from_capi(orig) as interpid: + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + _interpreters.get_config(interpid, restrict=True) config = _interpreters.get_config(interpid) self.assert_ns_equal(config, orig) @@ -1446,10 +1495,10 @@ def test_whence(self): with self.subTest('stdlib'): interpid = _interpreters.create() whence = _interpreters.whence(interpid) - self.assertEqual(whence, _interpreters.WHENCE_XI) + self.assertEqual(whence, _interpreters.WHENCE_STDLIB) for orig, name in { - # XXX Also check WHENCE_UNKNOWN. + _interpreters.WHENCE_UNKNOWN: 'not ready', _interpreters.WHENCE_LEGACY_CAPI: 'legacy C-API', _interpreters.WHENCE_CAPI: 'C-API', _interpreters.WHENCE_XI: 'cross-interpreter C-API', @@ -1481,38 +1530,40 @@ def test_whence(self): self.assertEqual(whence, _interpreters.WHENCE_LEGACY_CAPI) def test_is_running(self): - with self.subTest('main'): - interpid, *_ = _interpreters.get_main() + def check(interpid, expected): + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + _interpreters.is_running(interpid, restrict=True) running = _interpreters.is_running(interpid) - self.assertTrue(running) + self.assertIs(running, expected) with self.subTest('from _interpreters (running)'): interpid = _interpreters.create() with self.running(interpid): running = _interpreters.is_running(interpid) - self.assertTrue(running) + self.assertTrue(running) with self.subTest('from _interpreters (not running)'): interpid = _interpreters.create() running = _interpreters.is_running(interpid) self.assertFalse(running) + with self.subTest('main'): + interpid, *_ = _interpreters.get_main() + check(interpid, True) + with self.subTest('from C-API (running __main__)'): with self.interpreter_from_capi() as interpid: with self.running_from_capi(interpid, main=True): - running = _interpreters.is_running(interpid) - self.assertTrue(running) + check(interpid, True) with self.subTest('from C-API (running, but not __main__)'): with self.interpreter_from_capi() as interpid: with self.running_from_capi(interpid, main=False): - running = _interpreters.is_running(interpid) - self.assertFalse(running) + check(interpid, False) with self.subTest('from C-API (not running)'): with self.interpreter_from_capi() as interpid: - running = _interpreters.is_running(interpid) - self.assertFalse(running) + check(interpid, False) def test_exec(self): with self.subTest('run script'): @@ -1549,6 +1600,9 @@ def test_exec(self): with self.subTest('from C-API'): with self.interpreter_from_capi() as interpid: + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + _interpreters.exec(interpid, 'raise Exception("it worked!")', + restrict=True) exc = _interpreters.exec(interpid, 'raise Exception("it worked!")') self.assertIsNot(exc, None) self.assertEqual(exc.msg, 'it worked!') @@ -1574,6 +1628,7 @@ def test_call(self): errdisplay=exc.errdisplay, )) + @requires_test_modules def test_set___main___attrs(self): with self.subTest('from _interpreters'): interpid = _interpreters.create() @@ -1595,9 +1650,15 @@ def test_set___main___attrs(self): with self.subTest('from C-API'): with self.interpreter_from_capi() as interpid: + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + _interpreters.set___main___attrs(interpid, {'spam': True}, + restrict=True) _interpreters.set___main___attrs(interpid, {'spam': True}) - exc = _interpreters.exec(interpid, 'assert spam is True') - self.assertIsNone(exc) + rc = _testinternalcapi.exec_interpreter( + interpid, + 'assert spam is True', + ) + self.assertEqual(rc, 0) if __name__ == '__main__': diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py index d92179474959ef..08768c07c28ebf 100644 --- a/Lib/test/test_interpreters/utils.py +++ b/Lib/test/test_interpreters/utils.py @@ -509,6 +509,14 @@ def run_and_capture(self, interp, script): else: return text + def interp_exists(self, interpid): + try: + _interpreters.whence(interpid) + except _interpreters.InterpreterNotFoundError: + return False + else: + return True + @requires_test_modules @contextlib.contextmanager def interpreter_from_capi(self, config=None, whence=None): @@ -545,7 +553,12 @@ def interpreter_from_capi(self, config=None, whence=None): @contextlib.contextmanager def interpreter_obj_from_capi(self, config='legacy'): with self.interpreter_from_capi(config) as interpid: - yield interpreters.Interpreter(interpid), interpid + interp = interpreters.Interpreter( + interpid, + _whence=_interpreters.WHENCE_CAPI, + _ownsref=False, + ) + yield interp, interpid @contextlib.contextmanager def capturing(self, script): diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 538e4ceafd1743..36dad024d31c95 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1394,7 +1394,7 @@ static PyInterpreterState * _new_interpreter(PyInterpreterConfig *config, long whence) { if (whence == _PyInterpreterState_WHENCE_XI) { - return _PyXI_NewInterpreter(config, NULL, NULL); + return _PyXI_NewInterpreter(config, &whence, NULL, NULL); } PyObject *exc = NULL; PyInterpreterState *interp = NULL; @@ -1610,7 +1610,7 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) /* Create an interpreter, staying switched to it. */ PyInterpreterState *interp = \ - _PyXI_NewInterpreter(&config, &tstate, &save_tstate); + _PyXI_NewInterpreter(&config, NULL, &tstate, &save_tstate); if (interp == NULL) { return NULL; } diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 37ac5a3f28aba9..8fcd4fc4154882 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -491,6 +491,55 @@ _run_in_interpreter(PyInterpreterState *interp, /* module level code ********************************************************/ +static long +get_whence(PyInterpreterState *interp) +{ + return _PyInterpreterState_GetWhence(interp); +} + + +static PyInterpreterState * +resolve_interp(PyObject *idobj, int restricted, int reqready, const char *op) +{ + PyInterpreterState *interp; + if (idobj == NULL) { + interp = PyInterpreterState_Get(); + } + else { + interp = look_up_interp(idobj); + if (interp == NULL) { + return NULL; + } + } + + if (reqready && !_PyInterpreterState_IsReady(interp)) { + if (idobj == NULL) { + PyErr_Format(PyExc_InterpreterError, + "cannot %s current interpreter (not ready)", op); + } + else { + PyErr_Format(PyExc_InterpreterError, + "cannot %s interpreter %R (not ready)", op, idobj); + } + return NULL; + } + + if (restricted && get_whence(interp) != _PyInterpreterState_WHENCE_STDLIB) { + if (idobj == NULL) { + PyErr_Format(PyExc_InterpreterError, + "cannot %s unrecognized current interpreter", op); + } + else { + PyErr_Format(PyExc_InterpreterError, + "cannot %s unrecognized interpreter %R", op, idobj); + } + return NULL; + } + + return interp; +} + + static PyObject * get_summary(PyInterpreterState *interp) { @@ -498,8 +547,15 @@ get_summary(PyInterpreterState *interp) if (idobj == NULL) { return NULL; } - PyObject *res = PyTuple_Pack(1, idobj); + PyObject *whenceobj = PyLong_FromLong( + get_whence(interp)); + if (whenceobj == NULL) { + Py_DECREF(idobj); + return NULL; + } + PyObject *res = PyTuple_Pack(2, idobj, whenceobj); Py_DECREF(idobj); + Py_DECREF(whenceobj); return res; } @@ -564,7 +620,9 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - PyInterpreterState *interp = _PyXI_NewInterpreter(&config, NULL, NULL); + long whence = _PyInterpreterState_WHENCE_STDLIB; + PyInterpreterState *interp = \ + _PyXI_NewInterpreter(&config, &whence, NULL, NULL); if (interp == NULL) { // XXX Move the chained exception to interpreters.create()? PyObject *exc = PyErr_GetRaisedException(); @@ -573,6 +631,7 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds) _PyErr_ChainExceptions1(exc); return NULL; } + assert(_PyInterpreterState_IsReady(interp)); PyObject *idobj = _PyInterpreterState_GetIDObject(interp); if (idobj == NULL) { @@ -607,16 +666,20 @@ is \"isolated\"."); static PyObject * interp_destroy(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"id", NULL}; + static char *kwlist[] = {"id", "restrict", NULL}; PyObject *id; + int restricted = 0; // XXX Use "L" for id? if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O:destroy", kwlist, &id)) { + "O|$p:destroy", kwlist, &id, &restricted)) + { return NULL; } // Look up the interpreter. - PyInterpreterState *interp = look_up_interp(id); + int reqready = 0; + PyInterpreterState *interp = \ + resolve_interp(id, restricted, reqready, "destroy"); if (interp == NULL) { return NULL; } @@ -647,7 +710,7 @@ interp_destroy(PyObject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(destroy_doc, -"destroy(id)\n\ +"destroy(id, *, restrict=False)\n\ \n\ Destroy the identified interpreter.\n\ \n\ @@ -656,31 +719,39 @@ So does an unrecognized ID."); static PyObject * -interp_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) +interp_list_all(PyObject *self, PyObject *args, PyObject *kwargs) { - PyObject *ids; - PyInterpreterState *interp; + static char *kwlist[] = {"require_ready", NULL}; + int reqready = 0; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "|$p:" MODULE_NAME_STR ".list_all", + kwlist, &reqready)) + { + return NULL; + } - ids = PyList_New(0); + PyObject *ids = PyList_New(0); if (ids == NULL) { return NULL; } - interp = PyInterpreterState_Head(); + PyInterpreterState *interp = PyInterpreterState_Head(); while (interp != NULL) { - PyObject *item = get_summary(interp); - if (item == NULL) { - Py_DECREF(ids); - return NULL; - } - // insert at front of list - int res = PyList_Insert(ids, 0, item); - Py_DECREF(item); - if (res < 0) { - Py_DECREF(ids); - return NULL; - } + if (!reqready || _PyInterpreterState_IsReady(interp)) { + PyObject *item = get_summary(interp); + if (item == NULL) { + Py_DECREF(ids); + return NULL; + } + // insert at front of list + int res = PyList_Insert(ids, 0, item); + Py_DECREF(item); + if (res < 0) { + Py_DECREF(ids); + return NULL; + } + } interp = PyInterpreterState_Next(interp); } @@ -688,7 +759,7 @@ interp_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(list_all_doc, -"list_all() -> [(ID,)]\n\ +"list_all() -> [(ID, whence)]\n\ \n\ Return a list containing the ID of every existing interpreter."); @@ -700,11 +771,12 @@ interp_get_current(PyObject *self, PyObject *Py_UNUSED(ignored)) if (interp == NULL) { return NULL; } + assert(_PyInterpreterState_IsReady(interp)); return get_summary(interp); } PyDoc_STRVAR(get_current_doc, -"get_current() -> (ID,)\n\ +"get_current() -> (ID, whence)\n\ \n\ Return the ID of current interpreter."); @@ -713,26 +785,33 @@ static PyObject * interp_get_main(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyInterpreterState *interp = _PyInterpreterState_Main(); + assert(_PyInterpreterState_IsReady(interp)); return get_summary(interp); } PyDoc_STRVAR(get_main_doc, -"get_main() -> (ID,)\n\ +"get_main() -> (ID, whence)\n\ \n\ Return the ID of main interpreter."); + static PyObject * -interp_set___main___attrs(PyObject *self, PyObject *args) +interp_set___main___attrs(PyObject *self, PyObject *args, PyObject *kwargs) { + static char *kwlist[] = {"id", "updates", "restrict", NULL}; PyObject *id, *updates; - if (!PyArg_ParseTuple(args, "OO:" MODULE_NAME_STR ".set___main___attrs", - &id, &updates)) + int restricted = 0; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "OO|$p:" MODULE_NAME_STR ".set___main___attrs", + kwlist, &id, &updates, &restricted)) { return NULL; } // Look up the interpreter. - PyInterpreterState *interp = look_up_interp(id); + int reqready = 1; + PyInterpreterState *interp = \ + resolve_interp(id, restricted, reqready, "update __main__ for"); if (interp == NULL) { return NULL; } @@ -771,10 +850,11 @@ interp_set___main___attrs(PyObject *self, PyObject *args) } PyDoc_STRVAR(set___main___attrs_doc, -"set___main___attrs(id, ns)\n\ +"set___main___attrs(id, ns, *, restrict=False)\n\ \n\ Bind the given attributes in the interpreter's __main__ module."); + static PyUnicodeObject * convert_script_arg(PyObject *arg, const char *fname, const char *displayname, const char *expected) @@ -852,16 +932,9 @@ convert_code_arg(PyObject *arg, const char *fname, const char *displayname, } static int -_interp_exec(PyObject *self, - PyObject *id_arg, PyObject *code_arg, PyObject *shared_arg, - PyObject **p_excinfo) +_interp_exec(PyObject *self, PyInterpreterState *interp, + PyObject *code_arg, PyObject *shared_arg, PyObject **p_excinfo) { - // Look up the interpreter. - PyInterpreterState *interp = look_up_interp(id_arg); - if (interp == NULL) { - return -1; - } - // Extract code. Py_ssize_t codestrlen = -1; PyObject *bytes_obj = NULL; @@ -886,12 +959,21 @@ _interp_exec(PyObject *self, static PyObject * interp_exec(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"id", "code", "shared", NULL}; + static char *kwlist[] = {"id", "code", "shared", "restrict", NULL}; PyObject *id, *code; PyObject *shared = NULL; + int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OO|O:" MODULE_NAME_STR ".exec", kwlist, - &id, &code, &shared)) { + "OO|O$p:" MODULE_NAME_STR ".exec", kwlist, + &id, &code, &shared, &restricted)) + { + return NULL; + } + + int reqready = 1; + PyInterpreterState *interp = \ + resolve_interp(id, restricted, reqready, "exec code for"); + if (interp == NULL) { return NULL; } @@ -909,7 +991,7 @@ interp_exec(PyObject *self, PyObject *args, PyObject *kwds) } PyObject *excinfo = NULL; - int res = _interp_exec(self, id, code, shared, &excinfo); + int res = _interp_exec(self, interp, code, shared, &excinfo); Py_DECREF(code); if (res < 0) { assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); @@ -919,7 +1001,7 @@ interp_exec(PyObject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(exec_doc, -"exec(id, code, shared=None)\n\ +"exec(id, code, shared=None, *, restrict=False)\n\ \n\ Execute the provided code in the identified interpreter.\n\ This is equivalent to running the builtin exec() under the target\n\ @@ -938,13 +1020,24 @@ is ignored, including its __globals__ dict."); static PyObject * interp_call(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"id", "callable", "args", "kwargs", NULL}; + static char *kwlist[] = {"id", "callable", "args", "kwargs", + "restrict", NULL}; PyObject *id, *callable; PyObject *args_obj = NULL; PyObject *kwargs_obj = NULL; + int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OO|OO:" MODULE_NAME_STR ".call", kwlist, - &id, &callable, &args_obj, &kwargs_obj)) { + "OO|OO$p:" MODULE_NAME_STR ".call", kwlist, + &id, &callable, &args_obj, &kwargs_obj, + &restricted)) + { + return NULL; + } + + int reqready = 1; + PyInterpreterState *interp = \ + resolve_interp(id, restricted, reqready, "make a call in"); + if (interp == NULL) { return NULL; } @@ -964,7 +1057,7 @@ interp_call(PyObject *self, PyObject *args, PyObject *kwds) } PyObject *excinfo = NULL; - int res = _interp_exec(self, id, code, NULL, &excinfo); + int res = _interp_exec(self, interp, code, NULL, &excinfo); Py_DECREF(code); if (res < 0) { assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); @@ -974,7 +1067,7 @@ interp_call(PyObject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(call_doc, -"call(id, callable, args=None, kwargs=None)\n\ +"call(id, callable, args=None, kwargs=None, *, restrict=False)\n\ \n\ Call the provided object in the identified interpreter.\n\ Pass the given args and kwargs, if possible.\n\ @@ -988,12 +1081,21 @@ is ignored, including its __globals__ dict."); static PyObject * interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"id", "script", "shared", NULL}; + static char *kwlist[] = {"id", "script", "shared", "restrict", NULL}; PyObject *id, *script; PyObject *shared = NULL; + int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OU|O:" MODULE_NAME_STR ".run_string", kwlist, - &id, &script, &shared)) { + "OU|O$p:" MODULE_NAME_STR ".run_string", + kwlist, &id, &script, &shared, &restricted)) + { + return NULL; + } + + int reqready = 1; + PyInterpreterState *interp = \ + resolve_interp(id, restricted, reqready, "run a string in"); + if (interp == NULL) { return NULL; } @@ -1004,7 +1106,7 @@ interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) } PyObject *excinfo = NULL; - int res = _interp_exec(self, id, script, shared, &excinfo); + int res = _interp_exec(self, interp, script, shared, &excinfo); Py_DECREF(script); if (res < 0) { assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); @@ -1014,7 +1116,7 @@ interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(run_string_doc, -"run_string(id, script, shared=None)\n\ +"run_string(id, script, shared=None, *, restrict=False)\n\ \n\ Execute the provided string in the identified interpreter.\n\ \n\ @@ -1023,12 +1125,21 @@ Execute the provided string in the identified interpreter.\n\ static PyObject * interp_run_func(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"id", "func", "shared", NULL}; + static char *kwlist[] = {"id", "func", "shared", "restrict", NULL}; PyObject *id, *func; PyObject *shared = NULL; + int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OO|O:" MODULE_NAME_STR ".run_func", kwlist, - &id, &func, &shared)) { + "OO|O$p:" MODULE_NAME_STR ".run_func", + kwlist, &id, &func, &shared, &restricted)) + { + return NULL; + } + + int reqready = 1; + PyInterpreterState *interp = \ + resolve_interp(id, restricted, reqready, "run a function in"); + if (interp == NULL) { return NULL; } @@ -1040,7 +1151,7 @@ interp_run_func(PyObject *self, PyObject *args, PyObject *kwds) } PyObject *excinfo = NULL; - int res = _interp_exec(self, id, (PyObject *)code, shared, &excinfo); + int res = _interp_exec(self, interp, (PyObject *)code, shared, &excinfo); Py_DECREF(code); if (res < 0) { assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); @@ -1050,7 +1161,7 @@ interp_run_func(PyObject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(run_func_doc, -"run_func(id, func, shared=None)\n\ +"run_func(id, func, shared=None, *, restrict=False)\n\ \n\ Execute the body of the provided function in the identified interpreter.\n\ Code objects are also supported. In both cases, closures and args\n\ @@ -1086,17 +1197,23 @@ False otherwise."); static PyObject * interp_is_running(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"id", NULL}; + static char *kwlist[] = {"id", "restrict", NULL}; PyObject *id; + int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O:is_running", kwlist, &id)) { + "O|$p:is_running", kwlist, + &id, &restricted)) + { return NULL; } - PyInterpreterState *interp = look_up_interp(id); + int reqready = 1; + PyInterpreterState *interp = \ + resolve_interp(id, restricted, reqready, "check if running for"); if (interp == NULL) { return NULL; } + if (is_running_main(interp)) { Py_RETURN_TRUE; } @@ -1104,7 +1221,7 @@ interp_is_running(PyObject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(is_running_doc, -"is_running(id) -> bool\n\ +"is_running(id, *, restrict=False) -> bool\n\ \n\ Return whether or not the identified interpreter is running."); @@ -1112,23 +1229,24 @@ Return whether or not the identified interpreter is running."); static PyObject * interp_get_config(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"id", NULL}; + static char *kwlist[] = {"id", "restrict", NULL}; PyObject *idobj = NULL; + int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O:get_config", kwlist, &idobj)) + "O|$p:get_config", kwlist, + &idobj, &restricted)) { return NULL; } - - PyInterpreterState *interp; - if (idobj == NULL) { - interp = PyInterpreterState_Get(); + if (idobj == Py_None) { + idobj = NULL; } - else { - interp = _PyInterpreterState_LookUpIDObject(idobj); - if (interp == NULL) { - return NULL; - } + + int reqready = 0; + PyInterpreterState *interp = \ + resolve_interp(idobj, restricted, reqready, "get the config of"); + if (interp == NULL) { + return NULL; } PyInterpreterConfig config; @@ -1146,7 +1264,7 @@ interp_get_config(PyObject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(get_config_doc, -"get_config(id) -> types.SimpleNamespace\n\ +"get_config(id, *, restrict=False) -> types.SimpleNamespace\n\ \n\ Return a representation of the config used to initialize the interpreter."); @@ -1167,7 +1285,7 @@ interp_whence(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - long whence = _PyInterpreterState_GetWhence(interp); + long whence = get_whence(interp); return PyLong_FromLong(whence); } @@ -1180,17 +1298,20 @@ Return an identifier for where the interpreter was created."); static PyObject * interp_incref(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"id", "implieslink", NULL}; + static char *kwlist[] = {"id", "implieslink", "restrict", NULL}; PyObject *id; int implieslink = 0; + int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O|$p:incref", kwlist, - &id, &implieslink)) + "O|$pp:incref", kwlist, + &id, &implieslink, &restricted)) { return NULL; } - PyInterpreterState *interp = look_up_interp(id); + int reqready = 1; + PyInterpreterState *interp = \ + resolve_interp(id, restricted, reqready, "incref"); if (interp == NULL) { return NULL; } @@ -1208,17 +1329,22 @@ interp_incref(PyObject *self, PyObject *args, PyObject *kwds) static PyObject * interp_decref(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"id", NULL}; + static char *kwlist[] = {"id", "restrict", NULL}; PyObject *id; + int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O:decref", kwlist, &id)) { + "O|$p:decref", kwlist, &id, &restricted)) + { return NULL; } - PyInterpreterState *interp = look_up_interp(id); + int reqready = 1; + PyInterpreterState *interp = \ + resolve_interp(id, restricted, reqready, "decref"); if (interp == NULL) { return NULL; } + _PyInterpreterState_IDDecref(interp); Py_RETURN_NONE; @@ -1301,8 +1427,8 @@ static PyMethodDef module_functions[] = { METH_VARARGS | METH_KEYWORDS, create_doc}, {"destroy", _PyCFunction_CAST(interp_destroy), METH_VARARGS | METH_KEYWORDS, destroy_doc}, - {"list_all", interp_list_all, - METH_NOARGS, list_all_doc}, + {"list_all", _PyCFunction_CAST(interp_list_all), + METH_VARARGS | METH_KEYWORDS, list_all_doc}, {"get_current", interp_get_current, METH_NOARGS, get_current_doc}, {"get_main", interp_get_main, @@ -1324,7 +1450,7 @@ static PyMethodDef module_functions[] = { METH_VARARGS | METH_KEYWORDS, run_func_doc}, {"set___main___attrs", _PyCFunction_CAST(interp_set___main___attrs), - METH_VARARGS, set___main___attrs_doc}, + METH_VARARGS | METH_KEYWORDS, set___main___attrs_doc}, {"incref", _PyCFunction_CAST(interp_incref), METH_VARARGS | METH_KEYWORDS, NULL}, @@ -1364,6 +1490,7 @@ module_exec(PyObject *mod) ADD_WHENCE(LEGACY_CAPI) ADD_WHENCE(CAPI) ADD_WHENCE(XI) + ADD_WHENCE(STDLIB) #undef ADD_WHENCE // exceptions diff --git a/Python/crossinterp.c b/Python/crossinterp.c index fb0dae0bbb8f75..367e29d40d895a 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -1822,7 +1822,7 @@ _PyXI_FiniTypes(PyInterpreterState *interp) /*************/ PyInterpreterState * -_PyXI_NewInterpreter(PyInterpreterConfig *config, +_PyXI_NewInterpreter(PyInterpreterConfig *config, long *maybe_whence, PyThreadState **p_tstate, PyThreadState **p_save_tstate) { PyThreadState *save_tstate = PyThreadState_Swap(NULL); @@ -1845,7 +1845,11 @@ _PyXI_NewInterpreter(PyInterpreterConfig *config, assert(tstate != NULL); PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate); - _PyInterpreterState_SetWhence(interp, _PyInterpreterState_WHENCE_XI); + long whence = _PyInterpreterState_WHENCE_XI; + if (maybe_whence != NULL) { + whence = *maybe_whence; + } + _PyInterpreterState_SetWhence(interp, whence); if (p_tstate != NULL) { // We leave the new thread state as the current one. @@ -1868,6 +1872,22 @@ void _PyXI_EndInterpreter(PyInterpreterState *interp, PyThreadState *tstate, PyThreadState **p_save_tstate) { +#ifndef NDEBUG + long whence = _PyInterpreterState_GetWhence(interp); +#endif + assert(whence != _PyInterpreterState_WHENCE_RUNTIME); + + if (!_PyInterpreterState_IsReady(interp)) { + assert(whence == _PyInterpreterState_WHENCE_UNKNOWN); + // PyInterpreterState_Clear() requires the GIL, + // which a not-ready does not have, so we don't clear it. + // That means there may be leaks here until clearing the + // interpreter is fixed. + PyInterpreterState_Delete(interp); + return; + } + assert(whence != _PyInterpreterState_WHENCE_UNKNOWN); + PyThreadState *save_tstate = NULL; PyThreadState *cur_tstate = PyThreadState_GET(); if (tstate == NULL) { @@ -1889,18 +1909,7 @@ _PyXI_EndInterpreter(PyInterpreterState *interp, } } - long whence = _PyInterpreterState_GetWhence(interp); - assert(whence != _PyInterpreterState_WHENCE_RUNTIME); - if (whence == _PyInterpreterState_WHENCE_UNKNOWN) { - assert(!interp->_ready); - PyThreadState *tstate = PyThreadState_New(interp); - save_tstate = PyThreadState_Swap(tstate); - _PyInterpreterState_Clear(tstate); - PyInterpreterState_Delete(interp); - } - else { - Py_EndInterpreter(tstate); - } + Py_EndInterpreter(tstate); if (p_save_tstate != NULL) { save_tstate = *p_save_tstate; diff --git a/Python/pystate.c b/Python/pystate.c index b0fb210bb5f5f6..acec905484c21f 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1111,6 +1111,13 @@ _PyInterpreterState_ReinitRunningMain(PyThreadState *tstate) // accessors //---------- +int +_PyInterpreterState_IsReady(PyInterpreterState *interp) +{ + return interp->_ready; +} + + static inline int check_interpreter_whence(long whence) {