From f5b464a8cb689ec0f91c27bc9b5482617f93c60e Mon Sep 17 00:00:00 2001 From: Joannah Nanjekye Date: Tue, 10 Sep 2019 11:00:52 +0000 Subject: [PATCH 01/20] Add tests for the interpreters module --- Lib/test/test_interpreters.py | 2119 +++++++++++++++++++++++++++++++++ 1 file changed, 2119 insertions(+) create mode 100644 Lib/test/test_interpreters.py diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py new file mode 100644 index 00000000000000..65c6e77d69d1ee --- /dev/null +++ b/Lib/test/test_interpreters.py @@ -0,0 +1,2119 @@ +import interpreters +from collections import namedtuple +import contextlib +import itertools +import os +import pickle +import sys +from textwrap import dedent +import threading +import time +import unittest + +from test import support +from test.support import script_helper + + +################################## +# helpers + +def powerset(*sets): + return itertools.chain.from_iterable( + combinations(sets, r) + for r in range(len(sets)+1)) + + +def _captured_script(script): + r, w = os.pipe() + indented = script.replace('\n', '\n ') + wrapped = dedent(f""" + import contextlib + with open({w}, 'w') as spipe: + with contextlib.redirect_stdout(spipe): + {indented} + """) + return wrapped, open(r) + + +def _run_output(interp, request, shared=None): + script, rpipe = _captured_script(request) + with rpipe: + interpreters.run_string(interp, script, shared) + return rpipe.read() + + +@contextlib.contextmanager +def _running(interp): + r, w = os.pipe() + def run(): + interpreters.run_string(interp, dedent(f""" + # wait for "signal" + with open({r}) as rpipe: + rpipe.read() + """)) + + t = threading.Thread(target=run) + t.start() + + yield + + with open(w, 'w') as spipe: + spipe.write('done') + t.join() + + +#@contextmanager +#def run_threaded(id, source, **shared): +# def run(): +# run_interp(id, source, **shared) +# t = threading.Thread(target=run) +# t.start() +# yield +# t.join() + + +def run_interp(id, source, **shared): + _run_interp(id, source, shared) + + +def _run_interp(id, source, shared, _mainns={}): + source = dedent(source) + main = interpreters.get_main() + if main == id: + if interpreters.get_current() != main: + raise RuntimeError + # XXX Run a func? + exec(source, _mainns) + else: + interpreters.run_string(id, source, shared) + + +def run_interp_threaded(id, source, **shared): + def run(): + _run(id, source, shared) + t = threading.Thread(target=run) + t.start() + t.join() + + +class Interpreter(namedtuple('Interpreter', 'name id')): + + @classmethod + def from_raw(cls, raw): + if isinstance(raw, cls): + return raw + elif isinstance(raw, str): + return cls(raw) + else: + raise NotImplementedError + + def __new__(cls, name=None, id=None): + main = interpreters.get_main() + if id == main: + if not name: + name = 'main' + elif name != 'main': + raise ValueError( + 'name mismatch (expected "main", got "{}")'.format(name)) + id = main + elif id is not None: + if not name: + name = 'interp' + elif name == 'main': + raise ValueError('name mismatch (unexpected "main")') + if not isinstance(id, interpreters.InterpreterID): + id = interpreters.InterpreterID(id) + elif not name or name == 'main': + name = 'main' + id = main + else: + id = interpreters.create() + self = super().__new__(cls, name, id) + return self + + +# XXX expect_channel_closed() is unnecessary once we improve exc propagation. + +@contextlib.contextmanager +def expect_channel_closed(): + try: + yield + except interpreters.ChannelClosedError: + pass + else: + assert False, 'channel not closed' + + +class ChannelAction(namedtuple('ChannelAction', 'action end interp')): + + def __new__(cls, action, end=None, interp=None): + if not end: + end = 'both' + if not interp: + interp = 'main' + self = super().__new__(cls, action, end, interp) + return self + + def __init__(self, *args, **kwargs): + if self.action == 'use': + if self.end not in ('same', 'opposite', 'send', 'recv'): + raise ValueError(self.end) + elif self.action in ('close', 'force-close'): + if self.end not in ('both', 'same', 'opposite', 'send', 'recv'): + raise ValueError(self.end) + else: + raise ValueError(self.action) + if self.interp not in ('main', 'same', 'other', 'extra'): + raise ValueError(self.interp) + + def resolve_end(self, end): + if self.end == 'same': + return end + elif self.end == 'opposite': + return 'recv' if end == 'send' else 'send' + else: + return self.end + + def resolve_interp(self, interp, other, extra): + if self.interp == 'same': + return interp + elif self.interp == 'other': + if other is None: + raise RuntimeError + return other + elif self.interp == 'extra': + if extra is None: + raise RuntimeError + return extra + elif self.interp == 'main': + if interp.name == 'main': + return interp + elif other and other.name == 'main': + return other + else: + raise RuntimeError + # Per __init__(), there aren't any others. + + +class ChannelState(namedtuple('ChannelState', 'pending closed')): + + def __new__(cls, pending=0, *, closed=False): + self = super().__new__(cls, pending, closed) + return self + + def incr(self): + return type(self)(self.pending + 1, closed=self.closed) + + def decr(self): + return type(self)(self.pending - 1, closed=self.closed) + + def close(self, *, force=True): + if self.closed: + if not force or self.pending == 0: + return self + return type(self)(0 if force else self.pending, closed=True) + + +def run_action(cid, action, end, state, *, hideclosed=True): + if state.closed: + if action == 'use' and end == 'recv' and state.pending: + expectfail = False + else: + expectfail = True + else: + expectfail = False + + try: + result = _run_action(cid, action, end, state) + except interpreters.ChannelClosedError: + if not hideclosed and not expectfail: + raise + result = state.close() + else: + if expectfail: + raise ... # XXX + return result + + +def _run_action(cid, action, end, state): + if action == 'use': + if end == 'send': + interpreters.channel_send(cid, b'spam') + return state.incr() + elif end == 'recv': + if not state.pending: + try: + interpreters.channel_recv(cid) + except interpreters.ChannelEmptyError: + return state + else: + raise Exception('expected ChannelEmptyError') + else: + interpreters.channel_recv(cid) + return state.decr() + else: + raise ValueError(end) + elif action == 'close': + kwargs = {} + if end in ('recv', 'send'): + kwargs[end] = True + interpreters.channel_close(cid, **kwargs) + return state.close() + elif action == 'force-close': + kwargs = { + 'force': True, + } + if end in ('recv', 'send'): + kwargs[end] = True + interpreters.channel_close(cid, **kwargs) + return state.close(force=True) + else: + raise ValueError(action) + + +def clean_up_interpreters(): + for id in interpreters.list_all(): + if id == 0: # main + continue + try: + interpreters.destroy(id) + except RuntimeError: + pass # already destroyed + + +def clean_up_channels(): + for cid in interpreters.channel_list_all(): + try: + interpreters.channel_destroy(cid) + except interpreters.ChannelNotFoundError: + pass # already destroyed + + +class TestBase(unittest.TestCase): + + def tearDown(self): + clean_up_interpreters() + clean_up_channels() + + +################################## +# misc. tests + +class IsShareableTests(unittest.TestCase): + + def test_default_shareables(self): + shareables = [ + # singletons + None, + # builtin objects + b'spam', + 'spam', + 10, + -10, + ] + for obj in shareables: + with self.subTest(obj): + self.assertTrue( + interpreters.is_shareable(obj)) + + def test_not_shareable(self): + class Cheese: + def __init__(self, name): + self.name = name + def __str__(self): + return self.name + + class SubBytes(bytes): + """A subclass of a shareable type.""" + + not_shareables = [ + # singletons + True, + False, + NotImplemented, + ..., + # builtin types and objects + type, + object, + object(), + Exception(), + 100.0, + # user-defined types and objects + Cheese, + Cheese('Wensleydale'), + SubBytes(b'spam'), + ] + for obj in not_shareables: + with self.subTest(repr(obj)): + self.assertFalse( + interpreters.is_shareable(obj)) + + +class ShareableTypeTests(unittest.TestCase): + + def setUp(self): + super().setUp() + self.cid = interpreters.channel_create() + + def tearDown(self): + interpreters.channel_destroy(self.cid) + super().tearDown() + + def _assert_values(self, values): + for obj in values: + with self.subTest(obj): + interpreters.channel_send(self.cid, obj) + got = interpreters.channel_recv(self.cid) + + self.assertEqual(got, obj) + self.assertIs(type(got), type(obj)) + # XXX Check the following in the channel tests? + #self.assertIsNot(got, obj) + + def test_singletons(self): + for obj in [None]: + with self.subTest(obj): + interpreters.channel_send(self.cid, obj) + got = interpreters.channel_recv(self.cid) + + # XXX What about between interpreters? + self.assertIs(got, obj) + + def test_types(self): + self._assert_values([ + b'spam', + 9999, + self.cid, + ]) + + def test_bytes(self): + self._assert_values(i.to_bytes(2, 'little', signed=True) + for i in range(-1, 258)) + + def test_int(self): + self._assert_values(itertools.chain(range(-1, 258), + [sys.maxsize, -sys.maxsize - 1])) + + def test_non_shareable_int(self): + ints = [ + sys.maxsize + 1, + -sys.maxsize - 2, + 2**1000, + ] + for i in ints: + with self.subTest(i): + with self.assertRaises(OverflowError): + interpreters.channel_send(self.cid, i) + + +################################## +# interpreter tests + +class ListAllTests(TestBase): + + def test_initial(self): + main = interpreters.get_main() + ids = interpreters.list_all() + self.assertEqual(ids, [main]) + + def test_after_creating(self): + main = interpreters.get_main() + first = interpreters.create() + second = interpreters.create() + ids = interpreters.list_all() + self.assertEqual(ids, [main, first, second]) + + def test_after_destroying(self): + main = interpreters.get_main() + first = interpreters.create() + second = interpreters.create() + interpreters.destroy(first) + ids = interpreters.list_all() + self.assertEqual(ids, [main, second]) + + +class GetCurrentTests(TestBase): + + def test_main(self): + main = interpreters.get_main() + cur = interpreters.get_current() + self.assertEqual(cur, main) + self.assertIsInstance(cur, interpreters.InterpreterID) + + def test_subinterpreter(self): + main = interpreters.get_main() + interp = interpreters.create() + out = _run_output(interp, dedent(""" + import _xxsubinterpreters as _interpreters + cur = _interpreters.get_current() + print(cur) + assert isinstance(cur, _interpreters.InterpreterID) + """)) + cur = int(out.strip()) + _, expected = interpreters.list_all() + self.assertEqual(cur, expected) + self.assertNotEqual(cur, main) + + +class GetMainTests(TestBase): + + def test_from_main(self): + [expected] = interpreters.list_all() + main = interpreters.get_main() + self.assertEqual(main, expected) + self.assertIsInstance(main, interpreters.InterpreterID) + + def test_from_subinterpreter(self): + [expected] = interpreters.list_all() + interp = interpreters.create() + out = _run_output(interp, dedent(""" + import _xxsubinterpreters as _interpreters + main = _interpreters.get_main() + print(main) + assert isinstance(main, _interpreters.InterpreterID) + """)) + main = int(out.strip()) + self.assertEqual(main, expected) + + +class IsRunningTests(TestBase): + + def test_main(self): + main = interpreters.get_main() + self.assertTrue(interpreters.is_running(main)) + + def test_subinterpreter(self): + interp = interpreters.create() + self.assertFalse(interpreters.is_running(interp)) + + with _running(interp): + self.assertTrue(interpreters.is_running(interp)) + self.assertFalse(interpreters.is_running(interp)) + + def test_from_subinterpreter(self): + interp = interpreters.create() + out = _run_output(interp, dedent(f""" + import _xxsubinterpreters as _interpreters + if _interpreters.is_running({interp}): + print(True) + else: + print(False) + """)) + self.assertEqual(out.strip(), 'True') + + def test_already_destroyed(self): + interp = interpreters.create() + interpreters.destroy(interp) + with self.assertRaises(RuntimeError): + interpreters.is_running(interp) + + def test_does_not_exist(self): + with self.assertRaises(RuntimeError): + interpreters.is_running(1_000_000) + + def test_bad_id(self): + with self.assertRaises(RuntimeError): + interpreters.is_running(-1) + + +class InterpreterIDTests(TestBase): + + def test_with_int(self): + id = interpreters.InterpreterID(10, force=True) + + self.assertEqual(int(id), 10) + + def test_coerce_id(self): + id = interpreters.InterpreterID('10', force=True) + self.assertEqual(int(id), 10) + + id = interpreters.InterpreterID(10.0, force=True) + self.assertEqual(int(id), 10) + + class Int(str): + def __init__(self, value): + self._value = value + def __int__(self): + return self._value + + id = interpreters.InterpreterID(Int(10), force=True) + self.assertEqual(int(id), 10) + + def test_bad_id(self): + for id in [-1, 'spam']: + with self.subTest(id): + with self.assertRaises(ValueError): + interpreters.InterpreterID(id) + with self.assertRaises(OverflowError): + interpreters.InterpreterID(2**64) + with self.assertRaises(TypeError): + interpreters.InterpreterID(object()) + + def test_does_not_exist(self): + id = interpreters.channel_create() + with self.assertRaises(RuntimeError): + interpreters.InterpreterID(int(id) + 1) # unforced + + def test_str(self): + id = interpreters.InterpreterID(10, force=True) + self.assertEqual(str(id), '10') + + def test_repr(self): + id = interpreters.InterpreterID(10, force=True) + self.assertEqual(repr(id), 'InterpreterID(10)') + + def test_equality(self): + id1 = interpreters.create() + id2 = interpreters.InterpreterID(int(id1)) + id3 = interpreters.create() + + self.assertTrue(id1 == id1) + self.assertTrue(id1 == id2) + self.assertTrue(id1 == int(id1)) + self.assertFalse(id1 == id3) + + self.assertFalse(id1 != id1) + self.assertFalse(id1 != id2) + self.assertTrue(id1 != id3) + + +class CreateTests(TestBase): + + def test_in_main(self): + id = interpreters.create() + self.assertIsInstance(id, interpreters.InterpreterID) + + self.assertIn(id, interpreters.list_all()) + + @unittest.skip('enable this test when working on pystate.c') + def test_unique_id(self): + seen = set() + for _ in range(100): + id = interpreters.create() + interpreters.destroy(id) + seen.add(id) + + self.assertEqual(len(seen), 100) + + def test_in_thread(self): + lock = threading.Lock() + id = None + def f(): + nonlocal id + id = interpreters.create() + lock.acquire() + lock.release() + + t = threading.Thread(target=f) + with lock: + t.start() + t.join() + self.assertIn(id, interpreters.list_all()) + + def test_in_subinterpreter(self): + main, = interpreters.list_all() + id1 = interpreters.create() + out = _run_output(id1, dedent(""" + import _xxsubinterpreters as _interpreters + id = _interpreters.create() + print(id) + assert isinstance(id, _interpreters.InterpreterID) + """)) + id2 = int(out.strip()) + + self.assertEqual(set(interpreters.list_all()), {main, id1, id2}) + + def test_in_threaded_subinterpreter(self): + main, = interpreters.list_all() + id1 = interpreters.create() + id2 = None + def f(): + nonlocal id2 + out = _run_output(id1, dedent(""" + import _xxsubinterpreters as _interpreters + id = _interpreters.create() + print(id) + """)) + id2 = int(out.strip()) + + t = threading.Thread(target=f) + t.start() + t.join() + + self.assertEqual(set(interpreters.list_all()), {main, id1, id2}) + + def test_after_destroy_all(self): + before = set(interpreters.list_all()) + # Create 3 subinterpreters. + ids = [] + for _ in range(3): + id = interpreters.create() + ids.append(id) + # Now destroy them. + for id in ids: + interpreters.destroy(id) + # Finally, create another. + id = interpreters.create() + self.assertEqual(set(interpreters.list_all()), before | {id}) + + def test_after_destroy_some(self): + before = set(interpreters.list_all()) + # Create 3 subinterpreters. + id1 = interpreters.create() + id2 = interpreters.create() + id3 = interpreters.create() + # Now destroy 2 of them. + interpreters.destroy(id1) + interpreters.destroy(id3) + # Finally, create another. + id = interpreters.create() + self.assertEqual(set(interpreters.list_all()), before | {id, id2}) + + +class DestroyTests(TestBase): + + def test_one(self): + id1 = interpreters.create() + id2 = interpreters.create() + id3 = interpreters.create() + self.assertIn(id2, interpreters.list_all()) + interpreters.destroy(id2) + self.assertNotIn(id2, interpreters.list_all()) + self.assertIn(id1, interpreters.list_all()) + self.assertIn(id3, interpreters.list_all()) + + def test_all(self): + before = set(interpreters.list_all()) + ids = set() + for _ in range(3): + id = interpreters.create() + ids.add(id) + self.assertEqual(set(interpreters.list_all()), before | ids) + for id in ids: + interpreters.destroy(id) + self.assertEqual(set(interpreters.list_all()), before) + + def test_main(self): + main, = interpreters.list_all() + with self.assertRaises(RuntimeError): + interpreters.destroy(main) + + def f(): + with self.assertRaises(RuntimeError): + interpreters.destroy(main) + + t = threading.Thread(target=f) + t.start() + t.join() + + def test_already_destroyed(self): + id = interpreters.create() + interpreters.destroy(id) + with self.assertRaises(RuntimeError): + interpreters.destroy(id) + + def test_does_not_exist(self): + with self.assertRaises(RuntimeError): + interpreters.destroy(1_000_000) + + def test_bad_id(self): + with self.assertRaises(RuntimeError): + interpreters.destroy(-1) + + def test_from_current(self): + main, = interpreters.list_all() + id = interpreters.create() + script = dedent(f""" + import _xxsubinterpreters as _interpreters + try: + _interpreters.destroy({id}) + except RuntimeError: + pass + """) + + interpreters.run_string(id, script) + self.assertEqual(set(interpreters.list_all()), {main, id}) + + def test_from_sibling(self): + main, = interpreters.list_all() + id1 = interpreters.create() + id2 = interpreters.create() + script = dedent(f""" + import _xxsubinterpreters as _interpreters + _interpreters.destroy({id2}) + """) + interpreters.run_string(id1, script) + + self.assertEqual(set(interpreters.list_all()), {main, id1}) + + def test_from_other_thread(self): + id = interpreters.create() + def f(): + interpreters.destroy(id) + + t = threading.Thread(target=f) + t.start() + t.join() + + def test_still_running(self): + main, = interpreters.list_all() + interp = interpreters.create() + with _running(interp): + with self.assertRaises(RuntimeError): + interpreters.destroy(interp) + self.assertTrue(interpreters.is_running(interp)) + + +class RunStringTests(TestBase): + + SCRIPT = dedent(""" + with open('{}', 'w') as out: + out.write('{}') + """) + FILENAME = 'spam' + + def setUp(self): + super().setUp() + self.id = interpreters.create() + self._fs = None + + def tearDown(self): + if self._fs is not None: + self._fs.close() + super().tearDown() + + @property + def fs(self): + if self._fs is None: + self._fs = FSFixture(self) + return self._fs + + def test_success(self): + script, file = _captured_script('print("it worked!", end="")') + with file: + interpreters.run_string(self.id, script) + out = file.read() + + self.assertEqual(out, 'it worked!') + + def test_in_thread(self): + script, file = _captured_script('print("it worked!", end="")') + with file: + def f(): + interpreters.run_string(self.id, script) + + t = threading.Thread(target=f) + t.start() + t.join() + out = file.read() + + self.assertEqual(out, 'it worked!') + + def test_create_thread(self): + script, file = _captured_script(""" + import threading + def f(): + print('it worked!', end='') + + t = threading.Thread(target=f) + t.start() + t.join() + """) + with file: + interpreters.run_string(self.id, script) + out = file.read() + + self.assertEqual(out, 'it worked!') + + @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") + def test_fork(self): + import tempfile + with tempfile.NamedTemporaryFile('w+') as file: + file.write('') + file.flush() + + expected = 'spam spam spam spam spam' + script = dedent(f""" + import os + try: + os.fork() + except RuntimeError: + with open('{file.name}', 'w') as out: + out.write('{expected}') + """) + interpreters.run_string(self.id, script) + + file.seek(0) + content = file.read() + self.assertEqual(content, expected) + + def test_already_running(self): + with _running(self.id): + with self.assertRaises(RuntimeError): + interpreters.run_string(self.id, 'print("spam")') + + def test_does_not_exist(self): + id = 0 + while id in interpreters.list_all(): + id += 1 + with self.assertRaises(RuntimeError): + interpreters.run_string(id, 'print("spam")') + + def test_error_id(self): + with self.assertRaises(RuntimeError): + interpreters.run_string(-1, 'print("spam")') + + def test_bad_id(self): + with self.assertRaises(TypeError): + interpreters.run_string('spam', 'print("spam")') + + def test_bad_script(self): + with self.assertRaises(TypeError): + interpreters.run_string(self.id, 10) + + def test_bytes_for_script(self): + with self.assertRaises(TypeError): + interpreters.run_string(self.id, b'print("spam")') + + @contextlib.contextmanager + def assert_run_failed(self, exctype, msg=None): + with self.assertRaises(interpreters.RunFailedError) as caught: + yield + if msg is None: + self.assertEqual(str(caught.exception).split(':')[0], + str(exctype)) + else: + self.assertEqual(str(caught.exception), + "{}: {}".format(exctype, msg)) + + def test_invalid_syntax(self): + with self.assert_run_failed(SyntaxError): + # missing close paren + interpreters.run_string(self.id, 'print("spam"') + + def test_failure(self): + with self.assert_run_failed(Exception, 'spam'): + interpreters.run_string(self.id, 'raise Exception("spam")') + + def test_SystemExit(self): + with self.assert_run_failed(SystemExit, '42'): + interpreters.run_string(self.id, 'raise SystemExit(42)') + + def test_sys_exit(self): + with self.assert_run_failed(SystemExit): + interpreters.run_string(self.id, dedent(""" + import sys + sys.exit() + """)) + + with self.assert_run_failed(SystemExit, '42'): + interpreters.run_string(self.id, dedent(""" + import sys + sys.exit(42) + """)) + + def test_with_shared(self): + r, w = os.pipe() + + shared = { + 'spam': b'ham', + 'eggs': b'-1', + 'cheddar': None, + } + script = dedent(f""" + eggs = int(eggs) + spam = 42 + result = spam + eggs + + ns = dict(vars()) + del ns['__builtins__'] + import pickle + with open({w}, 'wb') as chan: + pickle.dump(ns, chan) + """) + interpreters.run_string(self.id, script, shared) + with open(r, 'rb') as chan: + ns = pickle.load(chan) + + self.assertEqual(ns['spam'], 42) + self.assertEqual(ns['eggs'], -1) + self.assertEqual(ns['result'], 41) + self.assertIsNone(ns['cheddar']) + + def test_shared_overwrites(self): + interpreters.run_string(self.id, dedent(""" + spam = 'eggs' + ns1 = dict(vars()) + del ns1['__builtins__'] + """)) + + shared = {'spam': b'ham'} + script = dedent(f""" + ns2 = dict(vars()) + del ns2['__builtins__'] + """) + interpreters.run_string(self.id, script, shared) + + r, w = os.pipe() + script = dedent(f""" + ns = dict(vars()) + del ns['__builtins__'] + import pickle + with open({w}, 'wb') as chan: + pickle.dump(ns, chan) + """) + interpreters.run_string(self.id, script) + with open(r, 'rb') as chan: + ns = pickle.load(chan) + + self.assertEqual(ns['ns1']['spam'], 'eggs') + self.assertEqual(ns['ns2']['spam'], b'ham') + self.assertEqual(ns['spam'], b'ham') + + def test_shared_overwrites_default_vars(self): + r, w = os.pipe() + + shared = {'__name__': b'not __main__'} + script = dedent(f""" + spam = 42 + + ns = dict(vars()) + del ns['__builtins__'] + import pickle + with open({w}, 'wb') as chan: + pickle.dump(ns, chan) + """) + interpreters.run_string(self.id, script, shared) + with open(r, 'rb') as chan: + ns = pickle.load(chan) + + self.assertEqual(ns['__name__'], b'not __main__') + + def test_main_reused(self): + r, w = os.pipe() + interpreters.run_string(self.id, dedent(f""" + spam = True + + ns = dict(vars()) + del ns['__builtins__'] + import pickle + with open({w}, 'wb') as chan: + pickle.dump(ns, chan) + del ns, pickle, chan + """)) + with open(r, 'rb') as chan: + ns1 = pickle.load(chan) + + r, w = os.pipe() + interpreters.run_string(self.id, dedent(f""" + eggs = False + + ns = dict(vars()) + del ns['__builtins__'] + import pickle + with open({w}, 'wb') as chan: + pickle.dump(ns, chan) + """)) + with open(r, 'rb') as chan: + ns2 = pickle.load(chan) + + self.assertIn('spam', ns1) + self.assertNotIn('eggs', ns1) + self.assertIn('eggs', ns2) + self.assertIn('spam', ns2) + + def test_execution_namespace_is_main(self): + r, w = os.pipe() + + script = dedent(f""" + spam = 42 + + ns = dict(vars()) + ns['__builtins__'] = str(ns['__builtins__']) + import pickle + with open({w}, 'wb') as chan: + pickle.dump(ns, chan) + """) + interpreters.run_string(self.id, script) + with open(r, 'rb') as chan: + ns = pickle.load(chan) + + ns.pop('__builtins__') + ns.pop('__loader__') + self.assertEqual(ns, { + '__name__': '__main__', + '__annotations__': {}, + '__doc__': None, + '__package__': None, + '__spec__': None, + 'spam': 42, + }) + + # XXX Fix this test! + @unittest.skip('blocking forever') + def test_still_running_at_exit(self): + script = dedent(f""" + from textwrap import dedent + import threading + import _xxsubinterpreters as _interpreters + id = _interpreters.create() + def f(): + _interpreters.run_string(id, dedent(''' + import time + # Give plenty of time for the main interpreter to finish. + time.sleep(1_000_000) + ''')) + + t = threading.Thread(target=f) + t.start() + """) + with support.temp_dir() as dirname: + filename = script_helper.make_script(dirname, 'interp', script) + with script_helper.spawn_python(filename) as proc: + retcode = proc.wait() + + self.assertEqual(retcode, 0) + + +################################## +# channel tests + +class ChannelIDTests(TestBase): + + def test_default_kwargs(self): + cid = interpreters._channel_id(10, force=True) + + self.assertEqual(int(cid), 10) + self.assertEqual(cid.end, 'both') + + def test_with_kwargs(self): + cid = interpreters._channel_id(10, send=True, force=True) + self.assertEqual(cid.end, 'send') + + cid = interpreters._channel_id(10, send=True, recv=False, force=True) + self.assertEqual(cid.end, 'send') + + cid = interpreters._channel_id(10, recv=True, force=True) + self.assertEqual(cid.end, 'recv') + + cid = interpreters._channel_id(10, recv=True, send=False, force=True) + self.assertEqual(cid.end, 'recv') + + cid = interpreters._channel_id(10, send=True, recv=True, force=True) + self.assertEqual(cid.end, 'both') + + def test_coerce_id(self): + cid = interpreters._channel_id('10', force=True) + self.assertEqual(int(cid), 10) + + cid = interpreters._channel_id(10.0, force=True) + self.assertEqual(int(cid), 10) + + class Int(str): + def __init__(self, value): + self._value = value + def __int__(self): + return self._value + + cid = interpreters._channel_id(Int(10), force=True) + self.assertEqual(int(cid), 10) + + def test_bad_id(self): + for cid in [-1, 'spam']: + with self.subTest(cid): + with self.assertRaises(ValueError): + interpreters._channel_id(cid) + with self.assertRaises(OverflowError): + interpreters._channel_id(2**64) + with self.assertRaises(TypeError): + interpreters._channel_id(object()) + + def test_bad_kwargs(self): + with self.assertRaises(ValueError): + interpreters._channel_id(10, send=False, recv=False) + + def test_does_not_exist(self): + cid = interpreters.channel_create() + with self.assertRaises(interpreters.ChannelNotFoundError): + interpreters._channel_id(int(cid) + 1) # unforced + + def test_str(self): + cid = interpreters._channel_id(10, force=True) + self.assertEqual(str(cid), '10') + + def test_repr(self): + cid = interpreters._channel_id(10, force=True) + self.assertEqual(repr(cid), 'ChannelID(10)') + + cid = interpreters._channel_id(10, send=True, force=True) + self.assertEqual(repr(cid), 'ChannelID(10, send=True)') + + cid = interpreters._channel_id(10, recv=True, force=True) + self.assertEqual(repr(cid), 'ChannelID(10, recv=True)') + + cid = interpreters._channel_id(10, send=True, recv=True, force=True) + self.assertEqual(repr(cid), 'ChannelID(10)') + + def test_equality(self): + cid1 = interpreters.channel_create() + cid2 = interpreters._channel_id(int(cid1)) + cid3 = interpreters.channel_create() + + self.assertTrue(cid1 == cid1) + self.assertTrue(cid1 == cid2) + self.assertTrue(cid1 == int(cid1)) + self.assertFalse(cid1 == cid3) + + self.assertFalse(cid1 != cid1) + self.assertFalse(cid1 != cid2) + self.assertTrue(cid1 != cid3) + + +class ChannelTests(TestBase): + + def test_create_cid(self): + cid = interpreters.channel_create() + self.assertIsInstance(cid, interpreters.ChannelID) + + def test_sequential_ids(self): + before = interpreters.channel_list_all() + id1 = interpreters.channel_create() + id2 = interpreters.channel_create() + id3 = interpreters.channel_create() + after = interpreters.channel_list_all() + + self.assertEqual(id2, int(id1) + 1) + self.assertEqual(id3, int(id2) + 1) + self.assertEqual(set(after) - set(before), {id1, id2, id3}) + + def test_ids_global(self): + id1 = interpreters.create() + out = _run_output(id1, dedent(""" + import _xxsubinterpreters as _interpreters + cid = _interpreters.channel_create() + print(cid) + """)) + cid1 = int(out.strip()) + + id2 = interpreters.create() + out = _run_output(id2, dedent(""" + import _xxsubinterpreters as _interpreters + cid = _interpreters.channel_create() + print(cid) + """)) + cid2 = int(out.strip()) + + self.assertEqual(cid2, int(cid1) + 1) + + #################### + + def test_send_recv_main(self): + cid = interpreters.channel_create() + orig = b'spam' + interpreters.channel_send(cid, orig) + obj = interpreters.channel_recv(cid) + + self.assertEqual(obj, orig) + self.assertIsNot(obj, orig) + + def test_send_recv_same_interpreter(self): + id1 = interpreters.create() + out = _run_output(id1, dedent(""" + import _xxsubinterpreters as _interpreters + cid = _interpreters.channel_create() + orig = b'spam' + _interpreters.channel_send(cid, orig) + obj = _interpreters.channel_recv(cid) + assert obj is not orig + assert obj == orig + """)) + + def test_send_recv_different_interpreters(self): + cid = interpreters.channel_create() + id1 = interpreters.create() + out = _run_output(id1, dedent(f""" + import _xxsubinterpreters as _interpreters + _interpreters.channel_send({cid}, b'spam') + """)) + obj = interpreters.channel_recv(cid) + + self.assertEqual(obj, b'spam') + + def test_send_recv_different_threads(self): + cid = interpreters.channel_create() + + def f(): + while True: + try: + obj = interpreters.channel_recv(cid) + break + except interpreters.ChannelEmptyError: + time.sleep(0.1) + interpreters.channel_send(cid, obj) + t = threading.Thread(target=f) + t.start() + + interpreters.channel_send(cid, b'spam') + t.join() + obj = interpreters.channel_recv(cid) + + self.assertEqual(obj, b'spam') + + def test_send_recv_different_interpreters_and_threads(self): + cid = interpreters.channel_create() + id1 = interpreters.create() + out = None + + def f(): + nonlocal out + out = _run_output(id1, dedent(f""" + import time + import _xxsubinterpreters as _interpreters + while True: + try: + obj = _interpreters.channel_recv({cid}) + break + except _interpreters.ChannelEmptyError: + time.sleep(0.1) + assert(obj == b'spam') + _interpreters.channel_send({cid}, b'eggs') + """)) + t = threading.Thread(target=f) + t.start() + + interpreters.channel_send(cid, b'spam') + t.join() + obj = interpreters.channel_recv(cid) + + self.assertEqual(obj, b'eggs') + + def test_send_not_found(self): + with self.assertRaises(interpreters.ChannelNotFoundError): + interpreters.channel_send(10, b'spam') + + def test_recv_not_found(self): + with self.assertRaises(interpreters.ChannelNotFoundError): + interpreters.channel_recv(10) + + def test_recv_empty(self): + cid = interpreters.channel_create() + with self.assertRaises(interpreters.ChannelEmptyError): + interpreters.channel_recv(cid) + + def test_run_string_arg_unresolved(self): + cid = interpreters.channel_create() + interp = interpreters.create() + + out = _run_output(interp, dedent(""" + import _xxsubinterpreters as _interpreters + print(cid.end) + _interpreters.channel_send(cid, b'spam') + """), + dict(cid=cid.send)) + obj = interpreters.channel_recv(cid) + + self.assertEqual(obj, b'spam') + self.assertEqual(out.strip(), 'send') + + # XXX For now there is no high-level channel into which the + # sent channel ID can be converted... + # Note: this test caused crashes on some buildbots (bpo-33615). + @unittest.skip('disabled until high-level channels exist') + def test_run_string_arg_resolved(self): + cid = interpreters.channel_create() + cid = interpreters._channel_id(cid, _resolve=True) + interp = interpreters.create() + + out = _run_output(interp, dedent(""" + import _xxsubinterpreters as _interpreters + print(chan.id.end) + _interpreters.channel_send(chan.id, b'spam') + """), + dict(chan=cid.send)) + obj = interpreters.channel_recv(cid) + + self.assertEqual(obj, b'spam') + self.assertEqual(out.strip(), 'send') + + # close + + def test_close_single_user(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_recv(cid) + interpreters.channel_close(cid) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'eggs') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_close_multiple_users(self): + cid = interpreters.channel_create() + id1 = interpreters.create() + id2 = interpreters.create() + interpreters.run_string(id1, dedent(f""" + import _xxsubinterpreters as _interpreters + _interpreters.channel_send({cid}, b'spam') + """)) + interpreters.run_string(id2, dedent(f""" + import _xxsubinterpreters as _interpreters + _interpreters.channel_recv({cid}) + """)) + interpreters.channel_close(cid) + with self.assertRaises(interpreters.RunFailedError) as cm: + interpreters.run_string(id1, dedent(f""" + _interpreters.channel_send({cid}, b'spam') + """)) + self.assertIn('ChannelClosedError', str(cm.exception)) + with self.assertRaises(interpreters.RunFailedError) as cm: + interpreters.run_string(id2, dedent(f""" + _interpreters.channel_send({cid}, b'spam') + """)) + self.assertIn('ChannelClosedError', str(cm.exception)) + + def test_close_multiple_times(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_recv(cid) + interpreters.channel_close(cid) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_close(cid) + + def test_close_empty(self): + tests = [ + (False, False), + (True, False), + (False, True), + (True, True), + ] + for send, recv in tests: + with self.subTest((send, recv)): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_recv(cid) + interpreters.channel_close(cid, send=send, recv=recv) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'eggs') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_close_defaults_with_unused_items(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_send(cid, b'ham') + + with self.assertRaises(interpreters.ChannelNotEmptyError): + interpreters.channel_close(cid) + interpreters.channel_recv(cid) + interpreters.channel_send(cid, b'eggs') + + def test_close_recv_with_unused_items_unforced(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_send(cid, b'ham') + + with self.assertRaises(interpreters.ChannelNotEmptyError): + interpreters.channel_close(cid, recv=True) + interpreters.channel_recv(cid) + interpreters.channel_send(cid, b'eggs') + interpreters.channel_recv(cid) + interpreters.channel_recv(cid) + interpreters.channel_close(cid, recv=True) + + def test_close_send_with_unused_items_unforced(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_send(cid, b'ham') + interpreters.channel_close(cid, send=True) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'eggs') + interpreters.channel_recv(cid) + interpreters.channel_recv(cid) + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_close_both_with_unused_items_unforced(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_send(cid, b'ham') + + with self.assertRaises(interpreters.ChannelNotEmptyError): + interpreters.channel_close(cid, recv=True, send=True) + interpreters.channel_recv(cid) + interpreters.channel_send(cid, b'eggs') + interpreters.channel_recv(cid) + interpreters.channel_recv(cid) + interpreters.channel_close(cid, recv=True) + + def test_close_recv_with_unused_items_forced(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_send(cid, b'ham') + interpreters.channel_close(cid, recv=True, force=True) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'eggs') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_close_send_with_unused_items_forced(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_send(cid, b'ham') + interpreters.channel_close(cid, send=True, force=True) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'eggs') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_close_both_with_unused_items_forced(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_send(cid, b'ham') + interpreters.channel_close(cid, send=True, recv=True, force=True) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'eggs') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_close_never_used(self): + cid = interpreters.channel_create() + interpreters.channel_close(cid) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'spam') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_close_by_unassociated_interp(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interp = interpreters.create() + interpreters.run_string(interp, dedent(f""" + import _xxsubinterpreters as _interpreters + _interpreters.channel_close({cid}, force=True) + """)) + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_close(cid) + + def test_close_used_multiple_times_by_single_user(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_send(cid, b'spam') + interpreters.channel_send(cid, b'spam') + interpreters.channel_recv(cid) + interpreters.channel_close(cid, force=True) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'eggs') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + +class ChannelReleaseTests(TestBase): + + # XXX Add more test coverage a la the tests for close(). + + """ + - main / interp / other + - run in: current thread / new thread / other thread / different threads + - end / opposite + - force / no force + - used / not used (associated / not associated) + - empty / emptied / never emptied / partly emptied + - closed / not closed + - released / not released + - creator (interp) / other + - associated interpreter not running + - associated interpreter destroyed + """ + + """ + use + pre-release + release + after + check + """ + + """ + release in: main, interp1 + creator: same, other (incl. interp2) + + use: None,send,recv,send/recv in None,same,other(incl. interp2),same+other(incl. interp2),all + pre-release: None,send,recv,both in None,same,other(incl. interp2),same+other(incl. interp2),all + pre-release forced: None,send,recv,both in None,same,other(incl. interp2),same+other(incl. interp2),all + + release: same + release forced: same + + use after: None,send,recv,send/recv in None,same,other(incl. interp2),same+other(incl. interp2),all + release after: None,send,recv,send/recv in None,same,other(incl. interp2),same+other(incl. interp2),all + check released: send/recv for same/other(incl. interp2) + check closed: send/recv for same/other(incl. interp2) + """ + + def test_single_user(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_recv(cid) + interpreters.channel_release(cid, send=True, recv=True) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'eggs') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_multiple_users(self): + cid = interpreters.channel_create() + id1 = interpreters.create() + id2 = interpreters.create() + interpreters.run_string(id1, dedent(f""" + import _xxsubinterpreters as _interpreters + _interpreters.channel_send({cid}, b'spam') + """)) + out = _run_output(id2, dedent(f""" + import _xxsubinterpreters as _interpreters + obj = _interpreters.channel_recv({cid}) + _interpreters.channel_release({cid}) + print(repr(obj)) + """)) + interpreters.run_string(id1, dedent(f""" + _interpreters.channel_release({cid}) + """)) + + self.assertEqual(out.strip(), "b'spam'") + + def test_no_kwargs(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_recv(cid) + interpreters.channel_release(cid) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'eggs') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_multiple_times(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_recv(cid) + interpreters.channel_release(cid, send=True, recv=True) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_release(cid, send=True, recv=True) + + def test_with_unused_items(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_send(cid, b'ham') + interpreters.channel_release(cid, send=True, recv=True) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_never_used(self): + cid = interpreters.channel_create() + interpreters.channel_release(cid) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'spam') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_by_unassociated_interp(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interp = interpreters.create() + interpreters.run_string(interp, dedent(f""" + import _xxsubinterpreters as _interpreters + _interpreters.channel_release({cid}) + """)) + obj = interpreters.channel_recv(cid) + interpreters.channel_release(cid) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'eggs') + self.assertEqual(obj, b'spam') + + def test_close_if_unassociated(self): + # XXX Something's not right with this test... + cid = interpreters.channel_create() + interp = interpreters.create() + interpreters.run_string(interp, dedent(f""" + import _xxsubinterpreters as _interpreters + obj = _interpreters.channel_send({cid}, b'spam') + _interpreters.channel_release({cid}) + """)) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_partially(self): + # XXX Is partial close too weird/confusing? + cid = interpreters.channel_create() + interpreters.channel_send(cid, None) + interpreters.channel_recv(cid) + interpreters.channel_send(cid, b'spam') + interpreters.channel_release(cid, send=True) + obj = interpreters.channel_recv(cid) + + self.assertEqual(obj, b'spam') + + def test_used_multiple_times_by_single_user(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_send(cid, b'spam') + interpreters.channel_send(cid, b'spam') + interpreters.channel_recv(cid) + interpreters.channel_release(cid, send=True, recv=True) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'eggs') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + +class ChannelCloseFixture(namedtuple('ChannelCloseFixture', + 'end interp other extra creator')): + + # Set this to True to avoid creating interpreters, e.g. when + # scanning through test permutations without running them. + QUICK = False + + def __new__(cls, end, interp, other, extra, creator): + assert end in ('send', 'recv') + if cls.QUICK: + known = {} + else: + interp = Interpreter.from_raw(interp) + other = Interpreter.from_raw(other) + extra = Interpreter.from_raw(extra) + known = { + interp.name: interp, + other.name: other, + extra.name: extra, + } + if not creator: + creator = 'same' + self = super().__new__(cls, end, interp, other, extra, creator) + self._prepped = set() + self._state = ChannelState() + self._known = known + return self + + @property + def state(self): + return self._state + + @property + def cid(self): + try: + return self._cid + except AttributeError: + creator = self._get_interpreter(self.creator) + self._cid = self._new_channel(creator) + return self._cid + + def get_interpreter(self, interp): + interp = self._get_interpreter(interp) + self._prep_interpreter(interp) + return interp + + def expect_closed_error(self, end=None): + if end is None: + end = self.end + if end == 'recv' and self.state.closed == 'send': + return False + return bool(self.state.closed) + + def prep_interpreter(self, interp): + self._prep_interpreter(interp) + + def record_action(self, action, result): + self._state = result + + def clean_up(self): + clean_up_interpreters() + clean_up_channels() + + # internal methods + + def _new_channel(self, creator): + if creator.name == 'main': + return interpreters.channel_create() + else: + ch = interpreters.channel_create() + run_interp(creator.id, f""" + import _xxsubinterpreters + cid = _xxsubinterpreters.channel_create() + # We purposefully send back an int to avoid tying the + # channel to the other interpreter. + _xxsubinterpreters.channel_send({ch}, int(cid)) + del _xxsubinterpreters + """) + self._cid = interpreters.channel_recv(ch) + return self._cid + + def _get_interpreter(self, interp): + if interp in ('same', 'interp'): + return self.interp + elif interp == 'other': + return self.other + elif interp == 'extra': + return self.extra + else: + name = interp + try: + interp = self._known[name] + except KeyError: + interp = self._known[name] = Interpreter(name) + return interp + + def _prep_interpreter(self, interp): + if interp.id in self._prepped: + return + self._prepped.add(interp.id) + if interp.name == 'main': + return + run_interp(interp.id, f""" + import _xxsubinterpreters as interpreters + import test.test__xxsubinterpreters as helpers + ChannelState = helpers.ChannelState + try: + cid + except NameError: + cid = interpreters._channel_id({self.cid}) + """) + + +@unittest.skip('these tests take several hours to run') +class ExhaustiveChannelTests(TestBase): + + """ + - main / interp / other + - run in: current thread / new thread / other thread / different threads + - end / opposite + - force / no force + - used / not used (associated / not associated) + - empty / emptied / never emptied / partly emptied + - closed / not closed + - released / not released + - creator (interp) / other + - associated interpreter not running + - associated interpreter destroyed + + - close after unbound + """ + + """ + use + pre-close + close + after + check + """ + + """ + close in: main, interp1 + creator: same, other, extra + + use: None,send,recv,send/recv in None,same,other,same+other,all + pre-close: None,send,recv in None,same,other,same+other,all + pre-close forced: None,send,recv in None,same,other,same+other,all + + close: same + close forced: same + + use after: None,send,recv,send/recv in None,same,other,extra,same+other,all + close after: None,send,recv,send/recv in None,same,other,extra,same+other,all + check closed: send/recv for same/other(incl. interp2) + """ + + def iter_action_sets(self): + # - used / not used (associated / not associated) + # - empty / emptied / never emptied / partly emptied + # - closed / not closed + # - released / not released + + # never used + yield [] + + # only pre-closed (and possible used after) + for closeactions in self._iter_close_action_sets('same', 'other'): + yield closeactions + for postactions in self._iter_post_close_action_sets(): + yield closeactions + postactions + for closeactions in self._iter_close_action_sets('other', 'extra'): + yield closeactions + for postactions in self._iter_post_close_action_sets(): + yield closeactions + postactions + + # used + for useactions in self._iter_use_action_sets('same', 'other'): + yield useactions + for closeactions in self._iter_close_action_sets('same', 'other'): + actions = useactions + closeactions + yield actions + for postactions in self._iter_post_close_action_sets(): + yield actions + postactions + for closeactions in self._iter_close_action_sets('other', 'extra'): + actions = useactions + closeactions + yield actions + for postactions in self._iter_post_close_action_sets(): + yield actions + postactions + for useactions in self._iter_use_action_sets('other', 'extra'): + yield useactions + for closeactions in self._iter_close_action_sets('same', 'other'): + actions = useactions + closeactions + yield actions + for postactions in self._iter_post_close_action_sets(): + yield actions + postactions + for closeactions in self._iter_close_action_sets('other', 'extra'): + actions = useactions + closeactions + yield actions + for postactions in self._iter_post_close_action_sets(): + yield actions + postactions + + def _iter_use_action_sets(self, interp1, interp2): + interps = (interp1, interp2) + + # only recv end used + yield [ + ChannelAction('use', 'recv', interp1), + ] + yield [ + ChannelAction('use', 'recv', interp2), + ] + yield [ + ChannelAction('use', 'recv', interp1), + ChannelAction('use', 'recv', interp2), + ] + + # never emptied + yield [ + ChannelAction('use', 'send', interp1), + ] + yield [ + ChannelAction('use', 'send', interp2), + ] + yield [ + ChannelAction('use', 'send', interp1), + ChannelAction('use', 'send', interp2), + ] + + # partially emptied + for interp1 in interps: + for interp2 in interps: + for interp3 in interps: + yield [ + ChannelAction('use', 'send', interp1), + ChannelAction('use', 'send', interp2), + ChannelAction('use', 'recv', interp3), + ] + + # fully emptied + for interp1 in interps: + for interp2 in interps: + for interp3 in interps: + for interp4 in interps: + yield [ + ChannelAction('use', 'send', interp1), + ChannelAction('use', 'send', interp2), + ChannelAction('use', 'recv', interp3), + ChannelAction('use', 'recv', interp4), + ] + + def _iter_close_action_sets(self, interp1, interp2): + ends = ('recv', 'send') + interps = (interp1, interp2) + for force in (True, False): + op = 'force-close' if force else 'close' + for interp in interps: + for end in ends: + yield [ + ChannelAction(op, end, interp), + ] + for recvop in ('close', 'force-close'): + for sendop in ('close', 'force-close'): + for recv in interps: + for send in interps: + yield [ + ChannelAction(recvop, 'recv', recv), + ChannelAction(sendop, 'send', send), + ] + + def _iter_post_close_action_sets(self): + for interp in ('same', 'extra', 'other'): + yield [ + ChannelAction('use', 'recv', interp), + ] + yield [ + ChannelAction('use', 'send', interp), + ] + + def run_actions(self, fix, actions): + for action in actions: + self.run_action(fix, action) + + def run_action(self, fix, action, *, hideclosed=True): + end = action.resolve_end(fix.end) + interp = action.resolve_interp(fix.interp, fix.other, fix.extra) + fix.prep_interpreter(interp) + if interp.name == 'main': + result = run_action( + fix.cid, + action.action, + end, + fix.state, + hideclosed=hideclosed, + ) + fix.record_action(action, result) + else: + _cid = interpreters.channel_create() + run_interp(interp.id, f""" + result = helpers.run_action( + {fix.cid}, + {repr(action.action)}, + {repr(end)}, + {repr(fix.state)}, + hideclosed={hideclosed}, + ) + interpreters.channel_send({_cid}, result.pending.to_bytes(1, 'little')) + interpreters.channel_send({_cid}, b'X' if result.closed else b'') + """) + result = ChannelState( + pending=int.from_bytes(interpreters.channel_recv(_cid), 'little'), + closed=bool(interpreters.channel_recv(_cid)), + ) + fix.record_action(action, result) + + def iter_fixtures(self): + # XXX threads? + interpreters = [ + ('main', 'interp', 'extra'), + ('interp', 'main', 'extra'), + ('interp1', 'interp2', 'extra'), + ('interp1', 'interp2', 'main'), + ] + for interp, other, extra in interpreters: + for creator in ('same', 'other', 'creator'): + for end in ('send', 'recv'): + yield ChannelCloseFixture(end, interp, other, extra, creator) + + def _close(self, fix, *, force): + op = 'force-close' if force else 'close' + close = ChannelAction(op, fix.end, 'same') + if not fix.expect_closed_error(): + self.run_action(fix, close, hideclosed=False) + else: + with self.assertRaises(interpreters.ChannelClosedError): + self.run_action(fix, close, hideclosed=False) + + def _assert_closed_in_interp(self, fix, interp=None): + if interp is None or interp.name == 'main': + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(fix.cid) + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(fix.cid, b'spam') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_close(fix.cid) + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_close(fix.cid, force=True) + else: + run_interp(interp.id, f""" + with helpers.expect_channel_closed(): + interpreters.channel_recv(cid) + """) + run_interp(interp.id, f""" + with helpers.expect_channel_closed(): + interpreters.channel_send(cid, b'spam') + """) + run_interp(interp.id, f""" + with helpers.expect_channel_closed(): + interpreters.channel_close(cid) + """) + run_interp(interp.id, f""" + with helpers.expect_channel_closed(): + interpreters.channel_close(cid, force=True) + """) + + def _assert_closed(self, fix): + self.assertTrue(fix.state.closed) + + for _ in range(fix.state.pending): + interpreters.channel_recv(fix.cid) + self._assert_closed_in_interp(fix) + + for interp in ('same', 'other'): + interp = fix.get_interpreter(interp) + if interp.name == 'main': + continue + self._assert_closed_in_interp(fix, interp) + + interp = fix.get_interpreter('fresh') + self._assert_closed_in_interp(fix, interp) + + def _iter_close_tests(self, verbose=False): + i = 0 + for actions in self.iter_action_sets(): + print() + for fix in self.iter_fixtures(): + i += 1 + if i > 1000: + return + if verbose: + if (i - 1) % 6 == 0: + print() + print(i, fix, '({} actions)'.format(len(actions))) + else: + if (i - 1) % 6 == 0: + print(' ', end='') + print('.', end=''); sys.stdout.flush() + yield i, fix, actions + if verbose: + print('---') + print() + + # This is useful for scanning through the possible tests. + def _skim_close_tests(self): + ChannelCloseFixture.QUICK = True + for i, fix, actions in self._iter_close_tests(): + pass + + def test_close(self): + for i, fix, actions in self._iter_close_tests(): + with self.subTest('{} {} {}'.format(i, fix, actions)): + fix.prep_interpreter(fix.interp) + self.run_actions(fix, actions) + + self._close(fix, force=False) + + self._assert_closed(fix) + # XXX Things slow down if we have too many interpreters. + fix.clean_up() + + def test_force_close(self): + for i, fix, actions in self._iter_close_tests(): + with self.subTest('{} {} {}'.format(i, fix, actions)): + fix.prep_interpreter(fix.interp) + self.run_actions(fix, actions) + + self._close(fix, force=True) + + self._assert_closed(fix) + # XXX Things slow down if we have too many interpreters. + fix.clean_up() + + +if __name__ == '__main__': + unittest.main() From 331be62729bad0037af12e484fe693685d22495b Mon Sep 17 00:00:00 2001 From: Joannah Nanjekye Date: Tue, 10 Sep 2019 12:54:38 +0000 Subject: [PATCH 02/20] create stdlib interpreters module and _interpreters internal module --- Doc/library/interpreters.rst | 9 +++++ Lib/interpreters.py | 0 Lib/test/test__xxsubinterpreters.py | 56 ++++++++++++++--------------- Modules/_xxsubinterpretersmodule.c | 18 +++++----- PC/config.c | 4 +-- setup.py | 2 +- 6 files changed, 49 insertions(+), 40 deletions(-) create mode 100644 Doc/library/interpreters.rst create mode 100644 Lib/interpreters.py diff --git a/Doc/library/interpreters.rst b/Doc/library/interpreters.rst new file mode 100644 index 00000000000000..bb9bce2de0f42d --- /dev/null +++ b/Doc/library/interpreters.rst @@ -0,0 +1,9 @@ +:mod:`interpreters` --- High-level Sub-interpreters Module +========================================================== + +.. module:: interpreters + :synopsis: High-level Sub-interpreters Module. + +**Source code:** :source:`Lib/interpreters.py` + +-------------- \ No newline at end of file diff --git a/Lib/interpreters.py b/Lib/interpreters.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index 78b2030a1f6d8b..20e6b0419ff411 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -13,7 +13,7 @@ from test.support import script_helper -interpreters = support.import_module('_xxsubinterpreters') +interpreters = support.import_module('_interpreters') ################################## @@ -446,7 +446,7 @@ def test_subinterpreter(self): main = interpreters.get_main() interp = interpreters.create() out = _run_output(interp, dedent(""" - import _xxsubinterpreters as _interpreters + import _interpreters cur = _interpreters.get_current() print(cur) assert isinstance(cur, _interpreters.InterpreterID) @@ -469,7 +469,7 @@ def test_from_subinterpreter(self): [expected] = interpreters.list_all() interp = interpreters.create() out = _run_output(interp, dedent(""" - import _xxsubinterpreters as _interpreters + import _interpreters main = _interpreters.get_main() print(main) assert isinstance(main, _interpreters.InterpreterID) @@ -495,7 +495,7 @@ def test_subinterpreter(self): def test_from_subinterpreter(self): interp = interpreters.create() out = _run_output(interp, dedent(f""" - import _xxsubinterpreters as _interpreters + import _interpreters if _interpreters.is_running({interp}): print(True) else: @@ -616,7 +616,7 @@ def test_in_subinterpreter(self): main, = interpreters.list_all() id1 = interpreters.create() out = _run_output(id1, dedent(""" - import _xxsubinterpreters as _interpreters + import _interpreters id = _interpreters.create() print(id) assert isinstance(id, _interpreters.InterpreterID) @@ -632,7 +632,7 @@ def test_in_threaded_subinterpreter(self): def f(): nonlocal id2 out = _run_output(id1, dedent(""" - import _xxsubinterpreters as _interpreters + import _interpreters id = _interpreters.create() print(id) """)) @@ -726,7 +726,7 @@ def test_from_current(self): main, = interpreters.list_all() id = interpreters.create() script = dedent(f""" - import _xxsubinterpreters as _interpreters + import _interpreters try: _interpreters.destroy({id}) except RuntimeError: @@ -741,7 +741,7 @@ def test_from_sibling(self): id1 = interpreters.create() id2 = interpreters.create() script = dedent(f""" - import _xxsubinterpreters as _interpreters + import _interpreters _interpreters.destroy({id2}) """) interpreters.run_string(id1, script) @@ -1057,7 +1057,7 @@ def test_still_running_at_exit(self): script = dedent(f""" from textwrap import dedent import threading - import _xxsubinterpreters as _interpreters + import _interpreters id = _interpreters.create() def f(): _interpreters.run_string(id, dedent(''' @@ -1191,7 +1191,7 @@ def test_sequential_ids(self): def test_ids_global(self): id1 = interpreters.create() out = _run_output(id1, dedent(""" - import _xxsubinterpreters as _interpreters + import _interpreters cid = _interpreters.channel_create() print(cid) """)) @@ -1199,7 +1199,7 @@ def test_ids_global(self): id2 = interpreters.create() out = _run_output(id2, dedent(""" - import _xxsubinterpreters as _interpreters + import _interpreters cid = _interpreters.channel_create() print(cid) """)) @@ -1221,7 +1221,7 @@ def test_send_recv_main(self): def test_send_recv_same_interpreter(self): id1 = interpreters.create() out = _run_output(id1, dedent(""" - import _xxsubinterpreters as _interpreters + import _interpreters cid = _interpreters.channel_create() orig = b'spam' _interpreters.channel_send(cid, orig) @@ -1234,7 +1234,7 @@ def test_send_recv_different_interpreters(self): cid = interpreters.channel_create() id1 = interpreters.create() out = _run_output(id1, dedent(f""" - import _xxsubinterpreters as _interpreters + import _interpreters _interpreters.channel_send({cid}, b'spam') """)) obj = interpreters.channel_recv(cid) @@ -1270,7 +1270,7 @@ def f(): nonlocal out out = _run_output(id1, dedent(f""" import time - import _xxsubinterpreters as _interpreters + import _interpreters while True: try: obj = _interpreters.channel_recv({cid}) @@ -1307,7 +1307,7 @@ def test_run_string_arg_unresolved(self): interp = interpreters.create() out = _run_output(interp, dedent(""" - import _xxsubinterpreters as _interpreters + import _interpreters print(cid.end) _interpreters.channel_send(cid, b'spam') """), @@ -1327,7 +1327,7 @@ def test_run_string_arg_resolved(self): interp = interpreters.create() out = _run_output(interp, dedent(""" - import _xxsubinterpreters as _interpreters + import _interpreters print(chan.id.end) _interpreters.channel_send(chan.id, b'spam') """), @@ -1355,11 +1355,11 @@ def test_close_multiple_users(self): id1 = interpreters.create() id2 = interpreters.create() interpreters.run_string(id1, dedent(f""" - import _xxsubinterpreters as _interpreters + import _interpreters _interpreters.channel_send({cid}, b'spam') """)) interpreters.run_string(id2, dedent(f""" - import _xxsubinterpreters as _interpreters + import _interpreters _interpreters.channel_recv({cid}) """)) interpreters.channel_close(cid) @@ -1498,7 +1498,7 @@ def test_close_by_unassociated_interp(self): interpreters.channel_send(cid, b'spam') interp = interpreters.create() interpreters.run_string(interp, dedent(f""" - import _xxsubinterpreters as _interpreters + import _interpreters _interpreters.channel_close({cid}, force=True) """)) with self.assertRaises(interpreters.ChannelClosedError): @@ -1579,11 +1579,11 @@ def test_multiple_users(self): id1 = interpreters.create() id2 = interpreters.create() interpreters.run_string(id1, dedent(f""" - import _xxsubinterpreters as _interpreters + import _interpreters _interpreters.channel_send({cid}, b'spam') """)) out = _run_output(id2, dedent(f""" - import _xxsubinterpreters as _interpreters + import _interpreters obj = _interpreters.channel_recv({cid}) _interpreters.channel_release({cid}) print(repr(obj)) @@ -1637,7 +1637,7 @@ def test_by_unassociated_interp(self): interpreters.channel_send(cid, b'spam') interp = interpreters.create() interpreters.run_string(interp, dedent(f""" - import _xxsubinterpreters as _interpreters + import _interpreters _interpreters.channel_release({cid}) """)) obj = interpreters.channel_recv(cid) @@ -1652,7 +1652,7 @@ def test_close_if_unassociated(self): cid = interpreters.channel_create() interp = interpreters.create() interpreters.run_string(interp, dedent(f""" - import _xxsubinterpreters as _interpreters + import _interpreters obj = _interpreters.channel_send({cid}, b'spam') _interpreters.channel_release({cid}) """)) @@ -1756,12 +1756,12 @@ def _new_channel(self, creator): else: ch = interpreters.channel_create() run_interp(creator.id, f""" - import _xxsubinterpreters - cid = _xxsubinterpreters.channel_create() + import _interpreters + cid = _interpreters.channel_create() # We purposefully send back an int to avoid tying the # channel to the other interpreter. - _xxsubinterpreters.channel_send({ch}, int(cid)) - del _xxsubinterpreters + _interpreters.channel_send({ch}, int(cid)) + del _interpreters """) self._cid = interpreters.channel_recv(ch) return self._cid @@ -1788,7 +1788,7 @@ def _prep_interpreter(self, interp): if interp.name == 'main': return run_interp(interp.id, f""" - import _xxsubinterpreters as interpreters + import _interpreters as interpreters import test.test__xxsubinterpreters as helpers ChannelState = helpers.ChannelState try: diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 19d98fd9693446..552d96a977f2a9 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -293,7 +293,7 @@ channel_exceptions_init(PyObject *ns) // XXX Move the exceptions into per-module memory? // A channel-related operation failed. - ChannelError = PyErr_NewException("_xxsubinterpreters.ChannelError", + ChannelError = PyErr_NewException("_interpreters.ChannelError", PyExc_RuntimeError, NULL); if (ChannelError == NULL) { return -1; @@ -304,7 +304,7 @@ channel_exceptions_init(PyObject *ns) // An operation tried to use a channel that doesn't exist. ChannelNotFoundError = PyErr_NewException( - "_xxsubinterpreters.ChannelNotFoundError", ChannelError, NULL); + "_interpreters.ChannelNotFoundError", ChannelError, NULL); if (ChannelNotFoundError == NULL) { return -1; } @@ -314,7 +314,7 @@ channel_exceptions_init(PyObject *ns) // An operation tried to use a closed channel. ChannelClosedError = PyErr_NewException( - "_xxsubinterpreters.ChannelClosedError", ChannelError, NULL); + "_interpreters.ChannelClosedError", ChannelError, NULL); if (ChannelClosedError == NULL) { return -1; } @@ -324,7 +324,7 @@ channel_exceptions_init(PyObject *ns) // An operation tried to pop from an empty channel. ChannelEmptyError = PyErr_NewException( - "_xxsubinterpreters.ChannelEmptyError", ChannelError, NULL); + "_interpreters.ChannelEmptyError", ChannelError, NULL); if (ChannelEmptyError == NULL) { return -1; } @@ -334,7 +334,7 @@ channel_exceptions_init(PyObject *ns) // An operation tried to close a non-empty channel. ChannelNotEmptyError = PyErr_NewException( - "_xxsubinterpreters.ChannelNotEmptyError", ChannelError, NULL); + "_interpreters.ChannelNotEmptyError", ChannelError, NULL); if (ChannelNotEmptyError == NULL) { return -1; } @@ -1736,7 +1736,7 @@ PyDoc_STRVAR(channelid_doc, static PyTypeObject ChannelIDtype = { PyVarObject_HEAD_INIT(&PyType_Type, 0) - "_xxsubinterpreters.ChannelID", /* tp_name */ + "_interpreters.ChannelID", /* tp_name */ sizeof(channelid), /* tp_basicsize */ 0, /* tp_itemsize */ (destructor)channelid_dealloc, /* tp_dealloc */ @@ -1793,7 +1793,7 @@ interp_exceptions_init(PyObject *ns) if (RunFailedError == NULL) { // An uncaught exception came out of interp_run_string(). - RunFailedError = PyErr_NewException("_xxsubinterpreters.RunFailedError", + RunFailedError = PyErr_NewException("_interpreters.RunFailedError", PyExc_RuntimeError, NULL); if (RunFailedError == NULL) { return -1; @@ -2519,7 +2519,7 @@ The 'interpreters' module provides a more convenient interface."); static struct PyModuleDef interpretersmodule = { PyModuleDef_HEAD_INIT, - "_xxsubinterpreters", /* m_name */ + "_interpreters", /* m_name */ module_doc, /* m_doc */ -1, /* m_size */ module_functions, /* m_methods */ @@ -2531,7 +2531,7 @@ static struct PyModuleDef interpretersmodule = { PyMODINIT_FUNC -PyInit__xxsubinterpreters(void) +PyInit__interpreters(void) { if (_init_globals() != 0) { return NULL; diff --git a/PC/config.c b/PC/config.c index 8eaeb31c9b934b..38d9506ffdd23e 100644 --- a/PC/config.c +++ b/PC/config.c @@ -35,7 +35,7 @@ extern PyObject* PyInit__codecs(void); extern PyObject* PyInit__weakref(void); /* XXX: These two should really be extracted to standalone extensions. */ extern PyObject* PyInit_xxsubtype(void); -extern PyObject* PyInit__xxsubinterpreters(void); +extern PyObject* PyInit__interpreters(void); extern PyObject* PyInit__random(void); extern PyObject* PyInit_itertools(void); extern PyObject* PyInit__collections(void); @@ -133,7 +133,7 @@ struct _inittab _PyImport_Inittab[] = { {"_json", PyInit__json}, {"xxsubtype", PyInit_xxsubtype}, - {"_xxsubinterpreters", PyInit__xxsubinterpreters}, + {"_interpreters", PyInit__interpreters}, #ifdef _Py_HAVE_ZLIB {"zlib", PyInit_zlib}, #endif diff --git a/setup.py b/setup.py index 02f523c42d355f..5243b41a3a828a 100644 --- a/setup.py +++ b/setup.py @@ -827,7 +827,7 @@ def detect_simple_extensions(self): self.add(Extension('syslog', ['syslogmodule.c'])) # Python interface to subinterpreter C-API. - self.add(Extension('_xxsubinterpreters', ['_xxsubinterpretersmodule.c'])) + self.add(Extension('_interpreters', ['_xxsubinterpretersmodule.c'])) # # Here ends the simple stuff. From here on, modules need certain From 3f2f2f4f862fde1455fdc178265172040cd7f7e8 Mon Sep 17 00:00:00 2001 From: Joannah Nanjekye Date: Tue, 10 Sep 2019 16:25:02 +0000 Subject: [PATCH 03/20] Global initial Sub-Interpreters functions --- Doc/library/interpreters.rst | 38 +- Lib/interpreters.py | 59 + Lib/test/test_interpreters.py | 2037 +-------------------------------- 3 files changed, 103 insertions(+), 2031 deletions(-) diff --git a/Doc/library/interpreters.rst b/Doc/library/interpreters.rst index bb9bce2de0f42d..28ab95c370ce66 100644 --- a/Doc/library/interpreters.rst +++ b/Doc/library/interpreters.rst @@ -2,8 +2,42 @@ ========================================================== .. module:: interpreters - :synopsis: High-level Sub-interpreters Module. + :synopsis: High-level Sub-Interpreters Module. **Source code:** :source:`Lib/interpreters.py` --------------- \ No newline at end of file +-------------- + +This module constructs higher-level interpreters interfaces on top of the lower +level :mod:`_interpreters` module. + +.. versionchanged:: 3.9 + + +This module defines the following functions: + + +.. function:: create() + + Create a new interpreter and return a unique generated ID. + +.. function:: list_all() + + Return a list containing the ID of every existing interpreter. + +.. function:: get_current() + + Return the ID of the currently running interpreter. + +.. function:: destroy(id) + + Destroy the interpreter whose ID is *id*. + +.. function:: get_main() + + Return the ID of the main interpreter. + +.. function:: run_string() + + Execute the provided string in the identified interpreter. + See `PyRun_SimpleStrings`. diff --git a/Lib/interpreters.py b/Lib/interpreters.py index e69de29bb2d1d6..95537bba6cbdae 100644 --- a/Lib/interpreters.py +++ b/Lib/interpreters.py @@ -0,0 +1,59 @@ +"""Sub-interpreters High Level Module.""" + +import _interpreters + +__all__ = ['create', 'list_all', 'get_current'] + +# Rename so that "from interpreters import *" is safe +_list_all = _interpreters.list_all +_get_current = _interpreters.get_current +_get_main = _interpreters.get_main +_run_string = _interpreters.run_string + +# Global API functions + +def create(): + """ create() -> Interpreter + + Create a new interpreter and return a unique generated ID. + """ + return _interpreters.create() + +def list_all(): + """list_all() -> [Interpreter] + + Return a list containing the ID of every existing interpreter. + """ + return _list_all() + +def get_current(): + """get_current() -> Interpreter + + Return the ID of the currently running interpreter. + """ + return _get_current() + +def get_main(): + """get_main() -> ID + + Return the ID of the main interpreter. + """ + + return _get_main() + +def destroy(id): + """destroy(id) -> None + + Destroy the identified interpreter. + """ + + return _interpreters.destroy(id) + +def run_string(id, script, shared): + """run_string(id, script, shared) -> None + + Execute the provided string in the identified interpreter. + See PyRun_SimpleStrings. + """ + + return _run_string(id, script, shared) diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index 65c6e77d69d1ee..32964ec435f4c0 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -1,4 +1,5 @@ import interpreters +import _interpreters #remove when all methods are implemented from collections import namedtuple import contextlib import itertools @@ -13,16 +14,6 @@ from test import support from test.support import script_helper - -################################## -# helpers - -def powerset(*sets): - return itertools.chain.from_iterable( - combinations(sets, r) - for r in range(len(sets)+1)) - - def _captured_script(script): r, w = os.pipe() indented = script.replace('\n', '\n ') @@ -34,243 +25,6 @@ def _captured_script(script): """) return wrapped, open(r) - -def _run_output(interp, request, shared=None): - script, rpipe = _captured_script(request) - with rpipe: - interpreters.run_string(interp, script, shared) - return rpipe.read() - - -@contextlib.contextmanager -def _running(interp): - r, w = os.pipe() - def run(): - interpreters.run_string(interp, dedent(f""" - # wait for "signal" - with open({r}) as rpipe: - rpipe.read() - """)) - - t = threading.Thread(target=run) - t.start() - - yield - - with open(w, 'w') as spipe: - spipe.write('done') - t.join() - - -#@contextmanager -#def run_threaded(id, source, **shared): -# def run(): -# run_interp(id, source, **shared) -# t = threading.Thread(target=run) -# t.start() -# yield -# t.join() - - -def run_interp(id, source, **shared): - _run_interp(id, source, shared) - - -def _run_interp(id, source, shared, _mainns={}): - source = dedent(source) - main = interpreters.get_main() - if main == id: - if interpreters.get_current() != main: - raise RuntimeError - # XXX Run a func? - exec(source, _mainns) - else: - interpreters.run_string(id, source, shared) - - -def run_interp_threaded(id, source, **shared): - def run(): - _run(id, source, shared) - t = threading.Thread(target=run) - t.start() - t.join() - - -class Interpreter(namedtuple('Interpreter', 'name id')): - - @classmethod - def from_raw(cls, raw): - if isinstance(raw, cls): - return raw - elif isinstance(raw, str): - return cls(raw) - else: - raise NotImplementedError - - def __new__(cls, name=None, id=None): - main = interpreters.get_main() - if id == main: - if not name: - name = 'main' - elif name != 'main': - raise ValueError( - 'name mismatch (expected "main", got "{}")'.format(name)) - id = main - elif id is not None: - if not name: - name = 'interp' - elif name == 'main': - raise ValueError('name mismatch (unexpected "main")') - if not isinstance(id, interpreters.InterpreterID): - id = interpreters.InterpreterID(id) - elif not name or name == 'main': - name = 'main' - id = main - else: - id = interpreters.create() - self = super().__new__(cls, name, id) - return self - - -# XXX expect_channel_closed() is unnecessary once we improve exc propagation. - -@contextlib.contextmanager -def expect_channel_closed(): - try: - yield - except interpreters.ChannelClosedError: - pass - else: - assert False, 'channel not closed' - - -class ChannelAction(namedtuple('ChannelAction', 'action end interp')): - - def __new__(cls, action, end=None, interp=None): - if not end: - end = 'both' - if not interp: - interp = 'main' - self = super().__new__(cls, action, end, interp) - return self - - def __init__(self, *args, **kwargs): - if self.action == 'use': - if self.end not in ('same', 'opposite', 'send', 'recv'): - raise ValueError(self.end) - elif self.action in ('close', 'force-close'): - if self.end not in ('both', 'same', 'opposite', 'send', 'recv'): - raise ValueError(self.end) - else: - raise ValueError(self.action) - if self.interp not in ('main', 'same', 'other', 'extra'): - raise ValueError(self.interp) - - def resolve_end(self, end): - if self.end == 'same': - return end - elif self.end == 'opposite': - return 'recv' if end == 'send' else 'send' - else: - return self.end - - def resolve_interp(self, interp, other, extra): - if self.interp == 'same': - return interp - elif self.interp == 'other': - if other is None: - raise RuntimeError - return other - elif self.interp == 'extra': - if extra is None: - raise RuntimeError - return extra - elif self.interp == 'main': - if interp.name == 'main': - return interp - elif other and other.name == 'main': - return other - else: - raise RuntimeError - # Per __init__(), there aren't any others. - - -class ChannelState(namedtuple('ChannelState', 'pending closed')): - - def __new__(cls, pending=0, *, closed=False): - self = super().__new__(cls, pending, closed) - return self - - def incr(self): - return type(self)(self.pending + 1, closed=self.closed) - - def decr(self): - return type(self)(self.pending - 1, closed=self.closed) - - def close(self, *, force=True): - if self.closed: - if not force or self.pending == 0: - return self - return type(self)(0 if force else self.pending, closed=True) - - -def run_action(cid, action, end, state, *, hideclosed=True): - if state.closed: - if action == 'use' and end == 'recv' and state.pending: - expectfail = False - else: - expectfail = True - else: - expectfail = False - - try: - result = _run_action(cid, action, end, state) - except interpreters.ChannelClosedError: - if not hideclosed and not expectfail: - raise - result = state.close() - else: - if expectfail: - raise ... # XXX - return result - - -def _run_action(cid, action, end, state): - if action == 'use': - if end == 'send': - interpreters.channel_send(cid, b'spam') - return state.incr() - elif end == 'recv': - if not state.pending: - try: - interpreters.channel_recv(cid) - except interpreters.ChannelEmptyError: - return state - else: - raise Exception('expected ChannelEmptyError') - else: - interpreters.channel_recv(cid) - return state.decr() - else: - raise ValueError(end) - elif action == 'close': - kwargs = {} - if end in ('recv', 'send'): - kwargs[end] = True - interpreters.channel_close(cid, **kwargs) - return state.close() - elif action == 'force-close': - kwargs = { - 'force': True, - } - if end in ('recv', 'send'): - kwargs[end] = True - interpreters.channel_close(cid, **kwargs) - return state.close(force=True) - else: - raise ValueError(action) - - def clean_up_interpreters(): for id in interpreters.list_all(): if id == 0: # main @@ -281,133 +35,11 @@ def clean_up_interpreters(): pass # already destroyed -def clean_up_channels(): - for cid in interpreters.channel_list_all(): - try: - interpreters.channel_destroy(cid) - except interpreters.ChannelNotFoundError: - pass # already destroyed - - class TestBase(unittest.TestCase): def tearDown(self): clean_up_interpreters() - clean_up_channels() - - -################################## -# misc. tests - -class IsShareableTests(unittest.TestCase): - - def test_default_shareables(self): - shareables = [ - # singletons - None, - # builtin objects - b'spam', - 'spam', - 10, - -10, - ] - for obj in shareables: - with self.subTest(obj): - self.assertTrue( - interpreters.is_shareable(obj)) - - def test_not_shareable(self): - class Cheese: - def __init__(self, name): - self.name = name - def __str__(self): - return self.name - - class SubBytes(bytes): - """A subclass of a shareable type.""" - - not_shareables = [ - # singletons - True, - False, - NotImplemented, - ..., - # builtin types and objects - type, - object, - object(), - Exception(), - 100.0, - # user-defined types and objects - Cheese, - Cheese('Wensleydale'), - SubBytes(b'spam'), - ] - for obj in not_shareables: - with self.subTest(repr(obj)): - self.assertFalse( - interpreters.is_shareable(obj)) - - -class ShareableTypeTests(unittest.TestCase): - - def setUp(self): - super().setUp() - self.cid = interpreters.channel_create() - - def tearDown(self): - interpreters.channel_destroy(self.cid) - super().tearDown() - def _assert_values(self, values): - for obj in values: - with self.subTest(obj): - interpreters.channel_send(self.cid, obj) - got = interpreters.channel_recv(self.cid) - - self.assertEqual(got, obj) - self.assertIs(type(got), type(obj)) - # XXX Check the following in the channel tests? - #self.assertIsNot(got, obj) - - def test_singletons(self): - for obj in [None]: - with self.subTest(obj): - interpreters.channel_send(self.cid, obj) - got = interpreters.channel_recv(self.cid) - - # XXX What about between interpreters? - self.assertIs(got, obj) - - def test_types(self): - self._assert_values([ - b'spam', - 9999, - self.cid, - ]) - - def test_bytes(self): - self._assert_values(i.to_bytes(2, 'little', signed=True) - for i in range(-1, 258)) - - def test_int(self): - self._assert_values(itertools.chain(range(-1, 258), - [sys.maxsize, -sys.maxsize - 1])) - - def test_non_shareable_int(self): - ints = [ - sys.maxsize + 1, - -sys.maxsize - 2, - 2**1000, - ] - for i in ints: - with self.subTest(i): - with self.assertRaises(OverflowError): - interpreters.channel_send(self.cid, i) - - -################################## -# interpreter tests class ListAllTests(TestBase): @@ -416,1704 +48,51 @@ def test_initial(self): ids = interpreters.list_all() self.assertEqual(ids, [main]) - def test_after_creating(self): - main = interpreters.get_main() - first = interpreters.create() - second = interpreters.create() - ids = interpreters.list_all() - self.assertEqual(ids, [main, first, second]) - - def test_after_destroying(self): - main = interpreters.get_main() - first = interpreters.create() - second = interpreters.create() - interpreters.destroy(first) - ids = interpreters.list_all() - self.assertEqual(ids, [main, second]) - class GetCurrentTests(TestBase): - def test_main(self): + def test_get_current(self): main = interpreters.get_main() cur = interpreters.get_current() self.assertEqual(cur, main) - self.assertIsInstance(cur, interpreters.InterpreterID) - - def test_subinterpreter(self): - main = interpreters.get_main() - interp = interpreters.create() - out = _run_output(interp, dedent(""" - import _xxsubinterpreters as _interpreters - cur = _interpreters.get_current() - print(cur) - assert isinstance(cur, _interpreters.InterpreterID) - """)) - cur = int(out.strip()) - _, expected = interpreters.list_all() - self.assertEqual(cur, expected) - self.assertNotEqual(cur, main) class GetMainTests(TestBase): - def test_from_main(self): + def test_get_main(self): [expected] = interpreters.list_all() main = interpreters.get_main() self.assertEqual(main, expected) - self.assertIsInstance(main, interpreters.InterpreterID) - - def test_from_subinterpreter(self): - [expected] = interpreters.list_all() - interp = interpreters.create() - out = _run_output(interp, dedent(""" - import _xxsubinterpreters as _interpreters - main = _interpreters.get_main() - print(main) - assert isinstance(main, _interpreters.InterpreterID) - """)) - main = int(out.strip()) - self.assertEqual(main, expected) - - -class IsRunningTests(TestBase): - - def test_main(self): - main = interpreters.get_main() - self.assertTrue(interpreters.is_running(main)) - - def test_subinterpreter(self): - interp = interpreters.create() - self.assertFalse(interpreters.is_running(interp)) - - with _running(interp): - self.assertTrue(interpreters.is_running(interp)) - self.assertFalse(interpreters.is_running(interp)) - - def test_from_subinterpreter(self): - interp = interpreters.create() - out = _run_output(interp, dedent(f""" - import _xxsubinterpreters as _interpreters - if _interpreters.is_running({interp}): - print(True) - else: - print(False) - """)) - self.assertEqual(out.strip(), 'True') - - def test_already_destroyed(self): - interp = interpreters.create() - interpreters.destroy(interp) - with self.assertRaises(RuntimeError): - interpreters.is_running(interp) - - def test_does_not_exist(self): - with self.assertRaises(RuntimeError): - interpreters.is_running(1_000_000) - - def test_bad_id(self): - with self.assertRaises(RuntimeError): - interpreters.is_running(-1) - - -class InterpreterIDTests(TestBase): - - def test_with_int(self): - id = interpreters.InterpreterID(10, force=True) - - self.assertEqual(int(id), 10) - - def test_coerce_id(self): - id = interpreters.InterpreterID('10', force=True) - self.assertEqual(int(id), 10) - - id = interpreters.InterpreterID(10.0, force=True) - self.assertEqual(int(id), 10) - - class Int(str): - def __init__(self, value): - self._value = value - def __int__(self): - return self._value - - id = interpreters.InterpreterID(Int(10), force=True) - self.assertEqual(int(id), 10) - - def test_bad_id(self): - for id in [-1, 'spam']: - with self.subTest(id): - with self.assertRaises(ValueError): - interpreters.InterpreterID(id) - with self.assertRaises(OverflowError): - interpreters.InterpreterID(2**64) - with self.assertRaises(TypeError): - interpreters.InterpreterID(object()) - - def test_does_not_exist(self): - id = interpreters.channel_create() - with self.assertRaises(RuntimeError): - interpreters.InterpreterID(int(id) + 1) # unforced - - def test_str(self): - id = interpreters.InterpreterID(10, force=True) - self.assertEqual(str(id), '10') - - def test_repr(self): - id = interpreters.InterpreterID(10, force=True) - self.assertEqual(repr(id), 'InterpreterID(10)') - - def test_equality(self): - id1 = interpreters.create() - id2 = interpreters.InterpreterID(int(id1)) - id3 = interpreters.create() - - self.assertTrue(id1 == id1) - self.assertTrue(id1 == id2) - self.assertTrue(id1 == int(id1)) - self.assertFalse(id1 == id3) - - self.assertFalse(id1 != id1) - self.assertFalse(id1 != id2) - self.assertTrue(id1 != id3) class CreateTests(TestBase): - def test_in_main(self): + def test_create(self): id = interpreters.create() - self.assertIsInstance(id, interpreters.InterpreterID) - self.assertIn(id, interpreters.list_all()) - @unittest.skip('enable this test when working on pystate.c') - def test_unique_id(self): - seen = set() - for _ in range(100): - id = interpreters.create() - interpreters.destroy(id) - seen.add(id) - - self.assertEqual(len(seen), 100) - - def test_in_thread(self): - lock = threading.Lock() - id = None - def f(): - nonlocal id - id = interpreters.create() - lock.acquire() - lock.release() - - t = threading.Thread(target=f) - with lock: - t.start() - t.join() - self.assertIn(id, interpreters.list_all()) - - def test_in_subinterpreter(self): - main, = interpreters.list_all() - id1 = interpreters.create() - out = _run_output(id1, dedent(""" - import _xxsubinterpreters as _interpreters - id = _interpreters.create() - print(id) - assert isinstance(id, _interpreters.InterpreterID) - """)) - id2 = int(out.strip()) - - self.assertEqual(set(interpreters.list_all()), {main, id1, id2}) - - def test_in_threaded_subinterpreter(self): - main, = interpreters.list_all() - id1 = interpreters.create() - id2 = None - def f(): - nonlocal id2 - out = _run_output(id1, dedent(""" - import _xxsubinterpreters as _interpreters - id = _interpreters.create() - print(id) - """)) - id2 = int(out.strip()) - - t = threading.Thread(target=f) - t.start() - t.join() - - self.assertEqual(set(interpreters.list_all()), {main, id1, id2}) - - def test_after_destroy_all(self): - before = set(interpreters.list_all()) - # Create 3 subinterpreters. - ids = [] - for _ in range(3): - id = interpreters.create() - ids.append(id) - # Now destroy them. - for id in ids: - interpreters.destroy(id) - # Finally, create another. - id = interpreters.create() - self.assertEqual(set(interpreters.list_all()), before | {id}) - - def test_after_destroy_some(self): - before = set(interpreters.list_all()) - # Create 3 subinterpreters. - id1 = interpreters.create() - id2 = interpreters.create() - id3 = interpreters.create() - # Now destroy 2 of them. - interpreters.destroy(id1) - interpreters.destroy(id3) - # Finally, create another. - id = interpreters.create() - self.assertEqual(set(interpreters.list_all()), before | {id, id2}) - - class DestroyTests(TestBase): - def test_one(self): + def test_destroy(self): id1 = interpreters.create() id2 = interpreters.create() id3 = interpreters.create() self.assertIn(id2, interpreters.list_all()) interpreters.destroy(id2) self.assertNotIn(id2, interpreters.list_all()) - self.assertIn(id1, interpreters.list_all()) - self.assertIn(id3, interpreters.list_all()) - - def test_all(self): - before = set(interpreters.list_all()) - ids = set() - for _ in range(3): - id = interpreters.create() - ids.add(id) - self.assertEqual(set(interpreters.list_all()), before | ids) - for id in ids: - interpreters.destroy(id) - self.assertEqual(set(interpreters.list_all()), before) - - def test_main(self): - main, = interpreters.list_all() - with self.assertRaises(RuntimeError): - interpreters.destroy(main) - - def f(): - with self.assertRaises(RuntimeError): - interpreters.destroy(main) - - t = threading.Thread(target=f) - t.start() - t.join() - - def test_already_destroyed(self): - id = interpreters.create() - interpreters.destroy(id) - with self.assertRaises(RuntimeError): - interpreters.destroy(id) - - def test_does_not_exist(self): - with self.assertRaises(RuntimeError): - interpreters.destroy(1_000_000) - - def test_bad_id(self): - with self.assertRaises(RuntimeError): - interpreters.destroy(-1) - - def test_from_current(self): - main, = interpreters.list_all() - id = interpreters.create() - script = dedent(f""" - import _xxsubinterpreters as _interpreters - try: - _interpreters.destroy({id}) - except RuntimeError: - pass - """) - - interpreters.run_string(id, script) - self.assertEqual(set(interpreters.list_all()), {main, id}) - - def test_from_sibling(self): - main, = interpreters.list_all() - id1 = interpreters.create() - id2 = interpreters.create() - script = dedent(f""" - import _xxsubinterpreters as _interpreters - _interpreters.destroy({id2}) - """) - interpreters.run_string(id1, script) - - self.assertEqual(set(interpreters.list_all()), {main, id1}) - - def test_from_other_thread(self): - id = interpreters.create() - def f(): - interpreters.destroy(id) - - t = threading.Thread(target=f) - t.start() - t.join() - - def test_still_running(self): - main, = interpreters.list_all() - interp = interpreters.create() - with _running(interp): - with self.assertRaises(RuntimeError): - interpreters.destroy(interp) - self.assertTrue(interpreters.is_running(interp)) class RunStringTests(TestBase): - SCRIPT = dedent(""" - with open('{}', 'w') as out: - out.write('{}') - """) - FILENAME = 'spam' - - def setUp(self): - super().setUp() - self.id = interpreters.create() - self._fs = None - - def tearDown(self): - if self._fs is not None: - self._fs.close() - super().tearDown() - - @property - def fs(self): - if self._fs is None: - self._fs = FSFixture(self) - return self._fs - - def test_success(self): - script, file = _captured_script('print("it worked!", end="")') - with file: - interpreters.run_string(self.id, script) - out = file.read() - - self.assertEqual(out, 'it worked!') - - def test_in_thread(self): + def test_run_string(self): script, file = _captured_script('print("it worked!", end="")') + id = interpreters.create() with file: - def f(): - interpreters.run_string(self.id, script) - - t = threading.Thread(target=f) - t.start() - t.join() - out = file.read() - - self.assertEqual(out, 'it worked!') - - def test_create_thread(self): - script, file = _captured_script(""" - import threading - def f(): - print('it worked!', end='') - - t = threading.Thread(target=f) - t.start() - t.join() - """) - with file: - interpreters.run_string(self.id, script) + interpreters.run_string(id, script, None) out = file.read() self.assertEqual(out, 'it worked!') - @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") - def test_fork(self): - import tempfile - with tempfile.NamedTemporaryFile('w+') as file: - file.write('') - file.flush() - - expected = 'spam spam spam spam spam' - script = dedent(f""" - import os - try: - os.fork() - except RuntimeError: - with open('{file.name}', 'w') as out: - out.write('{expected}') - """) - interpreters.run_string(self.id, script) - - file.seek(0) - content = file.read() - self.assertEqual(content, expected) - - def test_already_running(self): - with _running(self.id): - with self.assertRaises(RuntimeError): - interpreters.run_string(self.id, 'print("spam")') - - def test_does_not_exist(self): - id = 0 - while id in interpreters.list_all(): - id += 1 - with self.assertRaises(RuntimeError): - interpreters.run_string(id, 'print("spam")') - - def test_error_id(self): - with self.assertRaises(RuntimeError): - interpreters.run_string(-1, 'print("spam")') - - def test_bad_id(self): - with self.assertRaises(TypeError): - interpreters.run_string('spam', 'print("spam")') - - def test_bad_script(self): - with self.assertRaises(TypeError): - interpreters.run_string(self.id, 10) - - def test_bytes_for_script(self): - with self.assertRaises(TypeError): - interpreters.run_string(self.id, b'print("spam")') - - @contextlib.contextmanager - def assert_run_failed(self, exctype, msg=None): - with self.assertRaises(interpreters.RunFailedError) as caught: - yield - if msg is None: - self.assertEqual(str(caught.exception).split(':')[0], - str(exctype)) - else: - self.assertEqual(str(caught.exception), - "{}: {}".format(exctype, msg)) - - def test_invalid_syntax(self): - with self.assert_run_failed(SyntaxError): - # missing close paren - interpreters.run_string(self.id, 'print("spam"') - - def test_failure(self): - with self.assert_run_failed(Exception, 'spam'): - interpreters.run_string(self.id, 'raise Exception("spam")') - - def test_SystemExit(self): - with self.assert_run_failed(SystemExit, '42'): - interpreters.run_string(self.id, 'raise SystemExit(42)') - - def test_sys_exit(self): - with self.assert_run_failed(SystemExit): - interpreters.run_string(self.id, dedent(""" - import sys - sys.exit() - """)) - - with self.assert_run_failed(SystemExit, '42'): - interpreters.run_string(self.id, dedent(""" - import sys - sys.exit(42) - """)) - - def test_with_shared(self): - r, w = os.pipe() - - shared = { - 'spam': b'ham', - 'eggs': b'-1', - 'cheddar': None, - } - script = dedent(f""" - eggs = int(eggs) - spam = 42 - result = spam + eggs - - ns = dict(vars()) - del ns['__builtins__'] - import pickle - with open({w}, 'wb') as chan: - pickle.dump(ns, chan) - """) - interpreters.run_string(self.id, script, shared) - with open(r, 'rb') as chan: - ns = pickle.load(chan) - - self.assertEqual(ns['spam'], 42) - self.assertEqual(ns['eggs'], -1) - self.assertEqual(ns['result'], 41) - self.assertIsNone(ns['cheddar']) - - def test_shared_overwrites(self): - interpreters.run_string(self.id, dedent(""" - spam = 'eggs' - ns1 = dict(vars()) - del ns1['__builtins__'] - """)) - - shared = {'spam': b'ham'} - script = dedent(f""" - ns2 = dict(vars()) - del ns2['__builtins__'] - """) - interpreters.run_string(self.id, script, shared) - - r, w = os.pipe() - script = dedent(f""" - ns = dict(vars()) - del ns['__builtins__'] - import pickle - with open({w}, 'wb') as chan: - pickle.dump(ns, chan) - """) - interpreters.run_string(self.id, script) - with open(r, 'rb') as chan: - ns = pickle.load(chan) - - self.assertEqual(ns['ns1']['spam'], 'eggs') - self.assertEqual(ns['ns2']['spam'], b'ham') - self.assertEqual(ns['spam'], b'ham') - - def test_shared_overwrites_default_vars(self): - r, w = os.pipe() - - shared = {'__name__': b'not __main__'} - script = dedent(f""" - spam = 42 - - ns = dict(vars()) - del ns['__builtins__'] - import pickle - with open({w}, 'wb') as chan: - pickle.dump(ns, chan) - """) - interpreters.run_string(self.id, script, shared) - with open(r, 'rb') as chan: - ns = pickle.load(chan) - - self.assertEqual(ns['__name__'], b'not __main__') - - def test_main_reused(self): - r, w = os.pipe() - interpreters.run_string(self.id, dedent(f""" - spam = True - - ns = dict(vars()) - del ns['__builtins__'] - import pickle - with open({w}, 'wb') as chan: - pickle.dump(ns, chan) - del ns, pickle, chan - """)) - with open(r, 'rb') as chan: - ns1 = pickle.load(chan) - - r, w = os.pipe() - interpreters.run_string(self.id, dedent(f""" - eggs = False - - ns = dict(vars()) - del ns['__builtins__'] - import pickle - with open({w}, 'wb') as chan: - pickle.dump(ns, chan) - """)) - with open(r, 'rb') as chan: - ns2 = pickle.load(chan) - - self.assertIn('spam', ns1) - self.assertNotIn('eggs', ns1) - self.assertIn('eggs', ns2) - self.assertIn('spam', ns2) - - def test_execution_namespace_is_main(self): - r, w = os.pipe() - - script = dedent(f""" - spam = 42 - - ns = dict(vars()) - ns['__builtins__'] = str(ns['__builtins__']) - import pickle - with open({w}, 'wb') as chan: - pickle.dump(ns, chan) - """) - interpreters.run_string(self.id, script) - with open(r, 'rb') as chan: - ns = pickle.load(chan) - - ns.pop('__builtins__') - ns.pop('__loader__') - self.assertEqual(ns, { - '__name__': '__main__', - '__annotations__': {}, - '__doc__': None, - '__package__': None, - '__spec__': None, - 'spam': 42, - }) - - # XXX Fix this test! - @unittest.skip('blocking forever') - def test_still_running_at_exit(self): - script = dedent(f""" - from textwrap import dedent - import threading - import _xxsubinterpreters as _interpreters - id = _interpreters.create() - def f(): - _interpreters.run_string(id, dedent(''' - import time - # Give plenty of time for the main interpreter to finish. - time.sleep(1_000_000) - ''')) - - t = threading.Thread(target=f) - t.start() - """) - with support.temp_dir() as dirname: - filename = script_helper.make_script(dirname, 'interp', script) - with script_helper.spawn_python(filename) as proc: - retcode = proc.wait() - - self.assertEqual(retcode, 0) - - -################################## -# channel tests - -class ChannelIDTests(TestBase): - - def test_default_kwargs(self): - cid = interpreters._channel_id(10, force=True) - - self.assertEqual(int(cid), 10) - self.assertEqual(cid.end, 'both') - - def test_with_kwargs(self): - cid = interpreters._channel_id(10, send=True, force=True) - self.assertEqual(cid.end, 'send') - - cid = interpreters._channel_id(10, send=True, recv=False, force=True) - self.assertEqual(cid.end, 'send') - - cid = interpreters._channel_id(10, recv=True, force=True) - self.assertEqual(cid.end, 'recv') - - cid = interpreters._channel_id(10, recv=True, send=False, force=True) - self.assertEqual(cid.end, 'recv') - - cid = interpreters._channel_id(10, send=True, recv=True, force=True) - self.assertEqual(cid.end, 'both') - - def test_coerce_id(self): - cid = interpreters._channel_id('10', force=True) - self.assertEqual(int(cid), 10) - - cid = interpreters._channel_id(10.0, force=True) - self.assertEqual(int(cid), 10) - - class Int(str): - def __init__(self, value): - self._value = value - def __int__(self): - return self._value - - cid = interpreters._channel_id(Int(10), force=True) - self.assertEqual(int(cid), 10) - - def test_bad_id(self): - for cid in [-1, 'spam']: - with self.subTest(cid): - with self.assertRaises(ValueError): - interpreters._channel_id(cid) - with self.assertRaises(OverflowError): - interpreters._channel_id(2**64) - with self.assertRaises(TypeError): - interpreters._channel_id(object()) - - def test_bad_kwargs(self): - with self.assertRaises(ValueError): - interpreters._channel_id(10, send=False, recv=False) - - def test_does_not_exist(self): - cid = interpreters.channel_create() - with self.assertRaises(interpreters.ChannelNotFoundError): - interpreters._channel_id(int(cid) + 1) # unforced - - def test_str(self): - cid = interpreters._channel_id(10, force=True) - self.assertEqual(str(cid), '10') - - def test_repr(self): - cid = interpreters._channel_id(10, force=True) - self.assertEqual(repr(cid), 'ChannelID(10)') - - cid = interpreters._channel_id(10, send=True, force=True) - self.assertEqual(repr(cid), 'ChannelID(10, send=True)') - - cid = interpreters._channel_id(10, recv=True, force=True) - self.assertEqual(repr(cid), 'ChannelID(10, recv=True)') - - cid = interpreters._channel_id(10, send=True, recv=True, force=True) - self.assertEqual(repr(cid), 'ChannelID(10)') - - def test_equality(self): - cid1 = interpreters.channel_create() - cid2 = interpreters._channel_id(int(cid1)) - cid3 = interpreters.channel_create() - - self.assertTrue(cid1 == cid1) - self.assertTrue(cid1 == cid2) - self.assertTrue(cid1 == int(cid1)) - self.assertFalse(cid1 == cid3) - - self.assertFalse(cid1 != cid1) - self.assertFalse(cid1 != cid2) - self.assertTrue(cid1 != cid3) - - -class ChannelTests(TestBase): - - def test_create_cid(self): - cid = interpreters.channel_create() - self.assertIsInstance(cid, interpreters.ChannelID) - - def test_sequential_ids(self): - before = interpreters.channel_list_all() - id1 = interpreters.channel_create() - id2 = interpreters.channel_create() - id3 = interpreters.channel_create() - after = interpreters.channel_list_all() - - self.assertEqual(id2, int(id1) + 1) - self.assertEqual(id3, int(id2) + 1) - self.assertEqual(set(after) - set(before), {id1, id2, id3}) - - def test_ids_global(self): - id1 = interpreters.create() - out = _run_output(id1, dedent(""" - import _xxsubinterpreters as _interpreters - cid = _interpreters.channel_create() - print(cid) - """)) - cid1 = int(out.strip()) - - id2 = interpreters.create() - out = _run_output(id2, dedent(""" - import _xxsubinterpreters as _interpreters - cid = _interpreters.channel_create() - print(cid) - """)) - cid2 = int(out.strip()) - - self.assertEqual(cid2, int(cid1) + 1) - - #################### - - def test_send_recv_main(self): - cid = interpreters.channel_create() - orig = b'spam' - interpreters.channel_send(cid, orig) - obj = interpreters.channel_recv(cid) - - self.assertEqual(obj, orig) - self.assertIsNot(obj, orig) - - def test_send_recv_same_interpreter(self): - id1 = interpreters.create() - out = _run_output(id1, dedent(""" - import _xxsubinterpreters as _interpreters - cid = _interpreters.channel_create() - orig = b'spam' - _interpreters.channel_send(cid, orig) - obj = _interpreters.channel_recv(cid) - assert obj is not orig - assert obj == orig - """)) - - def test_send_recv_different_interpreters(self): - cid = interpreters.channel_create() - id1 = interpreters.create() - out = _run_output(id1, dedent(f""" - import _xxsubinterpreters as _interpreters - _interpreters.channel_send({cid}, b'spam') - """)) - obj = interpreters.channel_recv(cid) - - self.assertEqual(obj, b'spam') - - def test_send_recv_different_threads(self): - cid = interpreters.channel_create() - - def f(): - while True: - try: - obj = interpreters.channel_recv(cid) - break - except interpreters.ChannelEmptyError: - time.sleep(0.1) - interpreters.channel_send(cid, obj) - t = threading.Thread(target=f) - t.start() - - interpreters.channel_send(cid, b'spam') - t.join() - obj = interpreters.channel_recv(cid) - - self.assertEqual(obj, b'spam') - - def test_send_recv_different_interpreters_and_threads(self): - cid = interpreters.channel_create() - id1 = interpreters.create() - out = None - - def f(): - nonlocal out - out = _run_output(id1, dedent(f""" - import time - import _xxsubinterpreters as _interpreters - while True: - try: - obj = _interpreters.channel_recv({cid}) - break - except _interpreters.ChannelEmptyError: - time.sleep(0.1) - assert(obj == b'spam') - _interpreters.channel_send({cid}, b'eggs') - """)) - t = threading.Thread(target=f) - t.start() - - interpreters.channel_send(cid, b'spam') - t.join() - obj = interpreters.channel_recv(cid) - - self.assertEqual(obj, b'eggs') - - def test_send_not_found(self): - with self.assertRaises(interpreters.ChannelNotFoundError): - interpreters.channel_send(10, b'spam') - - def test_recv_not_found(self): - with self.assertRaises(interpreters.ChannelNotFoundError): - interpreters.channel_recv(10) - - def test_recv_empty(self): - cid = interpreters.channel_create() - with self.assertRaises(interpreters.ChannelEmptyError): - interpreters.channel_recv(cid) - - def test_run_string_arg_unresolved(self): - cid = interpreters.channel_create() - interp = interpreters.create() - - out = _run_output(interp, dedent(""" - import _xxsubinterpreters as _interpreters - print(cid.end) - _interpreters.channel_send(cid, b'spam') - """), - dict(cid=cid.send)) - obj = interpreters.channel_recv(cid) - - self.assertEqual(obj, b'spam') - self.assertEqual(out.strip(), 'send') - - # XXX For now there is no high-level channel into which the - # sent channel ID can be converted... - # Note: this test caused crashes on some buildbots (bpo-33615). - @unittest.skip('disabled until high-level channels exist') - def test_run_string_arg_resolved(self): - cid = interpreters.channel_create() - cid = interpreters._channel_id(cid, _resolve=True) - interp = interpreters.create() - - out = _run_output(interp, dedent(""" - import _xxsubinterpreters as _interpreters - print(chan.id.end) - _interpreters.channel_send(chan.id, b'spam') - """), - dict(chan=cid.send)) - obj = interpreters.channel_recv(cid) - - self.assertEqual(obj, b'spam') - self.assertEqual(out.strip(), 'send') - - # close - - def test_close_single_user(self): - cid = interpreters.channel_create() - interpreters.channel_send(cid, b'spam') - interpreters.channel_recv(cid) - interpreters.channel_close(cid) - - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_send(cid, b'eggs') - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_recv(cid) - - def test_close_multiple_users(self): - cid = interpreters.channel_create() - id1 = interpreters.create() - id2 = interpreters.create() - interpreters.run_string(id1, dedent(f""" - import _xxsubinterpreters as _interpreters - _interpreters.channel_send({cid}, b'spam') - """)) - interpreters.run_string(id2, dedent(f""" - import _xxsubinterpreters as _interpreters - _interpreters.channel_recv({cid}) - """)) - interpreters.channel_close(cid) - with self.assertRaises(interpreters.RunFailedError) as cm: - interpreters.run_string(id1, dedent(f""" - _interpreters.channel_send({cid}, b'spam') - """)) - self.assertIn('ChannelClosedError', str(cm.exception)) - with self.assertRaises(interpreters.RunFailedError) as cm: - interpreters.run_string(id2, dedent(f""" - _interpreters.channel_send({cid}, b'spam') - """)) - self.assertIn('ChannelClosedError', str(cm.exception)) - - def test_close_multiple_times(self): - cid = interpreters.channel_create() - interpreters.channel_send(cid, b'spam') - interpreters.channel_recv(cid) - interpreters.channel_close(cid) - - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_close(cid) - - def test_close_empty(self): - tests = [ - (False, False), - (True, False), - (False, True), - (True, True), - ] - for send, recv in tests: - with self.subTest((send, recv)): - cid = interpreters.channel_create() - interpreters.channel_send(cid, b'spam') - interpreters.channel_recv(cid) - interpreters.channel_close(cid, send=send, recv=recv) - - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_send(cid, b'eggs') - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_recv(cid) - - def test_close_defaults_with_unused_items(self): - cid = interpreters.channel_create() - interpreters.channel_send(cid, b'spam') - interpreters.channel_send(cid, b'ham') - - with self.assertRaises(interpreters.ChannelNotEmptyError): - interpreters.channel_close(cid) - interpreters.channel_recv(cid) - interpreters.channel_send(cid, b'eggs') - - def test_close_recv_with_unused_items_unforced(self): - cid = interpreters.channel_create() - interpreters.channel_send(cid, b'spam') - interpreters.channel_send(cid, b'ham') - - with self.assertRaises(interpreters.ChannelNotEmptyError): - interpreters.channel_close(cid, recv=True) - interpreters.channel_recv(cid) - interpreters.channel_send(cid, b'eggs') - interpreters.channel_recv(cid) - interpreters.channel_recv(cid) - interpreters.channel_close(cid, recv=True) - - def test_close_send_with_unused_items_unforced(self): - cid = interpreters.channel_create() - interpreters.channel_send(cid, b'spam') - interpreters.channel_send(cid, b'ham') - interpreters.channel_close(cid, send=True) - - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_send(cid, b'eggs') - interpreters.channel_recv(cid) - interpreters.channel_recv(cid) - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_recv(cid) - - def test_close_both_with_unused_items_unforced(self): - cid = interpreters.channel_create() - interpreters.channel_send(cid, b'spam') - interpreters.channel_send(cid, b'ham') - - with self.assertRaises(interpreters.ChannelNotEmptyError): - interpreters.channel_close(cid, recv=True, send=True) - interpreters.channel_recv(cid) - interpreters.channel_send(cid, b'eggs') - interpreters.channel_recv(cid) - interpreters.channel_recv(cid) - interpreters.channel_close(cid, recv=True) - - def test_close_recv_with_unused_items_forced(self): - cid = interpreters.channel_create() - interpreters.channel_send(cid, b'spam') - interpreters.channel_send(cid, b'ham') - interpreters.channel_close(cid, recv=True, force=True) - - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_send(cid, b'eggs') - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_recv(cid) - - def test_close_send_with_unused_items_forced(self): - cid = interpreters.channel_create() - interpreters.channel_send(cid, b'spam') - interpreters.channel_send(cid, b'ham') - interpreters.channel_close(cid, send=True, force=True) - - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_send(cid, b'eggs') - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_recv(cid) - - def test_close_both_with_unused_items_forced(self): - cid = interpreters.channel_create() - interpreters.channel_send(cid, b'spam') - interpreters.channel_send(cid, b'ham') - interpreters.channel_close(cid, send=True, recv=True, force=True) - - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_send(cid, b'eggs') - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_recv(cid) - - def test_close_never_used(self): - cid = interpreters.channel_create() - interpreters.channel_close(cid) - - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_send(cid, b'spam') - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_recv(cid) - - def test_close_by_unassociated_interp(self): - cid = interpreters.channel_create() - interpreters.channel_send(cid, b'spam') - interp = interpreters.create() - interpreters.run_string(interp, dedent(f""" - import _xxsubinterpreters as _interpreters - _interpreters.channel_close({cid}, force=True) - """)) - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_recv(cid) - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_close(cid) - - def test_close_used_multiple_times_by_single_user(self): - cid = interpreters.channel_create() - interpreters.channel_send(cid, b'spam') - interpreters.channel_send(cid, b'spam') - interpreters.channel_send(cid, b'spam') - interpreters.channel_recv(cid) - interpreters.channel_close(cid, force=True) - - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_send(cid, b'eggs') - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_recv(cid) - - -class ChannelReleaseTests(TestBase): - - # XXX Add more test coverage a la the tests for close(). - - """ - - main / interp / other - - run in: current thread / new thread / other thread / different threads - - end / opposite - - force / no force - - used / not used (associated / not associated) - - empty / emptied / never emptied / partly emptied - - closed / not closed - - released / not released - - creator (interp) / other - - associated interpreter not running - - associated interpreter destroyed - """ - - """ - use - pre-release - release - after - check - """ - - """ - release in: main, interp1 - creator: same, other (incl. interp2) - - use: None,send,recv,send/recv in None,same,other(incl. interp2),same+other(incl. interp2),all - pre-release: None,send,recv,both in None,same,other(incl. interp2),same+other(incl. interp2),all - pre-release forced: None,send,recv,both in None,same,other(incl. interp2),same+other(incl. interp2),all - - release: same - release forced: same - - use after: None,send,recv,send/recv in None,same,other(incl. interp2),same+other(incl. interp2),all - release after: None,send,recv,send/recv in None,same,other(incl. interp2),same+other(incl. interp2),all - check released: send/recv for same/other(incl. interp2) - check closed: send/recv for same/other(incl. interp2) - """ - - def test_single_user(self): - cid = interpreters.channel_create() - interpreters.channel_send(cid, b'spam') - interpreters.channel_recv(cid) - interpreters.channel_release(cid, send=True, recv=True) - - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_send(cid, b'eggs') - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_recv(cid) - - def test_multiple_users(self): - cid = interpreters.channel_create() - id1 = interpreters.create() - id2 = interpreters.create() - interpreters.run_string(id1, dedent(f""" - import _xxsubinterpreters as _interpreters - _interpreters.channel_send({cid}, b'spam') - """)) - out = _run_output(id2, dedent(f""" - import _xxsubinterpreters as _interpreters - obj = _interpreters.channel_recv({cid}) - _interpreters.channel_release({cid}) - print(repr(obj)) - """)) - interpreters.run_string(id1, dedent(f""" - _interpreters.channel_release({cid}) - """)) - - self.assertEqual(out.strip(), "b'spam'") - - def test_no_kwargs(self): - cid = interpreters.channel_create() - interpreters.channel_send(cid, b'spam') - interpreters.channel_recv(cid) - interpreters.channel_release(cid) - - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_send(cid, b'eggs') - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_recv(cid) - - def test_multiple_times(self): - cid = interpreters.channel_create() - interpreters.channel_send(cid, b'spam') - interpreters.channel_recv(cid) - interpreters.channel_release(cid, send=True, recv=True) - - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_release(cid, send=True, recv=True) - - def test_with_unused_items(self): - cid = interpreters.channel_create() - interpreters.channel_send(cid, b'spam') - interpreters.channel_send(cid, b'ham') - interpreters.channel_release(cid, send=True, recv=True) - - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_recv(cid) - - def test_never_used(self): - cid = interpreters.channel_create() - interpreters.channel_release(cid) - - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_send(cid, b'spam') - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_recv(cid) - - def test_by_unassociated_interp(self): - cid = interpreters.channel_create() - interpreters.channel_send(cid, b'spam') - interp = interpreters.create() - interpreters.run_string(interp, dedent(f""" - import _xxsubinterpreters as _interpreters - _interpreters.channel_release({cid}) - """)) - obj = interpreters.channel_recv(cid) - interpreters.channel_release(cid) - - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_send(cid, b'eggs') - self.assertEqual(obj, b'spam') - - def test_close_if_unassociated(self): - # XXX Something's not right with this test... - cid = interpreters.channel_create() - interp = interpreters.create() - interpreters.run_string(interp, dedent(f""" - import _xxsubinterpreters as _interpreters - obj = _interpreters.channel_send({cid}, b'spam') - _interpreters.channel_release({cid}) - """)) - - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_recv(cid) - - def test_partially(self): - # XXX Is partial close too weird/confusing? - cid = interpreters.channel_create() - interpreters.channel_send(cid, None) - interpreters.channel_recv(cid) - interpreters.channel_send(cid, b'spam') - interpreters.channel_release(cid, send=True) - obj = interpreters.channel_recv(cid) - - self.assertEqual(obj, b'spam') - - def test_used_multiple_times_by_single_user(self): - cid = interpreters.channel_create() - interpreters.channel_send(cid, b'spam') - interpreters.channel_send(cid, b'spam') - interpreters.channel_send(cid, b'spam') - interpreters.channel_recv(cid) - interpreters.channel_release(cid, send=True, recv=True) - - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_send(cid, b'eggs') - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_recv(cid) - - -class ChannelCloseFixture(namedtuple('ChannelCloseFixture', - 'end interp other extra creator')): - - # Set this to True to avoid creating interpreters, e.g. when - # scanning through test permutations without running them. - QUICK = False - - def __new__(cls, end, interp, other, extra, creator): - assert end in ('send', 'recv') - if cls.QUICK: - known = {} - else: - interp = Interpreter.from_raw(interp) - other = Interpreter.from_raw(other) - extra = Interpreter.from_raw(extra) - known = { - interp.name: interp, - other.name: other, - extra.name: extra, - } - if not creator: - creator = 'same' - self = super().__new__(cls, end, interp, other, extra, creator) - self._prepped = set() - self._state = ChannelState() - self._known = known - return self - - @property - def state(self): - return self._state - - @property - def cid(self): - try: - return self._cid - except AttributeError: - creator = self._get_interpreter(self.creator) - self._cid = self._new_channel(creator) - return self._cid - - def get_interpreter(self, interp): - interp = self._get_interpreter(interp) - self._prep_interpreter(interp) - return interp - - def expect_closed_error(self, end=None): - if end is None: - end = self.end - if end == 'recv' and self.state.closed == 'send': - return False - return bool(self.state.closed) - - def prep_interpreter(self, interp): - self._prep_interpreter(interp) - - def record_action(self, action, result): - self._state = result - - def clean_up(self): - clean_up_interpreters() - clean_up_channels() - - # internal methods - - def _new_channel(self, creator): - if creator.name == 'main': - return interpreters.channel_create() - else: - ch = interpreters.channel_create() - run_interp(creator.id, f""" - import _xxsubinterpreters - cid = _xxsubinterpreters.channel_create() - # We purposefully send back an int to avoid tying the - # channel to the other interpreter. - _xxsubinterpreters.channel_send({ch}, int(cid)) - del _xxsubinterpreters - """) - self._cid = interpreters.channel_recv(ch) - return self._cid - - def _get_interpreter(self, interp): - if interp in ('same', 'interp'): - return self.interp - elif interp == 'other': - return self.other - elif interp == 'extra': - return self.extra - else: - name = interp - try: - interp = self._known[name] - except KeyError: - interp = self._known[name] = Interpreter(name) - return interp - - def _prep_interpreter(self, interp): - if interp.id in self._prepped: - return - self._prepped.add(interp.id) - if interp.name == 'main': - return - run_interp(interp.id, f""" - import _xxsubinterpreters as interpreters - import test.test__xxsubinterpreters as helpers - ChannelState = helpers.ChannelState - try: - cid - except NameError: - cid = interpreters._channel_id({self.cid}) - """) - - -@unittest.skip('these tests take several hours to run') -class ExhaustiveChannelTests(TestBase): - - """ - - main / interp / other - - run in: current thread / new thread / other thread / different threads - - end / opposite - - force / no force - - used / not used (associated / not associated) - - empty / emptied / never emptied / partly emptied - - closed / not closed - - released / not released - - creator (interp) / other - - associated interpreter not running - - associated interpreter destroyed - - - close after unbound - """ - - """ - use - pre-close - close - after - check - """ - - """ - close in: main, interp1 - creator: same, other, extra - - use: None,send,recv,send/recv in None,same,other,same+other,all - pre-close: None,send,recv in None,same,other,same+other,all - pre-close forced: None,send,recv in None,same,other,same+other,all - - close: same - close forced: same - - use after: None,send,recv,send/recv in None,same,other,extra,same+other,all - close after: None,send,recv,send/recv in None,same,other,extra,same+other,all - check closed: send/recv for same/other(incl. interp2) - """ - - def iter_action_sets(self): - # - used / not used (associated / not associated) - # - empty / emptied / never emptied / partly emptied - # - closed / not closed - # - released / not released - - # never used - yield [] - - # only pre-closed (and possible used after) - for closeactions in self._iter_close_action_sets('same', 'other'): - yield closeactions - for postactions in self._iter_post_close_action_sets(): - yield closeactions + postactions - for closeactions in self._iter_close_action_sets('other', 'extra'): - yield closeactions - for postactions in self._iter_post_close_action_sets(): - yield closeactions + postactions - - # used - for useactions in self._iter_use_action_sets('same', 'other'): - yield useactions - for closeactions in self._iter_close_action_sets('same', 'other'): - actions = useactions + closeactions - yield actions - for postactions in self._iter_post_close_action_sets(): - yield actions + postactions - for closeactions in self._iter_close_action_sets('other', 'extra'): - actions = useactions + closeactions - yield actions - for postactions in self._iter_post_close_action_sets(): - yield actions + postactions - for useactions in self._iter_use_action_sets('other', 'extra'): - yield useactions - for closeactions in self._iter_close_action_sets('same', 'other'): - actions = useactions + closeactions - yield actions - for postactions in self._iter_post_close_action_sets(): - yield actions + postactions - for closeactions in self._iter_close_action_sets('other', 'extra'): - actions = useactions + closeactions - yield actions - for postactions in self._iter_post_close_action_sets(): - yield actions + postactions - - def _iter_use_action_sets(self, interp1, interp2): - interps = (interp1, interp2) - - # only recv end used - yield [ - ChannelAction('use', 'recv', interp1), - ] - yield [ - ChannelAction('use', 'recv', interp2), - ] - yield [ - ChannelAction('use', 'recv', interp1), - ChannelAction('use', 'recv', interp2), - ] - - # never emptied - yield [ - ChannelAction('use', 'send', interp1), - ] - yield [ - ChannelAction('use', 'send', interp2), - ] - yield [ - ChannelAction('use', 'send', interp1), - ChannelAction('use', 'send', interp2), - ] - - # partially emptied - for interp1 in interps: - for interp2 in interps: - for interp3 in interps: - yield [ - ChannelAction('use', 'send', interp1), - ChannelAction('use', 'send', interp2), - ChannelAction('use', 'recv', interp3), - ] - - # fully emptied - for interp1 in interps: - for interp2 in interps: - for interp3 in interps: - for interp4 in interps: - yield [ - ChannelAction('use', 'send', interp1), - ChannelAction('use', 'send', interp2), - ChannelAction('use', 'recv', interp3), - ChannelAction('use', 'recv', interp4), - ] - - def _iter_close_action_sets(self, interp1, interp2): - ends = ('recv', 'send') - interps = (interp1, interp2) - for force in (True, False): - op = 'force-close' if force else 'close' - for interp in interps: - for end in ends: - yield [ - ChannelAction(op, end, interp), - ] - for recvop in ('close', 'force-close'): - for sendop in ('close', 'force-close'): - for recv in interps: - for send in interps: - yield [ - ChannelAction(recvop, 'recv', recv), - ChannelAction(sendop, 'send', send), - ] - - def _iter_post_close_action_sets(self): - for interp in ('same', 'extra', 'other'): - yield [ - ChannelAction('use', 'recv', interp), - ] - yield [ - ChannelAction('use', 'send', interp), - ] - - def run_actions(self, fix, actions): - for action in actions: - self.run_action(fix, action) - - def run_action(self, fix, action, *, hideclosed=True): - end = action.resolve_end(fix.end) - interp = action.resolve_interp(fix.interp, fix.other, fix.extra) - fix.prep_interpreter(interp) - if interp.name == 'main': - result = run_action( - fix.cid, - action.action, - end, - fix.state, - hideclosed=hideclosed, - ) - fix.record_action(action, result) - else: - _cid = interpreters.channel_create() - run_interp(interp.id, f""" - result = helpers.run_action( - {fix.cid}, - {repr(action.action)}, - {repr(end)}, - {repr(fix.state)}, - hideclosed={hideclosed}, - ) - interpreters.channel_send({_cid}, result.pending.to_bytes(1, 'little')) - interpreters.channel_send({_cid}, b'X' if result.closed else b'') - """) - result = ChannelState( - pending=int.from_bytes(interpreters.channel_recv(_cid), 'little'), - closed=bool(interpreters.channel_recv(_cid)), - ) - fix.record_action(action, result) - - def iter_fixtures(self): - # XXX threads? - interpreters = [ - ('main', 'interp', 'extra'), - ('interp', 'main', 'extra'), - ('interp1', 'interp2', 'extra'), - ('interp1', 'interp2', 'main'), - ] - for interp, other, extra in interpreters: - for creator in ('same', 'other', 'creator'): - for end in ('send', 'recv'): - yield ChannelCloseFixture(end, interp, other, extra, creator) - - def _close(self, fix, *, force): - op = 'force-close' if force else 'close' - close = ChannelAction(op, fix.end, 'same') - if not fix.expect_closed_error(): - self.run_action(fix, close, hideclosed=False) - else: - with self.assertRaises(interpreters.ChannelClosedError): - self.run_action(fix, close, hideclosed=False) - - def _assert_closed_in_interp(self, fix, interp=None): - if interp is None or interp.name == 'main': - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_recv(fix.cid) - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_send(fix.cid, b'spam') - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_close(fix.cid) - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_close(fix.cid, force=True) - else: - run_interp(interp.id, f""" - with helpers.expect_channel_closed(): - interpreters.channel_recv(cid) - """) - run_interp(interp.id, f""" - with helpers.expect_channel_closed(): - interpreters.channel_send(cid, b'spam') - """) - run_interp(interp.id, f""" - with helpers.expect_channel_closed(): - interpreters.channel_close(cid) - """) - run_interp(interp.id, f""" - with helpers.expect_channel_closed(): - interpreters.channel_close(cid, force=True) - """) - - def _assert_closed(self, fix): - self.assertTrue(fix.state.closed) - - for _ in range(fix.state.pending): - interpreters.channel_recv(fix.cid) - self._assert_closed_in_interp(fix) - - for interp in ('same', 'other'): - interp = fix.get_interpreter(interp) - if interp.name == 'main': - continue - self._assert_closed_in_interp(fix, interp) - - interp = fix.get_interpreter('fresh') - self._assert_closed_in_interp(fix, interp) - - def _iter_close_tests(self, verbose=False): - i = 0 - for actions in self.iter_action_sets(): - print() - for fix in self.iter_fixtures(): - i += 1 - if i > 1000: - return - if verbose: - if (i - 1) % 6 == 0: - print() - print(i, fix, '({} actions)'.format(len(actions))) - else: - if (i - 1) % 6 == 0: - print(' ', end='') - print('.', end=''); sys.stdout.flush() - yield i, fix, actions - if verbose: - print('---') - print() - - # This is useful for scanning through the possible tests. - def _skim_close_tests(self): - ChannelCloseFixture.QUICK = True - for i, fix, actions in self._iter_close_tests(): - pass - - def test_close(self): - for i, fix, actions in self._iter_close_tests(): - with self.subTest('{} {} {}'.format(i, fix, actions)): - fix.prep_interpreter(fix.interp) - self.run_actions(fix, actions) - - self._close(fix, force=False) - - self._assert_closed(fix) - # XXX Things slow down if we have too many interpreters. - fix.clean_up() - - def test_force_close(self): - for i, fix, actions in self._iter_close_tests(): - with self.subTest('{} {} {}'.format(i, fix, actions)): - fix.prep_interpreter(fix.interp) - self.run_actions(fix, actions) - - self._close(fix, force=True) - - self._assert_closed(fix) - # XXX Things slow down if we have too many interpreters. - fix.clean_up() - if __name__ == '__main__': unittest.main() From 4618343e07ea2610ee2c8003521b15aeaf530d27 Mon Sep 17 00:00:00 2001 From: Joannah Nanjekye Date: Tue, 10 Sep 2019 16:50:27 +0000 Subject: [PATCH 04/20] update __all__ --- Doc/library/interpreters.rst | 3 ++- Lib/interpreters.py | 9 ++++----- Lib/test/test_interpreters.py | 12 +----------- 3 files changed, 7 insertions(+), 17 deletions(-) diff --git a/Doc/library/interpreters.rst b/Doc/library/interpreters.rst index 28ab95c370ce66..c2f8d19e495878 100644 --- a/Doc/library/interpreters.rst +++ b/Doc/library/interpreters.rst @@ -31,7 +31,8 @@ This module defines the following functions: .. function:: destroy(id) - Destroy the interpreter whose ID is *id*. + Destroy the interpreter whose ID is *id*. Attempting to destroy the current + interpreter results in a `RuntimeError`. So does an unrecognized ID. .. function:: get_main() diff --git a/Lib/interpreters.py b/Lib/interpreters.py index 95537bba6cbdae..62ca1466faa686 100644 --- a/Lib/interpreters.py +++ b/Lib/interpreters.py @@ -1,8 +1,10 @@ """Sub-interpreters High Level Module.""" -import _interpreters +__all__ = ['create', 'list_all', 'get_current', 'get_main', + 'run_string', 'destroy'] + -__all__ = ['create', 'list_all', 'get_current'] +import _interpreters # Rename so that "from interpreters import *" is safe _list_all = _interpreters.list_all @@ -38,7 +40,6 @@ def get_main(): Return the ID of the main interpreter. """ - return _get_main() def destroy(id): @@ -46,7 +47,6 @@ def destroy(id): Destroy the identified interpreter. """ - return _interpreters.destroy(id) def run_string(id, script, shared): @@ -55,5 +55,4 @@ def run_string(id, script, shared): Execute the provided string in the identified interpreter. See PyRun_SimpleStrings. """ - return _run_string(id, script, shared) diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index 32964ec435f4c0..d938235989720f 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -1,19 +1,8 @@ import interpreters -import _interpreters #remove when all methods are implemented -from collections import namedtuple -import contextlib -import itertools import os -import pickle -import sys from textwrap import dedent -import threading -import time import unittest -from test import support -from test.support import script_helper - def _captured_script(script): r, w = os.pipe() indented = script.replace('\n', '\n ') @@ -71,6 +60,7 @@ def test_create(self): id = interpreters.create() self.assertIn(id, interpreters.list_all()) + class DestroyTests(TestBase): def test_destroy(self): From ec63183c3b20fd4cd33b64bf37f7102be02648f2 Mon Sep 17 00:00:00 2001 From: Joannah Nanjekye Date: Thu, 12 Sep 2019 17:01:57 +0000 Subject: [PATCH 05/20] Implementation of all methods and relevant documentation --- Doc/library/interpreters.rst | 153 +++++++++++++++++++++-- Lib/interpreters.py | 234 ++++++++++++++++++++++++++++++----- 2 files changed, 340 insertions(+), 47 deletions(-) diff --git a/Doc/library/interpreters.rst b/Doc/library/interpreters.rst index c2f8d19e495878..9df38111a65ab8 100644 --- a/Doc/library/interpreters.rst +++ b/Doc/library/interpreters.rst @@ -13,32 +13,159 @@ level :mod:`_interpreters` module. .. versionchanged:: 3.9 +Interpreter Objects +------------------- -This module defines the following functions: +The Interpreter object represents a single interpreter. +.. class:: Interpreter(id) + + The class implementing a subinterpreter object. + + .. method:: is_running() + + Return whether or not the identified interpreter is running. + + .. method:: destroy() + + Destroy the identified interpreter. Attempting to destroy the current + interpreter results in a RuntimeError. So does an unrecognized ID. + + .. method:: run(self, src_str, /, *, channels=None): + + Run the given source code in the interpreter. This blocks the current + thread until done. + +RecvChannel Objects +------------------- + +The RecvChannel object represents a recieving channel. + +.. class:: RecvChannel(id) + + This class represents the receiving end of a channel. + + .. method:: recv() + + Get the next object from the channel, and wait if none have been + sent. Associate the interpreter with the channel. + + .. method:: recv_nowait(default=None) + + Like ``recv()``, but return the default instead of waiting. + + .. method:: release() + + No longer associate the current interpreter with the channel + (on the sending end). + + .. method:: close(force=False) + + Close the channel in all interpreters. + + +SendChannel Objects +-------------------- + +The SendChannel object represents a sending channel. + +.. class:: SendChannel(id) + + This class represents the receiving end of a channel. + + .. method:: send(obj) + + Send the object (i.e. its data) to the receiving end of the channel + and wait.Associate the interpreter with the channel. + + .. method:: send_nowait(obj) + + Like ``send()``, but return False if not received. + + .. method:: send_buffer(obj) + + Send the object's buffer to the receiving end of the channel and wait. + Associate the interpreter with the channel. + + .. method:: send_buffer_nowait(obj) + + Like ``send_buffer()``, but return False if not received. + + .. method:: release() + + No longer associate the current interpreter with the channel + (on the sending end). + + .. method:: close(force=False) + + Close the channel in all interpreters. + + +This module defines the following global functions: + + +.. function:: is_shareable(obj) + + Return `True` if the object's data can be shared between interpreters. + +.. function:: create_channel() + + Create a new channel for passing data between interpreters. + +.. function:: list_all_channels() + + Return all open channels. .. function:: create() - Create a new interpreter and return a unique generated ID. + Initialize a new (idle) Python interpreter. + +.. function:: get_current() + + Get the currently running interpreter. .. function:: list_all() - Return a list containing the ID of every existing interpreter. + Get all existing interpreters. -.. function:: get_current() +This module also defines the following exceptions. + +.. exception:: RunFailedError + + This exception, a subclass of :exc:`RuntimeError`, is raised when the + ``Interpreter.run()`` results in an uncaught exception. + +.. exception:: ChannelError + + This exception, a subclass of :exc:`Exception`, and is the base class for + channel-related exceptions. + +.. exception:: ChannelNotFoundError + + This exception, a subclass of :exc:`ChannelError`, is raised when the + the identified channel was not found. + +.. exception:: ChannelEmptyError + + This exception, a subclass of :exc:`ChannelError`, is raised when + the channel is unexpectedly empty. + +.. exception:: ChannelNotEmptyError - Return the ID of the currently running interpreter. + This exception, a subclass of :exc:`ChannelError`, is raised when + the channel is unexpectedly not empty. -.. function:: destroy(id) +.. exception:: NotReceivedError - Destroy the interpreter whose ID is *id*. Attempting to destroy the current - interpreter results in a `RuntimeError`. So does an unrecognized ID. + This exception, a subclass of :exc:`ChannelError`, is raised when + nothing was waiting to receive a sent object. -.. function:: get_main() +.. exception:: ChannelClosedError - Return the ID of the main interpreter. + This exception, a subclass of :exc:`ChannelError`, is raised when + the channel is closed. -.. function:: run_string() +.. exception:: ChannelReleasedError - Execute the provided string in the identified interpreter. - See `PyRun_SimpleStrings`. + This exception, a subclass of :exc:`ChannelClosedError`, is raised when + the channel is released (but not yet closed). diff --git a/Lib/interpreters.py b/Lib/interpreters.py index 62ca1466faa686..8bed804bb25b68 100644 --- a/Lib/interpreters.py +++ b/Lib/interpreters.py @@ -1,58 +1,224 @@ -"""Sub-interpreters High Level Module.""" +"""Subinterpreters High Level Module.""" -__all__ = ['create', 'list_all', 'get_current', 'get_main', - 'run_string', 'destroy'] +import _interpreters +import logger +__all__ = _all__ = ['create', 'list_all', 'get_current', 'get_main', + 'run_string', 'destroy'] -import _interpreters -# Rename so that "from interpreters import *" is safe -_list_all = _interpreters.list_all -_get_current = _interpreters.get_current -_get_main = _interpreters.get_main -_run_string = _interpreters.run_string +class Interpreter: + + def __init__(self, id): + self.id = id + + def is_running(self): + """is_running() -> bool + + Return whether or not the identified interpreter is running. + """ + return _interpreters.is_running(self.id) + + def destroy(self): + """destroy() + + Destroy the identified interpreter. + + Attempting to destroy the current + interpreter results in a RuntimeError. So does an unrecognized ID + """ + return _interpreters.destroy(self.id) + + def run(self, src_str, /, *, channels=None): + """run(src_str, /, *, channels=None) + + Run the given source code in the interpreter. + This blocks the current thread until done. + """ + try: + _interpreters.run_string(self.id, src_str) + except RunFailedError as err: + logger.error(err) + raise + +def wait(self, timeout): + #The implementation for wait + # will be non trivial to be useful + import time + time.sleep(timeout) + +def associate_interp_to_channel(id, cid): + pass + +class RecvChannel: + + def __init__(self, id): + self.id = id + self.interpreters = _interpreters.list_all() + + def recv(self, timeout=2): + """ channel_recv() -> obj + + + Get the next object from the channel, + and wait if none have been sent. + Associate the interpreter with the channel. + """ + obj = _interpreters.channel_recv(self.id) + if obj == None: + wait(timeout) + obj = obj = _interpreters.channel_recv(self.id) + + associate_interp_to_channel(interpId, Cid) + + return obj + + def recv_nowait(self, default=None): + """recv_nowait(default=None) -> object + + Like recv(), but return the default instead of waiting. + """ + return _interpreters.channel_recv(self.id) + + def release(self): + """ release() + + No longer associate the current interpreterwith the channel + (on the sending end). + """ + pass + + def close(self, force=False): + """close(force=False) + + Close the channel in all interpreters.. + """ + return _interpreters.channel_close(self.id, force) + + +class SendChannel: + + def __init__(self, id): + self.id = id + self.interpreters = _interpreters.list_all() + + def send(self, obj, timeout=2): + """ send(obj) + + Send the object (i.e. its data) to the receiving end of the channel + and wait. Associate the interpreter with the channel. + """ + obj = _interpreters.channel_send(self.id, obj) + wait(timeout) + associate_interp_to_channel(interpId, Cid) + + def send_nowait(self, obj): + """ send_nowait(obj) + + Like send(), but return False if not received. + """ + try: + obj = _interpreters.channel_send(self.id, obj) + except: + return False + + return obj + + def release(self): + """ release() + + No longer associate the current interpreterwith the channel + (on the sending end). + """ + pass + + def close(self, force=False): + """ close(force=False) + + No longer associate the current interpreterwith the channel + (on the sending end). + """ + return _interpreters.channel_close(self.id, force) + + +class ChannelError(Exception): + pass + + +class ChannelNotFoundError(ChannelError): + pass + + +class ChannelEmptyError(ChannelError): + pass + + +class ChannelNotEmptyError(ChannelError): + pass + + +class NotReceivedError(ChannelError): + pass + + +class ChannelClosedError(ChannelError): + pass + + +class ChannelReleasedError(ChannelClosedError): + pass + + +class RunFailedError(RuntimeError): + pass + # Global API functions -def create(): - """ create() -> Interpreter +def is_shareable(obj): + """ is_shareable(obj) -> Bool - Create a new interpreter and return a unique generated ID. + Return `True` if the object's data can be shared between + interpreters. """ - return _interpreters.create() + return _interpreters.is_shareable(obj) -def list_all(): - """list_all() -> [Interpreter] +def create_channel(): + """ create_channel() -> (RecvChannel, SendChannel) - Return a list containing the ID of every existing interpreter. + Create a new channel for passing data between interpreters. """ - return _list_all() -def get_current(): - """get_current() -> Interpreter + cid = _interpreters.channel_create() + return (RecvChannel(cid), SendChannel(cid)) - Return the ID of the currently running interpreter. +def list_all_channels(): + """ list_all_channels() -> [(RecvChannel, SendChannel)] + + Return all open channels. """ - return _get_current() + cid = _interpreters.channel_list_all() + return (RecvChannel(cid), SendChannel(cid)) -def get_main(): - """get_main() -> ID +def create(): + """ create() -> Interpreter - Return the ID of the main interpreter. + Initialize a new (idle) Python interpreter. """ - return _get_main() + id = _interpreters.create() + return Interpreter(id) -def destroy(id): - """destroy(id) -> None +def list_all(): + """ list_all() -> [Interpreter] - Destroy the identified interpreter. + Get all existing interpreters. """ - return _interpreters.destroy(id) + return [Interpreter(id) for id in _interpreters.list_all()] -def run_string(id, script, shared): - """run_string(id, script, shared) -> None +def get_current(): + """ get_current() -> Interpreter - Execute the provided string in the identified interpreter. - See PyRun_SimpleStrings. + Get the currently running interpreter. """ - return _run_string(id, script, shared) + id = _interpreters.get_current() + return Interpreter(id) From 80f680867593fd7562c642ed19881147006cdbba Mon Sep 17 00:00:00 2001 From: Joannah Nanjekye <33177550+nanjekyejoannah@users.noreply.github.com> Date: Fri, 13 Sep 2019 05:30:29 -0300 Subject: [PATCH 06/20] Update Lib/test/test_interpreters.py Co-Authored-By: Eric Snow --- Lib/test/test_interpreters.py | 38 +++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index d938235989720f..5f4dda331f0a5c 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -24,6 +24,44 @@ def clean_up_interpreters(): pass # already destroyed +class LowLevelStub: + # set these as appropriate in tests + errors = () + return_create = () + ... + def __init__(self): + self._calls = [] + def _add_call(self, name, args=(), kwargs=None): + self.calls.append( + (name, args, kwargs or {})) + def _maybe_error(self): + if not self.errors: + return + err = self.errors.pop(0) + if err is not None: + raise err + def check_calls(self, test, expected): + test.assertEqual(self._calls, expected) + for returns in [self.errors, self.return_create, ...]: + test.assertEqual(tuple(returns), ()) # make sure all were used + + # the stubbed methods + def create(self): + self._add_call('create') + self._maybe_error() + return self.return_create.pop(0) + def list_all(self): + ... + def get_current(self): + ... + def get_main(self): + ... + def destroy(self, id): + ... + def run_string(self, id, text, ...): + ... + + class TestBase(unittest.TestCase): def tearDown(self): From 724e618fc3e1e95c8e839c83a48ba1b2bc36e074 Mon Sep 17 00:00:00 2001 From: Joannah Nanjekye <33177550+nanjekyejoannah@users.noreply.github.com> Date: Fri, 13 Sep 2019 05:30:42 -0300 Subject: [PATCH 07/20] Update Lib/test/test_interpreters.py Co-Authored-By: Eric Snow --- Lib/test/test_interpreters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index 5f4dda331f0a5c..93b06bf13745d3 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -96,7 +96,7 @@ class CreateTests(TestBase): def test_create(self): id = interpreters.create() - self.assertIn(id, interpreters.list_all()) + self.assertIn(interp, interpreters.list_all()) class DestroyTests(TestBase): From abd0011e000b2b7f1dbfe8ad3abf720d24a9c13e Mon Sep 17 00:00:00 2001 From: Joannah Nanjekye <33177550+nanjekyejoannah@users.noreply.github.com> Date: Fri, 13 Sep 2019 05:30:57 -0300 Subject: [PATCH 08/20] Update Lib/test/test_interpreters.py Co-Authored-By: Eric Snow --- Lib/test/test_interpreters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index 93b06bf13745d3..6044977d608caa 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -95,7 +95,7 @@ def test_get_main(self): class CreateTests(TestBase): def test_create(self): - id = interpreters.create() + interp = interpreters.create() self.assertIn(interp, interpreters.list_all()) From d502415deafb2fdc42e6234e2fa237f223444eb8 Mon Sep 17 00:00:00 2001 From: Joannah Nanjekye <33177550+nanjekyejoannah@users.noreply.github.com> Date: Fri, 13 Sep 2019 05:31:13 -0300 Subject: [PATCH 09/20] Update Lib/test/test_interpreters.py Co-Authored-By: Eric Snow --- Lib/test/test_interpreters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index 6044977d608caa..2ca2d402d234d3 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -87,7 +87,7 @@ def test_get_current(self): class GetMainTests(TestBase): def test_get_main(self): - [expected] = interpreters.list_all() + expected, * = interpreters.list_all() main = interpreters.get_main() self.assertEqual(main, expected) From a751e153b2f81d5310d3d52a0e616dd708055f28 Mon Sep 17 00:00:00 2001 From: Joannah Nanjekye <33177550+nanjekyejoannah@users.noreply.github.com> Date: Fri, 13 Sep 2019 05:31:38 -0300 Subject: [PATCH 10/20] Update Doc/library/interpreters.rst Co-Authored-By: Eric Snow --- Doc/library/interpreters.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/Doc/library/interpreters.rst b/Doc/library/interpreters.rst index 9df38111a65ab8..e4938a69abef72 100644 --- a/Doc/library/interpreters.rst +++ b/Doc/library/interpreters.rst @@ -17,7 +17,6 @@ Interpreter Objects ------------------- The Interpreter object represents a single interpreter. - .. class:: Interpreter(id) The class implementing a subinterpreter object. From 56083c2079e73e639a3019e7b25b68fd5d819cb8 Mon Sep 17 00:00:00 2001 From: Joannah Nanjekye <33177550+nanjekyejoannah@users.noreply.github.com> Date: Fri, 13 Sep 2019 05:31:50 -0300 Subject: [PATCH 11/20] Update Doc/library/interpreters.rst Co-Authored-By: Eric Snow --- Doc/library/interpreters.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/interpreters.rst b/Doc/library/interpreters.rst index e4938a69abef72..ed935f16f0a315 100644 --- a/Doc/library/interpreters.rst +++ b/Doc/library/interpreters.rst @@ -11,7 +11,7 @@ This module constructs higher-level interpreters interfaces on top of the lower level :mod:`_interpreters` module. -.. versionchanged:: 3.9 +.. versionchanged:: added in 3.9 Interpreter Objects ------------------- From 14e97fcf29e26d347937cad1c594d0fe9f162202 Mon Sep 17 00:00:00 2001 From: Joannah Nanjekye <33177550+nanjekyejoannah@users.noreply.github.com> Date: Fri, 13 Sep 2019 05:32:07 -0300 Subject: [PATCH 12/20] Update Doc/library/interpreters.rst Co-Authored-By: Eric Snow --- Doc/library/interpreters.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/library/interpreters.rst b/Doc/library/interpreters.rst index ed935f16f0a315..8c4d44a43f2b42 100644 --- a/Doc/library/interpreters.rst +++ b/Doc/library/interpreters.rst @@ -8,7 +8,8 @@ -------------- -This module constructs higher-level interpreters interfaces on top of the lower +This module provides tools for working with sub-interpreters, such as creating them, +running code in them, or sending data between them. It is a wrapper around the low- level :mod:`_interpreters` module. .. versionchanged:: added in 3.9 From 9ab54eb47764f1508b23936375cc890f93c9c074 Mon Sep 17 00:00:00 2001 From: Joannah Nanjekye Date: Fri, 13 Sep 2019 10:52:53 +0000 Subject: [PATCH 13/20] Add tests --- Doc/library/interpreters.rst | 14 +-- Lib/interpreters.py | 32 +++++-- Lib/test/test_interpreters.py | 176 ++++++++++++++++++++-------------- 3 files changed, 138 insertions(+), 84 deletions(-) diff --git a/Doc/library/interpreters.rst b/Doc/library/interpreters.rst index 8c4d44a43f2b42..5d15484b6b4d78 100644 --- a/Doc/library/interpreters.rst +++ b/Doc/library/interpreters.rst @@ -1,8 +1,8 @@ -:mod:`interpreters` --- High-level Sub-interpreters Module +:mod:`interpreters` --- High-level Subinterpreters Module ========================================================== .. module:: interpreters - :synopsis: High-level Sub-Interpreters Module. + :synopsis: High-level SubInterpreters Module. **Source code:** :source:`Lib/interpreters.py` @@ -56,8 +56,9 @@ The RecvChannel object represents a recieving channel. .. method:: release() - No longer associate the current interpreter with the channel - (on the sending end). + Close the channel for the current interpreter. 'send' and 'recv' (bool) may + be used to indicate the ends to close. By default both ends are closed. + Closing an already closed end is a noop. .. method:: close(force=False) @@ -93,8 +94,9 @@ The SendChannel object represents a sending channel. .. method:: release() - No longer associate the current interpreter with the channel - (on the sending end). + Close the channel for the current interpreter. 'send' and 'recv' (bool) may + be used to indicate the ends to close. By default both ends are closed. + Closing an already closed end is a noop. .. method:: close(force=False) diff --git a/Lib/interpreters.py b/Lib/interpreters.py index 8bed804bb25b68..9fa3877540408f 100644 --- a/Lib/interpreters.py +++ b/Lib/interpreters.py @@ -3,8 +3,9 @@ import _interpreters import logger -__all__ = _all__ = ['create', 'list_all', 'get_current', 'get_main', - 'run_string', 'destroy'] +__all__ = ['Interpreter', 'SendChannel', 'RecvChannel', 'is_shareable', + 'create_channel', 'list_all_channels', 'list_all', 'get_current', + 'create'] class Interpreter: @@ -59,7 +60,6 @@ def __init__(self, id): def recv(self, timeout=2): """ channel_recv() -> obj - Get the next object from the channel, and wait if none have been sent. Associate the interpreter with the channel. @@ -69,6 +69,7 @@ def recv(self, timeout=2): wait(timeout) obj = obj = _interpreters.channel_recv(self.id) + # Pending: See issue 52 on multi-core python project associate_interp_to_channel(interpId, Cid) return obj @@ -80,13 +81,28 @@ def recv_nowait(self, default=None): """ return _interpreters.channel_recv(self.id) + def send_buffer(self, obj): + """ send_buffer(obj) + + Send the object's buffer to the receiving end of the channel + and wait. Associate the interpreter with the channel. + """ + pass + + def send_buffer_nowait(self, obj): + """ send_buffer_nowait(obj) + + Like send_buffer(), but return False if not received. + """ + pass + def release(self): """ release() No longer associate the current interpreterwith the channel (on the sending end). """ - pass + return _interpreters.(self.id) def close(self, force=False): """close(force=False) @@ -102,14 +118,14 @@ def __init__(self, id): self.id = id self.interpreters = _interpreters.list_all() - def send(self, obj, timeout=2): + def send(self, obj): """ send(obj) Send the object (i.e. its data) to the receiving end of the channel and wait. Associate the interpreter with the channel. """ obj = _interpreters.channel_send(self.id, obj) - wait(timeout) + wait(2) associate_interp_to_channel(interpId, Cid) def send_nowait(self, obj): @@ -127,10 +143,10 @@ def send_nowait(self, obj): def release(self): """ release() - No longer associate the current interpreterwith the channel + No longer associate the current interpreter with the channel (on the sending end). """ - pass + return _interpreters.channel_release(self.id) def close(self, force=False): """ close(force=False) diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index 2ca2d402d234d3..823af4c7527453 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -23,50 +23,86 @@ def clean_up_interpreters(): except RuntimeError: pass # already destroyed - -class LowLevelStub: - # set these as appropriate in tests - errors = () - return_create = () - ... - def __init__(self): - self._calls = [] - def _add_call(self, name, args=(), kwargs=None): - self.calls.append( - (name, args, kwargs or {})) - def _maybe_error(self): - if not self.errors: - return - err = self.errors.pop(0) - if err is not None: - raise err - def check_calls(self, test, expected): - test.assertEqual(self._calls, expected) - for returns in [self.errors, self.return_create, ...]: - test.assertEqual(tuple(returns), ()) # make sure all were used - - # the stubbed methods - def create(self): - self._add_call('create') - self._maybe_error() - return self.return_create.pop(0) - def list_all(self): - ... - def get_current(self): - ... - def get_main(self): - ... - def destroy(self, id): - ... - def run_string(self, id, text, ...): - ... - +def _run_output(interp, request, shared=None): + script, rpipe = _captured_script(request) + with rpipe: + interpreters.run_string(interp, script, shared) + return rpipe.read() class TestBase(unittest.TestCase): def tearDown(self): clean_up_interpreters() +class TestInterpreter(TestBase): + + def test_is_running(self): + interp = interpreters.Interpreter(1) + self.assertEqual(True, interp.is_running()) + + def test_destroy(self): + interp = interpreters.Interpreter(1) + interp2 = interpreters.Interpreter(2) + interp.destroy() + ids = interpreters.list_all() + self.assertEqual(ids, [interp2.id]) + + def test_run(self): + interp = interpreters.Interpreter(1) + interp.run(dedent(f""" + import _interpreters + _interpreters.channel_send({cid}, b'spam') + """)) + out = _run_output(id2, dedent(f""" + import _interpreters + obj = _interpreters.channel_recv({cid}) + _interpreters.channel_release({cid}) + print(repr(obj)) + """)) + self.assertEqual(out.strip(), "b'spam'") + +class RecvChannelTest(TestBase): + + def test_release(self): + import _interpreters as interpreters + + chanl = interpreters.RecvChannel(1) + interpreters.channel_send(chanl.id, b'spam') + interpreters.channel_recv(cid) + chanl.release(cid) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'eggs') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_close(self): + chanl = interpreters.RecvChannel(1) + chanl.close() + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(fix.cid) + +class SendChannelTest(TestBase): + + def test_release(self): + import _interpreters as interpreters + + chanl = interpreters.SendChannel(1) + interpreters.channel_send(chanl.id, b'spam') + interpreters.channel_recv(cid) + chanl.release(cid) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'eggs') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_close(self): + chanl = interpreters.RecvChannel(1) + chanl.close() + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(fix.cid) + class ListAllTests(TestBase): @@ -75,7 +111,6 @@ def test_initial(self): ids = interpreters.list_all() self.assertEqual(ids, [main]) - class GetCurrentTests(TestBase): def test_get_current(self): @@ -83,43 +118,44 @@ def test_get_current(self): cur = interpreters.get_current() self.assertEqual(cur, main) - -class GetMainTests(TestBase): - - def test_get_main(self): - expected, * = interpreters.list_all() - main = interpreters.get_main() - self.assertEqual(main, expected) - - class CreateTests(TestBase): def test_create(self): interp = interpreters.create() self.assertIn(interp, interpreters.list_all()) - -class DestroyTests(TestBase): - - def test_destroy(self): - id1 = interpreters.create() - id2 = interpreters.create() - id3 = interpreters.create() - self.assertIn(id2, interpreters.list_all()) - interpreters.destroy(id2) - self.assertNotIn(id2, interpreters.list_all()) - - -class RunStringTests(TestBase): - - def test_run_string(self): - script, file = _captured_script('print("it worked!", end="")') - id = interpreters.create() - with file: - interpreters.run_string(id, script, None) - out = file.read() - - self.assertEqual(out, 'it worked!') +class ExceptionTests(TestBase): + + def test_does_not_exist(self): + cid = interpreters.create_channel() + recvCha = interpreters.RecvChannel(cid) + with self.assertRaises(interpreters.ChannelNotFoundError): + interpreters._channel_id(int(cid) + 1) + + def test_recv_empty(self): + cid = interpreters.create_channel() + recvCha = interpreters.RecvChannel(cid) + with self.assertRaises(interpreters.ChannelEmptyError): + recvCha.recv() + + def test_channel_not_empty(self): + cid = interpreters.create_channel() + sendCha = interpreters.SendChannel(cid) + sendCha.send(b'spam') + sendCha.send(b'ham') + + with self.assertRaises(interpreters.ChannelNotEmptyError): + sendCha.close() + + def test_channel_closed(self): + cid = interpreters.channel_create() + sendCha = interpreters.SendChannel(cid) + sendCha.send(b'spam') + sendCha.send(b'ham') + sendCha.close() + + with self.assertRaises(interpreters.ChannelClosedError): + sendCha.send(b'spam') if __name__ == '__main__': From b7dce732b3b6f38f3fa7764700020180e867b5a5 Mon Sep 17 00:00:00 2001 From: Joannah Nanjekye Date: Tue, 3 Mar 2020 21:37:47 +0000 Subject: [PATCH 14/20] Fix tests --- Lib/interpreters.py | 107 ++++++++++----------- Lib/test/test__xxsubinterpreters.py | 98 +++++++++++++++++++ Lib/test/test_interpreters.py | 144 ++++++---------------------- Modules/_xxsubinterpretersmodule.c | 111 +++++++++++++++++++++ 4 files changed, 293 insertions(+), 167 deletions(-) diff --git a/Lib/interpreters.py b/Lib/interpreters.py index 9fa3877540408f..97779a89390c0e 100644 --- a/Lib/interpreters.py +++ b/Lib/interpreters.py @@ -1,13 +1,37 @@ """Subinterpreters High Level Module.""" import _interpreters -import logger +import logging __all__ = ['Interpreter', 'SendChannel', 'RecvChannel', 'is_shareable', 'create_channel', 'list_all_channels', 'list_all', 'get_current', 'create'] +def create(): + """ create() -> Interpreter + + Initialize a new (idle) Python interpreter. + """ + id = _interpreters.create() + return Interpreter(id) + +def list_all(): + """ list_all() -> [Interpreter] + + Get all existing interpreters. + """ + return [Interpreter(id) for id in _interpreters.list_all()] + +def get_current(): + """ get_current() -> Interpreter + + Get the currently running interpreter. + """ + id = _interpreters.get_current() + return Interpreter(id) + + class Interpreter: def __init__(self, id): @@ -42,6 +66,32 @@ def run(self, src_str, /, *, channels=None): logger.error(err) raise + +def is_shareable(obj): + """ is_shareable(obj) -> Bool + + Return `True` if the object's data can be shared between + interpreters. + """ + return _interpreters.is_shareable(obj) + +def create_channel(): + """ create_channel() -> (RecvChannel, SendChannel) + + Create a new channel for passing data between interpreters. + """ + + cid = _interpreters.channel_create() + return (RecvChannel(cid), SendChannel(cid)) + +def list_all_channels(): + """ list_all_channels() -> [(RecvChannel, SendChannel)] + + Return all open channels. + """ + cid = _interpreters.channel_list_all() + return (RecvChannel(cid), SendChannel(cid)) + def wait(self, timeout): #The implementation for wait # will be non trivial to be useful @@ -55,7 +105,7 @@ class RecvChannel: def __init__(self, id): self.id = id - self.interpreters = _interpreters.list_all() + self.interpreters = _interpreters.channel_list_interpreters(cid, send=False) def recv(self, timeout=2): """ channel_recv() -> obj @@ -102,7 +152,7 @@ def release(self): No longer associate the current interpreterwith the channel (on the sending end). """ - return _interpreters.(self.id) + return _interpreters(self.id) def close(self, force=False): """close(force=False) @@ -187,54 +237,3 @@ class ChannelReleasedError(ChannelClosedError): class RunFailedError(RuntimeError): pass - - -# Global API functions - -def is_shareable(obj): - """ is_shareable(obj) -> Bool - - Return `True` if the object's data can be shared between - interpreters. - """ - return _interpreters.is_shareable(obj) - -def create_channel(): - """ create_channel() -> (RecvChannel, SendChannel) - - Create a new channel for passing data between interpreters. - """ - - cid = _interpreters.channel_create() - return (RecvChannel(cid), SendChannel(cid)) - -def list_all_channels(): - """ list_all_channels() -> [(RecvChannel, SendChannel)] - - Return all open channels. - """ - cid = _interpreters.channel_list_all() - return (RecvChannel(cid), SendChannel(cid)) - -def create(): - """ create() -> Interpreter - - Initialize a new (idle) Python interpreter. - """ - id = _interpreters.create() - return Interpreter(id) - -def list_all(): - """ list_all() -> [Interpreter] - - Get all existing interpreters. - """ - return [Interpreter(id) for id in _interpreters.list_all()] - -def get_current(): - """ get_current() -> Interpreter - - Get the currently running interpreter. - """ - id = _interpreters.get_current() - return Interpreter(id) diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index 20e6b0419ff411..857ba48e38258a 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -1207,6 +1207,87 @@ def test_ids_global(self): self.assertEqual(cid2, int(cid1) + 1) + def test_channel_list_interpreters_none(self): + """Test listing interpreters for a channel with no associations.""" + # Test for channel with no associated interpreters. + cid = interpreters.channel_create() + send_interps = interpreters.channel_list_interpreters(cid, send=True) + recv_interps = interpreters.channel_list_interpreters(cid, send=False) + self.assertEqual(send_interps, []) + self.assertEqual(recv_interps, []) + + def test_channel_list_interpreters_basic(self): + """Test basic listing channel interpreters.""" + interp0 = interpreters.get_main() + cid = interpreters.channel_create() + interpreters.channel_send(cid, "send") + # Test for a channel that has one end associated to an interpreter. + send_interps = interpreters.channel_list_interpreters(cid, send=True) + recv_interps = interpreters.channel_list_interpreters(cid, send=False) + self.assertEqual(send_interps, [interp0]) + self.assertEqual(recv_interps, []) + + interp1 = interpreters.create() + _run_output(interp1, dedent(f""" + import _interpreters + obj = _interpreters.channel_recv({cid}) + """)) + # Test for channel that has boths ends associated to an interpreter. + send_interps = interpreters.channel_list_interpreters(cid, send=True) + recv_interps = interpreters.channel_list_interpreters(cid, send=False) + self.assertEqual(send_interps, [interp0]) + self.assertEqual(recv_interps, [interp1]) + + def test_channel_list_interpreters_multiple(self): + """Test listing interpreters for a channel with many associations.""" + interp0 = interpreters.get_main() + interp1 = interpreters.create() + interp2 = interpreters.create() + interp3 = interpreters.create() + cid = interpreters.channel_create() + + interpreters.channel_send(cid, "send") + _run_output(interp1, dedent(f""" + import _interpreters + obj = _interpreters.channel_send({cid}, "send") + """)) + _run_output(interp2, dedent(f""" + import _interpreters + obj = _interpreters.channel_recv({cid}) + """)) + _run_output(interp3, dedent(f""" + import _interpreters + obj = _interpreters.channel_recv({cid}) + """)) + send_interps = interpreters.channel_list_interpreters(cid, send=True) + recv_interps = interpreters.channel_list_interpreters(cid, send=False) + self.assertEqual(set(send_interps), {interp0, interp1}) + self.assertEqual(set(recv_interps), {interp2, interp3}) + + @unittest.skip("Failing due to handling of destroyed interpreters") + def test_channel_list_interpreters_destroyed(self): + """Test listing channel interpreters with a destroyed interpreter.""" + interp0 = interpreters.get_main() + interp1 = interpreters.create() + cid = interpreters.channel_create() + interpreters.channel_send(cid, "send") + _run_output(interp1, dedent(f""" + import _interpreters + obj = _interpreters.channel_recv({cid}) + """)) + # Should be one interpreter associated with each end. + send_interps = interpreters.channel_list_interpreters(cid, send=True) + recv_interps = interpreters.channel_list_interpreters(cid, send=False) + self.assertEqual(send_interps, [interp0]) + self.assertEqual(recv_interps, [interp1]) + + interpreters.destroy(interp1) + # Destroyed interpreter should not be listed. + send_interps = interpreters.channel_list_interpreters(cid, send=True) + recv_interps = interpreters.channel_list_interpreters(cid, send=False) + self.assertEqual(send_interps, [interp0]) + self.assertEqual(recv_interps, []) + #################### def test_send_recv_main(self): @@ -1519,6 +1600,23 @@ def test_close_used_multiple_times_by_single_user(self): with self.assertRaises(interpreters.ChannelClosedError): interpreters.channel_recv(cid) + def test_channel_list_interpreters_invalid_channel(self): + cid = interpreters.channel_create() + # Test for invalid channel ID. + with self.assertRaises(interpreters.ChannelNotFoundError): + interpreters.channel_list_interpreters(1000, send=True) + + interpreters.channel_close(cid) + # Test for a channel that has been closed. + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_list_interpreters(cid, send=True) + + def test_channel_list_interpreters_invalid_args(self): + # Tests for invalid arguments passed to the API. + cid = interpreters.channel_create() + with self.assertRaises(TypeError): + interpreters.channel_list_interpreters(cid) + class ChannelReleaseTests(TestBase): diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index 823af4c7527453..59c4ec0e8bf4a1 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -1,4 +1,5 @@ import interpreters +import _interpreters import os from textwrap import dedent import unittest @@ -15,11 +16,11 @@ def _captured_script(script): return wrapped, open(r) def clean_up_interpreters(): - for id in interpreters.list_all(): + for id in _interpreters.list_all(): if id == 0: # main continue try: - interpreters.destroy(id) + _interpreters.destroy(id) except RuntimeError: pass # already destroyed @@ -34,129 +35,46 @@ class TestBase(unittest.TestCase): def tearDown(self): clean_up_interpreters() -class TestInterpreter(TestBase): - - def test_is_running(self): - interp = interpreters.Interpreter(1) - self.assertEqual(True, interp.is_running()) +class CreateTests(TestBase): - def test_destroy(self): - interp = interpreters.Interpreter(1) - interp2 = interpreters.Interpreter(2) - interp.destroy() - ids = interpreters.list_all() - self.assertEqual(ids, [interp2.id]) + def test_create(self): + interp = interpreters.create() + lst = interpreters.list_all() + self.assertEqual(interp.id, lst[1].id) - def test_run(self): - interp = interpreters.Interpreter(1) - interp.run(dedent(f""" - import _interpreters - _interpreters.channel_send({cid}, b'spam') - """)) - out = _run_output(id2, dedent(f""" - import _interpreters - obj = _interpreters.channel_recv({cid}) - _interpreters.channel_release({cid}) - print(repr(obj)) - """)) - self.assertEqual(out.strip(), "b'spam'") - -class RecvChannelTest(TestBase): - - def test_release(self): - import _interpreters as interpreters - - chanl = interpreters.RecvChannel(1) - interpreters.channel_send(chanl.id, b'spam') - interpreters.channel_recv(cid) - chanl.release(cid) - - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_send(cid, b'eggs') - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_recv(cid) - - def test_close(self): - chanl = interpreters.RecvChannel(1) - chanl.close() - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_recv(fix.cid) - -class SendChannelTest(TestBase): - - def test_release(self): - import _interpreters as interpreters - - chanl = interpreters.SendChannel(1) - interpreters.channel_send(chanl.id, b'spam') - interpreters.channel_recv(cid) - chanl.release(cid) - - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_send(cid, b'eggs') - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_recv(cid) - - def test_close(self): - chanl = interpreters.RecvChannel(1) - chanl.close() - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_recv(fix.cid) +class GetCurrentTests(TestBase): + def test_get_current(self): + main_interp_id = _interpreters.get_main() + cur_interp_id = interpreters.get_current().id + self.assertEqual(cur_interp_id, main_interp_id) class ListAllTests(TestBase): def test_initial(self): - main = interpreters.get_main() - ids = interpreters.list_all() - self.assertEqual(ids, [main]) + interps = interpreters.list_all() + self.assertEqual(1, len(interps)) -class GetCurrentTests(TestBase): +class TestInterpreter(TestBase): - def test_get_current(self): - main = interpreters.get_main() - cur = interpreters.get_current() - self.assertEqual(cur, main) + def test_id_fields(self): + interp = interpreters.Interpreter(1) + self.assertEqual(1, interp.id) -class CreateTests(TestBase): + def test_is_running(self): + interp_de = interpreters.create() + self.assertEqual(False, interp_de.is_running()) - def test_create(self): + def test_destroy(self): interp = interpreters.create() - self.assertIn(interp, interpreters.list_all()) - -class ExceptionTests(TestBase): - - def test_does_not_exist(self): - cid = interpreters.create_channel() - recvCha = interpreters.RecvChannel(cid) - with self.assertRaises(interpreters.ChannelNotFoundError): - interpreters._channel_id(int(cid) + 1) - - def test_recv_empty(self): - cid = interpreters.create_channel() - recvCha = interpreters.RecvChannel(cid) - with self.assertRaises(interpreters.ChannelEmptyError): - recvCha.recv() - - def test_channel_not_empty(self): - cid = interpreters.create_channel() - sendCha = interpreters.SendChannel(cid) - sendCha.send(b'spam') - sendCha.send(b'ham') - - with self.assertRaises(interpreters.ChannelNotEmptyError): - sendCha.close() - - def test_channel_closed(self): - cid = interpreters.channel_create() - sendCha = interpreters.SendChannel(cid) - sendCha.send(b'spam') - sendCha.send(b'ham') - sendCha.close() + interp2 = interpreters.create() + interp.destroy() + interps = interpreters.list_all() + self.assertEqual(2, len(interps)) - with self.assertRaises(interpreters.ChannelClosedError): - sendCha.send(b'spam') + def test_run(self): + interp = interpreters.create() + interp.run("3") + self.assertEqual(False, interp.is_running()) -if __name__ == '__main__': - unittest.main() diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 552d96a977f2a9..ff3dfb713bc50c 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -627,6 +627,27 @@ _channelends_associate(_channelends *ends, int64_t interp, int send) return 0; } +static int64_t * +_channelends_list_interpreters(_channelends *ends, int64_t *count, int send) +{ + int64_t numopen = send ? ends->numsendopen : ends->numrecvopen; + + int64_t *ids = PyMem_NEW(int64_t, (Py_ssize_t)numopen); + if (ids == NULL) { + PyErr_NoMemory(); + return NULL; + } + + _channelend *ref = send ? ends->send : ends->recv; + for (int64_t i=0; ref != NULL; ref = ref->next, i++) { + ids[i] = ref->interp; + } + + *count = numopen; + + return ids; +} + static int _channelends_is_open(_channelends *ends) { @@ -1405,6 +1426,35 @@ typedef struct channelid { _channels *channels; } channelid; +static int +channel_id_converter(PyObject *arg, void *ptr) +{ + int64_t cid; + if (PyObject_TypeCheck(arg, &ChannelIDtype)) { + cid = ((channelid *)arg)->id; + } + else if (PyIndex_Check(arg)) { + cid = PyLong_AsLongLong(arg); + if (cid == -1 && PyErr_Occurred()) { + return 0; + } + if (cid < 0) { + PyErr_Format(PyExc_ValueError, + "channel ID must be a non-negative int, got %R", arg); + return 0; + } + } + else { + PyErr_Format(PyExc_TypeError, + "channel ID must be an int, got %.100s", + arg->ob_type->tp_name); + return 0; + } + *(int64_t *)ptr = cid; + return 1; +} + + static channelid * newchannelid(PyTypeObject *cls, int64_t cid, int end, _channels *channels, int force, int resolve) @@ -2327,6 +2377,65 @@ PyDoc_STRVAR(channel_list_all_doc, \n\ Return the list of all IDs for active channels."); + +static PyObject * +channel_list_interpreters(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"cid", "send", NULL}; + int64_t cid; /* Channel ID */ + int send = 0; /* Send or receive end? */ + PyObject *ret = NULL; + + if (!PyArg_ParseTupleAndKeywords( + args, kwds, "O&$p:channel_list_interpreters", + kwlist, channel_id_converter, &cid, &send)) { + return NULL; + } + + _PyChannelState *chan = _channels_lookup(&_globals.channels, cid, NULL); + if (chan == NULL) { + return NULL; + } + + int64_t count = 0; /* Number of interpreters */ + int64_t *ids = _channelends_list_interpreters(chan->ends, &count, send); + if (ids == NULL) { + goto except; + } + + ret = PyList_New((Py_ssize_t)count); + if (ret == NULL) { + goto except; + } + + for (int64_t i=0; i < count; i++) { + PyObject *id_obj = _PyInterpreterID_New(ids[i]); + if (id_obj == NULL) { + goto except; + } + PyList_SET_ITEM(ret, i, id_obj); + } + + goto finally; + +except: + Py_XDECREF(ret); + ret = NULL; + +finally: + PyMem_Free(ids); + return ret; +} + +PyDoc_STRVAR(channel_list_interpreters_doc, +"channel_list_interpreters(cid, *, send) -> [id]\n\ +\n\ +Return the list of all interpreter IDs associated with an end of the channel.\n\ +\n\ +The 'send' argument should be a boolean indicating whether to use the send or\n\ +receive end."); + + static PyObject * channel_send(PyObject *self, PyObject *args, PyObject *kwds) { @@ -2496,6 +2605,8 @@ static PyMethodDef module_functions[] = { METH_VARARGS | METH_KEYWORDS, channel_destroy_doc}, {"channel_list_all", channel_list_all, METH_NOARGS, channel_list_all_doc}, + {"channel_list_interpreters", (PyCFunction)(void(*)(void))channel_list_interpreters, + METH_VARARGS | METH_KEYWORDS, channel_list_interpreters_doc}, {"channel_send", (PyCFunction)(void(*)(void))channel_send, METH_VARARGS | METH_KEYWORDS, channel_send_doc}, {"channel_recv", (PyCFunction)(void(*)(void))channel_recv, From e9b56b8b954004ae9eb66b34d1019855c58ad0dd Mon Sep 17 00:00:00 2001 From: Joannah Nanjekye Date: Wed, 4 Mar 2020 22:39:43 +0000 Subject: [PATCH 15/20] Add send buffer functionality --- Lib/interpreters.py | 104 +++++++++++++++-------------- Modules/_xxsubinterpretersmodule.c | 93 ++++++++++++++++++++++++++ 2 files changed, 147 insertions(+), 50 deletions(-) diff --git a/Lib/interpreters.py b/Lib/interpreters.py index 97779a89390c0e..209dd44c0204b4 100644 --- a/Lib/interpreters.py +++ b/Lib/interpreters.py @@ -35,14 +35,18 @@ def get_current(): class Interpreter: def __init__(self, id): - self.id = id + self._id = id + + @property + def id(self): + return self._id def is_running(self): """is_running() -> bool Return whether or not the identified interpreter is running. """ - return _interpreters.is_running(self.id) + return _interpreters.is_running(self._id) def destroy(self): """destroy() @@ -52,7 +56,7 @@ def destroy(self): Attempting to destroy the current interpreter results in a RuntimeError. So does an unrecognized ID """ - return _interpreters.destroy(self.id) + return _interpreters.destroy(self._id) def run(self, src_str, /, *, channels=None): """run(src_str, /, *, channels=None) @@ -61,7 +65,7 @@ def run(self, src_str, /, *, channels=None): This blocks the current thread until done. """ try: - _interpreters.run_string(self.id, src_str) + _interpreters.run_string(self._id, src_str, channels) except RunFailedError as err: logger.error(err) raise @@ -71,7 +75,7 @@ def is_shareable(obj): """ is_shareable(obj) -> Bool Return `True` if the object's data can be shared between - interpreters. + interpreters and `False` otherwise. """ return _interpreters.is_shareable(obj) @@ -89,25 +93,21 @@ def list_all_channels(): Return all open channels. """ - cid = _interpreters.channel_list_all() - return (RecvChannel(cid), SendChannel(cid)) + return [(RecvChannel(cid), SendChannel(cid)) for cid in _interpreters.channel_list_all()] -def wait(self, timeout): +def wait(timeout): #The implementation for wait # will be non trivial to be useful import time time.sleep(timeout) -def associate_interp_to_channel(id, cid): - pass - class RecvChannel: def __init__(self, id): self.id = id - self.interpreters = _interpreters.channel_list_interpreters(cid, send=False) + self.interpreters = _interpreters.channel_list_interpreters(self.id, send=False) - def recv(self, timeout=2): + def recv(self): """ channel_recv() -> obj Get the next object from the channel, @@ -117,11 +117,7 @@ def recv(self, timeout=2): obj = _interpreters.channel_recv(self.id) if obj == None: wait(timeout) - obj = obj = _interpreters.channel_recv(self.id) - - # Pending: See issue 52 on multi-core python project - associate_interp_to_channel(interpId, Cid) - + obj = _interpreters.channel_recv(self.id) return obj def recv_nowait(self, default=None): @@ -129,44 +125,32 @@ def recv_nowait(self, default=None): Like recv(), but return the default instead of waiting. """ - return _interpreters.channel_recv(self.id) - - def send_buffer(self, obj): - """ send_buffer(obj) - - Send the object's buffer to the receiving end of the channel - and wait. Associate the interpreter with the channel. - """ - pass - - def send_buffer_nowait(self, obj): - """ send_buffer_nowait(obj) - - Like send_buffer(), but return False if not received. - """ - pass + obj = _interpreters.channel_recv(self.id) + if obj == None: + obj = default + return obj def release(self): """ release() No longer associate the current interpreterwith the channel - (on the sending end). + (on the receiving end). """ - return _interpreters(self.id) + return _interpreters.channel_release(self.id, recv=True) def close(self, force=False): """close(force=False) Close the channel in all interpreters.. """ - return _interpreters.channel_close(self.id, force) + return _interpreters.channel_close(self.id, recv=force) class SendChannel: def __init__(self, id): self.id = id - self.interpreters = _interpreters.list_all() + self.interpreters = _interpreters.channel_list_interpreters(self.id, send=True) def send(self, obj): """ send(obj) @@ -174,37 +158,57 @@ def send(self, obj): Send the object (i.e. its data) to the receiving end of the channel and wait. Associate the interpreter with the channel. """ - obj = _interpreters.channel_send(self.id, obj) + _interpreters.channel_send(self.id, obj) wait(2) - associate_interp_to_channel(interpId, Cid) def send_nowait(self, obj): """ send_nowait(obj) Like send(), but return False if not received. """ - try: - obj = _interpreters.channel_send(self.id, obj) - except: + _interpreters.channel_send(self.id, obj) + recv_obj = _interpreters.channel_recv(self.id) + if recv_obj: + return obj + else: return False - return obj + def send_buffer(self, obj): + """ ssend_buffer(obj) + + Send the object's buffer to the receiving + end of the channel and wait. Associate the interpreter + with the channel. + """ + _interpreters.channel_send_buffer(self.id, obj) + wait(2) + + def send_buffer_nowait(self, obj): + """ send_buffer_nowait(obj) + + Like send(), but return False if not received. + """ + _interpreters.channel_send_buffer(self.id, obj) + recv_obj = _interpreters.channel_recv(self.id) + if recv_obj: + return obj + else: + return False def release(self): """ release() - No longer associate the current interpreter with the channel + No longer associate the current interpreterwith the channel (on the sending end). """ - return _interpreters.channel_release(self.id) + return _interpreters.channel_release(self.id, send=True) def close(self, force=False): - """ close(force=False) + """close(force=False) - No longer associate the current interpreterwith the channel - (on the sending end). + Close the channel in all interpreters.. """ - return _interpreters.channel_close(self.id, force) + return _interpreters.channel_close(self.id, send=force) class ChannelError(Exception): diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index ff3dfb713bc50c..894e0641aa48ca 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -1303,6 +1303,71 @@ _channel_destroy(_channels *channels, int64_t id) return 0; } +static int +_channel_send_buffer(_channels *channels, int64_t id, PyObject *obj) +{ + const char *s = NULL; + Py_buffer view = {NULL, NULL}; + if (PyObject_GetBuffer(obj, &view, PyBUF_SIMPLE) != 0){ + return -1; + } + + s = view.buf; + if (s == NULL) { + PyBuffer_Release(&view); + return -1; + } + + PyInterpreterState *interp = _get_current(); + if (interp == NULL) { + PyBuffer_Release(&view); + return -1; + } + + // Look up the channel. + PyThread_type_lock mutex = NULL; + _PyChannelState *chan = _channels_lookup(channels, id, &mutex); + if (chan == NULL) { + PyBuffer_Release(&view); + return -1; + } + // Past this point we are responsible for releasing the mutex. + + if (chan->closing != NULL) { + PyErr_Format(ChannelClosedError, "channel %" PRId64 " closed", id); + PyThread_release_lock(mutex); + PyBuffer_Release(&view); + return -1; + } + + // Convert the buffer to cross-interpreter data. + _PyCrossInterpreterData *data = PyMem_NEW(_PyCrossInterpreterData, 1); + if (data == NULL) { + PyThread_release_lock(mutex); + PyBuffer_Release(&view); + return -1; + } + if (_PyObject_GetCrossInterpreterData((PyObject *)s, data) != 0) { + PyThread_release_lock(mutex); + PyMem_Free(data); + PyBuffer_Release(&view); + return -1; + } + + // Add the data to the channel. + int res = _channel_add(chan, PyInterpreterState_GetID(interp), data); + PyThread_release_lock(mutex); + if (res != 0) { + _PyCrossInterpreterData_Release(data); + PyMem_Free(data); + PyBuffer_Release(&view); + return -1; + } + + PyBuffer_Release(&view); + return 0; +} + static int _channel_send(_channels *channels, int64_t id, PyObject *obj) { @@ -2462,6 +2527,32 @@ PyDoc_STRVAR(channel_send_doc, \n\ Add the object's data to the channel's queue."); +static PyObject * +channel_send_buffer(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"cid", "obj", NULL}; + PyObject *id; + PyObject *obj; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "OO:channel_send_buffer", kwlist, &id, &obj)) { + return NULL; + } + int64_t cid = _Py_CoerceID(id); + if (cid < 0) { + return NULL; + } + + if (_channel_send_buffer(&_globals.channels, cid, obj) != 0) { + return NULL; + } + Py_RETURN_NONE; +} + +PyDoc_STRVAR(channel_send_buffer_doc, +"channel_send_buffer(cid, obj)\n\ +\n\ +Add the object's buffer to the channel's queue."); + static PyObject * channel_recv(PyObject *self, PyObject *args, PyObject *kwds) { @@ -2609,6 +2700,8 @@ static PyMethodDef module_functions[] = { METH_VARARGS | METH_KEYWORDS, channel_list_interpreters_doc}, {"channel_send", (PyCFunction)(void(*)(void))channel_send, METH_VARARGS | METH_KEYWORDS, channel_send_doc}, + {"channel_send_buffer", (PyCFunction)(void(*)(void))channel_send_buffer, + METH_VARARGS | METH_KEYWORDS, channel_send_buffer_doc}, {"channel_recv", (PyCFunction)(void(*)(void))channel_recv, METH_VARARGS | METH_KEYWORDS, channel_recv_doc}, {"channel_close", (PyCFunction)(void(*)(void))channel_close, From 2bf89d41507aab7e3daec1b558e6b69c19067913 Mon Sep 17 00:00:00 2001 From: Joannah Nanjekye Date: Thu, 5 Mar 2020 19:46:39 +0000 Subject: [PATCH 16/20] Add tests for Global functions and Interpreter --- Doc/library/interpreters.rst | 9 +- Lib/interpreters.py | 14 +- Lib/test/test_interpreters.py | 454 ++++++++++++++++++++++++++++++++-- 3 files changed, 453 insertions(+), 24 deletions(-) diff --git a/Doc/library/interpreters.rst b/Doc/library/interpreters.rst index 5d15484b6b4d78..3944b8d0712a42 100644 --- a/Doc/library/interpreters.rst +++ b/Doc/library/interpreters.rst @@ -8,9 +8,9 @@ -------------- -This module provides tools for working with sub-interpreters, such as creating them, -running code in them, or sending data between them. It is a wrapper around the low- -level :mod:`_interpreters` module. +This module provides highlevel tools for working with sub-interpreters, +such as creating them, running code in them, or sending data between them. +It is a wrapper around the low-level :mod:`_interpreters` module. .. versionchanged:: added in 3.9 @@ -124,7 +124,8 @@ This module defines the following global functions: .. function:: get_current() - Get the currently running interpreter. + Get the currently running interpreter. This method returns + an `interpreter` object. .. function:: list_all() diff --git a/Lib/interpreters.py b/Lib/interpreters.py index 209dd44c0204b4..10591c2bea0160 100644 --- a/Lib/interpreters.py +++ b/Lib/interpreters.py @@ -61,11 +61,23 @@ def destroy(self): def run(self, src_str, /, *, channels=None): """run(src_str, /, *, channels=None) + channel = (RecvChannel, SendChannel) + Run the given source code in the interpreter. This blocks the current thread until done. """ + if channels: + if channels[0] and channels[1] != None: + _interpreters.channel_recv(channels[0].id) + _interpreters.channel_send(channels[1].id, src_str) + elif channels[0] != None and channels[1] == None: + _interpreters.channel_recv(channels[0].id) + elif channels[0] == None and channels[1] != None: + _interpreters.channel_send(channels[1].id, src_str) + else: + pass try: - _interpreters.run_string(self._id, src_str, channels) + _interpreters.run_string(self._id, src_str) except RunFailedError as err: logger.error(err) raise diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index 59c4ec0e8bf4a1..0253dce6a04e89 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -1,9 +1,12 @@ -import interpreters -import _interpreters +import contextlib import os +import threading from textwrap import dedent import unittest +import interpreters +import _interpreters + def _captured_script(script): r, w = os.pipe() indented = script.replace('\n', '\n ') @@ -16,65 +19,478 @@ def _captured_script(script): return wrapped, open(r) def clean_up_interpreters(): - for id in _interpreters.list_all(): - if id == 0: # main + for interp in interpreters.list_all(): + if interp.id == 0: # main continue try: - _interpreters.destroy(id) + interp.destroy() except RuntimeError: pass # already destroyed def _run_output(interp, request, shared=None): script, rpipe = _captured_script(request) with rpipe: - interpreters.run_string(interp, script, shared) + interp.run(script) return rpipe.read() +@contextlib.contextmanager +def _running(interp): + r, w = os.pipe() + def run(): + interp.run(dedent(f""" + # wait for "signal" + with open({r}) as rpipe: + rpipe.read() + """)) + + t = threading.Thread(target=run) + t.start() + + yield + + with open(w, 'w') as spipe: + spipe.write('done') + t.join() + + class TestBase(unittest.TestCase): def tearDown(self): clean_up_interpreters() + class CreateTests(TestBase): - def test_create(self): + def test_in_main(self): interp = interpreters.create() lst = interpreters.list_all() self.assertEqual(interp.id, lst[1].id) + def test_in_thread(self): + lock = threading.Lock() + id = None + interp = interpreters.create() + lst = interpreters.list_all() + def f(): + nonlocal id + id = interp.id + lock.acquire() + lock.release() + + t = threading.Thread(target=f) + with lock: + t.start() + t.join() + self.assertEqual(interp.id, lst[1].id) + + def test_in_subinterpreter(self): + main, = interpreters.list_all() + interp = interpreters.create() + out = _run_output(interp, dedent(""" + import interpreters + interp = interpreters.create() + print(interp) + """)) + interp2 = out.strip() + + self.assertEqual(len(set(interpreters.list_all())), len({main, interp, interp2})) + + def test_in_threaded_subinterpreter(self): + main, = interpreters.list_all() + interp = interpreters.create() + interp2 = None + def f(): + nonlocal interp2 + out = _run_output(interp, dedent(""" + import interpreters + interp = interpreters.create() + print(interp) + """)) + interp2 = int(out.strip()) + + t = threading.Thread(target=f) + t.start() + t.join() + + self.assertEqual(len(set(interpreters.list_all())), len({main, interp, interp2})) + + def test_after_destroy_all(self): + before = set(interpreters.list_all()) + # Create 3 subinterpreters. + interp_lst = [] + for _ in range(3): + interps = interpreters.create() + interp_lst.append(interps) + # Now destroy them. + for interp in interp_lst: + interp.destroy() + # Finally, create another. + interp = interpreters.create() + self.assertEqual(len(set(interpreters.list_all())), len(before | {interp})) + + def test_after_destroy_some(self): + before = set(interpreters.list_all()) + # Create 3 subinterpreters. + interp1 = interpreters.create() + interp2 = interpreters.create() + interp3 = interpreters.create() + # Now destroy 2 of them. + interp1.destroy() + interp2.destroy() + # Finally, create another. + interp = interpreters.create() + self.assertEqual(len(set(interpreters.list_all())), len(before | {interp3, interp})) + + class GetCurrentTests(TestBase): - def test_get_current(self): + def test_main(self): main_interp_id = _interpreters.get_main() cur_interp_id = interpreters.get_current().id self.assertEqual(cur_interp_id, main_interp_id) + def test_subinterpreter(self): + main = _interpreters.get_main() + interp = interpreters.create() + out = _run_output(interp, dedent(""" + import interpreters + cur = interpreters.get_current() + print(cur) + """)) + cur = out.strip() + self.assertNotEqual(cur, main) + + class ListAllTests(TestBase): def test_initial(self): interps = interpreters.list_all() self.assertEqual(1, len(interps)) -class TestInterpreter(TestBase): + def test_after_creating(self): + main = interpreters.get_current() + first = interpreters.create() + second = interpreters.create() + + ids = [] + for interp in interpreters.list_all(): + ids.append(interp.id) + + self.assertEqual(ids, [main.id, first.id, second.id]) + + def test_after_destroying(self): + main = interpreters.get_current() + first = interpreters.create() + second = interpreters.create() + first.destroy() + + ids = [] + for interp in interpreters.list_all(): + ids.append(interp.id) + + self.assertEqual(ids, [main.id, second.id]) + + +class TestInterpreterId(TestBase): - def test_id_fields(self): + def test_id_field_in_main(self): + main = interpreters.get_current() + self.assertEqual(0, main.id) + + def test_id_field_custom(self): interp = interpreters.Interpreter(1) self.assertEqual(1, interp.id) - def test_is_running(self): - interp_de = interpreters.create() - self.assertEqual(False, interp_de.is_running()) + def test_id_field_readonly(self): + interp = interpreters.Interpreter(1) + with self.assertRaises(AttributeError): + interp.id = 2 + + +class TestInterpreterIsRunning(TestBase): + + def test_is_running_main(self): + main = interpreters.get_current() + self.assertTrue(main.is_running()) + + def test_is_running_subinterpreter(self): + interp = interpreters.create() + self.assertFalse(interp.is_running()) + + with _running(interp): + self.assertTrue(interp.is_running()) + self.assertFalse(interp.is_running()) + + def test_is_running_from_subinterpreter(self): + interp = interpreters.create() + out = _run_output(interp, dedent(f""" + import _interpreters + if _interpreters.is_running({interp.id}): + print(True) + else: + print(False) + """)) + self.assertEqual(out.strip(), 'True') - def test_destroy(self): + def test_is_running_already_destroyed(self): interp = interpreters.create() + interp.destroy() + with self.assertRaises(RuntimeError): + interp.is_running() + + def test_is_running_bad_id(self): + interp = interpreters.Interpreter(-1) + with self.assertRaises(RuntimeError): + interp.is_running() + + +class TestInterpreterDestroy(TestBase): + + def test_destroy_basic(self): + interp1 = interpreters.create() interp2 = interpreters.create() + interp3 = interpreters.create() + self.assertEqual(4, len(interpreters.list_all())) + interp2.destroy() + self.assertEqual(3, len(interpreters.list_all())) + + def test_destroy_all(self): + before = set(interpreters.list_all()) + interps = set() + for _ in range(3): + interp = interpreters.create() + interps.add(interp) + self.assertEqual(len(set(interpreters.list_all())), len(before | interps)) + for interp in interps: + interp.destroy() + self.assertEqual(len(set(interpreters.list_all())), len(before)) + + def test_destroy_main(self): + main, = interpreters.list_all() + with self.assertRaises(RuntimeError): + main.destroy() + + def f(): + with self.assertRaises(RuntimeError): + main.destroy() + + t = threading.Thread(target=f) + t.start() + t.join() + + def test_destroy_already_destroyed(self): + interp = interpreters.create() interp.destroy() - interps = interpreters.list_all() - self.assertEqual(2, len(interps)) + with self.assertRaises(RuntimeError): + interp.destroy() - def test_run(self): + def test_destroy_from_current(self): + main, = interpreters.list_all() interp = interpreters.create() - interp.run("3") - self.assertEqual(False, interp.is_running()) + script = dedent(f""" + import interpreters + try: + main = interpreters.get_current() + main.destroy() + except RuntimeError: + pass + """) + + interp.run(script) + self.assertEqual(len(set(interpreters.list_all())), len({main, interp})) + + def test_destroy_from_sibling(self): + main, = interpreters.list_all() + interp1 = interpreters.create() + script = dedent(f""" + import interpreters + interp2 = interpreters.create() + interp2.destroy() + """) + interp1.run(script) + + self.assertEqual(len(set(interpreters.list_all())), len({main, interp1})) + + def test_destroy_from_other_thread(self): + interp = interpreters.create() + def f(): + interp.destroy() + + t = threading.Thread(target=f) + t.start() + t.join() + + def test_destroy_still_running(self): + main, = interpreters.list_all() + interp = interpreters.create() + with _running(interp): + with self.assertRaises(RuntimeError): + interp.destroy() + self.assertTrue(interp.is_running()) + + +class TestInterpreterRun(TestBase): + + SCRIPT = dedent(""" + with open('{}', 'w') as out: + out.write('{}') + """) + FILENAME = 'spam' + + def setUp(self): + super().setUp() + self.interp = interpreters.create() + self._fs = None + + def tearDown(self): + if self._fs is not None: + self._fs.close() + super().tearDown() + + @property + def fs(self): + if self._fs is None: + self._fs = FSFixture(self) + return self._fs + + def test_success(self): + script, file = _captured_script('print("it worked!", end="")') + with file: + self.interp.run(script) + out = file.read() + + self.assertEqual(out, 'it worked!') + + def test_in_thread(self): + script, file = _captured_script('print("it worked!", end="")') + with file: + def f(): + self.interp.run(script) + + t = threading.Thread(target=f) + t.start() + t.join() + out = file.read() + + self.assertEqual(out, 'it worked!') + + def test_create_thread(self): + script, file = _captured_script(""" + import threading + def f(): + print('it worked!', end='') + + t = threading.Thread(target=f) + t.start() + t.join() + """) + with file: + self.interp.run(script) + out = file.read() + + self.assertEqual(out, 'it worked!') + + @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") + def test_fork(self): + import tempfile + with tempfile.NamedTemporaryFile('w+') as file: + file.write('') + file.flush() + + expected = 'spam spam spam spam spam' + script = dedent(f""" + import os + try: + os.fork() + except RuntimeError: + with open('{file.name}', 'w') as out: + out.write('{expected}') + """) + self.interp.run(script) + + file.seek(0) + content = file.read() + self.assertEqual(content, expected) + + def test_already_running(self): + with _running(self.interp): + with self.assertRaises(RuntimeError): + self.interp.run('print("spam")') + + def test_bad_script(self): + with self.assertRaises(TypeError): + self.interp.run(10) + + def test_bytes_for_script(self): + with self.assertRaises(TypeError): + self.interp.run(b'print("spam")') + + +class TestIsShareable(TestBase): + + def test_default_shareables(self): + shareables = [ + # singletons + None, + # builtin objects + b'spam', + 'spam', + 10, + -10, + ] + for obj in shareables: + with self.subTest(obj): + self.assertTrue( + interpreters.is_shareable(obj)) + + def test_not_shareable(self): + class Cheese: + def __init__(self, name): + self.name = name + def __str__(self): + return self.name + + class SubBytes(bytes): + """A subclass of a shareable type.""" + + not_shareables = [ + # singletons + True, + False, + NotImplemented, + ..., + # builtin types and objects + type, + object, + object(), + Exception(), + 100.0, + # user-defined types and objects + Cheese, + Cheese('Wensleydale'), + SubBytes(b'spam'), + ] + for obj in not_shareables: + with self.subTest(repr(obj)): + self.assertFalse( + interpreters.is_shareable(obj)) + + +class TestChannel(TestBase): + + def test_create_cid(self): + r, s = interpreters.create_channel() + self.assertIsInstance(r, interpreters.RecvChannel) + self.assertIsInstance(s, interpreters.SendChannel) + + def test_sequential_ids(self): + before = interpreters.list_all_channels() + channels1 = interpreters.create_channel() + channels2 = interpreters.create_channel() + channels3 = interpreters.create_channel() + after = interpreters.list_all_channels() + self.assertEqual(len(set(after) - set(before)), + len({channels1, channels2, channels3})) +class TestRecvChannelID(TestBase): \ No newline at end of file From 158e54fe848308287c66c6cc65683e46b3aae990 Mon Sep 17 00:00:00 2001 From: Joannah Nanjekye Date: Fri, 6 Mar 2020 21:35:43 +0000 Subject: [PATCH 17/20] Test coverage and documentation --- Doc/library/interpreters.rst | 92 ++++++---- Lib/interpreters.py | 177 +++++++++++++----- Lib/test/test_interpreters.py | 283 ++++++++++++++++++++++++++--- Modules/_xxsubinterpretersmodule.c | 84 +++++++-- 4 files changed, 515 insertions(+), 121 deletions(-) diff --git a/Doc/library/interpreters.rst b/Doc/library/interpreters.rst index 3944b8d0712a42..6b9c35b6250b89 100644 --- a/Doc/library/interpreters.rst +++ b/Doc/library/interpreters.rst @@ -8,7 +8,7 @@ -------------- -This module provides highlevel tools for working with sub-interpreters, +This module provides highlevel tools for working with sub-interpreters, such as creating them, running code in them, or sending data between them. It is a wrapper around the low-level :mod:`_interpreters` module. @@ -25,16 +25,18 @@ The Interpreter object represents a single interpreter. .. method:: is_running() Return whether or not the identified interpreter is running. + It returns `True` and `False` otherwise. .. method:: destroy() - Destroy the identified interpreter. Attempting to destroy the current - interpreter results in a RuntimeError. So does an unrecognized ID. + Destroy the interpreter. Attempting to destroy the current + interpreter results in a `RuntimeError`. .. method:: run(self, src_str, /, *, channels=None): - Run the given source code in the interpreter. This blocks the current - thread until done. + Run the given source code in the interpreter. This blocks + the current thread until done. `channels` should be in + the form : `(RecvChannel, SendChannel)`. RecvChannel Objects ------------------- @@ -47,60 +49,69 @@ The RecvChannel object represents a recieving channel. .. method:: recv() - Get the next object from the channel, and wait if none have been - sent. Associate the interpreter with the channel. + Get the next object from the channel, and wait if + none have been sent. Associate the interpreter + with the channel. .. method:: recv_nowait(default=None) - Like ``recv()``, but return the default instead of waiting. + Like ``recv()``, but return the default result + instead of waiting. - .. method:: release() + .. method:: release() - Close the channel for the current interpreter. 'send' and 'recv' (bool) may - be used to indicate the ends to close. By default both ends are closed. - Closing an already closed end is a noop. + Release the channel for the current interpreter. + By default both ends are released. Releasing an already + released end results in a ``ChannelReleasedError`` exception. .. method:: close(force=False) - Close the channel in all interpreters. + Close the channel in all interpreters. By default + both ends are closed. closing an already closed end + results in a ``ChannelClosedError`` exception. Without + seeting ``force`` to ``True`` a ``ChannelNotEmptyError`` + will be returned when a channel with data is closed. SendChannel Objects -------------------- -The SendChannel object represents a sending channel. +The ``SendChannel`` object represents a sending channel. .. class:: SendChannel(id) - This class represents the receiving end of a channel. + This class represents the sending end of a channel. .. method:: send(obj) - Send the object (i.e. its data) to the receiving end of the channel - and wait.Associate the interpreter with the channel. + Send the object ``obj`` to the receiving end of the channel + and wait. Associate the interpreter with the channel. .. method:: send_nowait(obj) - Like ``send()``, but return False if not received. + Like ``send()`` but return ``False`` if not received. .. method:: send_buffer(obj) - Send the object's buffer to the receiving end of the channel and wait. - Associate the interpreter with the channel. + Send the object's buffer to the receiving end of the + channel and wait. Associate the interpreter with the + channel. .. method:: send_buffer_nowait(obj) - Like ``send_buffer()``, but return False if not received. + Like ``send_buffer()`` but return ``False`` if not received. .. method:: release() - Close the channel for the current interpreter. 'send' and 'recv' (bool) may - be used to indicate the ends to close. By default both ends are closed. - Closing an already closed end is a noop. + Release the channel for the current interpreter. + By default both ends are released. Releasing an already + released end results in a ``ChannelReleasedError`` exception. .. method:: close(force=False) - Close the channel in all interpreters. + Close the channel in all interpreters. By default + both ends are closed. closing an already closed end + results in a ``ChannelClosedError`` exception. This module defines the following global functions: @@ -108,7 +119,8 @@ This module defines the following global functions: .. function:: is_shareable(obj) - Return `True` if the object's data can be shared between interpreters. + Return ``True`` if the object's data can be shared between + interpreters. .. function:: create_channel() @@ -120,16 +132,18 @@ This module defines the following global functions: .. function:: create() - Initialize a new (idle) Python interpreter. + Initialize a new (idle) Python interpreter. Get the currently + running interpreter. This method returns an ``Interpreter`` object. .. function:: get_current() Get the currently running interpreter. This method returns - an `interpreter` object. + an ``Interpreter`` object. .. function:: list_all() - Get all existing interpreters. + Get all existing interpreters. Returns a list + of ``Interpreter`` objects. This module also defines the following exceptions. @@ -140,35 +154,35 @@ This module also defines the following exceptions. .. exception:: ChannelError - This exception, a subclass of :exc:`Exception`, and is the base class for - channel-related exceptions. + This exception is a subclass of :exc:`Exception`, and is the base + class for all channel-related exceptions. .. exception:: ChannelNotFoundError - This exception, a subclass of :exc:`ChannelError`, is raised when the - the identified channel was not found. + This exception is a subclass of :exc:`ChannelError`, and is raised + when the the identified channel is not found. .. exception:: ChannelEmptyError - This exception, a subclass of :exc:`ChannelError`, is raised when + This exception is a subclass of :exc:`ChannelError`, and is raised when the channel is unexpectedly empty. .. exception:: ChannelNotEmptyError - This exception, a subclass of :exc:`ChannelError`, is raised when + This exception is a subclass of :exc:`ChannelError`, and is raised when the channel is unexpectedly not empty. .. exception:: NotReceivedError - This exception, a subclass of :exc:`ChannelError`, is raised when + This exception is a subclass of :exc:`ChannelError`, and is raised when nothing was waiting to receive a sent object. .. exception:: ChannelClosedError - This exception, a subclass of :exc:`ChannelError`, is raised when + This exception is a subclass of :exc:`ChannelError`, and is raised when the channel is closed. .. exception:: ChannelReleasedError - This exception, a subclass of :exc:`ChannelClosedError`, is raised when - the channel is released (but not yet closed). + This exception is a subclass of :exc:`ChannelClosedError`, and is raised + when the channel is released (but not yet closed). diff --git a/Lib/interpreters.py b/Lib/interpreters.py index 10591c2bea0160..04e9d551a547dc 100644 --- a/Lib/interpreters.py +++ b/Lib/interpreters.py @@ -3,8 +3,9 @@ import _interpreters import logging -__all__ = ['Interpreter', 'SendChannel', 'RecvChannel', 'is_shareable', - 'create_channel', 'list_all_channels', 'list_all', 'get_current', +__all__ = ['Interpreter', 'SendChannel', 'RecvChannel', + 'is_shareable', 'create_channel', + 'list_all_channels', 'get_current', 'create'] @@ -21,7 +22,8 @@ def list_all(): Get all existing interpreters. """ - return [Interpreter(id) for id in _interpreters.list_all()] + return [Interpreter(id) for id in + _interpreters.list_all()] def get_current(): """ get_current() -> Interpreter @@ -44,28 +46,24 @@ def id(self): def is_running(self): """is_running() -> bool - Return whether or not the identified interpreter is running. + Return whether or not the identified + interpreter is running. """ return _interpreters.is_running(self._id) def destroy(self): """destroy() - Destroy the identified interpreter. + Destroy the interpreter. Attempting to destroy the current - interpreter results in a RuntimeError. So does an unrecognized ID + interpreter results in a RuntimeError. """ return _interpreters.destroy(self._id) - def run(self, src_str, /, *, channels=None): - """run(src_str, /, *, channels=None) - - channel = (RecvChannel, SendChannel) - - Run the given source code in the interpreter. - This blocks the current thread until done. - """ + def _handle_channels(self, channels): + # Looks like the only way for an interpreter to be associated + # to a channel is through sending or recving data. if channels: if channels[0] and channels[1] != None: _interpreters.channel_recv(channels[0].id) @@ -76,6 +74,14 @@ def run(self, src_str, /, *, channels=None): _interpreters.channel_send(channels[1].id, src_str) else: pass + + def run(self, src_str, /, *, channels=None): + """run(src_str, /, *, channels=None) + + Run the given source code in the interpreter. + This blocks the current thread until done. + """ + self._handle_channels(channels) try: _interpreters.run_string(self._id, src_str) except RunFailedError as err: @@ -86,15 +92,16 @@ def run(self, src_str, /, *, channels=None): def is_shareable(obj): """ is_shareable(obj) -> Bool - Return `True` if the object's data can be shared between - interpreters and `False` otherwise. + Return `True` if the object's data can be + shared between interpreters and `False` otherwise. """ return _interpreters.is_shareable(obj) def create_channel(): """ create_channel() -> (RecvChannel, SendChannel) - Create a new channel for passing data between interpreters. + Create a new channel for passing data between + interpreters. """ cid = _interpreters.channel_create() @@ -105,7 +112,8 @@ def list_all_channels(): Return all open channels. """ - return [(RecvChannel(cid), SendChannel(cid)) for cid in _interpreters.channel_list_all()] + return [(RecvChannel(cid), SendChannel(cid)) + for cid in _interpreters.channel_list_all()] def wait(timeout): #The implementation for wait @@ -117,7 +125,9 @@ class RecvChannel: def __init__(self, id): self.id = id - self.interpreters = _interpreters.channel_list_interpreters(self.id, send=False) + self.interpreters = _interpreters.\ + channel_list_interpreters(self.id,\ + send=False) def recv(self): """ channel_recv() -> obj @@ -126,101 +136,174 @@ def recv(self): and wait if none have been sent. Associate the interpreter with the channel. """ - obj = _interpreters.channel_recv(self.id) - if obj == None: - wait(timeout) + try: obj = _interpreters.channel_recv(self.id) + if obj == None: + wait(2) + obj = _interpreters.channel_recv(self.id) + except _interpreters.ChannelEmptyError: + raise ChannelEmptyError + except _interpreters.ChannelNotFoundError: + raise ChannelNotFoundError + except _interpreters.ChannelClosedError: + raise ChannelClosedError + except _interpreters.RunFailedError: + raise RunFailedError return obj def recv_nowait(self, default=None): """recv_nowait(default=None) -> object - Like recv(), but return the default instead of waiting. + Like recv(), but return the default + instead of waiting. """ - obj = _interpreters.channel_recv(self.id) - if obj == None: - obj = default + try: + obj = _interpreters.channel_recv(self.id) + if obj == None: + obj = default + except _interpreters.ChannelEmptyError: + raise ChannelEmptyError + except _interpreters.ChannelNotFoundError: + raise ChannelNotFoundError + except _interpreters.ChannelClosedError: + raise ChannelClosedError + except _interpreters.RunFailedError: + raise RunFailedError return obj def release(self): """ release() - No longer associate the current interpreterwith the channel - (on the receiving end). + No longer associate the current interpreter + with the channel (on the receiving end). """ - return _interpreters.channel_release(self.id, recv=True) + try: + _interpreters.channel_release(self.id, recv=True) + except _interpreters.ChannelClosedError: + raise ChannelClosedError + except _interpreters.ChannelNotEmptyError: + raise ChannelNotEmptyError def close(self, force=False): """close(force=False) - Close the channel in all interpreters.. + Close the channel in all interpreters. """ - return _interpreters.channel_close(self.id, recv=force) + try: + _interpreters.channel_close(self.id, + force=force, recv=True) + except _interpreters.ChannelClosedError: + raise ChannelClosedError + except _interpreters.ChannelNotEmptyError: + raise ChannelNotEmptyError class SendChannel: def __init__(self, id): self.id = id - self.interpreters = _interpreters.channel_list_interpreters(self.id, send=True) + self.interpreters = _interpreters.\ + channel_list_interpreters(self.id,\ + send=True) def send(self, obj): """ send(obj) - Send the object (i.e. its data) to the receiving end of the channel - and wait. Associate the interpreter with the channel. + Send the object (i.e. its data) to the receiving + end of the channel and wait. Associate the interpreter + with the channel. """ - _interpreters.channel_send(self.id, obj) - wait(2) + try: + _interpreters.channel_send(self.id, obj) + wait(2) + except _interpreters.ChannelNotFoundError: + raise ChannelNotFoundError + except _interpreters.ChannelClosedError: + raise ChannelClosedError + except _interpreters.RunFailedError: + raise RunFailedError def send_nowait(self, obj): """ send_nowait(obj) Like send(), but return False if not received. """ - _interpreters.channel_send(self.id, obj) + try: + _interpreters.channel_send(self.id, obj) + except _interpreters.ChannelNotFoundError: + raise ChannelNotFoundError + except _interpreters.ChannelClosedError: + raise ChannelClosedError + except _interpreters.RunFailedError: + raise RunFailedError + recv_obj = _interpreters.channel_recv(self.id) if recv_obj: - return obj + return obj else: return False def send_buffer(self, obj): - """ ssend_buffer(obj) + """ send_buffer(obj) Send the object's buffer to the receiving end of the channel and wait. Associate the interpreter with the channel. """ - _interpreters.channel_send_buffer(self.id, obj) - wait(2) + try: + _interpreters.channel_send_buffer(self.id, obj) + wait(2) + except _interpreters.ChannelNotFoundError: + raise ChannelNotFoundError + except _interpreters.ChannelClosedError: + raise ChannelClosedError + except _interpreters.RunFailedError: + raise RunFailedError def send_buffer_nowait(self, obj): """ send_buffer_nowait(obj) Like send(), but return False if not received. """ - _interpreters.channel_send_buffer(self.id, obj) + try: + _interpreters.channel_send_buffer(self.id, obj) + except _interpreters.ChannelNotFoundError: + raise ChannelNotFoundError + except _interpreters.ChannelClosedError: + raise ChannelClosedError + except _interpreters.RunFailedError: + raise RunFailedError recv_obj = _interpreters.channel_recv(self.id) if recv_obj: - return obj + return obj else: return False def release(self): """ release() - No longer associate the current interpreterwith the channel - (on the sending end). + No longer associate the current interpreter + with the channel (on the sending end). """ - return _interpreters.channel_release(self.id, send=True) + try: + _interpreters.channel_release(self.id, send=True) + except _interpreters.ChannelClosedError: + raise ChannelClosedError + except _interpreters.ChannelNotEmptyError: + raise ChannelNotEmptyError def close(self, force=False): """close(force=False) Close the channel in all interpreters.. """ - return _interpreters.channel_close(self.id, send=force) + try: + _interpreters.channel_close(self.id, + force=force, send=False) + except _interpreters.ChannelClosedError: + raise ChannelClosedError + except _interpreters.ChannelNotEmptyError: + raise ChannelNotEmptyError class ChannelError(Exception): diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index 0253dce6a04e89..7570b925254ba8 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -3,7 +3,7 @@ import threading from textwrap import dedent import unittest - +import time import interpreters import _interpreters @@ -107,11 +107,11 @@ def f(): print(interp) """)) interp2 = int(out.strip()) - + t = threading.Thread(target=f) t.start() t.join() - + self.assertEqual(len(set(interpreters.list_all())), len({main, interp, interp2})) def test_after_destroy_all(self): @@ -193,15 +193,15 @@ def test_after_destroying(self): class TestInterpreterId(TestBase): - def test_id_field_in_main(self): + def test_in_main(self): main = interpreters.get_current() self.assertEqual(0, main.id) - def test_id_field_custom(self): + def test_with_custom_num(self): interp = interpreters.Interpreter(1) self.assertEqual(1, interp.id) - def test_id_field_readonly(self): + def test_for_readonly_property(self): interp = interpreters.Interpreter(1) with self.assertRaises(AttributeError): interp.id = 2 @@ -209,11 +209,11 @@ def test_id_field_readonly(self): class TestInterpreterIsRunning(TestBase): - def test_is_running_main(self): + def test_main(self): main = interpreters.get_current() self.assertTrue(main.is_running()) - def test_is_running_subinterpreter(self): + def test_subinterpreter(self): interp = interpreters.create() self.assertFalse(interp.is_running()) @@ -221,7 +221,7 @@ def test_is_running_subinterpreter(self): self.assertTrue(interp.is_running()) self.assertFalse(interp.is_running()) - def test_is_running_from_subinterpreter(self): + def test_from_subinterpreter(self): interp = interpreters.create() out = _run_output(interp, dedent(f""" import _interpreters @@ -232,13 +232,13 @@ def test_is_running_from_subinterpreter(self): """)) self.assertEqual(out.strip(), 'True') - def test_is_running_already_destroyed(self): + def test_already_destroyed(self): interp = interpreters.create() interp.destroy() with self.assertRaises(RuntimeError): interp.is_running() - def test_is_running_bad_id(self): + def test_bad_id(self): interp = interpreters.Interpreter(-1) with self.assertRaises(RuntimeError): interp.is_running() @@ -246,7 +246,7 @@ def test_is_running_bad_id(self): class TestInterpreterDestroy(TestBase): - def test_destroy_basic(self): + def test_basic(self): interp1 = interpreters.create() interp2 = interpreters.create() interp3 = interpreters.create() @@ -254,7 +254,7 @@ def test_destroy_basic(self): interp2.destroy() self.assertEqual(3, len(interpreters.list_all())) - def test_destroy_all(self): + def test_all(self): before = set(interpreters.list_all()) interps = set() for _ in range(3): @@ -265,7 +265,7 @@ def test_destroy_all(self): interp.destroy() self.assertEqual(len(set(interpreters.list_all())), len(before)) - def test_destroy_main(self): + def test_main(self): main, = interpreters.list_all() with self.assertRaises(RuntimeError): main.destroy() @@ -278,13 +278,13 @@ def f(): t.start() t.join() - def test_destroy_already_destroyed(self): + def test_already_destroyed(self): interp = interpreters.create() interp.destroy() with self.assertRaises(RuntimeError): interp.destroy() - def test_destroy_from_current(self): + def test_from_current(self): main, = interpreters.list_all() interp = interpreters.create() script = dedent(f""" @@ -299,7 +299,7 @@ def test_destroy_from_current(self): interp.run(script) self.assertEqual(len(set(interpreters.list_all())), len({main, interp})) - def test_destroy_from_sibling(self): + def test_from_sibling(self): main, = interpreters.list_all() interp1 = interpreters.create() script = dedent(f""" @@ -311,7 +311,7 @@ def test_destroy_from_sibling(self): self.assertEqual(len(set(interpreters.list_all())), len({main, interp1})) - def test_destroy_from_other_thread(self): + def test_from_other_thread(self): interp = interpreters.create() def f(): interp.destroy() @@ -320,7 +320,7 @@ def f(): t.start() t.join() - def test_destroy_still_running(self): + def test_still_running(self): main, = interpreters.list_all() interp = interpreters.create() with _running(interp): @@ -490,7 +490,248 @@ def test_sequential_ids(self): channels3 = interpreters.create_channel() after = interpreters.list_all_channels() - self.assertEqual(len(set(after) - set(before)), + self.assertEqual(len(set(after) - set(before)), len({channels1, channels2, channels3})) -class TestRecvChannelID(TestBase): \ No newline at end of file +class TestSendRecv(TestBase): + + def test_fields(self): + r, s = interpreters.create_channel() + self.assertGreaterEqual(r.id, 0) + self.assertEqual([], r.interpreters) + + def test_send_recv_main(self): + r, s = interpreters.create_channel() + orig = b'spam' + s.send(orig) + obj = r.recv() + + self.assertEqual(obj, orig) + self.assertIsNot(obj, orig) + + def test_send_recv_same_interpreter(self): + interp = interpreters.create() + out = _run_output(interp, dedent(""" + import interpreters + r, s = interpreters.create_channel() + orig = b'spam' + s.send(orig) + obj = r.recv() + assert obj is not orig + assert obj == orig + """)) + + def test_send_recv_different_threads(self): + r, s = interpreters.create_channel() + + def f(): + while True: + try: + obj = r.recv() + break + except interpreters.ChannelEmptyError: + time.sleep(0.1) + s.send(obj) + t = threading.Thread(target=f) + t.start() + + s.send(b'spam') + t.join() + obj = r.recv() + + self.assertEqual(obj, b'spam') + + def test_recv_empty(self): + r, s = interpreters.create_channel() + with self.assertRaises(interpreters.ChannelEmptyError): + r.recv() + + def test_send_recv_nowait_main(self): + r, s = interpreters.create_channel() + orig = b'spam' + s.send(orig) + obj = r.recv_nowait() + + self.assertEqual(obj, orig) + self.assertIsNot(obj, orig) + + def test_send_recv_nowait_same_interpreter(self): + interp = interpreters.create() + out = _run_output(interp, dedent(""" + import interpreters + r, s = interpreters.create_channel() + orig = b'spam' + s.send(orig) + obj = r.recv_nowait() + assert obj is not orig + assert obj == orig + """)) + + def test_send_recv_nowait_different_threads(self): + r, s = interpreters.create_channel() + + def f(): + while True: + try: + obj = r.recv_nowait() + break + except interpreters.ChannelEmptyError: + time.sleep(0.1) + s.send(obj) + t = threading.Thread(target=f) + t.start() + + s.send(b'spam') + t.join() + obj = r.recv_nowait() + + self.assertEqual(obj, b'spam') + + def test_recv_nowait_empty(self): + r, s = interpreters.create_channel() + with self.assertRaises(interpreters.ChannelEmptyError): + r.recv_nowait() + + # close + + def test_close_single_user(self): + r, s = interpreters.create_channel() + s.send(b'spam') + r.recv() + s.close() + + with self.assertRaises(interpreters.ChannelClosedError): + s.send(b'eggs') + with self.assertRaises(interpreters.ChannelClosedError): + r.recv() + + def test_close_multiple_times(self): + r, s = interpreters.create_channel() + s.send(b'spam') + r.recv() + s.close() + + with self.assertRaises(interpreters.ChannelClosedError): + s.close() + + def test_close_empty(self): + tests = [ + (False, False), + (True, False), + (False, True), + (True, True), + ] + for send, recv in tests: + with self.subTest((send, recv)): + r, s = interpreters.create_channel() + s.send(b'spam') + r.recv() + s.close() + + with self.assertRaises(interpreters.ChannelClosedError): + s.send(b'eggs') + with self.assertRaises(interpreters.ChannelClosedError): + r.recv() + + def test_close_defaults_with_unused_items(self): + r, s = interpreters.create_channel() + s.send(b'spam') + s.send(b'ham') + + with self.assertRaises(interpreters.ChannelNotEmptyError): + s.close() + r.recv() + s.send(b'eggs') + + def test_close_never_used(self): + r, s = interpreters.create_channel() + r.close() + + with self.assertRaises(interpreters.ChannelClosedError): + s.send(b'spam') + with self.assertRaises(interpreters.ChannelClosedError): + r.recv() + + def test_close_used_multiple_times_by_single_user(self): + r, s = interpreters.create_channel() + s.send(b'spam') + s.send(b'spam') + s.send(b'spam') + r.recv() + s.close(force=True) + + with self.assertRaises(interpreters.ChannelClosedError): + s.send(b'eggs') + with self.assertRaises(interpreters.ChannelClosedError): + r.recv() + + # release + + def test_single_user(self): + r, s = interpreters.create_channel() + s.send(b'spam') + r.recv() + s.release() + + with self.assertRaises(interpreters.ChannelClosedError): + s.send(b'eggs') + + def test_with_unused_items(self): + r, s = interpreters.create_channel() + s.send(b'spam') + s.send(b'ham') + s.release() + + with self.assertRaises(interpreters.ChannelClosedError): + r.recv() + + def test_never_used(self): + r, s = interpreters.create_channel() + s.release() + + with self.assertRaises(interpreters.ChannelClosedError): + s.send(b'spam') + with self.assertRaises(interpreters.ChannelClosedError): + r.recv() + +class TestSendBuffer(TestBase): + def test_send_recv_main(self): + r, s = interpreters.create_channel() + orig = b'spam' + s.send_buffer(orig) + obj = r.recv() + + self.assertEqual(obj, orig) + self.assertIsNot(obj, orig) + + def test_send_recv_same_interpreter(self): + interp = interpreters.create() + out = _run_output(interp, dedent(""" + import interpreters + r, s = interpreters.create_channel() + orig = b'spam' + s.send_buffer(orig) + obj = r.recv() + assert obj is not orig + assert obj == orig + """)) + + def test_send_recv_different_threads(self): + r, s = interpreters.create_channel() + + def f(): + while True: + try: + obj = r.recv() + break + except interpreters.ChannelEmptyError: + time.sleep(0.1) + s.send_buffer(obj) + t = threading.Thread(target=f) + t.start() + + s.send_buffer(b'spam') + t.join() + obj = r.recv() + + self.assertEqual(obj, b'spam') diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 894e0641aa48ca..008517c0a1d407 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -286,6 +286,8 @@ static PyObject *ChannelNotFoundError; static PyObject *ChannelClosedError; static PyObject *ChannelEmptyError; static PyObject *ChannelNotEmptyError; +static PyObject *ChannelReleasedError; +static PyObject *NotReceivedError; static int channel_exceptions_init(PyObject *ns) @@ -322,6 +324,27 @@ channel_exceptions_init(PyObject *ns) return -1; } + // An operation tried to use a released channel. + ChannelReleasedError = PyErr_NewException( + "_interpreters.ChannelReleasedError", ChannelClosedError, NULL); + if (ChannelReleasedError == NULL) { + return -1; + } + if (PyDict_SetItemString(ns, "ChannelReleasedError", ChannelReleasedError) != 0) { + return -1; + } + + // An operation trying to send an object when Nothing was waiting + // to receive it + NotReceivedError = PyErr_NewException( + "_interpreters.NotReceivedError", ChannelError, NULL); + if (NotReceivedError == NULL) { + return -1; + } + if (PyDict_SetItemString(ns, "NotReceivedError", NotReceivedError) != 0) { + return -1; + } + // An operation tried to pop from an empty channel. ChannelEmptyError = PyErr_NewException( "_interpreters.ChannelEmptyError", ChannelError, NULL); @@ -484,6 +507,7 @@ typedef struct _channelend { struct _channelend *next; int64_t interp; int open; + int release; } _channelend; static _channelend * @@ -618,6 +642,10 @@ _channelends_associate(_channelends *ends, int64_t interp, int send) PyErr_SetString(ChannelClosedError, "channel already closed"); return -1; } + if (end->release && !end->open) { + PyErr_SetString(ChannelReleasedError, "channel released"); + return -1; + } // already associated return 0; } @@ -732,6 +760,7 @@ typedef struct _channel { _channelqueue *queue; _channelends *ends; int open; + int release; struct _channel_closing *closing; } _PyChannelState; @@ -789,6 +818,10 @@ _channel_add(_PyChannelState *chan, int64_t interp, PyErr_SetString(ChannelClosedError, "channel closed"); goto done; } + if (chan->release && !chan->open) { + PyErr_SetString(ChannelReleasedError, "channel released"); + return -1; + } if (_channelends_associate(chan->ends, interp, 1) != 0) { goto done; } @@ -813,6 +846,10 @@ _channel_next(_PyChannelState *chan, int64_t interp) PyErr_SetString(ChannelClosedError, "channel closed"); goto done; } + if (chan->release && !chan->open) { + PyErr_SetString(ChannelReleasedError, "channel released"); + goto done; + } if (_channelends_associate(chan->ends, interp, 0) != 0) { goto done; } @@ -1306,21 +1343,24 @@ _channel_destroy(_channels *channels, int64_t id) static int _channel_send_buffer(_channels *channels, int64_t id, PyObject *obj) { - const char *s = NULL; - Py_buffer view = {NULL, NULL}; - if (PyObject_GetBuffer(obj, &view, PyBUF_SIMPLE) != 0){ + Py_buffer buffer; + PyObject *bytes; + + if (PyObject_GetBuffer(obj, &buffer, PyBUF_SIMPLE) < 0) { + PyErr_Format(PyExc_TypeError, + "Error creating object buffer, %.80s found", + Py_TYPE(obj)->tp_name); return -1; } - s = view.buf; - if (s == NULL) { - PyBuffer_Release(&view); + if (buffer.len == 0) { + PyBuffer_Release(&buffer); return -1; } PyInterpreterState *interp = _get_current(); if (interp == NULL) { - PyBuffer_Release(&view); + PyBuffer_Release(&buffer); return -1; } @@ -1328,7 +1368,7 @@ _channel_send_buffer(_channels *channels, int64_t id, PyObject *obj) PyThread_type_lock mutex = NULL; _PyChannelState *chan = _channels_lookup(channels, id, &mutex); if (chan == NULL) { - PyBuffer_Release(&view); + PyBuffer_Release(&buffer); return -1; } // Past this point we are responsible for releasing the mutex. @@ -1336,7 +1376,7 @@ _channel_send_buffer(_channels *channels, int64_t id, PyObject *obj) if (chan->closing != NULL) { PyErr_Format(ChannelClosedError, "channel %" PRId64 " closed", id); PyThread_release_lock(mutex); - PyBuffer_Release(&view); + PyBuffer_Release(&buffer); return -1; } @@ -1344,13 +1384,21 @@ _channel_send_buffer(_channels *channels, int64_t id, PyObject *obj) _PyCrossInterpreterData *data = PyMem_NEW(_PyCrossInterpreterData, 1); if (data == NULL) { PyThread_release_lock(mutex); - PyBuffer_Release(&view); + PyBuffer_Release(&buffer); return -1; } - if (_PyObject_GetCrossInterpreterData((PyObject *)s, data) != 0) { + + if (buffer.buf != NULL) + bytes = PyBytes_FromStringAndSize(buffer.buf, buffer.len); + else { + Py_INCREF(Py_None); + bytes = Py_None; + } + + if (_PyObject_GetCrossInterpreterData(bytes, data) != 0) { PyThread_release_lock(mutex); PyMem_Free(data); - PyBuffer_Release(&view); + PyBuffer_Release(&buffer); return -1; } @@ -1360,11 +1408,12 @@ _channel_send_buffer(_channels *channels, int64_t id, PyObject *obj) if (res != 0) { _PyCrossInterpreterData_Release(data); PyMem_Free(data); - PyBuffer_Release(&view); + PyBuffer_Release(&buffer); return -1; } - PyBuffer_Release(&view); + PyBuffer_Release(&buffer); + return 0; } @@ -1467,6 +1516,13 @@ _channel_drop(_channels *channels, int64_t id, int send, int recv) } // Past this point we are responsible for releasing the mutex. + // Release the channel + if (!chan->release) { + PyErr_SetString(ChannelClosedError, "channel already released"); + return -1; + } + chan->release = 1; + // Close one or both of the two ends. int res = _channel_close_interpreter(chan, PyInterpreterState_GetID(interp), send-recv); PyThread_release_lock(mutex); From 9e2f9d6d6986a7628087064fdbbe8d25d29fc297 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Fri, 6 Mar 2020 22:09:07 +0000 Subject: [PATCH 18/20] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Core and Builtins/2020-03-06-22-09-03.bpo-39881.Wh2TTV.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-03-06-22-09-03.bpo-39881.Wh2TTV.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-03-06-22-09-03.bpo-39881.Wh2TTV.rst b/Misc/NEWS.d/next/Core and Builtins/2020-03-06-22-09-03.bpo-39881.Wh2TTV.rst new file mode 100644 index 00000000000000..96cf52e5fc191b --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-03-06-22-09-03.bpo-39881.Wh2TTV.rst @@ -0,0 +1,2 @@ +High-level Implementation of PEP 554. +(Patch By Joannah Nanjekye) \ No newline at end of file From 71ad4d7e5c116e004843cefec1ee03a29686a94c Mon Sep 17 00:00:00 2001 From: Joannah Nanjekye <33177550+nanjekyejoannah@users.noreply.github.com> Date: Thu, 7 May 2020 18:01:36 -0300 Subject: [PATCH 19/20] Update Doc/library/interpreters.rst Co-authored-by: Kyle Stanley --- Doc/library/interpreters.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/interpreters.rst b/Doc/library/interpreters.rst index 6b9c35b6250b89..df314e019dce28 100644 --- a/Doc/library/interpreters.rst +++ b/Doc/library/interpreters.rst @@ -8,7 +8,7 @@ -------------- -This module provides highlevel tools for working with sub-interpreters, +This module provides high-level tools for working with sub-interpreters, such as creating them, running code in them, or sending data between them. It is a wrapper around the low-level :mod:`_interpreters` module. From b12e3d2bca61955895269ac0ef6b109d1c3e0b84 Mon Sep 17 00:00:00 2001 From: Joannah Nanjekye <33177550+nanjekyejoannah@users.noreply.github.com> Date: Thu, 7 May 2020 18:02:09 -0300 Subject: [PATCH 20/20] Update Doc/library/interpreters.rst Co-authored-by: Kyle Stanley --- Doc/library/interpreters.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/interpreters.rst b/Doc/library/interpreters.rst index df314e019dce28..4c12f343033a4a 100644 --- a/Doc/library/interpreters.rst +++ b/Doc/library/interpreters.rst @@ -10,7 +10,7 @@ This module provides high-level tools for working with sub-interpreters, such as creating them, running code in them, or sending data between them. -It is a wrapper around the low-level :mod:`_interpreters` module. +It is a wrapper around the low-level `_interpreters` module. .. versionchanged:: added in 3.9