From 7b4474861a469f3e66e12d3c28f09ae18e17fa5a Mon Sep 17 00:00:00 2001 From: Vishal Kuo Date: Fri, 26 Jun 2020 16:15:40 -0700 Subject: [PATCH 1/6] [multiprocessing] fixes test cases --- stdlib/3/multiprocessing/__init__.pyi | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/stdlib/3/multiprocessing/__init__.pyi b/stdlib/3/multiprocessing/__init__.pyi index fb8021ffe6d8..6a219980aa91 100644 --- a/stdlib/3/multiprocessing/__init__.pyi +++ b/stdlib/3/multiprocessing/__init__.pyi @@ -50,7 +50,7 @@ def Pool(processes: Optional[int] = ..., initargs: Iterable[Any] = ..., maxtasksperchild: Optional[int] = ...) -> pool.Pool: ... -class Array: +class _Array: value: Any = ... def __init__(self, typecode_or_type: Union[str, Type[_CData]], size_or_initializer: Union[int, Sequence[Any]], *, lock: Union[bool, _LockLike] = ...) -> None: ... @@ -66,8 +66,14 @@ class Array: def __getslice__(self, start: int, stop: int) -> Any: ... def __setitem__(self, key: int, value: Any) -> None: ... +def Array( + typecode_or_type: Union[str, Type[_CData]], + size_or_initializer: Union[int, Sequence[Any]], + *, + lock: Union[bool, _LockLike] = ...) -> _Array: ... -class Value(): + +class _Value(): value: Any = ... def __init__(self, typecode_or_type: Union[str, Type[_CData]], *args: Any, lock: Union[bool, _LockLike] = ...) -> None: ... def get_lock(self) -> _LockLike: ... @@ -75,6 +81,8 @@ class Value(): def acquire(self) -> bool: ... def release(self) -> bool: ... +def Value(typecode_or_type: Union[str, Type[_CData]], *args: Any, lock: Union[bool, _LockLike] = ...) -> _Value:... + # ----- multiprocessing function stubs ----- def allow_connection_pickling() -> None: ... def cpu_count() -> int: ... From 410e59b31a02976792346d7682a53066e6f60c13 Mon Sep 17 00:00:00 2001 From: Vishal Kuo Date: Fri, 26 Jun 2020 17:01:01 -0700 Subject: [PATCH 2/6] fix lint errors --- stdlib/3/multiprocessing/__init__.pyi | 2 +- stdlib/3/multiprocessing/sharedctypes.pyi | 0 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 stdlib/3/multiprocessing/sharedctypes.pyi diff --git a/stdlib/3/multiprocessing/__init__.pyi b/stdlib/3/multiprocessing/__init__.pyi index 6a219980aa91..ff2af0cb5e7e 100644 --- a/stdlib/3/multiprocessing/__init__.pyi +++ b/stdlib/3/multiprocessing/__init__.pyi @@ -81,7 +81,7 @@ class _Value(): def acquire(self) -> bool: ... def release(self) -> bool: ... -def Value(typecode_or_type: Union[str, Type[_CData]], *args: Any, lock: Union[bool, _LockLike] = ...) -> _Value:... +def Value(typecode_or_type: Union[str, Type[_CData]], *args: Any, lock: Union[bool, _LockLike] = ...) -> _Value: ... # ----- multiprocessing function stubs ----- def allow_connection_pickling() -> None: ... diff --git a/stdlib/3/multiprocessing/sharedctypes.pyi b/stdlib/3/multiprocessing/sharedctypes.pyi new file mode 100644 index 000000000000..e69de29bb2d1 From 7fb5cdb9f5b4dfb9fa7aa69bf007eaa969b378a9 Mon Sep 17 00:00:00 2001 From: Vishal Kuo Date: Fri, 26 Jun 2020 18:20:47 -0700 Subject: [PATCH 3/6] code review feedback - add shared ctypes --- stdlib/3/multiprocessing/__init__.pyi | 37 ++------------------ stdlib/3/multiprocessing/context.pyi | 5 +-- stdlib/3/multiprocessing/sharedctypes.pyi | 42 +++++++++++++++++++++++ 3 files changed, 48 insertions(+), 36 deletions(-) diff --git a/stdlib/3/multiprocessing/__init__.pyi b/stdlib/3/multiprocessing/__init__.pyi index ff2af0cb5e7e..d3b81fa01bdb 100644 --- a/stdlib/3/multiprocessing/__init__.pyi +++ b/stdlib/3/multiprocessing/__init__.pyi @@ -26,7 +26,6 @@ if sys.version_info >= (3, 8): if sys.platform != "win32": from multiprocessing.context import ForkContext, ForkServerContext - # N.B. The functions below are generated at runtime by partially applying # multiprocessing.context.BaseContext's methods, so the two signatures should # be identical (modulo self). @@ -50,39 +49,6 @@ def Pool(processes: Optional[int] = ..., initargs: Iterable[Any] = ..., maxtasksperchild: Optional[int] = ...) -> pool.Pool: ... -class _Array: - value: Any = ... - - def __init__(self, typecode_or_type: Union[str, Type[_CData]], size_or_initializer: Union[int, Sequence[Any]], *, lock: Union[bool, _LockLike] = ...) -> None: ... - def acquire(self) -> bool: ... - def release(self) -> bool: ... - def get_lock(self) -> _LockLike: ... - def get_obj(self) -> Any: ... - - @overload - def __getitem__(self, key: int) -> Any: ... - @overload - def __getitem__(self, key: slice) -> List[Any]: ... - def __getslice__(self, start: int, stop: int) -> Any: ... - def __setitem__(self, key: int, value: Any) -> None: ... - -def Array( - typecode_or_type: Union[str, Type[_CData]], - size_or_initializer: Union[int, Sequence[Any]], - *, - lock: Union[bool, _LockLike] = ...) -> _Array: ... - - -class _Value(): - value: Any = ... - def __init__(self, typecode_or_type: Union[str, Type[_CData]], *args: Any, lock: Union[bool, _LockLike] = ...) -> None: ... - def get_lock(self) -> _LockLike: ... - def get_obj(self) -> Any: ... - def acquire(self) -> bool: ... - def release(self) -> bool: ... - -def Value(typecode_or_type: Union[str, Type[_CData]], *args: Any, lock: Union[bool, _LockLike] = ...) -> _Value: ... - # ----- multiprocessing function stubs ----- def allow_connection_pickling() -> None: ... def cpu_count() -> int: ... @@ -94,6 +60,9 @@ def get_all_start_methods() -> List[str]: ... def get_start_method(allow_none: bool = ...) -> Optional[str]: ... def set_start_method(method: str, force: Optional[bool] = ...) -> None: ... +_default_context: DefaultContext +Array = DefaultContext.Array +Value = DefaultContext.Value if sys.platform != "win32": @overload diff --git a/stdlib/3/multiprocessing/context.pyi b/stdlib/3/multiprocessing/context.pyi index ced1d26ebae2..4c1264170efe 100644 --- a/stdlib/3/multiprocessing/context.pyi +++ b/stdlib/3/multiprocessing/context.pyi @@ -1,5 +1,6 @@ from logging import Logger import multiprocessing +from multiprocessing import sharedctypes from multiprocessing import synchronize from multiprocessing import queues from multiprocessing.process import BaseProcess @@ -79,7 +80,7 @@ class BaseContext(object): typecode_or_type: Any, *args: Any, lock: bool = ... - ) -> Any: ... + ) -> sharedctypes._Value: ... # TODO: typecode_or_type param is a ctype with a base class of _SimpleCData or array.typecode Need to figure out # how to handle the ctype # TODO: change return to Array once a stub exists in multiprocessing.sharedctypes @@ -89,7 +90,7 @@ class BaseContext(object): size_or_initializer: Union[int, Sequence[Any]], *, lock: bool = ... - ) -> Any: ... + ) -> sharedctypes._Array: ... def freeze_support(self) -> None: ... def get_logger(self) -> Logger: ... def log_to_stderr(self, level: Optional[str] = ...) -> Logger: ... diff --git a/stdlib/3/multiprocessing/sharedctypes.pyi b/stdlib/3/multiprocessing/sharedctypes.pyi index e69de29bb2d1..4eea77b3ce94 100644 --- a/stdlib/3/multiprocessing/sharedctypes.pyi +++ b/stdlib/3/multiprocessing/sharedctypes.pyi @@ -0,0 +1,42 @@ +from ctypes import _CData +from typing import Any, List, Optional, Sequence, Type, Union, overload + +from multiprocessing.synchronize import _LockLike +from multiprocessing.context import BaseContext + +class _Array: + value: Any = ... + + def __init__(self, typecode_or_type: Union[str, Type[_CData]], size_or_initializer: Union[int, Sequence[Any]], *, lock: Union[bool, _LockLike] = ...) -> None: ... + def acquire(self) -> bool: ... + def release(self) -> bool: ... + def get_lock(self) -> _LockLike: ... + def get_obj(self) -> Any: ... + + @overload + def __getitem__(self, key: int) -> Any: ... + @overload + def __getitem__(self, key: slice) -> List[Any]: ... + def __getslice__(self, start: int, stop: int) -> Any: ... + def __setitem__(self, key: int, value: Any) -> None: ... + +class _Value(): + value: Any = ... + def __init__(self, typecode_or_type: Union[str, Type[_CData]], *args: Any, lock: Union[bool, _LockLike] = ...) -> None: ... + def get_lock(self) -> _LockLike: ... + def get_obj(self) -> Any: ... + def acquire(self) -> bool: ... + def release(self) -> bool: ... + +def Array( + typecode_or_type: Union[str, Type[_CData]], + size_or_initializer: Union[int, Sequence[Any]], + *, + lock: Union[bool, _LockLike] = ..., + ctx = ...) -> _Array: ... + +def Value( + typecode_or_type: Union[str, Type[_CData]], + *args: Any, + lock: Union[bool, _LockLike] = ..., + ctx = ...) -> _Value: ... From b30d8afa02b83bd29614d6decf5c6cf6a8d7b1ad Mon Sep 17 00:00:00 2001 From: Vishal Kuo Date: Fri, 26 Jun 2020 18:42:52 -0700 Subject: [PATCH 4/6] types with base context --- stdlib/3/multiprocessing/sharedctypes.pyi | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stdlib/3/multiprocessing/sharedctypes.pyi b/stdlib/3/multiprocessing/sharedctypes.pyi index 4eea77b3ce94..86aea145eaec 100644 --- a/stdlib/3/multiprocessing/sharedctypes.pyi +++ b/stdlib/3/multiprocessing/sharedctypes.pyi @@ -33,10 +33,10 @@ def Array( size_or_initializer: Union[int, Sequence[Any]], *, lock: Union[bool, _LockLike] = ..., - ctx = ...) -> _Array: ... + ctx: Optional[BaseContext] = ...) -> _Array: ... def Value( typecode_or_type: Union[str, Type[_CData]], *args: Any, lock: Union[bool, _LockLike] = ..., - ctx = ...) -> _Value: ... + ctx: Optional[BaseContext] = ...) -> _Value: ... From 7a60063b8041a885d0d5a9cca4b355aa6579fb18 Mon Sep 17 00:00:00 2001 From: Vishal Kuo Date: Fri, 26 Jun 2020 19:00:31 -0700 Subject: [PATCH 5/6] tries to fix tests --- stdlib/3/multiprocessing/__init__.pyi | 6 +++--- stdlib/3/multiprocessing/context.pyi | 4 ++-- stdlib/3/multiprocessing/sharedctypes.pyi | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/stdlib/3/multiprocessing/__init__.pyi b/stdlib/3/multiprocessing/__init__.pyi index d3b81fa01bdb..2ec0b265ce6e 100644 --- a/stdlib/3/multiprocessing/__init__.pyi +++ b/stdlib/3/multiprocessing/__init__.pyi @@ -13,6 +13,7 @@ from multiprocessing.context import ( ProcessError as ProcessError, SpawnContext, TimeoutError as TimeoutError, + _default_context, ) from multiprocessing.managers import SyncManager from multiprocessing.process import active_children as active_children, current_process as current_process @@ -60,9 +61,8 @@ def get_all_start_methods() -> List[str]: ... def get_start_method(allow_none: bool = ...) -> Optional[str]: ... def set_start_method(method: str, force: Optional[bool] = ...) -> None: ... -_default_context: DefaultContext -Array = DefaultContext.Array -Value = DefaultContext.Value +Array = _default_context.Array +Value = _default_context.Value if sys.platform != "win32": @overload diff --git a/stdlib/3/multiprocessing/context.pyi b/stdlib/3/multiprocessing/context.pyi index 4c1264170efe..c27c13b99873 100644 --- a/stdlib/3/multiprocessing/context.pyi +++ b/stdlib/3/multiprocessing/context.pyi @@ -74,7 +74,6 @@ class BaseContext(object): def RawArray(self, typecode_or_type: Any, size_or_initializer: Union[int, Sequence[Any]]) -> Any: ... # TODO: typecode_or_type param is a ctype with a base class of _SimpleCData or array.typecode Need to figure out # how to handle the ctype - # TODO: change return to Value once a stub exists in multiprocessing.sharedctypes def Value( self, typecode_or_type: Any, @@ -83,7 +82,6 @@ class BaseContext(object): ) -> sharedctypes._Value: ... # TODO: typecode_or_type param is a ctype with a base class of _SimpleCData or array.typecode Need to figure out # how to handle the ctype - # TODO: change return to Array once a stub exists in multiprocessing.sharedctypes def Array( self, typecode_or_type: Any, @@ -179,3 +177,5 @@ def _force_start_method(method: str) -> None: ... def get_spawning_popen() -> Optional[Any]: ... def set_spawning_popen(popen: Any) -> None: ... def assert_spawning(obj: Any) -> None: ... + +_default_context: DefaultContext diff --git a/stdlib/3/multiprocessing/sharedctypes.pyi b/stdlib/3/multiprocessing/sharedctypes.pyi index 86aea145eaec..959cae5d12f2 100644 --- a/stdlib/3/multiprocessing/sharedctypes.pyi +++ b/stdlib/3/multiprocessing/sharedctypes.pyi @@ -1,4 +1,4 @@ -from ctypes import _CData +from ctypes import _CData from typing import Any, List, Optional, Sequence, Type, Union, overload from multiprocessing.synchronize import _LockLike From 8e416c76074f4b3e7dd04c888dbae2a6f6dd2dbc Mon Sep 17 00:00:00 2001 From: Vishal Kuo Date: Fri, 26 Jun 2020 19:59:37 -0700 Subject: [PATCH 6/6] duplicate function to workaround pytype --- stdlib/3/multiprocessing/__init__.pyi | 21 +++++++++++++++++---- stdlib/3/multiprocessing/context.pyi | 2 -- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/stdlib/3/multiprocessing/__init__.pyi b/stdlib/3/multiprocessing/__init__.pyi index 2ec0b265ce6e..f8d8f81a0e63 100644 --- a/stdlib/3/multiprocessing/__init__.pyi +++ b/stdlib/3/multiprocessing/__init__.pyi @@ -3,7 +3,7 @@ from typing import Any, Callable, Iterable, Optional, List, Union, Sequence, Tup from typing_extensions import Literal from ctypes import _CData from logging import Logger -from multiprocessing import connection, pool, spawn, synchronize +from multiprocessing import connection, pool, sharedctypes, spawn, synchronize from multiprocessing.context import ( AuthenticationError as AuthenticationError, BaseContext, @@ -13,7 +13,6 @@ from multiprocessing.context import ( ProcessError as ProcessError, SpawnContext, TimeoutError as TimeoutError, - _default_context, ) from multiprocessing.managers import SyncManager from multiprocessing.process import active_children as active_children, current_process as current_process @@ -50,6 +49,22 @@ def Pool(processes: Optional[int] = ..., initargs: Iterable[Any] = ..., maxtasksperchild: Optional[int] = ...) -> pool.Pool: ... +# Functions Array and Value are copied from context.pyi. +# See https://github.com/python/typeshed/blob/ac234f25927634e06d9c96df98d72d54dd80dfc4/stdlib/2and3/turtle.pyi#L284-L291 +# for rationale +def Array( + typecode_or_type: Any, + size_or_initializer: Union[int, Sequence[Any]], + *, + lock: bool = ... +) -> sharedctypes._Array: ... + +def Value( + typecode_or_type: Any, + *args: Any, + lock: bool = ... +) -> sharedctypes._Value: ... + # ----- multiprocessing function stubs ----- def allow_connection_pickling() -> None: ... def cpu_count() -> int: ... @@ -61,8 +76,6 @@ def get_all_start_methods() -> List[str]: ... def get_start_method(allow_none: bool = ...) -> Optional[str]: ... def set_start_method(method: str, force: Optional[bool] = ...) -> None: ... -Array = _default_context.Array -Value = _default_context.Value if sys.platform != "win32": @overload diff --git a/stdlib/3/multiprocessing/context.pyi b/stdlib/3/multiprocessing/context.pyi index c27c13b99873..9d391f922879 100644 --- a/stdlib/3/multiprocessing/context.pyi +++ b/stdlib/3/multiprocessing/context.pyi @@ -177,5 +177,3 @@ def _force_start_method(method: str) -> None: ... def get_spawning_popen() -> Optional[Any]: ... def set_spawning_popen(popen: Any) -> None: ... def assert_spawning(obj: Any) -> None: ... - -_default_context: DefaultContext