From 549fcdb8fa03216ef0919c074e94225bff84e1e3 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Mon, 18 Oct 2021 16:54:54 +0200 Subject: [PATCH 01/12] bpo-45459: Add Py_buffer to limited API --- Misc/NEWS.d/next/C API/2021-10-18-16-54-24.bpo-45459.Y1pEZs.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/C API/2021-10-18-16-54-24.bpo-45459.Y1pEZs.rst diff --git a/Misc/NEWS.d/next/C API/2021-10-18-16-54-24.bpo-45459.Y1pEZs.rst b/Misc/NEWS.d/next/C API/2021-10-18-16-54-24.bpo-45459.Y1pEZs.rst new file mode 100644 index 00000000000000..2da4d93f9bdea0 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2021-10-18-16-54-24.bpo-45459.Y1pEZs.rst @@ -0,0 +1 @@ +Add :c:type:`Py_buffer` to limited API / stable ABI. From 526c58c61a574c30b42514a662176464faf1c9d1 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Mon, 18 Oct 2021 16:39:59 +0200 Subject: [PATCH 02/12] Move Py_buffer APIs to buffer.h --- Include/Python.h | 1 + Include/buffer.h | 120 +++++++++++++++++++++++++++++++++++++ Include/cpython/abstract.h | 68 --------------------- Include/cpython/object.h | 34 +---------- Include/typeslots.h | 6 -- Makefile.pre.in | 1 + 6 files changed, 123 insertions(+), 107 deletions(-) create mode 100644 Include/buffer.h diff --git a/Include/Python.h b/Include/Python.h index 7260ae5cd0b4fe..5416b04e4bfb32 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -50,6 +50,7 @@ #include "longobject.h" #include "cpython/longintrepr.h" #include "boolobject.h" +#include "buffer.h" #include "floatobject.h" #include "complexobject.h" #include "rangeobject.h" diff --git a/Include/buffer.h b/Include/buffer.h new file mode 100644 index 00000000000000..59eef777f03c49 --- /dev/null +++ b/Include/buffer.h @@ -0,0 +1,120 @@ +/* Public Py_buffer API */ + +#ifndef Py_BUFFER_H +#define Py_BUFFER_H +#ifdef __cplusplus +extern "C" { +#endif + +/* === New Buffer API ============================================ + * Limited API since Python 3.11 + */ + +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030b0000 + +typedef struct bufferinfo Py_buffer; + +/* Return 1 if the getbuffer function is available, otherwise return 0. */ +PyAPI_FUNC(int) PyObject_CheckBuffer(PyObject *obj); + +/* This is a C-API version of the getbuffer function call. It checks + to make sure object has the required function pointer and issues the + call. + + Returns -1 and raises an error on failure and returns 0 on success. */ +PyAPI_FUNC(int) PyObject_GetBuffer(PyObject *obj, Py_buffer *view, + int flags); + +/* Get the memory area pointed to by the indices for the buffer given. + Note that view->ndim is the assumed size of indices. */ +PyAPI_FUNC(void *) PyBuffer_GetPointer(const Py_buffer *view, const Py_ssize_t *indices); + +/* Return the implied itemsize of the data-format area from a + struct-style description. */ +PyAPI_FUNC(Py_ssize_t) PyBuffer_SizeFromFormat(const char *format); + +/* Implementation in memoryobject.c */ +PyAPI_FUNC(int) PyBuffer_ToContiguous(void *buf, const Py_buffer *view, + Py_ssize_t len, char order); + +PyAPI_FUNC(int) PyBuffer_FromContiguous(const Py_buffer *view, const void *buf, + Py_ssize_t len, char order); + +/* Copy len bytes of data from the contiguous chunk of memory + pointed to by buf into the buffer exported by obj. Return + 0 on success and return -1 and raise a PyBuffer_Error on + error (i.e. the object does not have a buffer interface or + it is not working). + + If fort is 'F', then if the object is multi-dimensional, + then the data will be copied into the array in + Fortran-style (first dimension varies the fastest). If + fort is 'C', then the data will be copied into the array + in C-style (last dimension varies the fastest). If fort + is 'A', then it does not matter and the copy will be made + in whatever way is more efficient. */ +PyAPI_FUNC(int) PyObject_CopyData(PyObject *dest, PyObject *src); + +/* Copy the data from the src buffer to the buffer of destination. */ +PyAPI_FUNC(int) PyBuffer_IsContiguous(const Py_buffer *view, char fort); + +/*Fill the strides array with byte-strides of a contiguous + (Fortran-style if fort is 'F' or C-style otherwise) + array of the given shape with the given number of bytes + per element. */ +PyAPI_FUNC(void) PyBuffer_FillContiguousStrides(int ndims, + Py_ssize_t *shape, + Py_ssize_t *strides, + int itemsize, + char fort); + +/* Fills in a buffer-info structure correctly for an exporter + that can only share a contiguous chunk of memory of + "unsigned bytes" of the given length. + + Returns 0 on success and -1 (with raising an error) on error. */ +PyAPI_FUNC(int) PyBuffer_FillInfo(Py_buffer *view, PyObject *o, void *buf, + Py_ssize_t len, int readonly, + int flags); + +/* Releases a Py_buffer obtained from getbuffer ParseTuple's "s*". */ +PyAPI_FUNC(void) PyBuffer_Release(Py_buffer *view); + +/* Maximum number of dimensions */ +#define PyBUF_MAX_NDIM 64 + +/* Flags for getting buffers */ +#define PyBUF_SIMPLE 0 +#define PyBUF_WRITABLE 0x0001 +/* we used to include an E, backwards compatible alias */ +#define PyBUF_WRITEABLE PyBUF_WRITABLE +#define PyBUF_FORMAT 0x0004 +#define PyBUF_ND 0x0008 +#define PyBUF_STRIDES (0x0010 | PyBUF_ND) +#define PyBUF_C_CONTIGUOUS (0x0020 | PyBUF_STRIDES) +#define PyBUF_F_CONTIGUOUS (0x0040 | PyBUF_STRIDES) +#define PyBUF_ANY_CONTIGUOUS (0x0080 | PyBUF_STRIDES) +#define PyBUF_INDIRECT (0x0100 | PyBUF_STRIDES) + +#define PyBUF_CONTIG (PyBUF_ND | PyBUF_WRITABLE) +#define PyBUF_CONTIG_RO (PyBUF_ND) + +#define PyBUF_STRIDED (PyBUF_STRIDES | PyBUF_WRITABLE) +#define PyBUF_STRIDED_RO (PyBUF_STRIDES) + +#define PyBUF_RECORDS (PyBUF_STRIDES | PyBUF_WRITABLE | PyBUF_FORMAT) +#define PyBUF_RECORDS_RO (PyBUF_STRIDES | PyBUF_FORMAT) + +#define PyBUF_FULL (PyBUF_INDIRECT | PyBUF_WRITABLE | PyBUF_FORMAT) +#define PyBUF_FULL_RO (PyBUF_INDIRECT | PyBUF_FORMAT) + + +#define PyBUF_READ 0x100 +#define PyBUF_WRITE 0x200 + +#endif /* Py_LIMITED_API */ + +#ifdef __cplusplus +} +#endif +#endif /* Py_BUFFER_H */ \ No newline at end of file diff --git a/Include/cpython/abstract.h b/Include/cpython/abstract.h index 2876a7bb84f52c..b5a31392f2ed0e 100644 --- a/Include/cpython/abstract.h +++ b/Include/cpython/abstract.h @@ -168,74 +168,6 @@ PyAPI_FUNC(int) _PyObject_HasLen(PyObject *o); value. If one of the calls fails, this function returns -1. */ PyAPI_FUNC(Py_ssize_t) PyObject_LengthHint(PyObject *o, Py_ssize_t); -/* === New Buffer API ============================================ */ - -/* Return 1 if the getbuffer function is available, otherwise return 0. */ -PyAPI_FUNC(int) PyObject_CheckBuffer(PyObject *obj); - -/* This is a C-API version of the getbuffer function call. It checks - to make sure object has the required function pointer and issues the - call. - - Returns -1 and raises an error on failure and returns 0 on success. */ -PyAPI_FUNC(int) PyObject_GetBuffer(PyObject *obj, Py_buffer *view, - int flags); - -/* Get the memory area pointed to by the indices for the buffer given. - Note that view->ndim is the assumed size of indices. */ -PyAPI_FUNC(void *) PyBuffer_GetPointer(const Py_buffer *view, const Py_ssize_t *indices); - -/* Return the implied itemsize of the data-format area from a - struct-style description. */ -PyAPI_FUNC(Py_ssize_t) PyBuffer_SizeFromFormat(const char *format); - -/* Implementation in memoryobject.c */ -PyAPI_FUNC(int) PyBuffer_ToContiguous(void *buf, const Py_buffer *view, - Py_ssize_t len, char order); - -PyAPI_FUNC(int) PyBuffer_FromContiguous(const Py_buffer *view, const void *buf, - Py_ssize_t len, char order); - -/* Copy len bytes of data from the contiguous chunk of memory - pointed to by buf into the buffer exported by obj. Return - 0 on success and return -1 and raise a PyBuffer_Error on - error (i.e. the object does not have a buffer interface or - it is not working). - - If fort is 'F', then if the object is multi-dimensional, - then the data will be copied into the array in - Fortran-style (first dimension varies the fastest). If - fort is 'C', then the data will be copied into the array - in C-style (last dimension varies the fastest). If fort - is 'A', then it does not matter and the copy will be made - in whatever way is more efficient. */ -PyAPI_FUNC(int) PyObject_CopyData(PyObject *dest, PyObject *src); - -/* Copy the data from the src buffer to the buffer of destination. */ -PyAPI_FUNC(int) PyBuffer_IsContiguous(const Py_buffer *view, char fort); - -/*Fill the strides array with byte-strides of a contiguous - (Fortran-style if fort is 'F' or C-style otherwise) - array of the given shape with the given number of bytes - per element. */ -PyAPI_FUNC(void) PyBuffer_FillContiguousStrides(int ndims, - Py_ssize_t *shape, - Py_ssize_t *strides, - int itemsize, - char fort); - -/* Fills in a buffer-info structure correctly for an exporter - that can only share a contiguous chunk of memory of - "unsigned bytes" of the given length. - - Returns 0 on success and -1 (with raising an error) on error. */ -PyAPI_FUNC(int) PyBuffer_FillInfo(Py_buffer *view, PyObject *o, void *buf, - Py_ssize_t len, int readonly, - int flags); - -/* Releases a Py_buffer obtained from getbuffer ParseTuple's "s*". */ -PyAPI_FUNC(void) PyBuffer_Release(Py_buffer *view); - /* === Sequence protocol ================================================ */ /* Assume tp_as_sequence and sq_item exist and that 'i' does not diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 7b9f3acbc439d6..ac4f62b499b632 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -63,43 +63,11 @@ typedef struct bufferinfo { typedef int (*getbufferproc)(PyObject *, Py_buffer *, int); typedef void (*releasebufferproc)(PyObject *, Py_buffer *); +/* End buffer interface */ typedef PyObject *(*vectorcallfunc)(PyObject *callable, PyObject *const *args, size_t nargsf, PyObject *kwnames); -/* Maximum number of dimensions */ -#define PyBUF_MAX_NDIM 64 - -/* Flags for getting buffers */ -#define PyBUF_SIMPLE 0 -#define PyBUF_WRITABLE 0x0001 -/* we used to include an E, backwards compatible alias */ -#define PyBUF_WRITEABLE PyBUF_WRITABLE -#define PyBUF_FORMAT 0x0004 -#define PyBUF_ND 0x0008 -#define PyBUF_STRIDES (0x0010 | PyBUF_ND) -#define PyBUF_C_CONTIGUOUS (0x0020 | PyBUF_STRIDES) -#define PyBUF_F_CONTIGUOUS (0x0040 | PyBUF_STRIDES) -#define PyBUF_ANY_CONTIGUOUS (0x0080 | PyBUF_STRIDES) -#define PyBUF_INDIRECT (0x0100 | PyBUF_STRIDES) - -#define PyBUF_CONTIG (PyBUF_ND | PyBUF_WRITABLE) -#define PyBUF_CONTIG_RO (PyBUF_ND) - -#define PyBUF_STRIDED (PyBUF_STRIDES | PyBUF_WRITABLE) -#define PyBUF_STRIDED_RO (PyBUF_STRIDES) - -#define PyBUF_RECORDS (PyBUF_STRIDES | PyBUF_WRITABLE | PyBUF_FORMAT) -#define PyBUF_RECORDS_RO (PyBUF_STRIDES | PyBUF_FORMAT) - -#define PyBUF_FULL (PyBUF_INDIRECT | PyBUF_WRITABLE | PyBUF_FORMAT) -#define PyBUF_FULL_RO (PyBUF_INDIRECT | PyBUF_FORMAT) - - -#define PyBUF_READ 0x100 -#define PyBUF_WRITE 0x200 -/* End buffer interface */ - typedef struct { /* Number implementations must check *both* diff --git a/Include/typeslots.h b/Include/typeslots.h index 5800d0158bc924..506b05580de146 100644 --- a/Include/typeslots.h +++ b/Include/typeslots.h @@ -1,12 +1,6 @@ /* Do not renumber the file; these numbers are part of the stable ABI. */ -#if defined(Py_LIMITED_API) -/* Disabled, see #10181 */ -#undef Py_bf_getbuffer -#undef Py_bf_releasebuffer -#else #define Py_bf_getbuffer 1 #define Py_bf_releasebuffer 2 -#endif #define Py_mp_ass_subscript 3 #define Py_mp_length 4 #define Py_mp_subscript 5 diff --git a/Makefile.pre.in b/Makefile.pre.in index edc5fc3b6802f9..4dcedd684aa6da 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1439,6 +1439,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/abstract.h \ $(srcdir)/Include/bltinmodule.h \ $(srcdir)/Include/boolobject.h \ + $(srcdir)/Include/buffer.h \ $(srcdir)/Include/bytearrayobject.h \ $(srcdir)/Include/bytesobject.h \ $(srcdir)/Include/ceval.h \ From 1440b943f8fb28a1c9cc72a3f08a1a9a1e1de580 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Mon, 18 Oct 2021 16:40:50 +0200 Subject: [PATCH 03/12] Hide typo alias with limited API --- Include/buffer.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Include/buffer.h b/Include/buffer.h index 59eef777f03c49..eab44b4355e42e 100644 --- a/Include/buffer.h +++ b/Include/buffer.h @@ -86,8 +86,12 @@ PyAPI_FUNC(void) PyBuffer_Release(Py_buffer *view); /* Flags for getting buffers */ #define PyBUF_SIMPLE 0 #define PyBUF_WRITABLE 0x0001 + +#ifndef Py_LIMITED_API /* we used to include an E, backwards compatible alias */ #define PyBUF_WRITEABLE PyBUF_WRITABLE +#endif + #define PyBUF_FORMAT 0x0004 #define PyBUF_ND 0x0008 #define PyBUF_STRIDES (0x0010 | PyBUF_ND) From cf98a23544cf2fd55722b737b444057134957029 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Wed, 8 Dec 2021 14:27:12 +0100 Subject: [PATCH 04/12] Make Py_buffer public --- Include/buffer.h | 30 ++++++++++++++++++++++++------ Include/cpython/object.h | 19 ++----------------- 2 files changed, 26 insertions(+), 23 deletions(-) diff --git a/Include/buffer.h b/Include/buffer.h index eab44b4355e42e..dfbafbd9b9a4d7 100644 --- a/Include/buffer.h +++ b/Include/buffer.h @@ -6,13 +6,31 @@ extern "C" { #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030b0000 + /* === New Buffer API ============================================ - * Limited API since Python 3.11 + * Limited API and stable ABI since Python 3.11 + * + * Py_buffer struct layout and size is now part of the stable abi3. The + * struct layout and size must not be changed in any way, as it would + * break the ABI. + * */ -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030b0000 - -typedef struct bufferinfo Py_buffer; +typedef struct bufferinfo { + void *buf; + PyObject *obj; /* owned reference */ + Py_ssize_t len; + Py_ssize_t itemsize; /* This is Py_ssize_t so it can be + pointed to by strides in simple case.*/ + int readonly; + int ndim; + char *format; + Py_ssize_t *shape; + Py_ssize_t *strides; + Py_ssize_t *suboffsets; + void *internal; +} Py_buffer; /* Return 1 if the getbuffer function is available, otherwise return 0. */ PyAPI_FUNC(int) PyObject_CheckBuffer(PyObject *obj); @@ -116,9 +134,9 @@ PyAPI_FUNC(void) PyBuffer_Release(Py_buffer *view); #define PyBUF_READ 0x100 #define PyBUF_WRITE 0x200 -#endif /* Py_LIMITED_API */ +#endif /* !Py_LIMITED_API || Py_LIMITED_API >= 3.11 */ #ifdef __cplusplus } #endif -#endif /* Py_BUFFER_H */ \ No newline at end of file +#endif /* Py_BUFFER_H */ diff --git a/Include/cpython/object.h b/Include/cpython/object.h index ac4f62b499b632..22473edf49cc77 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -45,25 +45,10 @@ typedef struct _Py_Identifier { #define _Py_static_string(varname, value) static _Py_Identifier varname = _Py_static_string_init(value) #define _Py_IDENTIFIER(varname) _Py_static_string(PyId_##varname, #varname) -/* buffer interface */ -typedef struct bufferinfo { - void *buf; - PyObject *obj; /* owned reference */ - Py_ssize_t len; - Py_ssize_t itemsize; /* This is Py_ssize_t so it can be - pointed to by strides in simple case.*/ - int readonly; - int ndim; - char *format; - Py_ssize_t *shape; - Py_ssize_t *strides; - Py_ssize_t *suboffsets; - void *internal; -} Py_buffer; - +/* Py_buffer is defined in buffer.h */ +typedef struct bufferinfo Py_buffer; typedef int (*getbufferproc)(PyObject *, Py_buffer *, int); typedef void (*releasebufferproc)(PyObject *, Py_buffer *); -/* End buffer interface */ typedef PyObject *(*vectorcallfunc)(PyObject *callable, PyObject *const *args, size_t nargsf, PyObject *kwnames); From 0e67e2c94be6af3632c158dc8c6e5142a109d314 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Wed, 8 Dec 2021 14:28:59 +0100 Subject: [PATCH 05/12] Update stable_abi.txt and run make regen-limited-abi --- Doc/data/stable_abi.dat | 12 ++++++++++++ Lib/test/test_stable_abi_ctypes.py | 11 +++++++++++ Misc/stable_abi.txt | 26 ++++++++++++++++++++++++++ PC/python3dll.c | 11 +++++++++++ 4 files changed, 60 insertions(+) diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 02e54e5d7f14ab..2b6e5027a0686b 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -10,6 +10,14 @@ function,PyArg_ValidateKeywordArguments,3.2, var,PyBaseObject_Type,3.2, function,PyBool_FromLong,3.2, var,PyBool_Type,3.2, +function,PyBuffer_FillContiguousStrides,3.11, +function,PyBuffer_FillInfo,3.11, +function,PyBuffer_FromContiguous,3.11, +function,PyBuffer_GetPointer,3.11, +function,PyBuffer_IsContiguous,3.11, +function,PyBuffer_Release,3.11, +function,PyBuffer_SizeFromFormat,3.11, +function,PyBuffer_ToContiguous,3.11, var,PyByteArrayIter_Type,3.2, function,PyByteArray_AsString,3.2, function,PyByteArray_Concat,3.2, @@ -476,8 +484,10 @@ function,PyObject_CallMethodObjArgs,3.2, function,PyObject_CallNoArgs,3.10, function,PyObject_CallObject,3.2, function,PyObject_Calloc,3.7, +function,PyObject_CheckBuffer,3.11, function,PyObject_CheckReadBuffer,3.2, function,PyObject_ClearWeakRefs,3.2, +function,PyObject_CopyData,3.11, function,PyObject_DelItem,3.2, function,PyObject_DelItemString,3.2, function,PyObject_Dir,3.2, @@ -495,6 +505,7 @@ function,PyObject_GenericSetDict,3.7, function,PyObject_GetAIter,3.10, function,PyObject_GetAttr,3.2, function,PyObject_GetAttrString,3.2, +function,PyObject_GetBuffer,3.11, function,PyObject_GetItem,3.2, function,PyObject_GetIter,3.2, function,PyObject_HasAttr,3.2, @@ -832,6 +843,7 @@ var,Py_UTF8Mode,3.8, function,Py_VaBuildValue,3.2, var,Py_Version,3.11, function,Py_XNewRef,3.10, +type,Py_buffer,3.11, type,Py_intptr_t,3.2, type,Py_ssize_t,3.2, type,Py_uintptr_t,3.2, diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index 9fd6b14b0232a4..26b2461c394a72 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -28,6 +28,14 @@ def test_available_symbols(self): "PyBaseObject_Type", "PyBool_FromLong", "PyBool_Type", + "PyBuffer_FillContiguousStrides", + "PyBuffer_FillInfo", + "PyBuffer_FromContiguous", + "PyBuffer_GetPointer", + "PyBuffer_IsContiguous", + "PyBuffer_Release", + "PyBuffer_SizeFromFormat", + "PyBuffer_ToContiguous", "PyByteArrayIter_Type", "PyByteArray_AsString", "PyByteArray_Concat", @@ -470,8 +478,10 @@ def test_available_symbols(self): "PyObject_CallNoArgs", "PyObject_CallObject", "PyObject_Calloc", + "PyObject_CheckBuffer", "PyObject_CheckReadBuffer", "PyObject_ClearWeakRefs", + "PyObject_CopyData", "PyObject_DelItem", "PyObject_DelItemString", "PyObject_Dir", @@ -489,6 +499,7 @@ def test_available_symbols(self): "PyObject_GetAIter", "PyObject_GetAttr", "PyObject_GetAttrString", + "PyObject_GetBuffer", "PyObject_GetItem", "PyObject_GetIter", "PyObject_HasAttr", diff --git a/Misc/stable_abi.txt b/Misc/stable_abi.txt index c4f5318712a541..2c1c826e7b18c0 100644 --- a/Misc/stable_abi.txt +++ b/Misc/stable_abi.txt @@ -2191,6 +2191,32 @@ function PyType_GetQualName data PyStructSequence_UnnamedField added 3.11 +# Add stable Py_buffer API in Python 3.11 (https://bugs.python.org/issue45459) +struct Py_buffer + added 3.11 +function PyObject_CheckBuffer + added 3.11 +function PyObject_GetBuffer + added 3.11 +function PyBuffer_GetPointer + added 3.11 +function PyBuffer_SizeFromFormat + added 3.11 +function PyBuffer_ToContiguous + added 3.11 +function PyBuffer_FromContiguous + added 3.11 +function PyObject_CopyData + added 3.11 +function PyBuffer_IsContiguous + added 3.11 +function PyBuffer_FillContiguousStrides + added 3.11 +function PyBuffer_FillInfo + added 3.11 +function PyBuffer_Release + added 3.11 + # (Detailed comments aren't really needed for further entries: from here on # we can use version control logs.) diff --git a/PC/python3dll.c b/PC/python3dll.c index b2bb1706c4a2eb..d78d35cbea709c 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -94,6 +94,14 @@ EXPORT_FUNC(PyArg_ValidateKeywordArguments) EXPORT_FUNC(PyArg_VaParse) EXPORT_FUNC(PyArg_VaParseTupleAndKeywords) EXPORT_FUNC(PyBool_FromLong) +EXPORT_FUNC(PyBuffer_FillContiguousStrides) +EXPORT_FUNC(PyBuffer_FillInfo) +EXPORT_FUNC(PyBuffer_FromContiguous) +EXPORT_FUNC(PyBuffer_GetPointer) +EXPORT_FUNC(PyBuffer_IsContiguous) +EXPORT_FUNC(PyBuffer_Release) +EXPORT_FUNC(PyBuffer_SizeFromFormat) +EXPORT_FUNC(PyBuffer_ToContiguous) EXPORT_FUNC(PyByteArray_AsString) EXPORT_FUNC(PyByteArray_Concat) EXPORT_FUNC(PyByteArray_FromObject) @@ -426,8 +434,10 @@ EXPORT_FUNC(PyObject_CallMethodObjArgs) EXPORT_FUNC(PyObject_CallNoArgs) EXPORT_FUNC(PyObject_CallObject) EXPORT_FUNC(PyObject_Calloc) +EXPORT_FUNC(PyObject_CheckBuffer) EXPORT_FUNC(PyObject_CheckReadBuffer) EXPORT_FUNC(PyObject_ClearWeakRefs) +EXPORT_FUNC(PyObject_CopyData) EXPORT_FUNC(PyObject_DelItem) EXPORT_FUNC(PyObject_DelItemString) EXPORT_FUNC(PyObject_Dir) @@ -445,6 +455,7 @@ EXPORT_FUNC(PyObject_GenericSetDict) EXPORT_FUNC(PyObject_GetAIter) EXPORT_FUNC(PyObject_GetAttr) EXPORT_FUNC(PyObject_GetAttrString) +EXPORT_FUNC(PyObject_GetBuffer) EXPORT_FUNC(PyObject_GetItem) EXPORT_FUNC(PyObject_GetIter) EXPORT_FUNC(PyObject_HasAttr) From d5167819b1195de97ec85e347c14c05ba73b19cf Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Wed, 8 Dec 2021 15:12:15 +0100 Subject: [PATCH 06/12] Add simple tests for Py_buffer stable ABI --- Lib/test/test_xxlimited.py | 11 +++++++++ Modules/xxlimited.c | 50 +++++++++++++++++++++++++++++++++++++- 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_xxlimited.py b/Lib/test/test_xxlimited.py index e3f521d9b040dc..6dbfb3f439393c 100644 --- a/Lib/test/test_xxlimited.py +++ b/Lib/test/test_xxlimited.py @@ -58,6 +58,17 @@ def test_error(self): with self.assertRaises(self.module.Error): raise self.module.Error + def test_buffer(self): + xxo = self.module.Xxo() + self.assertEqual(xxo.x_exports, 0) + b1 = memoryview(xxo) + self.assertEqual(xxo.x_exports, 1) + b2 = memoryview(xxo) + self.assertEqual(xxo.x_exports, 2) + b1[0] = 1 + self.assertEqual(b1[0], 1) + self.assertEqual(b2[0], 1) + class TestXXLimited35(CommonTests, unittest.TestCase): module = xxlimited_35 diff --git a/Modules/xxlimited.c b/Modules/xxlimited.c index 93895c4f1214c3..4976b9d55d1701 100644 --- a/Modules/xxlimited.c +++ b/Modules/xxlimited.c @@ -19,6 +19,8 @@ def __init__(self): # In the C class, "_x_attr" is not accessible from Python code self._x_attr = {} + self._x_buffer = bytesarray(10) + self._x_exports = 0 def __getattr__(self, name): return self._x_attr[name] @@ -29,6 +31,10 @@ def __delattr__(self, name): del self._x_attr[name] + @property + def x_exports(self): + return self._x_exports + def demo(o, /): if isinstance(o, str): return o @@ -57,6 +63,9 @@ #define Py_LIMITED_API 0x030b0000 #include "Python.h" +#include + +#define BUFSIZE 10 // Module state typedef struct { @@ -70,7 +79,9 @@ typedef struct { // Instance state typedef struct { PyObject_HEAD - PyObject *x_attr; /* Attributes dictionary */ + PyObject *x_attr; /* Attributes dictionary */ + char x_buffer[BUFSIZE]; /* buffer for Py_buffer */ + Py_ssize_t x_exports; /* how many buffer are exported */ } XxoObject; // XXX: no good way to do this yet @@ -89,6 +100,8 @@ newXxoObject(PyObject *module) return NULL; } self->x_attr = NULL; + memset(self->x_buffer, 0, BUFSIZE); + self->x_exports = 0; return self; } @@ -212,11 +225,43 @@ static PyMethodDef Xxo_methods[] = { {NULL, NULL} /* sentinel */ }; +/* Xxo buffer interface */ + +static int +Xxo_getbuffer(XxoObject *self, Py_buffer *view, int flags) +{ + int res = PyBuffer_FillInfo(view, (PyObject*)self, + (void *)self->x_buffer, BUFSIZE, + 0, flags); + if (res == 0) { + self->x_exports++; + } + return res; +} + +static void +Xxo_releasebuffer(XxoObject *self, Py_buffer *view) +{ + self->x_exports--; +} + +static PyObject * +Xxo_get_x_exports(XxoObject *self, void *c) +{ + return PyLong_FromSsize_t(self->x_exports); +} + /* Xxo type definition */ PyDoc_STRVAR(Xxo_doc, "A class that explicitly stores attributes in an internal dict"); +static PyGetSetDef Xxo_getsetlist[] = { + {"x_exports", (getter) Xxo_get_x_exports, NULL, NULL}, + {NULL}, +}; + + static PyType_Slot Xxo_Type_slots[] = { {Py_tp_doc, (char *)Xxo_doc}, {Py_tp_traverse, Xxo_traverse}, @@ -226,6 +271,9 @@ static PyType_Slot Xxo_Type_slots[] = { {Py_tp_getattro, Xxo_getattro}, {Py_tp_setattro, Xxo_setattro}, {Py_tp_methods, Xxo_methods}, + {Py_bf_getbuffer, Xxo_getbuffer}, + {Py_bf_releasebuffer, Xxo_releasebuffer}, + {Py_tp_getset, Xxo_getsetlist}, {0, 0}, /* sentinel */ }; From c1391cf04456a793a0faef9d30512a015b986f76 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Wed, 8 Dec 2021 15:28:05 +0100 Subject: [PATCH 07/12] Add PyMemoryView_FromBuffer to limited API --- Doc/data/stable_abi.dat | 1 + Include/memoryobject.h | 2 +- Lib/test/test_stable_abi_ctypes.py | 1 + Misc/stable_abi.txt | 2 ++ PC/python3dll.c | 1 + 5 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 2b6e5027a0686b..18bbf03187b30f 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -383,6 +383,7 @@ function,PyMem_Malloc,3.2, function,PyMem_Realloc,3.2, type,PyMemberDef,3.2, var,PyMemberDescr_Type,3.2, +function,PyMemoryView_FromBuffer,3.11, function,PyMemoryView_FromMemory,3.7, function,PyMemoryView_FromObject,3.2, function,PyMemoryView_GetContiguous,3.2, diff --git a/Include/memoryobject.h b/Include/memoryobject.h index 0298cc93730684..154397ce1e56d2 100644 --- a/Include/memoryobject.h +++ b/Include/memoryobject.h @@ -25,7 +25,7 @@ PyAPI_FUNC(PyObject *) PyMemoryView_FromObject(PyObject *base); PyAPI_FUNC(PyObject *) PyMemoryView_FromMemory(char *mem, Py_ssize_t size, int flags); #endif -#ifndef Py_LIMITED_API +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030b0000 PyAPI_FUNC(PyObject *) PyMemoryView_FromBuffer(const Py_buffer *info); #endif PyAPI_FUNC(PyObject *) PyMemoryView_GetContiguous(PyObject *base, diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index 26b2461c394a72..a49235b81c1b07 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -389,6 +389,7 @@ def test_available_symbols(self): "PyMemberDescr_Type", "PyMember_GetOne", "PyMember_SetOne", + "PyMemoryView_FromBuffer", "PyMemoryView_FromMemory", "PyMemoryView_FromObject", "PyMemoryView_GetContiguous", diff --git a/Misc/stable_abi.txt b/Misc/stable_abi.txt index 2c1c826e7b18c0..cc3cc56d472d90 100644 --- a/Misc/stable_abi.txt +++ b/Misc/stable_abi.txt @@ -2216,6 +2216,8 @@ function PyBuffer_FillInfo added 3.11 function PyBuffer_Release added 3.11 +function PyMemoryView_FromBuffer + added 3.11 # (Detailed comments aren't really needed for further entries: from here on # we can use version control logs.) diff --git a/PC/python3dll.c b/PC/python3dll.c index d78d35cbea709c..70f11dc1905547 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -360,6 +360,7 @@ EXPORT_FUNC(PyMem_Malloc) EXPORT_FUNC(PyMem_Realloc) EXPORT_FUNC(PyMember_GetOne) EXPORT_FUNC(PyMember_SetOne) +EXPORT_FUNC(PyMemoryView_FromBuffer) EXPORT_FUNC(PyMemoryView_FromMemory) EXPORT_FUNC(PyMemoryView_FromObject) EXPORT_FUNC(PyMemoryView_GetContiguous) From 92d2ea5565a08b3f7f97e7d094da04159abcc738 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Wed, 8 Dec 2021 18:24:50 +0100 Subject: [PATCH 08/12] Add docs --- Doc/c-api/type.rst | 11 +++++----- Doc/whatsnew/3.11.rst | 20 +++++++++++++++++++ .../2021-10-18-16-54-24.bpo-45459.Y1pEZs.rst | 3 ++- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index f96886985e9326..97a818ab2ccd0b 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -272,12 +272,6 @@ The following functions and structs are used to create * :c:member:`~PyTypeObject.tp_vectorcall_offset` (see :ref:`PyMemberDef `) - The following fields cannot be set using :c:type:`PyType_Spec` and - :c:type:`PyType_Slot` under the limited API: - - * :c:member:`~PyBufferProcs.bf_getbuffer` - * :c:member:`~PyBufferProcs.bf_releasebuffer` - Setting :c:data:`Py_tp_bases` or :c:data:`Py_tp_base` may be problematic on some platforms. To avoid issues, use the *bases* argument of @@ -287,6 +281,11 @@ The following functions and structs are used to create Slots in :c:type:`PyBufferProcs` in may be set in the unlimited API. + .. versionchanged:: 3.11 + :c:member:`~PyBufferProcs.bf_getbuffer` and + :c:member:`~PyBufferProcs.bf_releasebuffer` are now available + under limited API. + .. c:member:: void *PyType_Slot.pfunc The desired value of the slot. In most cases, this is a pointer diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index e7f3dab2b51db4..3458ad63c9df8f 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -657,6 +657,26 @@ New Features :c:macro:`PY_VERSION_HEX`. (Contributed by Gabriele N. Tornetta in :issue:`43931`.) +* :c:type:`Py_buffer` and APIs are now part of the limited API and the stable + ABI: + + * :c:func:`PyObject_CheckBuffer` + * :c:func:`PyObject_GetBuffer` + * :c:func:`PyBuffer_GetPointer` + * :c:func:`PyBuffer_SizeFromFormat` + * :c:func:`PyBuffer_ToContiguous` + * :c:func:`PyBuffer_FromContiguous` + * :c:func:`PyBuffer_CopyData` + * :c:func:`PyBuffer_IsContiguous` + * :c:func:`PyBuffer_FillContiguousStrides` + * :c:func:`PyBuffer_FillInfo` + * :c:func:`PyBuffer_Release` + * :c:func:`PyMemoryView_FromBuffer` + * :c:member:`~PyBufferProcs.bf_getbuffer` and + :c:member:`~PyBufferProcs.bf_releasebuffer` type slots + + (Contributed by Christian Heimes in :issue:`45459`.) + Porting to Python 3.11 ---------------------- diff --git a/Misc/NEWS.d/next/C API/2021-10-18-16-54-24.bpo-45459.Y1pEZs.rst b/Misc/NEWS.d/next/C API/2021-10-18-16-54-24.bpo-45459.Y1pEZs.rst index 2da4d93f9bdea0..a8d93227817c46 100644 --- a/Misc/NEWS.d/next/C API/2021-10-18-16-54-24.bpo-45459.Y1pEZs.rst +++ b/Misc/NEWS.d/next/C API/2021-10-18-16-54-24.bpo-45459.Y1pEZs.rst @@ -1 +1,2 @@ -Add :c:type:`Py_buffer` to limited API / stable ABI. +:c:type:`Py_buffer` and various ``Py_buffer`` related functions are now +part of the limited API and stable ABI. From 7df9317a0b19e061439b252d8917758cfde3621c Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Wed, 2 Feb 2022 14:07:01 +0100 Subject: [PATCH 09/12] Document PyObject_CopyData --- Doc/c-api/buffer.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index 820a3a6f990ef0..05e131d06b909d 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -499,6 +499,13 @@ Buffer-related functions This function fails if *len* != *src->len*. +.. c:function:: int PyObject_CopyData(Py_buffer *dest, Py_buffer *src) + + Copy data from *src* to *dest* buffer. Can convert between C-style and + or Fortran-style buffers. + + ``0`` is returned on success, ``-1`` on error. + .. c:function:: void PyBuffer_FillContiguousStrides(int ndims, Py_ssize_t *shape, Py_ssize_t *strides, int itemsize, char order) Fill the *strides* array with byte-strides of a :term:`contiguous` (C-style if From 0ab2a44b5ee5f07c0df99ed6830fdf8db6557ef0 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Wed, 2 Feb 2022 14:07:46 +0100 Subject: [PATCH 10/12] Address Petr's code review of xxlimited.c --- Modules/xxlimited.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Modules/xxlimited.c b/Modules/xxlimited.c index 4976b9d55d1701..16d1b8311c62cc 100644 --- a/Modules/xxlimited.c +++ b/Modules/xxlimited.c @@ -19,7 +19,6 @@ def __init__(self): # In the C class, "_x_attr" is not accessible from Python code self._x_attr = {} - self._x_buffer = bytesarray(10) self._x_exports = 0 def __getattr__(self, name): @@ -33,6 +32,9 @@ @property def x_exports(self): + """Return the number of times an internal buffer is exported.""" + # Each Xxo instance has a 10-byte buffer that can be + # accessed via the buffer interface (e.g. `memoryview`). return self._x_exports def demo(o, /): From 98a4ee5bc7f41b976f0e87b0afb6f8f5e594d90e Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Wed, 2 Feb 2022 14:21:29 +0100 Subject: [PATCH 11/12] Make Py_Buffer struct anon --- Include/buffer.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/buffer.h b/Include/buffer.h index dfbafbd9b9a4d7..6893505e66e3e8 100644 --- a/Include/buffer.h +++ b/Include/buffer.h @@ -17,7 +17,7 @@ extern "C" { * */ -typedef struct bufferinfo { +typedef struct { void *buf; PyObject *obj; /* owned reference */ Py_ssize_t len; From 6119c2ba40eb4913d146cffda7c2d1d21490ceb1 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Wed, 2 Feb 2022 14:21:42 +0100 Subject: [PATCH 12/12] Remove forward declaration of Py_Buffer --- Include/cpython/object.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 22473edf49cc77..1554ac8aef1c44 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -2,6 +2,8 @@ # error "this header file must not be included directly" #endif +#include "buffer.h" // for Py_buffer, included after PyObject has been defined + PyAPI_FUNC(void) _Py_NewReference(PyObject *op); #ifdef Py_TRACE_REFS @@ -45,8 +47,6 @@ typedef struct _Py_Identifier { #define _Py_static_string(varname, value) static _Py_Identifier varname = _Py_static_string_init(value) #define _Py_IDENTIFIER(varname) _Py_static_string(PyId_##varname, #varname) -/* Py_buffer is defined in buffer.h */ -typedef struct bufferinfo Py_buffer; typedef int (*getbufferproc)(PyObject *, Py_buffer *, int); typedef void (*releasebufferproc)(PyObject *, Py_buffer *);