bpo-45459: Add Py_buffer to limited API (pythonGH-29991)

- [x] ``Py_buffer`` struct
- [x] ``PyBuffer_*()`` API functions
- [x] ``PyBUF_*`` constants
- [x] ``Py_bf_getbuffer`` and ``Py_bf_releasebuffer`` type slots
- [x] ``PyMemoryView_FromBuffer()`` API
- [x] tests for limited API
- [x] ``make regen-limited-abi``
- [x] documentation update
- [ ] export ``PyPickleBuffer*()`` API ???

Doc/c-api/buffer.rst

@@ -499,6 +499,13 @@ Buffer-related functions
    This function fails if *len* != *src->len*.
 
 
+.. c:function:: int PyObject_CopyData(Py_buffer *dest, Py_buffer *src)
+
+   Copy data from *src* to *dest* buffer. Can convert between C-style and
+   or Fortran-style buffers.
+
+   ``0`` is returned on success, ``-1`` on error.
+
 .. c:function:: void PyBuffer_FillContiguousStrides(int ndims, Py_ssize_t *shape, Py_ssize_t *strides, int itemsize, char order)
 
    Fill the *strides* array with byte-strides of a :term:`contiguous` (C-style if

Doc/c-api/type.rst

@@ -272,12 +272,6 @@ The following functions and structs are used to create
       * :c:member:`~PyTypeObject.tp_vectorcall_offset`
         (see :ref:`PyMemberDef <pymemberdef-offsets>`)
 
-      The following fields cannot be set using :c:type:`PyType_Spec` and
-      :c:type:`PyType_Slot` under the limited API:
-
-      * :c:member:`~PyBufferProcs.bf_getbuffer`
-      * :c:member:`~PyBufferProcs.bf_releasebuffer`
-
       Setting :c:data:`Py_tp_bases` or :c:data:`Py_tp_base` may be
       problematic on some platforms.
       To avoid issues, use the *bases* argument of
@@ -287,6 +281,11 @@ The following functions and structs are used to create
 
         Slots in :c:type:`PyBufferProcs` in may be set in the unlimited API.
 
+     .. versionchanged:: 3.11
+        :c:member:`~PyBufferProcs.bf_getbuffer` and
+        :c:member:`~PyBufferProcs.bf_releasebuffer` are now available
+        under limited API.
+
    .. c:member:: void *PyType_Slot.pfunc
 
       The desired value of the slot. In most cases, this is a pointer

Doc/data/stable_abi.dat

@@ -10,6 +10,14 @@ function,PyArg_ValidateKeywordArguments,3.2,
 var,PyBaseObject_Type,3.2,
 function,PyBool_FromLong,3.2,
 var,PyBool_Type,3.2,
+function,PyBuffer_FillContiguousStrides,3.11,
+function,PyBuffer_FillInfo,3.11,
+function,PyBuffer_FromContiguous,3.11,
+function,PyBuffer_GetPointer,3.11,
+function,PyBuffer_IsContiguous,3.11,
+function,PyBuffer_Release,3.11,
+function,PyBuffer_SizeFromFormat,3.11,
+function,PyBuffer_ToContiguous,3.11,
 var,PyByteArrayIter_Type,3.2,
 function,PyByteArray_AsString,3.2,
 function,PyByteArray_Concat,3.2,
@@ -375,6 +383,7 @@ function,PyMem_Malloc,3.2,
 function,PyMem_Realloc,3.2,
 type,PyMemberDef,3.2,
 var,PyMemberDescr_Type,3.2,
+function,PyMemoryView_FromBuffer,3.11,
 function,PyMemoryView_FromMemory,3.7,
 function,PyMemoryView_FromObject,3.2,
 function,PyMemoryView_GetContiguous,3.2,
@@ -476,8 +485,10 @@ function,PyObject_CallMethodObjArgs,3.2,
 function,PyObject_CallNoArgs,3.10,
 function,PyObject_CallObject,3.2,
 function,PyObject_Calloc,3.7,
+function,PyObject_CheckBuffer,3.11,
 function,PyObject_CheckReadBuffer,3.2,
 function,PyObject_ClearWeakRefs,3.2,
+function,PyObject_CopyData,3.11,
 function,PyObject_DelItem,3.2,
 function,PyObject_DelItemString,3.2,
 function,PyObject_Dir,3.2,
@@ -495,6 +506,7 @@ function,PyObject_GenericSetDict,3.7,
 function,PyObject_GetAIter,3.10,
 function,PyObject_GetAttr,3.2,
 function,PyObject_GetAttrString,3.2,
+function,PyObject_GetBuffer,3.11,
 function,PyObject_GetItem,3.2,
 function,PyObject_GetIter,3.2,
 function,PyObject_HasAttr,3.2,
@@ -832,6 +844,7 @@ var,Py_UTF8Mode,3.8,
 function,Py_VaBuildValue,3.2,
 var,Py_Version,3.11,
 function,Py_XNewRef,3.10,
+type,Py_buffer,3.11,
 type,Py_intptr_t,3.2,
 type,Py_ssize_t,3.2,
 type,Py_uintptr_t,3.2,

Doc/whatsnew/3.11.rst

@@ -657,6 +657,26 @@ New Features
   :c:macro:`PY_VERSION_HEX`.
   (Contributed by  Gabriele N. Tornetta in :issue:`43931`.)
 
+* :c:type:`Py_buffer` and APIs are now part of the limited API and the stable
+  ABI:
+
+  * :c:func:`PyObject_CheckBuffer`
+  * :c:func:`PyObject_GetBuffer`
+  * :c:func:`PyBuffer_GetPointer`
+  * :c:func:`PyBuffer_SizeFromFormat`
+  * :c:func:`PyBuffer_ToContiguous`
+  * :c:func:`PyBuffer_FromContiguous`
+  * :c:func:`PyBuffer_CopyData`
+  * :c:func:`PyBuffer_IsContiguous`
+  * :c:func:`PyBuffer_FillContiguousStrides`
+  * :c:func:`PyBuffer_FillInfo`
+  * :c:func:`PyBuffer_Release`
+  * :c:func:`PyMemoryView_FromBuffer`
+  * :c:member:`~PyBufferProcs.bf_getbuffer` and
+    :c:member:`~PyBufferProcs.bf_releasebuffer` type slots
+
+  (Contributed by Christian Heimes in :issue:`45459`.)
+
 
 Porting to Python 3.11
 ----------------------

Include/Python.h

@@ -50,6 +50,7 @@
 #include "longobject.h"
 #include "cpython/longintrepr.h"
 #include "boolobject.h"
+#include "buffer.h"
 #include "floatobject.h"
 #include "complexobject.h"
 #include "rangeobject.h"

Include/buffer.h

@@ -0,0 +1,142 @@
+/* Public Py_buffer API */
+
+#ifndef Py_BUFFER_H
+#define Py_BUFFER_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030b0000
+
+/* === New Buffer API ============================================
+ * Limited API and stable ABI since Python 3.11
+ *
+ * Py_buffer struct layout and size is now part of the stable abi3. The
+ * struct layout and size must not be changed in any way, as it would
+ * break the ABI.
+ *
+ */
+
+typedef struct {
+    void *buf;
+    PyObject *obj;        /* owned reference */
+    Py_ssize_t len;
+    Py_ssize_t itemsize;  /* This is Py_ssize_t so it can be
+                             pointed to by strides in simple case.*/
+    int readonly;
+    int ndim;
+    char *format;
+    Py_ssize_t *shape;
+    Py_ssize_t *strides;
+    Py_ssize_t *suboffsets;
+    void *internal;
+} Py_buffer;
+
+/* Return 1 if the getbuffer function is available, otherwise return 0. */
+PyAPI_FUNC(int) PyObject_CheckBuffer(PyObject *obj);
+
+/* This is a C-API version of the getbuffer function call.  It checks
+   to make sure object has the required function pointer and issues the
+   call.
+
+   Returns -1 and raises an error on failure and returns 0 on success. */
+PyAPI_FUNC(int) PyObject_GetBuffer(PyObject *obj, Py_buffer *view,
+                                   int flags);
+
+/* Get the memory area pointed to by the indices for the buffer given.
+   Note that view->ndim is the assumed size of indices. */
+PyAPI_FUNC(void *) PyBuffer_GetPointer(const Py_buffer *view, const Py_ssize_t *indices);
+
+/* Return the implied itemsize of the data-format area from a
+   struct-style description. */
+PyAPI_FUNC(Py_ssize_t) PyBuffer_SizeFromFormat(const char *format);
+
+/* Implementation in memoryobject.c */
+PyAPI_FUNC(int) PyBuffer_ToContiguous(void *buf, const Py_buffer *view,
+                                      Py_ssize_t len, char order);
+
+PyAPI_FUNC(int) PyBuffer_FromContiguous(const Py_buffer *view, const void *buf,
+                                        Py_ssize_t len, char order);
+
+/* Copy len bytes of data from the contiguous chunk of memory
+   pointed to by buf into the buffer exported by obj.  Return
+   0 on success and return -1 and raise a PyBuffer_Error on
+   error (i.e. the object does not have a buffer interface or
+   it is not working).
+
+   If fort is 'F', then if the object is multi-dimensional,
+   then the data will be copied into the array in
+   Fortran-style (first dimension varies the fastest).  If
+   fort is 'C', then the data will be copied into the array
+   in C-style (last dimension varies the fastest).  If fort
+   is 'A', then it does not matter and the copy will be made
+   in whatever way is more efficient. */
+PyAPI_FUNC(int) PyObject_CopyData(PyObject *dest, PyObject *src);
+
+/* Copy the data from the src buffer to the buffer of destination. */
+PyAPI_FUNC(int) PyBuffer_IsContiguous(const Py_buffer *view, char fort);
+
+/*Fill the strides array with byte-strides of a contiguous
+  (Fortran-style if fort is 'F' or C-style otherwise)
+  array of the given shape with the given number of bytes
+  per element. */
+PyAPI_FUNC(void) PyBuffer_FillContiguousStrides(int ndims,
+                                               Py_ssize_t *shape,
+                                               Py_ssize_t *strides,
+                                               int itemsize,
+                                               char fort);
+
+/* Fills in a buffer-info structure correctly for an exporter
+   that can only share a contiguous chunk of memory of
+   "unsigned bytes" of the given length.
+
+   Returns 0 on success and -1 (with raising an error) on error. */
+PyAPI_FUNC(int) PyBuffer_FillInfo(Py_buffer *view, PyObject *o, void *buf,
+                                  Py_ssize_t len, int readonly,
+                                  int flags);
+
+/* Releases a Py_buffer obtained from getbuffer ParseTuple's "s*". */
+PyAPI_FUNC(void) PyBuffer_Release(Py_buffer *view);
+
+/* Maximum number of dimensions */
+#define PyBUF_MAX_NDIM 64
+
+/* Flags for getting buffers */
+#define PyBUF_SIMPLE 0
+#define PyBUF_WRITABLE 0x0001
+
+#ifndef Py_LIMITED_API
+/*  we used to include an E, backwards compatible alias */
+#define PyBUF_WRITEABLE PyBUF_WRITABLE
+#endif
+
+#define PyBUF_FORMAT 0x0004
+#define PyBUF_ND 0x0008
+#define PyBUF_STRIDES (0x0010 | PyBUF_ND)
+#define PyBUF_C_CONTIGUOUS (0x0020 | PyBUF_STRIDES)
+#define PyBUF_F_CONTIGUOUS (0x0040 | PyBUF_STRIDES)
+#define PyBUF_ANY_CONTIGUOUS (0x0080 | PyBUF_STRIDES)
+#define PyBUF_INDIRECT (0x0100 | PyBUF_STRIDES)
+
+#define PyBUF_CONTIG (PyBUF_ND | PyBUF_WRITABLE)
+#define PyBUF_CONTIG_RO (PyBUF_ND)
+
+#define PyBUF_STRIDED (PyBUF_STRIDES | PyBUF_WRITABLE)
+#define PyBUF_STRIDED_RO (PyBUF_STRIDES)
+
+#define PyBUF_RECORDS (PyBUF_STRIDES | PyBUF_WRITABLE | PyBUF_FORMAT)
+#define PyBUF_RECORDS_RO (PyBUF_STRIDES | PyBUF_FORMAT)
+
+#define PyBUF_FULL (PyBUF_INDIRECT | PyBUF_WRITABLE | PyBUF_FORMAT)
+#define PyBUF_FULL_RO (PyBUF_INDIRECT | PyBUF_FORMAT)
+
+
+#define PyBUF_READ  0x100
+#define PyBUF_WRITE 0x200
+
+#endif /* !Py_LIMITED_API || Py_LIMITED_API >= 3.11 */
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* Py_BUFFER_H */

Include/cpython/abstract.h

@@ -168,74 +168,6 @@ PyAPI_FUNC(int) _PyObject_HasLen(PyObject *o);
    value.  If one of the calls fails, this function returns -1. */
 PyAPI_FUNC(Py_ssize_t) PyObject_LengthHint(PyObject *o, Py_ssize_t);
 
-/* === New Buffer API ============================================ */
-
-/* Return 1 if the getbuffer function is available, otherwise return 0. */
-PyAPI_FUNC(int) PyObject_CheckBuffer(PyObject *obj);
-
-/* This is a C-API version of the getbuffer function call.  It checks
-   to make sure object has the required function pointer and issues the
-   call.
-
-   Returns -1 and raises an error on failure and returns 0 on success. */
-PyAPI_FUNC(int) PyObject_GetBuffer(PyObject *obj, Py_buffer *view,
-                                   int flags);
-
-/* Get the memory area pointed to by the indices for the buffer given.
-   Note that view->ndim is the assumed size of indices. */
-PyAPI_FUNC(void *) PyBuffer_GetPointer(const Py_buffer *view, const Py_ssize_t *indices);
-
-/* Return the implied itemsize of the data-format area from a
-   struct-style description. */
-PyAPI_FUNC(Py_ssize_t) PyBuffer_SizeFromFormat(const char *format);
-
-/* Implementation in memoryobject.c */
-PyAPI_FUNC(int) PyBuffer_ToContiguous(void *buf, const Py_buffer *view,
-                                      Py_ssize_t len, char order);
-
-PyAPI_FUNC(int) PyBuffer_FromContiguous(const Py_buffer *view, const void *buf,
-                                        Py_ssize_t len, char order);
-
-/* Copy len bytes of data from the contiguous chunk of memory
-   pointed to by buf into the buffer exported by obj.  Return
-   0 on success and return -1 and raise a PyBuffer_Error on
-   error (i.e. the object does not have a buffer interface or
-   it is not working).
-
-   If fort is 'F', then if the object is multi-dimensional,
-   then the data will be copied into the array in
-   Fortran-style (first dimension varies the fastest).  If
-   fort is 'C', then the data will be copied into the array
-   in C-style (last dimension varies the fastest).  If fort
-   is 'A', then it does not matter and the copy will be made
-   in whatever way is more efficient. */
-PyAPI_FUNC(int) PyObject_CopyData(PyObject *dest, PyObject *src);
-
-/* Copy the data from the src buffer to the buffer of destination. */
-PyAPI_FUNC(int) PyBuffer_IsContiguous(const Py_buffer *view, char fort);
-
-/*Fill the strides array with byte-strides of a contiguous
-  (Fortran-style if fort is 'F' or C-style otherwise)
-  array of the given shape with the given number of bytes
-  per element. */
-PyAPI_FUNC(void) PyBuffer_FillContiguousStrides(int ndims,
-                                               Py_ssize_t *shape,
-                                               Py_ssize_t *strides,
-                                               int itemsize,
-                                               char fort);
-
-/* Fills in a buffer-info structure correctly for an exporter
-   that can only share a contiguous chunk of memory of
-   "unsigned bytes" of the given length.
-
-   Returns 0 on success and -1 (with raising an error) on error. */
-PyAPI_FUNC(int) PyBuffer_FillInfo(Py_buffer *view, PyObject *o, void *buf,
-                                  Py_ssize_t len, int readonly,
-                                  int flags);
-
-/* Releases a Py_buffer obtained from getbuffer ParseTuple's "s*". */
-PyAPI_FUNC(void) PyBuffer_Release(Py_buffer *view);
-
 /* === Sequence protocol ================================================ */
 
 /* Assume tp_as_sequence and sq_item exist and that 'i' does not