From 4ecc7bd3fce0642d6ffe6655a5b39cb0212efa49 Mon Sep 17 00:00:00 2001 From: David CARLIER Date: Sun, 16 Jan 2022 13:48:52 +0000 Subject: [PATCH 01/27] posixshmem module shm_rename freebsd support. --- .../_multiprocessing/clinic/posixshmem.c.h | 71 ++++++++++++++++++- Modules/_multiprocessing/posixshmem.c | 49 +++++++++++++ Modules/posixmodule.c | 10 +++ configure | 3 +- configure.ac | 2 +- pyconfig.h.in | 3 + 6 files changed, 134 insertions(+), 4 deletions(-) diff --git a/Modules/_multiprocessing/clinic/posixshmem.c.h b/Modules/_multiprocessing/clinic/posixshmem.c.h index 1b894ea4c67adc..da9436873053b4 100644 --- a/Modules/_multiprocessing/clinic/posixshmem.c.h +++ b/Modules/_multiprocessing/clinic/posixshmem.c.h @@ -42,6 +42,71 @@ _posixshmem_shm_open(PyObject *module, PyObject *args, PyObject *kwargs) #endif /* defined(HAVE_SHM_OPEN) */ +#if defined(HAVE_SHM_RENAME) + +PyDoc_STRVAR(_posixshmem_shm_rename__doc__, +"shm_rename($module, /, path_from, path_to, flags)\n" +"--\n" +"\n" +"Rename a shared memory object.\n" +"\n" +"Remove a shared memory object and relink to another path\n" +"By default, if the destination path already exist, it will be unlinked.\n" +"With the SHM_RENAME_EXCHANGE flag, source and destination paths\n" +"will be exchanged.\n" +"With the SHM_RENAME_NOREPLACE flag, an error will be triggered\n" +"if the destination alredady exists."); + +#define _POSIXSHMEM_SHM_RENAME_METHODDEF \ + {"shm_rename", (PyCFunction)(void(*)(void))_posixshmem_shm_rename, METH_FASTCALL|METH_KEYWORDS, _posixshmem_shm_rename__doc__}, + +static PyObject * +_posixshmem_shm_rename_impl(PyObject *module, PyObject *path_from, + PyObject *path_to, int flags); + +static PyObject * +_posixshmem_shm_rename(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"path_from", "path_to", "flags", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "shm_rename", 0}; + PyObject *argsbuf[3]; + PyObject *path_from; + PyObject *path_to; + int flags; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + if (!args) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("shm_rename", "argument 'path_from'", "str", args[0]); + goto exit; + } + if (PyUnicode_READY(args[0]) == -1) { + goto exit; + } + path_from = args[0]; + if (!PyUnicode_Check(args[1])) { + _PyArg_BadArgument("shm_rename", "argument 'path_to'", "str", args[1]); + goto exit; + } + if (PyUnicode_READY(args[1]) == -1) { + goto exit; + } + path_to = args[1]; + flags = _PyLong_AsInt(args[2]); + if (flags == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = _posixshmem_shm_rename_impl(module, path_from, path_to, flags); + +exit: + return return_value; +} + +#endif /* defined(HAVE_SHM_RENAME) */ + #if defined(HAVE_SHM_UNLINK) PyDoc_STRVAR(_posixshmem_shm_unlink__doc__, @@ -82,7 +147,11 @@ _posixshmem_shm_unlink(PyObject *module, PyObject *args, PyObject *kwargs) #define _POSIXSHMEM_SHM_OPEN_METHODDEF #endif /* !defined(_POSIXSHMEM_SHM_OPEN_METHODDEF) */ +#ifndef _POSIXSHMEM_SHM_RENAME_METHODDEF + #define _POSIXSHMEM_SHM_RENAME_METHODDEF +#endif /* !defined(_POSIXSHMEM_SHM_RENAME_METHODDEF) */ + #ifndef _POSIXSHMEM_SHM_UNLINK_METHODDEF #define _POSIXSHMEM_SHM_UNLINK_METHODDEF #endif /* !defined(_POSIXSHMEM_SHM_UNLINK_METHODDEF) */ -/*[clinic end generated code: output=be0661dbed83ea23 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=dedbfe53c5c203af input=a9049054013a1b77]*/ diff --git a/Modules/_multiprocessing/posixshmem.c b/Modules/_multiprocessing/posixshmem.c index 425ce10075c156..65110d5de4262c 100644 --- a/Modules/_multiprocessing/posixshmem.c +++ b/Modules/_multiprocessing/posixshmem.c @@ -68,6 +68,52 @@ _posixshmem_shm_open_impl(PyObject *module, PyObject *path, int flags, } #endif /* HAVE_SHM_OPEN */ +#ifdef HAVE_SHM_RENAME +/*[clinic input] +_posixshmem.shm_rename + path_from: unicode + path_to: unicode + flags: int + +Rename a shared memory object. + +Remove a shared memory object and relink to another path +By default, if the destination path already exist, it will be unlinked. +With the SHM_RENAME_EXCHANGE flag, source and destination paths +will be exchanged. +With the SHM_RENAME_NOREPLACE flag, an error will be triggered +if the destination alredady exists. + +[clinic start generated code]*/ + +static PyObject * +_posixshmem_shm_rename_impl(PyObject *module, PyObject *path_from, + PyObject *path_to, int flags) +/*[clinic end generated code: output=a9101f606826ad30 input=e32a44cee3e0326e]*/ +{ + int rv; + int async_err = 0; + const char *from = PyUnicode_AsUTF8(path_from); + const char *to = PyUnicode_AsUTF8(path_to); + if (from == NULL || to == NULL) { + return NULL; + } + do { + Py_BEGIN_ALLOW_THREADS + rv = shm_rename(from, to, flags); + Py_END_ALLOW_THREADS + } while (rv < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); + + if (rv < 0) { + if (!async_err) + PyErr_SetFromErrnoWithFilenameObjects(PyExc_OSError, path_from, path_to); + return NULL; + } + + Py_RETURN_NONE; +} +#endif /* HAVE_SHM_RENAME */ + #ifdef HAVE_SHM_UNLINK /*[clinic input] _posixshmem.shm_unlink @@ -111,6 +157,9 @@ _posixshmem_shm_unlink_impl(PyObject *module, PyObject *path) static PyMethodDef module_methods[ ] = { _POSIXSHMEM_SHM_OPEN_METHODDEF +#if defined(HAVE_SHM_RENAME) + _POSIXSHMEM_SHM_RENAME_METHODDEF +#endif _POSIXSHMEM_SHM_UNLINK_METHODDEF {NULL} /* Sentinel */ }; diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index fd70b38bddec79..a3d43316fb90c9 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -17288,6 +17288,16 @@ all_ins(PyObject *m) #endif #endif /* HAVE_MEMFD_CREATE */ +#ifdef HAVE_SHM_RENAME +#ifdef SHM_RENAME_EXCHANGE + if (PyModule_AddIntMacro(m, SHM_RENAME_EXCHANGE)) return -1; +#endif +#ifdef SHM_RENAME_NOREPLACE + if (PyModule_AddIntMacro(m, SHM_RENAME_NOREPLACE)) return -1; +#endif +#endif /* HAVE_SHM_RENAME */ + +#ifdef HAVE_EVENTFD #if defined(HAVE_EVENTFD) && defined(EFD_CLOEXEC) if (PyModule_AddIntMacro(m, EFD_CLOEXEC)) return -1; #ifdef EFD_NONBLOCK diff --git a/configure b/configure index fcf34f050861be..09298bc16bebb1 100755 --- a/configure +++ b/configure @@ -27375,8 +27375,7 @@ fi # endif #endif " - - for ac_func in shm_open shm_unlink + for ac_func in shm_open shm_rename shm_unlink do : as_ac_var=`printf "%s\n" "ac_cv_func_$ac_func" | $as_tr_sh` ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var" diff --git a/configure.ac b/configure.ac index f6df9b8bb41cc9..af9d625a7c0ef2 100644 --- a/configure.ac +++ b/configure.ac @@ -6873,7 +6873,7 @@ WITH_SAVE_ENV([ # endif #endif " - AC_CHECK_FUNCS([shm_open shm_unlink], [have_posix_shmem=yes], [have_posix_shmem=no]) + AC_CHECK_FUNCS([shm_open shm_rename shm_unlink], [have_posix_shmem=yes], [have_posix_shmem=no]) _RESTORE_VAR([ac_includes_default]) ]) diff --git a/pyconfig.h.in b/pyconfig.h.in index 63a3437ebb3eac..5e26d4206602e7 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -1138,6 +1138,9 @@ /* Define to 1 if you have the `shm_open' function. */ #undef HAVE_SHM_OPEN +/* Define to 1 if you have the `shm_rename' function. */ +#undef HAVE_SHM_RENAME + /* Define to 1 if you have the `shm_unlink' function. */ #undef HAVE_SHM_UNLINK From 1336f8c8cd257f21f59acc41f57ec004b4bd583b Mon Sep 17 00:00:00 2001 From: David CARLIER Date: Sun, 16 Jan 2022 16:31:47 +0000 Subject: [PATCH 02/27] update shared_memory library --- Lib/multiprocessing/shared_memory.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py index 67e70fdc27cf31..e29090422508c3 100644 --- a/Lib/multiprocessing/shared_memory.py +++ b/Lib/multiprocessing/shared_memory.py @@ -11,6 +11,7 @@ from functools import partial import mmap import os +import platform import errno import struct import secrets @@ -253,6 +254,32 @@ def unlink(self): if self._track: resource_tracker.unregister(self._name, "shared_memory") + def rename(self, newname, flags = 0): + """Renames a shared memory block. + + The policy how the operation is handled depends on the flag passed. + The default behavior is if the newname already exists, it will + be unlinked beforehand. + With the SHM_RENAME_EXCHANGE flag, the old and new name will + be exchanged. + With the SHM_RENAME_NOREPLACE flag, an error will be returned + if the new name exists. + """ + if platform.system() != "FreeBSD": + raise OSError("Unsupported operation on this platform") + + if newname: + newname = "/" + newname if self._prepend_leading_slash else newname + r = _posixshmem.shm_rename(self._name, newname, flags) + if r == None: + from .resource_tracker import unregister, register + unregister(self._name, "shared_memory") + self._name = newname + register(self._name, "shared_memory") + return True + + return False + _encoding = "utf8" From d11ff820ad728c00aefd9133d1f30afa05365b80 Mon Sep 17 00:00:00 2001 From: David Carlier Date: Sun, 16 Jan 2022 20:29:03 +0000 Subject: [PATCH 03/27] doc update --- Doc/library/multiprocessing.shared_memory.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst index 933fd07d62418a..fcc860c243b8c0 100644 --- a/Doc/library/multiprocessing.shared_memory.rst +++ b/Doc/library/multiprocessing.shared_memory.rst @@ -112,6 +112,15 @@ copying of data. This method has no effect on Windows, where the only way to delete a shared memory block is to close all handles. + .. method:: rename(newname, flags) + + Renames the underlying shared memory block to ``newname``. By + default, if ``newname`` already exists, it will be unlinked + beforehand. Passing the ``os.SHM_RENAME_EXCHANGE`` as ``flags``, + an exchange with ``newname`` will be proceeded instead. Passing + the ``os.SHM_RENAME_NOREPLACE`` as ``flags``, an error + will be triggered if ``newname`` already exists. + .. attribute:: buf A memoryview of contents of the shared memory block. From 13734b63a3f38fc4cf79a92e2503f20eee3c6fef Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Mon, 17 Jan 2022 20:03:58 +0000 Subject: [PATCH 04/27] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Lib/multiprocessing/shared_memory.py | 13 ++--- .../2022-01-17-20-03-56.bpo-46398.FATeqM.rst | 1 + .../_multiprocessing/clinic/posixshmem.c.h | 47 +++++-------------- Modules/_multiprocessing/posixshmem.c | 17 +++---- Modules/posixmodule.c | 1 - 5 files changed, 27 insertions(+), 52 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-01-17-20-03-56.bpo-46398.FATeqM.rst diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py index e29090422508c3..ccb8b4da508fe3 100644 --- a/Lib/multiprocessing/shared_memory.py +++ b/Lib/multiprocessing/shared_memory.py @@ -265,19 +265,14 @@ def rename(self, newname, flags = 0): With the SHM_RENAME_NOREPLACE flag, an error will be returned if the new name exists. """ - if platform.system() != "FreeBSD": + if !platform.hasattr("shm_rename"): raise OSError("Unsupported operation on this platform") if newname: newname = "/" + newname if self._prepend_leading_slash else newname - r = _posixshmem.shm_rename(self._name, newname, flags) - if r == None: - from .resource_tracker import unregister, register - unregister(self._name, "shared_memory") - self._name = newname - register(self._name, "shared_memory") - return True - + self._fd = _posixshmem.shm_rename(self._name, newname, flags) + self._name = newname + return True return False diff --git a/Misc/NEWS.d/next/Library/2022-01-17-20-03-56.bpo-46398.FATeqM.rst b/Misc/NEWS.d/next/Library/2022-01-17-20-03-56.bpo-46398.FATeqM.rst new file mode 100644 index 00000000000000..b0c968e1637e79 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-01-17-20-03-56.bpo-46398.FATeqM.rst @@ -0,0 +1 @@ +Add :method:`rename` to the `multiprocessing.shared_memory` library for FreeBSD. \ No newline at end of file diff --git a/Modules/_multiprocessing/clinic/posixshmem.c.h b/Modules/_multiprocessing/clinic/posixshmem.c.h index da9436873053b4..1451f9d902dbba 100644 --- a/Modules/_multiprocessing/clinic/posixshmem.c.h +++ b/Modules/_multiprocessing/clinic/posixshmem.c.h @@ -45,12 +45,12 @@ _posixshmem_shm_open(PyObject *module, PyObject *args, PyObject *kwargs) #if defined(HAVE_SHM_RENAME) PyDoc_STRVAR(_posixshmem_shm_rename__doc__, -"shm_rename($module, /, path_from, path_to, flags)\n" +"shm_rename($module, path_from, path_to, flags, /)\n" "--\n" "\n" "Rename a shared memory object.\n" "\n" -"Remove a shared memory object and relink to another path\n" +"Remove a shared memory object and relink to another path.\n" "By default, if the destination path already exist, it will be unlinked.\n" "With the SHM_RENAME_EXCHANGE flag, source and destination paths\n" "will be exchanged.\n" @@ -58,48 +58,30 @@ PyDoc_STRVAR(_posixshmem_shm_rename__doc__, "if the destination alredady exists."); #define _POSIXSHMEM_SHM_RENAME_METHODDEF \ - {"shm_rename", (PyCFunction)(void(*)(void))_posixshmem_shm_rename, METH_FASTCALL|METH_KEYWORDS, _posixshmem_shm_rename__doc__}, + {"shm_rename", (PyCFunction)(void(*)(void))_posixshmem_shm_rename, METH_FASTCALL, _posixshmem_shm_rename__doc__}, -static PyObject * +static int _posixshmem_shm_rename_impl(PyObject *module, PyObject *path_from, PyObject *path_to, int flags); static PyObject * -_posixshmem_shm_rename(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_posixshmem_shm_rename(PyObject *module, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; - static const char * const _keywords[] = {"path_from", "path_to", "flags", NULL}; - static _PyArg_Parser _parser = {NULL, _keywords, "shm_rename", 0}; - PyObject *argsbuf[3]; + static char *_keywords[] = {"path_from", "path_to", "flags", NULL}; PyObject *path_from; PyObject *path_to; int flags; + int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); - if (!args) { - goto exit; - } - if (!PyUnicode_Check(args[0])) { - _PyArg_BadArgument("shm_rename", "argument 'path_from'", "str", args[0]); - goto exit; - } - if (PyUnicode_READY(args[0]) == -1) { - goto exit; - } - path_from = args[0]; - if (!PyUnicode_Check(args[1])) { - _PyArg_BadArgument("shm_rename", "argument 'path_to'", "str", args[1]); - goto exit; - } - if (PyUnicode_READY(args[1]) == -1) { + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "Ui|i:shm_rename", _keywords, + &path_from, &path_to, &flags)) goto exit; - } - path_to = args[1]; - flags = _PyLong_AsInt(args[2]); - if (flags == -1 && PyErr_Occurred()) { + _return_value = _posixshmem_shm_rename_impl(module, path_from, path_to, flags); + if ((_return_value == -1) && PyErr_Occurred()) { goto exit; } - return_value = _posixshmem_shm_rename_impl(module, path_from, path_to, flags); + return_value = PyLong_FromLong((long)_return_value); exit: return return_value; @@ -122,9 +104,6 @@ PyDoc_STRVAR(_posixshmem_shm_unlink__doc__, #define _POSIXSHMEM_SHM_UNLINK_METHODDEF \ {"shm_unlink", (PyCFunction)(void(*)(void))_posixshmem_shm_unlink, METH_VARARGS|METH_KEYWORDS, _posixshmem_shm_unlink__doc__}, -static PyObject * -_posixshmem_shm_unlink_impl(PyObject *module, PyObject *path); - static PyObject * _posixshmem_shm_unlink(PyObject *module, PyObject *args, PyObject *kwargs) { @@ -154,4 +133,4 @@ _posixshmem_shm_unlink(PyObject *module, PyObject *args, PyObject *kwargs) #ifndef _POSIXSHMEM_SHM_UNLINK_METHODDEF #define _POSIXSHMEM_SHM_UNLINK_METHODDEF #endif /* !defined(_POSIXSHMEM_SHM_UNLINK_METHODDEF) */ -/*[clinic end generated code: output=dedbfe53c5c203af input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ca3677d5d1e7755f input=a9049054013a1b77]*/ diff --git a/Modules/_multiprocessing/posixshmem.c b/Modules/_multiprocessing/posixshmem.c index 65110d5de4262c..fe38dcf16973f4 100644 --- a/Modules/_multiprocessing/posixshmem.c +++ b/Modules/_multiprocessing/posixshmem.c @@ -74,10 +74,11 @@ _posixshmem.shm_rename path_from: unicode path_to: unicode flags: int + / Rename a shared memory object. -Remove a shared memory object and relink to another path +Remove a shared memory object and relink to another path. By default, if the destination path already exist, it will be unlinked. With the SHM_RENAME_EXCHANGE flag, source and destination paths will be exchanged. @@ -86,17 +87,17 @@ if the destination alredady exists. [clinic start generated code]*/ -static PyObject * +static int _posixshmem_shm_rename_impl(PyObject *module, PyObject *path_from, PyObject *path_to, int flags) -/*[clinic end generated code: output=a9101f606826ad30 input=e32a44cee3e0326e]*/ +/*[clinic end generated code: output=a9101f606826ad30 input=0373bfc9c491e123]*/ { int rv; int async_err = 0; - const char *from = PyUnicode_AsUTF8(path_from); - const char *to = PyUnicode_AsUTF8(path_to); + const char *from = PyUnicode_AsUTF8AndSize(path_from, NULL); + const char *to = PyUnicode_AsUTF8AndSize(path_to, NULL); if (from == NULL || to == NULL) { - return NULL; + return -1; } do { Py_BEGIN_ALLOW_THREADS @@ -107,10 +108,10 @@ _posixshmem_shm_rename_impl(PyObject *module, PyObject *path_from, if (rv < 0) { if (!async_err) PyErr_SetFromErrnoWithFilenameObjects(PyExc_OSError, path_from, path_to); - return NULL; + return -1; } - Py_RETURN_NONE; + return rv; } #endif /* HAVE_SHM_RENAME */ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index a3d43316fb90c9..0150a0d2cdb446 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -17297,7 +17297,6 @@ all_ins(PyObject *m) #endif #endif /* HAVE_SHM_RENAME */ -#ifdef HAVE_EVENTFD #if defined(HAVE_EVENTFD) && defined(EFD_CLOEXEC) if (PyModule_AddIntMacro(m, EFD_CLOEXEC)) return -1; #ifdef EFD_NONBLOCK From 69d583f82a1c240b1e95a72e6edb36d506d00e7a Mon Sep 17 00:00:00 2001 From: AN Long Date: Sat, 29 Mar 2025 18:26:41 +0800 Subject: [PATCH 05/27] Update generated files --- .../_multiprocessing/clinic/posixshmem.c.h | 33 +++++++++++-------- Modules/_multiprocessing/posixshmem.c | 2 +- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/Modules/_multiprocessing/clinic/posixshmem.c.h b/Modules/_multiprocessing/clinic/posixshmem.c.h index 0f4255a5e430a6..50a9a62789b571 100644 --- a/Modules/_multiprocessing/clinic/posixshmem.c.h +++ b/Modules/_multiprocessing/clinic/posixshmem.c.h @@ -60,28 +60,37 @@ PyDoc_STRVAR(_posixshmem_shm_rename__doc__, #define _POSIXSHMEM_SHM_RENAME_METHODDEF \ {"shm_rename", (PyCFunction)(void(*)(void))_posixshmem_shm_rename, METH_FASTCALL, _posixshmem_shm_rename__doc__}, -static int +static PyObject * _posixshmem_shm_rename_impl(PyObject *module, PyObject *path_from, PyObject *path_to, int flags); static PyObject * -_posixshmem_shm_rename(PyObject *module, PyObject *args, PyObject *kwargs) +_posixshmem_shm_rename(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - static char *_keywords[] = {"path_from", "path_to", "flags", NULL}; PyObject *path_from; PyObject *path_to; int flags; - int _return_value; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "Ui|i:shm_rename", _keywords, - &path_from, &path_to, &flags)) + if (nargs != 3) { + PyErr_Format(PyExc_TypeError, "shm_rename expected 3 arguments, got %zd", nargs); goto exit; - _return_value = _posixshmem_shm_rename_impl(module, path_from, path_to, flags); - if ((_return_value == -1) && PyErr_Occurred()) { + } + if (!PyUnicode_Check(args[0])) { + PyErr_Format(PyExc_TypeError, "shm_rename() argument 1 must be str, not %T", args[0]); goto exit; } - return_value = PyLong_FromLong((long)_return_value); + path_from = args[0]; + if (!PyUnicode_Check(args[1])) { + PyErr_Format(PyExc_TypeError, "shm_rename() argument 2 must be str, not %T", args[1]); + goto exit; + } + path_to = args[1]; + flags = PyLong_AsInt(args[2]); + if (flags == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = _posixshmem_shm_rename_impl(module, path_from, path_to, flags); exit: return return_value; @@ -105,14 +114,10 @@ PyDoc_STRVAR(_posixshmem_shm_unlink__doc__, {"shm_unlink", (PyCFunction)_posixshmem_shm_unlink, METH_O, _posixshmem_shm_unlink__doc__}, static PyObject * -<<<<<<< HEAD -_posixshmem_shm_unlink(PyObject *module, PyObject *args, PyObject *kwargs) -======= _posixshmem_shm_unlink_impl(PyObject *module, PyObject *path); static PyObject * _posixshmem_shm_unlink(PyObject *module, PyObject *arg) ->>>>>>> origin/main { PyObject *return_value = NULL; PyObject *path; @@ -141,4 +146,4 @@ _posixshmem_shm_unlink(PyObject *module, PyObject *arg) #ifndef _POSIXSHMEM_SHM_UNLINK_METHODDEF #define _POSIXSHMEM_SHM_UNLINK_METHODDEF #endif /* !defined(_POSIXSHMEM_SHM_UNLINK_METHODDEF) */ -/*[clinic end generated code: output=ca3677d5d1e7755f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c708ebf083cac0dc input=a9049054013a1b77]*/ diff --git a/Modules/_multiprocessing/posixshmem.c b/Modules/_multiprocessing/posixshmem.c index 7f36c554e3fe8e..f5c88b10b37144 100644 --- a/Modules/_multiprocessing/posixshmem.c +++ b/Modules/_multiprocessing/posixshmem.c @@ -92,7 +92,7 @@ if the destination alredady exists. [clinic start generated code]*/ -static int +static PyObject * _posixshmem_shm_rename_impl(PyObject *module, PyObject *path_from, PyObject *path_to, int flags) /*[clinic end generated code: output=a9101f606826ad30 input=0373bfc9c491e123]*/ From 83d63f4221c30c1a6f74b25ca6c9da75d3eb90a6 Mon Sep 17 00:00:00 2001 From: AN Long Date: Sat, 29 Mar 2025 20:26:50 +0800 Subject: [PATCH 06/27] Update by review --- Lib/multiprocessing/shared_memory.py | 11 ++++------- Modules/_multiprocessing/clinic/posixshmem.c.h | 11 ++++++++--- Modules/_multiprocessing/posixshmem.c | 16 +++++++++++----- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py index 30d7489f519f45..da54bf818c2969 100644 --- a/Lib/multiprocessing/shared_memory.py +++ b/Lib/multiprocessing/shared_memory.py @@ -265,15 +265,12 @@ def rename(self, newname, flags = 0): With the SHM_RENAME_NOREPLACE flag, an error will be returned if the new name exists. """ - if !platform.hasattr("shm_rename"): + if not hasattr(_posixshmem, "shm_rename"): raise OSError("Unsupported operation on this platform") - if newname: - newname = "/" + newname if self._prepend_leading_slash else newname - self._fd = _posixshmem.shm_rename(self._name, newname, flags) - self._name = newname - return True - return False + newname = "/" + newname if self._prepend_leading_slash else newname + self._fd = _posixshmem.shm_rename(self._name, newname, flags) + self._name = newname _encoding = "utf8" diff --git a/Modules/_multiprocessing/clinic/posixshmem.c.h b/Modules/_multiprocessing/clinic/posixshmem.c.h index 50a9a62789b571..e9109a3c2b8ba8 100644 --- a/Modules/_multiprocessing/clinic/posixshmem.c.h +++ b/Modules/_multiprocessing/clinic/posixshmem.c.h @@ -60,7 +60,7 @@ PyDoc_STRVAR(_posixshmem_shm_rename__doc__, #define _POSIXSHMEM_SHM_RENAME_METHODDEF \ {"shm_rename", (PyCFunction)(void(*)(void))_posixshmem_shm_rename, METH_FASTCALL, _posixshmem_shm_rename__doc__}, -static PyObject * +static int _posixshmem_shm_rename_impl(PyObject *module, PyObject *path_from, PyObject *path_to, int flags); @@ -71,6 +71,7 @@ _posixshmem_shm_rename(PyObject *module, PyObject *const *args, Py_ssize_t nargs PyObject *path_from; PyObject *path_to; int flags; + int _return_value; if (nargs != 3) { PyErr_Format(PyExc_TypeError, "shm_rename expected 3 arguments, got %zd", nargs); @@ -90,7 +91,11 @@ _posixshmem_shm_rename(PyObject *module, PyObject *const *args, Py_ssize_t nargs if (flags == -1 && PyErr_Occurred()) { goto exit; } - return_value = _posixshmem_shm_rename_impl(module, path_from, path_to, flags); + _return_value = _posixshmem_shm_rename_impl(module, path_from, path_to, flags); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromLong((long)_return_value); exit: return return_value; @@ -146,4 +151,4 @@ _posixshmem_shm_unlink(PyObject *module, PyObject *arg) #ifndef _POSIXSHMEM_SHM_UNLINK_METHODDEF #define _POSIXSHMEM_SHM_UNLINK_METHODDEF #endif /* !defined(_POSIXSHMEM_SHM_UNLINK_METHODDEF) */ -/*[clinic end generated code: output=c708ebf083cac0dc input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3b3a888f3ea27db0 input=a9049054013a1b77]*/ diff --git a/Modules/_multiprocessing/posixshmem.c b/Modules/_multiprocessing/posixshmem.c index f5c88b10b37144..e33e2ffd0eb895 100644 --- a/Modules/_multiprocessing/posixshmem.c +++ b/Modules/_multiprocessing/posixshmem.c @@ -75,7 +75,7 @@ _posixshmem_shm_open_impl(PyObject *module, PyObject *path, int flags, #ifdef HAVE_SHM_RENAME /*[clinic input] -_posixshmem.shm_rename +_posixshmem.shm_rename -> int path_from: unicode path_to: unicode flags: int @@ -92,18 +92,24 @@ if the destination alredady exists. [clinic start generated code]*/ -static PyObject * +static int _posixshmem_shm_rename_impl(PyObject *module, PyObject *path_from, PyObject *path_to, int flags) -/*[clinic end generated code: output=a9101f606826ad30 input=0373bfc9c491e123]*/ +/*[clinic end generated code: output=d9a710c512166e18 input=5fb42d1ce077caec]*/ { int rv; int async_err = 0; - const char *from = PyUnicode_AsUTF8AndSize(path_from, NULL); - const char *to = PyUnicode_AsUTF8AndSize(path_to, NULL); + Py_ssize_t from_size; + Py_ssize_t to_size; + const char *from = PyUnicode_AsUTF8AndSize(path_from, &from_size); + const char *to = PyUnicode_AsUTF8AndSize(path_to, &to_size); if (from == NULL || to == NULL) { return -1; } + if (strlen(from) != (size_t)from_size || strlen(to) != (size_t)to_size) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + return -1; + } do { Py_BEGIN_ALLOW_THREADS rv = shm_rename(from, to, flags); From ba0e6d0aba5a59b5e56caadc899f1c6f5526fc61 Mon Sep 17 00:00:00 2001 From: AN Long Date: Sat, 29 Mar 2025 21:21:37 +0800 Subject: [PATCH 07/27] Improve document by introduce more RST markers --- Doc/library/multiprocessing.shared_memory.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst index 3243fae40468c6..bf4e455e90cfb1 100644 --- a/Doc/library/multiprocessing.shared_memory.rst +++ b/Doc/library/multiprocessing.shared_memory.rst @@ -114,12 +114,12 @@ copying of data. .. method:: rename(newname, flags) - Renames the underlying shared memory block to ``newname``. By - default, if ``newname`` already exists, it will be unlinked - beforehand. Passing the ``os.SHM_RENAME_EXCHANGE`` as ``flags``, - an exchange with ``newname`` will be proceeded instead. Passing - the ``os.SHM_RENAME_NOREPLACE`` as ``flags``, an error - will be triggered if ``newname`` already exists. + Renames the underlying shared memory block to *newname*. + By default, if *newname* already exists, it will be unlinked beforehand. + Passing the :data:`os.SHM_RENAME_EXCHANGE` as *flags*, + an exchange with *newname* will be proceeded instead. + Passing the :data:`os.SHM_RENAME_NOREPLACE` as *flags*, + an error will be triggered if *newname* already exists. .. attribute:: buf From 514ed3d6c7cca0c80aa1e7bd7b50d082c1e13531 Mon Sep 17 00:00:00 2001 From: AN Long Date: Sat, 5 Apr 2025 22:36:32 +0800 Subject: [PATCH 08/27] Add test --- Lib/multiprocessing/shared_memory.py | 7 ++++++- Lib/test/_test_multiprocessing.py | 13 +++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py index da54bf818c2969..28001e3a1d4ef8 100644 --- a/Lib/multiprocessing/shared_memory.py +++ b/Lib/multiprocessing/shared_memory.py @@ -268,8 +268,13 @@ def rename(self, newname, flags = 0): if not hasattr(_posixshmem, "shm_rename"): raise OSError("Unsupported operation on this platform") - newname = "/" + newname if self._prepend_leading_slash else newname + oldname = self._name + if _USE_POSIX and self._prepend_leading_slash: + newname = "/" + newname self._fd = _posixshmem.shm_rename(self._name, newname, flags) + if self._track: + resource_tracker.unregister(oldname, "shared_memory") + resource_tracker.register(newname, "shared_memory") self._name = newname diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index dcce57629efe5b..98c7f26f75fd01 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -4376,6 +4376,19 @@ def test_shared_memory_recreate(self): self.addCleanup(shm2.unlink) self.assertEqual(shm2._name, names[1]) + def test_shared_memory_rename(self): + _posixshmem = import_helper.import_module("_posixshmem") + if not hasattr(_posixshmem, "shm_rename"): + raise unittest.SkipTest('requires _posixshmem.shm_rename') + + name1 = self._new_shm_name('testrename01_tsmb') + name2 = self._new_shm_name('testrename02_tsmb') + sms = shared_memory.SharedMemory(name1, create=True, size=512) + self.addCleanup(sms.unlink) + + sms.rename(name2) + self.assertEqual(sms.name, name2) + def test_invalid_shared_memory_creation(self): # Test creating a shared memory segment with negative size with self.assertRaises(ValueError): From 810e76569a9b752099833592c71fa650666e44b1 Mon Sep 17 00:00:00 2001 From: AN Long Date: Sat, 5 Apr 2025 22:45:53 +0800 Subject: [PATCH 09/27] update quote in .in file --- pyconfig.h.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyconfig.h.in b/pyconfig.h.in index 535995425f54c1..df1cb7e0966c8d 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -1191,10 +1191,10 @@ /* Define to 1 if you have the 'shm_open' function. */ #undef HAVE_SHM_OPEN -/* Define to 1 if you have the `shm_rename' function. */ +/* Define to 1 if you have the 'shm_rename' function. */ #undef HAVE_SHM_RENAME -/* Define to 1 if you have the `shm_unlink' function. */ +/* Define to 1 if you have the 'shm_unlink' function. */ #undef HAVE_SHM_UNLINK /* Define to 1 if you have the 'shutdown' function. */ From fa45c2e38b343120c0a6f1c8dd17ae3dc20060bb Mon Sep 17 00:00:00 2001 From: AN Long Date: Sat, 5 Apr 2025 23:00:50 +0800 Subject: [PATCH 10/27] regen-config --- configure | 1 + 1 file changed, 1 insertion(+) diff --git a/configure b/configure index f62e5485d7061c..7b2c1ee6495fad 100755 --- a/configure +++ b/configure @@ -29778,6 +29778,7 @@ fi # endif #endif " + for ac_func in shm_open shm_rename shm_unlink do : as_ac_var=`printf "%s\n" "ac_cv_func_$ac_func" | sed "$as_sed_sh"` From ca0714b13d7bf0be9939d7abf32fa949cd46df01 Mon Sep 17 00:00:00 2001 From: AN Long Date: Sat, 5 Apr 2025 23:23:41 +0800 Subject: [PATCH 11/27] Fix news entry --- .../next/Library/2022-01-17-20-03-56.bpo-46398.FATeqM.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2022-01-17-20-03-56.bpo-46398.FATeqM.rst b/Misc/NEWS.d/next/Library/2022-01-17-20-03-56.bpo-46398.FATeqM.rst index b0c968e1637e79..865b4e578dd639 100644 --- a/Misc/NEWS.d/next/Library/2022-01-17-20-03-56.bpo-46398.FATeqM.rst +++ b/Misc/NEWS.d/next/Library/2022-01-17-20-03-56.bpo-46398.FATeqM.rst @@ -1 +1 @@ -Add :method:`rename` to the `multiprocessing.shared_memory` library for FreeBSD. \ No newline at end of file +Add :meth:`rename` to the `multiprocessing.shared_memory` library for FreeBSD. From b3ec9783e592be2f6bd4946ce4e40e6ecd4c48f2 Mon Sep 17 00:00:00 2001 From: AN Long Date: Sat, 5 Apr 2025 23:26:39 +0800 Subject: [PATCH 12/27] Fix docs --- Doc/library/multiprocessing.shared_memory.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst index bf4e455e90cfb1..effac457c46696 100644 --- a/Doc/library/multiprocessing.shared_memory.rst +++ b/Doc/library/multiprocessing.shared_memory.rst @@ -116,9 +116,9 @@ copying of data. Renames the underlying shared memory block to *newname*. By default, if *newname* already exists, it will be unlinked beforehand. - Passing the :data:`os.SHM_RENAME_EXCHANGE` as *flags*, + Passing the :data:`!os.SHM_RENAME_EXCHANGE` as *flags*, an exchange with *newname* will be proceeded instead. - Passing the :data:`os.SHM_RENAME_NOREPLACE` as *flags*, + Passing the :data:`!os.SHM_RENAME_NOREPLACE` as *flags*, an error will be triggered if *newname* already exists. .. attribute:: buf From 778e64679a0b34b0ff1090ea4dda4fc7b5dd42f4 Mon Sep 17 00:00:00 2001 From: AN Long Date: Sat, 5 Apr 2025 23:30:53 +0800 Subject: [PATCH 13/27] fix news entry --- .../next/Library/2022-01-17-20-03-56.bpo-46398.FATeqM.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2022-01-17-20-03-56.bpo-46398.FATeqM.rst b/Misc/NEWS.d/next/Library/2022-01-17-20-03-56.bpo-46398.FATeqM.rst index 865b4e578dd639..b08e75f0f24aef 100644 --- a/Misc/NEWS.d/next/Library/2022-01-17-20-03-56.bpo-46398.FATeqM.rst +++ b/Misc/NEWS.d/next/Library/2022-01-17-20-03-56.bpo-46398.FATeqM.rst @@ -1 +1 @@ -Add :meth:`rename` to the `multiprocessing.shared_memory` library for FreeBSD. +Add :meth:`rename` to the :mod:`multiprocessing.shared_memory` library for FreeBSD. From 7e3242cb1378bd61c15bfc3d56e1db059ea96d67 Mon Sep 17 00:00:00 2001 From: AN Long Date: Sat, 5 Apr 2025 23:36:23 +0800 Subject: [PATCH 14/27] fix news entry --- .../next/Library/2022-01-17-20-03-56.bpo-46398.FATeqM.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2022-01-17-20-03-56.bpo-46398.FATeqM.rst b/Misc/NEWS.d/next/Library/2022-01-17-20-03-56.bpo-46398.FATeqM.rst index b08e75f0f24aef..82dc359c1def34 100644 --- a/Misc/NEWS.d/next/Library/2022-01-17-20-03-56.bpo-46398.FATeqM.rst +++ b/Misc/NEWS.d/next/Library/2022-01-17-20-03-56.bpo-46398.FATeqM.rst @@ -1 +1 @@ -Add :meth:`rename` to the :mod:`multiprocessing.shared_memory` library for FreeBSD. +Add :meth:`!rename` to the :mod:`multiprocessing.shared_memory` library for FreeBSD. From 6ec2152e6bcb073f878e5733fdf06320ad2b37a9 Mon Sep 17 00:00:00 2001 From: AN Long Date: Sun, 6 Apr 2025 13:24:22 +0900 Subject: [PATCH 15/27] Update Lib/multiprocessing/shared_memory.py Co-authored-by: Peter Bierma --- Lib/multiprocessing/shared_memory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py index 28001e3a1d4ef8..935c8a9bff7218 100644 --- a/Lib/multiprocessing/shared_memory.py +++ b/Lib/multiprocessing/shared_memory.py @@ -254,7 +254,7 @@ def unlink(self): if self._track: resource_tracker.unregister(self._name, "shared_memory") - def rename(self, newname, flags = 0): + def rename(self, newname, flags=0): """Renames a shared memory block. The policy how the operation is handled depends on the flag passed. From 81097750d350f1c038586f7a105822368ee231e6 Mon Sep 17 00:00:00 2001 From: AN Long Date: Sun, 6 Apr 2025 13:24:51 +0900 Subject: [PATCH 16/27] Update Doc/library/multiprocessing.shared_memory.rst Co-authored-by: Peter Bierma --- Doc/library/multiprocessing.shared_memory.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst index effac457c46696..8fbb9620740b1e 100644 --- a/Doc/library/multiprocessing.shared_memory.rst +++ b/Doc/library/multiprocessing.shared_memory.rst @@ -114,7 +114,7 @@ copying of data. .. method:: rename(newname, flags) - Renames the underlying shared memory block to *newname*. + Rename the underlying shared memory block to *newname*. By default, if *newname* already exists, it will be unlinked beforehand. Passing the :data:`!os.SHM_RENAME_EXCHANGE` as *flags*, an exchange with *newname* will be proceeded instead. From 8e86d210447b0ebf14686952ea4575bed4aa4494 Mon Sep 17 00:00:00 2001 From: AN Long Date: Sun, 6 Apr 2025 13:25:02 +0900 Subject: [PATCH 17/27] Update Doc/library/multiprocessing.shared_memory.rst Co-authored-by: Peter Bierma --- Doc/library/multiprocessing.shared_memory.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst index 8fbb9620740b1e..a2cd31c3a7ba5e 100644 --- a/Doc/library/multiprocessing.shared_memory.rst +++ b/Doc/library/multiprocessing.shared_memory.rst @@ -112,7 +112,7 @@ copying of data. This method has no effect on Windows, where the only way to delete a shared memory block is to close all handles. - .. method:: rename(newname, flags) + .. method:: rename(newname, flags=0) Rename the underlying shared memory block to *newname*. By default, if *newname* already exists, it will be unlinked beforehand. From ef3b5ce2219676cc0545abf42c8e6868186ef415 Mon Sep 17 00:00:00 2001 From: AN Long Date: Sun, 6 Apr 2025 13:25:24 +0900 Subject: [PATCH 18/27] Update Misc/NEWS.d/next/Library/2022-01-17-20-03-56.bpo-46398.FATeqM.rst Co-authored-by: Peter Bierma --- .../next/Library/2022-01-17-20-03-56.bpo-46398.FATeqM.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2022-01-17-20-03-56.bpo-46398.FATeqM.rst b/Misc/NEWS.d/next/Library/2022-01-17-20-03-56.bpo-46398.FATeqM.rst index 82dc359c1def34..d253aaf8ab35d8 100644 --- a/Misc/NEWS.d/next/Library/2022-01-17-20-03-56.bpo-46398.FATeqM.rst +++ b/Misc/NEWS.d/next/Library/2022-01-17-20-03-56.bpo-46398.FATeqM.rst @@ -1 +1 @@ -Add :meth:`!rename` to the :mod:`multiprocessing.shared_memory` library for FreeBSD. +Add the :mod:`multiprocessing.shared_memory.SharedMemory.rename` method for FreeBSD. From 86debb361bdabe5092d2bb76f8b59c5af6852dd0 Mon Sep 17 00:00:00 2001 From: AN Long Date: Sun, 6 Apr 2025 15:15:27 +0900 Subject: [PATCH 19/27] Update Doc/library/multiprocessing.shared_memory.rst Co-authored-by: Peter Bierma --- Doc/library/multiprocessing.shared_memory.rst | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst index a2cd31c3a7ba5e..13d3f28f62187b 100644 --- a/Doc/library/multiprocessing.shared_memory.rst +++ b/Doc/library/multiprocessing.shared_memory.rst @@ -116,10 +116,14 @@ copying of data. Rename the underlying shared memory block to *newname*. By default, if *newname* already exists, it will be unlinked beforehand. - Passing the :data:`!os.SHM_RENAME_EXCHANGE` as *flags*, - an exchange with *newname* will be proceeded instead. - Passing the :data:`!os.SHM_RENAME_NOREPLACE` as *flags*, - an error will be triggered if *newname* already exists. + + *flags* (0 by default) takes :ref:`bitwise ORed ` flags together. + + * :data:`!os.SHM_RENAME_EXCHANGE` will let + an exchange with *newname* will be processed instead. + * :data:`!os.SHM_RENAME_NOREPLACE` will raise + an error will be triggered if *newname* already exists. + .. attribute:: buf From cb181b092dd945f696c7222f426c2ff083bade64 Mon Sep 17 00:00:00 2001 From: AN Long Date: Sun, 6 Apr 2025 15:20:44 +0800 Subject: [PATCH 20/27] Remove whitespace in empty line --- Doc/library/multiprocessing.shared_memory.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst index 13d3f28f62187b..7e6b3fcece212c 100644 --- a/Doc/library/multiprocessing.shared_memory.rst +++ b/Doc/library/multiprocessing.shared_memory.rst @@ -118,7 +118,7 @@ copying of data. By default, if *newname* already exists, it will be unlinked beforehand. *flags* (0 by default) takes :ref:`bitwise ORed ` flags together. - + * :data:`!os.SHM_RENAME_EXCHANGE` will let an exchange with *newname* will be processed instead. * :data:`!os.SHM_RENAME_NOREPLACE` will raise From f6f831215b978838b424ac5380689436282ec107 Mon Sep 17 00:00:00 2001 From: AN Long Date: Sun, 6 Apr 2025 15:49:19 +0800 Subject: [PATCH 21/27] Define the rename method conditionally --- Lib/multiprocessing/shared_memory.py | 42 +++++++++++++--------------- Lib/test/_test_multiprocessing.py | 5 +--- 2 files changed, 21 insertions(+), 26 deletions(-) diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py index 935c8a9bff7218..4391cfb8b4b1c9 100644 --- a/Lib/multiprocessing/shared_memory.py +++ b/Lib/multiprocessing/shared_memory.py @@ -254,28 +254,26 @@ def unlink(self): if self._track: resource_tracker.unregister(self._name, "shared_memory") - def rename(self, newname, flags=0): - """Renames a shared memory block. - - The policy how the operation is handled depends on the flag passed. - The default behavior is if the newname already exists, it will - be unlinked beforehand. - With the SHM_RENAME_EXCHANGE flag, the old and new name will - be exchanged. - With the SHM_RENAME_NOREPLACE flag, an error will be returned - if the new name exists. - """ - if not hasattr(_posixshmem, "shm_rename"): - raise OSError("Unsupported operation on this platform") - - oldname = self._name - if _USE_POSIX and self._prepend_leading_slash: - newname = "/" + newname - self._fd = _posixshmem.shm_rename(self._name, newname, flags) - if self._track: - resource_tracker.unregister(oldname, "shared_memory") - resource_tracker.register(newname, "shared_memory") - self._name = newname + if hasattr(_posixshmem, "shm_rename"): + def rename(self, newname, flags=0): + """Renames a shared memory block. + + The policy how the operation is handled depends on the flag passed. + The default behavior is if the newname already exists, it will + be unlinked beforehand. + With the SHM_RENAME_EXCHANGE flag, the old and new name will + be exchanged. + With the SHM_RENAME_NOREPLACE flag, an error will be returned + if the new name exists. + """ + oldname = self._name + if _USE_POSIX and self._prepend_leading_slash: + newname = "/" + newname + self._fd = _posixshmem.shm_rename(self._name, newname, flags) + if self._track: + resource_tracker.unregister(oldname, "shared_memory") + resource_tracker.register(newname, "shared_memory") + self._name = newname _encoding = "utf8" diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 98c7f26f75fd01..da3ad6b0f96f7b 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -4376,11 +4376,8 @@ def test_shared_memory_recreate(self): self.addCleanup(shm2.unlink) self.assertEqual(shm2._name, names[1]) + @unittest.skipUnless(hasattr(shared_memory, "rename"), "requires SharedMomery.rename") def test_shared_memory_rename(self): - _posixshmem = import_helper.import_module("_posixshmem") - if not hasattr(_posixshmem, "shm_rename"): - raise unittest.SkipTest('requires _posixshmem.shm_rename') - name1 = self._new_shm_name('testrename01_tsmb') name2 = self._new_shm_name('testrename02_tsmb') sms = shared_memory.SharedMemory(name1, create=True, size=512) From 1cc08d7ef44c9de14adac700bb84c0b48bd20bed Mon Sep 17 00:00:00 2001 From: AN Long Date: Sun, 6 Apr 2025 16:18:31 +0800 Subject: [PATCH 22/27] Fix test on Windows --- Lib/test/_test_multiprocessing.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index da3ad6b0f96f7b..16863059fe5a34 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -4376,7 +4376,8 @@ def test_shared_memory_recreate(self): self.addCleanup(shm2.unlink) self.assertEqual(shm2._name, names[1]) - @unittest.skipUnless(hasattr(shared_memory, "rename"), "requires SharedMomery.rename") + @unittest.skipUnless(HAS_SHMEM and hasattr(shared_memory, "rename"), + "requires SharedMomery.rename") def test_shared_memory_rename(self): name1 = self._new_shm_name('testrename01_tsmb') name2 = self._new_shm_name('testrename02_tsmb') From d77049c1741db5d0231767bf75a4a48878c53fb1 Mon Sep 17 00:00:00 2001 From: AN Long Date: Sun, 6 Apr 2025 16:39:09 +0800 Subject: [PATCH 23/27] Fix codes on Windows --- Lib/multiprocessing/shared_memory.py | 2 +- Lib/test/_test_multiprocessing.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py index 4391cfb8b4b1c9..f07c26c6992b5d 100644 --- a/Lib/multiprocessing/shared_memory.py +++ b/Lib/multiprocessing/shared_memory.py @@ -254,7 +254,7 @@ def unlink(self): if self._track: resource_tracker.unregister(self._name, "shared_memory") - if hasattr(_posixshmem, "shm_rename"): + if _USE_POSIX and hasattr(_posixshmem, "shm_rename"): def rename(self, newname, flags=0): """Renames a shared memory block. diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 16863059fe5a34..b83dea615c1e25 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -4376,7 +4376,7 @@ def test_shared_memory_recreate(self): self.addCleanup(shm2.unlink) self.assertEqual(shm2._name, names[1]) - @unittest.skipUnless(HAS_SHMEM and hasattr(shared_memory, "rename"), + @unittest.skipUnless(hasattr(shared_memory, "rename"), "requires SharedMomery.rename") def test_shared_memory_rename(self): name1 = self._new_shm_name('testrename01_tsmb') From 416c6d8c751322238de4fa487af771443220de76 Mon Sep 17 00:00:00 2001 From: AN Long Date: Sun, 6 Apr 2025 18:58:40 +0900 Subject: [PATCH 24/27] Move constants --- Modules/_multiprocessing/posixshmem.c | 14 ++++++++++++++ Modules/posixmodule.c | 9 --------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/Modules/_multiprocessing/posixshmem.c b/Modules/_multiprocessing/posixshmem.c index e33e2ffd0eb895..55e762cd58683c 100644 --- a/Modules/_multiprocessing/posixshmem.c +++ b/Modules/_multiprocessing/posixshmem.c @@ -182,8 +182,22 @@ static PyMethodDef module_methods[ ] = { {NULL} /* Sentinel */ }; +static int +posixshmem_exec(PyObject *m) +{ +#ifdef HAVE_SHM_RENAME +#ifdef SHM_RENAME_EXCHANGE + if (PyModule_AddIntMacro(m, SHM_RENAME_EXCHANGE)) return -1; +#endif +#ifdef SHM_RENAME_NOREPLACE + if (PyModule_AddIntMacro(m, SHM_RENAME_NOREPLACE)) return -1; +#endif +#endif /* HAVE_SHM_RENAME */ + return 0; +} static PyModuleDef_Slot module_slots[] = { + {Py_mod_exec, posixshmem_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 8ded8f94712f24..b7300def8dc75f 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -17762,15 +17762,6 @@ all_ins(PyObject *m) #endif #endif /* HAVE_MEMFD_CREATE */ -#ifdef HAVE_SHM_RENAME -#ifdef SHM_RENAME_EXCHANGE - if (PyModule_AddIntMacro(m, SHM_RENAME_EXCHANGE)) return -1; -#endif -#ifdef SHM_RENAME_NOREPLACE - if (PyModule_AddIntMacro(m, SHM_RENAME_NOREPLACE)) return -1; -#endif -#endif /* HAVE_SHM_RENAME */ - #if defined(HAVE_EVENTFD) && defined(EFD_CLOEXEC) if (PyModule_AddIntMacro(m, EFD_CLOEXEC)) return -1; #ifdef EFD_NONBLOCK From 620da83a16ddbe0ea8abb0e8240396a085fcf4ad Mon Sep 17 00:00:00 2001 From: AN Long Date: Sun, 6 Apr 2025 18:44:55 +0800 Subject: [PATCH 25/27] Update document --- Doc/library/multiprocessing.shared_memory.rst | 21 +++++++++++++++---- Lib/multiprocessing/shared_memory.py | 3 +++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst index 7e6b3fcece212c..c4d498afc03deb 100644 --- a/Doc/library/multiprocessing.shared_memory.rst +++ b/Doc/library/multiprocessing.shared_memory.rst @@ -119,11 +119,24 @@ copying of data. *flags* (0 by default) takes :ref:`bitwise ORed ` flags together. - * :data:`!os.SHM_RENAME_EXCHANGE` will let - an exchange with *newname* will be processed instead. - * :data:`!os.SHM_RENAME_NOREPLACE` will raise - an error will be triggered if *newname* already exists. + .. availability:: FreeBSD >= 13.0 + .. versionadded:: next + + .. data:: SHM_RENAME_EXCHANGE + SHM_RENAME_NOREPLACE + + Parameters to the :func:`rename` function. + + :const:`SHM_RENAME_EXCHANGE` + Atomically exchange the SharedMemory to the *newname*. + + :const:`SHM_RENAME_NOREPLACE` + Raise an error if *newname* exists, rather than unlinking it. + + .. availability:: FreeBSD >= 13.0 + + .. versionadded:: next .. attribute:: buf diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py index f07c26c6992b5d..45cf58ed1c1920 100644 --- a/Lib/multiprocessing/shared_memory.py +++ b/Lib/multiprocessing/shared_memory.py @@ -255,6 +255,9 @@ def unlink(self): resource_tracker.unregister(self._name, "shared_memory") if _USE_POSIX and hasattr(_posixshmem, "shm_rename"): + from _posixshmem import SHM_RENAME_EXCHANGE + from _posixshmem import SHM_RENAME_NOREPLACE + def rename(self, newname, flags=0): """Renames a shared memory block. From 499368566e846c0e2134ee189c410a490b128b1c Mon Sep 17 00:00:00 2001 From: AN Long Date: Sun, 6 Apr 2025 18:55:33 +0800 Subject: [PATCH 26/27] Fix indent in document --- Doc/library/multiprocessing.shared_memory.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst index c4d498afc03deb..334d515503653a 100644 --- a/Doc/library/multiprocessing.shared_memory.rst +++ b/Doc/library/multiprocessing.shared_memory.rst @@ -129,7 +129,7 @@ copying of data. Parameters to the :func:`rename` function. :const:`SHM_RENAME_EXCHANGE` - Atomically exchange the SharedMemory to the *newname*. + Atomically exchange the SharedMemory to the *newname*. :const:`SHM_RENAME_NOREPLACE` Raise an error if *newname* exists, rather than unlinking it. From 8adc394ea8d368f0575800bfc93a80df64324ed7 Mon Sep 17 00:00:00 2001 From: AN Long Date: Mon, 7 Apr 2025 21:59:27 +0800 Subject: [PATCH 27/27] Add more tests --- Lib/multiprocessing/shared_memory.py | 7 ++++--- Lib/test/_test_multiprocessing.py | 26 +++++++++++++++++++++++++- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py index 45cf58ed1c1920..690e27aaeeef0d 100644 --- a/Lib/multiprocessing/shared_memory.py +++ b/Lib/multiprocessing/shared_memory.py @@ -37,6 +37,10 @@ else: _SHM_NAME_PREFIX = 'wnsm_' +if _USE_POSIX and hasattr(_posixshmem, "shm_rename"): + from _posixshmem import SHM_RENAME_EXCHANGE + from _posixshmem import SHM_RENAME_NOREPLACE + def _make_filename(): "Create a random filename for the shared memory object." @@ -255,9 +259,6 @@ def unlink(self): resource_tracker.unregister(self._name, "shared_memory") if _USE_POSIX and hasattr(_posixshmem, "shm_rename"): - from _posixshmem import SHM_RENAME_EXCHANGE - from _posixshmem import SHM_RENAME_NOREPLACE - def rename(self, newname, flags=0): """Renames a shared memory block. diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index b83dea615c1e25..fd39dd9614812e 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -4376,7 +4376,7 @@ def test_shared_memory_recreate(self): self.addCleanup(shm2.unlink) self.assertEqual(shm2._name, names[1]) - @unittest.skipUnless(hasattr(shared_memory, "rename"), + @unittest.skipUnless(hasattr(shared_memory.SharedMemory, "rename"), "requires SharedMomery.rename") def test_shared_memory_rename(self): name1 = self._new_shm_name('testrename01_tsmb') @@ -4387,6 +4387,30 @@ def test_shared_memory_rename(self): sms.rename(name2) self.assertEqual(sms.name, name2) + @unittest.skipUnless(hasattr(shared_memory.SharedMemory, "rename"), + "requires SharedMomery.rename") + def test_shared_memory_rename_noreplace(self): + name1 = self._new_shm_name('testrename01_tsmb') + name2 = self._new_shm_name('testrename02_tsmb') + sms1 = shared_memory.SharedMemory(name1, create=True, size=512) + sms2 = shared_memory.SharedMemory(name2, create=True, size=512) + self.addCleanup(sms1.unlink) + self.addCleanup(sms2.unlink) + + with self.assertRaises(FileExistsError): + sms1.rename(name2, flags=shared_memory.SHM_RENAME_NOREPLACE) + + @unittest.skipUnless(hasattr(shared_memory.SharedMemory, "rename"), + "requires SharedMomery.rename") + def test_shared_memory_rename_exchange(self): + name1 = self._new_shm_name('testrename01_tsmb') + name2 = self._new_shm_name('testrename02_tsmb') + sms1 = shared_memory.SharedMemory(name1, create=True, size=512) + sms2 = shared_memory.SharedMemory(name2, create=True, size=512) + self.addCleanup(sms2.unlink) + + sms1.rename(name2, flags=shared_memory.SHM_RENAME_EXCHANGE) + def test_invalid_shared_memory_creation(self): # Test creating a shared memory segment with negative size with self.assertRaises(ValueError):