Skip to content

bpo-1635741 - try to port pickle to multi-stage init #21319

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

bpo-1635741 - try to port pickle to multi-stage init #21319

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Misc/NEWS.d/next/Core and Builtins/2020-07-04-09-04-41.bpo-1635741.ZsP31Y.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Port :mod:`_pickle` to multiphase initialization
259 changes: 121 additions & 138 deletions Modules/_pickle.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,9 @@
PyDoc_STRVAR(pickle_module_doc,
"Optimized C implementation for the Python pickle module.");

//TODO should we remove Pickler_Type from the clinic input here?
//similar entry was not removed in pull 20960 for _bz2

/*[clinic input]
module _pickle
class _pickle.Pickler "PicklerObject *" "&Pickler_Type"
Expand Down Expand Up @@ -173,6 +176,10 @@ typedef struct {
/* functools.partial, used for implementing __newobj_ex__ with protocols
2 and 3 */
PyObject *partial;

/* needed for multi-state initialization*/
PyTypeObject *pickler_type;
PyTypeObject *unpickler_type;
} PickleState;

/* Forward declaration of the _pickle module definition. */
Expand Down Expand Up @@ -211,6 +218,8 @@ _Pickle_ClearState(PickleState *st)
Py_CLEAR(st->codecs_encode);
Py_CLEAR(st->getattr);
Py_CLEAR(st->partial);
Py_CLEAR(st->pickler_type);
Py_CLEAR(st->unpickler_type);
}

/* Initialize the given pickle module state. */
Expand Down Expand Up @@ -708,8 +717,6 @@ typedef struct {
/* Forward declarations */
static int save(PicklerObject *, PyObject *, int);
static int save_reduce(PicklerObject *, PyObject *, PyObject *);
static PyTypeObject Pickler_Type;
static PyTypeObject Unpickler_Type;

#include "clinic/_pickle.c.h"

Expand Down Expand Up @@ -1102,11 +1109,12 @@ _Pickler_Write(PicklerObject *self, const char *s, Py_ssize_t data_len)
}

static PicklerObject *
_Pickler_New(void)
_Pickler_New(PyObject *module)
{
PicklerObject *self;

self = PyObject_GC_New(PicklerObject, &Pickler_Type);
PickleState *st = _Pickle_GetState(module);
self = PyObject_GC_New(PicklerObject, st->pickler_type);
if (self == NULL)
return NULL;

Expand Down Expand Up @@ -1586,11 +1594,12 @@ _Unpickler_MemoCleanup(UnpicklerObject *self)
}

static UnpicklerObject *
_Unpickler_New(void)
_Unpickler_New(PyObject *module)
{
UnpicklerObject *self;
PickleState *st = _Pickle_GetState(module);

self = PyObject_GC_New(UnpicklerObject, &Unpickler_Type);
self = PyObject_GC_New(UnpicklerObject, st->unpickler_type);
if (self == NULL)
return NULL;

Expand Down Expand Up @@ -5071,47 +5080,28 @@ static PyGetSetDef Pickler_getsets[] = {
{NULL}
};

static PyTypeObject Pickler_Type = {
PyVarObject_HEAD_INIT(NULL, 0)
"_pickle.Pickler" , /*tp_name*/
sizeof(PicklerObject), /*tp_basicsize*/
0, /*tp_itemsize*/
(destructor)Pickler_dealloc, /*tp_dealloc*/
0, /*tp_vectorcall_offset*/
0, /*tp_getattr*/
0, /*tp_setattr*/
0, /*tp_as_async*/
0, /*tp_repr*/
0, /*tp_as_number*/
0, /*tp_as_sequence*/
0, /*tp_as_mapping*/
0, /*tp_hash*/
0, /*tp_call*/
0, /*tp_str*/
0, /*tp_getattro*/
0, /*tp_setattro*/
0, /*tp_as_buffer*/
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
_pickle_Pickler___init____doc__, /*tp_doc*/
(traverseproc)Pickler_traverse, /*tp_traverse*/
(inquiry)Pickler_clear, /*tp_clear*/
0, /*tp_richcompare*/
0, /*tp_weaklistoffset*/
0, /*tp_iter*/
0, /*tp_iternext*/
Pickler_methods, /*tp_methods*/
Pickler_members, /*tp_members*/
Pickler_getsets, /*tp_getset*/
0, /*tp_base*/
0, /*tp_dict*/
0, /*tp_descr_get*/
0, /*tp_descr_set*/
0, /*tp_dictoffset*/
_pickle_Pickler___init__, /*tp_init*/
PyType_GenericAlloc, /*tp_alloc*/
PyType_GenericNew, /*tp_new*/
PyObject_GC_Del, /*tp_free*/
0, /*tp_is_gc*/
static PyType_Slot pickle_pickler_type_slots[] = {
{Py_tp_dealloc, Pickler_dealloc},
{Py_tp_methods, Pickler_methods},
{Py_tp_members, Pickler_members},
{Py_tp_getset, Pickler_getsets},
{Py_tp_clear, Pickler_clear},
{Py_tp_doc, (char*)_pickle_Pickler___init____doc__},
{Py_tp_traverse, Pickler_traverse},
{0, 0}
};

//TODO - how to handle this??
// Calling PyType_GetModuleState() on a subclass is not safe.
// bz2_compressor_type_spec does not have Py_TPFLAGS_BASETYPE flag
// which prevents to create a subclass.
// So calling PyType_GetModuleState() in this file is always safe.

static PyType_Spec pickle_pickler_type_spec = {
.name = "_pickle.Pickler",
.basicsize = sizeof(PicklerObject),
.flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
.slots = pickle_pickler_type_slots,
};

/* Temporary helper for calling self.find_class().
Expand Down Expand Up @@ -7623,47 +7613,21 @@ static PyGetSetDef Unpickler_getsets[] = {
{NULL}
};

static PyTypeObject Unpickler_Type = {
PyVarObject_HEAD_INIT(NULL, 0)
"_pickle.Unpickler", /*tp_name*/
sizeof(UnpicklerObject), /*tp_basicsize*/
0, /*tp_itemsize*/
(destructor)Unpickler_dealloc, /*tp_dealloc*/
0, /*tp_vectorcall_offset*/
0, /*tp_getattr*/
0, /*tp_setattr*/
0, /*tp_as_async*/
0, /*tp_repr*/
0, /*tp_as_number*/
0, /*tp_as_sequence*/
0, /*tp_as_mapping*/
0, /*tp_hash*/
0, /*tp_call*/
0, /*tp_str*/
0, /*tp_getattro*/
0, /*tp_setattro*/
0, /*tp_as_buffer*/
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
_pickle_Unpickler___init____doc__, /*tp_doc*/
(traverseproc)Unpickler_traverse, /*tp_traverse*/
(inquiry)Unpickler_clear, /*tp_clear*/
0, /*tp_richcompare*/
0, /*tp_weaklistoffset*/
0, /*tp_iter*/
0, /*tp_iternext*/
Unpickler_methods, /*tp_methods*/
0, /*tp_members*/
Unpickler_getsets, /*tp_getset*/
0, /*tp_base*/
0, /*tp_dict*/
0, /*tp_descr_get*/
0, /*tp_descr_set*/
0, /*tp_dictoffset*/
_pickle_Unpickler___init__, /*tp_init*/
PyType_GenericAlloc, /*tp_alloc*/
PyType_GenericNew, /*tp_new*/
PyObject_GC_Del, /*tp_free*/
0, /*tp_is_gc*/
static PyType_Slot pickle_unpickler_type_slots[] = {
{Py_tp_dealloc, Unpickler_dealloc},
{Py_tp_doc, (char *)_pickle_Unpickler___init____doc__},
{Py_tp_traverse, Unpickler_traverse},
{Py_tp_clear, Unpickler_clear},
{Py_tp_methods, Unpickler_methods},
{Py_tp_getset, Unpickler_getsets},
{0,0}
};

static PyType_Spec pickle_unpickler_type_spec = {
.name = "_pickle.Unpickler",
.basicsize = sizeof(UnpicklerObject),
.flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
.slots = pickle_unpickler_type_slots,
};

/*[clinic input]
Expand Down Expand Up @@ -7712,7 +7676,7 @@ _pickle_dump_impl(PyObject *module, PyObject *obj, PyObject *file,
PyObject *buffer_callback)
/*[clinic end generated code: output=706186dba996490c input=5ed6653da99cd97c]*/
{
PicklerObject *pickler = _Pickler_New();
PicklerObject *pickler = _Pickler_New(module);

if (pickler == NULL)
return NULL;
Expand Down Expand Up @@ -7777,7 +7741,7 @@ _pickle_dumps_impl(PyObject *module, PyObject *obj, PyObject *protocol,
/*[clinic end generated code: output=fbab0093a5580fdf input=e543272436c6f987]*/
{
PyObject *result;
PicklerObject *pickler = _Pickler_New();
PicklerObject *pickler = _Pickler_New(module);

if (pickler == NULL)
return NULL;
Expand Down Expand Up @@ -7843,7 +7807,7 @@ _pickle_load_impl(PyObject *module, PyObject *file, int fix_imports,
/*[clinic end generated code: output=250452d141c23e76 input=46c7c31c92f4f371]*/
{
PyObject *result;
UnpicklerObject *unpickler = _Unpickler_New();
UnpicklerObject *unpickler = _Unpickler_New(module);

if (unpickler == NULL)
return NULL;
Expand Down Expand Up @@ -7903,7 +7867,7 @@ _pickle_loads_impl(PyObject *module, PyObject *data, int fix_imports,
/*[clinic end generated code: output=82ac1e6b588e6d02 input=b3615540d0535087]*/
{
PyObject *result;
UnpicklerObject *unpickler = _Unpickler_New();
UnpicklerObject *unpickler = _Unpickler_New(module);

if (unpickler == NULL)
return NULL;
Expand Down Expand Up @@ -7967,83 +7931,102 @@ pickle_traverse(PyObject *m, visitproc visit, void *arg)
Py_VISIT(st->codecs_encode);
Py_VISIT(st->getattr);
Py_VISIT(st->partial);
Py_VISIT(st->pickler_type);
Py_VISIT(st->unpickler_type);
return 0;
}

static struct PyModuleDef _picklemodule = {
PyModuleDef_HEAD_INIT,
"_pickle", /* m_name */
pickle_module_doc, /* m_doc */
sizeof(PickleState), /* m_size */
pickle_methods, /* m_methods */
NULL, /* m_reload */
pickle_traverse, /* m_traverse */
pickle_clear, /* m_clear */
(freefunc)pickle_free /* m_free */
};

PyMODINIT_FUNC
PyInit__pickle(void)
static int
_pickle_exec(PyObject *m)
{
PyObject *m;
PickleState *st;

m = PyState_FindModule(&_picklemodule);
if (m) {
Py_INCREF(m);
return m;
}

if (PyType_Ready(&Pdata_Type) < 0)
return NULL;
return -1;
if (PyType_Ready(&PicklerMemoProxyType) < 0)
return NULL;
return -1;
if (PyType_Ready(&UnpicklerMemoProxyType) < 0)
return NULL;
return -1;

/* Create the module and add the functions. */
m = PyModule_Create(&_picklemodule);
if (m == NULL)
return NULL;
st = _Pickle_GetState(m);

/* Add types */
if (PyModule_AddType(m, &Pickler_Type) < 0) {
return NULL;
if (PyModule_AddType(m, &PyPickleBuffer_Type) < 0) {
return -1;
}
if (PyModule_AddType(m, &Unpickler_Type) < 0) {
return NULL;

st->pickler_type = (PyTypeObject *)PyType_FromModuleAndSpec(m, &pickle_pickler_type_spec, NULL);
if (st->pickler_type == NULL) {
return -1;
}
if (PyModule_AddType(m, &PyPickleBuffer_Type) < 0) {
return NULL;

if (PyModule_AddType(m, st->pickler_type) < 0) {
return -1;
}

st = _Pickle_GetState(m);
st->unpickler_type = (PyTypeObject *)PyType_FromModuleAndSpec(m, &pickle_unpickler_type_spec, NULL);
if (st->unpickler_type == NULL) {
return -1;
}

if (PyModule_AddType(m, st->unpickler_type) < 0) {
return -1;
}

/* Initialize the exceptions. */
st->PickleError = PyErr_NewException("_pickle.PickleError", NULL, NULL);
if (st->PickleError == NULL)
return NULL;
return -1;
st->PicklingError = \
PyErr_NewException("_pickle.PicklingError", st->PickleError, NULL);
if (st->PicklingError == NULL)
return NULL;
return -1;
st->UnpicklingError = \
PyErr_NewException("_pickle.UnpicklingError", st->PickleError, NULL);
if (st->UnpicklingError == NULL)
return NULL;
return -1;

Py_INCREF(st->PickleError);
if (PyModule_AddObject(m, "PickleError", st->PickleError) < 0)
return NULL;
if (PyModule_AddObject(m, "PickleError", st->PickleError) < 0) {
Py_DECREF((PyObject *)st->PickleError);
return -1;
}
Py_INCREF(st->PicklingError);
if (PyModule_AddObject(m, "PicklingError", st->PicklingError) < 0)
return NULL;
if (PyModule_AddObject(m, "PicklingError", st->PicklingError) < 0) {
Py_DECREF((PyObject *)st->PicklingError);
return -1;
}
Py_INCREF(st->UnpicklingError);
if (PyModule_AddObject(m, "UnpicklingError", st->UnpicklingError) < 0)
return NULL;
if (PyModule_AddObject(m, "UnpicklingError", st->UnpicklingError) < 0) {
Py_DECREF((PyObject *)st->UnpicklingError);
return -1;
}

if (_Pickle_InitState(st) < 0)
return NULL;
return -1;

return 0;
}

return m;
static PyModuleDef_Slot pickle_slots[] = {
{Py_mod_exec, _pickle_exec},
{0, NULL}
};

static struct PyModuleDef _picklemodule = {
PyModuleDef_HEAD_INIT,
.m_name = "_pickle",
.m_doc = pickle_module_doc,
.m_size = sizeof(PickleState),
.m_methods = pickle_methods,
.m_slots = pickle_slots,
.m_traverse = pickle_traverse,
.m_clear = pickle_clear,
.m_free = (freefunc)pickle_free
};

PyMODINIT_FUNC
PyInit__pickle(void)
{
return PyModuleDef_Init(&_picklemodule);
}