From 02eddb90062706b15d7b4db278a5069a2118a327 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 22 Apr 2024 11:40:49 -0600 Subject: [PATCH 1/9] Add struct _Py_ext_module_loader_info. --- Include/internal/pycore_importdl.h | 16 +++- Python/importdl.c | 117 ++++++++++++++++++----------- 2 files changed, 87 insertions(+), 46 deletions(-) diff --git a/Include/internal/pycore_importdl.h b/Include/internal/pycore_importdl.h index c8583582b358ac..08d6fea2da6011 100644 --- a/Include/internal/pycore_importdl.h +++ b/Include/internal/pycore_importdl.h @@ -14,10 +14,24 @@ extern "C" { extern const char *_PyImport_DynLoadFiletab[]; -extern PyObject *_PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *); typedef PyObject *(*PyModInitFunction)(void); +struct _Py_ext_module_loader_info { + PyObject *path; + PyObject *name; + PyObject *name_encoded; + const char *hook_prefix; +}; +extern void _Py_ext_module_loader_info_clear( + struct _Py_ext_module_loader_info *info); +extern int _Py_ext_module_loader_info_from_spec( + PyObject *spec, + struct _Py_ext_module_loader_info *info); + +extern PyObject *_PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *); + + /* Max length of module suffix searched for -- accommodates "module.slb" */ #define MAXSUFFIXSIZE 12 diff --git a/Python/importdl.c b/Python/importdl.c index e512161d3071f2..580ea90a958f33 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -93,57 +93,90 @@ get_encoded_name(PyObject *name, const char **hook_prefix) { return NULL; } -PyObject * -_PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp) +void +_Py_ext_module_loader_info_clear(struct _Py_ext_module_loader_info *info) { -#ifndef MS_WINDOWS - PyObject *filename_bytes = NULL; - const char *filename_buf; -#endif - PyObject *name_unicode = NULL, *name = NULL, *filename = NULL, *m = NULL; - const char *name_buf, *hook_prefix; - const char *oldcontext, *newcontext; - dl_funcptr exportfunc; - PyModuleDef *def; - PyModInitFunction p0; + Py_CLEAR(info->path); + Py_CLEAR(info->name); + Py_CLEAR(info->name_encoded); +} + +int +_Py_ext_module_loader_info_from_spec(PyObject *spec, + struct _Py_ext_module_loader_info *info) +{ + PyObject *name_unicode = NULL, *name = NULL, *path = NULL; + const char *hook_prefix; name_unicode = PyObject_GetAttrString(spec, "name"); if (name_unicode == NULL) { - return NULL; + return -1; } if (!PyUnicode_Check(name_unicode)) { PyErr_SetString(PyExc_TypeError, "spec.name must be a string"); - goto error; - } - newcontext = PyUnicode_AsUTF8(name_unicode); - if (newcontext == NULL) { - goto error; + Py_DECREF(name_unicode); + return -1; } name = get_encoded_name(name_unicode, &hook_prefix); if (name == NULL) { - goto error; + Py_DECREF(name_unicode); + return -1; + } + + path = PyObject_GetAttrString(spec, "origin"); + if (path == NULL) { + Py_DECREF(name_unicode); + Py_DECREF(name); + return -1; } - name_buf = PyBytes_AS_STRING(name); - filename = PyObject_GetAttrString(spec, "origin"); - if (filename == NULL) { + *info = (struct _Py_ext_module_loader_info){ + .path=path, + .name=name_unicode, + .name_encoded=name, + .hook_prefix=hook_prefix, + }; + return 0; +} + +PyObject * +_PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp) +{ + struct _Py_ext_module_loader_info info; + if (_Py_ext_module_loader_info_from_spec(spec, &info) < 0) { + return NULL; + } + +#ifndef MS_WINDOWS + PyObject *pathbytes = NULL; + const char *path_buf; +#endif + PyObject *m = NULL; + const char *name_buf = PyBytes_AS_STRING(info.name_encoded); + const char *oldcontext, *newcontext; + dl_funcptr exportfunc; + PyModInitFunction p0; + PyModuleDef *def; + + newcontext = PyUnicode_AsUTF8(info.name); + if (newcontext == NULL) { goto error; } #ifdef MS_WINDOWS exportfunc = _PyImport_FindSharedFuncptrWindows( - hook_prefix, name_buf, filename, fp); + info.hook_prefix, name_buf, info.path, fp); #else - filename_bytes = PyUnicode_EncodeFSDefault(filename); - if (filename_bytes == NULL) { + pathbytes = PyUnicode_EncodeFSDefault(info.path); + if (pathbytes == NULL) { goto error; } - filename_buf = PyBytes_AS_STRING(filename_bytes); + path_buf = PyBytes_AS_STRING(pathbytes); exportfunc = _PyImport_FindSharedFuncptr( - hook_prefix, name_buf, filename_buf, fp); - Py_DECREF(filename_bytes); + info.hook_prefix, name_buf, path_buf, fp); + Py_DECREF(pathbytes); #endif if (exportfunc == NULL) { @@ -152,11 +185,11 @@ _PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp) msg = PyUnicode_FromFormat( "dynamic module does not define " "module export function (%s_%s)", - hook_prefix, name_buf); - if (msg == NULL) - goto error; - PyErr_SetImportError(msg, name_unicode, filename); - Py_DECREF(msg); + info.hook_prefix, name_buf); + if (msg != NULL) { + PyErr_SetImportError(msg, info.name, info.path); + Py_DECREF(msg); + } } goto error; } @@ -195,9 +228,7 @@ _PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp) goto error; } if (PyObject_TypeCheck(m, &PyModuleDef_Type)) { - Py_DECREF(name_unicode); - Py_DECREF(name); - Py_DECREF(filename); + _Py_ext_module_loader_info_clear(&info); return PyModule_FromDefAndSpec((PyModuleDef*)m, spec); } @@ -207,7 +238,7 @@ _PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp) goto error; } - if (hook_prefix == nonascii_prefix) { + if (info.hook_prefix == nonascii_prefix) { /* don't allow legacy init for non-ASCII module names */ PyErr_Format( PyExc_SystemError, @@ -232,19 +263,15 @@ _PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp) } PyObject *modules = PyImport_GetModuleDict(); - if (_PyImport_FixupExtensionObject(m, name_unicode, filename, modules) < 0) + if (_PyImport_FixupExtensionObject(m, info.name, info.path, modules) < 0) { goto error; + } - Py_DECREF(name_unicode); - Py_DECREF(name); - Py_DECREF(filename); - + _Py_ext_module_loader_info_clear(&info); return m; error: - Py_DECREF(name_unicode); - Py_XDECREF(name); - Py_XDECREF(filename); + _Py_ext_module_loader_info_clear(&info); Py_XDECREF(m); return NULL; } From c9600ba17a8dd12fff6b3505b4550ecc72bd5e0a Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 22 Apr 2024 11:48:07 -0600 Subject: [PATCH 2/9] Pass info to _PyImport_LoadDynamicModuleWithSpec(). --- Include/internal/pycore_importdl.h | 5 ++++- Python/import.c | 23 ++++++++-------------- Python/importdl.c | 31 ++++++++++++------------------ 3 files changed, 24 insertions(+), 35 deletions(-) diff --git a/Include/internal/pycore_importdl.h b/Include/internal/pycore_importdl.h index 08d6fea2da6011..fcb0639932e2a2 100644 --- a/Include/internal/pycore_importdl.h +++ b/Include/internal/pycore_importdl.h @@ -29,7 +29,10 @@ extern int _Py_ext_module_loader_info_from_spec( PyObject *spec, struct _Py_ext_module_loader_info *info); -extern PyObject *_PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *); +extern PyObject *_PyImport_LoadDynamicModuleWithSpec( + struct _Py_ext_module_loader_info *info, + PyObject *spec, + FILE *fp); /* Max length of module suffix searched for -- accommodates "module.slb" */ diff --git a/Python/import.c b/Python/import.c index 739f506fe03100..aa149d54880e43 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3878,28 +3878,22 @@ static PyObject * _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) /*[clinic end generated code: output=83249b827a4fde77 input=c31b954f4cf4e09d]*/ { - PyObject *mod, *name, *filename; + PyObject *mod = NULL; FILE *fp; - name = PyObject_GetAttrString(spec, "name"); - if (name == NULL) { - return NULL; - } - - filename = PyObject_GetAttrString(spec, "origin"); - if (filename == NULL) { - Py_DECREF(name); + struct _Py_ext_module_loader_info info; + if (_Py_ext_module_loader_info_from_spec(spec, &info) < 0) { return NULL; } PyThreadState *tstate = _PyThreadState_GET(); - mod = import_find_extension(tstate, name, filename); + mod = import_find_extension(tstate, info.name, info.path); if (mod != NULL || _PyErr_Occurred(tstate)) { assert(mod == NULL || !_PyErr_Occurred(tstate)); goto finally; } - if (PySys_Audit("import", "OOOOO", name, filename, + if (PySys_Audit("import", "OOOOO", info.name, info.path, Py_None, Py_None, Py_None) < 0) { goto finally; @@ -3911,7 +3905,7 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) * _PyImport_GetModInitFunc(), but it isn't clear if the intervening * code relies on fp still being open. */ if (file != NULL) { - fp = _Py_fopen_obj(filename, "r"); + fp = _Py_fopen_obj(info.path, "r"); if (fp == NULL) { goto finally; } @@ -3920,7 +3914,7 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) fp = NULL; } - mod = _PyImport_LoadDynamicModuleWithSpec(spec, fp); + mod = _PyImport_LoadDynamicModuleWithSpec(&info, spec, fp); // XXX Shouldn't this happen in the error cases too. if (fp) { @@ -3928,8 +3922,7 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) } finally: - Py_DECREF(name); - Py_DECREF(filename); + _Py_ext_module_loader_info_clear(&info); return mod; } diff --git a/Python/importdl.c b/Python/importdl.c index 580ea90a958f33..fb1932006ed531 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -142,40 +142,36 @@ _Py_ext_module_loader_info_from_spec(PyObject *spec, } PyObject * -_PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp) +_PyImport_LoadDynamicModuleWithSpec(struct _Py_ext_module_loader_info *info, + PyObject *spec, FILE *fp) { - struct _Py_ext_module_loader_info info; - if (_Py_ext_module_loader_info_from_spec(spec, &info) < 0) { - return NULL; - } - #ifndef MS_WINDOWS PyObject *pathbytes = NULL; const char *path_buf; #endif PyObject *m = NULL; - const char *name_buf = PyBytes_AS_STRING(info.name_encoded); + const char *name_buf = PyBytes_AS_STRING(info->name_encoded); const char *oldcontext, *newcontext; dl_funcptr exportfunc; PyModInitFunction p0; PyModuleDef *def; - newcontext = PyUnicode_AsUTF8(info.name); + newcontext = PyUnicode_AsUTF8(info->name); if (newcontext == NULL) { goto error; } #ifdef MS_WINDOWS exportfunc = _PyImport_FindSharedFuncptrWindows( - info.hook_prefix, name_buf, info.path, fp); + info->hook_prefix, name_buf, info->path, fp); #else - pathbytes = PyUnicode_EncodeFSDefault(info.path); + pathbytes = PyUnicode_EncodeFSDefault(info->path); if (pathbytes == NULL) { goto error; } path_buf = PyBytes_AS_STRING(pathbytes); exportfunc = _PyImport_FindSharedFuncptr( - info.hook_prefix, name_buf, path_buf, fp); + info->hook_prefix, name_buf, path_buf, fp); Py_DECREF(pathbytes); #endif @@ -185,9 +181,9 @@ _PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp) msg = PyUnicode_FromFormat( "dynamic module does not define " "module export function (%s_%s)", - info.hook_prefix, name_buf); + info->hook_prefix, name_buf); if (msg != NULL) { - PyErr_SetImportError(msg, info.name, info.path); + PyErr_SetImportError(msg, info->name, info->path); Py_DECREF(msg); } } @@ -228,7 +224,6 @@ _PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp) goto error; } if (PyObject_TypeCheck(m, &PyModuleDef_Type)) { - _Py_ext_module_loader_info_clear(&info); return PyModule_FromDefAndSpec((PyModuleDef*)m, spec); } @@ -238,7 +233,7 @@ _PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp) goto error; } - if (info.hook_prefix == nonascii_prefix) { + if (info->hook_prefix == nonascii_prefix) { /* don't allow legacy init for non-ASCII module names */ PyErr_Format( PyExc_SystemError, @@ -258,20 +253,18 @@ _PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp) def->m_base.m_init = p0; /* Remember the filename as the __file__ attribute */ - if (PyModule_AddObjectRef(m, "__file__", filename) < 0) { + if (PyModule_AddObjectRef(m, "__file__", info->path) < 0) { PyErr_Clear(); /* Not important enough to report */ } PyObject *modules = PyImport_GetModuleDict(); - if (_PyImport_FixupExtensionObject(m, info.name, info.path, modules) < 0) { + if (_PyImport_FixupExtensionObject(m, info->name, info->path, modules) < 0) { goto error; } - _Py_ext_module_loader_info_clear(&info); return m; error: - _Py_ext_module_loader_info_clear(&info); Py_XDECREF(m); return NULL; } From 5d927b76b00224187013e1158a2083807570e1eb Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 22 Apr 2024 12:03:31 -0600 Subject: [PATCH 3/9] Calculate path_bytes ahead of time. --- Include/internal/pycore_importdl.h | 3 ++ Python/importdl.c | 59 ++++++++++++++---------------- 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/Include/internal/pycore_importdl.h b/Include/internal/pycore_importdl.h index fcb0639932e2a2..bfdbd94fbe0aef 100644 --- a/Include/internal/pycore_importdl.h +++ b/Include/internal/pycore_importdl.h @@ -19,6 +19,9 @@ typedef PyObject *(*PyModInitFunction)(void); struct _Py_ext_module_loader_info { PyObject *path; +#ifndef MS_WINDOWS + PyObject *path_encoded; +#endif PyObject *name; PyObject *name_encoded; const char *hook_prefix; diff --git a/Python/importdl.c b/Python/importdl.c index fb1932006ed531..15764e5e25823c 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -97,47 +97,51 @@ void _Py_ext_module_loader_info_clear(struct _Py_ext_module_loader_info *info) { Py_CLEAR(info->path); +#ifndef MS_WINDOWS + Py_CLEAR(info->path_encoded); +#endif Py_CLEAR(info->name); Py_CLEAR(info->name_encoded); } int _Py_ext_module_loader_info_from_spec(PyObject *spec, - struct _Py_ext_module_loader_info *info) + struct _Py_ext_module_loader_info *p_info) { - PyObject *name_unicode = NULL, *name = NULL, *path = NULL; - const char *hook_prefix; + struct _Py_ext_module_loader_info info = {0}; - name_unicode = PyObject_GetAttrString(spec, "name"); - if (name_unicode == NULL) { + info.name = PyObject_GetAttrString(spec, "name"); + if (info.name == NULL) { return -1; } - if (!PyUnicode_Check(name_unicode)) { + if (!PyUnicode_Check(info.name)) { PyErr_SetString(PyExc_TypeError, "spec.name must be a string"); - Py_DECREF(name_unicode); + _Py_ext_module_loader_info_clear(&info); + return -1; + } + + info.name_encoded = get_encoded_name(info.name, &info.hook_prefix); + if (info.name_encoded == NULL) { + _Py_ext_module_loader_info_clear(&info); return -1; } - name = get_encoded_name(name_unicode, &hook_prefix); - if (name == NULL) { - Py_DECREF(name_unicode); + info.path = PyObject_GetAttrString(spec, "origin"); + if (info.path == NULL) { + _Py_ext_module_loader_info_clear(&info); return -1; } - path = PyObject_GetAttrString(spec, "origin"); - if (path == NULL) { - Py_DECREF(name_unicode); - Py_DECREF(name); +#ifndef MS_WINDOWS + info.path_encoded = PyUnicode_EncodeFSDefault(info.path); + if (info.path_encoded == NULL) { + _Py_ext_module_loader_info_clear(&info); return -1; } +#endif - *info = (struct _Py_ext_module_loader_info){ - .path=path, - .name=name_unicode, - .name_encoded=name, - .hook_prefix=hook_prefix, - }; + *p_info = info; return 0; } @@ -145,10 +149,6 @@ PyObject * _PyImport_LoadDynamicModuleWithSpec(struct _Py_ext_module_loader_info *info, PyObject *spec, FILE *fp) { -#ifndef MS_WINDOWS - PyObject *pathbytes = NULL; - const char *path_buf; -#endif PyObject *m = NULL; const char *name_buf = PyBytes_AS_STRING(info->name_encoded); const char *oldcontext, *newcontext; @@ -165,14 +165,11 @@ _PyImport_LoadDynamicModuleWithSpec(struct _Py_ext_module_loader_info *info, exportfunc = _PyImport_FindSharedFuncptrWindows( info->hook_prefix, name_buf, info->path, fp); #else - pathbytes = PyUnicode_EncodeFSDefault(info->path); - if (pathbytes == NULL) { - goto error; + { + const char *path_buf = PyBytes_AS_STRING(info->path_encoded); + exportfunc = _PyImport_FindSharedFuncptr( + info->hook_prefix, name_buf, path_buf, fp); } - path_buf = PyBytes_AS_STRING(pathbytes); - exportfunc = _PyImport_FindSharedFuncptr( - info->hook_prefix, name_buf, path_buf, fp); - Py_DECREF(pathbytes); #endif if (exportfunc == NULL) { From 427280fcb94dc8887ab96fae030d2fa9b33a8a04 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 16 Apr 2024 13:07:30 -0600 Subject: [PATCH 4/9] _Py_ext_module_loader_info_from_spec() -> _Py_ext_module_loader_info_init_from_spec() --- Include/internal/pycore_importdl.h | 6 +++--- Python/import.c | 2 +- Python/importdl.c | 5 +++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Include/internal/pycore_importdl.h b/Include/internal/pycore_importdl.h index bfdbd94fbe0aef..63e8d1902c253b 100644 --- a/Include/internal/pycore_importdl.h +++ b/Include/internal/pycore_importdl.h @@ -28,9 +28,9 @@ struct _Py_ext_module_loader_info { }; extern void _Py_ext_module_loader_info_clear( struct _Py_ext_module_loader_info *info); -extern int _Py_ext_module_loader_info_from_spec( - PyObject *spec, - struct _Py_ext_module_loader_info *info); +extern int _Py_ext_module_loader_info_init_from_spec( + struct _Py_ext_module_loader_info *info, + PyObject *spec); extern PyObject *_PyImport_LoadDynamicModuleWithSpec( struct _Py_ext_module_loader_info *info, diff --git a/Python/import.c b/Python/import.c index aa149d54880e43..d8bc89ffa66b27 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3882,7 +3882,7 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) FILE *fp; struct _Py_ext_module_loader_info info; - if (_Py_ext_module_loader_info_from_spec(spec, &info) < 0) { + if (_Py_ext_module_loader_info_init_from_spec(&info, spec) < 0) { return NULL; } diff --git a/Python/importdl.c b/Python/importdl.c index 15764e5e25823c..1c37361f6c7980 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -105,8 +105,9 @@ _Py_ext_module_loader_info_clear(struct _Py_ext_module_loader_info *info) } int -_Py_ext_module_loader_info_from_spec(PyObject *spec, - struct _Py_ext_module_loader_info *p_info) +_Py_ext_module_loader_info_init_from_spec( + struct _Py_ext_module_loader_info *p_info, + PyObject *spec) { struct _Py_ext_module_loader_info info = {0}; From d5f97e7e70b0c2f8298880d95da79b7b0e1995fa Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 16 Apr 2024 14:17:03 -0600 Subject: [PATCH 5/9] Add info.newcontext. --- Include/internal/pycore_importdl.h | 1 + Python/importdl.c | 15 ++++++++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Include/internal/pycore_importdl.h b/Include/internal/pycore_importdl.h index 63e8d1902c253b..1a8dc586eab33c 100644 --- a/Include/internal/pycore_importdl.h +++ b/Include/internal/pycore_importdl.h @@ -25,6 +25,7 @@ struct _Py_ext_module_loader_info { PyObject *name; PyObject *name_encoded; const char *hook_prefix; + const char *newcontext; }; extern void _Py_ext_module_loader_info_clear( struct _Py_ext_module_loader_info *info); diff --git a/Python/importdl.c b/Python/importdl.c index 1c37361f6c7980..e057e79f035495 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -128,6 +128,12 @@ _Py_ext_module_loader_info_init_from_spec( return -1; } + info.newcontext = PyUnicode_AsUTF8(info.name); + if (info.newcontext == NULL) { + _Py_ext_module_loader_info_clear(&info); + return -1; + } + info.path = PyObject_GetAttrString(spec, "origin"); if (info.path == NULL) { _Py_ext_module_loader_info_clear(&info); @@ -152,16 +158,11 @@ _PyImport_LoadDynamicModuleWithSpec(struct _Py_ext_module_loader_info *info, { PyObject *m = NULL; const char *name_buf = PyBytes_AS_STRING(info->name_encoded); - const char *oldcontext, *newcontext; + const char *oldcontext; dl_funcptr exportfunc; PyModInitFunction p0; PyModuleDef *def; - newcontext = PyUnicode_AsUTF8(info->name); - if (newcontext == NULL) { - goto error; - } - #ifdef MS_WINDOWS exportfunc = _PyImport_FindSharedFuncptrWindows( info->hook_prefix, name_buf, info->path, fp); @@ -191,7 +192,7 @@ _PyImport_LoadDynamicModuleWithSpec(struct _Py_ext_module_loader_info *info, p0 = (PyModInitFunction)exportfunc; /* Package context is needed for single-phase init */ - oldcontext = _PyImport_SwapPackageContext(newcontext); + oldcontext = _PyImport_SwapPackageContext(info->newcontext); m = p0(); _PyImport_SwapPackageContext(oldcontext); From 5d951b32c6758365ac4ba3dc867e891179976312 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 22 Apr 2024 16:25:20 -0600 Subject: [PATCH 6/9] path -> filename --- Include/internal/pycore_importdl.h | 4 ++-- Python/import.c | 6 +++--- Python/importdl.c | 24 +++++++++++++----------- 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/Include/internal/pycore_importdl.h b/Include/internal/pycore_importdl.h index 1a8dc586eab33c..d84a0fefcb9314 100644 --- a/Include/internal/pycore_importdl.h +++ b/Include/internal/pycore_importdl.h @@ -18,9 +18,9 @@ extern const char *_PyImport_DynLoadFiletab[]; typedef PyObject *(*PyModInitFunction)(void); struct _Py_ext_module_loader_info { - PyObject *path; + PyObject *filename; #ifndef MS_WINDOWS - PyObject *path_encoded; + PyObject *filename_encoded; #endif PyObject *name; PyObject *name_encoded; diff --git a/Python/import.c b/Python/import.c index d8bc89ffa66b27..90a5105b0e04b1 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3887,13 +3887,13 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) } PyThreadState *tstate = _PyThreadState_GET(); - mod = import_find_extension(tstate, info.name, info.path); + mod = import_find_extension(tstate, info.name, info.filename); if (mod != NULL || _PyErr_Occurred(tstate)) { assert(mod == NULL || !_PyErr_Occurred(tstate)); goto finally; } - if (PySys_Audit("import", "OOOOO", info.name, info.path, + if (PySys_Audit("import", "OOOOO", info.name, info.filename, Py_None, Py_None, Py_None) < 0) { goto finally; @@ -3905,7 +3905,7 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) * _PyImport_GetModInitFunc(), but it isn't clear if the intervening * code relies on fp still being open. */ if (file != NULL) { - fp = _Py_fopen_obj(info.path, "r"); + fp = _Py_fopen_obj(info.filename, "r"); if (fp == NULL) { goto finally; } diff --git a/Python/importdl.c b/Python/importdl.c index e057e79f035495..479906b800a699 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -96,9 +96,9 @@ get_encoded_name(PyObject *name, const char **hook_prefix) { void _Py_ext_module_loader_info_clear(struct _Py_ext_module_loader_info *info) { - Py_CLEAR(info->path); + Py_CLEAR(info->filename); #ifndef MS_WINDOWS - Py_CLEAR(info->path_encoded); + Py_CLEAR(info->filename_encoded); #endif Py_CLEAR(info->name); Py_CLEAR(info->name_encoded); @@ -134,15 +134,15 @@ _Py_ext_module_loader_info_init_from_spec( return -1; } - info.path = PyObject_GetAttrString(spec, "origin"); - if (info.path == NULL) { + info.filename = PyObject_GetAttrString(spec, "origin"); + if (info.filename == NULL) { _Py_ext_module_loader_info_clear(&info); return -1; } #ifndef MS_WINDOWS - info.path_encoded = PyUnicode_EncodeFSDefault(info.path); - if (info.path_encoded == NULL) { + info.filename_encoded = PyUnicode_EncodeFSDefault(info.filename); + if (info.filename_encoded == NULL) { _Py_ext_module_loader_info_clear(&info); return -1; } @@ -165,10 +165,10 @@ _PyImport_LoadDynamicModuleWithSpec(struct _Py_ext_module_loader_info *info, #ifdef MS_WINDOWS exportfunc = _PyImport_FindSharedFuncptrWindows( - info->hook_prefix, name_buf, info->path, fp); + info->hook_prefix, name_buf, info->filename, fp); #else { - const char *path_buf = PyBytes_AS_STRING(info->path_encoded); + const char *path_buf = PyBytes_AS_STRING(info->filename_encoded); exportfunc = _PyImport_FindSharedFuncptr( info->hook_prefix, name_buf, path_buf, fp); } @@ -182,7 +182,7 @@ _PyImport_LoadDynamicModuleWithSpec(struct _Py_ext_module_loader_info *info, "module export function (%s_%s)", info->hook_prefix, name_buf); if (msg != NULL) { - PyErr_SetImportError(msg, info->name, info->path); + PyErr_SetImportError(msg, info->name, info->filename); Py_DECREF(msg); } } @@ -252,12 +252,14 @@ _PyImport_LoadDynamicModuleWithSpec(struct _Py_ext_module_loader_info *info, def->m_base.m_init = p0; /* Remember the filename as the __file__ attribute */ - if (PyModule_AddObjectRef(m, "__file__", info->path) < 0) { + if (PyModule_AddObjectRef(m, "__file__", info->filename) < 0) { PyErr_Clear(); /* Not important enough to report */ } PyObject *modules = PyImport_GetModuleDict(); - if (_PyImport_FixupExtensionObject(m, info->name, info->path, modules) < 0) { + if (_PyImport_FixupExtensionObject( + m, info->name, info->filename, modules) < 0) + { goto error; } From 481bf5d9765c3105a26865d1ce9ad60e54b98222 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 23 Apr 2024 16:50:01 -0600 Subject: [PATCH 7/9] Factor out _Py_ext_module_loader_info_init(). --- Include/internal/pycore_importdl.h | 4 +++ Python/importdl.c | 56 ++++++++++++++++++++---------- 2 files changed, 41 insertions(+), 19 deletions(-) diff --git a/Include/internal/pycore_importdl.h b/Include/internal/pycore_importdl.h index d84a0fefcb9314..12e81509762c57 100644 --- a/Include/internal/pycore_importdl.h +++ b/Include/internal/pycore_importdl.h @@ -29,6 +29,10 @@ struct _Py_ext_module_loader_info { }; extern void _Py_ext_module_loader_info_clear( struct _Py_ext_module_loader_info *info); +extern int _Py_ext_module_loader_info_init( + struct _Py_ext_module_loader_info *info, + PyObject *name, + PyObject *filename); extern int _Py_ext_module_loader_info_init_from_spec( struct _Py_ext_module_loader_info *info, PyObject *spec); diff --git a/Python/importdl.c b/Python/importdl.c index 479906b800a699..51bb9808541a7a 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -105,22 +105,19 @@ _Py_ext_module_loader_info_clear(struct _Py_ext_module_loader_info *info) } int -_Py_ext_module_loader_info_init_from_spec( - struct _Py_ext_module_loader_info *p_info, - PyObject *spec) +_Py_ext_module_loader_info_init(struct _Py_ext_module_loader_info *p_info, + PyObject *name, PyObject *filename) { struct _Py_ext_module_loader_info info = {0}; - info.name = PyObject_GetAttrString(spec, "name"); - if (info.name == NULL) { - return -1; - } - if (!PyUnicode_Check(info.name)) { + assert(name != NULL); + if (!PyUnicode_Check(name)) { PyErr_SetString(PyExc_TypeError, - "spec.name must be a string"); + "module name must be a string"); _Py_ext_module_loader_info_clear(&info); return -1; } + info.name = Py_NewRef(name); info.name_encoded = get_encoded_name(info.name, &info.hook_prefix); if (info.name_encoded == NULL) { @@ -134,24 +131,45 @@ _Py_ext_module_loader_info_init_from_spec( return -1; } - info.filename = PyObject_GetAttrString(spec, "origin"); - if (info.filename == NULL) { - _Py_ext_module_loader_info_clear(&info); - return -1; - } + if (filename != NULL) { + if (!PyUnicode_Check(filename)) { + PyErr_SetString(PyExc_TypeError, + "module filename must be a string"); + _Py_ext_module_loader_info_clear(&info); + return -1; + } + info.filename = Py_NewRef(filename); #ifndef MS_WINDOWS - info.filename_encoded = PyUnicode_EncodeFSDefault(info.filename); - if (info.filename_encoded == NULL) { - _Py_ext_module_loader_info_clear(&info); - return -1; - } + info.filename_encoded = PyUnicode_EncodeFSDefault(info.filename); + if (info.filename_encoded == NULL) { + _Py_ext_module_loader_info_clear(&info); + return -1; + } #endif + } *p_info = info; return 0; } +int +_Py_ext_module_loader_info_init_from_spec( + struct _Py_ext_module_loader_info *p_info, + PyObject *spec) +{ + PyObject *name = PyObject_GetAttrString(spec, "name"); + if (name == NULL) { + return -1; + } + PyObject *filename = PyObject_GetAttrString(spec, "origin"); + if (filename == NULL) { + return -1; + } + return _Py_ext_module_loader_info_init(p_info, name, filename); +} + + PyObject * _PyImport_LoadDynamicModuleWithSpec(struct _Py_ext_module_loader_info *info, PyObject *spec, FILE *fp) From 425c5e068c6ec85699144b4dd179f6761fb0025c Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 23 Apr 2024 16:57:11 -0600 Subject: [PATCH 8/9] Add _Py_ext_module_loader_info.path. --- Include/internal/pycore_importdl.h | 3 +++ Python/importdl.c | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/Include/internal/pycore_importdl.h b/Include/internal/pycore_importdl.h index 12e81509762c57..8bf7c2a48f66be 100644 --- a/Include/internal/pycore_importdl.h +++ b/Include/internal/pycore_importdl.h @@ -24,6 +24,9 @@ struct _Py_ext_module_loader_info { #endif PyObject *name; PyObject *name_encoded; + /* path is always a borrowed ref of name or filename, + * depending on if it's builtin or not. */ + PyObject *path; const char *hook_prefix; const char *newcontext; }; diff --git a/Python/importdl.c b/Python/importdl.c index 51bb9808541a7a..65370249493325 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -147,6 +147,11 @@ _Py_ext_module_loader_info_init(struct _Py_ext_module_loader_info *p_info, return -1; } #endif + + info.path = info.filename; + } + else { + info.path = info.name; } *p_info = info; From 2a9b621fbc032195629fa054d471fd35a0465e96 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 23 Apr 2024 17:05:18 -0600 Subject: [PATCH 9/9] Pass loader info to import_find_extension(). --- Python/import.c | 73 +++++++++++++++++++++++++++++++------------------ 1 file changed, 47 insertions(+), 26 deletions(-) diff --git a/Python/import.c b/Python/import.c index 90a5105b0e04b1..56011295f95190 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1328,11 +1328,11 @@ _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, static PyObject * -import_find_extension(PyThreadState *tstate, PyObject *name, - PyObject *path) +import_find_extension(PyThreadState *tstate, + struct _Py_ext_module_loader_info *info) { /* Only single-phase init modules will be in the cache. */ - PyModuleDef *def = _extensions_cache_get(path, name); + PyModuleDef *def = _extensions_cache_get(info->path, info->name); if (def == NULL) { return NULL; } @@ -1340,7 +1340,7 @@ import_find_extension(PyThreadState *tstate, PyObject *name, /* It may have been successfully imported previously in an interpreter that allows legacy modules but is not allowed in the current interpreter. */ - const char *name_buf = PyUnicode_AsUTF8(name); + const char *name_buf = PyUnicode_AsUTF8(info->name); assert(name_buf != NULL); if (_PyImport_CheckSubinterpIncompatibleExtensionAllowed(name_buf) < 0) { return NULL; @@ -1355,12 +1355,13 @@ import_find_extension(PyThreadState *tstate, PyObject *name, if (m_copy == NULL) { /* It might be a core module (e.g. sys & builtins), for which we don't set m_copy. */ - m_copy = get_core_module_dict(tstate->interp, name, path); + m_copy = get_core_module_dict( + tstate->interp, info->name, info->path); if (m_copy == NULL) { return NULL; } } - mod = import_add_module(tstate, name); + mod = import_add_module(tstate, info->name); if (mod == NULL) { return NULL; } @@ -1378,15 +1379,16 @@ import_find_extension(PyThreadState *tstate, PyObject *name, if (def->m_base.m_init == NULL) return NULL; mod = def->m_base.m_init(); - if (mod == NULL) + if (mod == NULL) { return NULL; - if (PyObject_SetItem(modules, name, mod) == -1) { + } + if (PyObject_SetItem(modules, info->name, mod) == -1) { Py_DECREF(mod); return NULL; } } if (_modules_by_index_set(tstate->interp, def, mod) < 0) { - PyMapping_DelItem(modules, name); + PyMapping_DelItem(modules, info->name); Py_DECREF(mod); return NULL; } @@ -1394,7 +1396,7 @@ import_find_extension(PyThreadState *tstate, PyObject *name, int verbose = _PyInterpreterState_GetConfig(tstate->interp)->verbose; if (verbose) { PySys_FormatStderr("import %U # previously loaded (%R)\n", - name, path); + info->name, info->path); } return mod; } @@ -1505,44 +1507,56 @@ static PyObject* create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec) { PyModuleDef *def = NULL; - PyObject *mod = import_find_extension(tstate, name, name); + + struct _Py_ext_module_loader_info info; + if (_Py_ext_module_loader_info_init(&info, name, NULL) < 0) { + return NULL; + } + + PyObject *mod = import_find_extension(tstate, &info); if (mod || _PyErr_Occurred(tstate)) { - return mod; + goto finally; } struct _inittab *found = NULL; for (struct _inittab *p = INITTAB; p->name != NULL; p++) { - if (_PyUnicode_EqualToASCIIString(name, p->name)) { + if (_PyUnicode_EqualToASCIIString(info.name, p->name)) { found = p; } } if (found == NULL) { // not found - Py_RETURN_NONE; + mod = Py_NewRef(Py_None); + goto finally; } PyModInitFunction p0 = (PyModInitFunction)found->initfunc; if (p0 == NULL) { /* Cannot re-init internal module ("sys" or "builtins") */ - assert(is_core_module(tstate->interp, name, name)); - return import_add_module(tstate, name); + assert(is_core_module(tstate->interp, info.name, info.path)); + mod = import_add_module(tstate, info.name); + goto finally; } mod = p0(); if (mod == NULL) { - return NULL; + goto finally; } if (PyObject_TypeCheck(mod, &PyModuleDef_Type)) { def = (PyModuleDef*)mod; assert(!is_singlephase(def)); - return PyModule_FromDefAndSpec(def, spec); + mod = PyModule_FromDefAndSpec(def, spec); + if (mod == NULL) { + goto finally; + } } else { assert(PyModule_Check(mod)); def = PyModule_GetDef(mod); if (def == NULL) { - return NULL; + Py_CLEAR(mod); + goto finally; } assert(is_singlephase(def)); @@ -1553,22 +1567,29 @@ create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec) // gh-88216: Extensions and def->m_base.m_copy can be updated // when the extension module doesn't support sub-interpreters. if (def->m_size == -1 - && !is_core_module(tstate->interp, name, name)) + && !is_core_module(tstate->interp, info.name, info.path)) { singlephase.m_dict = PyModule_GetDict(mod); assert(singlephase.m_dict != NULL); } if (update_global_state_for_extension( - tstate, name, name, def, &singlephase) < 0) + tstate, info.name, info.path, def, &singlephase) < 0) { - return NULL; + Py_CLEAR(mod); + goto finally; } PyObject *modules = get_modules_dict(tstate, true); - if (finish_singlephase_extension(tstate, mod, def, name, modules) < 0) { - return NULL; + if (finish_singlephase_extension( + tstate, mod, def, info.name, modules) < 0) + { + Py_CLEAR(mod); + goto finally; } - return mod; } + +finally: + _Py_ext_module_loader_info_clear(&info); + return mod; } @@ -3887,7 +3908,7 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) } PyThreadState *tstate = _PyThreadState_GET(); - mod = import_find_extension(tstate, info.name, info.filename); + mod = import_find_extension(tstate, &info); if (mod != NULL || _PyErr_Occurred(tstate)) { assert(mod == NULL || !_PyErr_Occurred(tstate)); goto finally;