diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h
index 05e36ad18b..7c5137d05f 100644
--- a/include/pybind11/detail/internals.h
+++ b/include/pybind11/detail/internals.h
@@ -535,7 +535,11 @@ struct local_internals {
         loader_life_support_tls_key
             = static_cast<shared_loader_life_support_data *>(ptr)->loader_life_support_tls_key;
     }
+#else
+    local_internals() = default;
 #endif //  defined(WITH_THREAD) && PYBIND11_INTERNALS_VERSION == 4
+    local_internals(const local_internals &) = delete;
+    local_internals &operator=(const local_internals &) = delete;
 };
 
 /// Works like `get_internals`, but for things which are locally registered.
diff --git a/include/pybind11/embed.h b/include/pybind11/embed.h
index 5a175b1341..35fbce6f40 100644
--- a/include/pybind11/embed.h
+++ b/include/pybind11/embed.h
@@ -254,13 +254,25 @@ inline void finalize_interpreter() {
     if (builtins.contains(id) && isinstance<capsule>(builtins[id])) {
         internals_ptr_ptr = capsule(builtins[id]);
     }
-    // Local internals contains data managed by the current interpreter, so we must clear them to
-    // avoid undefined behaviors when initializing another interpreter
-    detail::get_local_internals().registered_types_cpp.clear();
-    detail::get_local_internals().registered_exception_translators.clear();
+
+    // We must clear this data after the interpreter is finalized but calling get_local_internals()
+    // is UB at that point, so he have to cache a references here.
+    auto &local_internals = detail::get_local_internals();
+
+    // Hack to materialize local internals from static method variable before Py_Finalize.
+    bool need_to_clear = !local_internals.registered_exception_translators.empty()
+                         || !local_internals.registered_exception_translators.empty();
 
     Py_Finalize();
 
+    // This local data is needed during Py_Finalize() for callbacks or hooks such as atexit.
+    // Local internals contains data managed by the current interpreter, so we must clear them to
+    // avoid undefined behaviors when initializing another interpreter
+    if (need_to_clear) {
+        local_internals.registered_types_cpp.clear();
+        local_internals.registered_exception_translators.clear();
+    }
+
     if (internals_ptr_ptr) {
         delete *internals_ptr_ptr;
         *internals_ptr_ptr = nullptr;
diff --git a/tests/test_embed/catch.cpp b/tests/test_embed/catch.cpp
index 558a7a35e5..8fec877788 100644
--- a/tests/test_embed/catch.cpp
+++ b/tests/test_embed/catch.cpp
@@ -34,7 +34,8 @@ int main(int argc, char *argv[]) {
 #else
     setenv("PYTHONPATH", updated_pythonpath.c_str(), /*replace=*/1);
 #endif
-
+    // Below is a test case for a potential segfault. See PR #4500.
+    { py::scoped_interpreter guard; }
     py::scoped_interpreter guard{};
 
     auto result = Catch::Session().run(argc, argv);