From d9f6f44f009282d12d1a8eb1dc480d0300bf537e Mon Sep 17 00:00:00 2001 From: Jon Burdo Date: Mon, 19 Dec 2022 09:40:38 -0500 Subject: [PATCH 01/24] fix os.walk RecursionError on deep trees --- Lib/os.py | 112 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 63 insertions(+), 49 deletions(-) diff --git a/Lib/os.py b/Lib/os.py index fd1e774fdcbcfa..0e2e8fd0b6a704 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -480,57 +480,71 @@ def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks): # Note: This uses O(depth of the directory tree) file descriptors: if # necessary, it can be adapted to only require O(1) FDs, see issue # #13734. - - scandir_it = scandir(topfd) - dirs = [] - nondirs = [] - entries = None if topdown or follow_symlinks else [] - for entry in scandir_it: - name = entry.name - if isbytes: - name = fsencode(name) - try: - if entry.is_dir(): - dirs.append(name) - if entries is not None: - entries.append(entry) + try: + stack = [(2, (topfd, toppath))] + while stack: + sig, val = stack.pop() + if sig == 1: + yield val + continue + elif sig == 2: + topfd, toppath = val + elif sig == 3: + close(val) + continue else: - nondirs.append(name) - except OSError: - try: - # Add dangling symlinks, ignore disappeared files - if entry.is_symlink(): - nondirs.append(name) - except OSError: - pass - - if topdown: - yield toppath, dirs, nondirs, topfd + raise ValueError(f"invalid sig: {sig}") + scandir_it = scandir(topfd) + dirs = [] + nondirs = [] + entries = None if topdown or follow_symlinks else [] + for entry in scandir_it: + name = entry.name + if isbytes: + name = fsencode(name) + try: + if entry.is_dir(): + dirs.append(name) + if entries is not None: + entries.append(entry) + else: + nondirs.append(name) + except OSError: + try: + # Add dangling symlinks, ignore disappeared files + if entry.is_symlink(): + nondirs.append(name) + except OSError: + pass + + if topdown: + yield toppath, dirs, nondirs, topfd + else: + stack.append((1, (toppath, dirs, nondirs, topfd))) - for name in dirs if entries is None else zip(dirs, entries): - try: - if not follow_symlinks: - if topdown: - orig_st = stat(name, dir_fd=topfd, follow_symlinks=False) - else: - assert entries is not None - name, entry = name - orig_st = entry.stat(follow_symlinks=False) - dirfd = open(name, O_RDONLY, dir_fd=topfd) - except OSError as err: - if onerror is not None: - onerror(err) - continue - try: - if follow_symlinks or path.samestat(orig_st, stat(dirfd)): - dirpath = path.join(toppath, name) - yield from _fwalk(dirfd, dirpath, isbytes, - topdown, onerror, follow_symlinks) - finally: - close(dirfd) - - if not topdown: - yield toppath, dirs, nondirs, topfd + for name in reversed(dirs) if entries is None else zip(reversed(dirs), reversed(entries)): + try: + if not follow_symlinks: + if topdown: + orig_st = stat(name, dir_fd=topfd, follow_symlinks=False) + else: + assert entries is not None + name, entry = name + orig_st = entry.stat(follow_symlinks=False) + dirfd = open(name, O_RDONLY, dir_fd=topfd) + except OSError as err: + if onerror is not None: + onerror(err) + continue + stack.append((3, dirfd)) + if follow_symlinks or path.samestat(orig_st, stat(dirfd)): + dirpath = path.join(toppath, name) + stack.append((2, (dirfd, dirpath))) + except: + for sig, val in reversed(stack): + if sig == 3: + close(val) + raise __all__.append("fwalk") From 61b8078a37a014d070b99fb9bfcfebb21b2f38ca Mon Sep 17 00:00:00 2001 From: Jon Burdo Date: Mon, 19 Dec 2022 10:40:13 -0500 Subject: [PATCH 02/24] use _WalkAction enum --- Lib/os.py | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/Lib/os.py b/Lib/os.py index 0e2e8fd0b6a704..bb27fae6b7e9e6 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -427,6 +427,13 @@ def _walk(top, topdown, onerror, followlinks): __all__.append("walk") if {open, stat} <= supports_dir_fd and {scandir, stat} <= supports_fd: + from enum import Enum, _simple_enum + + @_simple_enum(Enum) + class _WalkAction: + YIELD = 1 + CLOSE = 2 + WALK = 3 def fwalk(top=".", topdown=True, onerror=None, *, follow_symlinks=False, dir_fd=None): """Directory tree generator. @@ -481,19 +488,19 @@ def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks): # necessary, it can be adapted to only require O(1) FDs, see issue # #13734. try: - stack = [(2, (topfd, toppath))] + stack = [(_WalkAction.WALK, (topfd, toppath))] while stack: - sig, val = stack.pop() - if sig == 1: - yield val + action, value = stack.pop() + if action is _WalkAction.YIELD: + yield value continue - elif sig == 2: - topfd, toppath = val - elif sig == 3: - close(val) + elif action is _WalkAction.CLOSE: + close(value) continue + elif action is _WalkAction.WALK: + topfd, toppath = value else: - raise ValueError(f"invalid sig: {sig}") + raise ValueError(f"invalid walk action: {action}") scandir_it = scandir(topfd) dirs = [] nondirs = [] @@ -520,7 +527,7 @@ def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks): if topdown: yield toppath, dirs, nondirs, topfd else: - stack.append((1, (toppath, dirs, nondirs, topfd))) + stack.append((_WalkAction.YIELD, (toppath, dirs, nondirs, topfd))) for name in reversed(dirs) if entries is None else zip(reversed(dirs), reversed(entries)): try: @@ -536,14 +543,14 @@ def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks): if onerror is not None: onerror(err) continue - stack.append((3, dirfd)) + stack.append((_WalkAction.CLOSE, dirfd)) if follow_symlinks or path.samestat(orig_st, stat(dirfd)): dirpath = path.join(toppath, name) - stack.append((2, (dirfd, dirpath))) + stack.append((_WalkAction.WALK, (dirfd, dirpath))) except: - for sig, val in reversed(stack): - if sig == 3: - close(val) + for action, value in reversed(stack): + if action is _WalkAction.CLOSE: + close(value) raise __all__.append("fwalk") From 90c123ba387c65d044cd33db7d6e92f54f8f7f91 Mon Sep 17 00:00:00 2001 From: Jon Burdo Date: Tue, 20 Dec 2022 09:28:16 -0500 Subject: [PATCH 03/24] fix formatting --- Lib/os.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Lib/os.py b/Lib/os.py index bb27fae6b7e9e6..7553c401e268c2 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -527,13 +527,16 @@ def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks): if topdown: yield toppath, dirs, nondirs, topfd else: - stack.append((_WalkAction.YIELD, (toppath, dirs, nondirs, topfd))) + stack.append( + (_WalkAction.YIELD, (toppath, dirs, nondirs, topfd))) - for name in reversed(dirs) if entries is None else zip(reversed(dirs), reversed(entries)): + for name in (reversed(dirs) if entries is None + else zip(reversed(dirs), reversed(entries))): try: if not follow_symlinks: if topdown: - orig_st = stat(name, dir_fd=topfd, follow_symlinks=False) + orig_st = stat(name, dir_fd=topfd, + follow_symlinks=False) else: assert entries is not None name, entry = name From 570818b0cf609935787431ed7e0b7cdc0f8d1f88 Mon Sep 17 00:00:00 2001 From: Jon Burdo Date: Tue, 20 Dec 2022 09:29:58 -0500 Subject: [PATCH 04/24] remove enum from os --- Lib/os.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/Lib/os.py b/Lib/os.py index 7553c401e268c2..576b007a95f0f9 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -427,13 +427,10 @@ def _walk(top, topdown, onerror, followlinks): __all__.append("walk") if {open, stat} <= supports_dir_fd and {scandir, stat} <= supports_fd: - from enum import Enum, _simple_enum - - @_simple_enum(Enum) class _WalkAction: - YIELD = 1 - CLOSE = 2 - WALK = 3 + YIELD = object() + CLOSE = object() + WALK = object() def fwalk(top=".", topdown=True, onerror=None, *, follow_symlinks=False, dir_fd=None): """Directory tree generator. From 30aeed7fc6945e57946f6c7708546f04fcec7f04 Mon Sep 17 00:00:00 2001 From: Jon Burdo Date: Tue, 20 Dec 2022 09:37:49 -0500 Subject: [PATCH 05/24] add blurb --- .../next/Library/2022-12-20-09-36-29.gh-issue-89727.FpprK3.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2022-12-20-09-36-29.gh-issue-89727.FpprK3.rst diff --git a/Misc/NEWS.d/next/Library/2022-12-20-09-36-29.gh-issue-89727.FpprK3.rst b/Misc/NEWS.d/next/Library/2022-12-20-09-36-29.gh-issue-89727.FpprK3.rst new file mode 100644 index 00000000000000..3c17b5604d4c36 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-20-09-36-29.gh-issue-89727.FpprK3.rst @@ -0,0 +1,3 @@ +Fix issue with :func:`os.fwalk` where a :exc:`RecursionError` would occur on +deep directory structures by adjusting the implementation to be iterative +instead of recursive. From 55e17b83551db9414bd5418f577351333ff6292b Mon Sep 17 00:00:00 2001 From: Jon Burdo Date: Mon, 19 Dec 2022 13:59:01 -0500 Subject: [PATCH 06/24] gh-89727: Fix os.walk RecursionError on deep trees (#99803) Use a stack to implement os.walk iteratively instead of recursively to avoid hitting recursion limits on deeply nested trees. --- Lib/os.py | 162 +++++++++--------- Lib/test/support/__init__.py | 16 +- Lib/test/test_os.py | 43 +++++ ...2-11-29-20-44-54.gh-issue-89727.UJZjkk.rst | 3 + 4 files changed, 140 insertions(+), 84 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-11-29-20-44-54.gh-issue-89727.UJZjkk.rst diff --git a/Lib/os.py b/Lib/os.py index 576b007a95f0f9..9dc19ba601ffb5 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -340,89 +340,95 @@ def walk(top, topdown=True, onerror=None, followlinks=False): """ sys.audit("os.walk", top, topdown, onerror, followlinks) - return _walk(fspath(top), topdown, onerror, followlinks) - -def _walk(top, topdown, onerror, followlinks): - dirs = [] - nondirs = [] - walk_dirs = [] - - # We may not have read permission for top, in which case we can't - # get a list of the files the directory contains. os.walk - # always suppressed the exception then, rather than blow up for a - # minor reason when (say) a thousand readable directories are still - # left to visit. That logic is copied here. - try: - # Note that scandir is global in this module due - # to earlier import-*. - scandir_it = scandir(top) - except OSError as error: - if onerror is not None: - onerror(error) - return - with scandir_it: - while True: - try: + stack = [(False, fspath(top))] + islink, join = path.islink, path.join + while stack: + must_yield, top = stack.pop() + if must_yield: + yield top + continue + + dirs = [] + nondirs = [] + walk_dirs = [] + + # We may not have read permission for top, in which case we can't + # get a list of the files the directory contains. + # We suppress the exception here, rather than blow up for a + # minor reason when (say) a thousand readable directories are still + # left to visit. + try: + scandir_it = scandir(top) + except OSError as error: + if onerror is not None: + onerror(error) + continue + + cont = False + with scandir_it: + while True: try: - entry = next(scandir_it) - except StopIteration: + try: + entry = next(scandir_it) + except StopIteration: + break + except OSError as error: + if onerror is not None: + onerror(error) + cont = True break - except OSError as error: - if onerror is not None: - onerror(error) - return - - try: - is_dir = entry.is_dir() - except OSError: - # If is_dir() raises an OSError, consider that the entry is not - # a directory, same behaviour than os.path.isdir(). - is_dir = False - - if is_dir: - dirs.append(entry.name) - else: - nondirs.append(entry.name) - if not topdown and is_dir: - # Bottom-up: recurse into sub-directory, but exclude symlinks to - # directories if followlinks is False - if followlinks: - walk_into = True + try: + is_dir = entry.is_dir() + except OSError: + # If is_dir() raises an OSError, consider the entry not to + # be a directory, same behaviour as os.path.isdir(). + is_dir = False + + if is_dir: + dirs.append(entry.name) else: - try: - is_symlink = entry.is_symlink() - except OSError: - # If is_symlink() raises an OSError, consider that the - # entry is not a symbolic link, same behaviour than - # os.path.islink(). - is_symlink = False - walk_into = not is_symlink - - if walk_into: - walk_dirs.append(entry.path) - - # Yield before recursion if going top down - if topdown: - yield top, dirs, nondirs - - # Recurse into sub-directories - islink, join = path.islink, path.join - for dirname in dirs: - new_path = join(top, dirname) - # Issue #23605: os.path.islink() is used instead of caching - # entry.is_symlink() result during the loop on os.scandir() because - # the caller can replace the directory entry during the "yield" - # above. - if followlinks or not islink(new_path): - yield from _walk(new_path, topdown, onerror, followlinks) - else: - # Recurse into sub-directories - for new_path in walk_dirs: - yield from _walk(new_path, topdown, onerror, followlinks) - # Yield after recursion if going bottom up - yield top, dirs, nondirs + nondirs.append(entry.name) + + if not topdown and is_dir: + # Bottom-up: traverse into sub-directory, but exclude + # symlinks to directories if followlinks is False + if followlinks: + walk_into = True + else: + try: + is_symlink = entry.is_symlink() + except OSError: + # If is_symlink() raises an OSError, consider the + # entry not to be a symbolic link, same behaviour + # as os.path.islink(). + is_symlink = False + walk_into = not is_symlink + + if walk_into: + walk_dirs.append(entry.path) + if cont: + continue + + if topdown: + # Yield before sub-directory traversal if going top down + yield top, dirs, nondirs + # Traverse into sub-directories + for dirname in reversed(dirs): + new_path = join(top, dirname) + # bpo-23605: os.path.islink() is used instead of caching + # entry.is_symlink() result during the loop on os.scandir() because + # the caller can replace the directory entry during the "yield" + # above. + if followlinks or not islink(new_path): + stack.append((False, new_path)) + else: + # Yield after sub-directory traversal if going bottom up + stack.append((True, (top, dirs, nondirs))) + # Traverse into sub-directories + for new_path in reversed(walk_dirs): + stack.append((False, new_path)) __all__.append("walk") diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index a631bfc80cfaf0..b7186057990ac1 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2178,19 +2178,23 @@ def check_disallow_instantiation(testcase, tp, *args, **kwds): testcase.assertRaisesRegex(TypeError, msg, tp, *args, **kwds) @contextlib.contextmanager +def set_recursion_limit(limit): + """Temporarily change the recursion limit.""" + original_limit = sys.getrecursionlimit() + try: + sys.setrecursionlimit(limit) + yield + finally: + sys.setrecursionlimit(original_limit) + def infinite_recursion(max_depth=75): """Set a lower limit for tests that interact with infinite recursions (e.g test_ast.ASTHelpers_Test.test_recursion_direct) since on some debug windows builds, due to not enough functions being inlined the stack size might not handle the default recursion limit (1000). See bpo-11105 for details.""" + return set_recursion_limit(max_depth) - original_depth = sys.getrecursionlimit() - try: - sys.setrecursionlimit(max_depth) - yield - finally: - sys.setrecursionlimit(original_depth) def ignore_deprecations_from(module: str, *, like: str) -> object: token = object() diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index e0577916428a08..e6e25b507de051 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -33,6 +33,7 @@ from test.support import import_helper from test.support import os_helper from test.support import socket_helper +from test.support import set_recursion_limit from test.support import warnings_helper from platform import win32_is_iot @@ -1471,6 +1472,46 @@ def test_walk_many_open_files(self): self.assertEqual(next(it), expected) p = os.path.join(p, 'd') + def test_walk_above_recursion_limit(self): + depth = 50 + os.makedirs(os.path.join(self.walk_path, *(['d'] * depth))) + with set_recursion_limit(depth - 5): + all = list(self.walk(self.walk_path)) + + sub2_path = self.sub2_tree[0] + for root, dirs, files in all: + if root == sub2_path: + dirs.sort() + files.sort() + + d_entries = [] + d_path = self.walk_path + for _ in range(depth): + d_path = os.path.join(d_path, "d") + d_entries.append((d_path, ["d"], [])) + d_entries[-1][1].clear() + + # Sub-sequences where the order is known + sections = { + "SUB1": [ + (self.sub1_path, ["SUB11"], ["tmp2"]), + (self.sub11_path, [], []), + ], + "SUB2": [self.sub2_tree], + "d": d_entries, + } + + # The ordering of sub-dirs is arbitrary but determines the order in + # which sub-sequences appear + dirs = all[0][1] + expected = [(self.walk_path, dirs, ["tmp1"])] + for d in dirs: + expected.extend(sections[d]) + + self.assertEqual(len(all), depth + 4) + self.assertEqual(sorted(dirs), ["SUB1", "SUB2", "d"]) + self.assertEqual(all, expected) + @unittest.skipUnless(hasattr(os, 'fwalk'), "Test needs os.fwalk()") class FwalkTests(WalkTests): @@ -1545,6 +1586,8 @@ def test_fd_leak(self): # fwalk() keeps file descriptors open test_walk_many_open_files = None + # fwalk() still uses recursion + test_walk_above_recursion_limit = None class BytesWalkTests(WalkTests): diff --git a/Misc/NEWS.d/next/Library/2022-11-29-20-44-54.gh-issue-89727.UJZjkk.rst b/Misc/NEWS.d/next/Library/2022-11-29-20-44-54.gh-issue-89727.UJZjkk.rst new file mode 100644 index 00000000000000..8a5fdb64b87f82 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-29-20-44-54.gh-issue-89727.UJZjkk.rst @@ -0,0 +1,3 @@ +Fix issue with :func:`os.walk` where a :exc:`RecursionError` would occur on +deep directory structures by adjusting the implementation of +:func:`os.walk` to be iterative instead of recursive. From f41cef6705dd62bebd02505a671acefab949016c Mon Sep 17 00:00:00 2001 From: Stanley <46876382+slateny@users.noreply.github.com> Date: Mon, 19 Dec 2022 19:07:31 -0800 Subject: [PATCH 07/24] gh-69929: re docs: Add more specific definition of \w (#92015) Co-authored-by: Jelle Zijlstra --- Doc/library/re.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Doc/library/re.rst b/Doc/library/re.rst index f7d46586cf7570..cbee70b01d9f46 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -591,10 +591,9 @@ character ``'$'``. ``\w`` For Unicode (str) patterns: - Matches Unicode word characters; this includes most characters - that can be part of a word in any language, as well as numbers and - the underscore. If the :const:`ASCII` flag is used, only - ``[a-zA-Z0-9_]`` is matched. + Matches Unicode word characters; this includes alphanumeric characters (as defined by :meth:`str.isalnum`) + as well as the underscore (``_``). + If the :const:`ASCII` flag is used, only ``[a-zA-Z0-9_]`` is matched. For 8-bit (bytes) patterns: Matches characters considered alphanumeric in the ASCII character set; From 33ba6a5800628af22fa7ecf00fc5601296849a10 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Tue, 20 Dec 2022 07:10:30 +0000 Subject: [PATCH 08/24] gh-89051: Add ssl.OP_LEGACY_SERVER_CONNECT (#93927) Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Christian Heimes Co-authored-by: Hugo van Kemenade Fixes https://github.com/python/cpython/issues/89051 --- Doc/library/ssl.rst | 7 +++++++ Lib/test/test_ssl.py | 16 ++++++++++++++++ ...2022-06-17-08-00-34.gh-issue-89051.yP4Na0.rst | 1 + Modules/_ssl.c | 2 ++ 4 files changed, 26 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-06-17-08-00-34.gh-issue-89051.yP4Na0.rst diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 08824feeb3958f..78d44a23a83bf0 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -823,6 +823,13 @@ Constants .. versionadded:: 3.12 +.. data:: OP_LEGACY_SERVER_CONNECT + + Allow legacy insecure renegotiation between OpenSSL and unpatched servers + only. + + .. versionadded:: 3.12 + .. data:: HAS_ALPN Whether the OpenSSL library has built-in support for the *Application-Layer diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index e926fc5e88e584..d4eb2d2e81fe0f 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -1461,6 +1461,8 @@ def _assert_context_options(self, ctx): if OP_CIPHER_SERVER_PREFERENCE != 0: self.assertEqual(ctx.options & OP_CIPHER_SERVER_PREFERENCE, OP_CIPHER_SERVER_PREFERENCE) + self.assertEqual(ctx.options & ssl.OP_LEGACY_SERVER_CONNECT, + 0 if IS_OPENSSL_3_0_0 else ssl.OP_LEGACY_SERVER_CONNECT) def test_create_default_context(self): ctx = ssl.create_default_context() @@ -3815,6 +3817,20 @@ def test_compression_disabled(self): sni_name=hostname) self.assertIs(stats['compression'], None) + def test_legacy_server_connect(self): + client_context, server_context, hostname = testing_context() + client_context.options |= ssl.OP_LEGACY_SERVER_CONNECT + server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + + def test_no_legacy_server_connect(self): + client_context, server_context, hostname = testing_context() + client_context.options &= ~ssl.OP_LEGACY_SERVER_CONNECT + server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + @unittest.skipIf(Py_DEBUG_WIN32, "Avoid mixing debug/release CRT on Windows") def test_dh_params(self): # Check we can get a connection with ephemeral Diffie-Hellman diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-06-17-08-00-34.gh-issue-89051.yP4Na0.rst b/Misc/NEWS.d/next/Core and Builtins/2022-06-17-08-00-34.gh-issue-89051.yP4Na0.rst new file mode 100644 index 00000000000000..5c8164863b8192 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-06-17-08-00-34.gh-issue-89051.yP4Na0.rst @@ -0,0 +1 @@ +Add :data:`ssl.OP_LEGACY_SERVER_CONNECT` diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 591eb91dd0f340..8f03a846aed089 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -5845,6 +5845,8 @@ sslmodule_init_constants(PyObject *m) SSL_OP_CIPHER_SERVER_PREFERENCE); PyModule_AddIntConstant(m, "OP_SINGLE_DH_USE", SSL_OP_SINGLE_DH_USE); PyModule_AddIntConstant(m, "OP_NO_TICKET", SSL_OP_NO_TICKET); + PyModule_AddIntConstant(m, "OP_LEGACY_SERVER_CONNECT", + SSL_OP_LEGACY_SERVER_CONNECT); #ifdef SSL_OP_SINGLE_ECDH_USE PyModule_AddIntConstant(m, "OP_SINGLE_ECDH_USE", SSL_OP_SINGLE_ECDH_USE); #endif From f9b6796728684e242c795b23c0c8b04dedd7cbc1 Mon Sep 17 00:00:00 2001 From: Brad Wolfe Date: Tue, 20 Dec 2022 11:10:31 +0100 Subject: [PATCH 09/24] gh-88211: Change lower-case and upper-case to match recommendations in imaplib docs (#99625) --- Doc/library/imaplib.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 0c10e7afee401f..8c28fce99ff912 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -187,7 +187,7 @@ IMAP4 Objects ------------- All IMAP4rev1 commands are represented by methods of the same name, either -upper-case or lower-case. +uppercase or lowercase. All arguments to commands are converted to strings, except for ``AUTHENTICATE``, and the last argument to ``APPEND`` which is passed as an IMAP4 literal. If From 77d160fee528fa7666497ff35fa6b2e51675bf55 Mon Sep 17 00:00:00 2001 From: Richard Kojedzinszky Date: Tue, 20 Dec 2022 11:40:56 +0100 Subject: [PATCH 10/24] gh-100348: Fix ref cycle in `asyncio._SelectorSocketTransport` with `_read_ready_cb` (#100349) --- Lib/asyncio/selector_events.py | 4 ++++ .../Library/2022-12-19-19-30-06.gh-issue-100348.o7IAHh.rst | 2 ++ 2 files changed, 6 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2022-12-19-19-30-06.gh-issue-100348.o7IAHh.rst diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index 3d30006198f671..74f289f0e6f811 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -1133,6 +1133,10 @@ def _make_empty_waiter(self): def _reset_empty_waiter(self): self._empty_waiter = None + def close(self): + self._read_ready_cb = None + super().close() + class _SelectorDatagramTransport(_SelectorTransport, transports.DatagramTransport): diff --git a/Misc/NEWS.d/next/Library/2022-12-19-19-30-06.gh-issue-100348.o7IAHh.rst b/Misc/NEWS.d/next/Library/2022-12-19-19-30-06.gh-issue-100348.o7IAHh.rst new file mode 100644 index 00000000000000..b5d4f7ca998cb5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-19-19-30-06.gh-issue-100348.o7IAHh.rst @@ -0,0 +1,2 @@ +Fix ref cycle in :class:`!asyncio._SelectorSocketTransport` by removing ``_read_ready_cb`` in ``close``. + From db820a20070359f93181471c553f07a743876829 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franti=C5=A1ek=20Nesveda?= Date: Tue, 20 Dec 2022 10:54:56 +0000 Subject: [PATCH 11/24] gh-99925: Fix inconsistency in `json.dumps()` error messages (GH-99926) --- Lib/test/test_json/test_float.py | 3 ++- .../Library/2022-12-01-15-44-58.gh-issue-99925.x4y6pF.rst | 4 ++++ Modules/_json.c | 5 +++-- 3 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-12-01-15-44-58.gh-issue-99925.x4y6pF.rst diff --git a/Lib/test/test_json/test_float.py b/Lib/test/test_json/test_float.py index d0c7214334d6e5..61540a3a02c2c6 100644 --- a/Lib/test/test_json/test_float.py +++ b/Lib/test/test_json/test_float.py @@ -26,7 +26,8 @@ def test_allow_nan(self): res = self.loads(out) self.assertEqual(len(res), 1) self.assertNotEqual(res[0], res[0]) - self.assertRaises(ValueError, self.dumps, [val], allow_nan=False) + msg = f'Out of range float values are not JSON compliant: {val}' + self.assertRaisesRegex(ValueError, msg, self.dumps, [val], allow_nan=False) class TestPyFloat(TestFloat, PyTest): pass diff --git a/Misc/NEWS.d/next/Library/2022-12-01-15-44-58.gh-issue-99925.x4y6pF.rst b/Misc/NEWS.d/next/Library/2022-12-01-15-44-58.gh-issue-99925.x4y6pF.rst new file mode 100644 index 00000000000000..660635a039631c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-01-15-44-58.gh-issue-99925.x4y6pF.rst @@ -0,0 +1,4 @@ +Unify error messages in JSON serialization between +``json.dumps(float('nan'), allow_nan=False)`` and ``json.dumps(float('nan'), +allow_nan=False, indent=)``. Now both include the representation +of the value that could not be serialized. diff --git a/Modules/_json.c b/Modules/_json.c index 6879ad3d0722b6..fa8e2a936d2c33 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -1319,9 +1319,10 @@ encoder_encode_float(PyEncoderObject *s, PyObject *obj) double i = PyFloat_AS_DOUBLE(obj); if (!Py_IS_FINITE(i)) { if (!s->allow_nan) { - PyErr_SetString( + PyErr_Format( PyExc_ValueError, - "Out of range float values are not JSON compliant" + "Out of range float values are not JSON compliant: %R", + obj ); return NULL; } From c39ce6305ede472a412ba52c276fdb1074e1bdc9 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Tue, 20 Dec 2022 11:35:48 +0000 Subject: [PATCH 12/24] Clarify that every thread has its own default context in contextvars (#99246) --- Doc/library/contextvars.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Doc/library/contextvars.rst b/Doc/library/contextvars.rst index 08a7c7d74eab97..0ac2f3d85749b7 100644 --- a/Doc/library/contextvars.rst +++ b/Doc/library/contextvars.rst @@ -144,6 +144,11 @@ Manual Context Management To get a copy of the current context use the :func:`~contextvars.copy_context` function. + Every thread will have a different top-level :class:`~contextvars.Context` + object. This means that a :class:`ContextVar` object behaves in a similar + fashion to :func:`threading.local()` when values are assigned in different + threads. + Context implements the :class:`collections.abc.Mapping` interface. .. method:: run(callable, *args, **kwargs) From 8d2befbf48064809cafbfa762b33e45c02fe57b2 Mon Sep 17 00:00:00 2001 From: Jon Burdo Date: Tue, 20 Dec 2022 09:40:35 -0500 Subject: [PATCH 13/24] run test_walk_above_recursion_limit for fwalk --- Lib/test/test_os.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index e6e25b507de051..6ff38f31d142ac 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -1586,8 +1586,6 @@ def test_fd_leak(self): # fwalk() keeps file descriptors open test_walk_many_open_files = None - # fwalk() still uses recursion - test_walk_above_recursion_limit = None class BytesWalkTests(WalkTests): From d29188e8fea033757b6ee8b4ae62ed7760c7c207 Mon Sep 17 00:00:00 2001 From: Jon Burdo Date: Tue, 20 Dec 2022 09:45:30 -0500 Subject: [PATCH 14/24] set stack outside try-except in fwalk --- Lib/os.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/os.py b/Lib/os.py index 9dc19ba601ffb5..7bf23a3b1b277a 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -490,8 +490,8 @@ def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks): # Note: This uses O(depth of the directory tree) file descriptors: if # necessary, it can be adapted to only require O(1) FDs, see issue # #13734. + stack = [(_WalkAction.WALK, (topfd, toppath))] try: - stack = [(_WalkAction.WALK, (topfd, toppath))] while stack: action, value = stack.pop() if action is _WalkAction.YIELD: From 2cf7550aeb8de8a837e4ecf80e363e62ccba0ec5 Mon Sep 17 00:00:00 2001 From: Jon Burdo Date: Tue, 20 Dec 2022 10:02:39 -0500 Subject: [PATCH 15/24] fix comments --- Lib/os.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Lib/os.py b/Lib/os.py index 7bf23a3b1b277a..86acb03534443d 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -452,8 +452,8 @@ def fwalk(top=".", topdown=True, onerror=None, *, follow_symlinks=False, dir_fd= races (when follow_symlinks is False). If dir_fd is not None, it should be a file descriptor open to a directory, - and top should be relative; top will then be relative to that directory. - (dir_fd is always supported for fwalk.) + and top should be relative; top will then be relative to that directory. + (dir_fd is always supported for fwalk.) Caution: Since fwalk() yields file descriptors, those are only valid until the @@ -489,7 +489,7 @@ def fwalk(top=".", topdown=True, onerror=None, *, follow_symlinks=False, dir_fd= def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks): # Note: This uses O(depth of the directory tree) file descriptors: if # necessary, it can be adapted to only require O(1) FDs, see issue - # #13734. + # bpo-13734 (gh-57943). stack = [(_WalkAction.WALK, (topfd, toppath))] try: while stack: @@ -528,8 +528,10 @@ def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks): pass if topdown: + # Yield top immediately, before walking subdirs yield toppath, dirs, nondirs, topfd else: + # Yield top after walking subdirs stack.append( (_WalkAction.YIELD, (toppath, dirs, nondirs, topfd))) @@ -549,7 +551,11 @@ def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks): if onerror is not None: onerror(err) continue + # Close dirfd right after all subdirs have been traversed. + # Note that we use a stack, so actions appended first are + # executed last. stack.append((_WalkAction.CLOSE, dirfd)) + # Walk all subdirs if follow_symlinks or path.samestat(orig_st, stat(dirfd)): dirpath = path.join(toppath, name) stack.append((_WalkAction.WALK, (dirfd, dirpath))) From af18a1d882aeaac006b7d2eb9e077cd0f4548f8b Mon Sep 17 00:00:00 2001 From: Jon Burdo Date: Tue, 20 Dec 2022 10:45:12 -0500 Subject: [PATCH 16/24] change ValueError to AssertionError --- Lib/os.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/os.py b/Lib/os.py index 86acb03534443d..5a12154c56678a 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -503,7 +503,7 @@ def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks): elif action is _WalkAction.WALK: topfd, toppath = value else: - raise ValueError(f"invalid walk action: {action}") + raise AssertionError(f"invalid walk action: {action!r}") scandir_it = scandir(topfd) dirs = [] nondirs = [] From 5ee50c6d0c9b63f934cd4af49097beccea426cda Mon Sep 17 00:00:00 2001 From: Jon Burdo Date: Thu, 23 Mar 2023 10:18:12 -0400 Subject: [PATCH 17/24] add more reliable file descriptor closing logic in os.fwalk Make sure file descriptors get closed if an exception occurs right after the file descriptor was popped from the stack. Also catch exceptions thrown by calling os.close on an already-closed file-descriptor when doing file descriptor cleanup. --- Lib/os.py | 36 +++++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/Lib/os.py b/Lib/os.py index 8e012d32b2fe2d..68716aa325cf4c 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -494,16 +494,27 @@ def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks): try: while stack: action, value = stack.pop() - if action is _WalkAction.YIELD: - yield value - continue - elif action is _WalkAction.CLOSE: - close(value) - continue - elif action is _WalkAction.WALK: - topfd, toppath = value - else: - raise AssertionError(f"invalid walk action: {action!r}") + try: + if action is _WalkAction.YIELD: + yield value + continue + elif action is _WalkAction.CLOSE: + close(value) + continue + elif action is _WalkAction.WALK: + topfd, toppath = value + else: + raise AssertionError(f"invalid walk action: {action!r}") + except: + # make sure fd is closed when an exception occurs right + # after its popped from the stack + if action is _WalkAction.CLOSE: + try: + close(value) + except OSError: + pass + raise + scandir_it = scandir(topfd) dirs = [] nondirs = [] @@ -562,7 +573,10 @@ def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks): except: for action, value in reversed(stack): if action is _WalkAction.CLOSE: - close(value) + try: + close(value) + except OSError: + pass raise __all__.append("fwalk") From f0f9330f4c2a6bbb24957002f1442ca135022c37 Mon Sep 17 00:00:00 2001 From: Jon Burdo Date: Thu, 23 Mar 2023 10:50:51 -0400 Subject: [PATCH 18/24] use a separate fd_stack for simpler cleanup and error handling --- Lib/os.py | 48 +++++++++++++++++++++--------------------------- 1 file changed, 21 insertions(+), 27 deletions(-) diff --git a/Lib/os.py b/Lib/os.py index 68716aa325cf4c..f14d8404191998 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -491,29 +491,23 @@ def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks): # necessary, it can be adapted to only require O(1) FDs, see issue # bpo-13734 (gh-57943). stack = [(_WalkAction.WALK, (topfd, toppath))] + fd_stack = [] try: while stack: action, value = stack.pop() - try: - if action is _WalkAction.YIELD: - yield value - continue - elif action is _WalkAction.CLOSE: - close(value) - continue - elif action is _WalkAction.WALK: - topfd, toppath = value - else: - raise AssertionError(f"invalid walk action: {action!r}") - except: - # make sure fd is closed when an exception occurs right - # after its popped from the stack - if action is _WalkAction.CLOSE: - try: - close(value) - except OSError: - pass - raise + if action is _WalkAction.YIELD: + yield value + continue + elif action is _WalkAction.CLOSE: + # Don't remove any fd from fd_stack until after it + # is closed + close(fd_stack[-1]) + fd_stack.pop() + continue + elif action is _WalkAction.WALK: + topfd, toppath = value + else: + raise AssertionError(f"invalid walk action: {action!r}") scandir_it = scandir(topfd) dirs = [] @@ -565,18 +559,18 @@ def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks): # Close dirfd right after all subdirs have been traversed. # Note that we use a stack, so actions appended first are # executed last. - stack.append((_WalkAction.CLOSE, dirfd)) + fd_stack.append(dirfd) + stack.append((_WalkAction.CLOSE, None)) # Walk all subdirs if follow_symlinks or path.samestat(orig_st, stat(dirfd)): dirpath = path.join(toppath, name) stack.append((_WalkAction.WALK, (dirfd, dirpath))) except: - for action, value in reversed(stack): - if action is _WalkAction.CLOSE: - try: - close(value) - except OSError: - pass + for fd in reversed(fd_stack): + try: + close(fd) + except OSError: + pass raise __all__.append("fwalk") From 598bdf908ae91783d812edeeb6f1532e134f938a Mon Sep 17 00:00:00 2001 From: Jon Burdo Date: Thu, 23 Mar 2023 10:55:37 -0400 Subject: [PATCH 19/24] run test_walk_above_recursion_limit with fwalk tests --- Lib/test/test_os.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 74ece3ffb4ed17..3356233f6a6f26 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -1596,8 +1596,6 @@ def test_fd_leak(self): # fwalk() keeps file descriptors open test_walk_many_open_files = None - # fwalk() still uses recursion - test_walk_above_recursion_limit = None class BytesWalkTests(WalkTests): From 6608a6ad3eff2bc8882e66b40f993046453d6bbf Mon Sep 17 00:00:00 2001 From: Jon Burdo Date: Sun, 26 Mar 2023 09:49:12 -0400 Subject: [PATCH 20/24] make sure we don't close the same fd twice If an exception occurred between `close(fd_stack[-1])` and `fd_stack.pop()`, then we would close the fd a second time in the except clause. It would be better to risk leaving the fd open, as the previous implementation of fwalk could --- Lib/os.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Lib/os.py b/Lib/os.py index f14d8404191998..80d386e681004f 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -492,6 +492,7 @@ def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks): # bpo-13734 (gh-57943). stack = [(_WalkAction.WALK, (topfd, toppath))] fd_stack = [] + close_action = (_WalkAction.CLOSE, None) try: while stack: action, value = stack.pop() @@ -499,10 +500,7 @@ def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks): yield value continue elif action is _WalkAction.CLOSE: - # Don't remove any fd from fd_stack until after it - # is closed - close(fd_stack[-1]) - fd_stack.pop() + close(fd_stack.pop()) continue elif action is _WalkAction.WALK: topfd, toppath = value @@ -560,7 +558,7 @@ def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks): # Note that we use a stack, so actions appended first are # executed last. fd_stack.append(dirfd) - stack.append((_WalkAction.CLOSE, None)) + stack.append(close_action) # Walk all subdirs if follow_symlinks or path.samestat(orig_st, stat(dirfd)): dirpath = path.join(toppath, name) From d026146a50eef6c2137b2bd9f2e333a83a093e63 Mon Sep 17 00:00:00 2001 From: Jon Burdo Date: Sun, 26 Mar 2023 10:14:38 -0400 Subject: [PATCH 21/24] revert to using a single stack instead of a separate ffd stack --- Lib/os.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/Lib/os.py b/Lib/os.py index 80d386e681004f..1560d628397dac 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -491,7 +491,6 @@ def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks): # necessary, it can be adapted to only require O(1) FDs, see issue # bpo-13734 (gh-57943). stack = [(_WalkAction.WALK, (topfd, toppath))] - fd_stack = [] close_action = (_WalkAction.CLOSE, None) try: while stack: @@ -500,7 +499,7 @@ def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks): yield value continue elif action is _WalkAction.CLOSE: - close(fd_stack.pop()) + close(value) continue elif action is _WalkAction.WALK: topfd, toppath = value @@ -557,18 +556,18 @@ def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks): # Close dirfd right after all subdirs have been traversed. # Note that we use a stack, so actions appended first are # executed last. - fd_stack.append(dirfd) - stack.append(close_action) + stack.append((_WalkAction.CLOSE, dirfd)) # Walk all subdirs if follow_symlinks or path.samestat(orig_st, stat(dirfd)): dirpath = path.join(toppath, name) stack.append((_WalkAction.WALK, (dirfd, dirpath))) except: - for fd in reversed(fd_stack): - try: - close(fd) - except OSError: - pass + for action, value in reversed(stack): + if action is _WalkAction.CLOSE: + try: + close(value) + except OSError: + pass raise __all__.append("fwalk") From 762c03a939a42b2b9776a9230a5e2212deeddb28 Mon Sep 17 00:00:00 2001 From: Jon Burdo Date: Sun, 26 Mar 2023 10:16:04 -0400 Subject: [PATCH 22/24] remove unused variable --- Lib/os.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/os.py b/Lib/os.py index 1560d628397dac..8fdf16bf60c304 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -491,7 +491,6 @@ def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks): # necessary, it can be adapted to only require O(1) FDs, see issue # bpo-13734 (gh-57943). stack = [(_WalkAction.WALK, (topfd, toppath))] - close_action = (_WalkAction.CLOSE, None) try: while stack: action, value = stack.pop() From 045fd8878f14226b9b6904ba12784e036f01a0fa Mon Sep 17 00:00:00 2001 From: Jon Burdo Date: Sat, 1 Apr 2023 11:16:41 -0400 Subject: [PATCH 23/24] get rid of unnecessary os._fwalk --- Lib/os.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/Lib/os.py b/Lib/os.py index 8fdf16bf60c304..454584a02b2541 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -471,27 +471,23 @@ def fwalk(top=".", topdown=True, onerror=None, *, follow_symlinks=False, dir_fd= if 'CVS' in dirs: dirs.remove('CVS') # don't visit CVS directories """ + # Note: This uses O(depth of the directory tree) file descriptors: if + # necessary, it can be adapted to only require O(1) FDs, see issue + # bpo-13734 (gh-57943). sys.audit("os.fwalk", top, topdown, onerror, follow_symlinks, dir_fd) top = fspath(top) + isbytes = isinstance(top, bytes) # Note: To guard against symlink races, we use the standard # lstat()/open()/fstat() trick. if not follow_symlinks: orig_st = stat(top, follow_symlinks=False, dir_fd=dir_fd) topfd = open(top, O_RDONLY, dir_fd=dir_fd) + stack = [(_WalkAction.CLOSE, topfd)] try: if (follow_symlinks or (st.S_ISDIR(orig_st.st_mode) and path.samestat(orig_st, stat(topfd)))): - yield from _fwalk(topfd, top, isinstance(top, bytes), - topdown, onerror, follow_symlinks) - finally: - close(topfd) + stack.append((_WalkAction.WALK, (topfd, top))) - def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks): - # Note: This uses O(depth of the directory tree) file descriptors: if - # necessary, it can be adapted to only require O(1) FDs, see issue - # bpo-13734 (gh-57943). - stack = [(_WalkAction.WALK, (topfd, toppath))] - try: while stack: action, value = stack.pop() if action is _WalkAction.YIELD: @@ -501,7 +497,7 @@ def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks): close(value) continue elif action is _WalkAction.WALK: - topfd, toppath = value + topfd, top = value else: raise AssertionError(f"invalid walk action: {action!r}") @@ -530,11 +526,11 @@ def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks): if topdown: # Yield top immediately, before walking subdirs - yield toppath, dirs, nondirs, topfd + yield top, dirs, nondirs, topfd else: # Yield top after walking subdirs stack.append( - (_WalkAction.YIELD, (toppath, dirs, nondirs, topfd))) + (_WalkAction.YIELD, (top, dirs, nondirs, topfd))) for name in (reversed(dirs) if entries is None else zip(reversed(dirs), reversed(entries))): @@ -558,7 +554,7 @@ def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks): stack.append((_WalkAction.CLOSE, dirfd)) # Walk all subdirs if follow_symlinks or path.samestat(orig_st, stat(dirfd)): - dirpath = path.join(toppath, name) + dirpath = path.join(top, name) stack.append((_WalkAction.WALK, (dirfd, dirpath))) except: for action, value in reversed(stack): From f3f793a48634c1fe6f500e95f5f9bbed69c247e3 Mon Sep 17 00:00:00 2001 From: Jon Burdo Date: Sat, 1 Apr 2023 13:26:50 -0400 Subject: [PATCH 24/24] change except clause to finally clause in os.fwalk --- Lib/os.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/os.py b/Lib/os.py index 454584a02b2541..6e44b331fd9b89 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -556,14 +556,13 @@ def fwalk(top=".", topdown=True, onerror=None, *, follow_symlinks=False, dir_fd= if follow_symlinks or path.samestat(orig_st, stat(dirfd)): dirpath = path.join(top, name) stack.append((_WalkAction.WALK, (dirfd, dirpath))) - except: + finally: for action, value in reversed(stack): if action is _WalkAction.CLOSE: try: close(value) except OSError: pass - raise __all__.append("fwalk")