From 52ea05fadd717dbdf52a8e04e6f477f17d57972c Mon Sep 17 00:00:00 2001 From: Sacha Date: Mon, 15 May 2023 02:01:58 +0200 Subject: [PATCH 01/13] Fixed netrc comments handling --- Lib/netrc.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/Lib/netrc.py b/Lib/netrc.py index b285fd8e357ddb..20eb77e3c4bcae 100644 --- a/Lib/netrc.py +++ b/Lib/netrc.py @@ -2,7 +2,8 @@ # Module and documentation by Eric S. Raymond, 21 Dec 1998 -import os, stat +import os +import stat __all__ = ["netrc", "NetrcParseError"] @@ -66,7 +67,7 @@ def push_token(self, token): class netrc: def __init__(self, file=None): default_netrc = file is None - if file is None: + if default_netrc: file = os.path.join(os.path.expanduser("~"), ".netrc") self.hosts = {} self.macros = {} @@ -81,13 +82,15 @@ def _parse(self, file, fp, default_netrc): lexer = _netrclex(fp) while 1: # Look for a machine, default, or macdef top-level keyword - saved_lineno = lexer.lineno - toplevel = tt = lexer.get_token() + tt = lexer.get_token() if not tt: break elif tt[0] == '#': - if lexer.lineno == saved_lineno and len(tt) == 1: + # For top level tokens, we skip line if the # is followed + # by a space. Otherwise, we only skip the token. + if len(tt) == 1: lexer.instream.readline() + lexer.lineno++ continue elif tt == 'machine': entryname = lexer.get_token() @@ -98,6 +101,7 @@ def _parse(self, file, fp, default_netrc): self.macros[entryname] = [] while 1: line = lexer.instream.readline() + lexer.lineno++ if not line: raise NetrcParseError( "Macro definition missing null line terminator.", @@ -114,17 +118,17 @@ def _parse(self, file, fp, default_netrc): "bad toplevel token %r" % tt, file, lexer.lineno) if not entryname: - raise NetrcParseError("missing %r name" % tt, file, lexer.lineno) + raise NetrcParseError( + "missing %r name" % tt, file, lexer.lineno) # We're looking at start of an entry for a named machine or default. login = account = password = '' self.hosts[entryname] = {} while 1: - prev_lineno = lexer.lineno tt = lexer.get_token() - if tt.startswith('#'): - if lexer.lineno == prev_lineno: - lexer.instream.readline() + if tt[0] == '#': + lexer.instream.readline() + lexer.lineno++ continue if tt in {'', 'machine', 'default', 'macdef'}: self.hosts[entryname] = (login, account, password) @@ -165,12 +169,7 @@ def _security_check(self, fp, default_netrc, login): def authenticators(self, host): """Return a (user, account, password) tuple for given host.""" - if host in self.hosts: - return self.hosts[host] - elif 'default' in self.hosts: - return self.hosts['default'] - else: - return None + return self.hosts.get(host, self.hosts.get('default')) def __repr__(self): """Dump the class data in the format of a .netrc file.""" @@ -188,5 +187,6 @@ def __repr__(self): rep += "\n" return rep + if __name__ == '__main__': print(netrc()) From 2810677d4fc61f50d5f54448e82b040618cd945f Mon Sep 17 00:00:00 2001 From: Sacha Date: Mon, 15 May 2023 19:03:07 +0200 Subject: [PATCH 02/13] Implementation that passes all test cases. Rework on the `lexer.get_token()` method --- Lib/netrc.py | 49 +++++++++++++++++++++--------------------- Lib/test/test_netrc.py | 25 ++++++++++++++++++++- 2 files changed, 49 insertions(+), 25 deletions(-) diff --git a/Lib/netrc.py b/Lib/netrc.py index 20eb77e3c4bcae..ba286a2adb87fa 100644 --- a/Lib/netrc.py +++ b/Lib/netrc.py @@ -23,6 +23,7 @@ def __str__(self): class _netrclex: def __init__(self, fp): self.lineno = 1 + self.dontskip = False self.instream = fp self.whitespace = "\n\t\r " self.pushback = [] @@ -34,30 +35,29 @@ def _read_char(self): return ch def get_token(self): + self.dontskip = False if self.pushback: return self.pushback.pop(0) token = "" - fiter = iter(self._read_char, "") - for ch in fiter: - if ch in self.whitespace: + enquoted = False + while ch := self._read_char(): + if ch == '\\': + ch = self._read_char() + token += ch continue + if ch in self.whitespace and not enquoted: + if token == "": + continue + if ch == '\n': + self.dontskip = True + return token if ch == '"': - for ch in fiter: - if ch == '"': - return token - elif ch == "\\": - ch = self._read_char() - token += ch + if enquoted: + return token + enquoted = True + continue else: - if ch == "\\": - ch = self._read_char() token += ch - for ch in fiter: - if ch in self.whitespace: - return token - elif ch == "\\": - ch = self._read_char() - token += ch return token def push_token(self, token): @@ -87,10 +87,10 @@ def _parse(self, file, fp, default_netrc): break elif tt[0] == '#': # For top level tokens, we skip line if the # is followed - # by a space. Otherwise, we only skip the token. - if len(tt) == 1: + # by a space / newline. Otherwise, we only skip the token. + if tt == '#' and not lexer.dontskip: lexer.instream.readline() - lexer.lineno++ + lexer.lineno += 1 continue elif tt == 'machine': entryname = lexer.get_token() @@ -101,7 +101,7 @@ def _parse(self, file, fp, default_netrc): self.macros[entryname] = [] while 1: line = lexer.instream.readline() - lexer.lineno++ + lexer.lineno += 1 if not line: raise NetrcParseError( "Macro definition missing null line terminator.", @@ -126,9 +126,10 @@ def _parse(self, file, fp, default_netrc): self.hosts[entryname] = {} while 1: tt = lexer.get_token() - if tt[0] == '#': - lexer.instream.readline() - lexer.lineno++ + if tt.startswith('#'): + if not lexer.dontskip: + lexer.instream.readline() + lexer.lineno += 1 continue if tt in {'', 'machine', 'default', 'macdef'}: self.hosts[entryname] = (login, account, password) diff --git a/Lib/test/test_netrc.py b/Lib/test/test_netrc.py index 573d636de956d1..8273212e0573ce 100644 --- a/Lib/test/test_netrc.py +++ b/Lib/test/test_netrc.py @@ -1,4 +1,8 @@ -import netrc, os, unittest, sys, textwrap +import netrc +import os +import sys +import textwrap +import unittest from test.support import os_helper, run_unittest try: @@ -8,6 +12,7 @@ temp_filename = os_helper.TESTFN + class NetrcTestCase(unittest.TestCase): def make_nrc(self, test_data): @@ -191,6 +196,7 @@ def test_token_value_internal_hash(self): def _test_comment(self, nrc, passwd='pass'): nrc = self.make_nrc(nrc) + print(nrc.hosts) self.assertEqual(nrc.hosts['foo.domain.com'], ('bar', '', passwd)) self.assertEqual(nrc.hosts['bar.domain.com'], ('foo', '', 'pass')) @@ -215,6 +221,14 @@ def test_comment_before_machine_line_hash_only(self): machine bar.domain.com login foo password pass """) + def test_comment_after_new_line(self): + self._test_comment("""\ + machine foo.domain.com login bar password pass + + # TEST + machine bar.domain.com login foo password pass + """) + def test_comment_after_machine_line(self): self._test_comment("""\ machine foo.domain.com login bar password pass @@ -251,6 +265,13 @@ def test_comment_after_machine_line_hash_only(self): # """) + def test_comment_at_first_line(self): + self._test_comment(""" + # TEST + machine foo.domain.com login bar password pass + machine bar.domain.com login foo password pass + """) + def test_comment_at_end_of_machine_line(self): self._test_comment("""\ machine foo.domain.com login bar password pass # comment @@ -308,8 +329,10 @@ def test_security(self): self.assertEqual(nrc.hosts['foo.domain.com'], ('anonymous', '', 'pass')) + def test_main(): run_unittest(NetrcTestCase) + if __name__ == "__main__": test_main() From 68c946debb0a50120bd3347590081dfda74505c9 Mon Sep 17 00:00:00 2001 From: Sacha Date: Mon, 15 May 2023 19:05:43 +0200 Subject: [PATCH 03/13] Removed debug remnants --- Lib/test/test_netrc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_netrc.py b/Lib/test/test_netrc.py index 8273212e0573ce..863b00a51917a5 100644 --- a/Lib/test/test_netrc.py +++ b/Lib/test/test_netrc.py @@ -196,7 +196,6 @@ def test_token_value_internal_hash(self): def _test_comment(self, nrc, passwd='pass'): nrc = self.make_nrc(nrc) - print(nrc.hosts) self.assertEqual(nrc.hosts['foo.domain.com'], ('bar', '', passwd)) self.assertEqual(nrc.hosts['bar.domain.com'], ('foo', '', 'pass')) From a542e84070a7ed36f05342b545167637cbecc8a8 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Mon, 15 May 2023 17:22:54 +0000 Subject: [PATCH 04/13] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Library/2023-05-15-17-22-53.gh-issue-104306.YMiegg.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2023-05-15-17-22-53.gh-issue-104306.YMiegg.rst diff --git a/Misc/NEWS.d/next/Library/2023-05-15-17-22-53.gh-issue-104306.YMiegg.rst b/Misc/NEWS.d/next/Library/2023-05-15-17-22-53.gh-issue-104306.YMiegg.rst new file mode 100644 index 00000000000000..9cf42e6914a9a8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-15-17-22-53.gh-issue-104306.YMiegg.rst @@ -0,0 +1 @@ +Fix incorrect comment parsing in the `netrc`module From 27b83e9ea861373d4a357d52a1cd1f1ac857b672 Mon Sep 17 00:00:00 2001 From: Sacha Dupuydauby Date: Mon, 15 May 2023 19:27:05 +0200 Subject: [PATCH 05/13] Missing whitespace --- .../next/Library/2023-05-15-17-22-53.gh-issue-104306.YMiegg.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2023-05-15-17-22-53.gh-issue-104306.YMiegg.rst b/Misc/NEWS.d/next/Library/2023-05-15-17-22-53.gh-issue-104306.YMiegg.rst index 9cf42e6914a9a8..06bfecfb98b32c 100644 --- a/Misc/NEWS.d/next/Library/2023-05-15-17-22-53.gh-issue-104306.YMiegg.rst +++ b/Misc/NEWS.d/next/Library/2023-05-15-17-22-53.gh-issue-104306.YMiegg.rst @@ -1 +1 @@ -Fix incorrect comment parsing in the `netrc`module +Fix incorrect comment parsing in the `netrc` module From 1920abe8aa8bd8788c146c9236acbe6a9f531831 Mon Sep 17 00:00:00 2001 From: Sacha Dupuydauby Date: Mon, 15 May 2023 19:30:24 +0200 Subject: [PATCH 06/13] Update 2023-05-15-17-22-53.gh-issue-104306.YMiegg.rst Double backticks --- .../next/Library/2023-05-15-17-22-53.gh-issue-104306.YMiegg.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2023-05-15-17-22-53.gh-issue-104306.YMiegg.rst b/Misc/NEWS.d/next/Library/2023-05-15-17-22-53.gh-issue-104306.YMiegg.rst index 06bfecfb98b32c..c91776b05fbff8 100644 --- a/Misc/NEWS.d/next/Library/2023-05-15-17-22-53.gh-issue-104306.YMiegg.rst +++ b/Misc/NEWS.d/next/Library/2023-05-15-17-22-53.gh-issue-104306.YMiegg.rst @@ -1 +1 @@ -Fix incorrect comment parsing in the `netrc` module +Fix incorrect comment parsing in the ``netrc`` module From 8c243db4c359222ad092b5308549c88463af0051 Mon Sep 17 00:00:00 2001 From: Sacha Dupuydauby Date: Mon, 15 May 2023 20:08:19 +0200 Subject: [PATCH 07/13] Link to module Co-authored-by: Oleg Iarygin --- .../next/Library/2023-05-15-17-22-53.gh-issue-104306.YMiegg.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2023-05-15-17-22-53.gh-issue-104306.YMiegg.rst b/Misc/NEWS.d/next/Library/2023-05-15-17-22-53.gh-issue-104306.YMiegg.rst index c91776b05fbff8..60e75d1b64aa43 100644 --- a/Misc/NEWS.d/next/Library/2023-05-15-17-22-53.gh-issue-104306.YMiegg.rst +++ b/Misc/NEWS.d/next/Library/2023-05-15-17-22-53.gh-issue-104306.YMiegg.rst @@ -1 +1 @@ -Fix incorrect comment parsing in the ``netrc`` module +Fix incorrect comment parsing in the :mod:`netrc` module. From e83581982ca55b70bdffb7e9f815eee8f5b2592c Mon Sep 17 00:00:00 2001 From: Sacha Dupuydauby Date: Mon, 6 Jan 2025 21:37:43 +0000 Subject: [PATCH 08/13] fix netrc unit tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 🇺🇦 Sviatoslav Sydorenko (Святослав Сидоренко) --- Lib/test/test_netrc.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/Lib/test/test_netrc.py b/Lib/test/test_netrc.py index 1c5da1cc753ea5..c0450084a83d7b 100644 --- a/Lib/test/test_netrc.py +++ b/Lib/test/test_netrc.py @@ -1,9 +1,5 @@ -import netrc -import os -import sys -import textwrap -import unittest -from test.support import os_helper, run_unittest +import netrc, os, unittest, sys, textwrap +from test.support import os_helper try: import pwd @@ -12,7 +8,6 @@ temp_filename = os_helper.TESTFN - class NetrcTestCase(unittest.TestCase): def make_nrc(self, test_data): @@ -329,9 +324,5 @@ def test_security(self): ('anonymous', '', 'pass')) -def test_main(): - run_unittest(NetrcTestCase) - - if __name__ == "__main__": unittest.main() From 7687836f4e495926d90c0bb370351fe5d0b732fc Mon Sep 17 00:00:00 2001 From: Sacha Dupuydauby Date: Mon, 6 Jan 2025 21:39:32 +0000 Subject: [PATCH 09/13] remove extra blank line MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 🇺🇦 Sviatoslav Sydorenko (Святослав Сидоренко) --- Lib/netrc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/netrc.py b/Lib/netrc.py index ba286a2adb87fa..1b81dd4957c76e 100644 --- a/Lib/netrc.py +++ b/Lib/netrc.py @@ -188,6 +188,5 @@ def __repr__(self): rep += "\n" return rep - if __name__ == '__main__': print(netrc()) From 7f64f234eac8e7d17b7f11fd80f942ccc2e2b2ca Mon Sep 17 00:00:00 2001 From: Sacha Dupuydauby Date: Mon, 6 Jan 2025 21:40:45 +0000 Subject: [PATCH 10/13] remove extra blank line MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 🇺🇦 Sviatoslav Sydorenko (Святослав Сидоренко) --- Lib/netrc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/netrc.py b/Lib/netrc.py index 1b81dd4957c76e..4a05f46581d2bd 100644 --- a/Lib/netrc.py +++ b/Lib/netrc.py @@ -188,5 +188,4 @@ def __repr__(self): rep += "\n" return rep -if __name__ == '__main__': print(netrc()) From 9e631859385f0223feded066d92de298b8c50bd1 Mon Sep 17 00:00:00 2001 From: Sacha Dupuydauby Date: Mon, 6 Jan 2025 22:00:34 +0000 Subject: [PATCH 11/13] remove extra line --- Lib/netrc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/netrc.py b/Lib/netrc.py index 4a05f46581d2bd..6a251b5bcf3708 100644 --- a/Lib/netrc.py +++ b/Lib/netrc.py @@ -188,4 +188,3 @@ def __repr__(self): rep += "\n" return rep - print(netrc()) From dc72ad1dc164d19693aada72a5baeb266569891c Mon Sep 17 00:00:00 2001 From: Sacha Dupuydauby Date: Mon, 6 Jan 2025 22:03:05 +0000 Subject: [PATCH 12/13] remove extra blank line --- Lib/netrc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/netrc.py b/Lib/netrc.py index 6a251b5bcf3708..f40c88e25faabd 100644 --- a/Lib/netrc.py +++ b/Lib/netrc.py @@ -187,4 +187,3 @@ def __repr__(self): rep += line rep += "\n" return rep - From 5ed10a734a68135cc96e5738cf73ac7e268d6c22 Mon Sep 17 00:00:00 2001 From: Sacha Dupuydauby Date: Tue, 7 Jan 2025 22:42:58 +0000 Subject: [PATCH 13/13] fix: skip any trailing new line when lexing --- Lib/netrc.py | 81 ++++++++++++++++++++++++++---------------- Lib/test/test_netrc.py | 13 ++++++- 2 files changed, 62 insertions(+), 32 deletions(-) diff --git a/Lib/netrc.py b/Lib/netrc.py index f40c88e25faabd..52a9b4ddbe6283 100644 --- a/Lib/netrc.py +++ b/Lib/netrc.py @@ -2,8 +2,7 @@ # Module and documentation by Eric S. Raymond, 21 Dec 1998 -import os -import stat +import os, stat __all__ = ["netrc", "NetrcParseError"] @@ -23,41 +22,53 @@ def __str__(self): class _netrclex: def __init__(self, fp): self.lineno = 1 - self.dontskip = False self.instream = fp self.whitespace = "\n\t\r " self.pushback = [] + self.char_pushback = [] def _read_char(self): + if self.char_pushback: + return self.char_pushback.pop(0) ch = self.instream.read(1) if ch == "\n": self.lineno += 1 return ch + def skip_blank_lines(self): + fiter = iter(self._read_char, "") + for ch in fiter: + if ch == '\n': + self.lineno += 1 + else: + self.char_pushback.append(ch) + return + def get_token(self): - self.dontskip = False if self.pushback: return self.pushback.pop(0) token = "" - enquoted = False - while ch := self._read_char(): - if ch == '\\': - ch = self._read_char() - token += ch + fiter = iter(self._read_char, "") + for ch in fiter: + if ch in self.whitespace: continue - if ch in self.whitespace and not enquoted: - if token == "": - continue - if ch == '\n': - self.dontskip = True - return token if ch == '"': - if enquoted: - return token - enquoted = True - continue + for ch in fiter: + if ch == '"': + return token + elif ch == "\\": + ch = self._read_char() + token += ch else: + if ch == "\\": + ch = self._read_char() token += ch + for ch in fiter: + if ch in self.whitespace: + return token + elif ch == "\\": + ch = self._read_char() + token += ch return token def push_token(self, token): @@ -67,7 +78,7 @@ def push_token(self, token): class netrc: def __init__(self, file=None): default_netrc = file is None - if default_netrc: + if file is None: file = os.path.join(os.path.expanduser("~"), ".netrc") self.hosts = {} self.macros = {} @@ -82,15 +93,14 @@ def _parse(self, file, fp, default_netrc): lexer = _netrclex(fp) while 1: # Look for a machine, default, or macdef top-level keyword - tt = lexer.get_token() + lexer.skip_blank_lines() + saved_lineno = lexer.lineno + toplevel = tt = lexer.get_token() if not tt: break elif tt[0] == '#': - # For top level tokens, we skip line if the # is followed - # by a space / newline. Otherwise, we only skip the token. - if tt == '#' and not lexer.dontskip: + if lexer.lineno == saved_lineno and len(tt) == 1: lexer.instream.readline() - lexer.lineno += 1 continue elif tt == 'machine': entryname = lexer.get_token() @@ -101,7 +111,6 @@ def _parse(self, file, fp, default_netrc): self.macros[entryname] = [] while 1: line = lexer.instream.readline() - lexer.lineno += 1 if not line: raise NetrcParseError( "Macro definition missing null line terminator.", @@ -118,18 +127,20 @@ def _parse(self, file, fp, default_netrc): "bad toplevel token %r" % tt, file, lexer.lineno) if not entryname: - raise NetrcParseError( - "missing %r name" % tt, file, lexer.lineno) + raise NetrcParseError("missing %r name" % tt, file, lexer.lineno) # We're looking at start of an entry for a named machine or default. login = account = password = '' self.hosts[entryname] = {} while 1: + # Trailing blank lines would break the checks that determine if the token + # is the last one on its line. + lexer.skip_blank_lines() + prev_lineno = lexer.lineno tt = lexer.get_token() if tt.startswith('#'): - if not lexer.dontskip: + if lexer.lineno == prev_lineno: lexer.instream.readline() - lexer.lineno += 1 continue if tt in {'', 'machine', 'default', 'macdef'}: self.hosts[entryname] = (login, account, password) @@ -170,7 +181,12 @@ def _security_check(self, fp, default_netrc, login): def authenticators(self, host): """Return a (user, account, password) tuple for given host.""" - return self.hosts.get(host, self.hosts.get('default')) + if host in self.hosts: + return self.hosts[host] + elif 'default' in self.hosts: + return self.hosts['default'] + else: + return None def __repr__(self): """Dump the class data in the format of a .netrc file.""" @@ -187,3 +203,6 @@ def __repr__(self): rep += line rep += "\n" return rep + +if __name__ == '__main__': + print(netrc()) diff --git a/Lib/test/test_netrc.py b/Lib/test/test_netrc.py index c0450084a83d7b..774fccd40d251f 100644 --- a/Lib/test/test_netrc.py +++ b/Lib/test/test_netrc.py @@ -259,13 +259,24 @@ def test_comment_after_machine_line_hash_only(self): # """) - def test_comment_at_first_line(self): + def test_comment_at_first_line_trailing_new_line(self): self._test_comment(""" # TEST machine foo.domain.com login bar password pass machine bar.domain.com login foo password pass """) + def test_comment_multiple_trailing_new_lines(self): + self._test_comment(""" + # TEST + machine foo.domain.com login bar password pass + + + #FTP + + machine bar.domain.com login foo password pass + """) + def test_comment_at_end_of_machine_line(self): self._test_comment("""\ machine foo.domain.com login bar password pass # comment