From 15409c720be0503131713e3d3abc1acd0da07378 Mon Sep 17 00:00:00 2001 From: Emmanuel Arias Date: Wed, 17 Nov 2021 06:07:54 -0300 Subject: bpo-28806: Continue work: improve the netrc library (GH-26330) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Continue with the improvement of the library netrc Original work and report Xiang Zhang * 📜🤖 Added by blurb_it. Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> --- Doc/library/netrc.rst | 11 +- Lib/netrc.py | 131 ++++++--- Lib/test/test_netrc.py | 305 +++++++++++++++------ .../2021-05-24-13-48-34.bpo-28806.PkNw5D.rst | 1 + 4 files changed, 319 insertions(+), 129 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2021-05-24-13-48-34.bpo-28806.PkNw5D.rst diff --git a/Doc/library/netrc.rst b/Doc/library/netrc.rst index 4bf7de6..88265d9 100644 --- a/Doc/library/netrc.rst +++ b/Doc/library/netrc.rst @@ -41,6 +41,10 @@ the Unix :program:`ftp` program and other FTP clients. .. versionchanged:: 3.10 :class:`netrc` try UTF-8 encoding before using locale specific encoding. + The entry in the netrc file no longer needs to contain all tokens. The missing + tokens' value default to an empty string. All the tokens and their values now + can contain arbitrary characters, like whitespace and non-ASCII characters. + If the login name is anonymous, it won't trigger the security check. .. exception:: NetrcParseError @@ -85,10 +89,3 @@ Instances of :class:`~netrc.netrc` have public instance variables: .. attribute:: netrc.macros Dictionary mapping macro names to string lists. - -.. note:: - - Passwords are limited to a subset of the ASCII character set. All ASCII - punctuation is allowed in passwords, however, note that whitespace and - non-printable characters are not allowed in passwords. This is a limitation - of the way the .netrc file is parsed and may be removed in the future. diff --git a/Lib/netrc.py b/Lib/netrc.py index 734d94c..c1358aa 100644 --- a/Lib/netrc.py +++ b/Lib/netrc.py @@ -19,6 +19,50 @@ class NetrcParseError(Exception): return "%s (%s, line %s)" % (self.msg, self.filename, self.lineno) +class _netrclex: + def __init__(self, fp): + self.lineno = 1 + self.instream = fp + self.whitespace = "\n\t\r " + self.pushback = [] + + def _read_char(self): + ch = self.instream.read(1) + if ch == "\n": + self.lineno += 1 + return ch + + def get_token(self): + if self.pushback: + return self.pushback.pop(0) + token = "" + fiter = iter(self._read_char, "") + for ch in fiter: + if ch in self.whitespace: + continue + if ch == '"': + for ch in fiter: + if ch == '"': + return token + elif ch == "\\": + ch = self._read_char() + token += ch + else: + if ch == "\\": + ch = self._read_char() + token += ch + for ch in fiter: + if ch in self.whitespace: + return token + elif ch == "\\": + ch = self._read_char() + token += ch + return token + + def push_token(self, token): + self.pushback.append(token) + + class netrc: def __init__(self, file=None): default_netrc = file is None @@ -34,9 +78,7 @@ class netrc: self._parse(file, fp, default_netrc) def _parse(self, file, fp, default_netrc): - lexer = shlex.shlex(fp) - lexer.wordchars += r"""!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~""" - lexer.commenters = lexer.commenters.replace('#', '') + lexer = _netrclex(fp) while 1: # Look for a machine, default, or macdef top-level keyword saved_lineno = lexer.lineno @@ -51,14 +93,19 @@ class netrc: entryname = lexer.get_token() elif tt == 'default': entryname = 'default' - elif tt == 'macdef': # Just skip to end of macdefs + elif tt == 'macdef': entryname = lexer.get_token() self.macros[entryname] = [] - lexer.whitespace = ' \t' while 1: line = lexer.instream.readline() - if not line or line == '\012': - lexer.whitespace = ' \t\r\n' + if not line: + raise NetrcParseError( + "Macro definition missing null line terminator.", + file, lexer.lineno) + if line == '\n': + # a macro definition finished with consecutive new-line + # characters. The first \n is encountered by the + # readline() method and this is the second \n. break self.macros[entryname].append(line) continue @@ -66,53 +113,55 @@ class netrc: raise NetrcParseError( "bad toplevel token %r" % tt, file, lexer.lineno) + if not entryname: + raise NetrcParseError("missing %r name" % tt, file, lexer.lineno) + # We're looking at start of an entry for a named machine or default. - login = '' - account = password = None + login = account = password = '' self.hosts[entryname] = {} while 1: + prev_lineno = lexer.lineno tt = lexer.get_token() - if (tt.startswith('#') or - tt in {'', 'machine', 'default', 'macdef'}): - if password: - self.hosts[entryname] = (login, account, password) - lexer.push_token(tt) - break - else: - raise NetrcParseError( - "malformed %s entry %s terminated by %s" - % (toplevel, entryname, repr(tt)), - file, lexer.lineno) + if tt.startswith('#'): + if lexer.lineno == prev_lineno: + lexer.instream.readline() + continue + if tt in {'', 'machine', 'default', 'macdef'}: + self.hosts[entryname] = (login, account, password) + lexer.push_token(tt) + break elif tt == 'login' or tt == 'user': login = lexer.get_token() elif tt == 'account': account = lexer.get_token() elif tt == 'password': - if os.name == 'posix' and default_netrc: - prop = os.fstat(fp.fileno()) - if prop.st_uid != os.getuid(): - import pwd - try: - fowner = pwd.getpwuid(prop.st_uid)[0] - except KeyError: - fowner = 'uid %s' % prop.st_uid - try: - user = pwd.getpwuid(os.getuid())[0] - except KeyError: - user = 'uid %s' % os.getuid() - raise NetrcParseError( - ("~/.netrc file owner (%s) does not match" - " current user (%s)") % (fowner, user), - file, lexer.lineno) - if (prop.st_mode & (stat.S_IRWXG | stat.S_IRWXO)): - raise NetrcParseError( - "~/.netrc access too permissive: access" - " permissions must restrict access to only" - " the owner", file, lexer.lineno) password = lexer.get_token() else: raise NetrcParseError("bad follower token %r" % tt, file, lexer.lineno) + self._security_check(fp, default_netrc, self.hosts[entryname][0]) + + def _security_check(self, fp, default_netrc, login): + if os.name == 'posix' and default_netrc and login != "anonymous": + prop = os.fstat(fp.fileno()) + if prop.st_uid != os.getuid(): + import pwd + try: + fowner = pwd.getpwuid(prop.st_uid)[0] + except KeyError: + fowner = 'uid %s' % prop.st_uid + try: + user = pwd.getpwuid(os.getuid())[0] + except KeyError: + user = 'uid %s' % os.getuid() + raise NetrcParseError( + (f"~/.netrc file owner ({fowner}, {user}) does not match" + " current user")) + if (prop.st_mode & (stat.S_IRWXG | stat.S_IRWXO)): + raise NetrcParseError( + "~/.netrc access too permissive: access" + " permissions must restrict access to only" + " the owner") def authenticators(self, host): """Return a (user, account, password) tuple for given host.""" diff --git a/Lib/test/test_netrc.py b/Lib/test/test_netrc.py index 90ef5cd..a6b4bc4 100644 --- a/Lib/test/test_netrc.py +++ b/Lib/test/test_netrc.py @@ -1,7 +1,7 @@ -import netrc, os, unittest, sys, tempfile, textwrap -from test import support -from test.support import os_helper +import netrc, os, unittest, sys, textwrap +from test.support import os_helper, run_unittest +temp_filename = os_helper.TESTFN class NetrcTestCase(unittest.TestCase): @@ -10,26 +10,32 @@ class NetrcTestCase(unittest.TestCase): mode = 'w' if sys.platform != 'cygwin': mode += 't' - temp_fd, temp_filename = tempfile.mkstemp() - with os.fdopen(temp_fd, mode=mode) as fp: + with open(temp_filename, mode) as fp: fp.write(test_data) - self.addCleanup(os.unlink, temp_filename) - return netrc.netrc(temp_filename) + try: + nrc = netrc.netrc(temp_filename) + finally: + os.unlink(temp_filename) + return nrc - def test_default(self): + def test_toplevel_non_ordered_tokens(self): nrc = self.make_nrc("""\ - machine host1.domain.com login log1 password pass1 account acct1 - default login log2 password pass2 + machine host.domain.com password pass1 login log1 account acct1 + default login log2 password pass2 account acct2 """) - self.assertEqual(nrc.hosts['host1.domain.com'], - ('log1', 'acct1', 'pass1')) - self.assertEqual(nrc.hosts['default'], ('log2', None, 'pass2')) + self.assertEqual(nrc.hosts['host.domain.com'], ('log1', 'acct1', 'pass1')) + self.assertEqual(nrc.hosts['default'], ('log2', 'acct2', 'pass2')) - nrc2 = self.make_nrc(nrc.__repr__()) - self.assertEqual(nrc.hosts, nrc2.hosts) + def test_toplevel_tokens(self): + nrc = self.make_nrc("""\ + machine host.domain.com login log1 password pass1 account acct1 + default login log2 password pass2 account acct2 + """) + self.assertEqual(nrc.hosts['host.domain.com'], ('log1', 'acct1', 'pass1')) + self.assertEqual(nrc.hosts['default'], ('log2', 'acct2', 'pass2')) def test_macros(self): - nrc = self.make_nrc("""\ + data = """\ macdef macro1 line1 line2 @@ -37,33 +43,151 @@ class NetrcTestCase(unittest.TestCase): macdef macro2 line3 line4 - """) + + """ + nrc = self.make_nrc(data) self.assertEqual(nrc.macros, {'macro1': ['line1\n', 'line2\n'], 'macro2': ['line3\n', 'line4\n']}) + # strip the last \n + self.assertRaises(netrc.NetrcParseError, self.make_nrc, + data.rstrip(' ')[:-1]) + + def test_optional_tokens(self): + data = ( + "machine host.domain.com", + "machine host.domain.com login", + "machine host.domain.com account", + "machine host.domain.com password", + "machine host.domain.com login \"\" account", + "machine host.domain.com login \"\" password", + "machine host.domain.com account \"\" password" + ) + for item in data: + nrc = self.make_nrc(item) + self.assertEqual(nrc.hosts['host.domain.com'], ('', '', '')) + data = ( + "default", + "default login", + "default account", + "default password", + "default login \"\" account", + "default login \"\" password", + "default account \"\" password" + ) + for item in data: + nrc = self.make_nrc(item) + self.assertEqual(nrc.hosts['default'], ('', '', '')) - def _test_passwords(self, nrc, passwd): + def test_invalid_tokens(self): + data = ( + "invalid host.domain.com", + "machine host.domain.com invalid", + "machine host.domain.com login log password pass account acct invalid", + "default host.domain.com invalid", + "default host.domain.com login log password pass account acct invalid" + ) + for item in data: + self.assertRaises(netrc.NetrcParseError, self.make_nrc, item) + + def _test_token_x(self, nrc, token, value): nrc = self.make_nrc(nrc) - self.assertEqual(nrc.hosts['host.domain.com'], ('log', 'acct', passwd)) + if token == 'login': + self.assertEqual(nrc.hosts['host.domain.com'], (value, 'acct', 'pass')) + elif token == 'account': + self.assertEqual(nrc.hosts['host.domain.com'], ('log', value, 'pass')) + elif token == 'password': + self.assertEqual(nrc.hosts['host.domain.com'], ('log', 'acct', value)) + + def test_token_value_quotes(self): + self._test_token_x("""\ + machine host.domain.com login "log" password pass account acct + """, 'login', 'log') + self._test_token_x("""\ + machine host.domain.com login log password pass account "acct" + """, 'account', 'acct') + self._test_token_x("""\ + machine host.domain.com login log password "pass" account acct + """, 'password', 'pass') + + def test_token_value_escape(self): + self._test_token_x("""\ + machine host.domain.com login \\"log password pass account acct + """, 'login', '"log') + self._test_token_x("""\ + machine host.domain.com login "\\"log" password pass account acct + """, 'login', '"log') + self._test_token_x("""\ + machine host.domain.com login log password pass account \\"acct + """, 'account', '"acct') + self._test_token_x("""\ + machine host.domain.com login log password pass account "\\"acct" + """, 'account', '"acct') + self._test_token_x("""\ + machine host.domain.com login log password \\"pass account acct + """, 'password', '"pass') + self._test_token_x("""\ + machine host.domain.com login log password "\\"pass" account acct + """, 'password', '"pass') - def test_password_with_leading_hash(self): - self._test_passwords("""\ + def test_token_value_whitespace(self): + self._test_token_x("""\ + machine host.domain.com login "lo g" password pass account acct + """, 'login', 'lo g') + self._test_token_x("""\ + machine host.domain.com login log password "pas s" account acct + """, 'password', 'pas s') + self._test_token_x("""\ + machine host.domain.com login log password pass account "acc t" + """, 'account', 'acc t') + + def test_token_value_non_ascii(self): + self._test_token_x("""\ + machine host.domain.com login \xa1\xa2 password pass account acct + """, 'login', '\xa1\xa2') + self._test_token_x("""\ + machine host.domain.com login log password pass account \xa1\xa2 + """, 'account', '\xa1\xa2') + self._test_token_x("""\ + machine host.domain.com login log password \xa1\xa2 account acct + """, 'password', '\xa1\xa2') + + def test_token_value_leading_hash(self): + self._test_token_x("""\ + machine host.domain.com login #log password pass account acct + """, 'login', '#log') + self._test_token_x("""\ + machine host.domain.com login log password pass account #acct + """, 'account', '#acct') + self._test_token_x("""\ machine host.domain.com login log password #pass account acct - """, '#pass') + """, 'password', '#pass') - def test_password_with_trailing_hash(self): - self._test_passwords("""\ + def test_token_value_trailing_hash(self): + self._test_token_x("""\ + machine host.domain.com login log# password pass account acct + """, 'login', 'log#') + self._test_token_x("""\ + machine host.domain.com login log password pass account acct# + """, 'account', 'acct#') + self._test_token_x("""\ machine host.domain.com login log password pass# account acct - """, 'pass#') + """, 'password', 'pass#') - def test_password_with_internal_hash(self): - self._test_passwords("""\ + def test_token_value_internal_hash(self): + self._test_token_x("""\ + machine host.domain.com login lo#g password pass account acct + """, 'login', 'lo#g') + self._test_token_x("""\ + machine host.domain.com login log password pass account ac#ct + """, 'account', 'ac#ct') + self._test_token_x("""\ machine host.domain.com login log password pa#ss account acct - """, 'pa#ss') + """, 'password', 'pa#ss') def _test_comment(self, nrc, passwd='pass'): nrc = self.make_nrc(nrc) - self.assertEqual(nrc.hosts['foo.domain.com'], ('bar', None, passwd)) - self.assertEqual(nrc.hosts['bar.domain.com'], ('foo', None, 'pass')) + self.assertEqual(nrc.hosts['foo.domain.com'], ('bar', '', passwd)) + self.assertEqual(nrc.hosts['bar.domain.com'], ('foo', '', 'pass')) def test_comment_before_machine_line(self): self._test_comment("""\ @@ -86,6 +210,42 @@ class NetrcTestCase(unittest.TestCase): machine bar.domain.com login foo password pass """) + def test_comment_after_machine_line(self): + self._test_comment("""\ + machine foo.domain.com login bar password pass + # comment + machine bar.domain.com login foo password pass + """) + self._test_comment("""\ + machine foo.domain.com login bar password pass + machine bar.domain.com login foo password pass + # comment + """) + + def test_comment_after_machine_line_no_space(self): + self._test_comment("""\ + machine foo.domain.com login bar password pass + #comment + machine bar.domain.com login foo password pass + """) + self._test_comment("""\ + machine foo.domain.com login bar password pass + machine bar.domain.com login foo password pass + #comment + """) + + def test_comment_after_machine_line_hash_only(self): + self._test_comment("""\ + machine foo.domain.com login bar password pass + # + machine bar.domain.com login foo password pass + """) + self._test_comment("""\ + machine foo.domain.com login bar password pass + machine bar.domain.com login foo password pass + # + """) + def test_comment_at_end_of_machine_line(self): self._test_comment("""\ machine foo.domain.com login bar password pass # comment @@ -109,57 +269,40 @@ class NetrcTestCase(unittest.TestCase): def test_security(self): # This test is incomplete since we are normally not run as root and # therefore can't test the file ownership being wrong. - with os_helper.temp_cwd(None) as d: - fn = os.path.join(d, '.netrc') - with open(fn, 'wt') as f: - f.write("""\ - machine foo.domain.com login bar password pass - default login foo password pass - """) - with os_helper.EnvironmentVarGuard() as environ: - environ.set('HOME', d) - os.chmod(fn, 0o600) - nrc = netrc.netrc() - self.assertEqual(nrc.hosts['foo.domain.com'], - ('bar', None, 'pass')) - os.chmod(fn, 0o622) - self.assertRaises(netrc.NetrcParseError, netrc.netrc) - - def test_file_not_found_in_home(self): - with os_helper.temp_cwd(None) as d: - with os_helper.EnvironmentVarGuard() as environ: - environ.set('HOME', d) - self.assertRaises(FileNotFoundError, netrc.netrc) - - def test_file_not_found_explicit(self): - self.assertRaises(FileNotFoundError, netrc.netrc, - file='unlikely_netrc') - - def test_home_not_set(self): - with os_helper.temp_cwd(None) as fake_home: - fake_netrc_path = os.path.join(fake_home, '.netrc') - with open(fake_netrc_path, 'w') as f: - f.write('machine foo.domain.com login bar password pass') - os.chmod(fake_netrc_path, 0o600) - - orig_expanduser = os.path.expanduser - called = [] - - def fake_expanduser(s): - called.append(s) - with os_helper.EnvironmentVarGuard() as environ: - environ.set('HOME', fake_home) - environ.set('USERPROFILE', fake_home) - result = orig_expanduser(s) - return result - - with support.swap_attr(os.path, 'expanduser', fake_expanduser): - nrc = netrc.netrc() - login, account, password = nrc.authenticators('foo.domain.com') - self.assertEqual(login, 'bar') - - self.assertTrue(called) + d = os_helper.TESTFN + os.mkdir(d) + self.addCleanup(os_helper.rmtree, d) + fn = os.path.join(d, '.netrc') + with open(fn, 'wt') as f: + f.write("""\ + machine foo.domain.com login bar password pass + default login foo password pass + """) + with os_helper.EnvironmentVarGuard() as environ: + environ.set('HOME', d) + os.chmod(fn, 0o600) + nrc = netrc.netrc() + self.assertEqual(nrc.hosts['foo.domain.com'], + ('bar', '', 'pass')) + os.chmod(fn, 0o622) + self.assertRaises(netrc.NetrcParseError, netrc.netrc) + with open(fn, 'wt') as f: + f.write("""\ + machine foo.domain.com login anonymous password pass + default login foo password pass + """) + with os_helper.EnvironmentVarGuard() as environ: + environ.set('HOME', d) + os.chmod(fn, 0o600) + nrc = netrc.netrc() + self.assertEqual(nrc.hosts['foo.domain.com'], + ('anonymous', '', 'pass')) + os.chmod(fn, 0o622) + self.assertEqual(nrc.hosts['foo.domain.com'], + ('anonymous', '', 'pass')) +def test_main(): + run_unittest(NetrcTestCase) if __name__ == "__main__": - unittest.main() + test_main() diff --git a/Misc/NEWS.d/next/Library/2021-05-24-13-48-34.bpo-28806.PkNw5D.rst b/Misc/NEWS.d/next/Library/2021-05-24-13-48-34.bpo-28806.PkNw5D.rst new file mode 100644 index 0000000..7783845 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-05-24-13-48-34.bpo-28806.PkNw5D.rst @@ -0,0 +1 @@ +Improve netrc library. netrc file no longer needs to contain all tokens. And if the login name is anonymous, security check is no longer need. \ No newline at end of file -- cgit v0.12