From 19dca041212f9f58ee11833bff3f8c157d4fd3e8 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 30 Apr 2022 13:13:46 +0300 Subject: gh-91760: Deprecate group names and numbers which will be invalid in future (GH-91794) Only sequence of ASCII digits will be accepted as a numerical reference. The group name in bytes patterns and replacement strings could only contain ASCII letters and digits and underscore. --- Doc/library/re.rst | 10 ++++ Doc/whatsnew/3.11.rst | 8 ++++ Lib/re/_parser.py | 41 +++++++++++++--- Lib/test/test_re.py | 56 ++++++++++++++++++++++ .../2022-04-21-19-46-03.gh-issue-91760.zDtv1E.rst | 4 ++ 5 files changed, 112 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-04-21-19-46-03.gh-issue-91760.zDtv1E.rst diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 89de928..3cd9f25 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -417,6 +417,9 @@ The special characters are: | | * ``\1`` | +---------------------------------------+----------------------------------+ + .. deprecated:: 3.11 + Group names containing non-ASCII characters in bytes patterns. + .. index:: single: (?P=; in regular expressions ``(?P=name)`` @@ -486,6 +489,9 @@ The special characters are: will match with ``''`` as well as ``'user@host.com'``, but not with ``''``. + .. deprecated:: 3.11 + Group *id* containing anything except ASCII digits. + The special sequences consist of ``'\'`` and a character from the list below. If the ordinary character is not an ASCII digit or an ASCII letter, then the @@ -995,6 +1001,10 @@ form. Empty matches for the pattern are replaced when adjacent to a previous non-empty match. + .. deprecated:: 3.11 + Group *id* containing anything except ASCII digits. + Group names containing non-ASCII characters in bytes replacement strings. + .. function:: subn(pattern, repl, string, count=0, flags=0) diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index fdd35ce..1a692f2 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -1151,6 +1151,14 @@ Deprecated (Contributed by Brett Cannon in :issue:`47061` and Victor Stinner in :gh:`68966`.) +* More strict rules will be applied now applied for numerical group references + and group names in regular expressions in future Python versions. + Only sequence of ASCII digits will be now accepted as a numerical reference. + The group name in bytes patterns and replacement strings could only + contain ASCII letters and digits and underscore. + For now, a deprecation warning is raised for such syntax. + (Contributed by Serhiy Storchaka in :gh:`91760`.) + Removed ======= diff --git a/Lib/re/_parser.py b/Lib/re/_parser.py index 933d515..a393c50 100644 --- a/Lib/re/_parser.py +++ b/Lib/re/_parser.py @@ -287,8 +287,22 @@ class Tokenizer: self.__next() def error(self, msg, offset=0): + if not self.istext: + msg = msg.encode('ascii', 'backslashreplace').decode('ascii') return error(msg, self.string, self.tell() - offset) + def checkgroupname(self, name, offset, nested): + if not name.isidentifier(): + msg = "bad character in group name %r" % name + raise self.error(msg, len(name) + offset) + if not (self.istext or name.isascii()): + import warnings + warnings.warn( + "bad character in group name %a at position %d" % + (name, self.tell() - len(name) - offset), + DeprecationWarning, stacklevel=nested + 7 + ) + def _class_escape(source, escape): # handle escape code inside character class code = ESCAPES.get(escape) @@ -703,15 +717,11 @@ def _parse(source, state, verbose, nested, first=False): if sourcematch("<"): # named group: skip forward to end of name name = source.getuntil(">", "group name") - if not name.isidentifier(): - msg = "bad character in group name %r" % name - raise source.error(msg, len(name) + 1) + source.checkgroupname(name, 1, nested) elif sourcematch("="): # named backreference name = source.getuntil(")", "group name") - if not name.isidentifier(): - msg = "bad character in group name %r" % name - raise source.error(msg, len(name) + 1) + source.checkgroupname(name, 1, nested) gid = state.groupdict.get(name) if gid is None: msg = "unknown group name %r" % name @@ -773,6 +783,7 @@ def _parse(source, state, verbose, nested, first=False): # conditional backreference group condname = source.getuntil(")", "group name") if condname.isidentifier(): + source.checkgroupname(condname, 1, nested) condgroup = state.groupdict.get(condname) if condgroup is None: msg = "unknown group name %r" % condname @@ -795,6 +806,14 @@ def _parse(source, state, verbose, nested, first=False): state.grouprefpos[condgroup] = ( source.tell() - len(condname) - 1 ) + if not (condname.isdecimal() and condname.isascii()): + import warnings + warnings.warn( + "bad character in group name %s at position %d" % + (repr(condname) if source.istext else ascii(condname), + source.tell() - len(condname) - 1), + DeprecationWarning, stacklevel=nested + 6 + ) state.checklookbehindgroup(condgroup, source) item_yes = _parse(source, state, verbose, nested + 1) if source.match("|"): @@ -1000,11 +1019,11 @@ def parse_template(source, state): # group c = this[1] if c == "g": - name = "" if not s.match("<"): raise s.error("missing <") name = s.getuntil(">", "group name") if name.isidentifier(): + s.checkgroupname(name, 1, -1) try: index = groupindex[name] except KeyError: @@ -1020,6 +1039,14 @@ def parse_template(source, state): if index >= MAXGROUPS: raise s.error("invalid group reference %d" % index, len(name) + 1) + if not (name.isdecimal() and name.isascii()): + import warnings + warnings.warn( + "bad character in group name %s at position %d" % + (repr(name) if s.istext else ascii(name), + s.tell() - len(name) - 1), + DeprecationWarning, stacklevel=5 + ) addgroup(index, len(name) + 1) elif c == "0": if s.next in OCTDIGITS: diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index a4c2f1f3..c101475 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -135,6 +135,7 @@ class ReTests(unittest.TestCase): self.assertEqual(re.sub('(?Px)', r'\g\g<1>', 'xx'), 'xxxx') self.assertEqual(re.sub('(?Px)', r'\g\g', 'xx'), 'xxxx') self.assertEqual(re.sub('(?Px)', r'\g<1>\g<1>', 'xx'), 'xxxx') + self.assertEqual(re.sub('()x', r'\g<0>\g<0>', 'xx'), 'xxxx') self.assertEqual(re.sub('a', r'\t\n\v\r\f\a\b', 'a'), '\t\n\v\r\f\a\b') self.assertEqual(re.sub('a', '\t\n\v\r\f\a\b', 'a'), '\t\n\v\r\f\a\b') @@ -274,6 +275,21 @@ class ReTests(unittest.TestCase): self.checkPatternError('(?P<©>x)', "bad character in group name '©'", 4) self.checkPatternError('(?P=©)', "bad character in group name '©'", 4) self.checkPatternError('(?(©)y)', "bad character in group name '©'", 3) + with self.assertWarnsRegex(DeprecationWarning, + r"bad character in group name '\\xc2\\xb5' " + r"at position 4") as w: + re.compile(b'(?P<\xc2\xb5>x)') + self.assertEqual(w.filename, __file__) + with self.assertWarnsRegex(DeprecationWarning, + r"bad character in group name '\\xc2\\xb5' " + r"at position 4"): + self.checkPatternError(b'(?P=\xc2\xb5)', + r"unknown group name '\xc2\xb5'", 4) + with self.assertWarnsRegex(DeprecationWarning, + r"bad character in group name '\\xc2\\xb5' " + r"at position 3"): + self.checkPatternError(b'(?(\xc2\xb5)y)', + r"unknown group name '\xc2\xb5'", 3) def test_symbolic_refs(self): self.assertEqual(re.sub('(?Px)|(?Py)', r'\g', 'xx'), '') @@ -306,12 +322,35 @@ class ReTests(unittest.TestCase): re.sub('(?Px)', r'\g', 'xx') self.checkTemplateError('(?Px)', r'\g<-1>', 'xx', "bad character in group name '-1'", 3) + with self.assertWarnsRegex(DeprecationWarning, + r"bad character in group name '\+1' " + r"at position 3") as w: + re.sub('(?Px)', r'\g<+1>', 'xx') + self.assertEqual(w.filename, __file__) + with self.assertWarnsRegex(DeprecationWarning, + r"bad character in group name '1_0' " + r"at position 3"): + re.sub('()'*10, r'\g<1_0>', 'xx') + with self.assertWarnsRegex(DeprecationWarning, + r"bad character in group name ' 1 ' " + r"at position 3"): + re.sub('(?Px)', r'\g< 1 >', 'xx') self.checkTemplateError('(?Px)', r'\g<©>', 'xx', "bad character in group name '©'", 3) + with self.assertWarnsRegex(DeprecationWarning, + r"bad character in group name '\\xc2\\xb5' " + r"at position 3") as w: + with self.assertRaisesRegex(IndexError, "unknown group name '\xc2\xb5'"): + re.sub(b'(?Px)', b'\\g<\xc2\xb5>', b'xx') + self.assertEqual(w.filename, __file__) self.checkTemplateError('(?Px)', r'\g<㊀>', 'xx', "bad character in group name '㊀'", 3) self.checkTemplateError('(?Px)', r'\g<¹>', 'xx', "bad character in group name '¹'", 3) + with self.assertWarnsRegex(DeprecationWarning, + r"bad character in group name '१' " + r"at position 3"): + re.sub('(?Px)', r'\g<१>', 'xx') def test_re_subn(self): self.assertEqual(re.subn("(?i)b+", "x", "bbbb BBBB"), ('x x', 2)) @@ -577,10 +616,27 @@ class ReTests(unittest.TestCase): self.checkPatternError(r'(?P)(?(0)a|b)', 'bad group number', 10) self.checkPatternError(r'()(?(-1)a|b)', "bad character in group name '-1'", 5) + with self.assertWarnsRegex(DeprecationWarning, + r"bad character in group name '\+1' " + r"at position 5") as w: + re.compile(r'()(?(+1)a|b)') + self.assertEqual(w.filename, __file__) + with self.assertWarnsRegex(DeprecationWarning, + r"bad character in group name '1_0' " + r"at position 23"): + re.compile(r'()'*10 + r'(?(1_0)a|b)') + with self.assertWarnsRegex(DeprecationWarning, + r"bad character in group name ' 1 ' " + r"at position 5"): + re.compile(r'()(?( 1 )a|b)') self.checkPatternError(r'()(?(㊀)a|b)', "bad character in group name '㊀'", 5) self.checkPatternError(r'()(?(¹)a|b)', "bad character in group name '¹'", 5) + with self.assertWarnsRegex(DeprecationWarning, + r"bad character in group name '१' " + r"at position 5"): + re.compile(r'()(?(१)a|b)') self.checkPatternError(r'()(?(1', "missing ), unterminated name", 5) self.checkPatternError(r'()(?(1)a', diff --git a/Misc/NEWS.d/next/Library/2022-04-21-19-46-03.gh-issue-91760.zDtv1E.rst b/Misc/NEWS.d/next/Library/2022-04-21-19-46-03.gh-issue-91760.zDtv1E.rst new file mode 100644 index 0000000..0bddbbe --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-04-21-19-46-03.gh-issue-91760.zDtv1E.rst @@ -0,0 +1,4 @@ +More strict rules will be applied for numerical group references and group +names in regular expressions. For now, a deprecation warning is emitted for +group references and group names which will be errors in future Python +versions. -- cgit v0.12