bpo-31672: Fix string.Template accidentally matched non-ASCII identifiers (GH-3872)

Pattern `[a-z]` with `IGNORECASE` flag can match to some non-ASCII characters.

Straightforward solution for this is using `IGNORECASE | ASCII` flag.
But users may subclass `Template` and override only `idpattern`. So we want to
avoid changing `Template.flags`.

So this commit uses local flag `-i` for `idpattern` and change `[a-z]` to `[a-zA-Z]`.

2 files changed, 11 insertions, 1 deletions

diff --git a/Lib/string.py b/Lib/string.py
index b46e60c..a3e6d91 100644
--- a/Lib/string.py
+++ b/Lib/string.py
@@ -79,7 +79,11 @@ class Template(metaclass=_TemplateMetaclass):
     """A string class for supporting $-substitutions."""
 
     delimiter = '$'
-    idpattern = r'[_a-z][_a-z0-9]*'
+    # r'[a-z]' matches to non-ASCII letters when used with IGNORECASE,
+    # but without ASCII flag.  We can't add re.ASCII to flags because of
+    # backward compatibility.  So we use local -i flag and [a-zA-Z] pattern.
+    # See https://bugs.python.org/issue31672
+    idpattern = r'(?-i:[_a-zA-Z][_a-zA-Z0-9]*)'
     braceidpattern = None
     flags = _re.IGNORECASE
 
diff --git a/Lib/test/test_string.py b/Lib/test/test_string.py
index 6e241ac..3480459 100644
--- a/Lib/test/test_string.py
+++ b/Lib/test/test_string.py
@@ -270,6 +270,12 @@ class TestTemplate(unittest.TestCase):
         raises(ValueError, s.substitute, dict(who='tim'))
         s = Template('$who likes $100')
         raises(ValueError, s.substitute, dict(who='tim'))
+        # Template.idpattern should match to only ASCII characters.
+        # https://bugs.python.org/issue31672
+        s = Template("$who likes $\u0131")  # (DOTLESS I)
+        raises(ValueError, s.substitute, dict(who='tim'))
+        s = Template("$who likes $\u0130")  # (LATIN CAPITAL LETTER I WITH DOT ABOVE)
+        raises(ValueError, s.substitute, dict(who='tim'))
 
     def test_idpattern_override(self):
         class PathPattern(Template):