Many updates to PEP 292 templates. Summary:

- Template no longer inherits from unicode.

- SafeTemplate is removed.  Now Templates have both a substitute() and a
  safe_substitute() method, so we don't need separate classes.  No more
  __mod__() operator.

- Adopt Tim Peter's idea for giving Template a metaclass, which makes the
  delimiter, the identifier pattern, or the entire pattern easy to override
  and document, while retaining efficiency of class-time compilation of the
  regexp.

- More informative ValueError messages which will help a user narrow down the
  bogus delimiter to the line and column in the original string (helpful for
  long triple quoted strings).

2 files changed, 136 insertions, 63 deletions

diff --git a/Lib/string.py b/Lib/string.py
index 9965111..fd9cc99 100644
--- a/Lib/string.py
+++ b/Lib/string.py
@@ -82,60 +82,83 @@ def maketrans(fromstr, tostr):
 ####################################################################
 import re as _re
 
-class Template(unicode):
+class _TemplateMetaclass(type):
+    pattern = r"""
+    (?P<escaped>%(delim)s{2})       |   # Escape sequence of two delimiters
+    %(delim)s(?P<named>%(id)s)      |   # delimiter and a Python identifier
+    %(delim)s{(?P<braced>%(id)s)}   |   # delimiter and a braced identifier
+    (?P<bogus>%(delim)s)                # Other ill-formed delimiter exprs
+    """
+
+    def __init__(cls, name, bases, dct):
+        super(_TemplateMetaclass, cls).__init__(name, bases, dct)
+        if 'pattern' in dct:
+            pattern = cls.pattern
+        else:
+            pattern = _TemplateMetaclass.pattern % {
+                'delim' : cls.delimiter,
+                'id'    : cls.idpattern,
+                }
+        cls.pattern = _re.compile(pattern, _re.IGNORECASE | _re.VERBOSE)
+
+
+class Template:
     """A string class for supporting $-substitutions."""
-    __slots__ = []
+    __metaclass__ = _TemplateMetaclass
+
+    delimiter = r'\$'
+    idpattern = r'[_a-z][_a-z0-9]*'
+
+    def __init__(self, template):
+        self.template = template
 
     # Search for $$, $identifier, ${identifier}, and any bare $'s
-    pattern = _re.compile(r"""
-      (?P<escaped>\${2})|                # Escape sequence of two $ signs
-      \$(?P<named>[_a-z][_a-z0-9]*)|     # $ and a Python identifier
-      \${(?P<braced>[_a-z][_a-z0-9]*)}|  # $ and a brace delimited identifier
-      (?P<bogus>\$)                      # Other ill-formed $ expressions
-    """, _re.IGNORECASE | _re.VERBOSE)
-
-    def __mod__(self, mapping):
+
+    def _bogus(self, mo):
+        i = mo.start('bogus')
+        lines = self.template[:i].splitlines(True)
+        if not lines:
+            colno = 1
+            lineno = 1
+        else:
+            colno = i - len(''.join(lines[:-1]))
+            lineno = len(lines)
+        raise ValueError('Invalid placeholder in string: line %d, col %d' %
+                         (lineno, colno))
+
+    def substitute(self, mapping):
         def convert(mo):
             if mo.group('escaped') is not None:
                 return '$'
             if mo.group('bogus') is not None:
-                raise ValueError('Invalid placeholder at index %d' %
-                                 mo.start('bogus'))
+                self._bogus(mo)
             val = mapping[mo.group('named') or mo.group('braced')]
-            return unicode(val)
-        return self.pattern.sub(convert, self)
-
-
-class SafeTemplate(Template):
-    """A string class for supporting $-substitutions.
-
-    This class is 'safe' in the sense that you will never get KeyErrors if
-    there are placeholders missing from the interpolation dictionary.  In that
-    case, you will get the original placeholder in the value string.
-    """
-    __slots__ = []
+            # We use this idiom instead of str() because the latter will fail
+            # if val is a Unicode containing non-ASCII characters.
+            return '%s' % val
+        return self.pattern.sub(convert, self.template)
 
-    def __mod__(self, mapping):
+    def safe_substitute(self, mapping):
         def convert(mo):
             if mo.group('escaped') is not None:
                 return '$'
             if mo.group('bogus') is not None:
-                raise ValueError('Invalid placeholder at index %d' %
-                                 mo.start('bogus'))
+                self._bogus(mo)
             named = mo.group('named')
             if named is not None:
                 try:
-                    return unicode(mapping[named])
+                    # We use this idiom instead of str() because the latter
+                    # will fail if val is a Unicode containing non-ASCII
+                    return '%s' % mapping[named]
                 except KeyError:
                     return '$' + named
             braced = mo.group('braced')
             try:
-                return unicode(mapping[braced])
+                return '%s' % mapping[braced]
             except KeyError:
                 return '${' + braced + '}'
-        return self.pattern.sub(convert, self)
+        return self.pattern.sub(convert, self.template)
 
-del _re
 
 
 ####################################################################
diff --git a/Lib/test/test_pep292.py b/Lib/test/test_pep292.py
index 7eff309..56eb417 100644
--- a/Lib/test/test_pep292.py
+++ b/Lib/test/test_pep292.py
@@ -3,70 +3,120 @@
 # License: http://www.opensource.org/licenses/PythonSoftFoundation.php
 
 import unittest
-from string import Template, SafeTemplate
+from string import Template
+
+
+class Bag:
+    pass
+
+class Mapping:
+    def __getitem__(self, name):
+        obj = self
+        for part in name.split('.'):
+            try:
+                obj = getattr(obj, part)
+            except AttributeError:
+                raise KeyError(name)
+        return obj
 
-class TestTemplate(unittest.TestCase):
 
+class TestTemplate(unittest.TestCase):
     def test_regular_templates(self):
         s = Template('$who likes to eat a bag of $what worth $$100')
-        self.assertEqual(s % dict(who='tim', what='ham'),
+        self.assertEqual(s.substitute(dict(who='tim', what='ham')),
                          'tim likes to eat a bag of ham worth $100')
-        self.assertRaises(KeyError, lambda s, d: s % d, s, dict(who='tim'))
+        self.assertRaises(KeyError, s.substitute, dict(who='tim'))
 
     def test_regular_templates_with_braces(self):
         s = Template('$who likes ${what} for ${meal}')
-        self.assertEqual(s % dict(who='tim', what='ham', meal='dinner'),
-                         'tim likes ham for dinner')
-        self.assertRaises(KeyError, lambda s, d: s % d,
-                          s, dict(who='tim', what='ham'))
+        d = dict(who='tim', what='ham', meal='dinner')
+        self.assertEqual(s.substitute(d), 'tim likes ham for dinner')
+        self.assertRaises(KeyError, s.substitute,
+                          dict(who='tim', what='ham'))
 
     def test_escapes(self):
         eq = self.assertEqual
         s = Template('$who likes to eat a bag of $$what worth $$100')
-        eq(s % dict(who='tim', what='ham'),
+        eq(s.substitute(dict(who='tim', what='ham')),
            'tim likes to eat a bag of $what worth $100')
         s = Template('$who likes $$')
-        eq(s % dict(who='tim', what='ham'), 'tim likes $')
+        eq(s.substitute(dict(who='tim', what='ham')), 'tim likes $')
 
     def test_percents(self):
+        eq = self.assertEqual
         s = Template('%(foo)s $foo ${foo}')
-        self.assertEqual(s % dict(foo='baz'), '%(foo)s baz baz')
-        s = SafeTemplate('%(foo)s $foo ${foo}')
-        self.assertEqual(s % dict(foo='baz'), '%(foo)s baz baz')
+        d = dict(foo='baz')
+        eq(s.substitute(d), '%(foo)s baz baz')
+        eq(s.safe_substitute(d), '%(foo)s baz baz')
 
     def test_stringification(self):
+        eq = self.assertEqual
         s = Template('tim has eaten $count bags of ham today')
-        self.assertEqual(s % dict(count=7),
-                         'tim has eaten 7 bags of ham today')
-        s = SafeTemplate('tim has eaten $count bags of ham today')
-        self.assertEqual(s % dict(count=7),
-                         'tim has eaten 7 bags of ham today')
-        s = SafeTemplate('tim has eaten ${count} bags of ham today')
-        self.assertEqual(s % dict(count=7),
-                         'tim has eaten 7 bags of ham today')
+        d = dict(count=7)
+        eq(s.substitute(d), 'tim has eaten 7 bags of ham today')
+        eq(s.safe_substitute(d), 'tim has eaten 7 bags of ham today')
+        s = Template('tim has eaten ${count} bags of ham today')
+        eq(s.substitute(d), 'tim has eaten 7 bags of ham today')
 
     def test_SafeTemplate(self):
         eq = self.assertEqual
-        s = SafeTemplate('$who likes ${what} for ${meal}')
-        eq(s % dict(who='tim'),
-           'tim likes ${what} for ${meal}')
-        eq(s % dict(what='ham'),
-           '$who likes ham for ${meal}')
-        eq(s % dict(what='ham', meal='dinner'),
+        s = Template('$who likes ${what} for ${meal}')
+        eq(s.safe_substitute(dict(who='tim')), 'tim likes ${what} for ${meal}')
+        eq(s.safe_substitute(dict(what='ham')), '$who likes ham for ${meal}')
+        eq(s.safe_substitute(dict(what='ham', meal='dinner')),
            '$who likes ham for dinner')
-        eq(s % dict(who='tim', what='ham'),
+        eq(s.safe_substitute(dict(who='tim', what='ham')),
            'tim likes ham for ${meal}')
-        eq(s % dict(who='tim', what='ham', meal='dinner'),
+        eq(s.safe_substitute(dict(who='tim', what='ham', meal='dinner')),
            'tim likes ham for dinner')
 
     def test_invalid_placeholders(self):
         raises = self.assertRaises
         s = Template('$who likes $')
-        raises(ValueError, lambda s, d: s % d, s, dict(who='tim'))
+        raises(ValueError, s.substitute, dict(who='tim'))
         s = Template('$who likes ${what)')
-        raises(ValueError, lambda s, d: s % d, s, dict(who='tim'))
+        raises(ValueError, s.substitute, dict(who='tim'))
         s = Template('$who likes $100')
-        raises(ValueError, lambda s, d: s % d, s, dict(who='tim'))
+        raises(ValueError, s.substitute, dict(who='tim'))
+
+    def test_delimiter_override(self):
+        class PieDelims(Template):
+            delimiter = '@'
+        s = PieDelims('@who likes to eat a bag of @{what} worth $100')
+        self.assertEqual(s.substitute(dict(who='tim', what='ham')),
+                         'tim likes to eat a bag of ham worth $100')
+
+    def test_idpattern_override(self):
+        class PathPattern(Template):
+            idpattern = r'[_a-z][._a-z0-9]*'
+        m = Mapping()
+        m.bag = Bag()
+        m.bag.foo = Bag()
+        m.bag.foo.who = 'tim'
+        m.bag.what = 'ham'
+        s = PathPattern('$bag.foo.who likes to eat a bag of $bag.what')
+        self.assertEqual(s.substitute(m), 'tim likes to eat a bag of ham')
+
+    def test_pattern_override(self):
+        class MyPattern(Template):
+            pattern = r"""
+            (?P<escaped>@{2})                   |
+            @(?P<named>[_a-z][._a-z0-9]*)       |
+            @{(?P<braced>[_a-z][._a-z0-9]*)}    |
+            (?P<bogus>@)
+            """
+        m = Mapping()
+        m.bag = Bag()
+        m.bag.foo = Bag()
+        m.bag.foo.who = 'tim'
+        m.bag.what = 'ham'
+        s = MyPattern('@bag.foo.who likes to eat a bag of @bag.what')
+        self.assertEqual(s.substitute(m), 'tim likes to eat a bag of ham')
+
+    def test_unicode_values(self):
+        s = Template('$who likes $what')
+        d = dict(who=u't\xffm', what=u'f\xfe\fed')
+        self.assertEqual(s.substitute(d), u't\xffm likes f\xfe\x0ced')
 
 
 def suite():