#14731: refactor email policy framework.

This patch primarily does two things: (1) it adds some internal-interface
methods to Policy that allow for Policy to control the parsing and folding of
headers in such a way that we can construct a backward compatibility policy
that is 100% compatible with the 3.2 API, while allowing a new policy to
implement the email6 API.  (2) it adds that backward compatibility policy and
refactors the test suite so that the only differences between the 3.2
test_email.py file and the 3.3 test_email.py file is some small changes in
test framework and the addition of tests for bugs fixed that apply to the 3.2
API.

There are some additional teaks, such as moving just the code needed for the
compatibility policy into _policybase, so that the library code can import
only _policybase.  That way the new code that will be added for email6
will only get imported when a non-compatibility policy is imported.

1 files changed, 75 insertions, 10 deletions

diff --git a/Lib/test/test_email/test_generator.py b/Lib/test/test_email/test_generator.py
index 35ca6c5..8f5fde7 100644
--- a/Lib/test/test_email/test_generator.py
+++ b/Lib/test/test_email/test_generator.py
@@ -11,6 +11,8 @@ from test.test_email import TestEmailBase
 
 class TestGeneratorBase():
 
+    policy = policy.compat32
+
     long_subject = {
         0: textwrap.dedent("""\
             To: whom_it_may_concern@example.com
@@ -58,11 +60,11 @@ class TestGeneratorBase():
     long_subject[100] = long_subject[0]
 
     def maxheaderlen_parameter_test(self, n):
-        msg = self.msgmaker(self.long_subject[0])
+        msg = self.msgmaker(self.typ(self.long_subject[0]))
         s = self.ioclass()
         g = self.genclass(s, maxheaderlen=n)
         g.flatten(msg)
-        self.assertEqual(s.getvalue(), self.long_subject[n])
+        self.assertEqual(s.getvalue(), self.typ(self.long_subject[n]))
 
     def test_maxheaderlen_parameter_0(self):
         self.maxheaderlen_parameter_test(0)
@@ -77,11 +79,11 @@ class TestGeneratorBase():
         self.maxheaderlen_parameter_test(20)
 
     def maxheaderlen_policy_test(self, n):
-        msg = self.msgmaker(self.long_subject[0])
+        msg = self.msgmaker(self.typ(self.long_subject[0]))
         s = self.ioclass()
         g = self.genclass(s, policy=policy.default.clone(max_line_length=n))
         g.flatten(msg)
-        self.assertEqual(s.getvalue(), self.long_subject[n])
+        self.assertEqual(s.getvalue(), self.typ(self.long_subject[n]))
 
     def test_maxheaderlen_policy_0(self):
         self.maxheaderlen_policy_test(0)
@@ -96,12 +98,12 @@ class TestGeneratorBase():
         self.maxheaderlen_policy_test(20)
 
     def maxheaderlen_parm_overrides_policy_test(self, n):
-        msg = self.msgmaker(self.long_subject[0])
+        msg = self.msgmaker(self.typ(self.long_subject[0]))
         s = self.ioclass()
         g = self.genclass(s, maxheaderlen=n,
                           policy=policy.default.clone(max_line_length=10))
         g.flatten(msg)
-        self.assertEqual(s.getvalue(), self.long_subject[n])
+        self.assertEqual(s.getvalue(), self.typ(self.long_subject[n]))
 
     def test_maxheaderlen_parm_overrides_policy_0(self):
         self.maxheaderlen_parm_overrides_policy_test(0)
@@ -115,21 +117,84 @@ class TestGeneratorBase():
     def test_maxheaderlen_parm_overrides_policy_20(self):
         self.maxheaderlen_parm_overrides_policy_test(20)
 
+    def test_crlf_control_via_policy(self):
+        source = "Subject: test\r\n\r\ntest body\r\n"
+        expected = source
+        msg = self.msgmaker(self.typ(source))
+        s = self.ioclass()
+        g = self.genclass(s, policy=policy.SMTP)
+        g.flatten(msg)
+        self.assertEqual(s.getvalue(), self.typ(expected))
+
+    def test_flatten_linesep_overrides_policy(self):
+        source = "Subject: test\n\ntest body\n"
+        expected = source
+        msg = self.msgmaker(self.typ(source))
+        s = self.ioclass()
+        g = self.genclass(s, policy=policy.SMTP)
+        g.flatten(msg, linesep='\n')
+        self.assertEqual(s.getvalue(), self.typ(expected))
+
 
 class TestGenerator(TestGeneratorBase, TestEmailBase):
 
-    msgmaker = staticmethod(message_from_string)
     genclass = Generator
     ioclass = io.StringIO
+    typ = str
+
+    def msgmaker(self, msg, policy=None):
+        policy = self.policy if policy is None else policy
+        return message_from_string(msg, policy=policy)
 
 
 class TestBytesGenerator(TestGeneratorBase, TestEmailBase):
 
-    msgmaker = staticmethod(message_from_bytes)
     genclass = BytesGenerator
     ioclass = io.BytesIO
-    long_subject = {key: x.encode('ascii')
-        for key, x in TestGeneratorBase.long_subject.items()}
+    typ = lambda self, x: x.encode('ascii')
+
+    def msgmaker(self, msg, policy=None):
+        policy = self.policy if policy is None else policy
+        return message_from_bytes(msg, policy=policy)
+
+    def test_cte_type_7bit_handles_unknown_8bit(self):
+        source = ("Subject: Maintenant je vous présente mon "
+                 "collègue\n\n").encode('utf-8')
+        expected = ('Subject: =?unknown-8bit?q?Maintenant_je_vous_pr=C3=A9sente_mon_'
+                    'coll=C3=A8gue?=\n\n').encode('ascii')
+        msg = message_from_bytes(source)
+        s = io.BytesIO()
+        g = BytesGenerator(s, policy=self.policy.clone(cte_type='7bit'))
+        g.flatten(msg)
+        self.assertEqual(s.getvalue(), expected)
+
+    def test_cte_type_7bit_transforms_8bit_cte(self):
+        source = textwrap.dedent("""\
+            From: foo@bar.com
+            To: Dinsdale
+            Subject: Nudge nudge, wink, wink
+            Mime-Version: 1.0
+            Content-Type: text/plain; charset="latin-1"
+            Content-Transfer-Encoding: 8bit
+
+            oh là là, know what I mean, know what I mean?
+            """).encode('latin1')
+        msg = message_from_bytes(source)
+        expected =  textwrap.dedent("""\
+            From: foo@bar.com
+            To: Dinsdale
+            Subject: Nudge nudge, wink, wink
+            Mime-Version: 1.0
+            Content-Type: text/plain; charset="iso-8859-1"
+            Content-Transfer-Encoding: quoted-printable
+
+            oh l=E0 l=E0, know what I mean, know what I mean?
+            """).encode('ascii')
+        s = io.BytesIO()
+        g = BytesGenerator(s, policy=self.policy.clone(cte_type='7bit',
+                                                       linesep='\n'))
+        g.flatten(msg)
+        self.assertEqual(s.getvalue(), expected)
 
 
 if __name__ == '__main__':