A fix for issue 1974, inspired by the patch from Andi Albrecht (aalbrecht),

though with some changes by me.  This patch should not be back ported or
forward ported.  It's a bit too risky for 2.6 and 3.x does things fairly
differently.

3 files changed, 81 insertions, 61 deletions

diff --git a/Lib/email/generator.py b/Lib/email/generator.py
index ed832a3..594a257 100644
--- a/Lib/email/generator.py
+++ b/Lib/email/generator.py
@@ -1,5 +1,4 @@
-# Copyright (C) 2001-2006 Python Software Foundation
-# Author: Barry Warsaw
+# Copyright (C) 2001-2009 Python Software Foundation
 # Contact: email-sig@python.org
 
 """Classes to generate plain text from a message object tree."""
@@ -156,10 +155,13 @@ class Generator:
                 # be to not split the string and risk it being too long.
                 print >> self._fp, v
             else:
-                # Header's got lots of smarts, so use it.
+                # Header's got lots of smarts, so use it.  Note that this is
+                # fundamentally broken though because we lose idempotency when
+                # the header string is continued with tabs.  It will now be
+                # continued with spaces.  This was reversedly broken before we
+                # fixed bug 1974.  Either way, we lose.
                 print >> self._fp, Header(
-                    v, maxlinelen=self._maxheaderlen,
-                    header_name=h, continuation_ws='\t').encode()
+                    v, maxlinelen=self._maxheaderlen, header_name=h).encode()
         # A blank line always separates headers from body
         print >> self._fp
 
diff --git a/Lib/email/test/test_email.py b/Lib/email/test/test_email.py
index 3a68f02..f9cc6f0 100644
--- a/Lib/email/test/test_email.py
+++ b/Lib/email/test/test_email.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2001-2007 Python Software Foundation
+# Copyright (C) 2001-2009 Python Software Foundation
 # Contact: email-sig@python.org
 # email package unit tests
 
@@ -251,7 +251,16 @@ class TestMessageAPI(TestEmailBase):
         msg = self._msgobj('msg_01.txt')
         fp = openfile('msg_01.txt')
         try:
-            text = fp.read()
+            # BAW 30-Mar-2009 Evil be here.  So, the generator is broken with
+            # respect to long line breaking.  It's also not idempotent when a
+            # header from a parsed message is continued with tabs rather than
+            # spaces.  Before we fixed bug 1974 it was reversedly broken,
+            # i.e. headers that were continued with spaces got continued with
+            # tabs.  For Python 2.x there's really no good fix and in Python
+            # 3.x all this stuff is re-written to be right(er).  Chris Withers
+            # convinced me that using space as the default continuation
+            # character is less bad for more applications.
+            text = fp.read().replace('\t', ' ')
         finally:
             fp.close()
         eq(text, msg.as_string())
@@ -554,8 +563,8 @@ test
         g.flatten(msg)
         eq(sfp.getvalue(), """\
 Subject: bug demonstration
-\t12345678911234567892123456789312345678941234567895123456789612345678971234567898112345678911234567892123456789112345678911234567892123456789
-\tmore text
+ 12345678911234567892123456789312345678941234567895123456789612345678971234567898112345678911234567892123456789112345678911234567892123456789
+ more text
 
 test
 """)
@@ -655,7 +664,7 @@ Content-Type: text/plain; charset="us-ascii"
 MIME-Version: 1.0
 Content-Transfer-Encoding: 7bit
 X-Foobar-Spoink-Defrobnit: wasnipoop; giraffes="very-long-necked-animals";
-\tspooge="yummy"; hippos="gargantuan"; marshmallows="gooey"
+ spooge="yummy"; hippos="gargantuan"; marshmallows="gooey"
 
 ''')
 
@@ -671,7 +680,7 @@ X-Foobar-Spoink-Defrobnit: wasnipoop; giraffes="very-long-necked-animals";
         eq(sfp.getvalue(), """\
 From: test@dom.ain
 References: <0@dom.ain> <1@dom.ain> <2@dom.ain> <3@dom.ain> <4@dom.ain>
-\t<5@dom.ain> <6@dom.ain> <7@dom.ain> <8@dom.ain> <9@dom.ain>
+ <5@dom.ain> <6@dom.ain> <7@dom.ain> <8@dom.ain> <9@dom.ain>
 
 Test""")
 
@@ -749,9 +758,9 @@ Reply-To: Britische Regierung gibt gr\xfcnes Licht f\xfcr Offshore-Windkraftproj
         msg['To'] = to
         eq(msg.as_string(0), '''\
 To: "Someone Test #A" <someone@eecs.umich.edu>, <someone@eecs.umich.edu>,
-\t"Someone Test #B" <someone@umich.edu>,
-\t"Someone Test #C" <someone@eecs.umich.edu>,
-\t"Someone Test #D" <someone@eecs.umich.edu>
+ "Someone Test #B" <someone@umich.edu>,
+ "Someone Test #C" <someone@eecs.umich.edu>,
+ "Someone Test #D" <someone@eecs.umich.edu>
 
 ''')
 
@@ -794,22 +803,22 @@ Received-1: from FOO.TLD (vizworld.acl.foo.tld [123.452.678.9]) by
 \throthgar.la.mastaler.com (tmda-ofmipd) with ESMTP;
 \tWed, 05 Mar 2003 18:10:18 -0700
 Received-2: from FOO.TLD (vizworld.acl.foo.tld [123.452.678.9]) by
-\throthgar.la.mastaler.com (tmda-ofmipd) with ESMTP;
-\tWed, 05 Mar 2003 18:10:18 -0700
+ hrothgar.la.mastaler.com (tmda-ofmipd) with ESMTP;
+ Wed, 05 Mar 2003 18:10:18 -0700
 
 """)
 
     def test_string_headerinst_eq(self):
         h = '<15975.17901.207240.414604@sgigritzmann1.mathematik.tu-muenchen.de> (David Bremner\'s message of "Thu, 6 Mar 2003 13:58:21 +0100")'
         msg = Message()
-        msg['Received-1'] = Header(h, header_name='Received-1',
-                                   continuation_ws='\t')
-        msg['Received-2'] = h
-        self.assertEqual(msg.as_string(), """\
-Received-1: <15975.17901.207240.414604@sgigritzmann1.mathematik.tu-muenchen.de>
-\t(David Bremner's message of "Thu, 6 Mar 2003 13:58:21 +0100")
-Received-2: <15975.17901.207240.414604@sgigritzmann1.mathematik.tu-muenchen.de>
+        msg['Received'] = Header(h, header_name='Received',
+                                 continuation_ws='\t')
+        msg['Received'] = h
+        self.ndiffAssertEqual(msg.as_string(), """\
+Received: <15975.17901.207240.414604@sgigritzmann1.mathematik.tu-muenchen.de>
 \t(David Bremner's message of "Thu, 6 Mar 2003 13:58:21 +0100")
+Received: <15975.17901.207240.414604@sgigritzmann1.mathematik.tu-muenchen.de>
+ (David Bremner's message of "Thu, 6 Mar 2003 13:58:21 +0100")
 
 """)
 
@@ -823,7 +832,7 @@ Received-2: <15975.17901.207240.414604@sgigritzmann1.mathematik.tu-muenchen.de>
         msg['Face-2'] = Header(t, header_name='Face-2')
         eq(msg.as_string(), """\
 Face-1: iVBORw0KGgoAAAANSUhEUgAAADAAAAAwBAMAAAClLOS0AAAAGFBMVEUAAAAkHiJeRUIcGBi9
-\tlocQDQ4zJykFBAXJfWDjAAACYUlEQVR4nF2TQY/jIAyFc6lydlG5x8Nyp1Y69wj1PN2I5gzp
+ locQDQ4zJykFBAXJfWDjAAACYUlEQVR4nF2TQY/jIAyFc6lydlG5x8Nyp1Y69wj1PN2I5gzp
 Face-2: iVBORw0KGgoAAAANSUhEUgAAADAAAAAwBAMAAAClLOS0AAAAGFBMVEUAAAAkHiJeRUIcGBi9
  locQDQ4zJykFBAXJfWDjAAACYUlEQVR4nF2TQY/jIAyFc6lydlG5x8Nyp1Y69wj1PN2I5gzp
 
@@ -833,11 +842,11 @@ Face-2: iVBORw0KGgoAAAANSUhEUgAAADAAAAAwBAMAAAClLOS0AAAAGFBMVEUAAAAkHiJeRUIcGBi9
         eq = self.ndiffAssertEqual
         m = '''\
 Received: from siimage.com ([172.25.1.3]) by zima.siliconimage.com with Microsoft SMTPSVC(5.0.2195.4905);
-\tWed, 16 Oct 2002 07:41:11 -0700'''
+ Wed, 16 Oct 2002 07:41:11 -0700'''
         msg = email.message_from_string(m)
         eq(msg.as_string(), '''\
 Received: from siimage.com ([172.25.1.3]) by zima.siliconimage.com with
-\tMicrosoft SMTPSVC(5.0.2195.4905); Wed, 16 Oct 2002 07:41:11 -0700
+ Microsoft SMTPSVC(5.0.2195.4905); Wed, 16 Oct 2002 07:41:11 -0700
 
 ''')
 
@@ -851,7 +860,7 @@ List-Unsubscribe: <https://lists.sourceforge.net/lists/listinfo/spamassassin-tal
         msg['List'] = Header(h, header_name='List')
         eq(msg.as_string(), """\
 List: List-Unsubscribe: <https://lists.sourceforge.net/lists/listinfo/spamassassin-talk>,
-\t<mailto:spamassassin-talk-request@lists.sourceforge.net?subject=unsubscribe>
+ <mailto:spamassassin-talk-request@lists.sourceforge.net?subject=unsubscribe>
 List: List-Unsubscribe: <https://lists.sourceforge.net/lists/listinfo/spamassassin-talk>,
  <mailto:spamassassin-talk-request@lists.sourceforge.net?subject=unsubscribe>
 
@@ -2979,11 +2988,11 @@ class TestRFC2231(TestEmailBase):
         msg = self._msgobj('msg_01.txt')
         msg.set_param('title', 'This is even more ***fun*** isn\'t it!',
                       charset='us-ascii', language='en')
-        eq(msg.as_string(), """\
+        self.ndiffAssertEqual(msg.as_string(), """\
 Return-Path: <bbb@zzz.org>
 Delivered-To: bbb@zzz.org
 Received: by mail.zzz.org (Postfix, from userid 889)
-\tid 27CEAD38CC; Fri,  4 May 2001 14:05:44 -0400 (EDT)
+ id 27CEAD38CC; Fri,  4 May 2001 14:05:44 -0400 (EDT)
 MIME-Version: 1.0
 Content-Transfer-Encoding: 7bit
 Message-ID: <15090.61304.110929.45684@aaa.zzz.org>
@@ -2992,7 +3001,7 @@ To: bbb@zzz.org
 Subject: This is a test message
 Date: Fri, 4 May 2001 14:05:44 -0400
 Content-Type: text/plain; charset=us-ascii;
-\ttitle*="us-ascii'en'This%20is%20even%20more%20%2A%2A%2Afun%2A%2A%2A%20isn%27t%20it%21"
+ title*="us-ascii'en'This%20is%20even%20more%20%2A%2A%2Afun%2A%2A%2A%20isn%27t%20it%21"
 
 
 Hi,
@@ -3013,7 +3022,7 @@ Do you like this message?
 Return-Path: <bbb@zzz.org>
 Delivered-To: bbb@zzz.org
 Received: by mail.zzz.org (Postfix, from userid 889)
-\tid 27CEAD38CC; Fri,  4 May 2001 14:05:44 -0400 (EDT)
+ id 27CEAD38CC; Fri,  4 May 2001 14:05:44 -0400 (EDT)
 MIME-Version: 1.0
 Content-Transfer-Encoding: 7bit
 Message-ID: <15090.61304.110929.45684@aaa.zzz.org>
@@ -3022,7 +3031,7 @@ To: bbb@zzz.org
 Subject: This is a test message
 Date: Fri, 4 May 2001 14:05:44 -0400
 Content-Type: text/plain; charset="us-ascii";
-\ttitle*="us-ascii'en'This%20is%20even%20more%20%2A%2A%2Afun%2A%2A%2A%20isn%27t%20it%21"
+ title*="us-ascii'en'This%20is%20even%20more%20%2A%2A%2Afun%2A%2A%2A%20isn%27t%20it%21"
 
 
 Hi,
diff --git a/Lib/email/test/test_email_renamed.py b/Lib/email/test/test_email_renamed.py
index 37ce7af..61fe26e 100644
--- a/Lib/email/test/test_email_renamed.py
+++ b/Lib/email/test/test_email_renamed.py
@@ -239,10 +239,19 @@ class TestMessageAPI(TestEmailBase):
         msg = self._msgobj('msg_01.txt')
         fp = openfile('msg_01.txt')
         try:
-            text = fp.read()
+            # BAW 30-Mar-2009 Evil be here.  So, the generator is broken with
+            # respect to long line breaking.  It's also not idempotent when a
+            # header from a parsed message is continued with tabs rather than
+            # spaces.  Before we fixed bug 1974 it was reversedly broken,
+            # i.e. headers that were continued with spaces got continued with
+            # tabs.  For Python 2.x there's really no good fix and in Python
+            # 3.x all this stuff is re-written to be right(er).  Chris Withers
+            # convinced me that using space as the default continuation
+            # character is less bad for more applications.
+            text = fp.read().replace('\t', ' ')
         finally:
             fp.close()
-        eq(text, msg.as_string())
+        self.ndiffAssertEqual(text, msg.as_string())
         fullrepr = str(msg)
         lines = fullrepr.split('\n')
         self.failUnless(lines[0].startswith('From '))
@@ -535,8 +544,8 @@ test
         g.flatten(msg)
         eq(sfp.getvalue(), """\
 Subject: bug demonstration
-\t12345678911234567892123456789312345678941234567895123456789612345678971234567898112345678911234567892123456789112345678911234567892123456789
-\tmore text
+ 12345678911234567892123456789312345678941234567895123456789612345678971234567898112345678911234567892123456789112345678911234567892123456789
+ more text
 
 test
 """)
@@ -636,7 +645,7 @@ Content-Type: text/plain; charset="us-ascii"
 MIME-Version: 1.0
 Content-Transfer-Encoding: 7bit
 X-Foobar-Spoink-Defrobnit: wasnipoop; giraffes="very-long-necked-animals";
-\tspooge="yummy"; hippos="gargantuan"; marshmallows="gooey"
+ spooge="yummy"; hippos="gargantuan"; marshmallows="gooey"
 
 ''')
 
@@ -652,7 +661,7 @@ X-Foobar-Spoink-Defrobnit: wasnipoop; giraffes="very-long-necked-animals";
         eq(sfp.getvalue(), """\
 From: test@dom.ain
 References: <0@dom.ain> <1@dom.ain> <2@dom.ain> <3@dom.ain> <4@dom.ain>
-\t<5@dom.ain> <6@dom.ain> <7@dom.ain> <8@dom.ain> <9@dom.ain>
+ <5@dom.ain> <6@dom.ain> <7@dom.ain> <8@dom.ain> <9@dom.ain>
 
 Test""")
 
@@ -730,9 +739,9 @@ Reply-To: Britische Regierung gibt gr\xfcnes Licht f\xfcr Offshore-Windkraftproj
         msg['To'] = to
         eq(msg.as_string(0), '''\
 To: "Someone Test #A" <someone@eecs.umich.edu>, <someone@eecs.umich.edu>,
-\t"Someone Test #B" <someone@umich.edu>,
-\t"Someone Test #C" <someone@eecs.umich.edu>,
-\t"Someone Test #D" <someone@eecs.umich.edu>
+ "Someone Test #B" <someone@umich.edu>,
+ "Someone Test #C" <someone@eecs.umich.edu>,
+ "Someone Test #D" <someone@eecs.umich.edu>
 
 ''')
 
@@ -770,27 +779,27 @@ This is an example of string which has almost the limit of header length.
         msg = Message()
         msg['Received-1'] = Header(h, continuation_ws='\t')
         msg['Received-2'] = h
-        self.assertEqual(msg.as_string(), """\
+        self.ndiffAssertEqual(msg.as_string(), """\
 Received-1: from FOO.TLD (vizworld.acl.foo.tld [123.452.678.9]) by
 \throthgar.la.mastaler.com (tmda-ofmipd) with ESMTP;
 \tWed, 05 Mar 2003 18:10:18 -0700
 Received-2: from FOO.TLD (vizworld.acl.foo.tld [123.452.678.9]) by
-\throthgar.la.mastaler.com (tmda-ofmipd) with ESMTP;
-\tWed, 05 Mar 2003 18:10:18 -0700
+ hrothgar.la.mastaler.com (tmda-ofmipd) with ESMTP;
+ Wed, 05 Mar 2003 18:10:18 -0700
 
 """)
 
     def test_string_headerinst_eq(self):
         h = '<15975.17901.207240.414604@sgigritzmann1.mathematik.tu-muenchen.de> (David Bremner\'s message of "Thu, 6 Mar 2003 13:58:21 +0100")'
         msg = Message()
-        msg['Received-1'] = Header(h, header_name='Received-1',
-                                   continuation_ws='\t')
-        msg['Received-2'] = h
-        self.assertEqual(msg.as_string(), """\
-Received-1: <15975.17901.207240.414604@sgigritzmann1.mathematik.tu-muenchen.de>
-\t(David Bremner's message of "Thu, 6 Mar 2003 13:58:21 +0100")
-Received-2: <15975.17901.207240.414604@sgigritzmann1.mathematik.tu-muenchen.de>
+        msg['Received'] = Header(h, header_name='Received-1',
+                                 continuation_ws='\t')
+        msg['Received'] = h
+        self.ndiffAssertEqual(msg.as_string(), """\
+Received: <15975.17901.207240.414604@sgigritzmann1.mathematik.tu-muenchen.de>
 \t(David Bremner's message of "Thu, 6 Mar 2003 13:58:21 +0100")
+Received: <15975.17901.207240.414604@sgigritzmann1.mathematik.tu-muenchen.de>
+ (David Bremner's message of "Thu, 6 Mar 2003 13:58:21 +0100")
 
 """)
 
@@ -804,7 +813,7 @@ Received-2: <15975.17901.207240.414604@sgigritzmann1.mathematik.tu-muenchen.de>
         msg['Face-2'] = Header(t, header_name='Face-2')
         eq(msg.as_string(), """\
 Face-1: iVBORw0KGgoAAAANSUhEUgAAADAAAAAwBAMAAAClLOS0AAAAGFBMVEUAAAAkHiJeRUIcGBi9
-\tlocQDQ4zJykFBAXJfWDjAAACYUlEQVR4nF2TQY/jIAyFc6lydlG5x8Nyp1Y69wj1PN2I5gzp
+ locQDQ4zJykFBAXJfWDjAAACYUlEQVR4nF2TQY/jIAyFc6lydlG5x8Nyp1Y69wj1PN2I5gzp
 Face-2: iVBORw0KGgoAAAANSUhEUgAAADAAAAAwBAMAAAClLOS0AAAAGFBMVEUAAAAkHiJeRUIcGBi9
  locQDQ4zJykFBAXJfWDjAAACYUlEQVR4nF2TQY/jIAyFc6lydlG5x8Nyp1Y69wj1PN2I5gzp
 
@@ -814,11 +823,11 @@ Face-2: iVBORw0KGgoAAAANSUhEUgAAADAAAAAwBAMAAAClLOS0AAAAGFBMVEUAAAAkHiJeRUIcGBi9
         eq = self.ndiffAssertEqual
         m = '''\
 Received: from siimage.com ([172.25.1.3]) by zima.siliconimage.com with Microsoft SMTPSVC(5.0.2195.4905);
-\tWed, 16 Oct 2002 07:41:11 -0700'''
+ Wed, 16 Oct 2002 07:41:11 -0700'''
         msg = email.message_from_string(m)
         eq(msg.as_string(), '''\
 Received: from siimage.com ([172.25.1.3]) by zima.siliconimage.com with
-\tMicrosoft SMTPSVC(5.0.2195.4905); Wed, 16 Oct 2002 07:41:11 -0700
+ Microsoft SMTPSVC(5.0.2195.4905); Wed, 16 Oct 2002 07:41:11 -0700
 
 ''')
 
@@ -830,9 +839,9 @@ List-Unsubscribe: <https://lists.sourceforge.net/lists/listinfo/spamassassin-tal
         msg = Message()
         msg['List'] = h
         msg['List'] = Header(h, header_name='List')
-        eq(msg.as_string(), """\
+        self.ndiffAssertEqual(msg.as_string(), """\
 List: List-Unsubscribe: <https://lists.sourceforge.net/lists/listinfo/spamassassin-talk>,
-\t<mailto:spamassassin-talk-request@lists.sourceforge.net?subject=unsubscribe>
+ <mailto:spamassassin-talk-request@lists.sourceforge.net?subject=unsubscribe>
 List: List-Unsubscribe: <https://lists.sourceforge.net/lists/listinfo/spamassassin-talk>,
  <mailto:spamassassin-talk-request@lists.sourceforge.net?subject=unsubscribe>
 
@@ -2974,11 +2983,11 @@ class TestRFC2231(TestEmailBase):
         msg = self._msgobj('msg_01.txt')
         msg.set_param('title', 'This is even more ***fun*** isn\'t it!',
                       charset='us-ascii', language='en')
-        eq(msg.as_string(), """\
+        self.ndiffAssertEqual(msg.as_string(), """\
 Return-Path: <bbb@zzz.org>
 Delivered-To: bbb@zzz.org
 Received: by mail.zzz.org (Postfix, from userid 889)
-\tid 27CEAD38CC; Fri,  4 May 2001 14:05:44 -0400 (EDT)
+ id 27CEAD38CC; Fri,  4 May 2001 14:05:44 -0400 (EDT)
 MIME-Version: 1.0
 Content-Transfer-Encoding: 7bit
 Message-ID: <15090.61304.110929.45684@aaa.zzz.org>
@@ -2987,7 +2996,7 @@ To: bbb@zzz.org
 Subject: This is a test message
 Date: Fri, 4 May 2001 14:05:44 -0400
 Content-Type: text/plain; charset=us-ascii;
-\ttitle*="us-ascii'en'This%20is%20even%20more%20%2A%2A%2Afun%2A%2A%2A%20isn%27t%20it%21"
+ title*="us-ascii'en'This%20is%20even%20more%20%2A%2A%2Afun%2A%2A%2A%20isn%27t%20it%21"
 
 
 Hi,
@@ -3008,7 +3017,7 @@ Do you like this message?
 Return-Path: <bbb@zzz.org>
 Delivered-To: bbb@zzz.org
 Received: by mail.zzz.org (Postfix, from userid 889)
-\tid 27CEAD38CC; Fri,  4 May 2001 14:05:44 -0400 (EDT)
+ id 27CEAD38CC; Fri,  4 May 2001 14:05:44 -0400 (EDT)
 MIME-Version: 1.0
 Content-Transfer-Encoding: 7bit
 Message-ID: <15090.61304.110929.45684@aaa.zzz.org>
@@ -3017,7 +3026,7 @@ To: bbb@zzz.org
 Subject: This is a test message
 Date: Fri, 4 May 2001 14:05:44 -0400
 Content-Type: text/plain; charset="us-ascii";
-\ttitle*="us-ascii'en'This%20is%20even%20more%20%2A%2A%2Afun%2A%2A%2A%20isn%27t%20it%21"
+ title*="us-ascii'en'This%20is%20even%20more%20%2A%2A%2Afun%2A%2A%2A%20isn%27t%20it%21"
 
 
 Hi,