#21795: advertise 8BITMIME if decode_data is False.

Patch by Milan Oberkirch, with a few updates.  This changeset also
tweaks the smtpd and whatsnew docs for smtpd into what should be
the final form for the 3.5 release.

2 files changed, 164 insertions, 69 deletions

diff --git a/Lib/smtpd.py b/Lib/smtpd.py
index dd410b8..ff86e7d 100755
--- a/Lib/smtpd.py
+++ b/Lib/smtpd.py
@@ -381,10 +381,13 @@ class SMTPChannel(asynchat.async_chat):
                     data.append(text)
             self.received_data = self._newline.join(data)
             args = (self.peer, self.mailfrom, self.rcpttos, self.received_data)
-            if self.require_SMTPUTF8:
-                status = self.smtp_server.process_smtputf8_message(*args)
-            else:
-                status = self.smtp_server.process_message(*args)
+            kwargs = {}
+            if not self._decode_data:
+                kwargs = {
+                    'mail_options': self.mail_options,
+                    'rcpt_options': self.rcpt_options,
+                }
+            status = self.smtp_server.process_message(*args, **kwargs)
             self._set_post_data_state()
             if not status:
                 self.push('250 OK')
@@ -419,8 +422,9 @@ class SMTPChannel(asynchat.async_chat):
         if self.data_size_limit:
             self.push('250-SIZE %s' % self.data_size_limit)
             self.command_size_limits['MAIL'] += 26
-        if self.enable_SMTPUTF8:
+        if not self._decode_data:
             self.push('250-8BITMIME')
+        if self.enable_SMTPUTF8:
             self.push('250-SMTPUTF8')
             self.command_size_limits['MAIL'] += 10
         self.push('250 HELP')
@@ -454,11 +458,15 @@ class SMTPChannel(asynchat.async_chat):
         return address.addr_spec, rest
 
     def _getparams(self, params):
-        # Return any parameters that appear to be syntactically valid according
-        # to RFC 1869, ignore all others.  (Postel rule: accept what we can.)
-        params = [param.split('=', 1) if '=' in param else (param, True)
-                  for param in params.split()]
-        return {k: v for k, v in params if k.isalnum()}
+        # Return params as dictionary. Return None if not all parameters
+        # appear to be syntactically valid according to RFC 1869.
+        result = {}
+        for param in params:
+            param, eq, value = param.partition('=')
+            if not param.isalnum() or eq and not value:
+                return None
+            result[param] = value if eq else True
+        return result
 
     def smtp_HELP(self, arg):
         if arg:
@@ -508,7 +516,7 @@ class SMTPChannel(asynchat.async_chat):
 
     def smtp_MAIL(self, arg):
         if not self.seen_greeting:
-            self.push('503 Error: send HELO first');
+            self.push('503 Error: send HELO first')
             return
         print('===> MAIL', arg, file=DEBUGSTREAM)
         syntaxerr = '501 Syntax: MAIL FROM: <address>'
@@ -528,18 +536,23 @@ class SMTPChannel(asynchat.async_chat):
         if self.mailfrom:
             self.push('503 Error: nested MAIL command')
             return
-        params = self._getparams(params.upper())
+        self.mail_options = params.upper().split()
+        params = self._getparams(self.mail_options)
         if params is None:
             self.push(syntaxerr)
             return
-        body = params.pop('BODY', '7BIT')
-        if self.enable_SMTPUTF8 and params.pop('SMTPUTF8', False):
-            if body != '8BITMIME':
-                self.push('501 Syntax: MAIL FROM: <address>'
-                          ' [BODY=8BITMIME SMTPUTF8]')
+        if not self._decode_data:
+            body = params.pop('BODY', '7BIT')
+            if body not in ['7BIT', '8BITMIME']:
+                self.push('501 Error: BODY can only be one of 7BIT, 8BITMIME')
                 return
-            else:
+        if self.enable_SMTPUTF8:
+            smtputf8 = params.pop('SMTPUTF8', False)
+            if smtputf8 is True:
                 self.require_SMTPUTF8 = True
+            elif smtputf8 is not False:
+                self.push('501 Error: SMTPUTF8 takes no arguments')
+                return
         size = params.pop('SIZE', None)
         if size:
             if not size.isdigit():
@@ -574,16 +587,16 @@ class SMTPChannel(asynchat.async_chat):
         if not address:
             self.push(syntaxerr)
             return
-        if params:
-            if self.extended_smtp:
-                params = self._getparams(params.upper())
-                if params is None:
-                    self.push(syntaxerr)
-                    return
-            else:
-                self.push(syntaxerr)
-                return
-        if params and len(params.keys()) > 0:
+        if not self.extended_smtp and params:
+            self.push(syntaxerr)
+            return
+        self.rcpt_options = params.upper().split()
+        params = self._getparams(self.rcpt_options)
+        if params is None:
+            self.push(syntaxerr)
+            return
+        # XXX currently there are no options we recognize.
+        if len(params.keys()) > 0:
             self.push('555 RCPT TO parameters not recognized or not implemented')
             return
         self.rcpttos.append(address)
@@ -667,7 +680,7 @@ class SMTPServer(asyncore.dispatcher):
                                      self._decode_data)
 
     # API for "doing something useful with the message"
-    def process_message(self, peer, mailfrom, rcpttos, data):
+    def process_message(self, peer, mailfrom, rcpttos, data, **kwargs):
         """Override this abstract method to handle messages from the client.
 
         peer is a tuple containing (ipaddr, port) of the client that made the
@@ -685,21 +698,16 @@ class SMTPServer(asyncore.dispatcher):
         containing a `.' followed by other text has had the leading dot
         removed.
 
-        This function should return None for a normal `250 Ok' response;
-        otherwise, it should return the desired response string in RFC 821
-        format.
-
-        """
-        raise NotImplementedError
-
-    # API for processing messeges needing Unicode support (RFC 6531, RFC 6532).
-    def process_smtputf8_message(self, peer, mailfrom, rcpttos, data):
-        """Same as ``process_message`` but for messages for which the client
-        has sent the SMTPUTF8 parameter with the MAIL command (see the
-        enable_SMTPUTF8 parameter of the constructor).
+        kwargs is a dictionary containing additional information. It is empty
+        unless decode_data=False or enable_SMTPUTF8=True was given as init
+        parameter, in which case ut will contain the following keys:
+            'mail_options': list of parameters to the mail command.  All
+                            elements are uppercase strings.  Example:
+                            ['BODY=8BITMIME', 'SMTPUTF8'].
+            'rcpt_options': same, for the rcpt command.
 
         This function should return None for a normal `250 Ok' response;
-        otherwise, it should return the desired response string in RFC 6531
+        otherwise, it should return the desired response string in RFC 821
         format.
 
         """
@@ -725,13 +733,13 @@ class DebuggingServer(SMTPServer):
                 line = repr(line)
             print(line)
 
-    def process_message(self, peer, mailfrom, rcpttos, data):
+    def process_message(self, peer, mailfrom, rcpttos, data, **kwargs):
         print('---------- MESSAGE FOLLOWS ----------')
-        self._print_message_content(peer, data)
-        print('------------ END MESSAGE ------------')
-
-    def process_smtputf8_message(self, peer, mailfrom, rcpttos, data):
-        print('----- SMTPUTF8 MESSAGE FOLLOWS ------')
+        if kwargs:
+            if kwargs.get('mail_options'):
+                print('mail options: %s' % kwargs['mail_options'])
+            if kwargs.get('rcpt_options'):
+                print('rcpt options: %s\n' % kwargs['rcpt_options'])
         self._print_message_content(peer, data)
         print('------------ END MESSAGE ------------')
 
diff --git a/Lib/test/test_smtpd.py b/Lib/test/test_smtpd.py
index 6eb47f1..1aa55d2 100644
--- a/Lib/test/test_smtpd.py
+++ b/Lib/test/test_smtpd.py
@@ -16,13 +16,12 @@ class DummyServer(smtpd.SMTPServer):
         else:
             self.return_status = b'return status'
 
-    def process_message(self, peer, mailfrom, rcpttos, data):
+    def process_message(self, peer, mailfrom, rcpttos, data, **kw):
         self.messages.append((peer, mailfrom, rcpttos, data))
         if data == self.return_status:
             return '250 Okish'
-
-    def process_smtputf8_message(self, *args, **kwargs):
-        return '250 SMTPUTF8 message okish'
+        if 'mail_options' in kw and 'SMTPUTF8' in kw['mail_options']:
+            return '250 SMTPUTF8 message okish'
 
 
 class DummyDispatcherBroken(Exception):
@@ -54,22 +53,6 @@ class SMTPDServerTest(unittest.TestCase):
         write_line(b'DATA')
         self.assertRaises(NotImplementedError, write_line, b'spam\r\n.\r\n')
 
-    def test_process_smtputf8_message_unimplemented(self):
-        server = smtpd.SMTPServer((support.HOST, 0), ('b', 0),
-                                  enable_SMTPUTF8=True)
-        conn, addr = server.accept()
-        channel = smtpd.SMTPChannel(server, conn, addr, enable_SMTPUTF8=True)
-
-        def write_line(line):
-            channel.socket.queue_recv(line)
-            channel.handle_read()
-
-        write_line(b'EHLO example')
-        write_line(b'MAIL From: <eggs@example> BODY=8BITMIME SMTPUTF8')
-        write_line(b'RCPT To: <spam@example>')
-        write_line(b'DATA')
-        self.assertRaises(NotImplementedError, write_line, b'spam\r\n.\r\n')
-
     def test_decode_data_default_warns(self):
         with self.assertWarns(DeprecationWarning):
             smtpd.SMTPServer((support.HOST, 0), ('b', 0))
@@ -168,7 +151,8 @@ class DebuggingServerTest(unittest.TestCase):
                            enable_SMTPUTF8=True)
         stdout = s.getvalue()
         self.assertEqual(stdout, textwrap.dedent("""\
-             ----- SMTPUTF8 MESSAGE FOLLOWS ------
+             ---------- MESSAGE FOLLOWS ----------
+             mail options: ['BODY=8BITMIME', 'SMTPUTF8']
              b'From: test'
              b'X-Peer: peer-address'
              b''
@@ -201,6 +185,109 @@ class TestFamilyDetection(unittest.TestCase):
         self.assertEqual(server.socket.family, socket.AF_INET)
 
 
+class TestRcptOptionParsing(unittest.TestCase):
+    error_response = (b'555 RCPT TO parameters not recognized or not '
+                      b'implemented\r\n')
+
+    def setUp(self):
+        smtpd.socket = asyncore.socket = mock_socket
+        self.old_debugstream = smtpd.DEBUGSTREAM
+        self.debug = smtpd.DEBUGSTREAM = io.StringIO()
+
+    def tearDown(self):
+        asyncore.close_all()
+        asyncore.socket = smtpd.socket = socket
+        smtpd.DEBUGSTREAM = self.old_debugstream
+
+    def write_line(self, channel, line):
+        channel.socket.queue_recv(line)
+        channel.handle_read()
+
+    def test_params_rejected(self):
+        server = DummyServer((support.HOST, 0), ('b', 0), decode_data=False)
+        conn, addr = server.accept()
+        channel = smtpd.SMTPChannel(server, conn, addr, decode_data=False)
+        self.write_line(channel, b'EHLO example')
+        self.write_line(channel, b'MAIL from: <foo@example.com> size=20')
+        self.write_line(channel, b'RCPT to: <foo@example.com> foo=bar')
+        self.assertEqual(channel.socket.last, self.error_response)
+
+    def test_nothing_accepted(self):
+        server = DummyServer((support.HOST, 0), ('b', 0), decode_data=False)
+        conn, addr = server.accept()
+        channel = smtpd.SMTPChannel(server, conn, addr, decode_data=False)
+        self.write_line(channel, b'EHLO example')
+        self.write_line(channel, b'MAIL from: <foo@example.com> size=20')
+        self.write_line(channel, b'RCPT to: <foo@example.com>')
+        self.assertEqual(channel.socket.last, b'250 OK\r\n')
+
+
+class TestMailOptionParsing(unittest.TestCase):
+    error_response = (b'555 MAIL FROM parameters not recognized or not '
+                      b'implemented\r\n')
+
+    def setUp(self):
+        smtpd.socket = asyncore.socket = mock_socket
+        self.old_debugstream = smtpd.DEBUGSTREAM
+        self.debug = smtpd.DEBUGSTREAM = io.StringIO()
+
+    def tearDown(self):
+        asyncore.close_all()
+        asyncore.socket = smtpd.socket = socket
+        smtpd.DEBUGSTREAM = self.old_debugstream
+
+    def write_line(self, channel, line):
+        channel.socket.queue_recv(line)
+        channel.handle_read()
+
+    def test_with_decode_data_true(self):
+        server = DummyServer((support.HOST, 0), ('b', 0), decode_data=True)
+        conn, addr = server.accept()
+        channel = smtpd.SMTPChannel(server, conn, addr, decode_data=True)
+        self.write_line(channel, b'EHLO example')
+        for line in [
+            b'MAIL from: <foo@example.com> size=20 SMTPUTF8',
+            b'MAIL from: <foo@example.com> size=20 SMTPUTF8 BODY=8BITMIME',
+            b'MAIL from: <foo@example.com> size=20 BODY=UNKNOWN',
+            b'MAIL from: <foo@example.com> size=20 body=8bitmime',
+        ]:
+            self.write_line(channel, line)
+            self.assertEqual(channel.socket.last, self.error_response)
+        self.write_line(channel, b'MAIL from: <foo@example.com> size=20')
+        self.assertEqual(channel.socket.last, b'250 OK\r\n')
+
+    def test_with_decode_data_false(self):
+        server = DummyServer((support.HOST, 0), ('b', 0), decode_data=False)
+        conn, addr = server.accept()
+        channel = smtpd.SMTPChannel(server, conn, addr, decode_data=False)
+        self.write_line(channel, b'EHLO example')
+        for line in [
+            b'MAIL from: <foo@example.com> size=20 SMTPUTF8',
+            b'MAIL from: <foo@example.com> size=20 SMTPUTF8 BODY=8BITMIME',
+        ]:
+            self.write_line(channel, line)
+            self.assertEqual(channel.socket.last, self.error_response)
+        self.write_line(
+            channel,
+            b'MAIL from: <foo@example.com> size=20 SMTPUTF8 BODY=UNKNOWN')
+        self.assertEqual(
+            channel.socket.last,
+            b'501 Error: BODY can only be one of 7BIT, 8BITMIME\r\n')
+        self.write_line(
+            channel, b'MAIL from: <foo@example.com> size=20 body=8bitmime')
+        self.assertEqual(channel.socket.last, b'250 OK\r\n')
+
+    def test_with_enable_smtputf8_true(self):
+        server = DummyServer((support.HOST, 0), ('b', 0), enable_SMTPUTF8=True)
+        conn, addr = server.accept()
+        channel = smtpd.SMTPChannel(server, conn, addr, enable_SMTPUTF8=True)
+        self.write_line(channel, b'EHLO example')
+        self.write_line(
+            channel,
+            b'MAIL from: <foo@example.com> size=20 body=8bitmime smtputf8')
+        self.assertEqual(channel.socket.last, b'250 OK\r\n')
+
+
 class SMTPDChannelTest(unittest.TestCase):
     def setUp(self):
         smtpd.socket = asyncore.socket = mock_socket