#21725: Add RFC 6531 (SMTPUTF8) support to smtpd.

Patch by Milan Oberkirch, developed as part of his 2014 GSOC project.

Note that this also fixes a bug in mock_socket ('getpeername' was returning a
simple string instead of the tuple required for IPvX protocols), a bug in
DebugServer with respect to handling binary data (should have been fixed when
decode_data was introduced, but wasn't found until this patch was written),
and a long-standing bug in DebugServer (it was printing an extra blank line at
the end of the displayed message text).

1 files changed, 146 insertions, 58 deletions

diff --git a/Lib/smtpd.py b/Lib/smtpd.py
index f90fae8..33653d4 100755
--- a/Lib/smtpd.py
+++ b/Lib/smtpd.py
@@ -1,5 +1,5 @@
 #! /usr/bin/env python3
-"""An RFC 5321 smtp proxy.
+"""An RFC 5321 smtp proxy with optional RFC 1870 and RFC 6531 extensions.
 
 Usage: %(program)s [options] [localhost:localport [remotehost:remoteport]]
 
@@ -25,6 +25,10 @@ Options:
         Restrict the total size of the incoming message to "limit" number of
         bytes via the RFC 1870 SIZE extension.  Defaults to 33554432 bytes.
 
+    --smtputf8
+    -u
+        Enable the SMTPUTF8 extension and behave as an RFC 6531 smtp proxy.
+
     --debug
     -d
         Turn on debugging prints.
@@ -115,18 +119,27 @@ class SMTPChannel(asynchat.async_chat):
 
     command_size_limit = 512
     command_size_limits = collections.defaultdict(lambda x=command_size_limit: x)
-    command_size_limits.update({
-        'MAIL': command_size_limit + 26,
-        })
-    max_command_size_limit = max(command_size_limits.values())
+
+    @property
+    def max_command_size_limit(self):
+        try:
+            return max(self.command_size_limits.values())
+        except ValueError:
+            return self.command_size_limit
 
     def __init__(self, server, conn, addr, data_size_limit=DATA_SIZE_DEFAULT,
-                 map=None, decode_data=None):
+                 map=None, enable_SMTPUTF8=False, decode_data=None):
         asynchat.async_chat.__init__(self, conn, map=map)
         self.smtp_server = server
         self.conn = conn
         self.addr = addr
         self.data_size_limit = data_size_limit
+        self.enable_SMTPUTF8 = enable_SMTPUTF8
+        if enable_SMTPUTF8:
+            if decode_data:
+                ValueError("decode_data and enable_SMTPUTF8 cannot be set to"
+                           " True at the same time")
+            decode_data = False
         if decode_data is None:
             warn("The decode_data default of True will change to False in 3.6;"
                  " specify an explicit value for this keyword",
@@ -143,14 +156,11 @@ class SMTPChannel(asynchat.async_chat):
             self._linesep = b'\r\n'
             self._dotsep = b'.'
             self._newline = b'\n'
-        self.received_lines = []
-        self.smtp_state = self.COMMAND
+        self._set_rset_state()
         self.seen_greeting = ''
-        self.mailfrom = None
-        self.rcpttos = []
-        self.received_data = ''
+        self.extended_smtp = False
+        self.command_size_limits.clear()
         self.fqdn = socket.getfqdn()
-        self.num_bytes = 0
         try:
             self.peer = conn.getpeername()
         except OSError as err:
@@ -162,8 +172,22 @@ class SMTPChannel(asynchat.async_chat):
             return
         print('Peer:', repr(self.peer), file=DEBUGSTREAM)
         self.push('220 %s %s' % (self.fqdn, __version__))
+
+    def _set_post_data_state(self):
+        """Reset state variables to their post-DATA state."""
+        self.smtp_state = self.COMMAND
+        self.mailfrom = None
+        self.rcpttos = []
+        self.require_SMTPUTF8 = False
+        self.num_bytes = 0
         self.set_terminator(b'\r\n')
-        self.extended_smtp = False
+
+    def _set_rset_state(self):
+        """Reset all state variables except the greeting."""
+        self._set_post_data_state()
+        self.received_data = ''
+        self.received_lines = []
+
 
     # properties for backwards-compatibility
     @property
@@ -287,9 +311,10 @@ class SMTPChannel(asynchat.async_chat):
             "set 'addr' instead", DeprecationWarning, 2)
         self.addr = value
 
-    # Overrides base class for convenience
+    # Overrides base class for convenience.
     def push(self, msg):
-        asynchat.async_chat.push(self, bytes(msg + '\r\n', 'ascii'))
+        asynchat.async_chat.push(self, bytes(
+            msg + '\r\n', 'utf-8' if self.require_SMTPUTF8 else 'ascii'))
 
     # Implementation of base class abstract method
     def collect_incoming_data(self, data):
@@ -317,7 +342,6 @@ class SMTPChannel(asynchat.async_chat):
             if not line:
                 self.push('500 Error: bad syntax')
                 return
-            method = None
             if not self._decode_data:
                 line = str(line, 'utf-8')
             i = line.find(' ')
@@ -356,15 +380,12 @@ class SMTPChannel(asynchat.async_chat):
                 else:
                     data.append(text)
             self.received_data = self._newline.join(data)
-            status = self.smtp_server.process_message(self.peer,
-                                                      self.mailfrom,
-                                                      self.rcpttos,
-                                                      self.received_data)
-            self.rcpttos = []
-            self.mailfrom = None
-            self.smtp_state = self.COMMAND
-            self.num_bytes = 0
-            self.set_terminator(b'\r\n')
+            args = (self.peer, self.mailfrom, self.rcpttos, self.received_data)
+            if self.require_SMTPUTF8:
+                status = self.smtp_server.process_smtputf8_message(*args)
+            else:
+                status = self.smtp_server.process_message(*args)
+            self._set_post_data_state()
             if not status:
                 self.push('250 OK')
             else:
@@ -375,26 +396,34 @@ class SMTPChannel(asynchat.async_chat):
         if not arg:
             self.push('501 Syntax: HELO hostname')
             return
+        # See issue #21783 for a discussion of this behavior.
         if self.seen_greeting:
             self.push('503 Duplicate HELO/EHLO')
-        else:
-            self.seen_greeting = arg
-            self.extended_smtp = False
-            self.push('250 %s' % self.fqdn)
+            return
+        self._set_rset_state()
+        self.seen_greeting = arg
+        self.push('250 %s' % self.fqdn)
 
     def smtp_EHLO(self, arg):
         if not arg:
             self.push('501 Syntax: EHLO hostname')
             return
+        # See issue #21783 for a discussion of this behavior.
         if self.seen_greeting:
             self.push('503 Duplicate HELO/EHLO')
-        else:
-            self.seen_greeting = arg
-            self.extended_smtp = True
-            self.push('250-%s' % self.fqdn)
-            if self.data_size_limit:
-                self.push('250-SIZE %s' % self.data_size_limit)
-            self.push('250 HELP')
+            return
+        self._set_rset_state()
+        self.seen_greeting = arg
+        self.extended_smtp = True
+        self.push('250-%s' % self.fqdn)
+        if self.data_size_limit:
+            self.push('250-SIZE %s' % self.data_size_limit)
+            self.command_size_limits['MAIL'] += 26
+        if self.enable_SMTPUTF8:
+            self.push('250-8BITMIME')
+            self.push('250-SMTPUTF8')
+            self.command_size_limits['MAIL'] += 10
+        self.push('250 HELP')
 
     def smtp_NOOP(self, arg):
         if arg:
@@ -427,8 +456,8 @@ class SMTPChannel(asynchat.async_chat):
     def _getparams(self, params):
         # Return any parameters that appear to be syntactically valid according
         # to RFC 1869, ignore all others.  (Postel rule: accept what we can.)
-        params = [param.split('=', 1) for param in params.split()
-                                      if '=' in param]
+        params = [param.split('=', 1) if '=' in param else (param, True)
+                  for param in params.split()]
         return {k: v for k, v in params if k.isalnum()}
 
     def smtp_HELP(self, arg):
@@ -506,6 +535,14 @@ class SMTPChannel(asynchat.async_chat):
         if params is None:
             self.push(syntaxerr)
             return
+        body = params.pop('BODY', '7BIT')
+        if self.enable_SMTPUTF8 and params.pop('SMTPUTF8', False):
+            if body != '8BITMIME':
+                self.push('501 Syntax: MAIL FROM: <address>'
+                          ' [BODY=8BITMIME SMTPUTF8]')
+                return
+            else:
+                self.require_SMTPUTF8 = True
         size = params.pop('SIZE', None)
         if size:
             if not size.isdigit():
@@ -566,11 +603,7 @@ class SMTPChannel(asynchat.async_chat):
         if arg:
             self.push('501 Syntax: RSET')
             return
-        # Resets the sender, recipients, and data, but not the greeting
-        self.mailfrom = None
-        self.rcpttos = []
-        self.received_data = ''
-        self.smtp_state = self.COMMAND
+        self._set_rset_state()
         self.push('250 OK')
 
     def smtp_DATA(self, arg):
@@ -598,10 +631,17 @@ class SMTPServer(asyncore.dispatcher):
 
     def __init__(self, localaddr, remoteaddr,
                  data_size_limit=DATA_SIZE_DEFAULT, map=None,
-                 decode_data=None):
+                 enable_SMTPUTF8=False, decode_data=None):
         self._localaddr = localaddr
         self._remoteaddr = remoteaddr
         self.data_size_limit = data_size_limit
+        self.enable_SMTPUTF8 = enable_SMTPUTF8
+        if enable_SMTPUTF8:
+            if decode_data:
+                raise ValueError("The decode_data and enable_SMTPUTF8"
+                                 " parameters cannot be set to True at the"
+                                 " same time.")
+            decode_data = False
         if decode_data is None:
             warn("The decode_data default of True will change to False in 3.6;"
                  " specify an explicit value for this keyword",
@@ -627,8 +667,13 @@ class SMTPServer(asyncore.dispatcher):
 
     def handle_accepted(self, conn, addr):
         print('Incoming connection from %s' % repr(addr), file=DEBUGSTREAM)
-        channel = self.channel_class(self, conn, addr, self.data_size_limit,
-                                     self._map, self._decode_data)
+        channel = self.channel_class(self,
+                                     conn,
+                                     addr,
+                                     self.data_size_limit,
+                                     self._map,
+                                     self.enable_SMTPUTF8,
+                                     self._decode_data)
 
     # API for "doing something useful with the message"
     def process_message(self, peer, mailfrom, rcpttos, data):
@@ -649,29 +694,63 @@ class SMTPServer(asyncore.dispatcher):
         containing a `.' followed by other text has had the leading dot
         removed.
 
-        This function should return None, for a normal `250 Ok' response;
-        otherwise it returns the desired response string in RFC 821 format.
+        This function should return None for a normal `250 Ok' response;
+        otherwise, it should return the desired response string in RFC 821
+        format.
+
+        """
+        raise NotImplementedError
+
+    # API for processing messeges needing Unicode support (RFC 6531, RFC 6532).
+    def process_smtputf8_message(self, peer, mailfrom, rcpttos, data):
+        """Same as ``process_message`` but for messages for which the client
+        has sent the SMTPUTF8 parameter with the MAIL command (see the
+        enable_SMTPUTF8 parameter of the constructor).
+
+        This function should return None for a normal `250 Ok' response;
+        otherwise, it should return the desired response string in RFC 6531
+        format.
 
         """
         raise NotImplementedError
 
 
 class DebuggingServer(SMTPServer):
-    # Do something with the gathered message
-    def process_message(self, peer, mailfrom, rcpttos, data):
+
+    def _print_message_content(self, peer, data):
         inheaders = 1
-        lines = data.split('\n')
-        print('---------- MESSAGE FOLLOWS ----------')
+        lines = data.splitlines()
         for line in lines:
             # headers first
             if inheaders and not line:
-                print('X-Peer:', peer[0])
+                peerheader = 'X-Peer: ' + peer[0]
+                if not isinstance(data, str):
+                    # decoded_data=false; make header match other binary output
+                    peerheader = repr(peerheader.encode('utf-8'))
+                print(peerheader)
                 inheaders = 0
+            if not isinstance(data, str):
+                # Avoid spurious 'str on bytes instance' warning.
+                line = repr(line)
             print(line)
+
+    def process_message(self, peer, mailfrom, rcpttos, data):
+        print('---------- MESSAGE FOLLOWS ----------')
+        self._print_message_content(peer, data)
+        print('------------ END MESSAGE ------------')
+
+    def process_smtputf8_message(self, peer, mailfrom, rcpttos, data):
+        print('----- SMTPUTF8 MESSAGE FOLLOWS ------')
+        self._print_message_content(peer, data)
         print('------------ END MESSAGE ------------')
 
 
 class PureProxy(SMTPServer):
+    def __init__(self, *args, **kwargs):
+        if 'enable_SMTPUTF8' in kwargs and kwargs['enable_SMTPUTF8']:
+            raise ValueError("PureProxy does not support SMTPUTF8.")
+        super(PureProxy, self).__init__(*args, **kwargs)
+
     def process_message(self, peer, mailfrom, rcpttos, data):
         lines = data.split('\n')
         # Look for the last header
@@ -712,6 +791,11 @@ class PureProxy(SMTPServer):
 
 
 class MailmanProxy(PureProxy):
+    def __init__(self, *args, **kwargs):
+        if 'enable_SMTPUTF8' in kwargs and kwargs['enable_SMTPUTF8']:
+            raise ValueError("MailmanProxy does not support SMTPUTF8.")
+        super(PureProxy, self).__init__(*args, **kwargs)
+
     def process_message(self, peer, mailfrom, rcpttos, data):
         from io import StringIO
         from Mailman import Utils
@@ -790,17 +874,19 @@ class MailmanProxy(PureProxy):
 
 
 class Options:
-    setuid = 1
+    setuid = True
     classname = 'PureProxy'
     size_limit = None
+    enable_SMTPUTF8 = False
 
 
 def parseargs():
     global DEBUGSTREAM
     try:
         opts, args = getopt.getopt(
-            sys.argv[1:], 'nVhc:s:d',
-            ['class=', 'nosetuid', 'version', 'help', 'size=', 'debug'])
+            sys.argv[1:], 'nVhc:s:du',
+            ['class=', 'nosetuid', 'version', 'help', 'size=', 'debug',
+             'smtputf8'])
     except getopt.error as e:
         usage(1, e)
 
@@ -812,11 +898,13 @@ def parseargs():
             print(__version__)
             sys.exit(0)
         elif opt in ('-n', '--nosetuid'):
-            options.setuid = 0
+            options.setuid = False
         elif opt in ('-c', '--class'):
             options.classname = arg
         elif opt in ('-d', '--debug'):
             DEBUGSTREAM = sys.stderr
+        elif opt in ('-u', '--smtputf8'):
+            options.enable_SMTPUTF8 = True
         elif opt in ('-s', '--size'):
             try:
                 int_size = int(arg)
@@ -871,7 +959,7 @@ if __name__ == '__main__':
     class_ = getattr(mod, classname)
     proxy = class_((options.localhost, options.localport),
                    (options.remotehost, options.remoteport),
-                   options.size_limit)
+                   options.size_limit, enable_SMTPUTF8=options.enable_SMTPUTF8)
     if options.setuid:
         try:
             import pwd