#22027: Add RFC6531 support to smtplib.

Initial patch by Milan Oberkirch.

5 files changed, 194 insertions, 7 deletions

diff --git a/Doc/library/smtplib.rst b/Doc/library/smtplib.rst
index 44473fe..133fa56 100644
--- a/Doc/library/smtplib.rst
+++ b/Doc/library/smtplib.rst
@@ -61,6 +61,10 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions).
    .. versionchanged:: 3.3
       source_address argument was added.
 
+   .. versionadded:: 3.5
+      The SMTPUTF8 extension (:rfc:`6531`) is now supported.
+
+
 .. class:: SMTP_SSL(host='', port=0, local_hostname=None, keyfile=None, \
                     certfile=None [, timeout], context=None, \
                     source_address=None)
@@ -161,6 +165,13 @@ A nice selection of exceptions is defined as well:
    The server refused our ``HELO`` message.
 
 
+.. exception:: SMTPNotSupportedError
+
+    The command or option attempted is not supported by the server.
+
+    .. versionadded:: 3.5
+
+
 .. exception:: SMTPAuthenticationError
 
    SMTP authentication went wrong.  Most probably the server didn't accept the
@@ -291,6 +302,9 @@ An :class:`SMTP` instance has the following methods:
    :exc:`SMTPAuthenticationError`
       The server didn't accept the username/password combination.
 
+   :exc:`SMTPNotSupportedError`
+      The ``AUTH`` command is not supported by the server.
+
    :exc:`SMTPException`
       No suitable authentication method was found.
 
@@ -298,6 +312,9 @@ An :class:`SMTP` instance has the following methods:
    turn if they are advertised as supported by the server (see :meth:`auth`
    for a list of supported authentication methods).
 
+   .. versionchanged:: 3.5
+      :exc:`SMTPNotSupportedError` may be raised.
+
 
 .. method:: SMTP.auth(mechanism, authobject)
 
@@ -349,7 +366,7 @@ An :class:`SMTP` instance has the following methods:
    :exc:`SMTPHeloError`
       The server didn't reply properly to the ``HELO`` greeting.
 
-   :exc:`SMTPException`
+   :exc:`SMTPNotSupportedError`
      The server does not support the STARTTLS extension.
 
    :exc:`RuntimeError`
@@ -363,6 +380,11 @@ An :class:`SMTP` instance has the following methods:
       :attr:`SSLContext.check_hostname` and *Server Name Indicator* (see
       :data:`~ssl.HAS_SNI`).
 
+   .. versionchanged:: 3.5
+      The error raised for lack of STARTTLS support is now the
+      :exc:`SMTPNotSupportedError` subclass instead of the base
+      :exc:`SMTPException`.
+
 
 .. method:: SMTP.sendmail(from_addr, to_addrs, msg, mail_options=[], rcpt_options=[])
 
@@ -399,6 +421,9 @@ An :class:`SMTP` instance has the following methods:
    recipient that was refused.  Each entry contains a tuple of the SMTP error code
    and the accompanying error message sent by the server.
 
+   If ``SMTPUTF8`` is included in *mail_options*, and the server supports it,
+   *from_addr* and *to_addr* may contain non-ASCII characters.
+
    This method may raise the following exceptions:
 
    :exc:`SMTPRecipientsRefused`
@@ -417,12 +442,20 @@ An :class:`SMTP` instance has the following methods:
       The server replied with an unexpected error code (other than a refusal of a
       recipient).
 
+   :exc:`SMTPNotSupportedError`
+      ``SMTPUTF8`` was given in the *mail_options* but is not supported by the
+      server.
+
    Unless otherwise noted, the connection will be open even after an exception is
    raised.
 
    .. versionchanged:: 3.2
       *msg* may be a byte string.
 
+   .. versionchanged:: 3.5
+      ``SMTPUTF8`` support added, and :exc:`SMTPNotSupportedError` may be
+      raised if ``SMTPUTF8`` is specified but the server does not support it.
+
 
 .. method:: SMTP.send_message(msg, from_addr=None, to_addrs=None, \
                               mail_options=[], rcpt_options=[])
diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst
index 92e1142..86173a0 100644
--- a/Doc/whatsnew/3.5.rst
+++ b/Doc/whatsnew/3.5.rst
@@ -527,6 +527,9 @@ smtplib
   :class:`smtplib.SMTP`.  (Contributed by Gavin Chappell and Maciej Szulik in
   :issue:`16914`.)
 
+* :mod:`smtplib` now support :rfc:`6531` (SMTPUTF8).  (Contributed by
+  Milan Oberkirch and R. David Murray in :issue:`22027`.)
+
 sndhdr
 ------
 
diff --git a/Lib/smtplib.py b/Lib/smtplib.py
index 7f49f23..6895bed 100755
--- a/Lib/smtplib.py
+++ b/Lib/smtplib.py
@@ -71,6 +71,13 @@ OLDSTYLE_AUTH = re.compile(r"auth=(.*)", re.I)
 class SMTPException(OSError):
     """Base class for all exceptions raised by this module."""
 
+class SMTPNotSupportedError(SMTPException):
+    """The command or option is not supported by the SMTP server.
+
+    This exception is raised when an attempt is made to run a command or a
+    command with an option which is not supported by the server.
+    """
+
 class SMTPServerDisconnected(SMTPException):
     """Not connected to any SMTP server.
 
@@ -237,6 +244,7 @@ class SMTP:
         self._host = host
         self.timeout = timeout
         self.esmtp_features = {}
+        self.command_encoding = 'ascii'
         self.source_address = source_address
 
         if host:
@@ -337,7 +345,10 @@ class SMTP:
             self._print_debug('send:', repr(s))
         if hasattr(self, 'sock') and self.sock:
             if isinstance(s, str):
-                s = s.encode("ascii")
+                # send is used by the 'data' command, where command_encoding
+                # should not be used, but 'data' needs to convert the string to
+                # binary itself anyway, so that's not a problem.
+                s = s.encode(self.command_encoding)
             try:
                 self.sock.sendall(s)
             except OSError:
@@ -482,6 +493,7 @@ class SMTP:
 
     def rset(self):
         """SMTP 'rset' command -- resets session."""
+        self.command_encoding = 'ascii'
         return self.docmd("rset")
 
     def _rset(self):
@@ -501,9 +513,22 @@ class SMTP:
         return self.docmd("noop")
 
     def mail(self, sender, options=[]):
-        """SMTP 'mail' command -- begins mail xfer session."""
+        """SMTP 'mail' command -- begins mail xfer session.
+
+        This method may raise the following exceptions:
+
+         SMTPNotSupportedError  The options parameter includes 'SMTPUTF8'
+                                but the SMTPUTF8 extension is not supported by
+                                the server.
+        """
         optionlist = ''
         if options and self.does_esmtp:
+            if any(x.lower()=='smtputf8' for x in options):
+                if self.has_extn('smtputf8'):
+                    self.command_encoding = 'utf-8'
+                else:
+                    raise SMTPNotSupportedError(
+                        'SMTPUTF8 not supported by server')
             optionlist = ' ' + ' '.join(options)
         self.putcmd("mail", "FROM:%s%s" % (quoteaddr(sender), optionlist))
         return self.getreply()
@@ -642,13 +667,16 @@ class SMTP:
                                   the helo greeting.
          SMTPAuthenticationError  The server didn't accept the username/
                                   password combination.
+         SMTPNotSupportedError    The AUTH command is not supported by the
+                                  server.
          SMTPException            No suitable authentication method was
                                   found.
         """
 
         self.ehlo_or_helo_if_needed()
         if not self.has_extn("auth"):
-            raise SMTPException("SMTP AUTH extension not supported by server.")
+            raise SMTPNotSupportedError(
+                "SMTP AUTH extension not supported by server.")
 
         # Authentication methods the server claims to support
         advertised_authlist = self.esmtp_features["auth"].split()
@@ -700,7 +728,8 @@ class SMTP:
         """
         self.ehlo_or_helo_if_needed()
         if not self.has_extn("starttls"):
-            raise SMTPException("STARTTLS extension not supported by server.")
+            raise SMTPNotSupportedError(
+                "STARTTLS extension not supported by server.")
         (resp, reply) = self.docmd("STARTTLS")
         if resp == 220:
             if not _have_ssl:
@@ -765,6 +794,9 @@ class SMTP:
          SMTPDataError          The server replied with an unexpected
                                 error code (other than a refusal of
                                 a recipient).
+         SMTPNotSupportedError  The mail_options parameter includes 'SMTPUTF8'
+                                but the SMTPUTF8 extension is not supported by
+                                the server.
 
         Note: the connection will be open even after an exception is raised.
 
@@ -793,8 +825,6 @@ class SMTP:
         if isinstance(msg, str):
             msg = _fix_eols(msg).encode('ascii')
         if self.does_esmtp:
-            # Hmmm? what's this? -ddm
-            # self.esmtp_features['7bit']=""
             if self.has_extn('size'):
                 esmtp_opts.append("size=%d" % len(msg))
             for option in mail_options:
diff --git a/Lib/test/test_smtplib.py b/Lib/test/test_smtplib.py
index 9011042..e496371 100644
--- a/Lib/test/test_smtplib.py
+++ b/Lib/test/test_smtplib.py
@@ -977,6 +977,125 @@ class SMTPSimTests(unittest.TestCase):
         self.assertIsNone(smtp.sock)
         self.assertEqual(self.serv._SMTPchannel.rcpt_count, 0)
 
+    def test_smtputf8_NotSupportedError_if_no_server_support(self):
+        smtp = smtplib.SMTP(
+            HOST, self.port, local_hostname='localhost', timeout=3)
+        self.addCleanup(smtp.close)
+        smtp.ehlo()
+        self.assertTrue(smtp.does_esmtp)
+        self.assertFalse(smtp.has_extn('smtputf8'))
+        self.assertRaises(
+            smtplib.SMTPNotSupportedError,
+            smtp.sendmail,
+            'John', 'Sally', '', mail_options=['BODY=8BITMIME', 'SMTPUTF8'])
+        self.assertRaises(
+            smtplib.SMTPNotSupportedError,
+            smtp.mail, 'John', options=['BODY=8BITMIME', 'SMTPUTF8'])
+
+    def test_send_unicode_without_SMTPUTF8(self):
+        smtp = smtplib.SMTP(
+            HOST, self.port, local_hostname='localhost', timeout=3)
+        self.addCleanup(smtp.close)
+        self.assertRaises(UnicodeEncodeError, smtp.sendmail, 'Alice', 'Böb', '')
+        self.assertRaises(UnicodeEncodeError, smtp.mail, 'Älice')
+
+
+class SimSMTPUTF8Server(SimSMTPServer):
+
+    def __init__(self, *args, **kw):
+        # The base SMTP server turns these on automatically, but our test
+        # server is set up to munge the EHLO response, so we need to provide
+        # them as well.  And yes, the call is to SMTPServer not SimSMTPServer.
+        self._extra_features = ['SMTPUTF8', '8BITMIME']
+        smtpd.SMTPServer.__init__(self, *args, **kw)
+
+    def handle_accepted(self, conn, addr):
+        self._SMTPchannel = self.channel_class(
+            self._extra_features, self, conn, addr,
+            decode_data=self._decode_data,
+            enable_SMTPUTF8=self.enable_SMTPUTF8,
+        )
+
+    def process_message(self, peer, mailfrom, rcpttos, data, mail_options=None,
+                                                             rcpt_options=None):
+        self.last_peer = peer
+        self.last_mailfrom = mailfrom
+        self.last_rcpttos = rcpttos
+        self.last_message = data
+        self.last_mail_options = mail_options
+        self.last_rcpt_options = rcpt_options
+
+
+@unittest.skipUnless(threading, 'Threading required for this test.')
+class SMTPUTF8SimTests(unittest.TestCase):
+
+    def setUp(self):
+        self.real_getfqdn = socket.getfqdn
+        socket.getfqdn = mock_socket.getfqdn
+        self.serv_evt = threading.Event()
+        self.client_evt = threading.Event()
+        # Pick a random unused port by passing 0 for the port number
+        self.serv = SimSMTPUTF8Server((HOST, 0), ('nowhere', -1),
+                                      decode_data=False,
+                                      enable_SMTPUTF8=True)
+        # Keep a note of what port was assigned
+        self.port = self.serv.socket.getsockname()[1]
+        serv_args = (self.serv, self.serv_evt, self.client_evt)
+        self.thread = threading.Thread(target=debugging_server, args=serv_args)
+        self.thread.start()
+
+        # wait until server thread has assigned a port number
+        self.serv_evt.wait()
+        self.serv_evt.clear()
+
+    def tearDown(self):
+        socket.getfqdn = self.real_getfqdn
+        # indicate that the client is finished
+        self.client_evt.set()
+        # wait for the server thread to terminate
+        self.serv_evt.wait()
+        self.thread.join()
+
+    def test_test_server_supports_extensions(self):
+        smtp = smtplib.SMTP(
+            HOST, self.port, local_hostname='localhost', timeout=3)
+        self.addCleanup(smtp.close)
+        smtp.ehlo()
+        self.assertTrue(smtp.does_esmtp)
+        self.assertTrue(smtp.has_extn('smtputf8'))
+
+    def test_send_unicode_with_SMTPUTF8_via_sendmail(self):
+        m = '¡a test message containing unicode!'.encode('utf-8')
+        smtp = smtplib.SMTP(
+            HOST, self.port, local_hostname='localhost', timeout=3)
+        self.addCleanup(smtp.close)
+        smtp.sendmail('Jőhn', 'Sálly', m,
+                      mail_options=['BODY=8BITMIME', 'SMTPUTF8'])
+        self.assertEqual(self.serv.last_mailfrom, 'Jőhn')
+        self.assertEqual(self.serv.last_rcpttos, ['Sálly'])
+        self.assertEqual(self.serv.last_message, m)
+        self.assertIn('BODY=8BITMIME', self.serv.last_mail_options)
+        self.assertIn('SMTPUTF8', self.serv.last_mail_options)
+        self.assertEqual(self.serv.last_rcpt_options, [])
+
+    def test_send_unicode_with_SMTPUTF8_via_low_level_API(self):
+        m = '¡a test message containing unicode!'.encode('utf-8')
+        smtp = smtplib.SMTP(
+            HOST, self.port, local_hostname='localhost', timeout=3)
+        self.addCleanup(smtp.close)
+        smtp.ehlo()
+        self.assertEqual(
+            smtp.mail('Jő', options=['BODY=8BITMIME', 'SMTPUTF8']),
+            (250, b'OK'))
+        self.assertEqual(smtp.rcpt('János'), (250, b'OK'))
+        self.assertEqual(smtp.data(m), (250, b'OK'))
+        self.assertEqual(self.serv.last_mailfrom, 'Jő')
+        self.assertEqual(self.serv.last_rcpttos, ['János'])
+        self.assertEqual(self.serv.last_message, m)
+        self.assertIn('BODY=8BITMIME', self.serv.last_mail_options)
+        self.assertIn('SMTPUTF8', self.serv.last_mail_options)
+        self.assertEqual(self.serv.last_rcpt_options, [])
+
 
 @support.reap_threads
 def test_main(verbose=None):
diff --git a/Misc/NEWS b/Misc/NEWS
index 95c5332..a00d967 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -47,6 +47,8 @@ Core and Builtins
 Library
 -------
 
+- Issue #22027: smtplib now supports RFC 6531 (SMTPUTF8).
+
 - Issue #23488: Random generator objects now consume 2x less memory on 64-bit.
 
 - Issue #1322: platform.dist() and platform.linux_distribution() functions are