#24218: Add SMTPUTF8 support to send_message.

Reviewed by Maciej Szulik.

4 files changed, 86 insertions, 8 deletions

diff --git a/Doc/library/smtplib.rst b/Doc/library/smtplib.rst
index 133fa56..25279f2 100644
--- a/Doc/library/smtplib.rst
+++ b/Doc/library/smtplib.rst
@@ -467,7 +467,7 @@ An :class:`SMTP` instance has the following methods:
 
    If *from_addr* is ``None`` or *to_addrs* is ``None``, ``send_message`` fills
    those arguments with addresses extracted from the headers of *msg* as
-   specified in :rfc:`2822`\: *from_addr* is set to the :mailheader:`Sender`
+   specified in :rfc:`5322`\: *from_addr* is set to the :mailheader:`Sender`
    field if it is present, and otherwise to the :mailheader:`From` field.
    *to_adresses* combines the values (if any) of the :mailheader:`To`,
    :mailheader:`Cc`, and :mailheader:`Bcc` fields from *msg*.  If exactly one
@@ -482,10 +482,18 @@ An :class:`SMTP` instance has the following methods:
    calls :meth:`sendmail` to transmit the resulting message.  Regardless of the
    values of *from_addr* and *to_addrs*, ``send_message`` does not transmit any
    :mailheader:`Bcc` or :mailheader:`Resent-Bcc` headers that may appear
-   in *msg*.
+   in *msg*.  If any of the addresses in *from_addr* and *to_addrs* contain
+   non-ASCII characters and the server does not advertise ``SMTPUTF8`` support,
+   an :exc:`SMTPNotSupported` error is raised.  Otherwise the ``Message`` is
+   serialized with a clone of its :mod:`~email.policy` with the
+   :attr:`~email.policy.EmailPolicy.utf8` attribute set to ``True``, and
+   ``SMTPUTF8`` and ``BODY=8BITMIME`` are added to *mail_options*.
 
    .. versionadded:: 3.2
 
+   .. versionadded:: 3.5
+      Support for internationalized addresses (``SMTPUTF8``).
+
 
 .. method:: SMTP.quit()
 
diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst
index 1f8d90f..762ad22 100644
--- a/Doc/whatsnew/3.5.rst
+++ b/Doc/whatsnew/3.5.rst
@@ -557,8 +557,10 @@ smtplib
   :class:`smtplib.SMTP`.  (Contributed by Gavin Chappell and Maciej Szulik in
   :issue:`16914`.)
 
-* :mod:`smtplib` now support :rfc:`6531` (SMTPUTF8).  (Contributed by
-  Milan Oberkirch and R. David Murray in :issue:`22027`.)
+* :mod:`smtplib` now supports :rfc:`6531` (SMTPUTF8) in both the
+  :meth:`~smtplib.SMTP.sendmail` and :meth:`~smtplib.SMTP.send_message`
+  commands.  (Contributed by Milan Oberkirch and R. David Murray in
+  :issue:`22027`.)
 
 sndhdr
 ------
diff --git a/Lib/smtplib.py b/Lib/smtplib.py
index 6895bed..71ccd2a 100755
--- a/Lib/smtplib.py
+++ b/Lib/smtplib.py
@@ -872,7 +872,13 @@ class SMTP:
         to_addr, any Bcc field (or Resent-Bcc field, when the Message is a
         resent) of the Message object won't be transmitted.  The Message
         object is then serialized using email.generator.BytesGenerator and
-        sendmail is called to transmit the message.
+        sendmail is called to transmit the message.  If the sender or any of
+        the recipient addresses contain non-ASCII and the server advertises the
+        SMTPUTF8 capability, the policy is cloned with utf8 set to True for the
+        serialization, and SMTPUTF8 and BODY=8BITMIME are asserted on the send.
+        If the server does not support SMTPUTF8, an SMPTNotSupported error is
+        raised.  Otherwise the generator is called without modifying the
+        policy.
 
         """
         # 'Resent-Date' is a mandatory field if the Message is resent (RFC 2822
@@ -885,6 +891,7 @@ class SMTP:
         # option allowing the user to enable the heuristics.  (It should be
         # possible to guess correctly almost all of the time.)
 
+        self.ehlo_or_helo_if_needed()
         resent = msg.get_all('Resent-Date')
         if resent is None:
             header_prefix = ''
@@ -900,14 +907,30 @@ class SMTP:
         if to_addrs is None:
             addr_fields = [f for f in (msg[header_prefix + 'To'],
                                        msg[header_prefix + 'Bcc'],
-                                       msg[header_prefix + 'Cc']) if f is not None]
+                                       msg[header_prefix + 'Cc'])
+                           if f is not None]
             to_addrs = [a[1] for a in email.utils.getaddresses(addr_fields)]
         # Make a local copy so we can delete the bcc headers.
         msg_copy = copy.copy(msg)
         del msg_copy['Bcc']
         del msg_copy['Resent-Bcc']
+        international = False
+        try:
+            ''.join([from_addr, *to_addrs]).encode('ascii')
+        except UnicodeEncodeError:
+            if not self.has_extn('smtputf8'):
+                raise SMTPNotSupportedError(
+                    "One or more source or delivery addresses require"
+                    " internationalized email support, but the server"
+                    " does not advertise the required SMTPUTF8 capability")
+            international = True
         with io.BytesIO() as bytesmsg:
-            g = email.generator.BytesGenerator(bytesmsg)
+            if international:
+                g = email.generator.BytesGenerator(
+                    bytesmsg, policy=msg.policy.clone(utf8=True))
+                mail_options += ['SMTPUTF8', 'BODY=8BITMIME']
+            else:
+                g = email.generator.BytesGenerator(bytesmsg)
             g.flatten(msg_copy, linesep='\r\n')
             flatmsg = bytesmsg.getvalue()
         return self.sendmail(from_addr, to_addrs, flatmsg, mail_options,
diff --git a/Lib/test/test_smtplib.py b/Lib/test/test_smtplib.py
index e496371..e66ae9b 100644
--- a/Lib/test/test_smtplib.py
+++ b/Lib/test/test_smtplib.py
@@ -1,5 +1,6 @@
 import asyncore
 import email.mime.text
+from email.message import EmailMessage
 import email.utils
 import socket
 import smtpd
@@ -10,7 +11,7 @@ import sys
 import time
 import select
 import errno
-import base64
+import textwrap
 
 import unittest
 from test import support, mock_socket
@@ -1029,6 +1030,8 @@ class SimSMTPUTF8Server(SimSMTPServer):
 @unittest.skipUnless(threading, 'Threading required for this test.')
 class SMTPUTF8SimTests(unittest.TestCase):
 
+    maxDiff = None
+
     def setUp(self):
         self.real_getfqdn = socket.getfqdn
         socket.getfqdn = mock_socket.getfqdn
@@ -1096,6 +1099,48 @@ class SMTPUTF8SimTests(unittest.TestCase):
         self.assertIn('SMTPUTF8', self.serv.last_mail_options)
         self.assertEqual(self.serv.last_rcpt_options, [])
 
+    def test_send_message_uses_smtputf8_if_addrs_non_ascii(self):
+        msg = EmailMessage()
+        msg['From'] = "Páolo <főo@bar.com>"
+        msg['To'] = 'Dinsdale'
+        msg['Subject'] = 'Nudge nudge, wink, wink \u1F609'
+        # XXX I don't know why I need two \n's here, but this is an existing
+        # bug (if it is one) and not a problem with the new functionality.
+        msg.set_content("oh là là, know what I mean, know what I mean?\n\n")
+        # XXX smtpd converts received /r/n to /n, so we can't easily test that
+        # we are successfully sending /r/n :(.
+        expected = textwrap.dedent("""\
+            From: Páolo <főo@bar.com>
+            To: Dinsdale
+            Subject: Nudge nudge, wink, wink \u1F609
+            Content-Type: text/plain; charset="utf-8"
+            Content-Transfer-Encoding: 8bit
+            MIME-Version: 1.0
+
+            oh là là, know what I mean, know what I mean?
+            """)
+        smtp = smtplib.SMTP(
+            HOST, self.port, local_hostname='localhost', timeout=3)
+        self.addCleanup(smtp.close)
+        self.assertEqual(smtp.send_message(msg), {})
+        self.assertEqual(self.serv.last_mailfrom, 'főo@bar.com')
+        self.assertEqual(self.serv.last_rcpttos, ['Dinsdale'])
+        self.assertEqual(self.serv.last_message.decode(), expected)
+        self.assertIn('BODY=8BITMIME', self.serv.last_mail_options)
+        self.assertIn('SMTPUTF8', self.serv.last_mail_options)
+        self.assertEqual(self.serv.last_rcpt_options, [])
+
+    def test_send_message_error_on_non_ascii_addrs_if_no_smtputf8(self):
+        msg = EmailMessage()
+        msg['From'] = "Páolo <főo@bar.com>"
+        msg['To'] = 'Dinsdale'
+        msg['Subject'] = 'Nudge nudge, wink, wink \u1F609'
+        smtp = smtplib.SMTP(
+            HOST, self.port, local_hostname='localhost', timeout=3)
+        self.addCleanup(smtp.close)
+        self.assertRaises(smtplib.SMTPNotSupportedError,
+                          smtp.send_message(msg))
+
 
 @support.reap_threads
 def test_main(verbose=None):