From 317f64f048b6da8a53870ce6a1d63ad458ece95f Mon Sep 17 00:00:00 2001 From: R David Murray Date: Sat, 2 Jan 2016 17:18:34 -0500 Subject: #21815: violate IMAP RFC to be compatible with, e.g., gmail and others, including imaplib's own behavior. I'm applying this only to 3.6 because there's a potential backward compatibility concern: if there are servers that include ] characters in the 'text' portion of their imap responses, this code change could introduce a new bug. Patch by Lita Cho, reviewed by Jessica McKellar, Berker Peksag, Maciej Szulik, silentghost, and me (I fleshed out the comments with the additional info/concerns.) --- Doc/library/imaplib.rst | 11 +++++++++++ Lib/imaplib.py | 10 +++++++++- Lib/test/test_imaplib.py | 49 ++++++++++++++++++++++++++++++++++++++++++++++++ Misc/NEWS | 4 ++++ 4 files changed, 73 insertions(+), 1 deletion(-) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 15b0932..fbb802c 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -500,6 +500,17 @@ An :class:`IMAP4` instance has the following methods: M.store(num, '+FLAGS', '\\Deleted') M.expunge() + .. note:: + + Creating flags containing ']' (for example: "[test]") violates + :rfc:`3501` (the IMAP protocol). However, imaplib has historically + allowed creation of such tags, and popular IMAP servers, such as Gmail, + accept and produce such flags. There are non-Python programs which also + create such tags. Although it is an RFC violation and IMAP clients and + servers are supposed to be strict, imaplib nontheless continues to allow + such tags to be created for backward compatibility reasons, and as of + python 3.5.2/3.6.0, handles them if they are sent from the server, since + this improves real-world compatibility. .. method:: IMAP4.subscribe(mailbox) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 4e8a4bb..a63ba8d 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -111,7 +111,15 @@ InternalDate = re.compile(br'.*INTERNALDATE "' # Literal is no longer used; kept for backward compatibility. Literal = re.compile(br'.*{(?P\d+)}$', re.ASCII) MapCRLF = re.compile(br'\r\n|\r|\n') -Response_code = re.compile(br'\[(?P[A-Z-]+)( (?P[^\]]*))?\]') +# We no longer exclude the ']' character from the data portion of the response +# code, even though it violates the RFC. Popular IMAP servers such as Gmail +# allow flags with ']', and there are programs (including imaplib!) that can +# produce them. The problem with this is if the 'text' portion of the response +# includes a ']' we'll parse the response wrong (which is the point of the RFC +# restriction). However, that seems less likely to be a problem in practice +# than being unable to correctly parse flags that include ']' chars, which +# was reported as a real-world problem in issue #21815. +Response_code = re.compile(br'\[(?P[A-Z-]+)( (?P.*))?\]') Untagged_response = re.compile(br'\* (?P[A-Z-]+)( (?P.*))?') # Untagged_status is no longer used; kept for backward compatibility Untagged_status = re.compile( diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index 07157f5..8e4990b 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -243,6 +243,55 @@ class ThreadedNetworkedTests(unittest.TestCase): client.shutdown() @reap_threads + def test_bracket_flags(self): + + # This violates RFC 3501, which disallows ']' characters in tag names, + # but imaplib has allowed producing such tags forever, other programs + # also produce them (eg: OtherInbox's Organizer app as of 20140716), + # and Gmail, for example, accepts them and produces them. So we + # support them. See issue #21815. + + class BracketFlagHandler(SimpleIMAPHandler): + + def handle(self): + self.flags = ['Answered', 'Flagged', 'Deleted', 'Seen', 'Draft'] + super().handle() + + def cmd_AUTHENTICATE(self, tag, args): + self._send_textline('+') + self.server.response = yield + self._send_tagged(tag, 'OK', 'FAKEAUTH successful') + + def cmd_SELECT(self, tag, args): + flag_msg = ' \\'.join(self.flags) + self._send_line(('* FLAGS (%s)' % flag_msg).encode('ascii')) + self._send_line(b'* 2 EXISTS') + self._send_line(b'* 0 RECENT') + msg = ('* OK [PERMANENTFLAGS %s \\*)] Flags permitted.' + % flag_msg) + self._send_line(msg.encode('ascii')) + self._send_tagged(tag, 'OK', '[READ-WRITE] SELECT completed.') + + def cmd_STORE(self, tag, args): + new_flags = args[2].strip('(').strip(')').split() + self.flags.extend(new_flags) + flags_msg = '(FLAGS (%s))' % ' \\'.join(self.flags) + msg = '* %s FETCH %s' % (args[0], flags_msg) + self._send_line(msg.encode('ascii')) + self._send_tagged(tag, 'OK', 'STORE completed.') + + with self.reaped_pair(BracketFlagHandler) as (server, client): + code, data = client.authenticate('MYAUTH', lambda x: b'fake') + self.assertEqual(code, 'OK') + self.assertEqual(server.response, b'ZmFrZQ==\r\n') + client.select('test') + typ, [data] = client.store(b'1', "+FLAGS", "[test]") + self.assertIn(b'[test]', data) + client.select('test') + typ, [data] = client.response('PERMANENTFLAGS') + self.assertIn(b'[test]', data) + + @reap_threads def test_issue5949(self): class EOFHandler(socketserver.StreamRequestHandler): diff --git a/Misc/NEWS b/Misc/NEWS index 83c07f5..7277838 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -128,6 +128,10 @@ Core and Builtins Library ------- +- Issue #21815: Accept ] characters in the data portion of imap responses, + in order to handle the flags with square brackets accepted and produced + by servers such as gmail. + - Issue #25447: fileinput now uses sys.stdin as-is if it does not have a buffer attribute (restores backward compatibility). -- cgit v0.12