#21815: violate IMAP RFC to be compatible with, e.g., gmail

and others, including imaplib's own behavior.  I'm applying this only to 3.6
because there's a potential backward compatibility concern: if there are
servers that include ] characters in the 'text' portion of their imap
responses, this code change could introduce a new bug.

Patch by Lita Cho, reviewed by Jessica McKellar, Berker Peksag, Maciej Szulik,
silentghost, and me (I fleshed out the comments with the additional
info/concerns.)

4 files changed, 73 insertions, 1 deletions

diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst
index 15b0932973..fbb802ce2f 100644
--- a/Doc/library/imaplib.rst
+++ b/Doc/library/imaplib.rst
@@ -500,6 +500,17 @@ An :class:`IMAP4` instance has the following methods:
          M.store(num, '+FLAGS', '\\Deleted')
       M.expunge()
 
+   .. note::
+
+      Creating flags containing ']' (for example: "[test]") violates
+      :rfc:`3501` (the IMAP protocol).  However, imaplib has historically
+      allowed creation of such tags, and popular IMAP servers, such as Gmail,
+      accept and produce such flags.  There are non-Python programs which also
+      create such tags.  Although it is an RFC violation and IMAP clients and
+      servers are supposed to be strict, imaplib nontheless continues to allow
+      such tags to be created for backward compatibility reasons, and as of
+      python 3.5.2/3.6.0, handles them if they are sent from the server, since
+      this improves real-world compatibility.
 
 .. method:: IMAP4.subscribe(mailbox)
 
diff --git a/Lib/imaplib.py b/Lib/imaplib.py
index 4e8a4bb6fa..a63ba8dbe0 100644
--- a/Lib/imaplib.py
+++ b/Lib/imaplib.py
@@ -111,7 +111,15 @@ InternalDate = re.compile(br'.*INTERNALDATE "'
 # Literal is no longer used; kept for backward compatibility.
 Literal = re.compile(br'.*{(?P<size>\d+)}$', re.ASCII)
 MapCRLF = re.compile(br'\r\n|\r|\n')
-Response_code = re.compile(br'\[(?P<type>[A-Z-]+)( (?P<data>[^\]]*))?\]')
+# We no longer exclude the ']' character from the data portion of the response
+# code, even though it violates the RFC.  Popular IMAP servers such as Gmail
+# allow flags with ']', and there are programs (including imaplib!) that can
+# produce them.  The problem with this is if the 'text' portion of the response
+# includes a ']' we'll parse the response wrong (which is the point of the RFC
+# restriction).  However, that seems less likely to be a problem in practice
+# than being unable to correctly parse flags that include ']' chars, which
+# was reported as a real-world problem in issue #21815.
+Response_code = re.compile(br'\[(?P<type>[A-Z-]+)( (?P<data>.*))?\]')
 Untagged_response = re.compile(br'\* (?P<type>[A-Z-]+)( (?P<data>.*))?')
 # Untagged_status is no longer used; kept for backward compatibility
 Untagged_status = re.compile(
diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py
index 07157f52d6..8e4990b3cf 100644
--- a/Lib/test/test_imaplib.py
+++ b/Lib/test/test_imaplib.py
@@ -243,6 +243,55 @@ class ThreadedNetworkedTests(unittest.TestCase):
             client.shutdown()
 
     @reap_threads
+    def test_bracket_flags(self):
+
+        # This violates RFC 3501, which disallows ']' characters in tag names,
+        # but imaplib has allowed producing such tags forever, other programs
+        # also produce them (eg: OtherInbox's Organizer app as of 20140716),
+        # and Gmail, for example, accepts them and produces them.  So we
+        # support them.  See issue #21815.
+
+        class BracketFlagHandler(SimpleIMAPHandler):
+
+            def handle(self):
+                self.flags = ['Answered', 'Flagged', 'Deleted', 'Seen', 'Draft']
+                super().handle()
+
+            def cmd_AUTHENTICATE(self, tag, args):
+                self._send_textline('+')
+                self.server.response = yield
+                self._send_tagged(tag, 'OK', 'FAKEAUTH successful')
+
+            def cmd_SELECT(self, tag, args):
+                flag_msg = ' \\'.join(self.flags)
+                self._send_line(('* FLAGS (%s)' % flag_msg).encode('ascii'))
+                self._send_line(b'* 2 EXISTS')
+                self._send_line(b'* 0 RECENT')
+                msg = ('* OK [PERMANENTFLAGS %s \\*)] Flags permitted.'
+                        % flag_msg)
+                self._send_line(msg.encode('ascii'))
+                self._send_tagged(tag, 'OK', '[READ-WRITE] SELECT completed.')
+
+            def cmd_STORE(self, tag, args):
+                new_flags = args[2].strip('(').strip(')').split()
+                self.flags.extend(new_flags)
+                flags_msg = '(FLAGS (%s))' % ' \\'.join(self.flags)
+                msg = '* %s FETCH %s' % (args[0], flags_msg)
+                self._send_line(msg.encode('ascii'))
+                self._send_tagged(tag, 'OK', 'STORE completed.')
+
+        with self.reaped_pair(BracketFlagHandler) as (server, client):
+            code, data = client.authenticate('MYAUTH', lambda x: b'fake')
+            self.assertEqual(code, 'OK')
+            self.assertEqual(server.response, b'ZmFrZQ==\r\n')
+            client.select('test')
+            typ, [data] = client.store(b'1', "+FLAGS", "[test]")
+            self.assertIn(b'[test]', data)
+            client.select('test')
+            typ, [data] = client.response('PERMANENTFLAGS')
+            self.assertIn(b'[test]', data)
+
+    @reap_threads
     def test_issue5949(self):
 
         class EOFHandler(socketserver.StreamRequestHandler):
diff --git a/Misc/NEWS b/Misc/NEWS
index 83c07f5a2e..7277838f4e 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -128,6 +128,10 @@ Core and Builtins
 Library
 -------
 
+- Issue #21815: Accept ] characters in the data portion of imap responses,
+  in order to handle the flags with square brackets accepted and produced
+  by servers such as gmail.
+
 - Issue #25447: fileinput now uses sys.stdin as-is if it does not have a
   buffer attribute (restores backward compatibility).