🩹 Workaround invalid Gmail FLAGS response

Gmail allows (or allowed) users to create flags containing some invalid `atom-specials` chars, such as `"]"` or `SP`. Ultimately, this isn't unambiguously parsable. But we can workaround it. Prior to #212, the ResponseParser simply grabbed everything inside the parentheses and then scanned through with a very liberal "flag" regexp. This commit attempts to parse it strictly at first and then falls back to the "quirks mode" parser if that fails. Fixes #241. This also uses `delete_prefix!` to avoid another string allocation when converting flag strings into symbols. ----- Benchmark results: ``` Calculating ------------------------------------- v0.4.7-6-g3f88747 0.2.3 0.3.7 0.4.2 0.4.7 fetch_msg_att_flags_and_uid 53.343k 33.978k 33.562k 34.063k 52.615k i/s - 162.079k times in 3.038415s 4.770048s 4.829303s 4.758267s 3.080481s flags_with_various_flag_types 95.497k 73.549k 73.802k 73.777k 93.892k i/s - 289.950k times in 3.036226s 3.942294s 3.928770s 3.930078s 3.088128s imap.gmail.com allows invalid atom-specials in flags 32.155k 41.518k 38.704k 37.842k ERROR i/s - 100.595k times in 3.128400s 2.422950s 2.599085s 2.658261s 0.000000s list_with_various_flag_capitalizations 42.105k 48.496k 45.108k 31.679k 39.903k i/s - 118.998k times in 2.826218s 2.453753s 2.638075s 3.756420s 2.982145s resp_code_PERMANENTFLAGS_rfc3501_6.3.1_example 56.608k 44.082k 44.029k 46.168k 56.743k i/s - 184.142k times in 3.252913s 4.177284s 4.182247s 3.988496s 3.245177s resp_code_PERMANENTFLAGS_rfc3501_6.3.2_example 62.031k 47.918k 49.941k 56.477k 63.410k i/s - 197.518k times in 3.184188s 4.122017s 3.955044s 3.497315s 3.114928s resp_text_PERMANENTFLAGS_with_various_flag_types 55.224k 41.582k 41.547k 46.085k 54.984k i/s - 178.322k times in 3.229081s 4.288422s 4.292034s 3.869394s 3.243159s rfc3501_7.2.6_FLAGS_response_example 79.586k 61.576k 62.134k 62.640k 79.194k i/s - 268.613k times in 3.375129s 4.362270s 4.323131s 4.288183s 3.391847s Comparison: fetch_msg_att_flags_and_uid v0.4.7-6-g3f88747: 53343.3 i/s 0.4.7: 52614.8 i/s - 1.01x slower 0.4.2: 34062.6 i/s - 1.57x slower 0.2.3: 33978.5 i/s - 1.57x slower 0.3.7: 33561.6 i/s - 1.59x slower flags_with_various_flag_types v0.4.7-6-g3f88747: 95496.8 i/s 0.4.7: 93891.8 i/s - 1.02x slower 0.3.7: 73801.7 i/s - 1.29x slower 0.4.2: 73777.2 i/s - 1.29x slower 0.2.3: 73548.5 i/s - 1.30x slower imap.gmail.com allows invalid atom-specials in flags 0.2.3: 41517.6 i/s 0.3.7: 38704.0 i/s - 1.07x slower 0.4.2: 37842.4 i/s - 1.10x slower v0.4.7-6-g3f88747: 32155.4 i/s - 1.29x slower 0.4.7: 0.0 i/s list_with_various_flag_capitalizations 0.2.3: 48496.3 i/s 0.3.7: 45107.9 i/s - 1.08x slower v0.4.7-6-gd7bf81d-dirty: 42105.0 i/s - 1.15x slower 0.4.7: 39903.5 i/s - 1.22x slower 0.4.2: 31678.6 i/s - 1.53x slower resp_code_PERMANENTFLAGS_rfc3501_6.3.1_example 0.4.7: 56743.3 i/s v0.4.7-6-g3f88747: 56608.3 i/s - 1.00x slower 0.4.2: 46168.3 i/s - 1.23x slower 0.2.3: 44081.8 i/s - 1.29x slower 0.3.7: 44029.4 i/s - 1.29x slower resp_code_PERMANENTFLAGS_rfc3501_6.3.2_example 0.4.7: 63410.1 i/s v0.4.7-6-g3f88747: 62030.9 i/s - 1.02x slower 0.4.2: 56477.0 i/s - 1.12x slower 0.3.7: 49940.8 i/s - 1.27x slower 0.2.3: 47917.8 i/s - 1.32x slower resp_text_PERMANENTFLAGS_with_various_flag_types v0.4.7-6-g3f88747: 55223.8 i/s 0.4.7: 54984.0 i/s - 1.00x slower 0.4.2: 46085.3 i/s - 1.20x slower 0.2.3: 41582.2 i/s - 1.33x slower 0.3.7: 41547.2 i/s - 1.33x slower rfc3501_7.2.6_FLAGS_response_example v0.4.7-6-g3f88747: 79586.0 i/s 0.4.7: 79193.7 i/s - 1.00x slower 0.4.2: 62640.3 i/s - 1.27x slower 0.3.7: 62133.9 i/s - 1.28x slower 0.2.3: 61576.4 i/s - 1.29x slower ```
ruby · Dec 12, 2023 · cc19514 · cc19514
1 parent bb6ced5
commit cc19514
Show file tree

Hide file tree

Showing 2 changed files with 60 additions and 7 deletions.
diff --git a/lib/net/imap/response_parser.rb b/lib/net/imap/response_parser.rb
@@ -231,6 +231,10 @@ module RFC3629
         FLAG_PERM_LIST    = /\G\((#{FLAG_PERM}(?:#{SP}#{FLAG_PERM})*|)\)/ni
         MBX_LIST_FLAGS    = /\G  (#{MBX_FLAG }(?:#{SP}#{MBX_FLAG })*)   /nix
 
+        # Gmail allows SP and "]" in flags.......
+        QUIRKY_FLAG       = Regexp.union(/\\?#{ASTRING_CHARS}/n, "\\*")
+        QUIRKY_FLAGS_LIST = /\G\((   [^)]*   )\)/nx
+
         # RFC3501:
         #   QUOTED-CHAR   = <any TEXT-CHAR except quoted-specials> /
         #                   "\" quoted-specials
@@ -1901,22 +1905,35 @@ def address
 
       # flag-list       = "(" [flag *(SP flag)] ")"
       def flag_list
-        match_re(Patterns::FLAG_LIST, "flag-list")[1]
-          .split(nil)
-          .map! { _1.start_with?("\\") ? _1[1..].capitalize.to_sym : _1 }
+        if (match = accept_re(Patterns::FLAG_LIST))
+          match[1].split(nil)
+            .map! { _1.delete_prefix!("\\") ? _1.capitalize.to_sym : _1 }
+        else
+          quirky__flag_list "flags-list"
+        end
       end
 
       #   "(" [flag-perm *(SP flag-perm)] ")"
       def flag_perm__list
-        match_re(Patterns::FLAG_PERM_LIST, "PERMANENTFLAGS flag-perm list")[1]
-          .split(nil)
-          .map! { _1.start_with?("\\") ? _1[1..].capitalize.to_sym : _1 }
+        if (match = accept_re(Patterns::FLAG_PERM_LIST))
+          match[1].split(nil)
+            .map! { _1.delete_prefix!("\\") ? _1.capitalize.to_sym : _1 }
+        else
+          quirky__flag_list "PERMANENTFLAGS flag-perm list"
+        end
+      end
+
+      def quirky__flag_list(name)
+        match_re(Patterns::QUIRKY_FLAGS_LIST, "quirks mode #{name}")[1]
+          .scan(Patterns::QUIRKY_FLAG)
+          .map! { _1.delete_prefix!("\\") ? _1.capitalize.to_sym : _1 }
       end
 
       # See Patterns::MBX_LIST_FLAGS
       def mbx_list_flags
         match_re(Patterns::MBX_LIST_FLAGS, "mbx-list-flags")[1]
-          .split(nil).map! { _1[1..].capitalize.to_sym }
+          .split(nil)
+          .map! { _1.delete_prefix!("\\"); _1.capitalize.to_sym }
       end
 
       # See https://developers.google.com/gmail/imap/imap-extensions

diff --git a/test/net/imap/fixtures/response_parser/quirky_behaviors.yml b/test/net/imap/fixtures/response_parser/quirky_behaviors.yml
@@ -102,3 +102,39 @@
         (\"[email protected]\" NIL \"numbers.ryan\" \"satterfield.test\")
         (\"[email protected]\" NIL \"keneth_feeney\" \"will-walter.test\"))
         NIL NIL NIL \"<aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa@bbbbbbbbbbbbb.ccccccc.PROD.OUTLOOK.COM>\"))\r\n"
+
+  imap.gmail.com allows invalid atom-specials in flags:
+    comment: |
+      Upstream bug report: https://issuetracker.google.com/issues/315160951
+      net-imap issue: https://github.com/ruby/net-imap/issues/241
+    :response: "* FLAGS (\\Answered \\Flagged \\Draft \\Deleted \\Seen $Forwarded $Junk
+      $MailFlagBit0 $MailFlagBit2 $NotJunk $NotPhishing $Phishing Forwarded
+      JunkRecorded NotJunk OIB-Seen-INBOX OIB-Seen-Unsubscribe
+      OIB-Seen-[Google Mail]/Alle Nachrichten)\r\n"
+    :expected: !ruby/struct:Net::IMAP::UntaggedResponse
+      name: FLAGS
+      data:
+      - :Answered
+      - :Flagged
+      - :Draft
+      - :Deleted
+      - :Seen
+      - "$Forwarded"
+      - "$Junk"
+      - "$MailFlagBit0"
+      - "$MailFlagBit2"
+      - "$NotJunk"
+      - "$NotPhishing"
+      - "$Phishing"
+      - Forwarded
+      - JunkRecorded
+      - NotJunk
+      - OIB-Seen-INBOX
+      - OIB-Seen-Unsubscribe
+      - OIB-Seen-[Google
+      - Mail]/Alle
+      - Nachrichten
+      raw_data: "* FLAGS (\\Answered \\Flagged \\Draft \\Deleted \\Seen $Forwarded
+        $Junk $MailFlagBit0 $MailFlagBit2 $NotJunk $NotPhishing $Phishing Forwarded
+        JunkRecorded NotJunk OIB-Seen-INBOX OIB-Seen-Unsubscribe OIB-Seen-[Google
+        Mail]/Alle Nachrichten)\r\n"