✨ Add UTF-8 support for both quoted and text

nevans · nevans · commit bf71d85f4fd3 · 2023-02-12T00:31:15.000-05:00
The parser update supports both RFC6855 (UTF8=ALLOW, UTF8=ONLY) and the
UTF8 requirements of IMAP4rev2 (resp-text).

Also updated #enable documentation and method signature:
* document `UTF8=ACCEPT` as "supported"
* use `*rest` args =&gt; flatten =&gt; map(aliases) =&gt; uniq
* add `:utf8` as an alias for `UTF8=ACCEPT`
diff --git a/benchmarks/generate_parser_benchmarks b/benchmarks/generate_parser_benchmarks
@@ -28,7 +28,8 @@ init = <<RUBY
   require "net/imap"
 
   def load_response(file, name)
-    YAML.unsafe_load_file(file).dig(:tests, name, :response) \\
+    YAML.unsafe_load_file(file).dig(:tests, name, :response)
+      .force_encoding "ASCII-8BIT" \\
       or abort "ERRORO: missing %p fixture data in %p" % [name, file]
   end
 
diff --git a/benchmarks/parser.yml b/benchmarks/parser.yml
@@ -4,7 +4,8 @@ prelude: |2
     require "net/imap"
 
     def load_response(file, name)
-      YAML.unsafe_load_file(file).dig(:tests, name, :response) \
+      YAML.unsafe_load_file(file).dig(:tests, name, :response)
+        .force_encoding "ASCII-8BIT" \
         or abort "ERRORO: missing %p fixture data in %p" % [name, file]
     end
 
@@ -560,6 +561,16 @@ benchmark:
       response = load_response("../test/net/imap/fixtures/response_parser/thread_responses.yml",
                                "thread_rfc5256_example5")
   script: parser.parse(response)
+- name: utf8_in_list_mailbox
+  prelude: |2
+      response = load_response("../test/net/imap/fixtures/response_parser/utf8_responses.yml",
+                               "test_utf8_in_list_mailbox")
+  script: parser.parse(response)
+- name: utf8_in_resp_text
+  prelude: |2
+      response = load_response("../test/net/imap/fixtures/response_parser/utf8_responses.yml",
+                               "test_utf8_in_resp_text")
+  script: parser.parse(response)
 - name: xlist_inbox
   prelude: |2
       response = load_response("../test/net/imap/fixtures/response_parser/list_responses.yml",
diff --git a/lib/net/imap.rb b/lib/net/imap.rb
@@ -489,12 +489,9 @@ module Net
   # - #move, #uid_move: Moves the specified messages to the end of the
   #   specified destination mailbox, expunging them from the current mailbox.
   #
-  #--
-  # ==== RFC6855: UTF8=ACCEPT
-  # TODO...
-  # ==== RFC6855: UTF8=ONLY
-  # TODO...
-  #++
+  # ==== RFC6855: <tt>UTF8=ACCEPT</tt>, <tt>UTF8=ONLY</tt>
+  #
+  # - See #enable for information about support foi UTF-8 string encoding.
   #
   #--
   # ==== RFC7888: <tt>LITERAL+</tt>, +LITERAL-+
@@ -679,6 +676,11 @@ module Net
   #   Gulbrandsen, A. and N. Freed, Ed., "Internet Message Access Protocol
   #   (\IMAP) - MOVE Extension", RFC 6851, DOI 10.17487/RFC6851, January 2013,
   #   <https://www.rfc-editor.org/info/rfc6851>.
+  # [UTF8=ACCEPT[https://tools.ietf.org/html/rfc6855]]::
+  # [UTF8=ONLY[https://tools.ietf.org/html/rfc6855]]::
+  #   Resnick, P., Ed., Newman, C., Ed., and S. Shen, Ed.,
+  #   "IMAP Support for UTF-8", RFC 6855, DOI 10.17487/RFC6855, March 2013,
+  #   <https://www.rfc-editor.org/info/rfc6855>.
   #
   # === IANA registries
   #
@@ -705,6 +707,12 @@ module Net
   class IMAP < Protocol
     VERSION = "0.3.4"
 
+    # Aliases for supported capabilities, to be used with the #enable command.
+    ENABLE_ALIASES = {
+      utf8:          "UTF8=ACCEPT",
+      "UTF8=ONLY" => "UTF8=ACCEPT",
+    }.freeze
+
     autoload :SASL,       File.expand_path("imap/sasl",       __dir__)
     autoload :StringPrep, File.expand_path("imap/stringprep", __dir__)
 
@@ -812,12 +820,14 @@ def disconnected?
     # Capability requirements—other than +IMAP4rev1+—are listed in the
     # documentation for each command method.
     #
+    # Related: #enable
+    #
     # ===== Basic IMAP4rev1 capabilities
     #
     # All IMAP4rev1 servers must include +IMAP4rev1+ in their capabilities list.
     # All IMAP4rev1 servers must _implement_ the +STARTTLS+,
     # <tt>AUTH=PLAIN</tt>, and +LOGINDISABLED+ capabilities, and clients must
-    # respect their presence or absence.  See the capabilites requirements on
+    # respect their presence or absence.  See the capabilities requirements on
     # #starttls, #login, and #authenticate.
     #
     # ===== Using IMAP4rev1 extensions
@@ -1886,26 +1896,84 @@ def uid_thread(algorithm, search_keys, charset)
 
     # Sends an {ENABLE command [RFC5161 §3.2]}[https://www.rfc-editor.org/rfc/rfc5161#section-3.1]
     # {[IMAP4rev2 §6.3.1]}[https://www.rfc-editor.org/rfc/rfc9051#section-6.3.1]
-    # to enable the specified extenstions, which may be either an
-    # array or a string.  Returns a list of the extensions that were enabled.
-    #
-    # Some of the extensions that use ENABLE permit the server to send
-    # syntax that this class cannot parse. Caution is advised.
+    # to enable the specified server +capabilities+.  Each capability may be an
+    # array, string, or symbol.  Returns a list of the capabilities that were
+    # enabled.
     #
     # The +ENABLE+ command is only valid in the _authenticated_ state, before
     # any mailbox is selected.
     #
+    # Related: #capability
+    #
     # ===== Capabilities
     #
-    # The server's capabilities must include +ENABLE+
-    # [RFC5161[https://tools.ietf.org/html/rfc5161]] or IMAP4REV2
-    # [RFC9051[https://tools.ietf.org/html/rfc9051]].
+    # The server's capabilities must include
+    # +ENABLE+ [RFC5161[https://tools.ietf.org/html/rfc5161]]
+    # or +IMAP4REV2+ [RFC9051[https://tools.ietf.org/html/rfc9051]].
     #
     # Additionally, the server capabilities must include a capability matching
     # each enabled extension (usually the same name as the enabled extension).
-    def enable(extensions)
+    # The following capabilities may be enabled:
+    #
+    # [+:utf8+ --- an alias for <tt>"UTF8=ACCEPT"</tt>]
+    #
+    #   In a future release, <tt>enable(:utf8)</tt> will enable either
+    #   <tt>"UTF8=ACCEPT"</tt> or <tt>"IMAP4rev2"</tt>, depending on server
+    #   capabilities.
+    #
+    # [<tt>"UTF8=ACCEPT"</tt> [RFC6855[https://tools.ietf.org/html/rfc6855]]]
+    #
+    #   The server's capabilities must include <tt>UTF8=ACCEPT</tt> _or_
+    #   <tt>UTF8=ONLY</tt>.
+    #
+    #   This allows the server to send strings encoded as UTF-8 which might
+    #   otherwise need to use a 7-bit encoding, such as {modified
+    #   UTF-7}[::decode_utf7] for mailbox names, or RFC2047 encoded-words for
+    #   message headers.
+    #
+    #   *Note:* For now, strings with 8-bit characters are still _sent_ using
+    #   "literal" syntax.  A future update will change how commands send UTF-8
+    #   strings when <tt>UTF8=ACCEPT</tt> is enabled.  This update should be
+    #   backward-compatible.
+    #
+    #   *Note:* <em>A future update may set string encodings slightly
+    #   differently</em>, e.g: "US-ASCII" when UTF-8 is not enabled, and "UTF-8"
+    #   when it is.  Currently, the encoding of strings sent as "quoted" or
+    #   "text" will _always_ be "UTF-8", even when a 7-bit encoding is used
+    #   (e.g. UTF-7, encoded-words, quoted-printable, base64).  And currently,
+    #   string "literals" sent by the server will always have an "ASCII-8BIT"
+    #   (binary) encoding, even if they must contain UTF-8 data---although a
+    #   server _should_ use "quoted" strings once <tt>UTF8=ACCEPT</tt> is
+    #   enabled.
+    #
+    # [<tt>"UTF8=ONLY"</tt> [RFC6855[https://tools.ietf.org/html/rfc6855]]]
+    #
+    #   A server that reports the <tt>UTF8=ONLY</tt> #capability _requires_ that
+    #   the client <tt>enable("UTF8=ACCEPT")</tt> before any mailboxes may be
+    #   selected.  For convenience, <tt>enable("UTF8=ONLY")</tt> is aliased to
+    #   <tt>enable("UTF8=ACCEPT")</tt>.
+    #
+    # ===== Unsupported capabilities
+    #
+    # *Note:* Some extensions that use ENABLE permit the server to send syntax
+    # that Net::IMAP cannot parse, which may raise an exception and disconnect.
+    # Some extensions may work, but the support may be incomplete, untested, or
+    # experimental.
+    #
+    # Until a capability is documented here as supported, enabling it may result
+    # in undocumented behavior and a future release may update with incompatible
+    # behavior <em>without warning or deprecation</em>.
+    #
+    # <em>Caution is advised.</em>
+    #
+    def enable(*capabilities)
+      capabilities = capabilities
+        .flatten
+        .map {|e| ENABLE_ALIASES[e] || e }
+        .uniq
+        .join(' ')
       synchronize do
-        send_command("ENABLE #{[extensions].flatten.join(' ')}")
+        send_command("ENABLE #{capabilities}")
         return @responses.delete("ENABLED")[-1]
       end
     end
diff --git a/lib/net/imap/response_parser.rb b/lib/net/imap/response_parser.rb
@@ -94,8 +94,44 @@ module RFC5234
           SP        = / /n
         end
 
+        # UTF-8, a transformation format of ISO 10646
+        # >>>
+        #   UTF8-1      = %x00-7F
+        #   UTF8-tail   = %x80-BF
+        #   UTF8-2      = %xC2-DF UTF8-tail
+        #   UTF8-3      = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) /
+        #                 %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail )
+        #   UTF8-4      = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) /
+        #                 %xF4 %x80-8F 2( UTF8-tail )
+        #   UTF8-char   = UTF8-1 / UTF8-2 / UTF8-3 / UTF8-4
+        #   UTF8-octets = *( UTF8-char )
+        #
+        # n.b. String * Integer is used for repetition, rather than /x{3}/,
+        # because ruby 3.2's linear-time cache-based optimization doesn't work
+        # with "bounded or fixed times repetition nesting in another repetition
+        # (e.g. /(a{2,3})*/). It is an implementation issue entirely, but we
+        # believe it is hard to support this case correctly."
+        # See https://bugs.ruby-lang.org/issues/19104
+        module RFC3629
+          UTF8_1      = /[\x00-\x7f]/n # aka ASCII 7bit
+          UTF8_TAIL   = /[\x80-\xBF]/n
+          UTF8_2      = /[\xC2-\xDF]#{UTF8_TAIL}/n
+          UTF8_3      = Regexp.union(/\xE0[\xA0-\xBF]#{UTF8_TAIL}/n,
+                                     /\xED[\x80-\x9F]#{UTF8_TAIL}/n,
+                                     /[\xE1-\xEC]#{    UTF8_TAIL.source * 2}/n,
+                                     /[\xEE-\xEF]#{    UTF8_TAIL.source * 2}/n)
+          UTF8_4      = Regexp.union(/[\xF1-\xF3]#{    UTF8_TAIL.source * 3}/n,
+                                     /\xF0[\x90-\xBF]#{UTF8_TAIL.source * 2}/n,
+                                     /\xF4[\x80-\x8F]#{UTF8_TAIL.source * 2}/n)
+          UTF8_CHAR   = Regexp.union(UTF8_1, UTF8_2, UTF8_3, UTF8_4)
+          UTF8_OCTETS = /#{UTF8_CHAR}*/n
+        end
+
         include RFC5234
+        include RFC3629
 
+        # quoted-specials = DQUOTE / "\"
+        QUOTED_SPECIALS   = /["\\]/n
         # resp-specials   = "]"
         RESP_SPECIALS     = /[\]]/n
 
@@ -106,9 +142,44 @@ module RFC5234
         CODE_TEXT_CHAR    = TEXT_CHAR - RESP_SPECIALS
         CODE_TEXT         = /#{CODE_TEXT_CHAR}+/n
 
+        # RFC3501:
+        #   QUOTED-CHAR   = <any TEXT-CHAR except quoted-specials> /
+        #                   "\" quoted-specials
+        # RFC9051:
+        #   QUOTED-CHAR   = <any TEXT-CHAR except quoted-specials> /
+        #                   "\" quoted-specials / UTF8-2 / UTF8-3 / UTF8-4
+        # RFC3501 & RFC9051:
+        #   quoted          = DQUOTE *QUOTED-CHAR DQUOTE
+        QUOTED_CHAR_safe  = TEXT_CHAR - QUOTED_SPECIALS
+        QUOTED_CHAR_esc   = /\\#{QUOTED_SPECIALS}/n
+        QUOTED_CHAR_rev1  = Regexp.union(QUOTED_CHAR_safe, QUOTED_CHAR_esc)
+        QUOTED_CHAR_rev2  = Regexp.union(QUOTED_CHAR_rev1,
+                                         UTF8_2, UTF8_3, UTF8_4)
+        QUOTED_rev1       = /"(#{QUOTED_CHAR_rev1}*)"/n
+        QUOTED_rev2       = /"(#{QUOTED_CHAR_rev2}*)"/n
+
         # RFC3501:
         #   text          = 1*TEXT-CHAR
+        # RFC9051:
+        #   text          = 1*(TEXT-CHAR / UTF8-2 / UTF8-3 / UTF8-4)
+        #                     ; Non-ASCII text can only be returned
+        #                     ; after ENABLE IMAP4rev2 command
         TEXT_rev1         = /#{TEXT_CHAR}+/
+        TEXT_rev2         = /#{Regexp.union TEXT_CHAR, UTF8_2, UTF8_3, UTF8_4}+/
+
+        module_function
+
+        def unescape_quoted!(quoted)
+          quoted
+            &.gsub!(/\\(#{QUOTED_SPECIALS})/n, "\\1")
+            &.force_encoding("UTF-8")
+        end
+
+        def unescape_quoted(quoted)
+          quoted
+            &.gsub(/\\(#{QUOTED_SPECIALS})/n, "\\1")
+            &.force_encoding("UTF-8")
+        end
 
       end
 
@@ -118,7 +189,7 @@ module RFC5234
 (?# 2:  NIL     )(NIL)(?=[\x80-\xff(){ \x00-\x1f\x7f%*"\\\[\]+])|\
 (?# 3:  NUMBER  )(\d+)(?=[\x80-\xff(){ \x00-\x1f\x7f%*"\\\[\]+])|\
 (?# 4:  ATOM    )([^\x80-\xff(){ \x00-\x1f\x7f%*"\\\[\]+]+)|\
-(?# 5:  QUOTED  )"((?:[^\x00\r\n"\\]|\\["\\])*)"|\
+(?# 5:  QUOTED  )#{Patterns::QUOTED_rev2}|\
 (?# 6:  LPAR    )(\()|\
 (?# 7:  RPAR    )(\))|\
 (?# 8:  BSLASH  )(\\)|\
@@ -136,13 +207,13 @@ module RFC5234
 (?# 1:  SPACE   )( )|\
 (?# 2:  NIL     )(NIL)|\
 (?# 3:  NUMBER  )(\d+)|\
-(?# 4:  QUOTED  )"((?:[^\x00\r\n"\\]|\\["\\])*)"|\
+(?# 4:  QUOTED  )#{Patterns::QUOTED_rev2}|\
 (?# 5:  LITERAL )\{(\d+)\}\r\n|\
 (?# 6:  LPAR    )(\()|\
 (?# 7:  RPAR    )(\)))/ni
 
       # text, after 'resp-text-code "]"'
-      TEXT_REGEXP = /\G(#{Patterns::TEXT_rev1})/n
+      TEXT_REGEXP = /\G(#{Patterns::TEXT_rev2})/n
 
       # resp-text-code, after 'atom SP'
       CTEXT_REGEXP = /\G(#{Patterns::CODE_TEXT})/n
@@ -1190,15 +1261,20 @@ def namespace_response_extensions
         data
       end
 
-      # text            = 1*TEXT-CHAR
-      # TEXT-CHAR       = <any CHAR except CR and LF>
+      #   TEXT-CHAR       = <any CHAR except CR and LF>
+      # RFC3501:
+      #   text            = 1*TEXT-CHAR
+      # RFC9051:
+      #   text            = 1*(TEXT-CHAR / UTF8-2 / UTF8-3 / UTF8-4)
+      #                     ; Non-ASCII text can only be returned
+      #                     ; after ENABLE IMAP4rev2 command
       def text
-        match_re(TEXT_REGEXP, "text")[0]
+        match_re(TEXT_REGEXP, "text")[0].force_encoding("UTF-8")
       end
 
       # an "accept" versiun of #text
       def text?
-        accept_re(TEXT_REGEXP)&.[](0)
+        accept_re(TEXT_REGEXP)&.[](0)&.force_encoding("UTF-8")
       end
 
       # RFC3501:
@@ -1349,9 +1425,7 @@ def address
           mailbox = $3
           host = $4
           for s in [name, route, mailbox, host]
-            if s
-              s.gsub!(/\\(["\\])/n, "\\1")
-            end
+            Patterns.unescape_quoted! s
           end
         else
           name = nstring
@@ -1533,8 +1607,7 @@ def next_token
             elsif $4
               return Token.new(T_ATOM, $+)
             elsif $5
-              return Token.new(T_QUOTED,
-                               $+.gsub(/\\(["\\])/n, "\\1"))
+              return Token.new(T_QUOTED, Patterns.unescape_quoted($+))
             elsif $6
               return Token.new(T_LPAR, $+)
             elsif $7
@@ -1577,8 +1650,7 @@ def next_token
             elsif $3
               return Token.new(T_NUMBER, $+)
             elsif $4
-              return Token.new(T_QUOTED,
-                               $+.gsub(/\\(["\\])/n, "\\1"))
+              return Token.new(T_QUOTED, Patterns.unescape_quoted($+))
             elsif $5
               len = $+.to_i
               val = @str[@pos, len]
diff --git a/test/net/imap/fixtures/response_parser/utf8_responses.yml b/test/net/imap/fixtures/response_parser/utf8_responses.yml
@@ -0,0 +1,23 @@
+
+---
+:tests:
+
+  test_utf8_in_list_mailbox:
+    :response: "* LIST () \"/\" \"☃️&☺️\"\r\n"
+    :expected: !ruby/struct:Net::IMAP::UntaggedResponse
+      name: LIST
+      data: !ruby/struct:Net::IMAP::MailboxList
+        attr: []
+        delim: "/"
+        name: "☃️&☺️"
+      raw_data: !binary |-
+        KiBMSVNUICgpICIvIiAi4piD77iPJuKYuu+4jyINCg==
+
+  test_utf8_in_resp_text:
+    :response: "* OK 𝖀𝖓𝖎𝖈𝖔𝖉𝖊 «α-ω» ほげ ふが ʇɐɥʍ\r\n"
+    :expected: !ruby/struct:Net::IMAP::UntaggedResponse
+      name: OK
+      data: !ruby/struct:Net::IMAP::ResponseText
+        text: "𝖀𝖓𝖎𝖈𝖔𝖉𝖊 «α-ω» ほげ ふが ʇɐɥʍ"
+      raw_data: !binary |-
+        KiBPSyDwnZaA8J2Wk/Cdlo7wnZaI8J2WlPCdlonwnZaKIMKrzrEtz4nCuyDjgbvjgZIg44G144 GMIMqHyZDJpcqNDQo=
diff --git a/test/net/imap/net_imap_test_helpers.rb b/test/net/imap/net_imap_test_helpers.rb
@@ -40,7 +40,7 @@ def generate_tests_from(fixture_data: nil, fixture_file: nil)
         case type
 
         when :parser_assert_equal
-          response = test.fetch(:response)
+          response = test.fetch(:response).force_encoding "ASCII-8BIT"
           expected = test.fetch(:expected)
 
           define_method name do
diff --git a/test/net/imap/test_imap.rb b/test/net/imap/test_imap.rb
diff --git a/test/net/imap/test_imap_response_parser.rb b/test/net/imap/test_imap_response_parser.rb
