🔀 Merge pull request #199 from ruby/parser/simplify-response

♻️ Update `response` and `continue-req` to new parser style

Author: nevans

lib/net/imap/response_data.rb

@@ -108,6 +108,9 @@ class TaggedResponse < Struct.new(:tag, :name, :data, :raw_data)
     # UntaggedResponse#data when the response type is a "condition" ("OK", "NO",
     # "BAD", "PREAUTH", or "BYE").
     class ResponseText < Struct.new(:code, :text)
+      # Used to avoid an allocation when ResponseText is empty
+      EMPTY = new(nil, "").freeze
+
       ##
       # method: code
       # :call-seq: code -> ResponseCode or nil

lib/net/imap/response_parser.rb

@@ -283,6 +283,7 @@ def unescape_quoted(quoted)
       Token = Struct.new(:symbol, :value)
 
       def_char_matchers :SP,   " ", :T_SPACE
+      def_char_matchers :PLUS, "+", :T_PLUS
 
       def_char_matchers :lpar, "(", :T_LPAR
       def_char_matchers :rpar, ")", :T_RPAR
@@ -325,6 +326,9 @@ def unescape_quoted(quoted)
       # TODO: add to lexer and only match tagged-ext-label
       def_token_matchers :tagged_ext_label, T_ATOM, T_NIL, send: :upcase
 
+      def_token_matchers :CRLF, T_CRLF
+      def_token_matchers :EOF,  T_EOF
+
       # atom            = 1*ATOM-CHAR
       # ATOM-CHAR       = <any CHAR except atom-specials>
       ATOM_TOKENS = [T_ATOM, T_NUMBER, T_NIL, T_LBRA, T_PLUS]
@@ -402,37 +406,40 @@ def case_insensitive__nstring
       alias number64    number
       alias number64?   number?
 
+      # [RFC3501 & RFC9051:]
+      #   response        = *(continue-req / response-data) response-done
+      #
+      # For simplicity, response isn't interpreted as the combination of the
+      # three response types, but instead represents any individual server
+      # response.  Our simplified interpretation is defined as:
+      #   response        = continue-req | response_data | response-tagged
+      #
+      # n.b: our "response-tagged" definition parses "greeting" too.
       def response
-        token = lookahead
-        case token.symbol
-        when T_PLUS
-          result = continue_req
-        when T_STAR
-          result = response_untagged
-        else
-          result = response_tagged
-        end
-        while lookahead.symbol == T_SPACE
-          # Ignore trailing space for Microsoft Exchange Server
-          shift_token
-        end
-        match(T_CRLF)
-        match(T_EOF)
-        return result
+        resp = case lookahead!(T_PLUS, T_STAR, *TAG_TOKENS).symbol
+               when T_PLUS then continue_req
+               when T_STAR then response_data
+               else             response_tagged
+               end
+        accept_spaces # QUIRKY: Ignore trailing space (MS Exchange Server?)
+        CRLF!
+        EOF!
+        resp
       end
 
+      # RFC3501 & RFC9051:
+      #   continue-req    = "+" SP (resp-text / base64) CRLF
+      #
+      # n.b: base64 is valid resp-text.  And in the spirit of RFC9051 Appx E 23
+      # (and to workaround existing servers), we use the following grammar:
+      #
+      #   continue-req    = "+" (SP (resp-text)) CRLF
       def continue_req
-        match(T_PLUS)
-        token = lookahead
-        if token.symbol == T_SPACE
-          shift_token
-          return ContinuationRequest.new(resp_text, @str)
-        else
-          return ContinuationRequest.new(ResponseText.new(nil, ""), @str)
-        end
+        PLUS!
+        ContinuationRequest.new(SP? ? resp_text : ResponseText::EMPTY, @str)
       end
 
-      def response_untagged
+      def response_data
         match(T_STAR)
         match(T_SPACE)
         token = lookahead
@@ -1564,10 +1571,10 @@ def nil_atom
       #
       # This advances @pos directly so it's safe before changing @lex_state.
       def accept_spaces
-        shift_token if @token&.symbol == T_SPACE
-        if @str.index(SPACES_REGEXP, @pos)
+        return false unless SP?
+        @str.index(SPACES_REGEXP, @pos) and
           @pos = $~.end(0)
-        end
+        true
       end
 
       def next_token

lib/net/imap/response_parser/parser_utils.rb

@@ -170,6 +170,16 @@ def lookahead
           @token ||= next_token
         end
 
+        # like match, without consuming the token
+        def lookahead!(*args)
+          if args.include?((@token ||= next_token)&.symbol)
+            @token
+          else
+            parse_error('unexpected token %s (expected %s)',
+                        @token&.symbol, args.join(" or "))
+          end
+        end
+
         def peek_str?(str)
           assert_no_lookahead if Net::IMAP.debug
           @str[@pos, str.length] == str