♻️ Update response to use new parser style

* Fix `accept_spaces?` and use it from `response`.
* Add `lookahead!(*tokens)`, an unconsuming version of `match`.
  The main benefit is that it produces more helpful errors.
* Rename `response_untagged` => `response_data`, to match the RFC ABNF

Author: nevans

lib/net/imap/response_parser.rb

@@ -326,6 +326,9 @@ def unescape_quoted(quoted)
       # TODO: add to lexer and only match tagged-ext-label
       def_token_matchers :tagged_ext_label, T_ATOM, T_NIL, send: :upcase
 
+      def_token_matchers :CRLF, T_CRLF
+      def_token_matchers :EOF,  T_EOF
+
       # atom            = 1*ATOM-CHAR
       # ATOM-CHAR       = <any CHAR except atom-specials>
       ATOM_TOKENS = [T_ATOM, T_NUMBER, T_NIL, T_LBRA, T_PLUS]
@@ -403,23 +406,25 @@ def case_insensitive__nstring
       alias number64    number
       alias number64?   number?
 
+      # [RFC3501 & RFC9051:]
+      #   response        = *(continue-req / response-data) response-done
+      #
+      # For simplicity, response isn't interpreted as the combination of the
+      # three response types, but instead represents any individual server
+      # response.  Our simplified interpretation is defined as:
+      #   response        = continue-req | response_data | response-tagged
+      #
+      # n.b: our "response-tagged" definition parses "greeting" too.
       def response
-        token = lookahead
-        case token.symbol
-        when T_PLUS
-          result = continue_req
-        when T_STAR
-          result = response_untagged
-        else
-          result = response_tagged
-        end
-        while lookahead.symbol == T_SPACE
-          # Ignore trailing space for Microsoft Exchange Server
-          shift_token
-        end
-        match(T_CRLF)
-        match(T_EOF)
-        return result
+        resp = case lookahead!(T_PLUS, T_STAR, *TAG_TOKENS).symbol
+               when T_PLUS then continue_req
+               when T_STAR then response_data
+               else             response_tagged
+               end
+        accept_spaces # QUIRKY: Ignore trailing space (MS Exchange Server?)
+        CRLF!
+        EOF!
+        resp
       end
 
       # RFC3501 & RFC9051:
@@ -434,7 +439,7 @@ def continue_req
         ContinuationRequest.new(SP? ? resp_text : ResponseText::EMPTY, @str)
       end
 
-      def response_untagged
+      def response_data
         match(T_STAR)
         match(T_SPACE)
         token = lookahead
@@ -1566,10 +1571,10 @@ def nil_atom
       #
       # This advances @pos directly so it's safe before changing @lex_state.
       def accept_spaces
-        shift_token if @token&.symbol == T_SPACE
-        if @str.index(SPACES_REGEXP, @pos)
+        return false unless SP?
+        @str.index(SPACES_REGEXP, @pos) and
           @pos = $~.end(0)
-        end
+        true
       end
 
       def next_token

lib/net/imap/response_parser/parser_utils.rb

@@ -170,6 +170,16 @@ def lookahead
           @token ||= next_token
         end
 
+        # like match, without consuming the token
+        def lookahead!(*args)
+          if args.include?((@token ||= next_token)&.symbol)
+            @token
+          else
+            parse_error('unexpected token %s (expected %s)',
+                        @token&.symbol, args.join(" or "))
+          end
+        end
+
         def peek_str?(str)
           assert_no_lookahead if Net::IMAP.debug
           @str[@pos, str.length] == str