Merge pull request rails#52629 from Shopify/opt_scanner

Optimize ActionDispatch::Journey::Scanner

Author: gmcgibbon

actionpack/lib/action_dispatch/journey/parser.rb

@@ -36,7 +36,7 @@ def parse_expressions
         node = parse_expression
 
         while @next_token
-          case @next_token.first
+          case @next_token
           when :RPAREN
             break
           when :OR
@@ -56,43 +56,43 @@ def parse_or(lhs)
       end
 
       def parse_expression
-        if @next_token.first == :STAR
+        if @next_token == :STAR
           parse_star
-        elsif @next_token.first == :LPAREN
+        elsif @next_token == :LPAREN
           parse_group
         else
           parse_terminal
         end
       end
 
       def parse_star
-        node = Star.new(Symbol.new(@next_token.last, Symbol::GREEDY_EXP))
+        node = Star.new(Symbol.new(@scanner.last_string, Symbol::GREEDY_EXP))
         advance_token
         node
       end
 
       def parse_group
         advance_token
         node = parse_expressions
-        if @next_token.first == :RPAREN
+        if @next_token == :RPAREN
           node = Group.new(node)
           advance_token
           node
         else
-          raise "error"
+          raise ArgumentError, "missing right parenthesis."
         end
       end
 
       def parse_terminal
-        node = case @next_token.first
+        node = case @next_token
         when :SYMBOL
-          Symbol.new(@next_token.last)
+          Symbol.new(@scanner.last_string)
         when :LITERAL
-          Literal.new(@next_token.last)
+          Literal.new(@scanner.last_literal)
         when :SLASH
-          Slash.new(@next_token.last)
+          Slash.new("/")
         when :DOT
-          Dot.new(@next_token.last)
+          Dot.new(".")
         end
 
         advance_token

actionpack/lib/action_dispatch/journey/scanner.rb

@@ -7,64 +7,62 @@
 module ActionDispatch
   module Journey # :nodoc:
     class Scanner # :nodoc:
+      STATIC_TOKENS = Array.new(150)
+      STATIC_TOKENS[".".ord] = :DOT
+      STATIC_TOKENS["/".ord] = :SLASH
+      STATIC_TOKENS["(".ord] = :LPAREN
+      STATIC_TOKENS[")".ord] = :RPAREN
+      STATIC_TOKENS["|".ord] = :OR
+      STATIC_TOKENS[":".ord] = :SYMBOL
+      STATIC_TOKENS["*".ord] = :STAR
+      STATIC_TOKENS.freeze
+
+      class Scanner < StringScanner
+        unless method_defined?(:peek_byte) # https://github.com/ruby/strscan/pull/89
+          def peek_byte
+            string.getbyte(pos)
+          end
+        end
+      end
+
       def initialize
-        @ss = nil
+        @scanner = nil
+        @length = nil
       end
 
       def scan_setup(str)
-        @ss = StringScanner.new(str)
+        @scanner = Scanner.new(str)
       end
 
-      def eos?
-        @ss.eos?
-      end
+      def next_token
+        return if @scanner.eos?
 
-      def pos
-        @ss.pos
+        until token = scan || @scanner.eos?; end
+        token
       end
 
-      def pre_match
-        @ss.pre_match
+      def last_string
+        -@scanner.string.byteslice(@scanner.pos - @length, @length)
       end
 
-      def next_token
-        return if @ss.eos?
-
-        until token = scan || @ss.eos?; end
-        token
+      def last_literal
+        last_str = @scanner.string.byteslice(@scanner.pos - @length, @length)
+        last_str.tr! "\\", ""
+        -last_str
       end
 
       private
-        # takes advantage of String @- deduping capabilities in Ruby 2.5 upwards see:
-        # https://bugs.ruby-lang.org/issues/13077
-        def dedup_scan(regex)
-          r = @ss.scan(regex)
-          r ? -r : nil
-        end
-
         def scan
+          next_byte = @scanner.peek_byte
           case
-            # /
-          when @ss.skip(/\//)
-            [:SLASH, "/"]
-          when @ss.skip(/\(/)
-            [:LPAREN, "("]
-          when @ss.skip(/\)/)
-            [:RPAREN, ")"]
-          when @ss.skip(/\|/)
-            [:OR, "|"]
-          when @ss.skip(/\./)
-            [:DOT, "."]
-          when text = dedup_scan(/:\w+/)
-            [:SYMBOL, text]
-          when text = dedup_scan(/\*\w+/)
-            [:STAR, text]
-          when text = @ss.scan(/(?:[\w%\-~!$&'*+,;=@]|\\[:()])+/)
-            text.tr! "\\", ""
-            [:LITERAL, -text]
-            # any char
-          when text = dedup_scan(/./)
-            [:LITERAL, text]
+          when (token = STATIC_TOKENS[next_byte])
+            @scanner.pos += 1
+            @length = @scanner.skip(/\w+/).to_i + 1 if token == :SYMBOL || token == :STAR
+            token
+          when @length = @scanner.skip(/(?:[\w%\-~!$&'*+,;=@]|\\[:()])+/)
+            :LITERAL
+          when @length = @scanner.skip(/./)
+            :LITERAL
           end
         end
     end

actionpack/test/journey/route/definition/scanner_test.rb

@@ -11,50 +11,54 @@ def setup
         end
 
         CASES = [
-          ["/",       [[:SLASH, "/"]]],
-          ["*omg",    [[:STAR, "*omg"]]],
-          ["/page",   [[:SLASH, "/"], [:LITERAL, "page"]]],
-          ["/page!",  [[:SLASH, "/"], [:LITERAL, "page!"]]],
-          ["/page$",  [[:SLASH, "/"], [:LITERAL, "page$"]]],
-          ["/page&",  [[:SLASH, "/"], [:LITERAL, "page&"]]],
-          ["/page'",  [[:SLASH, "/"], [:LITERAL, "page'"]]],
-          ["/page*",  [[:SLASH, "/"], [:LITERAL, "page*"]]],
-          ["/page+",  [[:SLASH, "/"], [:LITERAL, "page+"]]],
-          ["/page,",  [[:SLASH, "/"], [:LITERAL, "page,"]]],
-          ["/page;",  [[:SLASH, "/"], [:LITERAL, "page;"]]],
-          ["/page=",  [[:SLASH, "/"], [:LITERAL, "page="]]],
-          ["/page@",  [[:SLASH, "/"], [:LITERAL, "page@"]]],
-          ['/page\:', [[:SLASH, "/"], [:LITERAL, "page:"]]],
-          ['/page\(', [[:SLASH, "/"], [:LITERAL, "page("]]],
-          ['/page\)', [[:SLASH, "/"], [:LITERAL, "page)"]]],
-          ["/~page",  [[:SLASH, "/"], [:LITERAL, "~page"]]],
-          ["/pa-ge",  [[:SLASH, "/"], [:LITERAL, "pa-ge"]]],
-          ["/:page",  [[:SLASH, "/"], [:SYMBOL, ":page"]]],
+          ["/",       [:SLASH]],
+          ["*omg",    [:STAR]],
+          ["/page",   [:SLASH, :LITERAL]],
+          ["/page!",  [:SLASH, :LITERAL]],
+          ["/page$",  [:SLASH, :LITERAL]],
+          ["/page&",  [:SLASH, :LITERAL]],
+          ["/page'",  [:SLASH, :LITERAL]],
+          ["/page*",  [:SLASH, :LITERAL]],
+          ["/page+",  [:SLASH, :LITERAL]],
+          ["/page,",  [:SLASH, :LITERAL]],
+          ["/page;",  [:SLASH, :LITERAL]],
+          ["/page=",  [:SLASH, :LITERAL]],
+          ["/page@",  [:SLASH, :LITERAL]],
+          ['/page\:', [:SLASH, :LITERAL]],
+          ['/page\(', [:SLASH, :LITERAL]],
+          ['/page\)', [:SLASH, :LITERAL]],
+          ["/~page",  [:SLASH, :LITERAL]],
+          ["/pa-ge",  [:SLASH, :LITERAL]],
+          ["/:page",  [:SLASH, :SYMBOL]],
           ["/:page|*foo", [
-                            [:SLASH, "/"],
-                            [:SYMBOL, ":page"],
-                            [:OR, "|"],
-                            [:STAR, "*foo"]
+                            :SLASH,
+                            :SYMBOL,
+                            :OR,
+                            :STAR
                           ]],
           ["/(:page)", [
-                        [:SLASH, "/"],
-                        [:LPAREN, "("],
-                        [:SYMBOL, ":page"],
-                        [:RPAREN, ")"],
+                        :SLASH,
+                        :LPAREN,
+                        :SYMBOL,
+                        :RPAREN,
                       ]],
           ["(/:action)", [
-                          [:LPAREN, "("],
-                          [:SLASH, "/"],
-                          [:SYMBOL, ":action"],
-                          [:RPAREN, ")"],
+                          :LPAREN,
+                          :SLASH,
+                          :SYMBOL,
+                          :RPAREN,
                          ]],
-          ["(())", [[:LPAREN, "("],
-                   [:LPAREN, "("], [:RPAREN, ")"], [:RPAREN, ")"]]],
+          ["(())", [
+                    :LPAREN,
+                    :LPAREN,
+                    :RPAREN,
+                    :RPAREN,
+                  ]],
           ["(.:format)", [
-                          [:LPAREN, "("],
-                          [:DOT, "."],
-                          [:SYMBOL, ":format"],
-                          [:RPAREN, ")"],
+                          :LPAREN,
+                          :DOT,
+                          :SYMBOL,
+                          :RPAREN,
                         ]],
         ]