Optimizing lexer

Author: mtdowling

src/Lexer.php

@@ -8,8 +8,7 @@ class Lexer
 {
     private $regex, $offsetToToken;
 
-    /** @var array Map of regular expressions to their token match value */
-    private $tokenMap = [
+    private $tokens = [
         '[a-zA-Z_][a-zA-Z_0-9]*'     => 'identifier',
         '\.'                         => 'dot',
         '\*'                         => 'star',
@@ -41,16 +40,13 @@ class Lexer
 
     public function __construct()
     {
-        $this->regex = '((' .
-            implode(')|(', array_keys($this->tokenMap)) . '))';
-        $this->offsetToToken = array_values($this->tokenMap);
+        $this->regex = '((' . implode(')|(', array_keys($this->tokens)) . '))';
+        $this->offsetToToken = array_values($this->tokens);
     }
 
     /**
-     * Tokenize the JMESPath expression into an array of tokens.
-     *
-     * Each token array contains a type, value, and pos key along with any
-     * other keys that might be relevant to the particular token.
+     * Tokenize the JMESPath expression into an array of tokens hashes that
+     * contain a 'type', 'value', and 'key'.
      *
      * @param string $input JMESPath input
      *
@@ -59,23 +55,16 @@ public function __construct()
      */
     public function tokenize($input)
     {
-        $offset = 0;
-        $tokens = [];
-
         if (!preg_match_all($this->regex, $input, $matches, PREG_SET_ORDER)) {
-            $this->throwSyntax('Invalid expression', $offset, $input);
+            throw $this->throwSyntax('Invalid expression', 0, $input);
         }
 
+        $offset = 0;
+        $tokens = [];
         foreach ($matches as $match) {
             $type = $this->offsetToToken[count($match) - 2];
-
             if ($type !== 'skip') {
-                $token = [
-                    'type'  => $type,
-                    'value' => $match[0],
-                    'pos'   => $offset
-                ];
-
+                $token = ['type' => $type, 'value' => $match[0], 'pos' => $offset];
                 switch ($token['type']) {
                     case 'quoted_identifier':
                         $token['value'] = $this->decodeJson(
@@ -101,10 +90,8 @@ public function tokenize($input)
             $offset += strlen($match[0]);
         }
 
-        // Always end the token stream with an EOF token
         $tokens[] = ['type' => 'eof', 'pos' => $offset, 'value' => null];
 
-        // Ensure that the expression did not contain invalid characters
         if (strlen($input) != $offset) {
             $this->invalidExpression($input);
         }
@@ -114,31 +101,16 @@ public function tokenize($input)
 
     private function takeLiteral($value, $offset, $input)
     {
-        // Maps common JavaScript primitives with a native PHP primitive
-        static $primitives = ['true' => 0, 'false' => 1, 'null' => 2];
-        static $primitiveMap = [true, false, null];
-        // If a literal starts with these characters, it is JSON decoded
-        static $decodeCharacters = ['"' => 1, '[' => 1, '{' => 1];
-
+        static $valid = '/(true|false|null)|(^[\["{])|(^\-?[0-9]*(\.[0-9]+)?([e|E][+|\-][0-9]+)?$)/';
         $value = str_replace('\\`', '`', ltrim(substr($value, 1, -1)));
 
-        if (isset($primitives[$value])) {
-            // Fast lookups for common JSON primitives
-            return $primitiveMap[$primitives[$value]];
-        } elseif (strlen($value) == 0) {
-            $this->throwSyntax('Empty JSON literal', $offset, $input);
-        } elseif (isset($decodeCharacters[$value[0]])) {
-            // Always decode the JSON directly if it starts with these chars
-            return $this->decodeJson($value, $offset, $input);
-        } elseif (preg_match(
-            '/^\-?[0-9]*(\.[0-9]+)?([e|E][+|\-][0-9]+)?$/',
-            $value
-        )) {
-            // If it starts with a "-" or numbers, then attempt to JSON decode
-            return $this->decodeJson($value, $offset, $input);
+        if ($value === '') {
+            throw $this->throwSyntax('Empty JSON literal', $offset, $input);
         }
 
-        return $this->decodeJson('"' . $value . '"', $offset, $input);
+        return preg_match($valid, $value)
+            ? $this->decodeJson($value, $offset, $input)
+            : $this->decodeJson('"' . $value . '"', $offset, $input);
     }
 
     private function decodeJson($json, $offset, $input)
@@ -155,7 +127,7 @@ private function decodeJson($json, $offset, $input)
 
         if ($error = json_last_error()) {
             $message = isset($errs[$error]) ? $errs[$error] : 'Unknown error';
-            $this->throwSyntax(
+            throw $this->throwSyntax(
                 "Error decoding JSON: ({$error}) {$message}, given {$json}",
                 $offset,
                 $input
@@ -167,7 +139,7 @@ private function decodeJson($json, $offset, $input)
 
     private function throwSyntax($message, $offset, $input)
     {
-        throw new SyntaxErrorException(
+        return new SyntaxErrorException(
             $message,
             ['value' => substr($input, $offset, 1), 'pos' => $offset],
             $input
@@ -177,12 +149,10 @@ private function throwSyntax($message, $offset, $input)
     private function invalidExpression($input)
     {
         $offset = 0;
-        $regex = $this->regex . 'A';
-
-        while (preg_match($regex, $input, $matches, 0, $offset)) {
+        while (preg_match("{$this->regex}A", $input, $matches, 0, $offset)) {
             $offset += strlen($matches[0]);
         }
 
-        $this->throwSyntax('Unexpected character', $offset, $input);
+        throw $this->throwSyntax('Unexpected character', $offset, $input);
     }
 }