Merge pull request #35 from staabm/PhpTokenTokenize

Utilize `PhpToken::tokenize()` - requires PHP8+

Author: theseer

.github/workflows/ci.yml

@@ -15,7 +15,7 @@ jobs:
       # Keys:
       # - experimental: Whether the build is "allowed to fail".
       matrix:
-        php: ['7.3', '7.4', '8.0', '8.1', '8.2', '8.3', '8.4', '8.5']
+        php: ['8.0', '8.1', '8.2', '8.3', '8.4', '8.5']
         experimental: [false]
 
         include:

composer.json

@@ -13,7 +13,7 @@
     "issues": "https://github.com/theseer/tokenizer/issues"
   },
   "require": {
-    "php": "^7.2 || ^8.0",
+    "php": "^8.0",
     "ext-xmlwriter": "*",
     "ext-dom": "*",
     "ext-tokenizer": "*"

src/Tokenizer.php

@@ -46,35 +46,35 @@ public function parse(string $source): TokenCollection {
             return $result;
         }
 
-        $tokens = \token_get_all($source);
+        $tokens = \PhpToken::tokenize($source);
 
         $lastToken = new Token(
-            $tokens[0][2],
+            $tokens[0]->line,
             'Placeholder',
             ''
         );
 
-        foreach ($tokens as $pos => $tok) {
-            if (\is_string($tok)) {
+        foreach ($tokens as $tok) {
+            if (isset(self::MAP[$tok->text])) {
                 $token = new Token(
                     $lastToken->getLine(),
-                    self::MAP[$tok],
-                    $tok
+                    self::MAP[$tok->text],
+                    $tok->text,
                 );
                 $result->addToken($token);
                 $lastToken = $token;
 
                 continue;
             }
 
-            $line   = $tok[2];
-            $values = \preg_split('/\R+/Uu', $tok[1]);
+            $line   = $tok->line;
+            $values = \preg_split('/\R+/Uu', $tok->text);
 
             if (!$values) {
                 $result->addToken(
                     new Token(
                         $line,
-                        \token_name($tok[0]),
+                        $tok->getTokenName(),
                         '{binary data}'
                     )
                 );
@@ -85,7 +85,7 @@ public function parse(string $source): TokenCollection {
             foreach ($values as $v) {
                 $token = new Token(
                     $line,
-                    \token_name($tok[0]),
+                    $tok->getTokenName(),
                     $v
                 );
                 $lastToken = $token;