PHP 8.4 | Tokenizer/PHP: fix union/intersection/DNF type tokenization with final properties

Turns out there are tokenizer changes needed to support `final` properties as the retokenization of `|` to `T_TYPE_UNION`, `&` to T_TYPE_INTERSECTION` and the parenthesis in DNF types to `T_TYPE_*_PARENTHESIS` was broken.

Fixed now.

Includes tests.

Author: jrfnl

src/Tokenizers/PHP.php

@@ -3227,7 +3227,8 @@ protected function processAdditional()
                         && (isset(Tokens::$scopeModifiers[$this->tokens[$x]['code']]) === true
                         || $this->tokens[$x]['code'] === T_VAR
                         || $this->tokens[$x]['code'] === T_STATIC
-                        || $this->tokens[$x]['code'] === T_READONLY)
+                        || $this->tokens[$x]['code'] === T_READONLY
+                        || $this->tokens[$x]['code'] === T_FINAL)
                     ) {
                         // This will also confirm constructor property promotion parameters, but that's fine.
                         $confirmed = true;

tests/Core/Tokenizers/PHP/BitwiseOrTest.inc

@@ -75,6 +75,15 @@ class TypeUnion
     /* testTypeUnionPropertyWithOnlyStaticKeyword */
     static Foo|Bar $obj;
 
+    /* testTypeUnionWithPHP84FinalKeyword */
+    final int|string $finalKeywordA;
+
+    /* testTypeUnionWithPHP84FinalKeywordFirst */
+    final private float|null $finalKeywordB;
+
+    /* testTypeUnionWithPHP84FinalKeywordAndFQN */
+    final \MyClass|false $finalKeywordC;
+
     public function paramTypes(
         /* testTypeUnionParam1 */
         int|float $paramA /* testBitwiseOrParamDefaultValue */ = CONSTANT_A | CONSTANT_B,

tests/Core/Tokenizers/PHP/BitwiseOrTest.php

@@ -125,6 +125,9 @@ public static function dataTypeUnion()
             'type for readonly property, reversed modifier order'      => ['/* testTypeUnionPropertyWithReadOnlyKeywordFirst */'],
             'type for readonly property, no visibility'                => ['/* testTypeUnionPropertyWithOnlyReadOnlyKeyword */'],
             'type for static property, no visibility'                  => ['/* testTypeUnionPropertyWithOnlyStaticKeyword */'],
+            'type for final property, no visibility'                   => ['/* testTypeUnionWithPHP84FinalKeyword */'],
+            'type for final property, reversed modifier order'         => ['/* testTypeUnionWithPHP84FinalKeywordFirst */'],
+            'type for final property, no visibility, FQN type'         => ['/* testTypeUnionWithPHP84FinalKeywordAndFQN */'],
             'type for method parameter'                                => ['/* testTypeUnionParam1 */'],
             'type for method parameter, first in multi-union'          => ['/* testTypeUnionParam2 */'],
             'type for method parameter, last in multi-union'           => ['/* testTypeUnionParam3 */'],

tests/Core/Tokenizers/PHP/DNFTypesTest.inc

@@ -175,6 +175,12 @@ abstract class DNFTypes {
     /* testDNFTypePropertyWithOnlyStaticKeyword */
     static (A&B&C)|true $onlyStaticModified;
 
+    /* testDNFTypeWithPHP84FinalKeyword */
+    final (className&InterfaceName)|false $finalKeywordA;
+
+    /* testDNFTypeWithPHP84FinalKeywordAndStatic */
+    final static (\className&\InterfaceName)|false $finalKeywordB;
+
     public function paramTypes(
         /* testDNFTypeParam1WithAttribute */
         #[MyAttribute]

tests/Core/Tokenizers/PHP/DNFTypesTest.php

@@ -432,7 +432,6 @@ public static function dataDNFTypeParentheses()
             'OO property type: multi-dnf fully qualified classes'         => [
                 'testMarker'  => '/* testDNFTypePropertyMultiFullyQualified */',
             ],
-
             'OO property type: multi-dnf with readonly keyword 1'         => [
                 'testMarker'  => '/* testDNFTypePropertyWithReadOnlyKeyword1 */',
             ],
@@ -445,6 +444,13 @@ public static function dataDNFTypeParentheses()
             'OO property type: with only static keyword'                  => [
                 'testMarker'  => '/* testDNFTypePropertyWithOnlyStaticKeyword */',
             ],
+            'OO property type: with only final keyword'                   => [
+                'testMarker'  => '/* testDNFTypeWithPHP84FinalKeyword */',
+            ],
+            'OO property type: with final and static keyword'             => [
+                'testMarker'  => '/* testDNFTypeWithPHP84FinalKeywordAndStatic */',
+            ],
+
             'OO method param type: first param'                           => [
                 'testMarker'  => '/* testDNFTypeParam1WithAttribute */',
             ],

tests/Core/Tokenizers/PHP/TypeIntersectionTest.inc

@@ -63,6 +63,12 @@ class TypeIntersection
     /* testTypeIntersectionPropertyWithStaticKeyword */
     static Foo&Bar $obj;
 
+    /* testTypeIntersectionWithPHP84FinalKeyword */
+    final className&InterfaceName $finalKeywordA;
+
+    /* testTypeIntersectionWithPHP84FinalKeywordFirst */
+    final private \className&InterfaceName $finalKeywordB;
+
     public function paramTypes(
         /* testTypeIntersectionParam1 */
         Foo&Bar $paramA /* testBitwiseAndParamDefaultValue */ = CONSTANT_A & CONSTANT_B,

tests/Core/Tokenizers/PHP/TypeIntersectionTest.php

@@ -124,6 +124,8 @@ public static function dataTypeIntersection()
             'type for property using fully qualified names'               => ['/* testTypeIntersectionPropertyFullyQualified */'],
             'type for readonly property'                                  => ['/* testTypeIntersectionPropertyWithReadOnlyKeyword */'],
             'type for static readonly property'                           => ['/* testTypeIntersectionPropertyWithStaticKeyword */'],
+            'type for final property'                                     => ['/* testTypeIntersectionWithPHP84FinalKeyword */'],
+            'type for final property reversed modifier order'             => ['/* testTypeIntersectionWithPHP84FinalKeywordFirst */'],
             'type for method parameter'                                   => ['/* testTypeIntersectionParam1 */'],
             'type for method parameter, first in multi-intersect'         => ['/* testTypeIntersectionParam2 */'],
             'type for method parameter, last in multi-intersect'          => ['/* testTypeIntersectionParam3 */'],