Skip to content

Commit 08a946f

Browse files
jrfnljrfnl
committed
PHP 8.0 | Tokenizer/PHP: backfill the T_MATCH tokenization
PHP 8.0 introduces a new type of control structure: match expressions. > A match expression is similar to a `switch` control structure but with safer semantics and the ability to return values. Ref: https://wiki.php.net/rfc/match_expression_v2 This commit adds initial support for match expressions to PHPCS. * In PHP < 8: Retokenizes `T_STRING` tokens containing the `match` keyword to `T_MATCH` when they are in actual fact match expressions. * In PHP 8: Retokenizes `T_MATCH` tokens to `T_STRING` when the `match` keyword is used outside the context of a match expression, like in a method declaration or call. * Ensures that the `match` keyword for match expressions will be recognized as a scope owner and that the appropriate `scope_*` array indexes are set for the curly braces belonging to the match expression, as well as that the `match` condition is added to the tokens within the match expression. Note: in contrast to `switch` control structures, "cases" in a match expression will not be recognized as scopes and no `scope_owner`, `scope_opener` or `scope_closer` array indexes will be set for the individual cases in a match expression. This also applies to the `default` case when used in a match expression. Includes extensive unit tests. Note: at this point, not all unit tests will pass, this will be fixed in follow-on commits.
1 parent 966ae7c commit 08a946f

File tree

4 files changed

+949
-0
lines changed

4 files changed

+949
-0
lines changed

‎package.xml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
196196
<file baseinstalldir="" name="AnonClassParenthesisOwnerTest.php" role="test" />
197197
<file baseinstalldir="" name="BackfillFnTokenTest.inc" role="test" />
198198
<file baseinstalldir="" name="BackfillFnTokenTest.php" role="test" />
199+
<file baseinstalldir="" name="BackfillMatchTokenTest.inc" role="test" />
200+
<file baseinstalldir="" name="BackfillMatchTokenTest.php" role="test" />
199201
<file baseinstalldir="" name="BackfillNumericSeparatorTest.inc" role="test" />
200202
<file baseinstalldir="" name="BackfillNumericSeparatorTest.php" role="test" />
201203
<file baseinstalldir="" name="BitwiseOrTest.inc" role="test" />
@@ -2100,6 +2102,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
21002102
<install as="CodeSniffer/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" name="tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" />
21012103
<install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.php" name="tests/Core/Tokenizer/BackfillFnTokenTest.php" />
21022104
<install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.inc" name="tests/Core/Tokenizer/BackfillFnTokenTest.inc" />
2105+
<install as="CodeSniffer/Core/Tokenizer/BackfillMatchTokenTest.php" name="tests/Core/Tokenizer/BackfillMatchTokenTest.php" />
2106+
<install as="CodeSniffer/Core/Tokenizer/BackfillMatchTokenTest.inc" name="tests/Core/Tokenizer/BackfillMatchTokenTest.inc" />
21032107
<install as="CodeSniffer/Core/Tokenizer/BackfillNumericSeparatorTest.php" name="tests/Core/Tokenizer/BackfillNumericSeparatorTest.php" />
21042108
<install as="CodeSniffer/Core/Tokenizer/BackfillNumericSeparatorTest.inc" name="tests/Core/Tokenizer/BackfillNumericSeparatorTest.inc" />
21052109
<install as="CodeSniffer/Core/Tokenizer/BitwiseOrTest.php" name="tests/Core/Tokenizer/BitwiseOrTest.php" />
@@ -2176,6 +2180,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
21762180
<install as="CodeSniffer/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" name="tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" />
21772181
<install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.php" name="tests/Core/Tokenizer/BackfillFnTokenTest.php" />
21782182
<install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.inc" name="tests/Core/Tokenizer/BackfillFnTokenTest.inc" />
2183+
<install as="CodeSniffer/Core/Tokenizer/BackfillMatchTokenTest.php" name="tests/Core/Tokenizer/BackfillMatchTokenTest.php" />
2184+
<install as="CodeSniffer/Core/Tokenizer/BackfillMatchTokenTest.inc" name="tests/Core/Tokenizer/BackfillMatchTokenTest.inc" />
21792185
<install as="CodeSniffer/Core/Tokenizer/BackfillNumericSeparatorTest.php" name="tests/Core/Tokenizer/BackfillNumericSeparatorTest.php" />
21802186
<install as="CodeSniffer/Core/Tokenizer/BackfillNumericSeparatorTest.inc" name="tests/Core/Tokenizer/BackfillNumericSeparatorTest.inc" />
21812187
<install as="CodeSniffer/Core/Tokenizer/BitwiseOrTest.php" name="tests/Core/Tokenizer/BitwiseOrTest.php" />

‎src/Tokenizers/PHP.php

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -251,6 +251,13 @@ class PHP extends Tokenizer
251251
T_SWITCH => T_SWITCH,
252252
],
253253
],
254+
T_MATCH => [
255+
'start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET],
256+
'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET],
257+
'strict' => true,
258+
'shared' => false,
259+
'with' => [],
260+
],
254261
T_START_HEREDOC => [
255262
'start' => [T_START_HEREDOC => T_START_HEREDOC],
256263
'end' => [T_END_HEREDOC => T_END_HEREDOC],
@@ -365,6 +372,7 @@ class PHP extends Tokenizer
365372
T_LOGICAL_AND => 3,
366373
T_LOGICAL_OR => 2,
367374
T_LOGICAL_XOR => 3,
375+
T_MATCH => 5,
368376
T_METHOD_C => 10,
369377
T_MINUS_EQUAL => 2,
370378
T_POW_EQUAL => 3,
@@ -1254,6 +1262,87 @@ protected function tokenize($string)
12541262
continue;
12551263
}//end if
12561264

1265+
/*
1266+
Backfill the T_MATCH token for PHP versions < 8.0 and
1267+
do initial correction for non-match expression T_MATCH tokens
1268+
to T_STRING for PHP >= 8.0.
1269+
A final check for non-match expression T_MATCH tokens is done
1270+
in PHP::processAdditional().
1271+
*/
1272+
1273+
if ($tokenIsArray === true
1274+
&& (($token[0] === T_STRING
1275+
&& strtolower($token[1]) === 'match')
1276+
|| $token[0] === T_MATCH)
1277+
) {
1278+
$isMatch = false;
1279+
for ($x = ($stackPtr + 1); $x < $numTokens; $x++) {
1280+
if (isset($tokens[$x][0], Util\Tokens::$emptyTokens[$tokens[$x][0]]) === true) {
1281+
continue;
1282+
}
1283+
1284+
if ($tokens[$x] !== '(') {
1285+
// This is not a match expression.
1286+
break;
1287+
}
1288+
1289+
// Next was an open parenthesis, now check what is before the match keyword.
1290+
for ($y = ($stackPtr - 1); $y >= 0; $y--) {
1291+
if (isset(Util\Tokens::$emptyTokens[$tokens[$y][0]]) === true) {
1292+
continue;
1293+
}
1294+
1295+
if (is_array($tokens[$y]) === true
1296+
&& ($tokens[$y][0] === T_PAAMAYIM_NEKUDOTAYIM
1297+
|| $tokens[$y][0] === T_OBJECT_OPERATOR
1298+
|| $tokens[$y][0] === T_NS_SEPARATOR
1299+
|| $tokens[$y][0] === T_NEW
1300+
|| $tokens[$y][0] === T_FUNCTION
1301+
|| $tokens[$y][0] === T_CLASS
1302+
|| $tokens[$y][0] === T_INTERFACE
1303+
|| $tokens[$y][0] === T_TRAIT
1304+
|| $tokens[$y][0] === T_NAMESPACE
1305+
|| $tokens[$y][0] === T_CONST)
1306+
) {
1307+
// This is not a match expression.
1308+
break 2;
1309+
}
1310+
1311+
$isMatch = true;
1312+
break 2;
1313+
}//end for
1314+
}//end for
1315+
1316+
if ($isMatch === true && $token[0] === T_STRING) {
1317+
$newToken = [];
1318+
$newToken['code'] = T_MATCH;
1319+
$newToken['type'] = 'T_MATCH';
1320+
$newToken['content'] = $token[1];
1321+
1322+
if (PHP_CODESNIFFER_VERBOSITY > 1) {
1323+
echo "\t\t* token $stackPtr changed from T_STRING to T_MATCH".PHP_EOL;
1324+
}
1325+
1326+
$finalTokens[$newStackPtr] = $newToken;
1327+
$newStackPtr++;
1328+
continue;
1329+
} else if ($isMatch === false && $token[0] === T_MATCH) {
1330+
// PHP 8.0, match keyword, but not a match expression.
1331+
$newToken = [];
1332+
$newToken['code'] = T_STRING;
1333+
$newToken['type'] = 'T_STRING';
1334+
$newToken['content'] = $token[1];
1335+
1336+
if (PHP_CODESNIFFER_VERBOSITY > 1) {
1337+
echo "\t\t* token $stackPtr changed from T_MATCH to T_STRING".PHP_EOL;
1338+
}
1339+
1340+
$finalTokens[$newStackPtr] = $newToken;
1341+
$newStackPtr++;
1342+
continue;
1343+
}//end if
1344+
}//end if
1345+
12571346
/*
12581347
Convert ? to T_NULLABLE OR T_INLINE_THEN
12591348
*/
@@ -2265,6 +2354,36 @@ protected function processAdditional()
22652354
}
22662355
}
22672356

2357+
continue;
2358+
} else if ($this->tokens[$i]['code'] === T_MATCH) {
2359+
if (isset($this->tokens[$i]['scope_opener'], $this->tokens[$i]['scope_closer']) === false) {
2360+
// Not a match expression after all.
2361+
$this->tokens[$i]['code'] = T_STRING;
2362+
$this->tokens[$i]['type'] = 'T_STRING';
2363+
2364+
if (PHP_CODESNIFFER_VERBOSITY > 1) {
2365+
echo "\t\t* token $i changed from T_MATCH to T_STRING".PHP_EOL;
2366+
}
2367+
2368+
if (isset($this->tokens[$i]['parenthesis_opener'], $this->tokens[$i]['parenthesis_closer']) === true) {
2369+
$opener = $this->tokens[$i]['parenthesis_opener'];
2370+
$closer = $this->tokens[$i]['parenthesis_closer'];
2371+
unset(
2372+
$this->tokens[$opener]['parenthesis_owner'],
2373+
$this->tokens[$closer]['parenthesis_owner']
2374+
);
2375+
unset(
2376+
$this->tokens[$i]['parenthesis_opener'],
2377+
$this->tokens[$i]['parenthesis_closer'],
2378+
$this->tokens[$i]['parenthesis_owner']
2379+
);
2380+
2381+
if (PHP_CODESNIFFER_VERBOSITY > 1) {
2382+
echo "\t\t* cleaned parenthesis of token $i *".PHP_EOL;
2383+
}
2384+
}
2385+
}//end if
2386+
22682387
continue;
22692388
} else if ($this->tokens[$i]['code'] === T_BITWISE_OR) {
22702389
/*

0 commit comments

Comments
 (0)