Skip to content

Commit 694c569

Browse files
jrfnljrfnl
committed
PHP 8.0 | Tokenizer/PHP: retokenize default keywords within match expressions as T_MATCH_DEFAULT
The `default` keyword in match expressions is tokenized as `T_DEFAULT` by PHP. However, for `switch` control structures, `default` (and `case`) are treated as scope owners/openers by PHPCS. If the `default` keyword tokenization in match expressions was left as-is, the `Tokenizer::recurseScopeMap()` would search for the wrong scope opener/closer, throwing the scope setting for scope owners in the rest of the file off. Adding the typical opener/closers for match expression `default` cases to the `PHP::$scopeOpeners` array would not prevent this and once the scope setting is out of kilter, a lot of sniffs start failing, including the `ScopeIndent` sniffs, `ScopeCloserBrace` sniffs etc. The only stable solution I could find to prevent the scope setting potentially getting borked and existing sniffs breaking badly, was to retokenize the `default` keyword when used in a `match` expression to `T_MATCH_DEFAULT`. Includes a separate set of unit tests which specifically tests the tokenization of the `default` keyword and verifies both the retokenization to `T_MATCH_DEFAULT` when used in `match` expressions, as well as the tokenization of the `default` keyword within `switch` control structures, including the scope setting for the `default` case.
1 parent 08a946f commit 694c569

File tree

5 files changed

+328
-0
lines changed

5 files changed

+328
-0
lines changed

‎package.xml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
202202
<file baseinstalldir="" name="BackfillNumericSeparatorTest.php" role="test" />
203203
<file baseinstalldir="" name="BitwiseOrTest.inc" role="test" />
204204
<file baseinstalldir="" name="BitwiseOrTest.php" role="test" />
205+
<file baseinstalldir="" name="DefaultKeywordTest.inc" role="test" />
206+
<file baseinstalldir="" name="DefaultKeywordTest.php" role="test" />
205207
<file baseinstalldir="" name="GotoLabelTest.inc" role="test" />
206208
<file baseinstalldir="" name="GotoLabelTest.php" role="test" />
207209
<file baseinstalldir="" name="NamedFunctionCallArgumentsTest.inc" role="test" />
@@ -2108,6 +2110,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
21082110
<install as="CodeSniffer/Core/Tokenizer/BackfillNumericSeparatorTest.inc" name="tests/Core/Tokenizer/BackfillNumericSeparatorTest.inc" />
21092111
<install as="CodeSniffer/Core/Tokenizer/BitwiseOrTest.php" name="tests/Core/Tokenizer/BitwiseOrTest.php" />
21102112
<install as="CodeSniffer/Core/Tokenizer/BitwiseOrTest.inc" name="tests/Core/Tokenizer/BitwiseOrTest.inc" />
2113+
<install as="CodeSniffer/Core/Tokenizer/DefaultKeywordTest.php" name="tests/Core/Tokenizer/DefaultKeywordTest.php" />
2114+
<install as="CodeSniffer/Core/Tokenizer/DefaultKeywordTest.inc" name="tests/Core/Tokenizer/DefaultKeywordTest.inc" />
21112115
<install as="CodeSniffer/Core/Tokenizer/GotoLabelTest.php" name="tests/Core/Tokenizer/GotoLabelTest.php" />
21122116
<install as="CodeSniffer/Core/Tokenizer/GotoLabelTest.inc" name="tests/Core/Tokenizer/GotoLabelTest.inc" />
21132117
<install as="CodeSniffer/Core/Tokenizer/NamedFunctionCallArgumentsTest.php" name="tests/Core/Tokenizer/NamedFunctionCallArgumentsTest.php" />
@@ -2186,6 +2190,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
21862190
<install as="CodeSniffer/Core/Tokenizer/BackfillNumericSeparatorTest.inc" name="tests/Core/Tokenizer/BackfillNumericSeparatorTest.inc" />
21872191
<install as="CodeSniffer/Core/Tokenizer/BitwiseOrTest.php" name="tests/Core/Tokenizer/BitwiseOrTest.php" />
21882192
<install as="CodeSniffer/Core/Tokenizer/BitwiseOrTest.inc" name="tests/Core/Tokenizer/BitwiseOrTest.inc" />
2193+
<install as="CodeSniffer/Core/Tokenizer/DefaultKeywordTest.php" name="tests/Core/Tokenizer/DefaultKeywordTest.php" />
2194+
<install as="CodeSniffer/Core/Tokenizer/DefaultKeywordTest.inc" name="tests/Core/Tokenizer/DefaultKeywordTest.inc" />
21892195
<install as="CodeSniffer/Core/Tokenizer/GotoLabelTest.php" name="tests/Core/Tokenizer/GotoLabelTest.php" />
21902196
<install as="CodeSniffer/Core/Tokenizer/GotoLabelTest.inc" name="tests/Core/Tokenizer/GotoLabelTest.inc" />
21912197
<install as="CodeSniffer/Core/Tokenizer/NamedFunctionCallArgumentsTest.php" name="tests/Core/Tokenizer/NamedFunctionCallArgumentsTest.php" />

‎src/Tokenizers/PHP.php

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -373,6 +373,7 @@ class PHP extends Tokenizer
373373
T_LOGICAL_OR => 2,
374374
T_LOGICAL_XOR => 3,
375375
T_MATCH => 5,
376+
T_MATCH_DEFAULT => 7,
376377
T_METHOD_C => 10,
377378
T_MINUS_EQUAL => 2,
378379
T_POW_EQUAL => 3,
@@ -1343,6 +1344,48 @@ protected function tokenize($string)
13431344
}//end if
13441345
}//end if
13451346

1347+
/*
1348+
Retokenize the T_DEFAULT in match control structures as T_MATCH_DEFAULT
1349+
to prevent scope being set and the scope for switch default statements
1350+
breaking.
1351+
*/
1352+
1353+
if ($tokenIsArray === true
1354+
&& $token[0] === T_DEFAULT
1355+
) {
1356+
for ($x = ($stackPtr + 1); $x < $numTokens; $x++) {
1357+
if ($tokens[$x] === ',') {
1358+
// Skip over potential trailing comma (supported in PHP).
1359+
continue;
1360+
}
1361+
1362+
if (is_array($tokens[$x]) === false
1363+
|| isset(Util\Tokens::$emptyTokens[$tokens[$x][0]]) === false
1364+
) {
1365+
// Non-empty, non-comma content.
1366+
break;
1367+
}
1368+
}
1369+
1370+
if (isset($tokens[$x]) === true
1371+
&& is_array($tokens[$x]) === true
1372+
&& $tokens[$x][0] === T_DOUBLE_ARROW
1373+
) {
1374+
$newToken = [];
1375+
$newToken['code'] = T_MATCH_DEFAULT;
1376+
$newToken['type'] = 'T_MATCH_DEFAULT';
1377+
$newToken['content'] = $token[1];
1378+
1379+
if (PHP_CODESNIFFER_VERBOSITY > 1) {
1380+
echo "\t\t* token $stackPtr changed from T_DEFAULT to T_MATCH_DEFAULT".PHP_EOL;
1381+
}
1382+
1383+
$finalTokens[$newStackPtr] = $newToken;
1384+
$newStackPtr++;
1385+
continue;
1386+
}//end if
1387+
}//end if
1388+
13461389
/*
13471390
Convert ? to T_NULLABLE OR T_INLINE_THEN
13481391
*/

‎src/Util/Tokens.php

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@
7777
define('T_FN_ARROW', 'PHPCS_T_FN_ARROW');
7878
define('T_TYPE_UNION', 'PHPCS_T_TYPE_UNION');
7979
define('T_PARAM_NAME', 'PHPCS_T_PARAM_NAME');
80+
define('T_MATCH_DEFAULT', 'PHPCS_T_MATCH_DEFAULT');
8081

8182
// Some PHP 5.5 tokens, replicated for lower versions.
8283
if (defined('T_FINALLY') === false) {

‎tests/Core/Tokenizer/DefaultKeywordTest.inc

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
<?php
2+
3+
function matchWithDefault($i) {
4+
return match ($i) {
5+
1 => 1,
6+
2 => 2,
7+
/* testSimpleMatchDefault */
8+
default => 'default',
9+
};
10+
}
11+
12+
function switchWithDefault($i) {
13+
switch ($i) {
14+
case 1:
15+
return 1;
16+
case 2:
17+
return 2;
18+
/* testSimpleSwitchDefault */
19+
default:
20+
return 'default';
21+
}
22+
}
23+
24+
function switchWithDefaultAndCurlies($i) {
25+
switch ($i) {
26+
case 1:
27+
return 1;
28+
case 2:
29+
return 2;
30+
/* testSimpleSwitchDefaultWithCurlies */
31+
default: {
32+
return 'default';
33+
}
34+
}
35+
}
36+
37+
function matchWithDefaultInSwitch() {
38+
switch ($something) {
39+
case 'foo':
40+
$var = [1, 2, 3];
41+
$var = match ($i) {
42+
1 => 1,
43+
/* testMatchDefaultNestedInSwitchCase1 */
44+
default => 'default',
45+
};
46+
continue;
47+
48+
case 'bar' :
49+
$i = callMe($a, $b);
50+
return match ($i) {
51+
1 => 1,
52+
/* testMatchDefaultNestedInSwitchCase2 */
53+
default => 'default',
54+
};
55+
56+
/* testSwitchDefault */
57+
default;
58+
echo 'something', match ($i) {
59+
1, => 1,
60+
/* testMatchDefaultNestedInSwitchDefault */
61+
default, => 'default',
62+
};
63+
break;
64+
}
65+
}
66+
67+
function switchWithDefaultInMatch() {
68+
$x = match ($y) {
69+
5, 8 => function($z) {
70+
switch($z) {
71+
case 'a';
72+
$var = [1, 2, 3];
73+
return 'a';
74+
/* testSwitchDefaultNestedInMatchCase */
75+
default:
76+
$var = [1, 2, 3];
77+
return 'default1';
78+
}
79+
},
80+
/* testMatchDefault */
81+
default => function($z) {
82+
switch($z) {
83+
case 'a':
84+
$i = callMe($a, $b);
85+
return 'b';
86+
/* testSwitchDefaultNestedInMatchDefault */
87+
default:
88+
$i = callMe($a, $b);
89+
return 'default2';
90+
}
91+
}
92+
};
93+
}

‎tests/Core/Tokenizer/DefaultKeywordTest.php

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
<?php
2+
/**
3+
* Tests the retokenization of the `default` keyword to T_MATCH_DEFAULT for PHP 8.0 match structures
4+
* and makes sure that the tokenization of switch `T_DEFAULT` structures is not aversely affected.
5+
*
6+
* @author Juliette Reinders Folmer <[email protected]>
7+
* @copyright 2020-2021 Squiz Pty Ltd (ABN 77 084 670 600)
8+
* @license https://github.com/squizlabs/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
9+
*/
10+
11+
namespace PHP_CodeSniffer\Tests\Core\Tokenizer;
12+
13+
use PHP_CodeSniffer\Tests\Core\AbstractMethodUnitTest;
14+
15+
class DefaultKeywordTest extends AbstractMethodUnitTest
16+
{
17+
18+
19+
/**
20+
* Test the retokenization of the `default` keyword for match structure to `T_MATCH_DEFAULT`.
21+
*
22+
* Note: Cases and default structures within a match structure do *NOT* get case/default scope
23+
* conditions, in contrast to case and default structures in switch control structures.
24+
*
25+
* @param string $testMarker The comment prefacing the target token.
26+
*
27+
* @dataProvider dataMatchDefault
28+
* @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize
29+
* @covers PHP_CodeSniffer\Tokenizers\Tokenizer::recurseScopeMap
30+
*
31+
* @return void
32+
*/
33+
public function testMatchDefault($testMarker)
34+
{
35+
$tokens = self::$phpcsFile->getTokens();
36+
37+
$token = $this->getTargetToken($testMarker, [T_MATCH_DEFAULT, T_DEFAULT]);
38+
$tokenArray = $tokens[$token];
39+
40+
$this->assertSame(T_MATCH_DEFAULT, $tokenArray['code'], 'Token tokenized as '.$tokenArray['type'].', not T_MATCH_DEFAULT (code)');
41+
$this->assertSame('T_MATCH_DEFAULT', $tokenArray['type'], 'Token tokenized as '.$tokenArray['type'].', not T_MATCH_DEFAULT (type)');
42+
43+
$this->assertArrayNotHasKey('scope_condition', $tokenArray, 'Scope condition is set');
44+
$this->assertArrayNotHasKey('scope_opener', $tokenArray, 'Scope opener is set');
45+
$this->assertArrayNotHasKey('scope_closer', $tokenArray, 'Scope closer is set');
46+
47+
}//end testMatchDefault()
48+
49+
50+
/**
51+
* Data provider.
52+
*
53+
* @see testMatchDefault()
54+
*
55+
* @return array
56+
*/
57+
public function dataMatchDefault()
58+
{
59+
return [
60+
'simple_match_default' => ['/* testSimpleMatchDefault */'],
61+
'match_default_in_switch_case_1' => ['/* testMatchDefaultNestedInSwitchCase1 */'],
62+
'match_default_in_switch_case_2' => ['/* testMatchDefaultNestedInSwitchCase2 */'],
63+
'match_default_in_switch_default' => ['/* testMatchDefaultNestedInSwitchDefault */'],
64+
'match_default_containing_switch' => ['/* testMatchDefault */'],
65+
];
66+
67+
}//end dataMatchDefault()
68+
69+
70+
/**
71+
* Verify that the retokenization of `T_DEFAULT` tokens in match constructs, doesn't negatively
72+
* impact the tokenization of `T_DEFAULT` tokens in switch control structures.
73+
*
74+
* Note: Cases and default structures within a switch control structure *do* get case/default scope
75+
* conditions.
76+
*
77+
* @param string $testMarker The comment prefacing the target token.
78+
* @param int $openerOffset The expected offset of the scope opener in relation to the testMarker.
79+
* @param int $closerOffset The expected offset of the scope closer in relation to the testMarker.
80+
* @param int|null $conditionStop The expected offset at which tokens stop having T_DEFAULT as a scope condition.
81+
*
82+
* @dataProvider dataSwitchDefault
83+
* @covers PHP_CodeSniffer\Tokenizers\Tokenizer::recurseScopeMap
84+
*
85+
* @return void
86+
*/
87+
public function testSwitchDefault($testMarker, $openerOffset, $closerOffset, $conditionStop=null)
88+
{
89+
$tokens = self::$phpcsFile->getTokens();
90+
91+
$token = $this->getTargetToken($testMarker, [T_MATCH_DEFAULT, T_DEFAULT]);
92+
$tokenArray = $tokens[$token];
93+
$expectedScopeOpener = ($token + $openerOffset);
94+
$expectedScopeCloser = ($token + $closerOffset);
95+
96+
$this->assertSame(T_DEFAULT, $tokenArray['code'], 'Token tokenized as '.$tokenArray['type'].', not T_DEFAULT (code)');
97+
$this->assertSame('T_DEFAULT', $tokenArray['type'], 'Token tokenized as '.$tokenArray['type'].', not T_DEFAULT (type)');
98+
99+
$this->assertArrayHasKey('scope_condition', $tokenArray, 'Scope condition is not set');
100+
$this->assertArrayHasKey('scope_opener', $tokenArray, 'Scope opener is not set');
101+
$this->assertArrayHasKey('scope_closer', $tokenArray, 'Scope closer is not set');
102+
$this->assertSame($token, $tokenArray['scope_condition'], 'Scope condition is not the T_DEFAULT token');
103+
$this->assertSame($expectedScopeOpener, $tokenArray['scope_opener'], 'Scope opener of the T_DEFAULT token incorrect');
104+
$this->assertSame($expectedScopeCloser, $tokenArray['scope_closer'], 'Scope closer of the T_DEFAULT token incorrect');
105+
106+
$opener = $tokenArray['scope_opener'];
107+
$this->assertArrayHasKey('scope_condition', $tokens[$opener], 'Opener scope condition is not set');
108+
$this->assertArrayHasKey('scope_opener', $tokens[$opener], 'Opener scope opener is not set');
109+
$this->assertArrayHasKey('scope_closer', $tokens[$opener], 'Opener scope closer is not set');
110+
$this->assertSame($token, $tokens[$opener]['scope_condition'], 'Opener scope condition is not the T_DEFAULT token');
111+
$this->assertSame($expectedScopeOpener, $tokens[$opener]['scope_opener'], 'T_DEFAULT opener scope opener token incorrect');
112+
$this->assertSame($expectedScopeCloser, $tokens[$opener]['scope_closer'], 'T_DEFAULT opener scope closer token incorrect');
113+
114+
$closer = $tokenArray['scope_closer'];
115+
$this->assertArrayHasKey('scope_condition', $tokens[$closer], 'Closer scope condition is not set');
116+
$this->assertArrayHasKey('scope_opener', $tokens[$closer], 'Closer scope opener is not set');
117+
$this->assertArrayHasKey('scope_closer', $tokens[$closer], 'Closer scope closer is not set');
118+
$this->assertSame($token, $tokens[$closer]['scope_condition'], 'Closer scope condition is not the T_DEFAULT token');
119+
$this->assertSame($expectedScopeOpener, $tokens[$closer]['scope_opener'], 'T_DEFAULT closer scope opener token incorrect');
120+
$this->assertSame($expectedScopeCloser, $tokens[$closer]['scope_closer'], 'T_DEFAULT closer scope closer token incorrect');
121+
122+
if (($opener + 1) !== $closer) {
123+
$end = $closer;
124+
if (isset($conditionStop) === true) {
125+
$end = $conditionStop;
126+
}
127+
128+
for ($i = ($opener + 1); $i < $end; $i++) {
129+
$this->assertArrayHasKey(
130+
$token,
131+
$tokens[$i]['conditions'],
132+
'T_DEFAULT condition not added for token belonging to the T_DEFAULT structure'
133+
);
134+
}
135+
}
136+
137+
}//end testSwitchDefault()
138+
139+
140+
/**
141+
* Data provider.
142+
*
143+
* @see testSwitchDefault()
144+
*
145+
* @return array
146+
*/
147+
public function dataSwitchDefault()
148+
{
149+
return [
150+
'simple_switch_default' => [
151+
'/* testSimpleSwitchDefault */',
152+
1,
153+
4,
154+
],
155+
'simple_switch_default_with_curlies' => [
156+
// For a default structure with curly braces, the scope opener
157+
// will be the open curly and the closer the close curly.
158+
// However, scope conditions will not be set for open to close,
159+
// but only for the open token up to the "break/return/continue" etc.
160+
'/* testSimpleSwitchDefaultWithCurlies */',
161+
3,
162+
12,
163+
6,
164+
],
165+
'switch_default_toplevel' => [
166+
'/* testSwitchDefault */',
167+
1,
168+
43,
169+
],
170+
'switch_default_nested_in_match_case' => [
171+
'/* testSwitchDefaultNestedInMatchCase */',
172+
1,
173+
20,
174+
],
175+
'switch_default_nested_in_match_default' => [
176+
'/* testSwitchDefaultNestedInMatchDefault */',
177+
1,
178+
18,
179+
],
180+
];
181+
182+
}//end dataSwitchDefault()
183+
184+
185+
}//end class

0 commit comments

Comments
 (0)