Skip to content

Commit eafb5ff

Browse files
jrfnljrfnl
committed
Tokenizer/PHP: bug fix - handling of \true/\false/\null
Improve handling of `true`, `false` and `null` when prefixed with a namespace separator. In PHP, `true === \true` and `\null === null`, however, the PHPCS tokenizer would not tokenize this correctly. In normal circumstances, `true` is tokenized as `T_TRUE`, `false` as `T_FALSE` and `null` as `T_NULL`, but when prefixed with a namespace separator, these would be tokenized as `T_STRING`. While this is intentional when the name is part of a larger namespaced name, for stand-alone `\true`/`\false`/`\null`, the tokenizer should use the appropriate tokens for the keywords. Fixed now. Includes tests. Note: the changes in the `PHP` class will be most straight-forward to review while ignoring whitespace changes.
1 parent be74da1 commit eafb5ff

File tree

5 files changed

+144
-18
lines changed

5 files changed

+144
-18
lines changed

‎src/Tokenizers/PHP.php‎

Lines changed: 59 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1247,6 +1247,14 @@ protected function tokenize($string)
12471247
++$newStackPtr;
12481248

12491249
$name = ltrim($name, '\\');
1250+
1251+
$nameLc = strtolower($name);
1252+
if ($nameLc === 'true' || $nameLc === 'false' || $nameLc === 'null') {
1253+
$newToken = self::standardiseToken([T_STRING, $name]);
1254+
$finalTokens[$newStackPtr] = $newToken;
1255+
++$newStackPtr;
1256+
$name = '';
1257+
}
12501258
}
12511259

12521260
if ($token[0] === T_NAME_RELATIVE) {
@@ -1267,27 +1275,29 @@ protected function tokenize($string)
12671275
$name = substr($name, 10);
12681276
}
12691277

1270-
$parts = explode('\\', $name);
1271-
$partCount = count($parts);
1272-
$lastPart = ($partCount - 1);
1278+
if ($name !== '') {
1279+
$parts = explode('\\', $name);
1280+
$partCount = count($parts);
1281+
$lastPart = ($partCount - 1);
12731282

1274-
foreach ($parts as $i => $part) {
1275-
$newToken = [];
1276-
$newToken['code'] = T_STRING;
1277-
$newToken['type'] = 'T_STRING';
1278-
$newToken['content'] = $part;
1279-
$finalTokens[$newStackPtr] = $newToken;
1280-
++$newStackPtr;
1281-
1282-
if ($i !== $lastPart) {
1283+
foreach ($parts as $i => $part) {
12831284
$newToken = [];
1284-
$newToken['code'] = T_NS_SEPARATOR;
1285-
$newToken['type'] = 'T_NS_SEPARATOR';
1286-
$newToken['content'] = '\\';
1285+
$newToken['code'] = T_STRING;
1286+
$newToken['type'] = 'T_STRING';
1287+
$newToken['content'] = $part;
12871288
$finalTokens[$newStackPtr] = $newToken;
12881289
++$newStackPtr;
1290+
1291+
if ($i !== $lastPart) {
1292+
$newToken = [];
1293+
$newToken['code'] = T_NS_SEPARATOR;
1294+
$newToken['type'] = 'T_NS_SEPARATOR';
1295+
$newToken['content'] = '\\';
1296+
$finalTokens[$newStackPtr] = $newToken;
1297+
++$newStackPtr;
1298+
}
12891299
}
1290-
}
1300+
}//end if
12911301

12921302
if (PHP_CODESNIFFER_VERBOSITY > 1) {
12931303
$type = Tokens::tokenName($token[0]);
@@ -2427,16 +2437,47 @@ function return types. We want to keep the parenthesis map clean,
24272437
} else if (isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true
24282438
&& $finalTokens[$lastNotEmptyToken]['code'] !== T_CONST
24292439
) {
2430-
$preserveTstring = true;
2440+
$preserveTstring = true;
2441+
$tokenContentLower = strtolower($token[1]);
24312442

24322443
// Special case for syntax like: return new self/new parent
24332444
// where self/parent should not be a string.
2434-
$tokenContentLower = strtolower($token[1]);
24352445
if ($finalTokens[$lastNotEmptyToken]['code'] === T_NEW
24362446
&& ($tokenContentLower === 'self' || $tokenContentLower === 'parent')
24372447
) {
24382448
$preserveTstring = false;
24392449
}
2450+
2451+
// Special case for syntax like: \null or \false
2452+
// where null/false/true should not be a string,
2453+
// however leave as T_STRING if part of a larger namespaced name.
2454+
if ($finalTokens[$lastNotEmptyToken]['code'] === T_NS_SEPARATOR
2455+
&& ($tokenContentLower === 'null' || $tokenContentLower === 'true' || $tokenContentLower === 'false')
2456+
) {
2457+
for ($i = ($lastNotEmptyToken - 1); $i >= 0; $i--) {
2458+
if (isset(Tokens::$emptyTokens[$finalTokens[$i]['code']]) === true) {
2459+
continue;
2460+
}
2461+
2462+
break;
2463+
}
2464+
2465+
if ($finalTokens[$i]['code'] !== T_STRING && $finalTokens[$i]['code'] !== T_NAMESPACE) {
2466+
for ($j = ($stackPtr + 1); $j < $numTokens; $j++) {
2467+
if (is_array($tokens[$j]) === true
2468+
&& isset(Tokens::$emptyTokens[$tokens[$j][0]]) === true
2469+
) {
2470+
continue;
2471+
}
2472+
2473+
break;
2474+
}
2475+
2476+
if (is_array($tokens[$j]) === false || $tokens[$j][0] !== T_NS_SEPARATOR) {
2477+
$preserveTstring = false;
2478+
}
2479+
}
2480+
}//end if
24402481
} else if ($finalTokens[$lastNotEmptyToken]['content'] === '&') {
24412482
// Function names for functions declared to return by reference.
24422483
for ($i = ($lastNotEmptyToken - 1); $i >= 0; $i--) {

‎tests/Core/Tokenizers/PHP/OtherContextSensitiveKeywordsTest.inc‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,14 @@ $instantiated4 = new /* testClassInstantiationFalseIsString */ False();
4141
$instantiated5 = new /* testClassInstantiationTrueIsString */ true ();
4242
$instantiated6 = new /* testClassInstantiationNullIsString */ null();
4343

44+
$null = /* testFullyQualifiedNullIsKeyword */ \NULL;
45+
$true = /* testFullyQualifiedTrueIsKeyword */ \True;
46+
$false = /* testFullyQualifiedFalseIsKeyword */ \false;
47+
48+
$notNull = /* testNamespacedNameNullIsString */ \NULL\CONSTANT_NAME;
49+
$notTrue = /* testNamespacedNameTrueIsString */ Namespace\True\Relative;
50+
$notFalse = /* testNamespacedNameFalseIsString */ \Fully\Qualified\false;
51+
4452
function standAloneFalseTrueNullTypesAndMore(
4553
/* testFalseIsKeywordAsParamType */ false $paramA,
4654
/* testTrueIsKeywordAsParamType */ true $paramB,

‎tests/Core/Tokenizers/PHP/OtherContextSensitiveKeywordsTest.php‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,10 @@ public static function dataStrings()
8989
'class instantiation: true' => ['/* testClassInstantiationTrueIsString */'],
9090
'class instantiation: null' => ['/* testClassInstantiationNullIsString */'],
9191

92+
'part of namespaced name: null' => ['/* testNamespacedNameNullIsString */'],
93+
'part of namespaced name: true' => ['/* testNamespacedNameTrueIsString */'],
94+
'part of namespaced name: false' => ['/* testNamespacedNameFalseIsString */'],
95+
9296
'constant declaration: false as name after type' => ['/* testFalseIsNameForTypedConstant */'],
9397
'constant declaration: true as name after type' => ['/* testTrueIsNameForTypedConstant */'],
9498
'constant declaration: null as name after type' => ['/* testNullIsNameForTypedConstant */'],
@@ -157,6 +161,19 @@ public static function dataKeywords()
157161
'expectedTokenType' => 'T_SELF',
158162
],
159163

164+
'null: fully qualified' => [
165+
'testMarker' => '/* testFullyQualifiedNullIsKeyword */',
166+
'expectedTokenType' => 'T_NULL',
167+
],
168+
'true: fully qualified' => [
169+
'testMarker' => '/* testFullyQualifiedTrueIsKeyword */',
170+
'expectedTokenType' => 'T_TRUE',
171+
],
172+
'false: fully qualified' => [
173+
'testMarker' => '/* testFullyQualifiedFalseIsKeyword */',
174+
'expectedTokenType' => 'T_FALSE',
175+
],
176+
160177
'false: param type declaration' => [
161178
'testMarker' => '/* testFalseIsKeywordAsParamType */',
162179
'expectedTokenType' => 'T_FALSE',

‎tests/Core/Tokenizers/PHP/UndoNamespacedNameSingleTokenTest.inc‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,15 @@ class MyClass
130130

131131
/* testInstanceOfPartiallyQualified */
132132
$is = $obj instanceof Partially\ClassName;
133+
134+
/* testQualifiedNull */
135+
$a = \NULL;
136+
137+
/* testQualifiedFalse */
138+
$a = \false;
139+
140+
/* testQualifiedTrue */
141+
$a = \True;
133142
}
134143
}
135144

‎tests/Core/Tokenizers/PHP/UndoNamespacedNameSingleTokenTest.php‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1172,6 +1172,57 @@ public static function dataIdentifierTokenization()
11721172
],
11731173
],
11741174
],
1175+
'qualified "null"' => [
1176+
'testMarker' => '/* testQualifiedNull */',
1177+
'expectedTokens' => [
1178+
[
1179+
'type' => 'T_NS_SEPARATOR',
1180+
'content' => '\\',
1181+
],
1182+
[
1183+
'type' => 'T_NULL',
1184+
'content' => 'NULL',
1185+
],
1186+
[
1187+
'type' => 'T_SEMICOLON',
1188+
'content' => ';',
1189+
],
1190+
],
1191+
],
1192+
'qualified "false"' => [
1193+
'testMarker' => '/* testQualifiedFalse */',
1194+
'expectedTokens' => [
1195+
[
1196+
'type' => 'T_NS_SEPARATOR',
1197+
'content' => '\\',
1198+
],
1199+
[
1200+
'type' => 'T_FALSE',
1201+
'content' => 'false',
1202+
],
1203+
[
1204+
'type' => 'T_SEMICOLON',
1205+
'content' => ';',
1206+
],
1207+
],
1208+
],
1209+
'qualified "true"' => [
1210+
'testMarker' => '/* testQualifiedTrue */',
1211+
'expectedTokens' => [
1212+
[
1213+
'type' => 'T_NS_SEPARATOR',
1214+
'content' => '\\',
1215+
],
1216+
[
1217+
'type' => 'T_TRUE',
1218+
'content' => 'True',
1219+
],
1220+
[
1221+
'type' => 'T_SEMICOLON',
1222+
'content' => ';',
1223+
],
1224+
],
1225+
],
11751226
'function call, namespace relative, with whitespace (invalid in PHP 8)' => [
11761227
'testMarker' => '/* testInvalidInPHP8Whitespace */',
11771228
'expectedTokens' => [

0 commit comments

Comments
 (0)