Skip to content

Commit 9fcdd17

Browse files
freddenfredden
committed
Define polyfilled constants from Tokenizer as int
The value of these constants are not stable, and therefore already cannot be relied upon. This is because the specific values that PHP assigns can change with different versions of PHP. PHPCS does not use the values of these constants (other than to look up their name using the Tokens::tokenName() method). There are other tools which also polyfill these constants. Some of those tools also perform validation on the value for these constants. In order to play nicely with the arbitrary validation that other tools perform on these constants, we are switching from string values to integer values. All PHPCS 'native' tokens currently have reliable values. In line with PHP T_* constants, the values of these tokens should never be relied upon. In a future version of PHPCS, the values for these tokens will switch from strings to integers. Existing tests already cover the use of these constants and do not require adjustment for the code being changed here.
1 parent c4adbbc commit 9fcdd17

File tree

2 files changed

+77
-82
lines changed

2 files changed

+77
-82
lines changed

‎src/Util/Tokens.php‎

Lines changed: 76 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010

1111
namespace PHP_CodeSniffer\Util;
1212

13+
// PHPCS native tokens.
1314
define('T_NONE', 'PHPCS_T_NONE');
1415
define('T_OPEN_CURLY_BRACKET', 'PHPCS_T_OPEN_CURLY_BRACKET');
1516
define('T_CLOSE_CURLY_BRACKET', 'PHPCS_T_CLOSE_CURLY_BRACKET');
@@ -70,84 +71,6 @@
7071
define('T_TYPE_OPEN_PARENTHESIS', 'PHPCS_T_TYPE_OPEN_PARENTHESIS');
7172
define('T_TYPE_CLOSE_PARENTHESIS', 'PHPCS_T_TYPE_CLOSE_PARENTHESIS');
7273

73-
/*
74-
* {@internal IMPORTANT: all PHP native polyfilled tokens MUST be added to the
75-
* `PHP_CodeSniffer\Tests\Core\Util\Tokens\TokenNameTest::dataPolyfilledPHPNativeTokens()` test method!}
76-
*/
77-
78-
// Some PHP 7.4 tokens, replicated for lower versions.
79-
if (defined('T_COALESCE_EQUAL') === false) {
80-
define('T_COALESCE_EQUAL', 'PHPCS_T_COALESCE_EQUAL');
81-
}
82-
83-
if (defined('T_BAD_CHARACTER') === false) {
84-
define('T_BAD_CHARACTER', 'PHPCS_T_BAD_CHARACTER');
85-
}
86-
87-
if (defined('T_FN') === false) {
88-
define('T_FN', 'PHPCS_T_FN');
89-
}
90-
91-
// Some PHP 8.0 tokens, replicated for lower versions.
92-
if (defined('T_NULLSAFE_OBJECT_OPERATOR') === false) {
93-
define('T_NULLSAFE_OBJECT_OPERATOR', 'PHPCS_T_NULLSAFE_OBJECT_OPERATOR');
94-
}
95-
96-
if (defined('T_NAME_QUALIFIED') === false) {
97-
define('T_NAME_QUALIFIED', 'PHPCS_T_NAME_QUALIFIED');
98-
}
99-
100-
if (defined('T_NAME_FULLY_QUALIFIED') === false) {
101-
define('T_NAME_FULLY_QUALIFIED', 'PHPCS_T_NAME_FULLY_QUALIFIED');
102-
}
103-
104-
if (defined('T_NAME_RELATIVE') === false) {
105-
define('T_NAME_RELATIVE', 'PHPCS_T_NAME_RELATIVE');
106-
}
107-
108-
if (defined('T_MATCH') === false) {
109-
define('T_MATCH', 'PHPCS_T_MATCH');
110-
}
111-
112-
if (defined('T_ATTRIBUTE') === false) {
113-
define('T_ATTRIBUTE', 'PHPCS_T_ATTRIBUTE');
114-
}
115-
116-
// Some PHP 8.1 tokens, replicated for lower versions.
117-
if (defined('T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG') === false) {
118-
define('T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG', 'PHPCS_T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG');
119-
}
120-
121-
if (defined('T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG') === false) {
122-
define('T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG', 'PHPCS_T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG');
123-
}
124-
125-
if (defined('T_READONLY') === false) {
126-
define('T_READONLY', 'PHPCS_T_READONLY');
127-
}
128-
129-
if (defined('T_ENUM') === false) {
130-
define('T_ENUM', 'PHPCS_T_ENUM');
131-
}
132-
133-
// Some PHP 8.4 tokens, replicated for lower versions.
134-
if (defined('T_PUBLIC_SET') === false) {
135-
define('T_PUBLIC_SET', 'PHPCS_T_PUBLIC_SET');
136-
}
137-
138-
if (defined('T_PROTECTED_SET') === false) {
139-
define('T_PROTECTED_SET', 'PHPCS_T_PROTECTED_SET');
140-
}
141-
142-
if (defined('T_PRIVATE_SET') === false) {
143-
define('T_PRIVATE_SET', 'PHPCS_T_PRIVATE_SET');
144-
}
145-
146-
// Some PHP 8.5 tokens, replicated for lower versions.
147-
if (defined('T_VOID_CAST') === false) {
148-
define('T_VOID_CAST', 'PHPCS_T_VOID_CAST');
149-
}
150-
15174
// Tokens used for parsing doc blocks.
15275
define('T_DOC_COMMENT_STAR', 'PHPCS_T_DOC_COMMENT_STAR');
15376
define('T_DOC_COMMENT_WHITESPACE', 'PHPCS_T_DOC_COMMENT_WHITESPACE');
@@ -163,6 +86,8 @@
16386
define('T_PHPCS_IGNORE', 'PHPCS_T_PHPCS_IGNORE');
16487
define('T_PHPCS_IGNORE_FILE', 'PHPCS_T_PHPCS_IGNORE_FILE');
16588

89+
Tokens::polyfillTokenizerConstants();
90+
16691
final class Tokens
16792
{
16893

@@ -612,6 +537,13 @@ final class Tokens
612537
T_YIELD_FROM => T_YIELD_FROM,
613538
];
614539

540+
/**
541+
* Mapping table for polyfilled constants
542+
*
543+
* @var array<int, string>
544+
*/
545+
private static $polyfillMappingTable = [];
546+
615547
/**
616548
* The token weightings.
617549
*
@@ -943,12 +875,12 @@ final class Tokens
943875
*/
944876
public static function tokenName($token)
945877
{
946-
if (is_string($token) === false) {
947-
// PHP-supplied token name.
948-
return token_name($token);
878+
if (is_string($token) === true) {
879+
// PHPCS native token.
880+
return substr($token, 6);
949881
}
950882

951-
return substr($token, 6);
883+
return (self::$polyfillMappingTable[$token] ?? token_name($token));
952884
}
953885

954886

@@ -991,4 +923,66 @@ public static function getHighestWeightedToken(array $tokens)
991923

992924
return $highestType;
993925
}
926+
927+
928+
/**
929+
* Polyfill tokenizer (T_*) constants.
930+
*
931+
* {@internal IMPORTANT: all PHP native polyfilled tokens MUST be added to the
932+
* `PHP_CodeSniffer\Tests\Core\Util\Tokens\TokenNameTest::dataPolyfilledPHPNativeTokens()` test method!}
933+
*
934+
* @return void
935+
*/
936+
public static function polyfillTokenizerConstants(): void
937+
{
938+
// Ideally this would be a private class constant. We cannot do that
939+
// here as the constants that we are polyfilling in this method are
940+
// used in some of the class constants for this class. If we reference
941+
// any class constants or properties before this method has fully run,
942+
// PHP will intitialise the class, leading to warnings about undefined
943+
// T_* constants.
944+
$tokensToPolyfill = [
945+
'T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG',
946+
'T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG',
947+
'T_ATTRIBUTE',
948+
'T_BAD_CHARACTER',
949+
'T_COALESCE_EQUAL',
950+
'T_ENUM',
951+
'T_FN',
952+
'T_MATCH',
953+
'T_NAME_FULLY_QUALIFIED',
954+
'T_NAME_QUALIFIED',
955+
'T_NAME_RELATIVE',
956+
'T_NULLSAFE_OBJECT_OPERATOR',
957+
'T_PRIVATE_SET',
958+
'T_PROTECTED_SET',
959+
'T_PUBLIC_SET',
960+
'T_READONLY',
961+
'T_VOID_CAST',
962+
];
963+
964+
// <https://www.php.net/manual/en/tokens.php>
965+
// The PHP manual suggests "using big numbers like 10000" for
966+
// polyfilled T_* constants. We have arbitrarily chosen to start our
967+
// numbering scheme from 135_000.
968+
$nextTokenNumber = 135000;
969+
970+
$polyfillMappingTable = [];
971+
972+
foreach ($tokensToPolyfill as $tokenName) {
973+
if (defined($tokenName) === false) {
974+
while (isset($polyfillMappingTable[$nextTokenNumber]) === true) {
975+
$nextTokenNumber++;
976+
}
977+
978+
define($tokenName, $nextTokenNumber);
979+
}
980+
981+
$polyfillMappingTable[constant($tokenName)] = $tokenName;
982+
}
983+
984+
// Be careful to not reference this class anywhere in this method until
985+
// *after* all constants have been polyfilled.
986+
self::$polyfillMappingTable = $polyfillMappingTable;
987+
}
994988
}

‎tests/Core/Util/Tokens/TokenNameTest.php‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
* Tests for the \PHP_CodeSniffer\Util\Tokens::tokenName() method.
1717
*
1818
* @covers \PHP_CodeSniffer\Util\Tokens::tokenName
19+
* @covers \PHP_CodeSniffer\Util\Tokens::polyfillTokenizerConstants
1920
*/
2021
final class TokenNameTest extends TestCase
2122
{

0 commit comments

Comments
 (0)