Skip to content

Define polyfilled T_* constants from Tokenizer as int #1292

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: 4.x
Choose a base branch
from
Open

Define polyfilled T_* constants from Tokenizer as int #1292

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
152 changes: 75 additions & 77 deletions src/Util/Tokens.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

namespace PHP_CodeSniffer\Util;

// PHPCS native tokens.
define('T_NONE', 'PHPCS_T_NONE');
define('T_OPEN_CURLY_BRACKET', 'PHPCS_T_OPEN_CURLY_BRACKET');
define('T_CLOSE_CURLY_BRACKET', 'PHPCS_T_CLOSE_CURLY_BRACKET');
Expand Down Expand Up @@ -70,79 +71,6 @@
define('T_TYPE_OPEN_PARENTHESIS', 'PHPCS_T_TYPE_OPEN_PARENTHESIS');
define('T_TYPE_CLOSE_PARENTHESIS', 'PHPCS_T_TYPE_CLOSE_PARENTHESIS');

/*
* {@internal IMPORTANT: all PHP native polyfilled tokens MUST be added to the
* `PHP_CodeSniffer\Tests\Core\Util\Tokens\TokenNameTest::dataPolyfilledPHPNativeTokens()` test method!}
*/

// Some PHP 7.4 tokens, replicated for lower versions.
if (defined('T_COALESCE_EQUAL') === false) {
define('T_COALESCE_EQUAL', 'PHPCS_T_COALESCE_EQUAL');
}

if (defined('T_BAD_CHARACTER') === false) {
define('T_BAD_CHARACTER', 'PHPCS_T_BAD_CHARACTER');
}

if (defined('T_FN') === false) {
define('T_FN', 'PHPCS_T_FN');
}

// Some PHP 8.0 tokens, replicated for lower versions.
if (defined('T_NULLSAFE_OBJECT_OPERATOR') === false) {
define('T_NULLSAFE_OBJECT_OPERATOR', 'PHPCS_T_NULLSAFE_OBJECT_OPERATOR');
}

if (defined('T_NAME_QUALIFIED') === false) {
define('T_NAME_QUALIFIED', 'PHPCS_T_NAME_QUALIFIED');
}

if (defined('T_NAME_FULLY_QUALIFIED') === false) {
define('T_NAME_FULLY_QUALIFIED', 'PHPCS_T_NAME_FULLY_QUALIFIED');
}

if (defined('T_NAME_RELATIVE') === false) {
define('T_NAME_RELATIVE', 'PHPCS_T_NAME_RELATIVE');
}

if (defined('T_MATCH') === false) {
define('T_MATCH', 'PHPCS_T_MATCH');
}

if (defined('T_ATTRIBUTE') === false) {
define('T_ATTRIBUTE', 'PHPCS_T_ATTRIBUTE');
}

// Some PHP 8.1 tokens, replicated for lower versions.
if (defined('T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG') === false) {
define('T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG', 'PHPCS_T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG');
}

if (defined('T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG') === false) {
define('T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG', 'PHPCS_T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG');
}

if (defined('T_READONLY') === false) {
define('T_READONLY', 'PHPCS_T_READONLY');
}

if (defined('T_ENUM') === false) {
define('T_ENUM', 'PHPCS_T_ENUM');
}

// Some PHP 8.4 tokens, replicated for lower versions.
if (defined('T_PUBLIC_SET') === false) {
define('T_PUBLIC_SET', 'PHPCS_T_PUBLIC_SET');
}

if (defined('T_PROTECTED_SET') === false) {
define('T_PROTECTED_SET', 'PHPCS_T_PROTECTED_SET');
}

if (defined('T_PRIVATE_SET') === false) {
define('T_PRIVATE_SET', 'PHPCS_T_PRIVATE_SET');
}

// Tokens used for parsing doc blocks.
define('T_DOC_COMMENT_STAR', 'PHPCS_T_DOC_COMMENT_STAR');
define('T_DOC_COMMENT_WHITESPACE', 'PHPCS_T_DOC_COMMENT_WHITESPACE');
Expand All @@ -158,6 +86,8 @@
define('T_PHPCS_IGNORE', 'PHPCS_T_PHPCS_IGNORE');
define('T_PHPCS_IGNORE_FILE', 'PHPCS_T_PHPCS_IGNORE_FILE');

Tokens::polyfillTokenizerConstants();

final class Tokens
{

Expand Down Expand Up @@ -606,6 +536,13 @@ final class Tokens
T_YIELD_FROM => T_YIELD_FROM,
];

/**
* Mapping table for polyfilled constants
*
* @var array<int, string>
*/
private static $polyfillMappingTable = [];

/**
* The token weightings.
*
Expand Down Expand Up @@ -937,12 +874,12 @@ final class Tokens
*/
public static function tokenName($token)
{
if (is_string($token) === false) {
// PHP-supplied token name.
return token_name($token);
if (is_string($token) === true) {
// PHPCS native token.
return substr($token, 6);
}

return substr($token, 6);
return (self::$polyfillMappingTable[$token] ?? token_name($token));
}


Expand Down Expand Up @@ -985,4 +922,65 @@ public static function getHighestWeightedToken(array $tokens)

return $highestType;
}


/**
* Polyfill tokenizer (T_*) constants.
*
* {@internal IMPORTANT: all PHP native polyfilled tokens MUST be added to the
* `PHP_CodeSniffer\Tests\Core\Util\Tokens\TokenNameTest::dataPolyfilledPHPNativeTokens()` test method!}
*
* @return void
*/
public static function polyfillTokenizerConstants(): void
{
// Ideally this would be a private class constant. We cannot do that
// here as the constants that we are polyfilling in this method are
// used in some of the class constants for this class. If we reference
// any class constants or properties before this method has fully run,
// PHP will intitialise the class, leading to warnings about undefined
// T_* constants.
$tokensToPolyfill = [
'T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG',
'T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG',
'T_ATTRIBUTE',
'T_BAD_CHARACTER',
'T_COALESCE_EQUAL',
'T_ENUM',
'T_FN',
'T_MATCH',
'T_NAME_FULLY_QUALIFIED',
'T_NAME_QUALIFIED',
'T_NAME_RELATIVE',
'T_NULLSAFE_OBJECT_OPERATOR',
'T_PRIVATE_SET',
'T_PROTECTED_SET',
'T_PUBLIC_SET',
'T_READONLY',
];

// <https://www.php.net/manual/en/tokens.php>
// The PHP manual suggests "using big numbers like 10000" for
// polyfilled T_* constants. We have arbitrarily chosen to start our
// numbering scheme from 135_000.
$nextTokenNumber = 135000;

$polyfillMappingTable = [];

foreach ($tokensToPolyfill as $tokenName) {
if (defined($tokenName) === false) {
while (isset($polyfillMappingTable[$nextTokenNumber]) === true) {
$nextTokenNumber++;
}

define($tokenName, $nextTokenNumber);
}

$polyfillMappingTable[constant($tokenName)] = $tokenName;
}

// Be careful to not reference this class anywhere in this method until
// *after* all constants have been polyfilled.
self::$polyfillMappingTable = $polyfillMappingTable;
}
}
1 change: 1 addition & 0 deletions tests/Core/Util/Tokens/TokenNameTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
* Tests for the \PHP_CodeSniffer\Util\Tokens::tokenName() method.
*
* @covers \PHP_CodeSniffer\Util\Tokens::tokenName
* @covers \PHP_CodeSniffer\Util\Tokens::polyfillTokenizerConstants
*/
final class TokenNameTest extends TestCase
{
Expand Down