Skip to content

Commit 013943d

Browse files
gsherwoodgsherwood
committed
Merge branch 'feature/tokenizer_attributes' of https://github.com/alekitto/PHP_CodeSniffer 
2 parents fe68f36 + 6367e71 commit 013943d

File tree

9 files changed

+852
-0
lines changed

9 files changed

+852
-0
lines changed

‎package.xml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
227227
<dir name="Tokenizer">
228228
<file baseinstalldir="" name="AnonClassParenthesisOwnerTest.inc" role="test" />
229229
<file baseinstalldir="" name="AnonClassParenthesisOwnerTest.php" role="test" />
230+
<file baseinstalldir="" name="AttributesTest.inc" role="test" />
231+
<file baseinstalldir="" name="AttributesTest.php" role="test" />
230232
<file baseinstalldir="" name="BackfillFnTokenTest.inc" role="test" />
231233
<file baseinstalldir="" name="BackfillFnTokenTest.php" role="test" />
232234
<file baseinstalldir="" name="BackfillMatchTokenTest.inc" role="test" />
@@ -2139,6 +2141,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
21392141
<install as="CodeSniffer/Core/Sniffs/AbstractArraySniffTestable.php" name="tests/Core/Sniffs/AbstractArraySniffTestable.php" />
21402142
<install as="CodeSniffer/Core/Tokenizer/AnonClassParenthesisOwnerTest.php" name="tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.php" />
21412143
<install as="CodeSniffer/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" name="tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" />
2144+
<install as="CodeSniffer/Core/Tokenizer/AttributesTest.php" name="tests/Core/Tokenizer/AttributesTest.php" />
2145+
<install as="CodeSniffer/Core/Tokenizer/AttributesTest.inc" name="tests/Core/Tokenizer/AttributesTest.inc" />
21422146
<install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.php" name="tests/Core/Tokenizer/BackfillFnTokenTest.php" />
21432147
<install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.inc" name="tests/Core/Tokenizer/BackfillFnTokenTest.inc" />
21442148
<install as="CodeSniffer/Core/Tokenizer/BackfillMatchTokenTest.php" name="tests/Core/Tokenizer/BackfillMatchTokenTest.php" />
@@ -2223,6 +2227,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
22232227
<install as="CodeSniffer/Core/Sniffs/AbstractArraySniffTestable.php" name="tests/Core/Sniffs/AbstractArraySniffTestable.php" />
22242228
<install as="CodeSniffer/Core/Tokenizer/AnonClassParenthesisOwnerTest.php" name="tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.php" />
22252229
<install as="CodeSniffer/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" name="tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" />
2230+
<install as="CodeSniffer/Core/Tokenizer/AttributesTest.php" name="tests/Core/Tokenizer/AttributesTest.php" />
2231+
<install as="CodeSniffer/Core/Tokenizer/AttributesTest.inc" name="tests/Core/Tokenizer/AttributesTest.inc" />
22262232
<install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.php" name="tests/Core/Tokenizer/BackfillFnTokenTest.php" />
22272233
<install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.inc" name="tests/Core/Tokenizer/BackfillFnTokenTest.inc" />
22282234
<install as="CodeSniffer/Core/Tokenizer/BackfillMatchTokenTest.php" name="tests/Core/Tokenizer/BackfillMatchTokenTest.php" />

‎src/Files/File.php

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1813,6 +1813,7 @@ public function getMemberProperties($stackPtr)
18131813
T_SEMICOLON,
18141814
T_OPEN_CURLY_BRACKET,
18151815
T_CLOSE_CURLY_BRACKET,
1816+
T_ATTRIBUTE_END,
18161817
],
18171818
($stackPtr - 1)
18181819
);

‎src/Tokenizers/PHP.php

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -903,6 +903,50 @@ protected function tokenize($string)
903903
continue;
904904
}//end if
905905

906+
/*
907+
PHP 8.0 Attributes
908+
*/
909+
910+
if (PHP_VERSION_ID < 80000
911+
&& $token[0] === T_COMMENT
912+
&& strpos($token[1], '#[') === 0
913+
) {
914+
$subTokens = $this->parsePhpAttribute($tokens, $stackPtr);
915+
if ($subTokens !== null) {
916+
array_splice($tokens, $stackPtr, 1, $subTokens);
917+
$numTokens = count($tokens);
918+
919+
$tokenIsArray = true;
920+
$token = $tokens[$stackPtr];
921+
} else {
922+
$token[0] = T_ATTRIBUTE;
923+
}
924+
}
925+
926+
if ($tokenIsArray === true
927+
&& $token[0] === T_ATTRIBUTE
928+
) {
929+
// Go looking for the close bracket.
930+
$bracketCloser = $this->findCloser($tokens, ($stackPtr + 1), ['[', '#['], ']');
931+
932+
$newToken = [];
933+
$newToken['code'] = T_ATTRIBUTE;
934+
$newToken['type'] = 'T_ATTRIBUTE';
935+
$newToken['content'] = '#[';
936+
$finalTokens[$newStackPtr] = $newToken;
937+
938+
$tokens[$bracketCloser] = [];
939+
$tokens[$bracketCloser][0] = T_ATTRIBUTE_END;
940+
$tokens[$bracketCloser][1] = ']';
941+
942+
if (PHP_CODESNIFFER_VERBOSITY > 1) {
943+
echo "\t\t* token $bracketCloser changed from T_CLOSE_SQUARE_BRACKET to T_ATTRIBUTE_END".PHP_EOL;
944+
}
945+
946+
$newStackPtr++;
947+
continue;
948+
}//end if
949+
906950
/*
907951
Tokenize the parameter labels for PHP 8.0 named parameters as a special T_PARAM_NAME
908952
token and ensure that the colon after it is always T_COLON.
@@ -1857,6 +1901,7 @@ function return types. We want to keep the parenthesis map clean,
18571901
T_CLASS => true,
18581902
T_EXTENDS => true,
18591903
T_IMPLEMENTS => true,
1904+
T_ATTRIBUTE => true,
18601905
T_NEW => true,
18611906
T_CONST => true,
18621907
T_NS_SEPARATOR => true,
@@ -2114,6 +2159,8 @@ protected function processAdditional()
21142159
echo "\t*** START ADDITIONAL PHP PROCESSING ***".PHP_EOL;
21152160
}
21162161

2162+
$this->createAttributesNestingMap();
2163+
21172164
$numTokens = count($this->tokens);
21182165
for ($i = ($numTokens - 1); $i >= 0; $i--) {
21192166
// Check for any unset scope conditions due to alternate IF/ENDIF syntax.
@@ -3089,4 +3136,128 @@ public static function resolveSimpleToken($token)
30893136
}//end resolveSimpleToken()
30903137

30913138

3139+
/**
3140+
* Finds a "closer" token (closing parenthesis or square bracket for example)
3141+
* Handle parenthesis balancing while searching for closing token
3142+
*
3143+
* @param array $tokens The list of tokens to iterate searching the closing token (as returned by token_get_all)
3144+
* @param int $start The starting position
3145+
* @param string|string[] $openerTokens The opening character
3146+
* @param string $closerChar The closing character
3147+
*
3148+
* @return int|null The position of the closing token, if found. NULL otherwise.
3149+
*/
3150+
private function findCloser(array &$tokens, $start, $openerTokens, $closerChar)
3151+
{
3152+
$numTokens = count($tokens);
3153+
$stack = [0];
3154+
$closer = null;
3155+
$openerTokens = (array) $openerTokens;
3156+
3157+
for ($x = $start; $x < $numTokens; $x++) {
3158+
if (in_array($tokens[$x], $openerTokens, true) === true
3159+
|| (is_array($tokens[$x]) === true && in_array($tokens[$x][1], $openerTokens, true) === true)
3160+
) {
3161+
$stack[] = $x;
3162+
} else if ($tokens[$x] === $closerChar) {
3163+
array_pop($stack);
3164+
if (empty($stack) === true) {
3165+
$closer = $x;
3166+
break;
3167+
}
3168+
}
3169+
}
3170+
3171+
return $closer;
3172+
3173+
}//end findCloser()
3174+
3175+
3176+
/**
3177+
* PHP 8 attributes parser for PHP < 8
3178+
* Handles single-line and multiline attributes.
3179+
*
3180+
* @param array $tokens The original array of tokens (as returned by token_get_all)
3181+
* @param int $stackPtr The current position in token array
3182+
*
3183+
* @return array|null The array of parsed attribute tokens
3184+
*/
3185+
private function parsePhpAttribute(array &$tokens, $stackPtr)
3186+
{
3187+
3188+
$token = $tokens[$stackPtr];
3189+
3190+
$commentBody = substr($token[1], 2);
3191+
$subTokens = @token_get_all('<?php '.$commentBody);
3192+
3193+
foreach ($subTokens as $i => $subToken) {
3194+
if (is_array($subToken) === true
3195+
&& $subToken[0] === T_COMMENT
3196+
&& strpos($subToken[1], '#[') === 0
3197+
) {
3198+
$reparsed = $this->parsePhpAttribute($subTokens, $i);
3199+
if ($reparsed !== null) {
3200+
array_splice($subTokens, $i, 1, $reparsed);
3201+
} else {
3202+
$subToken[0] = T_ATTRIBUTE;
3203+
}
3204+
}
3205+
}
3206+
3207+
array_splice($subTokens, 0, 1, [[T_ATTRIBUTE, '#[']]);
3208+
3209+
// Go looking for the close bracket.
3210+
$bracketCloser = $this->findCloser($subTokens, 1, '[', ']');
3211+
if ($bracketCloser === null) {
3212+
$bracketCloser = $this->findCloser($tokens, $stackPtr, '[', ']');
3213+
if ($bracketCloser === null) {
3214+
return null;
3215+
}
3216+
3217+
$subTokens = array_merge($subTokens, array_slice($tokens, ($stackPtr + 1), ($bracketCloser - $stackPtr)));
3218+
array_splice($tokens, ($stackPtr + 1), ($bracketCloser - $stackPtr));
3219+
}
3220+
3221+
return $subTokens;
3222+
3223+
}//end parsePhpAttribute()
3224+
3225+
3226+
/**
3227+
* Creates a map for the attributes tokens that surround other tokens.
3228+
*
3229+
* @return void
3230+
*/
3231+
private function createAttributesNestingMap()
3232+
{
3233+
$map = [];
3234+
for ($i = 0; $i < $this->numTokens; $i++) {
3235+
if (isset($this->tokens[$i]['attribute_opener']) === true
3236+
&& $i === $this->tokens[$i]['attribute_opener']
3237+
) {
3238+
if (empty($map) === false) {
3239+
$this->tokens[$i]['nested_attributes'] = $map;
3240+
}
3241+
3242+
if (isset($this->tokens[$i]['attribute_closer']) === true) {
3243+
$map[$this->tokens[$i]['attribute_opener']]
3244+
= $this->tokens[$i]['attribute_closer'];
3245+
}
3246+
} else if (isset($this->tokens[$i]['attribute_closer']) === true
3247+
&& $i === $this->tokens[$i]['attribute_closer']
3248+
) {
3249+
array_pop($map);
3250+
if (empty($map) === false) {
3251+
$this->tokens[$i]['nested_attributes'] = $map;
3252+
}
3253+
} else {
3254+
if (empty($map) === false) {
3255+
$this->tokens[$i]['nested_attributes'] = $map;
3256+
}
3257+
}//end if
3258+
}//end for
3259+
3260+
}//end createAttributesNestingMap()
3261+
3262+
30923263
}//end class

‎src/Tokenizers/Tokenizer.php

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -740,6 +740,40 @@ private function createTokenMap()
740740
$this->tokens[$i]['parenthesis_closer'] = $i;
741741
$this->tokens[$opener]['parenthesis_closer'] = $i;
742742
}//end if
743+
} else if ($this->tokens[$i]['code'] === T_ATTRIBUTE) {
744+
$openers[] = $i;
745+
if (PHP_CODESNIFFER_VERBOSITY > 1) {
746+
echo str_repeat("\t", count($openers));
747+
echo "=> Found attribute opener at $i".PHP_EOL;
748+
}
749+
750+
$this->tokens[$i]['attribute_opener'] = $i;
751+
$this->tokens[$i]['attribute_closer'] = null;
752+
} else if ($this->tokens[$i]['code'] === T_ATTRIBUTE_END) {
753+
$numOpeners = count($openers);
754+
if ($numOpeners !== 0) {
755+
$opener = array_pop($openers);
756+
if (isset($this->tokens[$opener]['attribute_opener']) === true) {
757+
$this->tokens[$opener]['attribute_closer'] = $i;
758+
759+
if (PHP_CODESNIFFER_VERBOSITY > 1) {
760+
echo str_repeat("\t", (count($openers) + 1));
761+
echo "=> Found attribute closer at $i for $opener".PHP_EOL;
762+
}
763+
764+
for ($x = ($opener + 1); $x <= $i; ++$x) {
765+
if (isset($this->tokens[$x]['attribute_closer']) === true) {
766+
continue;
767+
}
768+
769+
$this->tokens[$x]['attribute_opener'] = $opener;
770+
$this->tokens[$x]['attribute_closer'] = $i;
771+
}
772+
} else if (PHP_CODESNIFFER_VERBOSITY > 1) {
773+
echo str_repeat("\t", (count($openers) + 1));
774+
echo "=> Found unowned attribute closer at $i for $opener".PHP_EOL;
775+
}
776+
}//end if
743777
}//end if
744778

745779
/*

‎src/Util/Tokens.php

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@
7979
define('T_PARAM_NAME', 'PHPCS_T_PARAM_NAME');
8080
define('T_MATCH_ARROW', 'PHPCS_T_MATCH_ARROW');
8181
define('T_MATCH_DEFAULT', 'PHPCS_T_MATCH_DEFAULT');
82+
define('T_ATTRIBUTE_END', 'PHPCS_T_ATTRIBUTE_END');
8283

8384
// Some PHP 5.5 tokens, replicated for lower versions.
8485
if (defined('T_FINALLY') === false) {
@@ -149,6 +150,10 @@
149150
define('T_MATCH', 'PHPCS_T_MATCH');
150151
}
151152

153+
if (defined('T_ATTRIBUTE') === false) {
154+
define('T_ATTRIBUTE', 'PHPCS_T_ATTRIBUTE');
155+
}
156+
152157
// Tokens used for parsing doc blocks.
153158
define('T_DOC_COMMENT_STAR', 'PHPCS_T_DOC_COMMENT_STAR');
154159
define('T_DOC_COMMENT_WHITESPACE', 'PHPCS_T_DOC_COMMENT_WHITESPACE');

‎tests/Core/File/GetMemberPropertiesTest.inc

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -240,3 +240,19 @@ $anon = class() {
240240
// Intentional fatal error - duplicate types are not allowed in union types, but that's not the concern of the method.
241241
public int |string| /*comment*/ INT $duplicateTypeInUnion;
242242
};
243+
244+
$anon = class {
245+
/* testPHP8PropertySingleAttribute */
246+
#[PropertyWithAttribute]
247+
public string $foo;
248+
249+
/* testPHP8PropertyMultipleAttributes */
250+
#[PropertyWithAttribute(foo: 'bar'), MyAttribute]
251+
protected ?int|float $bar;
252+
253+
/* testPHP8PropertyMultilineAttribute */
254+
#[
255+
PropertyWithAttribute(/* comment */ 'baz')
256+
]
257+
private mixed $baz;
258+
};

‎tests/Core/File/GetMemberPropertiesTest.php

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -610,6 +610,36 @@ public function dataGetMemberProperties()
610610
'nullable_type' => false,
611611
],
612612
],
613+
[
614+
'/* testPHP8PropertySingleAttribute */',
615+
[
616+
'scope' => 'public',
617+
'scope_specified' => true,
618+
'is_static' => false,
619+
'type' => 'string',
620+
'nullable_type' => false,
621+
],
622+
],
623+
[
624+
'/* testPHP8PropertyMultipleAttributes */',
625+
[
626+
'scope' => 'protected',
627+
'scope_specified' => true,
628+
'is_static' => false,
629+
'type' => '?int|float',
630+
'nullable_type' => true,
631+
],
632+
],
633+
[
634+
'/* testPHP8PropertyMultilineAttribute */',
635+
[
636+
'scope' => 'private',
637+
'scope_specified' => true,
638+
'is_static' => false,
639+
'type' => 'mixed',
640+
'nullable_type' => false,
641+
],
642+
],
613643
];
614644

615645
}//end dataGetMemberProperties()

0 commit comments

Comments
 (0)