Skip to content

Commit 5ecca2d

Browse files
jrfnljrfnl
committed
Tokenizer/Comment: add docblock open + close ptr to all docblock tokens
Previously, only tokens of type `T_DOC_COMMENT_OPEN_TAG ` would receive a `comment_closer` index and tokens of type `T_DOC_COMMENT_CLOSE_TAG ` would receive a `comment_opener` index (asymmetrically). This commit changes this to set both the `comment_opener` as well as the `comment_closer` indexes on all docblock tokens. This should allow for more flexibility for sniffs examining docblocks. Includes updated tests to safeguard this change. Fixes 484
1 parent 884353b commit 5ecca2d

File tree

3 files changed

+44
-27
lines changed

3 files changed

+44
-27
lines changed

‎src/Tokenizers/Comment.php‎

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -51,10 +51,11 @@ public function tokenizeString($string, $eolChar, $stackPtr)
5151
}
5252

5353
$tokens[$stackPtr] = [
54-
'content' => $openTag,
55-
'code' => T_DOC_COMMENT_OPEN_TAG,
56-
'type' => 'T_DOC_COMMENT_OPEN_TAG',
57-
'comment_tags' => [],
54+
'content' => $openTag,
55+
'code' => T_DOC_COMMENT_OPEN_TAG,
56+
'type' => 'T_DOC_COMMENT_OPEN_TAG',
57+
'comment_opener' => $stackPtr,
58+
'comment_tags' => [],
5859
];
5960

6061
$openPtr = $stackPtr;
@@ -103,6 +104,7 @@ public function tokenizeString($string, $eolChar, $stackPtr)
103104
$space = $this->collectWhitespace($string, $char, $numChars);
104105
if ($space !== null) {
105106
$tokens[$stackPtr] = $space;
107+
$tokens[$stackPtr]['comment_opener'] = $openPtr;
106108
$stackPtr++;
107109
if (PHP_CODESNIFFER_VERBOSITY > 1) {
108110
$content = Common::prepareForOutput($space['content']);
@@ -123,9 +125,10 @@ public function tokenizeString($string, $eolChar, $stackPtr)
123125
// This is a function or class doc block line.
124126
$char++;
125127
$tokens[$stackPtr] = [
126-
'content' => '*',
127-
'code' => T_DOC_COMMENT_STAR,
128-
'type' => 'T_DOC_COMMENT_STAR',
128+
'content' => '*',
129+
'code' => T_DOC_COMMENT_STAR,
130+
'type' => 'T_DOC_COMMENT_STAR',
131+
'comment_opener' => $openPtr,
129132
];
130133

131134
$stackPtr++;
@@ -139,6 +142,7 @@ public function tokenizeString($string, $eolChar, $stackPtr)
139142
$lineTokens = $this->processLine($string, $eolChar, $char, $numChars);
140143
foreach ($lineTokens as $lineToken) {
141144
$tokens[$stackPtr] = $lineToken;
145+
$tokens[$stackPtr]['comment_opener'] = $openPtr;
142146
if (PHP_CODESNIFFER_VERBOSITY > 1) {
143147
$content = Common::prepareForOutput($lineToken['content']);
144148
$type = $lineToken['type'];
@@ -154,12 +158,17 @@ public function tokenizeString($string, $eolChar, $stackPtr)
154158
}//end foreach
155159

156160
$tokens[$stackPtr] = $closeTag;
157-
$tokens[$openPtr]['comment_closer'] = $stackPtr;
158161
if (PHP_CODESNIFFER_VERBOSITY > 1) {
159162
$content = Common::prepareForOutput($closeTag['content']);
160163
StatusWriter::write("Create comment token: T_DOC_COMMENT_CLOSE_TAG => $content", 2);
161164
}
162165

166+
// Only now do we know the stack pointer to the docblock closer,
167+
// so add it to all previously created comment tokens.
168+
foreach ($tokens as $ptr => $token) {
169+
$tokens[$ptr]['comment_closer'] = $stackPtr;
170+
}
171+
163172
if (PHP_CODESNIFFER_VERBOSITY > 1) {
164173
StatusWriter::write('*** END COMMENT TOKENIZING ***', 2);
165174
}

‎src/Tokenizers/PHP.php‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -887,11 +887,8 @@ protected function tokenize($string)
887887
|| ($token[0] === T_COMMENT && strpos($token[1], '/**') === 0 && $token[1] !== '/**/'))
888888
) {
889889
$commentTokens = $commentTokenizer->tokenizeString($token[1], $this->eolChar, $newStackPtr);
890-
foreach ($commentTokens as $commentToken) {
891-
$finalTokens[$newStackPtr] = $commentToken;
892-
$newStackPtr++;
893-
}
894-
890+
$finalTokens += $commentTokens;
891+
$newStackPtr += count($commentTokens);
895892
continue;
896893
}
897894

‎tests/Core/Tokenizers/Comment/CommentTestCase.php‎

Lines changed: 25 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@ abstract class CommentTestCase extends AbstractTokenizerTestCase
2222

2323

2424
/**
25-
* Test whether the docblock opener and closer have the expected extra keys.
25+
* Test whether the docblock tokens have the extra `comment_opener` and `comment_closer` keys,
26+
* and the docblock opener has the `comment_tags` key.
2627
*
2728
* @param string $marker The comment prefacing the target token.
2829
* @param int $closerOffset The offset of the closer from the opener.
@@ -34,28 +35,38 @@ abstract class CommentTestCase extends AbstractTokenizerTestCase
3435
*/
3536
public function testDocblockOpenerCloser($marker, $closerOffset, $expectedTags)
3637
{
37-
$tokens = $this->phpcsFile->getTokens();
38-
$target = $this->getTargetToken($marker, [T_DOC_COMMENT_OPEN_TAG]);
39-
40-
$opener = $tokens[$target];
38+
$tokens = $this->phpcsFile->getTokens();
39+
$expectedOpener = $this->getTargetToken($marker, [T_DOC_COMMENT_OPEN_TAG]);
40+
$expectedCloser = ($expectedOpener + $closerOffset);
4141

42-
$this->assertArrayHasKey('comment_closer', $opener, 'Comment opener: comment_closer index is not set');
43-
$this->assertArrayHasKey('comment_tags', $opener, 'Comment opener: comment_tags index is not set');
44-
45-
$expectedCloser = ($target + $closerOffset);
46-
$this->assertSame($expectedCloser, $opener['comment_closer'], 'Comment opener: comment_closer not set to the expected stack pointer');
42+
$opener = $tokens[$expectedOpener];
4743

4844
// Update the tags expectations.
4945
foreach ($expectedTags as $i => $ptr) {
50-
$expectedTags[$i] += $target;
46+
$expectedTags[$i] += $expectedOpener;
5147
}
5248

49+
// Verify that the comment opener has the `comment_tags` key.
50+
$this->assertArrayHasKey('comment_tags', $opener, 'Comment opener: comment_tags index is not set');
5351
$this->assertSame($expectedTags, $opener['comment_tags'], 'Comment opener: recorded tags do not match expected tags');
5452

55-
$closer = $tokens[$opener['comment_closer']];
53+
// Check that the comment_opener and comment_closer keys have been added to all docblock tokens.
54+
for ($i = $expectedOpener; $i <= $expectedCloser; $i++) {
55+
$token = $tokens[$i];
56+
$this->assertArrayHasKey('comment_opener', $token, 'Comment_opener index is not set (for stackPtr '.$i.')');
57+
$this->assertArrayHasKey('comment_closer', $token, 'Comment_closer index is not set (for stackPtr '.$i.')');
5658

57-
$this->assertArrayHasKey('comment_opener', $closer, 'Comment closer: comment_opener index is not set');
58-
$this->assertSame($target, $closer['comment_opener'], 'Comment closer: comment_opener not set to the expected stack pointer');
59+
$this->assertSame(
60+
$expectedOpener,
61+
$token['comment_opener'],
62+
'Comment_opener not set to the expected stack pointer (for stackPtr '.$i.')'
63+
);
64+
$this->assertSame(
65+
$expectedCloser,
66+
$token['comment_closer'],
67+
'Comment_closer not set to the expected stack pointer (for stackPtr '.$i.')'
68+
);
69+
}
5970

6071
}//end testDocblockOpenerCloser()
6172

0 commit comments

Comments
 (0)