diff --git a/src/Tokenizers/Comment.php b/src/Tokenizers/Comment.php index bac3111e28..c3dc8aa28d 100644 --- a/src/Tokenizers/Comment.php +++ b/src/Tokenizers/Comment.php @@ -51,10 +51,11 @@ public function tokenizeString($string, $eolChar, $stackPtr) } $tokens[$stackPtr] = [ - 'content' => $openTag, - 'code' => T_DOC_COMMENT_OPEN_TAG, - 'type' => 'T_DOC_COMMENT_OPEN_TAG', - 'comment_tags' => [], + 'content' => $openTag, + 'code' => T_DOC_COMMENT_OPEN_TAG, + 'type' => 'T_DOC_COMMENT_OPEN_TAG', + 'comment_opener' => $stackPtr, + 'comment_tags' => [], ]; $openPtr = $stackPtr; @@ -103,6 +104,7 @@ public function tokenizeString($string, $eolChar, $stackPtr) $space = $this->collectWhitespace($string, $char, $numChars); if ($space !== null) { $tokens[$stackPtr] = $space; + $tokens[$stackPtr]['comment_opener'] = $openPtr; $stackPtr++; if (PHP_CODESNIFFER_VERBOSITY > 1) { $content = Common::prepareForOutput($space['content']); @@ -123,9 +125,10 @@ public function tokenizeString($string, $eolChar, $stackPtr) // This is a function or class doc block line. $char++; $tokens[$stackPtr] = [ - 'content' => '*', - 'code' => T_DOC_COMMENT_STAR, - 'type' => 'T_DOC_COMMENT_STAR', + 'content' => '*', + 'code' => T_DOC_COMMENT_STAR, + 'type' => 'T_DOC_COMMENT_STAR', + 'comment_opener' => $openPtr, ]; $stackPtr++; @@ -139,6 +142,7 @@ public function tokenizeString($string, $eolChar, $stackPtr) $lineTokens = $this->processLine($string, $eolChar, $char, $numChars); foreach ($lineTokens as $lineToken) { $tokens[$stackPtr] = $lineToken; + $tokens[$stackPtr]['comment_opener'] = $openPtr; if (PHP_CODESNIFFER_VERBOSITY > 1) { $content = Common::prepareForOutput($lineToken['content']); $type = $lineToken['type']; @@ -154,12 +158,17 @@ public function tokenizeString($string, $eolChar, $stackPtr) }//end foreach $tokens[$stackPtr] = $closeTag; - $tokens[$openPtr]['comment_closer'] = $stackPtr; if (PHP_CODESNIFFER_VERBOSITY > 1) { $content = Common::prepareForOutput($closeTag['content']); StatusWriter::write("Create comment token: T_DOC_COMMENT_CLOSE_TAG => $content", 2); } + // Only now do we know the stack pointer to the docblock closer, + // so add it to all previously created comment tokens. + foreach ($tokens as $ptr => $token) { + $tokens[$ptr]['comment_closer'] = $stackPtr; + } + if (PHP_CODESNIFFER_VERBOSITY > 1) { StatusWriter::write('*** END COMMENT TOKENIZING ***', 2); } diff --git a/src/Tokenizers/PHP.php b/src/Tokenizers/PHP.php index b5f800117c..64a6504a6f 100644 --- a/src/Tokenizers/PHP.php +++ b/src/Tokenizers/PHP.php @@ -887,11 +887,8 @@ protected function tokenize($string) || ($token[0] === T_COMMENT && strpos($token[1], '/**') === 0 && $token[1] !== '/**/')) ) { $commentTokens = $commentTokenizer->tokenizeString($token[1], $this->eolChar, $newStackPtr); - foreach ($commentTokens as $commentToken) { - $finalTokens[$newStackPtr] = $commentToken; - $newStackPtr++; - } - + $finalTokens += $commentTokens; + $newStackPtr += count($commentTokens); continue; } diff --git a/tests/Core/Tokenizers/Comment/CommentTestCase.php b/tests/Core/Tokenizers/Comment/CommentTestCase.php index 625029db16..b6677b4d1e 100644 --- a/tests/Core/Tokenizers/Comment/CommentTestCase.php +++ b/tests/Core/Tokenizers/Comment/CommentTestCase.php @@ -22,7 +22,8 @@ abstract class CommentTestCase extends AbstractTokenizerTestCase /** - * Test whether the docblock opener and closer have the expected extra keys. + * Test whether the docblock tokens have the extra `comment_opener` and `comment_closer` keys, + * and the docblock opener has the `comment_tags` key. * * @param string $marker The comment prefacing the target token. * @param int $closerOffset The offset of the closer from the opener. @@ -34,28 +35,38 @@ abstract class CommentTestCase extends AbstractTokenizerTestCase */ public function testDocblockOpenerCloser($marker, $closerOffset, $expectedTags) { - $tokens = $this->phpcsFile->getTokens(); - $target = $this->getTargetToken($marker, [T_DOC_COMMENT_OPEN_TAG]); - - $opener = $tokens[$target]; + $tokens = $this->phpcsFile->getTokens(); + $expectedOpener = $this->getTargetToken($marker, [T_DOC_COMMENT_OPEN_TAG]); + $expectedCloser = ($expectedOpener + $closerOffset); - $this->assertArrayHasKey('comment_closer', $opener, 'Comment opener: comment_closer index is not set'); - $this->assertArrayHasKey('comment_tags', $opener, 'Comment opener: comment_tags index is not set'); - - $expectedCloser = ($target + $closerOffset); - $this->assertSame($expectedCloser, $opener['comment_closer'], 'Comment opener: comment_closer not set to the expected stack pointer'); + $opener = $tokens[$expectedOpener]; // Update the tags expectations. foreach ($expectedTags as $i => $ptr) { - $expectedTags[$i] += $target; + $expectedTags[$i] += $expectedOpener; } + // Verify that the comment opener has the `comment_tags` key. + $this->assertArrayHasKey('comment_tags', $opener, 'Comment opener: comment_tags index is not set'); $this->assertSame($expectedTags, $opener['comment_tags'], 'Comment opener: recorded tags do not match expected tags'); - $closer = $tokens[$opener['comment_closer']]; + // Check that the comment_opener and comment_closer keys have been added to all docblock tokens. + for ($i = $expectedOpener; $i <= $expectedCloser; $i++) { + $token = $tokens[$i]; + $this->assertArrayHasKey('comment_opener', $token, 'Comment_opener index is not set (for stackPtr '.$i.')'); + $this->assertArrayHasKey('comment_closer', $token, 'Comment_closer index is not set (for stackPtr '.$i.')'); - $this->assertArrayHasKey('comment_opener', $closer, 'Comment closer: comment_opener index is not set'); - $this->assertSame($target, $closer['comment_opener'], 'Comment closer: comment_opener not set to the expected stack pointer'); + $this->assertSame( + $expectedOpener, + $token['comment_opener'], + 'Comment_opener not set to the expected stack pointer (for stackPtr '.$i.')' + ); + $this->assertSame( + $expectedCloser, + $token['comment_closer'], + 'Comment_closer not set to the expected stack pointer (for stackPtr '.$i.')' + ); + } }//end testDocblockOpenerCloser()