Merge pull request #1019 from PHPCSStandards/phpcs-4.0/feature/484-tokenizer-docblock-improvements

Tokenizer/Comment: add docblock open + close ptr to all docblock tokens

Author: jrfnl

src/Tokenizers/Comment.php

@@ -51,10 +51,11 @@ public function tokenizeString($string, $eolChar, $stackPtr)
         }
 
         $tokens[$stackPtr] = [
-            'content'      => $openTag,
-            'code'         => T_DOC_COMMENT_OPEN_TAG,
-            'type'         => 'T_DOC_COMMENT_OPEN_TAG',
-            'comment_tags' => [],
+            'content'        => $openTag,
+            'code'           => T_DOC_COMMENT_OPEN_TAG,
+            'type'           => 'T_DOC_COMMENT_OPEN_TAG',
+            'comment_opener' => $stackPtr,
+            'comment_tags'   => [],
         ];
 
         $openPtr = $stackPtr;
@@ -103,6 +104,7 @@ public function tokenizeString($string, $eolChar, $stackPtr)
             $space = $this->collectWhitespace($string, $char, $numChars);
             if ($space !== null) {
                 $tokens[$stackPtr] = $space;
+                $tokens[$stackPtr]['comment_opener'] = $openPtr;
                 $stackPtr++;
                 if (PHP_CODESNIFFER_VERBOSITY > 1) {
                     $content = Common::prepareForOutput($space['content']);
@@ -123,9 +125,10 @@ public function tokenizeString($string, $eolChar, $stackPtr)
                 // This is a function or class doc block line.
                 $char++;
                 $tokens[$stackPtr] = [
-                    'content' => '*',
-                    'code'    => T_DOC_COMMENT_STAR,
-                    'type'    => 'T_DOC_COMMENT_STAR',
+                    'content'        => '*',
+                    'code'           => T_DOC_COMMENT_STAR,
+                    'type'           => 'T_DOC_COMMENT_STAR',
+                    'comment_opener' => $openPtr,
                 ];
 
                 $stackPtr++;
@@ -139,6 +142,7 @@ public function tokenizeString($string, $eolChar, $stackPtr)
             $lineTokens = $this->processLine($string, $eolChar, $char, $numChars);
             foreach ($lineTokens as $lineToken) {
                 $tokens[$stackPtr] = $lineToken;
+                $tokens[$stackPtr]['comment_opener'] = $openPtr;
                 if (PHP_CODESNIFFER_VERBOSITY > 1) {
                     $content = Common::prepareForOutput($lineToken['content']);
                     $type    = $lineToken['type'];
@@ -154,12 +158,17 @@ public function tokenizeString($string, $eolChar, $stackPtr)
         }//end foreach
 
         $tokens[$stackPtr] = $closeTag;
-        $tokens[$openPtr]['comment_closer'] = $stackPtr;
         if (PHP_CODESNIFFER_VERBOSITY > 1) {
             $content = Common::prepareForOutput($closeTag['content']);
             StatusWriter::write("Create comment token: T_DOC_COMMENT_CLOSE_TAG => $content", 2);
         }
 
+        // Only now do we know the stack pointer to the docblock closer,
+        // so add it to all previously created comment tokens.
+        foreach ($tokens as $ptr => $token) {
+            $tokens[$ptr]['comment_closer'] = $stackPtr;
+        }
+
         if (PHP_CODESNIFFER_VERBOSITY > 1) {
             StatusWriter::write('*** END COMMENT TOKENIZING ***', 2);
         }

src/Tokenizers/PHP.php

@@ -887,11 +887,8 @@ protected function tokenize($string)
                 || ($token[0] === T_COMMENT && strpos($token[1], '/**') === 0 && $token[1] !== '/**/'))
             ) {
                 $commentTokens = $commentTokenizer->tokenizeString($token[1], $this->eolChar, $newStackPtr);
-                foreach ($commentTokens as $commentToken) {
-                    $finalTokens[$newStackPtr] = $commentToken;
-                    $newStackPtr++;
-                }
-
+                $finalTokens  += $commentTokens;
+                $newStackPtr  += count($commentTokens);
                 continue;
             }
 

tests/Core/Tokenizers/Comment/CommentTestCase.php

@@ -22,7 +22,8 @@ abstract class CommentTestCase extends AbstractTokenizerTestCase
 
 
     /**
-     * Test whether the docblock opener and closer have the expected extra keys.
+     * Test whether the docblock tokens have the extra `comment_opener` and `comment_closer` keys,
+     * and the docblock opener has the `comment_tags` key.
      *
      * @param string     $marker       The comment prefacing the target token.
      * @param int        $closerOffset The offset of the closer from the opener.
@@ -34,28 +35,38 @@ abstract class CommentTestCase extends AbstractTokenizerTestCase
      */
     public function testDocblockOpenerCloser($marker, $closerOffset, $expectedTags)
     {
-        $tokens = $this->phpcsFile->getTokens();
-        $target = $this->getTargetToken($marker, [T_DOC_COMMENT_OPEN_TAG]);
-
-        $opener = $tokens[$target];
+        $tokens         = $this->phpcsFile->getTokens();
+        $expectedOpener = $this->getTargetToken($marker, [T_DOC_COMMENT_OPEN_TAG]);
+        $expectedCloser = ($expectedOpener + $closerOffset);
 
-        $this->assertArrayHasKey('comment_closer', $opener, 'Comment opener: comment_closer index is not set');
-        $this->assertArrayHasKey('comment_tags', $opener, 'Comment opener: comment_tags index is not set');
-
-        $expectedCloser = ($target + $closerOffset);
-        $this->assertSame($expectedCloser, $opener['comment_closer'], 'Comment opener: comment_closer not set to the expected stack pointer');
+        $opener = $tokens[$expectedOpener];
 
         // Update the tags expectations.
         foreach ($expectedTags as $i => $ptr) {
-            $expectedTags[$i] += $target;
+            $expectedTags[$i] += $expectedOpener;
         }
 
+        // Verify that the comment opener has the `comment_tags` key.
+        $this->assertArrayHasKey('comment_tags', $opener, 'Comment opener: comment_tags index is not set');
         $this->assertSame($expectedTags, $opener['comment_tags'], 'Comment opener: recorded tags do not match expected tags');
 
-        $closer = $tokens[$opener['comment_closer']];
+        // Check that the comment_opener and comment_closer keys have been added to all docblock tokens.
+        for ($i = $expectedOpener; $i <= $expectedCloser; $i++) {
+            $token = $tokens[$i];
+            $this->assertArrayHasKey('comment_opener', $token, 'Comment_opener index is not set (for stackPtr '.$i.')');
+            $this->assertArrayHasKey('comment_closer', $token, 'Comment_closer index is not set (for stackPtr '.$i.')');
 
-        $this->assertArrayHasKey('comment_opener', $closer, 'Comment closer: comment_opener index is not set');
-        $this->assertSame($target, $closer['comment_opener'], 'Comment closer: comment_opener not set to the expected stack pointer');
+            $this->assertSame(
+                $expectedOpener,
+                $token['comment_opener'],
+                'Comment_opener not set to the expected stack pointer (for stackPtr '.$i.')'
+            );
+            $this->assertSame(
+                $expectedCloser,
+                $token['comment_closer'],
+                'Comment_closer not set to the expected stack pointer (for stackPtr '.$i.')'
+            );
+        }
 
     }//end testDocblockOpenerCloser()