Merge pull request #647 from PHPCSStandards/feature/529-tokenizer-yield-from-tokenization-review

PHP 8.3 | Tokenizer/PHP: support "yield from" with comments

Author: jrfnl

src/Standards/Generic/Sniffs/WhiteSpace/DisallowTabIndentSniff.php

@@ -78,6 +78,7 @@ public function process(File $phpcsFile, $stackPtr)
             T_COMMENT                => true,
             T_END_HEREDOC            => true,
             T_END_NOWDOC             => true,
+            T_YIELD_FROM             => true,
         ];
 
         for ($i = 0; $i < $phpcsFile->numTokens; $i++) {

src/Standards/Generic/Sniffs/WhiteSpace/LanguageConstructSpacingSniff.php

@@ -80,42 +80,57 @@ public function process(File $phpcsFile, $stackPtr)
         if ($tokens[$stackPtr]['code'] === T_YIELD_FROM
             && strtolower($content) !== 'yield from'
         ) {
-            if ($tokens[($stackPtr - 1)]['code'] === T_YIELD_FROM) {
-                // A multi-line statement that has already been processed.
-                return;
-            }
+            $found        = $content;
+            $hasComment   = false;
+            $yieldFromEnd = $stackPtr;
+
+            // Handle potentially multi-line/multi-token "yield from" expressions.
+            if (preg_match('`yield\s+from`i', $content) !== 1) {
+                for ($i = ($stackPtr + 1); $i < $phpcsFile->numTokens; $i++) {
+                    if (isset(Tokens::$emptyTokens[$tokens[$i]['code']]) === false
+                        && $tokens[$i]['code'] !== T_YIELD_FROM
+                    ) {
+                        break;
+                    }
+
+                    if (isset(Tokens::$commentTokens[$tokens[$i]['code']]) === true) {
+                        $hasComment = true;
+                    }
 
-            $found = $content;
-            if ($tokens[($stackPtr + 1)]['code'] === T_YIELD_FROM) {
-                // This yield from statement is split over multiple lines.
-                $i = ($stackPtr + 1);
-                do {
                     $found .= $tokens[$i]['content'];
-                    $i++;
-                } while ($tokens[$i]['code'] === T_YIELD_FROM);
-            }
+
+                    if ($tokens[$i]['code'] === T_YIELD_FROM
+                        && strtolower(trim($tokens[$i]['content'])) === 'from'
+                    ) {
+                        break;
+                    }
+                }
+
+                $yieldFromEnd = $i;
+            }//end if
 
             $error = 'Language constructs must be followed by a single space; expected 1 space between YIELD FROM found "%s"';
             $data  = [Common::prepareForOutput($found)];
-            $fix   = $phpcsFile->addFixableError($error, $stackPtr, 'IncorrectYieldFrom', $data);
-            if ($fix === true) {
-                preg_match('/yield/i', $found, $yield);
-                preg_match('/from/i', $found, $from);
-                $phpcsFile->fixer->beginChangeset();
-                $phpcsFile->fixer->replaceToken($stackPtr, $yield[0].' '.$from[0]);
-
-                if ($tokens[($stackPtr + 1)]['code'] === T_YIELD_FROM) {
-                    $i = ($stackPtr + 1);
-                    do {
+
+            if ($hasComment === true) {
+                $phpcsFile->addError($error, $stackPtr, 'IncorrectYieldFromWithComment', $data);
+            } else {
+                $fix = $phpcsFile->addFixableError($error, $stackPtr, 'IncorrectYieldFrom', $data);
+                if ($fix === true) {
+                    preg_match('/yield/i', $found, $yield);
+                    preg_match('/from/i', $found, $from);
+                    $phpcsFile->fixer->beginChangeset();
+                    $phpcsFile->fixer->replaceToken($stackPtr, $yield[0].' '.$from[0]);
+
+                    for ($i = ($stackPtr + 1); $i <= $yieldFromEnd; $i++) {
                         $phpcsFile->fixer->replaceToken($i, '');
-                        $i++;
-                    } while ($tokens[$i]['code'] === T_YIELD_FROM);
-                }
+                    }
 
-                $phpcsFile->fixer->endChangeset();
-            }
+                    $phpcsFile->fixer->endChangeset();
+                }
+            }//end if
 
-            return;
+            return ($yieldFromEnd + 1);
         }//end if
 
         if ($tokens[($stackPtr + 1)]['code'] === T_WHITESPACE) {

src/Standards/Generic/Tests/PHP/LowerCaseKeywordUnitTest.inc

@@ -53,5 +53,12 @@ if (ISSET($a) && !Empty($a)) { UnSeT($a); }
 eval('foo');
 eVaL('foo');
 
+$c = function() {
+    Yield /*comment*/ From fun();
+    YIELD
+    /*comment*/
+    FROM fun();
+}
+
 __HALT_COMPILER(); // An exception due to phar support.
 function

src/Standards/Generic/Tests/PHP/LowerCaseKeywordUnitTest.inc.fixed

@@ -53,5 +53,12 @@ if (isset($a) && !empty($a)) { unset($a); }
 eval('foo');
 eval('foo');
 
+$c = function() {
+    yield /*comment*/ from fun();
+    yield
+    /*comment*/
+    from fun();
+}
+
 __HALT_COMPILER(); // An exception due to phar support.
 function

src/Standards/Generic/Tests/PHP/LowerCaseKeywordUnitTest.php

@@ -52,6 +52,9 @@ public function getErrorList()
             48 => 1,
             52 => 3,
             54 => 1,
+            57 => 2,
+            58 => 1,
+            60 => 1,
         ];
 
     }//end getErrorList()

src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.1.inc

@@ -116,3 +116,10 @@ $x = 1;
 
      Another line.
      */
+
+// A `yield from` can be multiline and may contain spaces in the indentation whitespace between the keywords.
+function myGenerator() {
+	yield
+    from
+	gen2();
+}

src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.1.inc.fixed

@@ -116,3 +116,10 @@ $x = 1;
 
 	 Another line.
 	 */
+
+// A `yield from` can be multiline and may contain spaces in the indentation whitespace between the keywords.
+function myGenerator() {
+	yield
+	from
+	gen2();
+}

src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.2.inc

@@ -116,3 +116,10 @@ $x = 1;
 
      Another line.
      */
+
+// A `yield from` can be multiline and may contain spaces in the indentation whitespace between the keywords.
+function myGenerator() {
+	yield
+    from
+	gen2();
+}

src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.2.inc.fixed

@@ -116,3 +116,10 @@ $x = 1;
 
 	 Another line.
 	 */
+
+// A `yield from` can be multiline and may contain spaces in the indentation whitespace between the keywords.
+function myGenerator() {
+	yield
+	from
+	gen2();
+}

src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.php

@@ -89,6 +89,7 @@ public function getErrorList($testFile='')
                 115 => 1,
                 117 => 1,
                 118 => 1,
+                123 => 1,
             ];
 
         case 'DisallowSpaceIndentUnitTest.3.inc':