Tokenizer: apply tab replacement to "yield from"

The `yield from` keyword(s) can be separated by spaces, tabs, new lines and since PHP 8.3, even comments.

The PHPCS `Tokenizer` previously did not execute tab replacement on this token leading to unexpected `'content'` and incorrect `'length'` values in the `File::$tokens` array, which in turn could lead to incorrect sniff results and incorrect fixes.

Previously, this affected all `T_YIELD_FROM` tokens. After the change to support PHP 8.3 "yield ... comment... from" syntax, this now only affects single line `yield from` expressions where the keywords are separated by a tab or a mix of tabs and spaces.

All the same, consistency is key, so this commit adds the `T_YIELD_FROM` token to the array of tokens for which to do tab replacement, which should make them more consistent with the rest of PHPCS.

Includes unit tests safeguarding this change.

Author: jrfnl

src/Tokenizers/Tokenizer.php

@@ -200,6 +200,7 @@ private function createPositionMap()
             T_END_HEREDOC              => true,
             T_END_NOWDOC               => true,
             T_INLINE_HTML              => true,
+            T_YIELD_FROM               => true,
         ];
 
         $this->numTokens = count($this->tokens);

tests/Core/Tokenizers/Tokenizer/CreatePositionMapYieldFromTest.inc

@@ -0,0 +1,15 @@
+<?php
+
+function myGenerator() {
+    /* testYieldFromHasSingleSpace */
+    yield from gen2();
+
+    /* testYieldFromHasMultiSpace */
+    yield     from gen2();
+
+    /* testYieldFromHasTabs */
+    yield		from gen2();
+
+    /* testYieldFromMixedTabsSpaces */
+    Yield	   	 	From gen2();
+}

tests/Core/Tokenizers/Tokenizer/CreatePositionMapYieldFromTest.php

@@ -0,0 +1,100 @@
+<?php
+/**
+ * Tests the tokenization of "yield from" tokens.
+ *
+ * @author    Juliette Reinders Folmer <[email protected]>
+ * @copyright 2024 PHPCSStandards and contributors
+ * @license   https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
+ */
+
+namespace PHP_CodeSniffer\Tests\Core\Tokenizers\Tokenizer;
+
+use PHP_CodeSniffer\Tests\Core\Tokenizers\AbstractTokenizerTestCase;
+
+/**
+ * Yield from token test.
+ *
+ * @covers PHP_CodeSniffer\Tokenizers\Tokenizer::createPositionMap
+ */
+final class CreatePositionMapYieldFromTest extends AbstractTokenizerTestCase
+{
+
+
+    /**
+     * Verify that spaces/tabs in "yield from" tokens get the tab replacement treatment.
+     *
+     * @param string                         $testMarker The comment prefacing the target token.
+     * @param array<string, int|string|null> $expected   Expectations for the token array.
+     * @param string                         $content    Optional. The test token content to search for.
+     *                                                   Defaults to null.
+     *
+     * @dataProvider dataYieldFromTabReplacement
+     *
+     * @return void
+     */
+    public function testYieldFromTabReplacement($testMarker, $expected, $content=null)
+    {
+        $tokens = $this->phpcsFile->getTokens();
+        $target = $this->getTargetToken($testMarker, [T_YIELD_FROM], $content);
+
+        foreach ($expected as $key => $value) {
+            if ($key === 'orig_content' && $value === null) {
+                $this->assertArrayNotHasKey($key, $tokens[$target], "Unexpected 'orig_content' key found in the token array.");
+                continue;
+            }
+
+            $this->assertArrayHasKey($key, $tokens[$target], "Key $key not found in the token array.");
+            $this->assertSame($value, $tokens[$target][$key], "Value for key $key does not match expectation.");
+        }
+
+    }//end testYieldFromTabReplacement()
+
+
+    /**
+     * Data provider.
+     *
+     * @see testYieldFromTabReplacement()
+     *
+     * @return array<string, array<string, string|array<string, int|string|null>>>
+     */
+    public static function dataYieldFromTabReplacement()
+    {
+        return [
+            'Yield from, single line, single space' => [
+                'testMarker' => '/* testYieldFromHasSingleSpace */',
+                'expected'   => [
+                    'length'       => 10,
+                    'content'      => 'yield from',
+                    'orig_content' => null,
+                ],
+            ],
+            'Yield from, single line, multiple spaces'  => [
+                'testMarker' => '/* testYieldFromHasMultiSpace */',
+                'expected'   => [
+                    'length'       => 14,
+                    'content'      => 'yield     from',
+                    'orig_content' => null,
+                ],
+            ],
+            'Yield from, single line, has tabs' => [
+                'testMarker' => '/* testYieldFromHasTabs */',
+                'expected'   => [
+                    'length'       => 16,
+                    'content'      => 'yield       from',
+                    'orig_content' => 'yield		from',
+                ],
+            ],
+            'Yield from, single line, mix of tabs and spaces'  => [
+                'testMarker' => '/* testYieldFromMixedTabsSpaces */',
+                'expected'   => [
+                    'length'       => 20,
+                    'content'      => 'Yield           From',
+                    'orig_content' => 'Yield	   	 	From',
+                ],
+            ],
+        ];
+
+    }//end dataYieldFromTabReplacement()
+
+
+}//end class