Skip to content

Tokenizer::replaceTabsInToken(): add tests #644

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 25, 2024
Merged

Tokenizer::replaceTabsInToken(): add tests #644

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/Tokenizers/Tokenizer.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -620,7 +620,7 @@ public function replaceTabsInToken(&$token, $prefix=' ', $padding=' ', $tabWidth
if ($content !== '') {
$newContent .= $content;
if ($checkEncoding === true) {
// Not using the default encoding, so take a bit more care.
// Not using ASCII encoding, so take a bit more care.
$oldLevel = error_reporting();
error_reporting(0);
$contentLength = iconv_strlen($content, $this->config->encoding);
Expand Down
107 changes: 107 additions & 0 deletions tests/Core/Tokenizers/Tokenizer/CreatePositionMapTabWidth0Test.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
<?php
/**
* Tests the tab replacement logic.
*
* @author Juliette Reinders Folmer <[email protected]>
* @copyright 2024 PHPCSStandards and contributors
* @license https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
*/

namespace PHP_CodeSniffer\Tests\Core\Tokenizers\Tokenizer;

/**
* Tab replacement test using tab width 0, means no tab replacement will take place.
*
* @covers PHP_CodeSniffer\Tokenizers\Tokenizer::createPositionMap
*/
final class CreatePositionMapTabWidth0Test extends ReplaceTabsInTokenTestCase
{

/**
* The tab width setting to use when tokenizing the file.
*
* @var integer
*/
protected $tabWidth = 0;


/**
* Data provider helper.
*
* @see ReplaceTabsInTokenTestCase::dataTabReplacement()
*
* @return array<string, array<string, int|string|null>>
*/
public static function getTabReplacementExpected()
{
return [
'Tab indentation' => [
'length' => 2,
'content' => ' ',
'orig_content' => null,
],
'Mixed tab/space indentation' => [
'length' => 3,
'content' => ' ',
'orig_content' => null,
],
'Inline: single tab in text string' => [
'length' => 15,
'content' => "'tab separated'",
'orig_content' => null,
],
'Inline: single tab between each word in text string' => [
'length' => 24,
'content' => '"tab $between each word"',
'orig_content' => null,
],
'Inline: multiple tabs in heredoc' => [
'length' => 15,
'content' => 'tab separated
',
'orig_content' => null,
],
'Inline: multiple tabs between each word in nowdoc' => [
'length' => 27,
'content' => 'tab between each word
',
'orig_content' => null,
],
'Inline: mixed spaces/tabs in text string' => [
'length' => 20,
'content' => "'tab separated'",
'orig_content' => null,
],
'Inline: mixed spaces/tabs between each word in text string' => [
'length' => 31,
'content' => '"tab $between each word"',
'orig_content' => null,
],
'Inline: tab becomes single space in comment (with tabwidth 4)' => [
'length' => 50,
'content' => '// -123 With tabwidth 4, the tab size should be 1.
',
'orig_content' => null,
],
'Inline: tab becomes 2 spaces in comment (with tabwidth 4)' => [
'length' => 52,
'content' => '/* -12 With tabwidth 4, the tab size should be 2. */',
'orig_content' => null,
],
'Inline: tab becomes 3 spaces in doc comment string (with tabwidth 4)' => [
'length' => 45,
'content' => '-1 With tabwidth 4, the tab size should be 3.',
'orig_content' => null,
],
'Inline: tab becomes 4 spaces in comment (with tabwidth 4)' => [
'length' => 47,
'content' => '// - With tabwidth 4, the tab size should be 4.
',
'orig_content' => null,
],
];

}//end getTabReplacementExpected()


}//end class
124 changes: 124 additions & 0 deletions tests/Core/Tokenizers/Tokenizer/ReplaceTabsInTokenMiscTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
<?php
/**
* Tests the tab replacement logic.
*
* @author Juliette Reinders Folmer <[email protected]>
* @copyright 2024 PHPCSStandards and contributors
* @license https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
*/

namespace PHP_CodeSniffer\Tests\Core\Tokenizers\Tokenizer;

use PHP_CodeSniffer\Files\DummyFile;
use PHP_CodeSniffer\Ruleset;
use PHP_CodeSniffer\Tests\ConfigDouble;
use PHPUnit\Framework\TestCase;

/**
* Miscellaneous tests for tab replacement.
*
* @covers PHP_CodeSniffer\Tokenizers\Tokenizer::replaceTabsInToken
*/
final class ReplaceTabsInTokenMiscTest extends TestCase
{


/**
* Test that when no tab width is set or passed, the tab width will be set to 1.
*
* @return void
*/
public function testTabWidthNotSet()
{
$config = new ConfigDouble();
$ruleset = new Ruleset($config);

$content = <<<EOD
<?php
echo 'foo';
EOD;
$phpcsFile = new DummyFile($content, $ruleset, $config);
$phpcsFile->parse();

$tokens = $phpcsFile->getTokens();
$target = $phpcsFile->findNext(T_WHITESPACE, 0);

// Verify initial state.
$this->assertTrue(is_int($target), 'Target token was not found');
$this->assertSame(' ', $tokens[$target]['content'], 'Content after initial parsing does not contain tabs');
$this->assertSame(2, $tokens[$target]['length'], 'Length after initial parsing is not as expected');
$this->assertArrayNotHasKey('orig_content', $tokens[$target], "Key 'orig_content' found in the initial token array.");

$phpcsFile->tokenizer->replaceTabsInToken($tokens[$target]);

// Verify tab replacement.
$this->assertSame(' ', $tokens[$target]['content'], 'Content after tab replacement is not as expected');
$this->assertSame(2, $tokens[$target]['length'], 'Length after tab replacement is not as expected');
$this->assertArrayHasKey('orig_content', $tokens[$target], "Key 'orig_content' not found in the token array.");

}//end testTabWidthNotSet()


/**
* Test that the length calculation handles text in non-ascii encodings correctly.
*
* @requires extension iconv
*
* @return void
*/
public function testLengthSettingRespectsEncoding()
{
$config = new ConfigDouble();
$config->tabWidth = 4;
$ruleset = new Ruleset($config);

$content = <<<EOD
<?php
echo 'пасха пасха';
EOD;
$phpcsFile = new DummyFile($content, $ruleset, $config);
$phpcsFile->parse();

$tokens = $phpcsFile->getTokens();
$target = $phpcsFile->findNext(T_CONSTANT_ENCAPSED_STRING, 0);

$this->assertTrue(is_int($target), 'Target token was not found');
$this->assertSame("'пасха пасха'", $tokens[$target]['content'], 'Content is not as expected');
$this->assertSame(17, $tokens[$target]['length'], 'Length is not as expected');
$this->assertArrayHasKey('orig_content', $tokens[$target], "Key 'orig_content' not found in the token array.");
$this->assertSame("'пасха пасха'", $tokens[$target]['orig_content'], 'Orig_content is not as expected');

}//end testLengthSettingRespectsEncoding()


/**
* Test that the length calculation falls back to byte length if iconv detects an illegal character.
*
* @requires extension iconv
*
* @return void
*/
public function testLengthSettingFallsBackToBytesWhenTextContainsIllegalChars()
{
$config = new ConfigDouble();
$config->tabWidth = 4;
$ruleset = new Ruleset($config);

$content = <<<EOD
<?php
echo "aa\xC3\xC3 \xC3\xB8aa";
EOD;
$phpcsFile = new DummyFile($content, $ruleset, $config);
$phpcsFile->parse();

$tokens = $phpcsFile->getTokens();
$target = $phpcsFile->findNext(T_CONSTANT_ENCAPSED_STRING, 0);

$this->assertTrue(is_int($target), 'Target token was not found');
$this->assertSame(11, $tokens[$target]['length'], 'Length is not as expected');
$this->assertArrayHasKey('orig_content', $tokens[$target], "Key 'orig_content' not found in the token array.");

}//end testLengthSettingFallsBackToBytesWhenTextContainsIllegalChars()


}//end class
111 changes: 111 additions & 0 deletions tests/Core/Tokenizers/Tokenizer/ReplaceTabsInTokenTabWidth1Test.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
<?php
/**
* Tests the tab replacement logic.
*
* @author Juliette Reinders Folmer <[email protected]>
* @copyright 2024 PHPCSStandards and contributors
* @license https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
*/

namespace PHP_CodeSniffer\Tests\Core\Tokenizers\Tokenizer;

/**
* Tab replacement test using tab width 1.
*
* @covers PHP_CodeSniffer\Tokenizers\Tokenizer::replaceTabsInToken
*/
final class ReplaceTabsInTokenTabWidth1Test extends ReplaceTabsInTokenTestCase
{

/**
* The tab width setting to use when tokenizing the file.
*
* @var integer
*/
protected $tabWidth = 1;


/**
* Data provider helper.
*
* @see ReplaceTabsInTokenTestCase::dataTabReplacement()
*
* @return array<string, array<string, int|string>>
*/
public static function getTabReplacementExpected()
{
return [
'Tab indentation' => [
'length' => 2,
'content' => ' ',
'orig_content' => ' ',
],
'Mixed tab/space indentation' => [
'length' => 3,
'content' => ' ',
'orig_content' => ' ',
],
'Inline: single tab in text string' => [
'length' => 15,
'content' => "'tab separated'",
'orig_content' => "'tab separated'",
],
'Inline: single tab between each word in text string' => [
'length' => 24,
'content' => '"tab $between each word"',
'orig_content' => '"tab $between each word"',
],
'Inline: multiple tabs in heredoc' => [
'length' => 15,
'content' => 'tab separated
',
'orig_content' => 'tab separated
',
],
'Inline: multiple tabs between each word in nowdoc' => [
'length' => 27,
'content' => 'tab between each word
',
'orig_content' => 'tab between each word
',
],
'Inline: mixed spaces/tabs in text string' => [
'length' => 20,
'content' => "'tab separated'",
'orig_content' => "'tab separated'",
],
'Inline: mixed spaces/tabs between each word in text string' => [
'length' => 31,
'content' => '"tab $between each word"',
'orig_content' => '"tab $between each word"',
],
'Inline: tab becomes single space in comment (with tabwidth 4)' => [
'length' => 50,
'content' => '// -123 With tabwidth 4, the tab size should be 1.
',
'orig_content' => '// -123 With tabwidth 4, the tab size should be 1.
',
],
'Inline: tab becomes 2 spaces in comment (with tabwidth 4)' => [
'length' => 52,
'content' => '/* -12 With tabwidth 4, the tab size should be 2. */',
'orig_content' => '/* -12 With tabwidth 4, the tab size should be 2. */',
],
'Inline: tab becomes 3 spaces in doc comment string (with tabwidth 4)' => [
'length' => 45,
'content' => '-1 With tabwidth 4, the tab size should be 3.',
'orig_content' => '-1 With tabwidth 4, the tab size should be 3.',
],
'Inline: tab becomes 4 spaces in comment (with tabwidth 4)' => [
'length' => 47,
'content' => '// - With tabwidth 4, the tab size should be 4.
',
'orig_content' => '// - With tabwidth 4, the tab size should be 4.
',
],
];

}//end getTabReplacementExpected()


}//end class
Loading