Skip to content

Commit 8acf004

Browse files
jrfnljrfnl
committed
PHP 7.4 | Tokenizer/PHP: handle PHP tag at end of file consistently
Prior to PHP 7.4, a PHP open tag at the end of a file was not tokenized correctly in PHP itself. From the PHP 7.4 changelog: > `<?php` at the end of the file (without trailing newline) will now be > interpreted as an opening PHP tag. Previously it was interpreted either as > `<? php` and resulted in a syntax error (with short_open_tag=1) or was > interpreted as a literal `<?php` string (with short_open_tag=0). This commit makes the tokenization of PHP open tags at the end of a file consistent in all PHP versions. Includes tests. Refs: * https://www.php.net/manual/en/migration74.incompatible.php#migration74.incompatible.core.php-tag * https://github.com/php/php-src/blob/30de357fa14480468132bbc22a272aeb91789ba8/UPGRADING#L37-L40
1 parent 9981876 commit 8acf004

File tree

7 files changed

+215
-0
lines changed

7 files changed

+215
-0
lines changed

‎src/Tokenizers/PHP.php‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -803,6 +803,47 @@ protected function tokenize($string)
803803
}
804804
}//end if
805805

806+
/*
807+
Prior to PHP 7.4, PHP didn't support stand-alone PHP open tags at the end of a file
808+
(without a new line), so we need to make sure that the tokenization in PHPCS is consistent
809+
cross-version PHP by retokenizing to T_OPEN_TAG.
810+
*/
811+
812+
if (PHP_VERSION_ID < 70400
813+
&& $tokenIsArray === true
814+
// PHP < 7.4 with short open tags off.
815+
&& (($stackPtr === ($numTokens - 1)
816+
&& $token[0] === T_INLINE_HTML
817+
&& stripos($token[1], '<?php') === 0)
818+
// PHP < 7.4 with short open tags on.
819+
|| ($stackPtr === ($numTokens - 2)
820+
&& $token[0] === T_OPEN_TAG
821+
&& $token[1] === '<?'
822+
&& is_array($tokens[($stackPtr + 1)]) === true
823+
&& $tokens[($stackPtr + 1)][0] === T_STRING
824+
&& strtolower($tokens[($stackPtr + 1)][1]) === 'php'))
825+
) {
826+
if ($token[0] === T_INLINE_HTML) {
827+
$finalTokens[$newStackPtr] = [
828+
'code' => T_OPEN_TAG,
829+
'type' => 'T_OPEN_TAG',
830+
'content' => $token[1],
831+
];
832+
833+
} else {
834+
$finalTokens[$newStackPtr] = [
835+
'code' => T_OPEN_TAG,
836+
'type' => 'T_OPEN_TAG',
837+
'content' => $token[1].$tokens[($stackPtr + 1)][1],
838+
];
839+
840+
$stackPtr++;
841+
}
842+
843+
$newStackPtr++;
844+
continue;
845+
}
846+
806847
/*
807848
Parse doc blocks into something that can be easily iterated over.
808849
*/

‎tests/Core/Tokenizers/PHP/PHPOpenTagEOF1Test.inc‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
<?php
2+
// This test case must be the last (only) test in the file without a new line after it!
3+
/* testLongOpenTagEndOfFileSpaceNoNewLine */ ?>
4+
<?php

‎tests/Core/Tokenizers/PHP/PHPOpenTagEOF1Test.php‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
<?php
2+
/**
3+
* Tests the tokenization of PHP open tags.
4+
*
5+
* Prior to PHP 7.4, PHP didn't support stand-alone PHP open tags at the end of a file (without a new line),
6+
* so we need to make sure that the tokenization in PHPCS is consistent and correct.
7+
*
8+
* @copyright 2025 PHPCSStandards and contributors
9+
* @license https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
10+
*/
11+
12+
namespace PHP_CodeSniffer\Tests\Core\Tokenizers\PHP;
13+
14+
use PHP_CodeSniffer\Tests\Core\Tokenizers\AbstractTokenizerTestCase;
15+
use PHP_CodeSniffer\Util\Tokens;
16+
17+
/**
18+
* Tests the tokenization of PHP open tags.
19+
*
20+
* @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize
21+
*/
22+
final class PHPOpenTagEOF1Test extends AbstractTokenizerTestCase
23+
{
24+
25+
26+
/**
27+
* Test that the tokenization of a long PHP open tag at the very end of a file is correct and consistent.
28+
*
29+
* @return void
30+
*/
31+
public function testLongOpenTagAtEndOfFile()
32+
{
33+
$tokens = $this->phpcsFile->getTokens();
34+
$stackPtr = $this->getTargetToken('/* testLongOpenTagEndOfFileSpaceNoNewLine */', [T_OPEN_TAG, T_STRING, T_INLINE_HTML]);
35+
36+
$this->assertSame(
37+
T_OPEN_TAG,
38+
$tokens[$stackPtr]['code'],
39+
'Token tokenized as '.Tokens::tokenName($tokens[$stackPtr]['code']).', not T_OPEN_TAG (code)'
40+
);
41+
$this->assertSame(
42+
'T_OPEN_TAG',
43+
$tokens[$stackPtr]['type'],
44+
'Token tokenized as '.$tokens[$stackPtr]['type'].', not T_OPEN_TAG (type)'
45+
);
46+
$this->assertSame('<?php ', $tokens[$stackPtr]['content']);
47+
48+
// Now make sure that this is the very last token in the file and there are no tokens after it.
49+
$this->assertArrayNotHasKey(($stackPtr + 1), $tokens);
50+
51+
}//end testLongOpenTagAtEndOfFile()
52+
53+
54+
}//end class

‎tests/Core/Tokenizers/PHP/PHPOpenTagEOF2Test.inc‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
<?php
2+
// This test case must be the last (only) test in the file without a new line after it!
3+
/* testLongOpenTagEndOfFileNoSpaceNoNewLine */ ?>
4+
<?php

‎tests/Core/Tokenizers/PHP/PHPOpenTagEOF2Test.php‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
<?php
2+
/**
3+
* Tests the tokenization of PHP open tags.
4+
*
5+
* Prior to PHP 7.4, PHP didn't support stand-alone PHP open tags at the end of a file (without a new line),
6+
* so we need to make sure that the tokenization in PHPCS is consistent and correct.
7+
*
8+
* @copyright 2025 PHPCSStandards and contributors
9+
* @license https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
10+
*/
11+
12+
namespace PHP_CodeSniffer\Tests\Core\Tokenizers\PHP;
13+
14+
use PHP_CodeSniffer\Tests\Core\Tokenizers\AbstractTokenizerTestCase;
15+
use PHP_CodeSniffer\Util\Tokens;
16+
17+
/**
18+
* Tests the tokenization of PHP open tags.
19+
*
20+
* @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize
21+
*/
22+
final class PHPOpenTagEOF2Test extends AbstractTokenizerTestCase
23+
{
24+
25+
26+
/**
27+
* Test that the tokenization of a long PHP open tag at the very end of a file is correct and consistent.
28+
*
29+
* @return void
30+
*/
31+
public function testLongOpenTagAtEndOfFile()
32+
{
33+
$tokens = $this->phpcsFile->getTokens();
34+
$stackPtr = $this->getTargetToken('/* testLongOpenTagEndOfFileNoSpaceNoNewLine */', [T_OPEN_TAG, T_STRING, T_INLINE_HTML]);
35+
36+
$this->assertSame(
37+
T_OPEN_TAG,
38+
$tokens[$stackPtr]['code'],
39+
'Token tokenized as '.Tokens::tokenName($tokens[$stackPtr]['code']).', not T_OPEN_TAG (code)'
40+
);
41+
$this->assertSame(
42+
'T_OPEN_TAG',
43+
$tokens[$stackPtr]['type'],
44+
'Token tokenized as '.$tokens[$stackPtr]['type'].', not T_OPEN_TAG (type)'
45+
);
46+
$this->assertSame('<?php', $tokens[$stackPtr]['content']);
47+
48+
// Now make sure that this is the very last token in the file and there are no tokens after it.
49+
$this->assertArrayNotHasKey(($stackPtr + 1), $tokens);
50+
51+
}//end testLongOpenTagAtEndOfFile()
52+
53+
54+
}//end class

‎tests/Core/Tokenizers/PHP/PHPOpenTagEOF3Test.inc‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
<?php
2+
// This test case must be the last (only) test in the file without a new line after it!
3+
/* testLongOpenTagEndOfFileNoSpaceNoNewLineUppercase */ ?>
4+
<?PHP

‎tests/Core/Tokenizers/PHP/PHPOpenTagEOF3Test.php‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
<?php
2+
/**
3+
* Tests the tokenization of PHP open tags.
4+
*
5+
* Prior to PHP 7.4, PHP didn't support stand-alone PHP open tags at the end of a file (without a new line),
6+
* so we need to make sure that the tokenization in PHPCS is consistent and correct.
7+
*
8+
* @copyright 2025 PHPCSStandards and contributors
9+
* @license https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
10+
*/
11+
12+
namespace PHP_CodeSniffer\Tests\Core\Tokenizers\PHP;
13+
14+
use PHP_CodeSniffer\Tests\Core\Tokenizers\AbstractTokenizerTestCase;
15+
use PHP_CodeSniffer\Util\Tokens;
16+
17+
/**
18+
* Tests the tokenization of PHP open tags.
19+
*
20+
* @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize
21+
*/
22+
final class PHPOpenTagEOF3Test extends AbstractTokenizerTestCase
23+
{
24+
25+
26+
/**
27+
* Test that the tokenization of a long PHP open tag at the very end of a file is correct and consistent.
28+
*
29+
* @return void
30+
*/
31+
public function testLongOpenTagAtEndOfFile()
32+
{
33+
$tokens = $this->phpcsFile->getTokens();
34+
$stackPtr = $this->getTargetToken('/* testLongOpenTagEndOfFileNoSpaceNoNewLineUppercase */', [T_OPEN_TAG, T_STRING, T_INLINE_HTML]);
35+
36+
$this->assertSame(
37+
T_OPEN_TAG,
38+
$tokens[$stackPtr]['code'],
39+
'Token tokenized as '.Tokens::tokenName($tokens[$stackPtr]['code']).', not T_OPEN_TAG (code)'
40+
);
41+
$this->assertSame(
42+
'T_OPEN_TAG',
43+
$tokens[$stackPtr]['type'],
44+
'Token tokenized as '.$tokens[$stackPtr]['type'].', not T_OPEN_TAG (type)'
45+
);
46+
$this->assertSame('<?PHP', $tokens[$stackPtr]['content']);
47+
48+
// Now make sure that this is the very last token in the file and there are no tokens after it.
49+
$this->assertArrayNotHasKey(($stackPtr + 1), $tokens);
50+
51+
}//end testLongOpenTagAtEndOfFile()
52+
53+
54+
}//end class

0 commit comments

Comments
 (0)