Skip to content

Commit c288675

Browse files
jrfnljrfnl
committed
PHP 7.4 | Tokenizer/PHP: handle PHP tag at end of file consistently
Prior to PHP 7.4, a PHP open tag at the end of a file was not tokenized correctly in PHP itself. From the PHP 7.4 changelog: > `<?php` at the end of the file (without trailing newline) will now be > interpreted as an opening PHP tag. Previously it was interpreted either as > `<? php` and resulted in a syntax error (with short_open_tag=1) or was > interpreted as a literal `<?php` string (with short_open_tag=0). This commit makes the tokenization of PHP open tags at the end of a file consistent in all PHP versions. Includes tests. Includes adding a few extra test builds which test with `short_open_tag=On` for various PHP versions. Refs: * https://www.php.net/manual/en/migration74.incompatible.php#migration74.incompatible.core.php-tag * https://github.com/php/php-src/blob/30de357fa14480468132bbc22a272aeb91789ba8/UPGRADING#L37-L40
1 parent 7c193ac commit c288675

File tree

8 files changed

+218
-0
lines changed

8 files changed

+218
-0
lines changed

‎.github/workflows/test.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,9 @@ jobs:
8080
- php: '7.4'
8181
os: 'ubuntu-latest'
8282
custom_ini: true
83+
- php: '8.3'
84+
os: 'ubuntu-latest'
85+
custom_ini: true
8386

8487
# yamllint disable-line rule:line-length
8588
name: "PHP: ${{ matrix.php }} ${{ matrix.custom_ini && ' with custom ini settings' || '' }}${{ matrix.libxml_minor && format( ' with libxml {0}', matrix.libxml_minor ) || '' }} (${{ matrix.os == 'ubuntu-latest' && 'Linux' || 'Win' }})"
@@ -204,6 +207,7 @@ jobs:
204207
PHP_CODESNIFFER_CBF: '1'
205208

206209
- name: 'PHPCS: check code style without cache, no parallel'
210+
if: ${{ matrix.custom_ini == false }}
207211
id: phpcs
208212
run: >
209213
php "bin/phpcs" --no-cache --parallel=1

‎src/Tokenizers/PHP.php‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -790,6 +790,46 @@ protected function tokenize($string)
790790
}
791791
}//end if
792792

793+
/*
794+
Prior to PHP 7.4, PHP didn't support stand-alone PHP open tags at the end of a file
795+
(without a new line), so we need to make sure that the tokenization in PHPCS is consistent
796+
cross-version PHP by retokenizing to T_OPEN_TAG.
797+
*/
798+
799+
if (PHP_VERSION_ID < 70400
800+
&& $tokenIsArray === true
801+
// PHP < 7.4 with short open tags off.
802+
&& (($stackPtr === ($numTokens - 1)
803+
&& $token[0] === T_INLINE_HTML
804+
&& stripos($token[1], '<?php') === 0)
805+
// PHP < 7.4 with short open tags on.
806+
|| ($stackPtr === ($numTokens - 2)
807+
&& $token[0] === T_OPEN_TAG
808+
&& $token[1] === '<?'
809+
&& is_array($tokens[($stackPtr + 1)]) === true
810+
&& $tokens[($stackPtr + 1)][0] === T_STRING
811+
&& strtolower($tokens[($stackPtr + 1)][1]) === 'php'))
812+
) {
813+
if ($token[0] === T_INLINE_HTML) {
814+
$finalTokens[$newStackPtr] = [
815+
'code' => T_OPEN_TAG,
816+
'type' => 'T_OPEN_TAG',
817+
'content' => $token[1],
818+
];
819+
} else {
820+
$finalTokens[$newStackPtr] = [
821+
'code' => T_OPEN_TAG,
822+
'type' => 'T_OPEN_TAG',
823+
'content' => $token[1].$tokens[($stackPtr + 1)][1],
824+
];
825+
826+
$stackPtr++;
827+
}
828+
829+
$newStackPtr++;
830+
continue;
831+
}//end if
832+
793833
/*
794834
Parse doc blocks into something that can be easily iterated over.
795835
*/

‎tests/Core/Tokenizers/PHP/PHPOpenTagEOF1Test.inc‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
<?php
2+
// This test case must be the last (only) test in the file without a new line after it!
3+
/* testLongOpenTagEndOfFileSpaceNoNewLine */ ?>
4+
<?php

‎tests/Core/Tokenizers/PHP/PHPOpenTagEOF1Test.php‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
<?php
2+
/**
3+
* Tests the tokenization of PHP open tags.
4+
*
5+
* Prior to PHP 7.4, PHP didn't support stand-alone PHP open tags at the end of a file (without a new line),
6+
* so we need to make sure that the tokenization in PHPCS is consistent and correct.
7+
*
8+
* @copyright 2025 PHPCSStandards and contributors
9+
* @license https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
10+
*/
11+
12+
namespace PHP_CodeSniffer\Tests\Core\Tokenizers\PHP;
13+
14+
use PHP_CodeSniffer\Tests\Core\Tokenizers\AbstractTokenizerTestCase;
15+
use PHP_CodeSniffer\Util\Tokens;
16+
17+
/**
18+
* Tests the tokenization of PHP open tags.
19+
*
20+
* @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize
21+
*/
22+
final class PHPOpenTagEOF1Test extends AbstractTokenizerTestCase
23+
{
24+
25+
26+
/**
27+
* Test that the tokenization of a long PHP open tag at the very end of a file is correct and consistent.
28+
*
29+
* @return void
30+
*/
31+
public function testLongOpenTagAtEndOfFile()
32+
{
33+
$tokens = $this->phpcsFile->getTokens();
34+
$stackPtr = $this->getTargetToken('/* testLongOpenTagEndOfFileSpaceNoNewLine */', [T_OPEN_TAG, T_STRING, T_INLINE_HTML]);
35+
36+
$this->assertSame(
37+
T_OPEN_TAG,
38+
$tokens[$stackPtr]['code'],
39+
'Token tokenized as '.Tokens::tokenName($tokens[$stackPtr]['code']).', not T_OPEN_TAG (code)'
40+
);
41+
$this->assertSame(
42+
'T_OPEN_TAG',
43+
$tokens[$stackPtr]['type'],
44+
'Token tokenized as '.$tokens[$stackPtr]['type'].', not T_OPEN_TAG (type)'
45+
);
46+
$this->assertSame('<?php ', $tokens[$stackPtr]['content']);
47+
48+
// Now make sure that this is the very last token in the file and there are no tokens after it.
49+
$this->assertArrayNotHasKey(($stackPtr + 1), $tokens);
50+
51+
}//end testLongOpenTagAtEndOfFile()
52+
53+
54+
}//end class

‎tests/Core/Tokenizers/PHP/PHPOpenTagEOF2Test.inc‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
<?php
2+
// This test case must be the last (only) test in the file without a new line after it!
3+
/* testLongOpenTagEndOfFileNoSpaceNoNewLine */ ?>
4+
<?php

‎tests/Core/Tokenizers/PHP/PHPOpenTagEOF2Test.php‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
<?php
2+
/**
3+
* Tests the tokenization of PHP open tags.
4+
*
5+
* Prior to PHP 7.4, PHP didn't support stand-alone PHP open tags at the end of a file (without a new line),
6+
* so we need to make sure that the tokenization in PHPCS is consistent and correct.
7+
*
8+
* @copyright 2025 PHPCSStandards and contributors
9+
* @license https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
10+
*/
11+
12+
namespace PHP_CodeSniffer\Tests\Core\Tokenizers\PHP;
13+
14+
use PHP_CodeSniffer\Tests\Core\Tokenizers\AbstractTokenizerTestCase;
15+
use PHP_CodeSniffer\Util\Tokens;
16+
17+
/**
18+
* Tests the tokenization of PHP open tags.
19+
*
20+
* @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize
21+
*/
22+
final class PHPOpenTagEOF2Test extends AbstractTokenizerTestCase
23+
{
24+
25+
26+
/**
27+
* Test that the tokenization of a long PHP open tag at the very end of a file is correct and consistent.
28+
*
29+
* @return void
30+
*/
31+
public function testLongOpenTagAtEndOfFile()
32+
{
33+
$tokens = $this->phpcsFile->getTokens();
34+
$stackPtr = $this->getTargetToken('/* testLongOpenTagEndOfFileNoSpaceNoNewLine */', [T_OPEN_TAG, T_STRING, T_INLINE_HTML]);
35+
36+
$this->assertSame(
37+
T_OPEN_TAG,
38+
$tokens[$stackPtr]['code'],
39+
'Token tokenized as '.Tokens::tokenName($tokens[$stackPtr]['code']).', not T_OPEN_TAG (code)'
40+
);
41+
$this->assertSame(
42+
'T_OPEN_TAG',
43+
$tokens[$stackPtr]['type'],
44+
'Token tokenized as '.$tokens[$stackPtr]['type'].', not T_OPEN_TAG (type)'
45+
);
46+
$this->assertSame('<?php', $tokens[$stackPtr]['content']);
47+
48+
// Now make sure that this is the very last token in the file and there are no tokens after it.
49+
$this->assertArrayNotHasKey(($stackPtr + 1), $tokens);
50+
51+
}//end testLongOpenTagAtEndOfFile()
52+
53+
54+
}//end class

‎tests/Core/Tokenizers/PHP/PHPOpenTagEOF3Test.inc‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
<?php
2+
// This test case must be the last (only) test in the file without a new line after it!
3+
/* testLongOpenTagEndOfFileNoSpaceNoNewLineUppercase */ ?>
4+
<?PHP

‎tests/Core/Tokenizers/PHP/PHPOpenTagEOF3Test.php‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
<?php
2+
/**
3+
* Tests the tokenization of PHP open tags.
4+
*
5+
* Prior to PHP 7.4, PHP didn't support stand-alone PHP open tags at the end of a file (without a new line),
6+
* so we need to make sure that the tokenization in PHPCS is consistent and correct.
7+
*
8+
* @copyright 2025 PHPCSStandards and contributors
9+
* @license https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
10+
*/
11+
12+
namespace PHP_CodeSniffer\Tests\Core\Tokenizers\PHP;
13+
14+
use PHP_CodeSniffer\Tests\Core\Tokenizers\AbstractTokenizerTestCase;
15+
use PHP_CodeSniffer\Util\Tokens;
16+
17+
/**
18+
* Tests the tokenization of PHP open tags.
19+
*
20+
* @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize
21+
*/
22+
final class PHPOpenTagEOF3Test extends AbstractTokenizerTestCase
23+
{
24+
25+
26+
/**
27+
* Test that the tokenization of a long PHP open tag at the very end of a file is correct and consistent.
28+
*
29+
* @return void
30+
*/
31+
public function testLongOpenTagAtEndOfFile()
32+
{
33+
$tokens = $this->phpcsFile->getTokens();
34+
$stackPtr = $this->getTargetToken('/* testLongOpenTagEndOfFileNoSpaceNoNewLineUppercase */', [T_OPEN_TAG, T_STRING, T_INLINE_HTML]);
35+
36+
$this->assertSame(
37+
T_OPEN_TAG,
38+
$tokens[$stackPtr]['code'],
39+
'Token tokenized as '.Tokens::tokenName($tokens[$stackPtr]['code']).', not T_OPEN_TAG (code)'
40+
);
41+
$this->assertSame(
42+
'T_OPEN_TAG',
43+
$tokens[$stackPtr]['type'],
44+
'Token tokenized as '.$tokens[$stackPtr]['type'].', not T_OPEN_TAG (type)'
45+
);
46+
$this->assertSame('<?PHP', $tokens[$stackPtr]['content']);
47+
48+
// Now make sure that this is the very last token in the file and there are no tokens after it.
49+
$this->assertArrayNotHasKey(($stackPtr + 1), $tokens);
50+
51+
}//end testLongOpenTagAtEndOfFile()
52+
53+
54+
}//end class

0 commit comments

Comments
 (0)