Skip to content

Commit bc630c3

Browse files
jrfnlgsherwood
jrfnl
authored and
gsherwood
committed
PHP 8.0 | Stabilize comment tokenization for hash comments
Follow up on 3027 which handled this for slash-style comments. Hash-style comments were not addressed in the earlier change. This oversight has now been fixed. Includes additional unit tests. Fixes 3069
1 parent 3099054 commit bc630c3

File tree

5 files changed

+246
-2
lines changed

5 files changed

+246
-2
lines changed

‎src/Tokenizers/PHP.php‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -571,12 +571,12 @@ protected function tokenize($string)
571571
}
572572

573573
/*
574-
PHP 8 tokenizes a new line after a slash comment to the next whitespace token.
574+
PHP 8 tokenizes a new line after a slash and hash comment to the next whitespace token.
575575
*/
576576

577577
if (PHP_VERSION_ID >= 80000
578578
&& $tokenIsArray === true
579-
&& ($token[0] === T_COMMENT && strpos($token[1], '//') === 0)
579+
&& ($token[0] === T_COMMENT && (strpos($token[1], '//') === 0 || strpos($token[1], '#') === 0))
580580
&& isset($tokens[($stackPtr + 1)]) === true
581581
&& is_array($tokens[($stackPtr + 1)]) === true
582582
&& $tokens[($stackPtr + 1)][0] === T_WHITESPACE

‎tests/Core/Tokenizer/StableCommentWhitespaceTest.inc‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,9 +111,29 @@ $prop = 123; /** Comment */
111111
* @tag Comment
112112
*/
113113

114+
/* testSingleLineHashComment */
115+
# Comment
116+
117+
/* testSingleLineHashCommentTrailing */
118+
echo 'a'; # Comment
119+
120+
/* testMultiLineHashComment */
121+
# Comment1
122+
# Comment2
123+
# Comment3
124+
125+
/* testMultiLineHashCommentWithIndent */
126+
# Comment1
127+
# Comment2
128+
# Comment3
129+
114130
/* testSingleLineSlashCommentNoNewLineAtEnd */
115131
// Slash ?>
116132
<?php
117133

134+
/* testSingleLineHashCommentNoNewLineAtEnd */
135+
# Hash ?>
136+
<?php
137+
118138
/* testCommentAtEndOfFile */
119139
/* Comment

‎tests/Core/Tokenizer/StableCommentWhitespaceTest.php‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -919,6 +919,94 @@ public function dataCommentTokenization()
919919
[
920920
'type' => 'T_WHITESPACE',
921921
'content' => '
922+
',
923+
],
924+
],
925+
],
926+
[
927+
'/* testSingleLineHashComment */',
928+
[
929+
[
930+
'type' => 'T_COMMENT',
931+
'content' => '# Comment
932+
',
933+
],
934+
[
935+
'type' => 'T_WHITESPACE',
936+
'content' => '
937+
',
938+
],
939+
],
940+
],
941+
[
942+
'/* testSingleLineHashCommentTrailing */',
943+
[
944+
[
945+
'type' => 'T_COMMENT',
946+
'content' => '# Comment
947+
',
948+
],
949+
[
950+
'type' => 'T_WHITESPACE',
951+
'content' => '
952+
',
953+
],
954+
],
955+
],
956+
[
957+
'/* testMultiLineHashComment */',
958+
[
959+
[
960+
'type' => 'T_COMMENT',
961+
'content' => '# Comment1
962+
',
963+
],
964+
[
965+
'type' => 'T_COMMENT',
966+
'content' => '# Comment2
967+
',
968+
],
969+
[
970+
'type' => 'T_COMMENT',
971+
'content' => '# Comment3
972+
',
973+
],
974+
[
975+
'type' => 'T_WHITESPACE',
976+
'content' => '
977+
',
978+
],
979+
],
980+
],
981+
[
982+
'/* testMultiLineHashCommentWithIndent */',
983+
[
984+
[
985+
'type' => 'T_COMMENT',
986+
'content' => '# Comment1
987+
',
988+
],
989+
[
990+
'type' => 'T_WHITESPACE',
991+
'content' => ' ',
992+
],
993+
[
994+
'type' => 'T_COMMENT',
995+
'content' => '# Comment2
996+
',
997+
],
998+
[
999+
'type' => 'T_WHITESPACE',
1000+
'content' => ' ',
1001+
],
1002+
[
1003+
'type' => 'T_COMMENT',
1004+
'content' => '# Comment3
1005+
',
1006+
],
1007+
[
1008+
'type' => 'T_WHITESPACE',
1009+
'content' => '
9221010
',
9231011
],
9241012
],
@@ -933,6 +1021,20 @@ public function dataCommentTokenization()
9331021
[
9341022
'type' => 'T_CLOSE_TAG',
9351023
'content' => '?>
1024+
',
1025+
],
1026+
],
1027+
],
1028+
[
1029+
'/* testSingleLineHashCommentNoNewLineAtEnd */',
1030+
[
1031+
[
1032+
'type' => 'T_COMMENT',
1033+
'content' => '# Hash ',
1034+
],
1035+
[
1036+
'type' => 'T_CLOSE_TAG',
1037+
'content' => '?>
9361038
',
9371039
],
9381040
],

‎tests/Core/Tokenizer/StableCommentWhitespaceWinTest.inc‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,5 +39,25 @@ echo 'a'; // Comment
3939
// Slash ?>
4040
<?php
4141

42+
/* testSingleLineHashComment */
43+
# Comment
44+
45+
/* testSingleLineHashCommentTrailing */
46+
echo 'a'; # Comment
47+
48+
/* testMultiLineHashComment */
49+
# Comment1
50+
# Comment2
51+
# Comment3
52+
53+
/* testMultiLineHashCommentWithIndent */
54+
# Comment1
55+
# Comment2
56+
# Comment3
57+
58+
/* testSingleLineHashCommentNoNewLineAtEnd */
59+
# Hash ?>
60+
<?php
61+
4262
/* testCommentAtEndOfFile */
4363
/* Comment

‎tests/Core/Tokenizer/StableCommentWhitespaceWinTest.php‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -244,6 +244,108 @@ public function dataCommentTokenization()
244244
[
245245
'type' => 'T_CLOSE_TAG',
246246
'content' => '?>
247+
',
248+
],
249+
],
250+
],
251+
[
252+
'/* testSingleLineHashComment */',
253+
[
254+
[
255+
'type' => 'T_COMMENT',
256+
'content' => '# Comment
257+
',
258+
],
259+
[
260+
'type' => 'T_WHITESPACE',
261+
'content' => '
262+
',
263+
],
264+
],
265+
],
266+
[
267+
'/* testSingleLineHashCommentTrailing */',
268+
[
269+
[
270+
'type' => 'T_COMMENT',
271+
'content' => '# Comment
272+
',
273+
],
274+
[
275+
'type' => 'T_WHITESPACE',
276+
'content' => '
277+
',
278+
],
279+
],
280+
],
281+
[
282+
'/* testMultiLineHashComment */',
283+
[
284+
[
285+
'type' => 'T_COMMENT',
286+
'content' => '# Comment1
287+
',
288+
],
289+
[
290+
'type' => 'T_COMMENT',
291+
'content' => '# Comment2
292+
',
293+
],
294+
[
295+
'type' => 'T_COMMENT',
296+
'content' => '# Comment3
297+
',
298+
],
299+
[
300+
'type' => 'T_WHITESPACE',
301+
'content' => '
302+
',
303+
],
304+
],
305+
],
306+
[
307+
'/* testMultiLineHashCommentWithIndent */',
308+
[
309+
[
310+
'type' => 'T_COMMENT',
311+
'content' => '# Comment1
312+
',
313+
],
314+
[
315+
'type' => 'T_WHITESPACE',
316+
'content' => ' ',
317+
],
318+
[
319+
'type' => 'T_COMMENT',
320+
'content' => '# Comment2
321+
',
322+
],
323+
[
324+
'type' => 'T_WHITESPACE',
325+
'content' => ' ',
326+
],
327+
[
328+
'type' => 'T_COMMENT',
329+
'content' => '# Comment3
330+
',
331+
],
332+
[
333+
'type' => 'T_WHITESPACE',
334+
'content' => '
335+
',
336+
],
337+
],
338+
],
339+
[
340+
'/* testSingleLineHashCommentNoNewLineAtEnd */',
341+
[
342+
[
343+
'type' => 'T_COMMENT',
344+
'content' => '# Hash ',
345+
],
346+
[
347+
'type' => 'T_CLOSE_TAG',
348+
'content' => '?>
247349
',
248350
],
249351
],

0 commit comments

Comments
 (0)