Support <?php followed by EOF

This is an annoying edge-case for canonicalization.

Author: nikic

UPGRADING

@@ -34,6 +34,10 @@ PHP 7.4 UPGRADE NOTES
   . Passing the result of a (non-reference) list() assignment by reference is
     consistently disallowed now. Previously this worked if the right hand side
     was a simple (CV) variable and did not occur as part of the list().
+  . `<?php` at the end of the file (without trailing newline) will now be
+    interpreted as an opening PHP tag. Previously it was interpreted either as
+    `<? php` and resulted in a syntax error (with short_open_tag=1) or was
+    interpreted as a literal `<?php` string (with short_open_tag=0).
 
 - BCMath:
   . BCMath functions will now warn if a non well-formed number is passed, such

Zend/tests/php_tag_only.inc

@@ -0,0 +1 @@
+<?php

Zend/tests/php_tag_only.phpt

@@ -0,0 +1,5 @@
+--TEST--
+File with just a <?php tag should be valid
+--FILE_EXTERNAL--
+php_tag_only.inc
+--EXPECT--

Zend/zend_language_scanner.l

@@ -2031,6 +2031,20 @@ string:
 	RETURN_OR_SKIP_TOKEN(T_OPEN_TAG);
 }
 
+<INITIAL>"<?php" {
+	/* Allow <?php followed by end of file. */
+	if (YYCURSOR == YYLIMIT) {
+		BEGIN(ST_IN_SCRIPTING);
+		RETURN_OR_SKIP_TOKEN(T_OPEN_TAG);
+	}
+	/* Degenerate case: <?phpX is interpreted as <? phpX with short tags. */
+	if (CG(short_tags)) {
+		yyless(2);
+		BEGIN(ST_IN_SCRIPTING);
+		RETURN_OR_SKIP_TOKEN(T_OPEN_TAG);
+	}
+	goto inline_char_handler;
+}
 
 <INITIAL>"<?" {
 	if (CG(short_tags)) {

ext/tokenizer/tests/php_tag_only.phpt

@@ -0,0 +1,19 @@
+--TEST--
+Tokenization of only the <?php tag
+--FILE--
+<?php
+
+foreach (token_get_all("<?php") as $token) {
+    echo token_name($token[0]), "\n";
+}
+echo "\n";
+foreach (token_get_all("Foobar<?php") as $token) {
+    echo token_name($token[0]), "\n";
+}
+
+?>
+--EXPECT--
+T_OPEN_TAG
+
+T_INLINE_HTML
+T_OPEN_TAG