Skip to content

Commit 93ac4e2

Browse files
rlerdorfrlerdorf
authored
Merge pull request #12 from phan/parser-gaps
Fix various parser gaps
2 parents ef8b5b9 + 32d169e commit 93ac4e2

37 files changed

+974
-84
lines changed

‎src/DiagnosticsProvider.php‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,11 @@ private static function checkDiagnosticForUnexpectedToken($token) {
9696
public static function getDiagnostics(Node $n) : array {
9797
$diagnostics = [];
9898

99+
// Check the root node itself first
100+
if (($diagnostic = self::checkDiagnostics($n)) !== null) {
101+
$diagnostics[] = $diagnostic;
102+
}
103+
99104
/**
100105
* @param \Microsoft\PhpParser\Node|\Microsoft\PhpParser\Token $node
101106
*/

‎src/Node/PropertyDeclaration.php‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,9 @@ class PropertyDeclaration extends Node implements ModifiedTypeInterface {
2020
/** @var AttributeGroup[]|null */
2121
public $attributes;
2222

23+
/** @var Token|null asymmetric visibility for set operations (PHP 8.4+) */
24+
public $setVisibilityToken;
25+
2326
/** @var Token|null question token for PHP 7.4 type declaration */
2427
public $questionToken;
2528

@@ -35,6 +38,7 @@ class PropertyDeclaration extends Node implements ModifiedTypeInterface {
3538
const CHILD_NAMES = [
3639
'attributes',
3740
'modifiers',
41+
'setVisibilityToken',
3842
'questionToken',
3943
'typeDeclarationList',
4044
'propertyElements',

‎src/Node/SourceFileNode.php‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,5 +22,12 @@ class SourceFileNode extends Node {
2222
/** @var Token */
2323
public $endOfFileToken;
2424

25+
/** @var \Microsoft\PhpParser\Diagnostic|null */
26+
public $unterminatedCommentDiagnostic;
27+
2528
const CHILD_NAMES = ['statementList', 'endOfFileToken'];
29+
30+
public function getDiagnosticForNode() {
31+
return $this->unterminatedCommentDiagnostic;
32+
}
2633
}

‎src/Parser.php‎

Lines changed: 152 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,10 @@ public function __construct() {
153153
TokenKind::FloatReservedWord, TokenKind::IntReservedWord, TokenKind::StringReservedWord,
154154
TokenKind::ObjectReservedWord, TokenKind::NullReservedWord, TokenKind::FalseReservedWord,
155155
TokenKind::TrueReservedWord, TokenKind::IterableReservedWord, TokenKind::MixedReservedWord,
156-
TokenKind::VoidReservedWord, TokenKind::NeverReservedWord]; // TODO update spec
156+
TokenKind::VoidReservedWord, TokenKind::NeverReservedWord,
157+
// Legacy type aliases (invalid in actual code, but parser should accept them for error recovery)
158+
TokenKind::BooleanReservedWord, TokenKind::IntegerReservedWord, TokenKind::DoubleReservedWord,
159+
TokenKind::RealReservedWord, TokenKind::BinaryReservedWord]; // TODO update spec
157160
$this->returnTypeDeclarationTokens = \array_merge([TokenKind::StaticKeyword], $this->parameterTypeDeclarationTokens);
158161
}
159162

@@ -204,11 +207,55 @@ public function parseSourceFile(string $fileContents, ?string $uri = null) : Sou
204207
$this->sourceFile->endOfFileToken = $this->eat1(TokenKind::EndOfFileToken);
205208
$this->advanceToken();
206209

210+
// Check for unterminated comments in the trivia before EOF
211+
$this->checkForUnterminatedComment($sourceFile);
212+
207213
$sourceFile->parent = null;
208214

209215
return $sourceFile;
210216
}
211217

218+
private function checkForUnterminatedComment($sourceFile) {
219+
// Check if the trivia before EOF contains an unterminated /* or /** comment
220+
$eofToken = $sourceFile->endOfFileToken;
221+
$trivia = substr($sourceFile->fileContents, $eofToken->fullStart, $eofToken->start - $eofToken->fullStart);
222+
223+
// Look for /* or /** that doesn't have a matching */
224+
// We need to make sure it's not inside a single-line comment
225+
if (preg_match('/\/\*\*?(?:(?!\*\/).)*$/s', $trivia, $matches, PREG_OFFSET_CAPTURE)) {
226+
// Found potential unterminated comment - verify it's not in a // comment
227+
$commentStart = $eofToken->fullStart + $matches[0][1];
228+
229+
// Check if this /* appears after a // on the same line
230+
// Get the line containing the /* by looking back to the last newline
231+
// Clamp the offset to avoid ValueError when $commentStart is near the beginning
232+
$offset = $commentStart - strlen($sourceFile->fileContents) - 1;
233+
$offset = max($offset, -strlen($sourceFile->fileContents));
234+
$lastNewline = strrpos($sourceFile->fileContents, "\n", $offset);
235+
$lineStart = $lastNewline !== false ? $lastNewline + 1 : 0;
236+
$currentLine = substr($sourceFile->fileContents, $lineStart, $commentStart - $lineStart + 2); // +2 to include /*
237+
238+
// If there's a // before /* on the same line, it's inside a single-line comment
239+
$doubleSlashPos = strpos($currentLine, '//');
240+
$slashStarPos = strpos($currentLine, '/*');
241+
if ($doubleSlashPos !== false && $doubleSlashPos < $slashStarPos) {
242+
return; // The /* is inside a // comment
243+
}
244+
245+
$commentText = $matches[0][0];
246+
247+
// Find the line number where the comment starts
248+
$lineNumber = 1 + substr_count(substr($sourceFile->fileContents, 0, $commentStart), "\n");
249+
250+
$sourceFile->unterminatedCommentDiagnostic = new Diagnostic(
251+
DiagnosticKind::Error,
252+
"Unterminated comment starting line $lineNumber",
253+
$commentStart,
254+
strlen($commentText)
255+
);
256+
}
257+
}
258+
212259
private function reset() {
213260
$this->advanceToken();
214261

@@ -664,6 +711,10 @@ private function parseClassElementFn() {
664711
case TokenKind::UseKeyword:
665712
return $this->parseTraitUseClause($parentNode);
666713

714+
case TokenKind::CaseKeyword:
715+
// enum case in a class - error but parse for recovery
716+
return $this->parseEnumCaseDeclaration($parentNode);
717+
667718
case TokenKind::AttributeToken:
668719
return $this->parseAttributeStatement($parentNode);
669720

@@ -857,7 +908,7 @@ private function parseParameterFn() {
857908
// PHP 8.5+ allows final/readonly to precede visibility, so we need to parse all modifiers first
858909
// then extract visibility from the modifiers list
859910
$parameter->visibilityToken = $this->eatOptional([TokenKind::PublicKeyword, TokenKind::ProtectedKeyword, TokenKind::PrivateKeyword]);
860-
$parameter->setVisibilityToken = $this->eatOptional([TokenKind::ProtectedSetKeyword, TokenKind::PrivateSetKeyword]);
911+
$parameter->setVisibilityToken = $this->eatOptional([TokenKind::PublicSetKeyword, TokenKind::ProtectedSetKeyword, TokenKind::PrivateSetKeyword]);
861912
$parameter->modifiers = $this->parseParameterModifiers() ?: null;
862913

863914
// If visibilityToken is null but we have modifiers, check if there's a visibility modifier
@@ -887,6 +938,16 @@ private function parseParameterFn() {
887938
array_pop($parameter->typeDeclarationList->children);
888939
$parameter->byRefToken = array_pop($parameter->typeDeclarationList->children);
889940
}
941+
// Check for invalid ?Type|OtherType syntax (nullable with union types)
942+
if ($parameter->questionToken) {
943+
foreach ($children as $index => $child) {
944+
if ($child instanceof Token && $child->kind === TokenKind::BarToken) {
945+
// Mark the BarToken as a SkippedToken to generate a diagnostic error
946+
$parameter->typeDeclarationList->children[$index] = new SkippedToken($child);
947+
break;
948+
}
949+
}
950+
}
890951
} elseif ($parameter->questionToken) {
891952
// TODO ParameterType?
892953
$parameter->typeDeclarationList = new MissingToken(TokenKind::PropertyType, $this->token->fullStart);
@@ -1097,6 +1158,9 @@ private function isClassMemberDeclarationStart(Token $token) {
10971158

10981159
case TokenKind::UseKeyword:
10991160

1161+
// case (enum case in class - error but accept for recovery)
1162+
case TokenKind::CaseKeyword:
1163+
11001164
// attributes
11011165
case TokenKind::AttributeToken:
11021166
return true;
@@ -1205,6 +1269,9 @@ private function isExpressionStartFn() {
12051269
case TokenKind::YieldKeyword:
12061270
case TokenKind::YieldFromKeyword:
12071271

1272+
// throw-expression (PHP 8.0+)
1273+
case TokenKind::ThrowKeyword:
1274+
12081275
// object-creation-expression
12091276
case TokenKind::NewKeyword:
12101277
case TokenKind::CloneKeyword:
@@ -1584,6 +1651,11 @@ private function isModifier($token): bool {
15841651
case TokenKind::ProtectedKeyword:
15851652
case TokenKind::PrivateKeyword:
15861653

1654+
// asymmetric visibility (PHP 8.4+)
1655+
case TokenKind::PrivateSetKeyword:
1656+
case TokenKind::ProtectedSetKeyword:
1657+
case TokenKind::PublicSetKeyword:
1658+
15871659
// static-modifier
15881660
case TokenKind::StaticKeyword:
15891661

@@ -1688,6 +1760,11 @@ private function isParameterStartFn() {
16881760
case TokenKind::FinalKeyword:
16891761
case TokenKind::AttributeToken:
16901762

1763+
// asymmetric visibility (PHP 8.4+)
1764+
case TokenKind::PublicSetKeyword:
1765+
case TokenKind::ProtectedSetKeyword:
1766+
case TokenKind::PrivateSetKeyword:
1767+
16911768
// dnf types (A&B)|C
16921769
case TokenKind::OpenParenToken:
16931770
return true;
@@ -3349,7 +3426,12 @@ private function parseObjectCreationExpression($parentNode) {
33493426
}
33503427

33513428
// PHP 8.4: new without parenthesis - allow chaining directly after new expression
3352-
if ($this->getCurrentToken()->kind === TokenKind::ArrowToken) {
3429+
$tokenKind = $this->getCurrentToken()->kind;
3430+
if ($tokenKind === TokenKind::ArrowToken ||
3431+
$tokenKind === TokenKind::QuestionArrowToken ||
3432+
$tokenKind === TokenKind::OpenBracketToken ||
3433+
$tokenKind === TokenKind::OpenParenToken ||
3434+
$tokenKind === TokenKind::ColonColonToken) {
33533435
return $this->parsePostfixExpressionRest($objectCreationExpression);
33543436
}
33553437

@@ -3510,7 +3592,16 @@ private function parsePropertyDeclaration($parentNode, $modifiers, $questionToke
35103592
$propertyDeclaration = new PropertyDeclaration();
35113593
$propertyDeclaration->parent = $parentNode;
35123594

3513-
$propertyDeclaration->modifiers = $modifiers;
3595+
// Extract setVisibilityToken from modifiers (PHP 8.4+)
3596+
$filteredModifiers = [];
3597+
foreach ($modifiers as $modifier) {
3598+
if ($modifier->kind === TokenKind::PublicSetKeyword || $modifier->kind === TokenKind::PrivateSetKeyword || $modifier->kind === TokenKind::ProtectedSetKeyword) {
3599+
$propertyDeclaration->setVisibilityToken = $modifier;
3600+
} else {
3601+
$filteredModifiers[] = $modifier;
3602+
}
3603+
}
3604+
$propertyDeclaration->modifiers = $filteredModifiers ?: null;
35143605
$propertyDeclaration->questionToken = $questionToken;
35153606
if ($typeDeclarationList) {
35163607
$propertyDeclaration->typeDeclarationList = $typeDeclarationList;
@@ -3648,6 +3739,9 @@ private function parsePropertyHook(PropertyHookList $parentNode): ?PropertyHook
36483739
$hook->arrowToken = $this->eat1(TokenKind::DoubleArrowToken);
36493740
$hook->expression = $this->parseExpression($hook);
36503741
$hook->semicolon = $this->eatOptional1(TokenKind::SemicolonToken) ?? new MissingToken(TokenKind::SemicolonToken, $this->getCurrentToken()->fullStart);
3742+
} elseif ($this->checkToken(TokenKind::SemicolonToken)) {
3743+
// Abstract hook (interface or abstract class)
3744+
$hook->semicolon = $this->eat1(TokenKind::SemicolonToken);
36513745
} else {
36523746
$hook->compoundStatement = $this->parseCompoundStatement($hook);
36533747
}
@@ -3727,7 +3821,43 @@ private function isInterfaceMemberDeclarationStart(Token $token) {
37273821

37283822
case TokenKind::FunctionKeyword:
37293823

3824+
// trait use (not allowed but parse for error recovery)
3825+
case TokenKind::UseKeyword:
3826+
37303827
case TokenKind::AttributeToken:
3828+
3829+
// PHP 8.4: interface property hooks without explicit visibility
3830+
case TokenKind::QuestionToken: // nullable type: ?Foo $bar
3831+
case TokenKind::VariableName: // no type: $bar
3832+
3833+
// Type tokens that can start a property declaration
3834+
case TokenKind::Name: // qualified names: Foo $bar
3835+
case TokenKind::BackslashToken: // fully qualified: \Foo $bar
3836+
case TokenKind::NamespaceKeyword: // relative: namespace\Foo $bar
3837+
case TokenKind::OpenParenToken: // DNF types: (A&B)|C $bar
3838+
3839+
// Built-in type keywords
3840+
case TokenKind::ArrayKeyword:
3841+
case TokenKind::CallableKeyword:
3842+
case TokenKind::BoolReservedWord:
3843+
case TokenKind::FloatReservedWord:
3844+
case TokenKind::IntReservedWord:
3845+
case TokenKind::StringReservedWord:
3846+
case TokenKind::ObjectReservedWord:
3847+
case TokenKind::NullReservedWord:
3848+
case TokenKind::FalseReservedWord:
3849+
case TokenKind::TrueReservedWord:
3850+
case TokenKind::IterableReservedWord:
3851+
case TokenKind::MixedReservedWord:
3852+
case TokenKind::VoidReservedWord:
3853+
case TokenKind::NeverReservedWord:
3854+
3855+
// Legacy type aliases (for error recovery)
3856+
case TokenKind::BooleanReservedWord:
3857+
case TokenKind::IntegerReservedWord:
3858+
case TokenKind::DoubleReservedWord:
3859+
case TokenKind::RealReservedWord:
3860+
case TokenKind::BinaryReservedWord:
37313861
return true;
37323862
}
37333863
return false;
@@ -3745,14 +3875,28 @@ private function parseInterfaceElementFn() {
37453875
case TokenKind::FunctionKeyword:
37463876
return $this->parseMethodDeclaration($parentNode, $modifiers);
37473877

3878+
case TokenKind::QuestionToken:
3879+
// PHP 8.4: property hooks in interfaces
3880+
return $this->parseRemainingPropertyDeclarationOrMissingMemberDeclaration(
3881+
$parentNode,
3882+
$modifiers,
3883+
$this->eat1(TokenKind::QuestionToken)
3884+
);
3885+
3886+
case TokenKind::VariableName:
3887+
// PHP 8.4: property hooks in interfaces
3888+
return $this->parsePropertyDeclaration($parentNode, $modifiers);
3889+
3890+
case TokenKind::UseKeyword:
3891+
// Trait use in interface - not allowed in PHP but parse for error recovery
3892+
return $this->parseTraitUseClause($parentNode);
3893+
37483894
case TokenKind::AttributeToken:
37493895
return $this->parseAttributeStatement($parentNode);
37503896

37513897
default:
3752-
$missingInterfaceMemberDeclaration = new MissingMemberDeclaration();
3753-
$missingInterfaceMemberDeclaration->parent = $parentNode;
3754-
$missingInterfaceMemberDeclaration->modifiers = $modifiers;
3755-
return $missingInterfaceMemberDeclaration;
3898+
// PHP 8.4: property hooks in interfaces - handle type declarations
3899+
return $this->parseRemainingPropertyDeclarationOrMissingMemberDeclaration($parentNode, $modifiers);
37563900
}
37573901
};
37583902
}

‎src/PhpTokenizer.php‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -259,6 +259,8 @@ public static function getTokensArrayFromContent(
259259
$newTokenKind = TokenKind::PrivateSetKeyword;
260260
} elseif ($tokenKind === 328 && defined('T_PROTECTED_SET') && T_PROTECTED_SET === 328) {
261261
$newTokenKind = TokenKind::ProtectedSetKeyword;
262+
} elseif ($tokenKind === 329 && defined('T_PUBLIC_SET') && T_PUBLIC_SET === 329) {
263+
$newTokenKind = TokenKind::PublicSetKeyword;
262264
} else {
263265
$newTokenKind = TokenKind::Unknown;
264266
}

‎src/TokenKind.php‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,7 @@ class TokenKind {
9494
const HaltCompilerKeyword = 173;
9595
const PrivateSetKeyword = 174;
9696
const ProtectedSetKeyword = 175;
97+
const PublicSetKeyword = 176;
9798

9899
const OpenBracketToken = 201;
99100
const CloseBracketToken = 202;

‎tests/cases/parser/dnfTypesProperty1.php.tree‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
"textLength": 6
4343
}
4444
],
45+
"setVisibilityToken": null,
4546
"questionToken": null,
4647
"typeDeclarationList": {
4748
"QualifiedNameList": {

‎tests/cases/parser/propertyDeclaration1.php.tree‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
"textLength": 6
4343
}
4444
],
45+
"setVisibilityToken": null,
4546
"questionToken": null,
4647
"typeDeclarationList": null,
4748
"propertyElements": {

‎tests/cases/parser/propertyDeclaration10.php.tree‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
"textLength": 6
4343
}
4444
],
45+
"setVisibilityToken": null,
4546
"questionToken": null,
4647
"typeDeclarationList": null,
4748
"propertyElements": {

‎tests/cases/parser/propertyDeclaration11.php.tree‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
"textLength": 6
4343
}
4444
],
45+
"setVisibilityToken": null,
4546
"questionToken": null,
4647
"typeDeclarationList": null,
4748
"propertyElements": {

0 commit comments

Comments
 (0)