Skip to content

Add native parameter types to Token constructor #526

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jan 16, 2024
Merged

Add native parameter types to Token constructor #526

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
974c487
Update tests/data/* files
MauricioFauth Jan 15, 2024
16389b2
Add native parameter types to Token constructor
MauricioFauth Jan 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
10 changes: 0 additions & 10 deletions phpstan-baseline.neon
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -385,16 +385,6 @@ parameters:
count: 3
path: src/Lexer.php

-
message: "#^Parameter \\#1 \\$token of class PhpMyAdmin\\\\SqlParser\\\\Token constructor expects string, null given\\.$#"
count: 1
path: src/Lexer.php

-
message: "#^Parameter \\#3 \\$flags of class PhpMyAdmin\\\\SqlParser\\\\Token constructor expects int, int\\|null given\\.$#"
count: 1
path: src/Lexer.php

-
message: "#^Property PhpMyAdmin\\\\SqlParser\\\\Lexer\\:\\:\\$delimiter \\(string\\) does not accept null\\.$#"
count: 1
Expand Down
4 changes: 0 additions & 4 deletions psalm-baseline.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -577,11 +577,7 @@
<code><![CDATA[$this->str[$this->last]]]></code>
<code><![CDATA[$token->value]]></code>
</MixedOperand>
<NullArgument>
<code>null</code>
</NullArgument>
<PossiblyNullArgument>
<code>$flags</code>
<code><![CDATA[$this->str[$this->last + 1]]]></code>
<code><![CDATA[$this->str[$this->last]]]></code>
<code><![CDATA[$this->str[$this->last]]]></code>
Expand Down
6 changes: 3 additions & 3 deletions src/Lexer.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -282,7 +282,7 @@ public function lex(): void
}

// Adding a final delimiter to mark the ending.
$list->tokens[$list->count++] = new Token(null, TokenType::Delimiter);
$list->tokens[$list->count++] = new Token('', TokenType::Delimiter);

// Saving the tokens list.
$this->list = $list;
Expand Down Expand Up @@ -360,7 +360,7 @@ private function solveAmbiguityOnFunctionKeywords(): void
&& ($next->type !== TokenType::Operator
|| ! in_array($next->value, self::OPERATOR_NAME_INDICATORS, true)
)
&& ($next->value !== null)
&& ($next->value !== '')
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I hope that this ok, but I feel like the phpdoc should be updated.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Token::$value type should definitely be updated. That value is only created by Lexer line 285.

sql-parser/src/Lexer.php

Lines 284 to 285 in 94f2db2

// Adding a final delimiter to mark the ending.
$list->tokens[$list->count++] = new Token(null, TokenType::Delimiter);

) {
continue;
}
Expand Down Expand Up @@ -915,7 +915,7 @@ public function parseString(string $quote = ''): Token|null
$token .= $this->str[$this->last];
}

return new Token($token, TokenType::String, $flags);
return new Token($token, TokenType::String, $flags ?? Token::FLAG_NONE);
}

/**
Expand Down
10 changes: 3 additions & 7 deletions src/Token.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,10 +64,8 @@ class Token

/**
* The token it its raw string representation.
*
* @var string
*/
public $token;
public string $token;

/**
* The value this token contains (i.e. token after some evaluation).
Expand All @@ -90,10 +88,8 @@ class Token

/**
* The flags of this token.
*
* @var int
*/
public $flags;
public int $flags;

/**
* The position in the initial string where this token started.
Expand All @@ -110,7 +106,7 @@ class Token
* @param TokenType $type the type of the token
* @param int $flags the flags of the token
*/
public function __construct($token, TokenType $type = TokenType::None, $flags = 0)
public function __construct(string $token, TokenType $type = TokenType::None, int $flags = self::FLAG_NONE)
{
$this->token = $token;
$this->type = $type;
Expand Down
4 changes: 2 additions & 2 deletions tests/Utils/CLITest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -329,7 +329,7 @@ public static function tokenizeParamsProvider(): array
$result = "[TOKEN 0]\nType = 1\nFlags = 3\nValue = 'SELECT'\nToken = 'SELECT'\n\n"
. "[TOKEN 1]\nType = 3\nFlags = 0\nValue = ' '\nToken = ' '\n\n"
. "[TOKEN 2]\nType = 6\nFlags = 0\nValue = 1\nToken = '1'\n\n"
. "[TOKEN 3]\nType = 9\nFlags = 0\nValue = NULL\nToken = NULL\n\n";
. "[TOKEN 3]\nType = 9\nFlags = 0\nValue = ''\nToken = ''\n\n";

return [
[
Expand Down Expand Up @@ -381,7 +381,7 @@ public static function tokenizeParamsStdInProvider(): array
$result = "[TOKEN 0]\nType = 1\nFlags = 3\nValue = 'SELECT'\nToken = 'SELECT'\n\n"
. "[TOKEN 1]\nType = 3\nFlags = 0\nValue = ' '\nToken = ' '\n\n"
. "[TOKEN 2]\nType = 6\nFlags = 0\nValue = 1\nToken = '1'\n\n"
. "[TOKEN 3]\nType = 9\nFlags = 0\nValue = NULL\nToken = NULL\n\n";
. "[TOKEN 3]\nType = 9\nFlags = 0\nValue = ''\nToken = ''\n\n";

return [
[
Expand Down
16 changes: 8 additions & 8 deletions tests/data/bugs/fuzz1.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
"query": "ALTER..2",
"lexer": {
"@type": "PhpMyAdmin\\SqlParser\\Lexer",
"strict": false,
"errors": [],
"str": "ALTER..2",
"len": 8,
"last": 8,
Expand Down Expand Up @@ -51,8 +53,8 @@
},
{
"@type": "PhpMyAdmin\\SqlParser\\Token",
"token": null,
"value": null,
"token": "",
"value": "",
"keyword": null,
"type": {
"@type": "PhpMyAdmin\\SqlParser\\TokenType",
Expand All @@ -65,12 +67,12 @@
]
},
"delimiter": ";",
"delimiterLen": 1,
"strict": false,
"errors": []
"delimiterLen": 1
},
"parser": {
"@type": "PhpMyAdmin\\SqlParser\\Parser",
"strict": false,
"errors": [],
"list": {
"@type": "@1"
},
Expand All @@ -84,9 +86,7 @@
"last": 0
}
],
"brackets": 0,
"strict": false,
"errors": []
"brackets": 0
},
"errors": {
"lexer": [],
Expand Down
16 changes: 8 additions & 8 deletions tests/data/bugs/fuzz2.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
"query": "WITH](",
"lexer": {
"@type": "PhpMyAdmin\\SqlParser\\Lexer",
"strict": false,
"errors": [],
"str": "WITH](",
"len": 6,
"last": 6,
Expand Down Expand Up @@ -51,8 +53,8 @@
},
{
"@type": "PhpMyAdmin\\SqlParser\\Token",
"token": null,
"value": null,
"token": "",
"value": "",
"keyword": null,
"type": {
"@type": "PhpMyAdmin\\SqlParser\\TokenType",
Expand All @@ -65,12 +67,12 @@
]
},
"delimiter": ";",
"delimiterLen": 1,
"strict": false,
"errors": []
"delimiterLen": 1
},
"parser": {
"@type": "PhpMyAdmin\\SqlParser\\Parser",
"strict": false,
"errors": [],
"list": {
"@type": "@1"
},
Expand All @@ -87,9 +89,7 @@
"last": 0
}
],
"brackets": 1,
"strict": false,
"errors": []
"brackets": 1
},
"errors": {
"lexer": [
Expand Down
16 changes: 8 additions & 8 deletions tests/data/bugs/fuzz3.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
"query": "WITH*/A(",
"lexer": {
"@type": "PhpMyAdmin\\SqlParser\\Lexer",
"strict": false,
"errors": [],
"str": "WITH*/A(",
"len": 8,
"last": 8,
Expand Down Expand Up @@ -64,8 +66,8 @@
},
{
"@type": "PhpMyAdmin\\SqlParser\\Token",
"token": null,
"value": null,
"token": "",
"value": "",
"keyword": null,
"type": {
"@type": "PhpMyAdmin\\SqlParser\\TokenType",
Expand All @@ -78,12 +80,12 @@
]
},
"delimiter": ";",
"delimiterLen": 1,
"strict": false,
"errors": []
"delimiterLen": 1
},
"parser": {
"@type": "PhpMyAdmin\\SqlParser\\Parser",
"strict": false,
"errors": [],
"list": {
"@type": "@1"
},
Expand All @@ -107,9 +109,7 @@
"last": 3
}
],
"brackets": 0,
"strict": false,
"errors": []
"brackets": 0
},
"errors": {
"lexer": [],
Expand Down
16 changes: 8 additions & 8 deletions tests/data/bugs/fuzz4.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
"query": "ALTeR=SET",
"lexer": {
"@type": "PhpMyAdmin\\SqlParser\\Lexer",
"strict": false,
"errors": [],
"str": "ALTeR=SET",
"len": 9,
"last": 9,
Expand Down Expand Up @@ -49,8 +51,8 @@
},
{
"@type": "PhpMyAdmin\\SqlParser\\Token",
"token": null,
"value": null,
"token": "",
"value": "",
"keyword": null,
"type": {
"@type": "PhpMyAdmin\\SqlParser\\TokenType",
Expand All @@ -63,12 +65,12 @@
]
},
"delimiter": ";",
"delimiterLen": 1,
"strict": false,
"errors": []
"delimiterLen": 1
},
"parser": {
"@type": "PhpMyAdmin\\SqlParser\\Parser",
"strict": false,
"errors": [],
"list": {
"@type": "@1"
},
Expand All @@ -93,9 +95,7 @@
"last": 2
}
],
"brackets": 0,
"strict": false,
"errors": []
"brackets": 0
},
"errors": {
"lexer": [],
Expand Down
16 changes: 8 additions & 8 deletions tests/data/bugs/fuzz5.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
"query": "+0xO",
"lexer": {
"@type": "PhpMyAdmin\\SqlParser\\Lexer",
"strict": false,
"errors": [],
"str": "+0xO",
"len": 4,
"last": 4,
Expand Down Expand Up @@ -38,8 +40,8 @@
},
{
"@type": "PhpMyAdmin\\SqlParser\\Token",
"token": null,
"value": null,
"token": "",
"value": "",
"keyword": null,
"type": {
"@type": "PhpMyAdmin\\SqlParser\\TokenType",
Expand All @@ -52,19 +54,17 @@
]
},
"delimiter": ";",
"delimiterLen": 1,
"strict": false,
"errors": []
"delimiterLen": 1
},
"parser": {
"@type": "PhpMyAdmin\\SqlParser\\Parser",
"strict": false,
"errors": [],
"list": {
"@type": "@1"
},
"statements": [],
"brackets": 0,
"strict": false,
"errors": []
"brackets": 0
},
"errors": {
"lexer": [],
Expand Down
16 changes: 8 additions & 8 deletions tests/data/bugs/fuzz6.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
"query": "-+0x!",
"lexer": {
"@type": "PhpMyAdmin\\SqlParser\\Lexer",
"strict": false,
"errors": [],
"str": "-+0x!",
"len": 5,
"last": 5,
Expand Down Expand Up @@ -38,8 +40,8 @@
},
{
"@type": "PhpMyAdmin\\SqlParser\\Token",
"token": null,
"value": null,
"token": "",
"value": "",
"keyword": null,
"type": {
"@type": "PhpMyAdmin\\SqlParser\\TokenType",
Expand All @@ -52,19 +54,17 @@
]
},
"delimiter": ";",
"delimiterLen": 1,
"strict": false,
"errors": []
"delimiterLen": 1
},
"parser": {
"@type": "PhpMyAdmin\\SqlParser\\Parser",
"strict": false,
"errors": [],
"list": {
"@type": "@1"
},
"statements": [],
"brackets": 0,
"strict": false,
"errors": []
"brackets": 0
},
"errors": {
"lexer": [],
Expand Down
Loading