Skip to content

Commit 70c6836

Browse files
nilportuguesnilportugues
committed
Tokenizer complexity reduced
1 parent 426a2cb commit 70c6836

File tree

9 files changed

+166
-78
lines changed

9 files changed

+166
-78
lines changed

‎src/Tokenizer/Parser/Boundary.php‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,28 @@
1818
*/
1919
final class Boundary
2020
{
21+
/**
22+
* @param Tokenizer $tokenizer
23+
* @param $string
24+
* @param array $matches
25+
*/
26+
public static function isBoundary(Tokenizer $tokenizer, $string, array &$matches)
27+
{
28+
if (!$tokenizer->getNextToken() &&
29+
self::isBoundaryCharacter($string, $matches, $tokenizer->getRegexBoundaries())
30+
) {
31+
$tokenizer->setNextToken(self::getBoundaryCharacter($matches));
32+
}
33+
}
34+
2135
/**
2236
* @param string $string
2337
* @param array $matches
2438
* @param string $regexBoundaries
2539
*
2640
* @return bool
2741
*/
28-
public static function isBoundaryCharacter($string, array &$matches, $regexBoundaries)
42+
protected static function isBoundaryCharacter($string, array &$matches, $regexBoundaries)
2943
{
3044
return (1 == preg_match('/^(' . $regexBoundaries . ')/', $string, $matches));
3145
}
@@ -35,7 +49,7 @@ public static function isBoundaryCharacter($string, array &$matches, $regexBound
3549
*
3650
* @return array
3751
*/
38-
public static function getBoundaryCharacter(array &$matches)
52+
protected static function getBoundaryCharacter(array &$matches)
3953
{
4054
return [
4155
Tokenizer::TOKEN_VALUE => $matches[1],

‎src/Tokenizer/Parser/Comment.php‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,23 @@
1818
*/
1919
final class Comment
2020
{
21+
/**
22+
* @param Tokenizer $tokenizer
23+
* @param $string
24+
*/
25+
public static function isComment(Tokenizer $tokenizer, $string)
26+
{
27+
if (!$tokenizer->getNextToken() && Comment::isCommentString($string)) {
28+
$tokenizer->setNextToken(Comment::getCommentString($string));
29+
}
30+
}
31+
2132
/**
2233
* @param string $string
2334
*
2435
* @return bool
2536
*/
26-
public static function isCommentString($string)
37+
protected static function isCommentString($string)
2738
{
2839
return $string[0] === '#' || self::isTwoCharacterComment($string);
2940
}
@@ -63,7 +74,7 @@ protected static function startsAsBlock($string)
6374
*
6475
* @return array
6576
*/
66-
public static function getCommentString($string)
77+
protected static function getCommentString($string)
6778
{
6879
$last = strpos($string, "*/", 2) + 2;
6980
$type = Tokenizer::TOKEN_TYPE_BLOCK_COMMENT;

‎src/Tokenizer/Parser/Numeral.php‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,28 @@
1818
*/
1919
final class Numeral
2020
{
21+
/**
22+
* @param Tokenizer $tokenizer
23+
* @param $string
24+
* @param array $matches
25+
*
26+
* @return array
27+
*/
28+
public static function isNumeral(Tokenizer $tokenizer, $string, array &$matches)
29+
{
30+
if (!$tokenizer->getNextToken() && self::isNumeralString($string, $matches, $tokenizer->getRegexBoundaries())) {
31+
$tokenizer->setNextToken(self::getNumeralString($matches));
32+
}
33+
}
34+
2135
/**
2236
* @param string $string
2337
* @param array $matches
2438
* @param string $regexBoundaries
2539
*
2640
* @return bool
2741
*/
28-
public static function isNumeralString($string, array &$matches, $regexBoundaries)
42+
protected static function isNumeralString($string, array &$matches, $regexBoundaries)
2943
{
3044
return (1 == preg_match(
3145
'/^([0-9]+(\.[0-9]+)?|0x[0-9a-fA-F]+|0b[01]+)($|\s|"\'`|' . $regexBoundaries . ')/',
@@ -39,7 +53,7 @@ public static function isNumeralString($string, array &$matches, $regexBoundarie
3953
*
4054
* @return array
4155
*/
42-
public static function getNumeralString(array &$matches)
56+
protected static function getNumeralString(array &$matches)
4357
{
4458
return [Tokenizer::TOKEN_VALUE => $matches[1], Tokenizer::TOKEN_TYPE => Tokenizer::TOKEN_TYPE_NUMBER];
4559
}

‎src/Tokenizer/Parser/Quoted.php‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,23 @@
1818
*/
1919
final class Quoted
2020
{
21+
/**
22+
* @param Tokenizer $tokenizer
23+
* @param $string
24+
*/
25+
public static function isQuoted(Tokenizer $tokenizer, $string)
26+
{
27+
if (!$tokenizer->getNextToken() && self::isQuotedString($string)) {
28+
$tokenizer->setNextToken(self::getQuotedString($string));
29+
}
30+
}
2131

2232
/**
2333
* @param string $string
2434
*
2535
* @return bool
2636
*/
27-
public static function isQuotedString($string)
37+
protected static function isQuotedString($string)
2838
{
2939
return $string[0] === '"' || $string[0] === '\'' || $string[0] === '`' || $string[0] === '[';
3040
}
@@ -34,7 +44,7 @@ public static function isQuotedString($string)
3444
*
3545
* @return array
3646
*/
37-
public static function getQuotedString($string)
47+
protected static function getQuotedString($string)
3848
{
3949
$tokenType = Tokenizer::TOKEN_TYPE_QUOTE;
4050

‎src/Tokenizer/Parser/Reserved.php‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -19,19 +19,19 @@
1919
final class Reserved
2020
{
2121
/**
22+
* @param Tokenizer $tokenizer
23+
* @param string $string
2224
* @param array $matches
2325
* @param array|null $previous
24-
* @param string $string
25-
* @param Tokenizer $tokenizer
2626
*
2727
* @return array
2828
*/
29-
public static function isReserved(array &$matches, $previous, $string, Tokenizer $tokenizer)
29+
public static function isReserved(Tokenizer $tokenizer, $string, array &$matches, $previous)
3030
{
3131
$tokenData = [];
3232

33-
if (Reserved::isReservedPrecededByDotCharacter($previous)) {
34-
Reserved::getReservedString(
33+
if (!$tokenizer->getNextToken() && self::isReservedPrecededByDotCharacter($previous)) {
34+
self::getReservedString(
3535
$tokenData,
3636
Tokenizer::TOKEN_TYPE_RESERVED_TOP_LEVEL,
3737
$string,
@@ -40,7 +40,7 @@ public static function isReserved(array &$matches, $previous, $string, Tokenizer
4040
$tokenizer->getRegexBoundaries()
4141
);
4242

43-
Reserved::getReservedString(
43+
self::getReservedString(
4444
$tokenData,
4545
Tokenizer::TOKEN_TYPE_RESERVED_NEWLINE,
4646
strtoupper($string),
@@ -49,17 +49,17 @@ public static function isReserved(array &$matches, $previous, $string, Tokenizer
4949
$tokenizer->getRegexBoundaries()
5050
);
5151

52-
Reserved::getReservedString(
52+
self::getReservedString(
5353
$tokenData,
5454
Tokenizer::TOKEN_TYPE_RESERVED,
5555
$string,
5656
$matches,
5757
$tokenizer->getRegexReserved(),
5858
$tokenizer->getRegexBoundaries()
5959
);
60-
}
6160

62-
return $tokenData;
61+
$tokenizer->setNextToken($tokenData);
62+
}
6363
}
6464

6565
/**
@@ -69,7 +69,7 @@ public static function isReserved(array &$matches, $previous, $string, Tokenizer
6969
*
7070
* @return bool
7171
*/
72-
public static function isReservedPrecededByDotCharacter($previous)
72+
protected static function isReservedPrecededByDotCharacter($previous)
7373
{
7474
return !$previous || !isset($previous[Tokenizer::TOKEN_VALUE]) || $previous[Tokenizer::TOKEN_VALUE] !== '.';
7575
}
@@ -83,7 +83,7 @@ public static function isReservedPrecededByDotCharacter($previous)
8383
*
8484
* @return bool
8585
*/
86-
public static function isReservedString($upper, array &$matches, $regexReserved, $regexBoundaries)
86+
protected static function isReservedString($upper, array &$matches, $regexReserved, $regexBoundaries)
8787
{
8888
return 1 == preg_match(
8989
'/^(' . $regexReserved . ')($|\s|' . $regexBoundaries . ')/',
@@ -117,9 +117,9 @@ protected static function getStringTypeArray($type, $string, array &$matches)
117117
*
118118
* @return array
119119
*/
120-
public static function getReservedString(array &$tokenData, $type, $string, array &$matches, $regex, $boundaries)
120+
protected static function getReservedString(array &$tokenData, $type, $string, array &$matches, $regex, $boundaries)
121121
{
122-
if (empty($tokenData) && Reserved::isReservedString($string, $matches, $regex, $boundaries)) {
122+
if (empty($tokenData) && self::isReservedString($string, $matches, $regex, $boundaries)) {
123123
$tokenData = self::getStringTypeArray($type, $string, $matches);
124124
}
125125
}

‎src/Tokenizer/Parser/String.php‎

Lines changed: 28 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,18 @@
1818
*/
1919
final class String
2020
{
21+
/**
22+
* @param Tokenizer $tokenizer
23+
* @param $string
24+
* @param array $matches
25+
*/
26+
public static function isFunction(Tokenizer $tokenizer, $string, array &$matches)
27+
{
28+
if (!$tokenizer->getNextToken() && self::isFunctionString($string, $matches, $tokenizer->getRegexFunction())) {
29+
$tokenizer->setNextToken(self::getFunctionString($string, $matches));
30+
}
31+
}
32+
2133
/**
2234
* A function must be succeeded by '('.
2335
* This makes it so that a function such as "COUNT(" is considered a function, but "COUNT" alone is not function.
@@ -28,7 +40,7 @@ final class String
2840
*
2941
* @return bool
3042
*/
31-
public static function isFunctionString($string, array &$matches, $regexFunction)
43+
protected static function isFunctionString($string, array &$matches, $regexFunction)
3244
{
3345
return (1 == preg_match('/^(' . $regexFunction . '[(]|\s|[)])/', strtoupper($string), $matches));
3446
}
@@ -39,7 +51,7 @@ public static function isFunctionString($string, array &$matches, $regexFunction
3951
*
4052
* @return array
4153
*/
42-
public static function getFunctionString($string, array &$matches)
54+
protected static function getFunctionString($string, array &$matches)
4355
{
4456
return [
4557
Tokenizer::TOKEN_TYPE => Tokenizer::TOKEN_TYPE_RESERVED,
@@ -48,23 +60,23 @@ public static function getFunctionString($string, array &$matches)
4860
}
4961

5062
/**
51-
* @param string $string
52-
* @param string $regexBoundaries
53-
*
54-
* @return array
63+
* @param Tokenizer $tokenizer
64+
* @param $string
65+
* @param array $matches
5566
*/
56-
public static function getNonReservedString($string, $regexBoundaries)
67+
public static function getNonReservedString(Tokenizer $tokenizer, $string, array &$matches)
5768
{
58-
$data = [];
59-
$matches = [];
69+
if (!$tokenizer->getNextToken()) {
70+
$data = [];
6071

61-
if (1 == preg_match('/^(.*?)($|\s|["\'`]|' . $regexBoundaries . ')/', $string, $matches)) {
62-
$data = [
63-
Tokenizer::TOKEN_VALUE => $matches[1],
64-
Tokenizer::TOKEN_TYPE => Tokenizer::TOKEN_TYPE_WORD
65-
];
66-
}
72+
if (1 == preg_match('/^(.*?)($|\s|["\'`]|' . $tokenizer->getRegexBoundaries() . ')/', $string, $matches)) {
73+
$data = [
74+
Tokenizer::TOKEN_VALUE => $matches[1],
75+
Tokenizer::TOKEN_TYPE => Tokenizer::TOKEN_TYPE_WORD
76+
];
77+
}
6778

68-
return $data;
79+
$tokenizer->setNextToken($data);
80+
}
6981
}
7082
}

‎src/Tokenizer/Parser/UserDefined.php‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,25 @@
1818
*/
1919
final class UserDefined
2020
{
21+
/**
22+
* @param Tokenizer $tokenizer
23+
* @param $string
24+
*
25+
* @return array
26+
*/
27+
public static function isUserDefinedVariable(Tokenizer $tokenizer, $string)
28+
{
29+
if (!$tokenizer->getNextToken() && self::isUserDefinedVariableString($string)) {
30+
$tokenizer->setNextToken(self::getUserDefinedVariableString($string));
31+
}
32+
}
2133

2234
/**
2335
* @param string $string
2436
*
2537
* @return bool
2638
*/
27-
public static function isUserDefinedVariableString(&$string)
39+
protected static function isUserDefinedVariableString(&$string)
2840
{
2941
return $string[0] === '@' && isset($string[1]);
3042
}
@@ -36,7 +48,7 @@ public static function isUserDefinedVariableString(&$string)
3648
*
3749
* @return array
3850
*/
39-
public static function getUserDefinedVariableString(&$string)
51+
protected static function getUserDefinedVariableString(&$string)
4052
{
4153
$returnData = [
4254
Tokenizer::TOKEN_VALUE => null,

‎src/Tokenizer/Parser/WhiteSpace.php‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,18 @@
1818
*/
1919
final class WhiteSpace
2020
{
21+
/**
22+
* @param Tokenizer $tokenizer
23+
* @param $string
24+
* @param array $matches
25+
*/
26+
public static function isWhiteSpace(Tokenizer $tokenizer, $string, array &$matches)
27+
{
28+
if (self::isWhiteSpaceString($string, $matches)) {
29+
$tokenizer->setNextToken(self::getWhiteSpaceString($matches));
30+
}
31+
}
32+
2133
/**
2234
* @param string $string
2335
* @param array $matches

0 commit comments

Comments
 (0)