feature symfony#54978 [ExpressionLanguage] Add comment support to expression language (valtzu)

This PR was squashed before being merged into the 7.2 branch.

Discussion
----------

[ExpressionLanguage] Add comment support to expression language

| Q             | A
| ------------- | ---
| Branch?       | 7.2
| Bug fix?      | no
| New feature?  | yes
| Deprecations? | no
| License       | MIT

When expression language expressions are provided by the user, it would be really helpful if we could include comments in the expression. Many times the user will write something like

```
customer.group == 'good_customer' or customer.id == 123
```

which in my opinion, would be much better with a comment

```
customer.group == 'good_customer'
or customer.id == 123 /* 123 is the test customer */
```

so that the next person reading the expression knows what the magical 123 is.

---

The implementation is quite simple, given it's ok to strip the comments from AST & the compiled version.

### Why `/*` & `*/`

This seems to be the most common way among different programming languages, including JS, C#, Go, C, PHP, Java, Rust – also in business scenarios you often times want multi-line comments, for which this syntax usually stands for. Also, this makes it possible to use comments in the middle of single-line expressions.

---

At the moment, unlike multi-line comments in PHP, we always require closing `*/`.

Commits
-------

0f8dee7 [ExpressionLanguage] Add comment support to expression language

Author: fabpot

src/Symfony/Component/ExpressionLanguage/CHANGELOG.md

@@ -5,6 +5,7 @@ CHANGELOG
 ---
 
  * Add support for null-coalescing unknown variables
+ * Add support for comments using `/*` & `*/`
 
 7.1
 ---

src/Symfony/Component/ExpressionLanguage/Lexer.php

@@ -69,6 +69,9 @@ public function tokenize(string $expression): TokenStream
                 // strings
                 $tokens[] = new Token(Token::STRING_TYPE, stripcslashes(substr($match[0], 1, -1)), $cursor + 1);
                 $cursor += \strlen($match[0]);
+            } elseif (preg_match('{/\*.*?\*/}A', $expression, $match, 0, $cursor)) {
+                // comments
+                $cursor += \strlen($match[0]);
             } elseif (preg_match('/(?<=^|[\s(])starts with(?=[\s(])|(?<=^|[\s(])ends with(?=[\s(])|(?<=^|[\s(])contains(?=[\s(])|(?<=^|[\s(])matches(?=[\s(])|(?<=^|[\s(])not in(?=[\s(])|(?<=^|[\s(])not(?=[\s(])|(?<=^|[\s(])and(?=[\s(])|\=\=\=|\!\=\=|(?<=^|[\s(])or(?=[\s(])|\|\||&&|\=\=|\!\=|\>\=|\<\=|(?<=^|[\s(])in(?=[\s(])|\.\.|\*\*|\!|\||\^|&|\<|\>|\+|\-|~|\*|\/|%/A', $expression, $match, 0, $cursor)) {
                 // operators
                 $tokens[] = new Token(Token::OPERATOR_TYPE, $match[0], $cursor + 1);

src/Symfony/Component/ExpressionLanguage/Tests/ExpressionLanguageTest.php

@@ -459,6 +459,43 @@ public function testRegisterAfterCompile($registerCallback)
         $registerCallback($el);
     }
 
+    public static function validCommentProvider()
+    {
+        yield ['1 /* comment */ + 1'];
+        yield ['1 /* /* comment with spaces */'];
+        yield ['1 /** extra stars **/ + 1'];
+        yield ["/* multi\nline */ 'foo'"];
+    }
+
+    /**
+     * @dataProvider validCommentProvider
+     */
+    public function testLintAllowsComments($expression)
+    {
+        $el = new ExpressionLanguage();
+        $el->lint($expression, []);
+
+        $this->expectNotToPerformAssertions();
+    }
+
+    public static function invalidCommentProvider()
+    {
+        yield ['1 + no start */'];
+        yield ['1 /* no closing'];
+        yield ['1 /* double closing */ */'];
+    }
+
+    /**
+     * @dataProvider invalidCommentProvider
+     */
+    public function testLintThrowsOnInvalidComments($expression)
+    {
+        $el = new ExpressionLanguage();
+
+        $this->expectException(SyntaxError::class);
+        $el->lint($expression, []);
+    }
+
     public function testLintDoesntThrowOnValidExpression()
     {
         $el = new ExpressionLanguage();
@@ -477,6 +514,13 @@ public function testLintThrowsOnInvalidExpression()
         $el->lint('node.', ['node']);
     }
 
+    public function testCommentsIgnored()
+    {
+        $expressionLanguage = new ExpressionLanguage();
+        $this->assertSame(3, $expressionLanguage->evaluate('1 /* foo */ + 2'));
+        $this->assertSame('(1 + 2)', $expressionLanguage->compile('1 /* foo */ + 2'));
+    }
+
     public static function getRegisterCallbacks()
     {
         return [

src/Symfony/Component/ExpressionLanguage/Tests/LexerTest.php

@@ -151,6 +151,34 @@ public static function getTokenizeData()
                 ],
                 '-.7_189e+10',
             ],
+            [
+                [
+                    new Token('number', 65536, 1),
+                ],
+                '65536 /* this is 2^16 */',
+            ],
+            [
+                [
+                    new Token('number', 2, 1),
+                    new Token('operator', '*', 21),
+                    new Token('number', 4, 23),
+                ],
+                '2 /* /* comment1 */ * 4',
+            ],
+            [
+                [
+                    new Token('string', '/* this is', 1),
+                    new Token('operator', '~', 14),
+                    new Token('string', 'not a comment */', 16),
+                ],
+                '"/* this is" ~ "not a comment */"',
+            ],
+            [
+                [
+                    new Token('string', '/* this is not a comment */', 1),
+                ],
+                '"/* this is not a comment */"',
+            ],
         ];
     }
 }