Always include whitespace in string literals (#688)

* Always include whitespace in string literals

Signed-off-by: Shane Loretz <[email protected]>

* Add tests for #676

Signed-off-by: Shane Loretz <[email protected]>

Author: sloretz

rosidl_parser/rosidl_parser/grammar.lark

@@ -17,6 +17,14 @@
 %import common.WS
 %ignore WS
 
+// Copied from lark-parser instead of imported so wide string doesn't match: `L   "white space between L and quote"`
+// https://github.com/lark-parser/lark/blob/953171821ed307f700fddf27f3fcb9483346bd46/lark/grammars/common.lark#L26-L29
+_STRING_INNER: /.*?/
+_STRING_ESC_INNER: _STRING_INNER /(?<!\\)(\\\\)*?/
+
+ESCAPED_STRING : "\"" _STRING_ESC_INNER "\""
+ESCAPED_WIDE_STRING: "L\"" _STRING_ESC_INNER "\""
+
 
 // 7.2.2 Comments
 COMMENT: "//" /[^\n]/*
@@ -71,11 +79,8 @@ _ESCAPE_SEQUENCES: "\\n" | "\\t" | "\\v" | "\\b" | "\\r" | "\\f" | "\\a" | "\\\\
 // adjacent string literals are concatenated
 string_literals: string_literal+
 wide_string_literals: wide_string_literal+
-// string_literal: "\"" CHAR* "\""
-// wide_string_literal:  "L\"" CHAR* "\""
-// replace precise rules based on the spec with regex for parsing performance
-string_literal: "\"\"" | "\"" /(\\\"|[^"])+/ "\""
-wide_string_literal: "L\"\"" | "L\"" /(\\\"|[^"])+/ "\""
+string_literal: ESCAPED_STRING
+wide_string_literal: ESCAPED_WIDE_STRING
 
 // 7.2.6.4 Floating-point Literals
 floating_pt_literal: FLOAT

rosidl_parser/rosidl_parser/parser.py

@@ -628,6 +628,16 @@ def get_string_literal_value(string_literal, *, allow_unicode=False):
     assert isinstance(child, Token)
     value = child.value
 
+    assert child.type in ('ESCAPED_STRING', 'ESCAPED_WIDE_STRING')
+    if 'ESCAPED_WIDE_STRING' == child.type:
+        assert len(value) >= 3
+        # Get rid of leading L" and trailing "
+        value = value[2:-1]
+    else:
+        assert len(value) >= 2
+        # Get rid of leading " and trailing "
+        value = value[1:-1]
+
     regex = _get_escape_sequences_regex(allow_unicode=allow_unicode)
     value = regex.sub(_decode_escape_sequence, value)
     # unescape double quote and backslash if preceeded by a backslash

rosidl_parser/test/test_parser.py

@@ -30,6 +30,8 @@
 from rosidl_parser.definition import UnboundedSequence
 from rosidl_parser.definition import UnboundedString
 from rosidl_parser.definition import UnboundedWString
+from rosidl_parser.parser import get_ast_from_idl_string
+from rosidl_parser.parser import get_string_literals_value
 from rosidl_parser.parser import parse_idl_file
 
 MESSAGE_IDL_LOCATOR = IdlLocator(
@@ -45,6 +47,38 @@ def message_idl_file():
     return parse_idl_file(MESSAGE_IDL_LOCATOR)
 
 
+def test_whitespace_at_start_of_string():
+    # Repeat to check ros2/rosidl#676
+    for _ in range(10):
+        ast = get_ast_from_idl_string('const string foo = " e";')
+        token = next(ast.find_pred(lambda t: 'string_literals' == t.data))
+        assert ' e' == get_string_literals_value(token)
+
+
+def test_whitespace_at_start_of_wide_string():
+    # Repeat to check ros2/rosidl#676
+    for _ in range(10):
+        ast = get_ast_from_idl_string('const wstring foo = L" e";')
+        token = next(ast.find_pred(lambda t: 'wide_string_literals' == t.data))
+        assert ' e' == get_string_literals_value(token, allow_unicode=True)
+
+
+def test_whitespace_at_end_of_string():
+    # Repeat to check ros2/rosidl#676
+    for _ in range(10):
+        ast = get_ast_from_idl_string('const string foo = "e ";')
+        token = next(ast.find_pred(lambda t: 'string_literals' == t.data))
+        assert 'e ' == get_string_literals_value(token)
+
+
+def test_whitespace_at_end_of_wide_string():
+    # Repeat to check ros2/rosidl#676
+    for _ in range(10):
+        ast = get_ast_from_idl_string('const wstring foo = L"e ";')
+        token = next(ast.find_pred(lambda t: 'wide_string_literals' == t.data))
+        assert 'e ' == get_string_literals_value(token, allow_unicode=True)
+
+
 def test_message_parser(message_idl_file):
     messages = message_idl_file.content.get_elements_of_type(Message)
     assert len(messages) == 1