(lexer) add: multi-line strings

Strings on multiple lines, with '\' escape or simply with successive double quotes

Author: t-webber

src/lexer/lex_content.rs

@@ -189,7 +189,9 @@ fn lex_line(
             break;
         }
     }
-    end_current(lex_status, lex_data, location);
+    if escape_state != EscapeStatus::Single {
+        end_current(lex_status, lex_data, location);
+    }
     if line.trim_end().ends_with('\\') {
         if line.ends_with(char::is_whitespace) {
             lex_data.push_err(location.to_suggestion(

src/lexer/types/lexing_data.rs

@@ -37,7 +37,13 @@ impl LexingData {
     }
 
     pub fn push_token(&mut self, token: Token) {
-        self.tokens.push(token);
+        if let TokenValue::Str(val) = token.get_value()
+            && let Some(TokenValue::Str(old)) = self.tokens.last_mut().map(Token::get_value_mut)
+        {
+            old.push_str(val);
+        } else {
+            self.tokens.push(token);
+        }
     }
 
     pub const fn set_end_line(&mut self) {

src/lexer/types/tokens_types.rs

@@ -137,6 +137,10 @@ impl Token {
         &self.value
     }
 
+    pub(crate) const fn get_value_mut(&mut self) -> &mut TokenValue {
+        &mut self.value
+    }
+
     pub(crate) fn into_value_location(self) -> (TokenValue, Location) {
         (self.value, self.location)
     }

src/parser/tree/mod.rs

@@ -116,8 +116,8 @@ impl fmt::Display for Literal {
         match self {
             Self::Empty => EMPTY.fmt(f),
             Self::Nullptr => "NULL".fmt(f),
-            Self::Char(val) => val.fmt(f),
-            Self::Str(val) => val.fmt(f),
+            Self::Char(val) => write!(f, "'{val}'"),
+            Self::Str(val) => write!(f, "\"{val}\""),
             Self::Number(val) => val.fmt(f),
             Self::ConstantBool(val) => val.fmt(f),
             Self::Variable(val) => val.fmt(f),

tests/strings.rs

@@ -1,14 +1,17 @@
 use c_parser::*;
 
+const SEP: &str = "\n--------------------\n";
+
 fn test_string(content: &str, output: &str) {
     let files = &[(String::new(), content)];
     let mut location = Location::from(String::new());
+    eprintln!("{SEP}Content = {content}{SEP}");
     let tokens = lex_file(content, &mut location).unwrap_or_display(files, "lexer");
-    eprintln!("Tokens = {}", display_tokens(&tokens));
+    eprintln!("{SEP}Tokens = {}{SEP}", display_tokens(&tokens));
     let node = parse_tokens(tokens).unwrap_or_display(files, "parser");
     assert!(
         output == format!("{node}"),
-        "Mismatch! Expected:\n{output}\n!= Computed\n{node}\n"
+        "{SEP}Mismatch! Expected:\n{output}\n!= Computed\n{node}{SEP}"
     );
 }
 
@@ -40,6 +43,14 @@ macro_rules! make_string_tests {
 
 make_string_tests!(
 
+multiline_string:
+    "\"multi\"
+     \"line\\
+     strings\" 
+    "
+    =>
+    "[\"multiline     strings\"..]"
+
 unary_binary:
     "a + b * c - d / e % f + g - h * i + j % k * l ^ !m++ & n | o || p && q"
     =>