(lexer) fix: error displays for trigraphs + ??' ignored (quote is not a symbol)

Author: t-webber

src/lexer/lex_content.rs

@@ -66,6 +66,11 @@ fn lex_char(
 
         /* Static strings and chars */
         // open/close
+        ('\'', Symbols(symbol_state), _) if symbol_state.is_trigraph() => {
+            if let Some((size, symbol)) = symbol_state.push(ch, lex_data, location) {
+                lex_data.push_token(Token::from_symbol(symbol, size, location));
+            }
+        }
         ('\'', state @ Char(_), _) => end_current(state, lex_data, location),
         ('\'', state, _) if !matches!(state, Str(_)) => {
             end_current(state, lex_data, location);
@@ -165,12 +170,12 @@ fn lex_line(
 ) {
     lex_data.newline();
     let mut escape_state = EscapeState::False;
-    let trimed = line.trim_end();
-    if trimed.is_empty() {
+    let trimmed = line.trim_end();
+    if trimmed.is_empty() {
         return;
     }
-    let last = trimed.len() - 1;
-    for (idx, ch) in trimed.chars().enumerate() {
+    let last = trimmed.len() - 1;
+    for (idx, ch) in trimmed.chars().enumerate() {
         lex_char(
             ch,
             location,

src/lexer/numbers/from_literal.rs

@@ -8,16 +8,16 @@ use super::types::arch_types::*;
 use super::types::{Base, ERR_PREFIX, Number, NumberType};
 use crate::errors::api::{CompileError, Location};
 
-/// Finds a invalid character with the base found with the prefix of the
+/// Finds an invalid character with the base found with the prefix of the
 /// constant.
 ///
 /// # Examples
 ///
 /// ```ignore
-/// assert!(check_with_base("1032", &Base::Binary) == Some('3'));
-/// assert!(check_with_base("1032", &Base::Octal) == None);
+/// assert!(get_first_invalid_char("1032", &Base::Binary) == Some('3'));
+/// assert!(get_first_invalid_char("1032", &Base::Octal) == None);
 /// ```
-fn check_with_base(literal: &str, base: &Base) -> Option<char> {
+fn get_first_invalid_char(literal: &str, base: &Base) -> Option<char> {
     let mut chars = literal.chars();
     match base {
         Base::Binary => chars.find(|ch| !matches!(ch, '0' | '1')),
@@ -213,7 +213,7 @@ fn literal_to_number_err(literal: &str, location: Location, signed: bool) -> Par
         )));
     }
 
-    if let Some(ch) = check_with_base(value, &base) {
+    if let Some(ch) = get_first_invalid_char(value, &base) {
         return ParseRes::Err(location.into_error(format!(
             "{ERR_PREFIX}found invalid character '{ch}' in {} base.",
             base.repr(),

src/lexer/state/end_state.rs

@@ -51,10 +51,11 @@ pub fn end_symbols(symbols: &mut SymbolState, lex_data: &mut LexingData, locatio
             let token = Token::from_symbol(symbol, size, location);
             lex_data.push_token(token);
         } else {
-            panic!(
-                "This can't happen, as lex_data is not empty! LexingData: {:?}",
-                &lex_data
-            );
+            /* This happens when the 3 characters formed a trigraph. If this
+             * is the case, they were ignored. */
+            //TODO: the characters are meant to be printed as they are
+            //TODO: it is only for a case not yet implemented: trigraphs inside
+            // string literals.
         }
     }
 }

src/lexer/state/symbol.rs

@@ -24,7 +24,7 @@ impl SymbolState {
         }
     }
 
-    fn handle_digraphs_trigraphs(&mut self) -> Option<(String, bool)> {
+    fn handle_digraphs_trigraphs(&mut self) -> Option<(String, usize, bool)> {
         let symbols = (self.first, self.second, self.third);
         let (graph, is_trigraph) = match symbols {
             ('?', '?', '=') => (Some('#'), true),
@@ -43,20 +43,22 @@ impl SymbolState {
             ('%', ':', _) => {
                 return Some((
                     "Found invalid character '#', found by replacing digraph '%:'.".to_owned(),
+                    2,
                     true,
                 ));
             }
             _ => (None, false),
         };
         if let Some(symbol) = graph {
             if is_trigraph {
-                return Some((
-                    format!(
-                        "Trigraphs are deprecated in C23. Please remove them: Replace \"{}{}{}\" by '{symbol}'.",
-                        self.first, self.second, self.third
-                    ),
-                    false,
-                ));
+                let msg = format!(
+                    "Trigraphs are deprecated in C23. Please remove them: replace '{}{}{}' by '{symbol}'.",
+                    self.first, self.second, self.third
+                );
+                self.first = NULL;
+                self.second = NULL;
+                self.third = NULL;
+                return Some((msg, 3, false));
             }
             self.first = symbol;
             self.second = self.third;
@@ -69,6 +71,13 @@ impl SymbolState {
         self.first == NULL && self.second == NULL && self.third == NULL
     }
 
+    pub const fn is_trigraph(&self) -> bool {
+        matches!(
+            (self.first, self.second, self.third),
+            ('?', '?', NULL) | (_, '?', '?')
+        )
+    }
+
     pub const fn last(&self) -> Option<char> {
         if self.third == NULL {
             if self.second == NULL {
@@ -123,11 +132,12 @@ impl SymbolState {
         lex_data: &mut LexingData,
         location: &Location,
     ) -> Option<(usize, Symbol)> {
-        if let Some((msg, error)) = self.handle_digraphs_trigraphs() {
+        if let Some((msg, len, error)) = self.handle_digraphs_trigraphs() {
+            let new_location = location.to_owned().into_past_with_length(len);
             if error {
-                lex_data.push_err(location.to_error(msg));
+                lex_data.push_err(new_location.to_error(msg));
             } else {
-                lex_data.push_err(location.to_warning(msg));
+                lex_data.push_err(new_location.to_warning(msg));
             }
         }
         let result = match (self.first, self.second, self.third) {

tests/strings.rs

@@ -1,3 +1,5 @@
+use std::fs;
+
 use c_parser::*;
 
 const SEP: &str = "\n--------------------\n";
@@ -25,6 +27,8 @@ fn test_string_error(content: &str, output: &str) {
     } else {
         res.get_displayed_errors(files, "lexer")
     };
+    fs::write("displayed.txt", &displayed).unwrap();
+    fs::write("expected.txt", output).unwrap();
     assert!(
         output == displayed,
         "Mismatch! Expected:\n!{output}!\n!= Computed\n!{displayed}!"
@@ -49,9 +53,8 @@ make_string_tests!(
 
 digraphs:
     "
-    int arr<:3:> = {1, 2, 3}; // Equivalent to int arr[3];
+    int arr<:3:> = <%1, 2, 3%>; // Equivalent to int arr[3];
     arr<:1:> = 42;            // Equivalent to arr[1] = 42;
-    // int map<%2%>;          // Equivalent to int map{2}; //TODO
     "
     =>
     "[(((int arr)[3]) = {1, 2, 3}), ((arr[1]) = 42), \u{2205} ..]"
@@ -114,6 +117,10 @@ nested_braces:
     =>
     "[[\u{2205} , \u{2205} , \u{2205} , \u{2205} , [(a = 1), (b = 2), \u{2205} ], (c = 3), \u{2205} ]..]"
 
+char_array:
+    "char x[4] = {'b', 12+'5', '3', '\0' };"
+    =>
+    "[(((char x)[4]) = {'b', (12 + '5'), '3', '\0'}), \u{2205} ..]"
 
 nested_block_functions:
         "f(a+b) { g(!x) {     a = 1;     b = 2; } c = 3;
@@ -184,29 +191,60 @@ macro_rules! make_string_error_tests {
 
 make_string_error_tests!(
 
-lengths:
-    "x = \"blob\" bob;"
+lengths_literal:
+"x = 'c' blob;"
+=>
+":1:9: parser error: Found 2 consecutive literals: block [(x = 'c')..] followed by blob.
+    1 | x = 'c' blob;
+                ^~~~
+"
+
+lengths_symbols:
+    "<<="
     =>
-":1:12: parser error: Found 2 consecutive literals: block [(x = \"blob\")..] followed by bob.
-    1 | x = \"blob\" bob;
-                   ^~~
+":1:1: parser error: Tried to call binary operator <<= on without a left argument.
+    1 | <<=
+        ^~~
 "
 
 digraphs:
     "%:include <stdio.h>"
     =>
-":1:3: lexer error: Found invalid character '#', found by replacing digraph '%:'.
+":1:1: lexer error: Found invalid character '#', found by replacing digraph '%:'.
     1 | %:include <stdio.h>
-          ^
+        ^~
 "
 
-// trigraphs:
-// "
-// int ??= 1;
-// int a ??( 10 ??);
-// char b = '??/';
-// char q ??'!'??';
-// int c ??- 5;
-// " => ""
+trigraphs:
+    "
+char b??(5??) = ??< 'b', 'l', 'o',??/
+                    'b', '\0' ??>;
+int x = 1 ??' ??- 2 ??! 3; 
+" =>
+":2:7: lexer warning: Trigraphs are deprecated in C23. Please remove them: replace '??(' by '['.
+    2 | char b??(5??) = ??< 'b', 'l', 'o',??/
+              ^~~
+:2:11: lexer warning: Trigraphs are deprecated in C23. Please remove them: replace '??)' by ']'.
+    2 | char b??(5??) = ??< 'b', 'l', 'o',??/
+                  ^~~
+:2:17: lexer warning: Trigraphs are deprecated in C23. Please remove them: replace '??<' by '{'.
+    2 | char b??(5??) = ??< 'b', 'l', 'o',??/
+                        ^~~
+:2:35: lexer warning: Trigraphs are deprecated in C23. Please remove them: replace '??/' by '\\'.
+    2 | char b??(5??) = ??< 'b', 'l', 'o',??/
+                                          ^~~
+:3:30: lexer warning: Trigraphs are deprecated in C23. Please remove them: replace '??>' by '}'.
+    3 |                     'b', '\0' ??>;
+                                     ^~~
+:4:11: lexer warning: Trigraphs are deprecated in C23. Please remove them: replace '??'' by '^'.
+    4 | int x = 1 ??' ??- 2 ??! 3; 
+                  ^~~
+:4:15: lexer warning: Trigraphs are deprecated in C23. Please remove them: replace '??-' by '~'.
+    4 | int x = 1 ??' ??- 2 ??! 3; 
+                      ^~~
+:4:21: lexer warning: Trigraphs are deprecated in C23. Please remove them: replace '??!' by '|'.
+    4 | int x = 1 ??' ??- 2 ??! 3; 
+                            ^~~
+"
 
 );