(errors+lexer) fix: length of tokens not kept for parser errors

Author: t-webber

src/errors/compile.rs

@@ -24,8 +24,6 @@ use crate::errors::api::Location;
 pub struct CompileError {
     /// Severity of the error
     err_lvl: ErrorLevel,
-    /// Length of the erroneous token or expression
-    length: usize,
     /// Location of the error in the C source file
     location: Location,
     /// Error message to be displayed to the user
@@ -34,46 +32,23 @@ pub struct CompileError {
 
 impl CompileError {
     /// Returns the owned data of a `CompileError`.
-    pub(super) fn into_values(self) -> (Location, String, String, usize) {
-        (
-            self.location,
-            self.message,
-            self.err_lvl.to_string(),
-            self.length,
-        )
+    pub(super) fn into_values(self) -> (Location, String, String) {
+        (self.location, self.message, self.err_lvl.to_string())
     }
 
     /// Checks if the error is of severity [`ErrorLevel::Error`].
     pub(crate) fn is_error(&self) -> bool {
         self.err_lvl == ErrorLevel::Error
     }
-
-    // Replaces length of the token or expression concerned by the `CompileError`.
-    pub(crate) fn specify_length(&mut self, length: usize) {
-        self.length = length;
-    }
-}
-
-impl From<(Location, String, ErrorLevel, usize)> for CompileError {
-    #[inline]
-    fn from((location, message, err_lvl, length): (Location, String, ErrorLevel, usize)) -> Self {
-        Self {
-            err_lvl,
-            length,
-            location,
-            message,
-        }
-    }
 }
 
 impl From<(Location, String, ErrorLevel)> for CompileError {
     #[inline]
     fn from((location, message, err_lvl): (Location, String, ErrorLevel)) -> Self {
         Self {
-            message,
-            length: 0,
-            location,
             err_lvl,
+            location,
+            message,
         }
     }
 }

src/errors/display.rs

@@ -18,8 +18,8 @@ pub(super) fn display_errors(
         files_state.insert(filename.to_owned(), content.lines().collect());
     }
     for error in errors {
-        let (location, message, err_lvl, length) = error.into_values();
-        let (filename, line_nb, column_nb) = location.into_values();
+        let (location, message, err_lvl) = error.into_values();
+        let (filename, line_nb, column_nb, length) = location.into_values();
         let code_lines = files_state
             .get(&filename)
             .expect("Never happens: File of error doesn't exist");
@@ -30,7 +30,7 @@ pub(super) fn display_errors(
             res,
             "{filename}:{line_nb}:{column_nb}: {err_type} {err_lvl}: {message}\n{line_nb:5} | {code_line}\n{}^{}",
             " ".repeat(8 + column_nb - 1),
-            "~".repeat(length)
+            "~".repeat(length - 1)
         )
         .map_err(|_| ())?;
     }

src/errors/location.rs

@@ -1,5 +1,3 @@
-use core::fmt;
-
 use super::compile::{CompileError, ErrorLevel};
 
 /// Struct to pinpoint a precise character in the C source file.
@@ -16,6 +14,7 @@ use super::compile::{CompileError, ErrorLevel};
 pub struct Location {
     col: usize,
     file: String,
+    length: usize,
     line: usize,
 }
 
@@ -44,16 +43,17 @@ impl Location {
     ///
     /// If the offset is too big, the column is set to minimal (1) without any
     /// warnings or errors.
-    pub(crate) fn into_past(self, offset: usize) -> Self {
+    pub(crate) fn into_past_with_length(self, len: usize) -> Self {
         Self {
-            col: self.col.checked_sub(offset).unwrap_or(1),
+            col: self.col.checked_sub(len).unwrap_or(1),
+            length: len,
             ..self
         }
     }
 
     /// Returns the owned data of a `Location`.
-    pub(crate) fn into_values(self) -> (String, usize, usize) {
-        (self.file, self.line, self.col)
+    pub(crate) fn into_values(self) -> (String, usize, usize, usize) {
+        (self.file, self.line, self.col, self.length)
     }
 
     /// Creates an error by cloning the location.
@@ -83,6 +83,7 @@ impl From<&str> for Location {
             file: value.to_owned(),
             line: 1,
             col: 1,
+            length: 1,
         }
     }
 }
@@ -94,14 +95,7 @@ impl From<String> for Location {
             file: value,
             line: 1,
             col: 1,
+            length: 1,
         }
     }
 }
-
-#[expect(clippy::min_ident_chars)]
-impl fmt::Display for Location {
-    #[inline]
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{}:{}:{}", self.file, self.line, self.col)
-    }
-}

src/errors/result.rs

@@ -36,8 +36,7 @@ impl<T> Res<T> {
     /// let content = "int m@in() { }";
     /// let res = lex_file(&content, &mut Location::from("filename.c"));
     /// let errors = res.get_displayed_errors(&[("filename.c".to_owned(), content)], "lexer");
-    /// let expected =
-    ///     "filename.c:1:6: lexer error: Character '@' not supported in context of a 'identifier'.
+    /// let expected = "filename.c:1:6: lexer error: Character '@' not supported.
     ///     1 | int m@in() { }
     ///              ^
     /// ";

src/lexer/lex_content.rs

@@ -130,23 +130,20 @@ fn lex_char(
             // dbg!("there", &val);
         }
         (_, state, _) if ch.is_alphanumeric() || matches!(ch, '_') => {
-            if let Symbols(symb) = state
-                && symb.last() == Some('.')
+            if let Symbols(symbol) = state
+                && symbol.last() == Some('.')
                 && ch.is_ascii_digit()
             {
-                symb.clear_last();
+                symbol.clear_last();
                 end_current(state, lex_data, location);
                 state.new_ident_str(format!("0.{ch}"));
             } else {
                 end_current(state, lex_data, location);
                 state.new_ident(ch);
             }
         }
-        (_, state, _) => {
-            lex_data.push_err(location.to_error(format!(
-                "Character '{ch}' not supported in context of a '{}'.",
-                state.repr(),
-            )));
+        (_, _, _) => {
+            lex_data.push_err(location.to_error(format!("Character '{ch}' not supported.")));
         }
     }
 }

src/lexer/numbers/from_literal.rs

@@ -182,11 +182,9 @@ pub fn literal_to_number(
             .map_or_else(|_| None, |x| Some(Number::Int(x)));
     };
 
-    let begin_location = location.to_owned().into_past(literal.len() - 1);
+    let begin_location = location.to_owned().into_past_with_length(literal.len());
 
-    let mut res = literal_to_number_err(literal.value(), begin_location, lex_data.last_is_minus());
-    res.edit_err(|err| err.specify_length(literal.len() - 1));
-    match res {
+    match literal_to_number_err(literal.value(), begin_location, lex_data.last_is_minus()) {
         ParseRes::Value(val) => Some(val),
         ParseRes::Err(err) => {
             lex_data.push_err(err);

src/lexer/numbers/parse.rs

@@ -124,15 +124,6 @@ pub enum ParseRes<T> {
 }
 
 impl<T> ParseRes<T> {
-    /// Apply a function on the error if it exists.
-    #[expect(clippy::min_ident_chars)]
-    pub fn edit_err<F: Fn(&mut CompileError)>(&mut self, f: F) {
-        match self {
-            Self::Value(_) => (),
-            Self::Err(err) | Self::ValueErr(_, err) => f(err),
-        }
-    }
-
     /// Returns the values of the parse result.
     fn into_elts(self) -> (Option<T>, Option<CompileError>) {
         match self {

src/lexer/types/tokens.rs

@@ -69,7 +69,7 @@ impl Token {
     pub(crate) fn from_char(ch: char, location: &Location) -> Self {
         Self {
             value: TokenValue::Char(ch),
-            location: location.to_owned().into_past(1),
+            location: location.to_owned().into_past_with_length(1),
         }
     }
 
@@ -95,13 +95,13 @@ impl Token {
                         }
                     })
                     .collect::<String>();
-                lex_data.push_err(location.to_owned().into_past(len).to_warning(format!("Underscore operators are deprecated since C23. Consider using the new keyword: {new_keyword}")));
+                lex_data.push_err(location.to_owned().into_past_with_length(len).to_warning(format!("Underscore operators are deprecated since C23. Consider using the new keyword: {new_keyword}")));
                 TokenValue::Keyword(keyword)
             }
             TryKeywordType::Failure => TokenValue::Identifier(value),
         };
         Self {
-            location: location.to_owned().into_past(len),
+            location: location.to_owned().into_past_with_length(len),
             value: token_value,
         }
     }
@@ -115,15 +115,15 @@ impl Token {
 
     pub(crate) fn from_str(str: String, location: &Location) -> Self {
         Self {
-            location: location.to_owned().into_past(str.len()),
+            location: location.to_owned().into_past_with_length(str.len()),
             value: TokenValue::Str(str),
         }
     }
 
     pub(crate) fn from_symbol(symbol: Symbol, size: usize, location: &Location) -> Self {
         Self {
             value: TokenValue::Symbol(symbol),
-            location: location.to_owned().into_past(size),
+            location: location.to_owned().into_past_with_length(size),
         }
     }
 

tests/strings.rs

@@ -180,6 +180,14 @@ macro_rules! make_string_error_tests {
 
 make_string_error_tests!(
 
+lengths:
+    "x = \"blob\" bob;"
+    =>
+":1:12: parser error: Found 2 consecutive literals: block [(x = \"blob\")..] followed by bob.
+    1 | x = \"blob\" bob;
+                   ^~~
+"
+
 // digraphs:
     // "%:include <stdio.h>"
     // =>