Skip to content

Add source snippets to missing_from_lexer/parser errors. #571

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ltratt merged 1 commit into from
May 22, 2025
Merged

Add source snippets to missing_from_lexer/parser errors. #571

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 77 additions & 19 deletions lrlex/src/lib/ctbuilder.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ use cfgrammar::{
Setting, Value,
},
markmap::MergeBehavior,
span::Location,
span::{Location, Span},
};
use glob::glob;
use lazy_static::lazy_static;
Expand Down Expand Up @@ -521,15 +521,15 @@ where
}
};

if let Some(ref lrcfg) = self.lrpar_config {
let mut lexerdef = lexerdef.clone();
let ct_parser = if let Some(ref lrcfg) = self.lrpar_config {
let mut closure_lexerdef = lexerdef.clone();
let mut ctp = CTParserBuilder::<LexerTypesT>::new().inspect_rt(Box::new(
move |yacc_header, rtpb, rule_ids_map, grm_path| {
let owned_map = rule_ids_map
.iter()
.map(|(x, y)| (&**x, *y))
.collect::<HashMap<_, _>>();
lexerdef.set_rule_ids(&owned_map);
closure_lexerdef.set_rule_ids(&owned_map);
yacc_header.mark_used(&"test_files".to_string());
let test_glob = yacc_header.get("test_files");
match test_glob {
Expand All @@ -540,7 +540,8 @@ where
{
let path = path?;
let input = fs::read_to_string(&path)?;
let l: LRNonStreamingLexer<LexerTypesT> = lexerdef.lexer(&input);
let l: LRNonStreamingLexer<LexerTypesT> =
closure_lexerdef.lexer(&input);
for e in rtpb.parse_noaction(&l) {
Err(format!("parsing {}: {}", path.display(), e))?
}
Expand All @@ -553,9 +554,12 @@ where
},
));
ctp = lrcfg(ctp);
let map = ctp.build()?;
self.rule_ids_map = Some(map.token_map().to_owned());
}
let ct_parser = ctp.build()?;
self.rule_ids_map = Some(ct_parser.token_map().to_owned());
Some(ct_parser)
} else {
None
};

let mut lexerdef = Box::new(lexerdef);
let unused_header_values = header.unused();
Expand All @@ -572,30 +576,83 @@ where
.iter()
.map(|(x, y)| (&**x, *y))
.collect::<HashMap<_, _>>();
let (x, y) = lexerdef.set_rule_ids(&owned_map);
let (x, y) = lexerdef.set_rule_ids_spanned(&owned_map);
(
x.map(|a| a.iter().map(|&b| b.to_string()).collect::<HashSet<_>>()),
y.map(|a| a.iter().map(|&b| b.to_string()).collect::<HashSet<_>>()),
y.map(|a| {
a.iter()
.map(|(b, span)| (b.to_string(), *span))
.collect::<HashSet<_>>()
}),
)
}
None => (None, None),
};

let mut has_unallowed_missing = false;
let err_indent = " ".repeat(ERROR.len());
if !self.allow_missing_terms_in_lexer {
if let Some(ref mfl) = missing_from_lexer {
eprintln!("Error: the following tokens are used in the grammar but are not defined in the lexer:");
for n in mfl {
eprintln!(" {}", n);
if let Some(ct_parser) = &ct_parser {
let grm = ct_parser.yacc_grammar();
let token_spans = mfl
.iter()
.map(|name| {
ct_parser
.yacc_grammar()
.token_span(*grm.tokens_map().get(name.as_str()).unwrap())
.expect("Given token should have a span")
})
.collect::<Vec<_>>();

let yacc_diag = SpannedDiagnosticFormatter::new(
ct_parser.grammar_src(),
ct_parser.grammar_path(),
);

eprintln!("{ERROR} these tokens are not referenced in the lexer but defined as follows");
eprintln!(
"{err_indent} {}",
yacc_diag.file_location_msg("in the grammar", None)
);
for span in token_spans {
eprintln!(
"{}",
yacc_diag.underline_span_with_text(
span,
"Missing from lexer".to_string(),
'^'
)
);
}
eprintln!();
} else {
eprintln!("{ERROR} the following tokens are used in the grammar but are not defined in the lexer:");
for n in mfl {
eprintln!(" {}", n);
}
}
has_unallowed_missing = true;
}
}
if !self.allow_missing_tokens_in_parser {
if let Some(ref mfp) = missing_from_parser {
eprintln!("Error: the following tokens are defined in the lexer but not used in the grammar:");
for n in mfp {
eprintln!(" {}", n);
eprintln!(
"{ERROR} these tokens are not referenced in the grammar but defined as follows"
);
eprintln!(
"{err_indent} {}",
lex_diag.file_location_msg("in the lexer", None)
);
for (_, span) in mfp {
eprintln!(
"{}",
lex_diag.underline_span_with_text(
*span,
"Missing from parser".to_string(),
'^'
)
);
}
has_unallowed_missing = true;
}
Expand Down Expand Up @@ -830,7 +887,8 @@ where
let cl = self.build()?;
Ok((
cl.missing_from_lexer().map(|x| x.to_owned()),
cl.missing_from_parser().map(|x| x.to_owned()),
cl.missing_from_parser()
.map(|x| x.iter().map(|(n, _)| n.to_owned()).collect::<HashSet<_>>()),
Copy link
Collaborator Author

@ratmice ratmice May 21, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Perhaps worth noting that this ugly bit is in deprecated code.

))
}

Expand Down Expand Up @@ -1066,15 +1124,15 @@ where
/// An interface to the result of [CTLexerBuilder::build()].
pub struct CTLexer {
missing_from_lexer: Option<HashSet<String>>,
missing_from_parser: Option<HashSet<String>>,
missing_from_parser: Option<HashSet<(String, Span)>>,
}

impl CTLexer {
fn missing_from_lexer(&self) -> Option<&HashSet<String>> {
self.missing_from_lexer.as_ref()
}

fn missing_from_parser(&self) -> Option<&HashSet<String>> {
fn missing_from_parser(&self) -> Option<&HashSet<(String, Span)>> {
self.missing_from_parser.as_ref()
}
}
Expand Down
39 changes: 31 additions & 8 deletions lrpar/src/lib/ctbuilder.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -692,6 +692,9 @@ where
return Ok(CTParser {
regenerated: false,
rule_ids,
yacc_grammar: grm,
grammar_src: inc,
grammar_path: self.grammar_path.unwrap(),
conflicts: None,
});
} else {
Expand Down Expand Up @@ -779,13 +782,16 @@ where
&format!("/* CACHE INFORMATION {} */\n", cache),
)?;
let conflicts = if stable.conflicts().is_some() {
Some((grm, sgraph, stable))
Some((sgraph, stable))
} else {
None
};
Ok(CTParser {
regenerated: true,
rule_ids,
yacc_grammar: grm,
grammar_src: inc,
grammar_path: self.grammar_path.unwrap(),
conflicts,
})
}
Expand Down Expand Up @@ -1489,11 +1495,10 @@ where
{
regenerated: bool,
rule_ids: HashMap<String, StorageT>,
conflicts: Option<(
YaccGrammar<StorageT>,
StateGraph<StorageT>,
StateTable<StorageT>,
)>,
yacc_grammar: YaccGrammar<StorageT>,
grammar_src: String,
grammar_path: PathBuf,
conflicts: Option<(StateGraph<StorageT>, StateTable<StorageT>)>,
}

impl<StorageT> CTParser<StorageT>
Expand Down Expand Up @@ -1527,11 +1532,29 @@ where
&StateTable<StorageT>,
&Conflicts<StorageT>,
)> {
if let Some((grm, sgraph, stable)) = &self.conflicts {
return Some((grm, sgraph, stable, stable.conflicts().unwrap()));
if let Some((sgraph, stable)) = &self.conflicts {
return Some((
&self.yacc_grammar,
sgraph,
stable,
stable.conflicts().unwrap(),
));
}
None
}

#[doc(hidden)]
pub fn yacc_grammar(&self) -> &YaccGrammar<StorageT> {
&self.yacc_grammar
}
#[doc(hidden)]
pub fn grammar_src(&self) -> &str {
&self.grammar_src
}
#[doc(hidden)]
pub fn grammar_path(&self) -> &Path {
self.grammar_path.as_path()
}
}

/// Indents a multi-line string and trims any trailing newline.
Expand Down