-
-
Notifications
You must be signed in to change notification settings - Fork 884
refactor(markdown-parser): promote fenced code block skipped trivia to explicit CST nodes #9321
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from 2 commits
041e82b
8d084b9
582f51a
870df65
27f50ef
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change | ||||||||
|---|---|---|---|---|---|---|---|---|---|---|
|
|
@@ -33,6 +33,7 @@ use biome_parser::{ | |||||||||
| }; | ||||||||||
|
|
||||||||||
| use crate::syntax::parse_error::unterminated_fenced_code; | ||||||||||
| use crate::syntax::quote::try_bump_quote_marker; | ||||||||||
| use crate::syntax::{MAX_BLOCK_PREFIX_INDENT, TAB_STOP_SPACES}; | ||||||||||
|
|
||||||||||
| /// Minimum number of fence characters required per CommonMark §4.5. | ||||||||||
|
|
@@ -276,70 +277,131 @@ fn parse_code_content( | |||||||||
|
|
||||||||||
| // Consume all tokens until we see the matching closing fence or EOF | ||||||||||
| while !p.at(T![EOF]) { | ||||||||||
| if at_line_start && quote_depth > 0 { | ||||||||||
| let prev_virtual = p.state().virtual_line_start; | ||||||||||
| p.state_mut().virtual_line_start = Some(p.cur_range().start()); | ||||||||||
| p.skip_line_indent(MAX_BLOCK_PREFIX_INDENT); | ||||||||||
| p.state_mut().virtual_line_start = prev_virtual; | ||||||||||
|
|
||||||||||
| let mut ok = true; | ||||||||||
| for _ in 0..quote_depth { | ||||||||||
| if p.at(MD_TEXTUAL_LITERAL) && p.cur_text().starts_with('>') { | ||||||||||
| p.force_relex_regular(); | ||||||||||
| } | ||||||||||
| match prepare_next_code_content_token( | ||||||||||
| p, | ||||||||||
| is_tilde_fence, | ||||||||||
| fence_len, | ||||||||||
| fence_indent, | ||||||||||
| quote_depth, | ||||||||||
| &mut at_line_start, | ||||||||||
| ) { | ||||||||||
| CodeContentLoopAction::Break => break, | ||||||||||
| CodeContentLoopAction::Continue => continue, | ||||||||||
| CodeContentLoopAction::ConsumeText => {} | ||||||||||
|
Comment on lines
+289
to
+290
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||||||
| } | ||||||||||
|
|
||||||||||
| if p.at(T![>]) { | ||||||||||
| p.parse_as_skipped_trivia_tokens(|p| p.bump(T![>])); | ||||||||||
| } else if p.at(MD_TEXTUAL_LITERAL) && p.cur_text() == ">" { | ||||||||||
| p.parse_as_skipped_trivia_tokens(|p| p.bump_remap(T![>])); | ||||||||||
| } else { | ||||||||||
| ok = false; | ||||||||||
| break; | ||||||||||
| } | ||||||||||
| consume_code_textual(p); | ||||||||||
| at_line_start = false; | ||||||||||
| } | ||||||||||
|
|
||||||||||
| if p.at(MD_TEXTUAL_LITERAL) { | ||||||||||
| let text = p.cur_text(); | ||||||||||
| if text == " " || text == "\t" { | ||||||||||
| p.parse_as_skipped_trivia_tokens(|p| p.bump(MD_TEXTUAL_LITERAL)); | ||||||||||
| } | ||||||||||
| } | ||||||||||
| } | ||||||||||
| m.complete(p, MD_INLINE_ITEM_LIST); | ||||||||||
| } | ||||||||||
|
|
||||||||||
| if !ok { | ||||||||||
| break; | ||||||||||
| } | ||||||||||
| at_line_start = false; | ||||||||||
| } | ||||||||||
| enum CodeContentLoopAction { | ||||||||||
| Break, | ||||||||||
| Continue, | ||||||||||
| ConsumeText, | ||||||||||
| } | ||||||||||
|
|
||||||||||
| if p.at(NEWLINE) { | ||||||||||
| // Preserve newlines as code content and reset virtual line start. | ||||||||||
| let text_m = p.start(); | ||||||||||
| p.bump_remap(MD_TEXTUAL_LITERAL); | ||||||||||
| text_m.complete(p, MD_TEXTUAL); | ||||||||||
| p.set_virtual_line_start(); | ||||||||||
| at_line_start = true; | ||||||||||
| continue; | ||||||||||
| fn prepare_next_code_content_token( | ||||||||||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. docstrings |
||||||||||
| p: &mut MarkdownParser, | ||||||||||
| is_tilde_fence: bool, | ||||||||||
| fence_len: usize, | ||||||||||
| fence_indent: usize, | ||||||||||
| quote_depth: usize, | ||||||||||
| at_line_start: &mut bool, | ||||||||||
| ) -> CodeContentLoopAction { | ||||||||||
| if *at_line_start && quote_depth > 0 { | ||||||||||
| if !consume_quote_prefixes_in_code_content(p, quote_depth) { | ||||||||||
| return CodeContentLoopAction::Break; | ||||||||||
| } | ||||||||||
| *at_line_start = false; | ||||||||||
| } | ||||||||||
|
|
||||||||||
| if consume_code_newline(p) { | ||||||||||
| *at_line_start = true; | ||||||||||
| return CodeContentLoopAction::Continue; | ||||||||||
| } | ||||||||||
|
|
||||||||||
| if at_closing_fence(p, is_tilde_fence, fence_len) { | ||||||||||
| return CodeContentLoopAction::Break; | ||||||||||
| } | ||||||||||
|
|
||||||||||
| if *at_line_start && fence_indent > 0 { | ||||||||||
| skip_fenced_content_indent(p, fence_indent); | ||||||||||
| if at_closing_fence(p, is_tilde_fence, fence_len) { | ||||||||||
| break; | ||||||||||
| return CodeContentLoopAction::Break; | ||||||||||
| } | ||||||||||
| } | ||||||||||
coderabbitai[bot] marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||||||||||
|
|
||||||||||
| if at_line_start && fence_indent > 0 { | ||||||||||
| skip_fenced_content_indent(p, fence_indent); | ||||||||||
| if at_closing_fence(p, is_tilde_fence, fence_len) { | ||||||||||
| break; | ||||||||||
| } | ||||||||||
| CodeContentLoopAction::ConsumeText | ||||||||||
| } | ||||||||||
|
|
||||||||||
| fn consume_quote_prefixes_in_code_content(p: &mut MarkdownParser, quote_depth: usize) -> bool { | ||||||||||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Same here, let's document and explain the boolean |
||||||||||
| let prev_virtual = p.state().virtual_line_start; | ||||||||||
| p.state_mut().virtual_line_start = Some(p.cur_range().start()); | ||||||||||
| p.skip_line_indent(MAX_BLOCK_PREFIX_INDENT); | ||||||||||
| p.state_mut().virtual_line_start = prev_virtual; | ||||||||||
|
|
||||||||||
| for _ in 0..quote_depth { | ||||||||||
| if !consume_quote_prefix_in_code_content(p) { | ||||||||||
| return false; | ||||||||||
| } | ||||||||||
| } | ||||||||||
|
|
||||||||||
| // Consume the token as code content (including NEWLINE tokens) | ||||||||||
| let text_m = p.start(); | ||||||||||
| p.bump_remap(MD_TEXTUAL_LITERAL); | ||||||||||
| text_m.complete(p, MD_TEXTUAL); | ||||||||||
| at_line_start = false; | ||||||||||
| true | ||||||||||
| } | ||||||||||
|
|
||||||||||
| fn consume_quote_prefix_in_code_content(p: &mut MarkdownParser) -> bool { | ||||||||||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Can you add some docstrings? Document what the returned value means so we know how to interpret it |
||||||||||
| if p.at(MD_TEXTUAL_LITERAL) && p.cur_text().starts_with('>') { | ||||||||||
| p.force_relex_regular(); | ||||||||||
| } | ||||||||||
|
|
||||||||||
| m.complete(p, MD_INLINE_ITEM_LIST); | ||||||||||
| if !(p.at(T![>]) || (p.at(MD_TEXTUAL_LITERAL) && p.cur_text() == ">")) { | ||||||||||
| return false; | ||||||||||
| } | ||||||||||
|
|
||||||||||
| let prefix_m = p.start(); | ||||||||||
|
|
||||||||||
| // Empty pre-marker indent list (initial indent handled by skip_line_indent). | ||||||||||
| let indent_list_m = p.start(); | ||||||||||
| indent_list_m.complete(p, MD_QUOTE_INDENT_LIST); | ||||||||||
|
|
||||||||||
| debug_assert!( | ||||||||||
| try_bump_quote_marker(p), | ||||||||||
| "guard above guarantees marker present" | ||||||||||
| ); | ||||||||||
coderabbitai[bot] marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||||||||||
|
|
||||||||||
| // Optional post-marker space | ||||||||||
| if p.at(MD_TEXTUAL_LITERAL) { | ||||||||||
| let text = p.cur_text(); | ||||||||||
| if text == " " || text == "\t" { | ||||||||||
| p.bump_remap(MD_QUOTE_POST_MARKER_SPACE); | ||||||||||
| } | ||||||||||
| } | ||||||||||
|
|
||||||||||
| prefix_m.complete(p, MD_QUOTE_PREFIX); | ||||||||||
| true | ||||||||||
| } | ||||||||||
|
|
||||||||||
| fn consume_code_newline(p: &mut MarkdownParser) -> bool { | ||||||||||
| if !p.at(NEWLINE) { | ||||||||||
| return false; | ||||||||||
| } | ||||||||||
|
|
||||||||||
| // Preserve newlines as code content and reset virtual line start. | ||||||||||
| let text_m = p.start(); | ||||||||||
| p.bump_remap(MD_TEXTUAL_LITERAL); | ||||||||||
| text_m.complete(p, MD_TEXTUAL); | ||||||||||
| p.set_virtual_line_start(); | ||||||||||
| true | ||||||||||
| } | ||||||||||
|
|
||||||||||
| fn consume_code_textual(p: &mut MarkdownParser) { | ||||||||||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. There's a bit of misalignment among these new functions. Some return a boolean, some don't, but they all start with |
||||||||||
| // Consume the token as code content (including NEWLINE tokens). | ||||||||||
| let text_m = p.start(); | ||||||||||
| p.bump_remap(MD_TEXTUAL_LITERAL); | ||||||||||
| text_m.complete(p, MD_TEXTUAL); | ||||||||||
| } | ||||||||||
|
|
||||||||||
| pub(crate) fn info_string_has_backtick(p: &mut MarkdownParser) -> bool { | ||||||||||
|
|
@@ -390,7 +452,9 @@ fn skip_fenced_content_indent(p: &mut MarkdownParser, indent: usize) { | |||||||||
| } | ||||||||||
|
|
||||||||||
| consumed += width; | ||||||||||
| p.parse_as_skipped_trivia_tokens(|p| p.bump(MD_TEXTUAL_LITERAL)); | ||||||||||
| let char_m = p.start(); | ||||||||||
| p.bump_remap(MD_INDENT_CHAR); | ||||||||||
| char_m.complete(p, MD_INDENT_TOKEN); | ||||||||||
| } | ||||||||||
| } | ||||||||||
|
|
||||||||||
|
|
||||||||||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Is this comment still applicable, now that we changed the logic?