Add a simple markdown parser for formatting rustc --explain

Currently, the output of `rustc --explain foo` displays the raw markdown in a
pager. This is acceptable, but using actual formatting makes it easier to
understand.

This patch consists of three major components:

1.  A markdown parser. This is an extremely simple non-backtracking recursive
    implementation that requires normalization of the final token stream
2.  A utility to write the token stream to an output buffer
3.  Configuration within rustc_driver_impl to invoke this combination for
    `--explain`. Like the current implementation, it first attempts to print to
    a pager with a fallback colorized terminal, and standard print as a last
    resort.

    If color is disabled, or if the output does not support it, or if printing
    with color fails, it will write the raw markdown (which matches current
    behavior).

    Pagers known to support color are: `less` (with `-r`), `bat` (aka `catbat`),
    and `delta`.

The markdown parser does not support the entire markdown specification, but
should support the following with reasonable accuracy:

-   Headings, including formatting
-   Comments
-   Code, inline and fenced block (no indented block)
-   Strong, emphasis, and strikethrough formatted text
-   Links, anchor, inline, and reference-style
-   Horizontal rules
-   Unordered and ordered list items, including formatting

This parser and writer should be reusable by other systems if ever needed.

Author: tgross35

Cargo.lock

@@ -4746,9 +4746,9 @@ dependencies = [
 
 [[package]]
 name = "termcolor"
-version = "1.1.3"
+version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755"
+checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6"
 dependencies = [
  "winapi-util",
 ]

compiler/rustc_driver_impl/src/lib.rs

@@ -24,6 +24,7 @@ use rustc_data_structures::profiling::{
 };
 use rustc_data_structures::sync::SeqCst;
 use rustc_errors::registry::{InvalidErrorCode, Registry};
+use rustc_errors::{markdown, ColorConfig};
 use rustc_errors::{
     DiagnosticMessage, ErrorGuaranteed, Handler, PResult, SubdiagnosticMessage, TerminalUrl,
 };
@@ -282,7 +283,7 @@ fn run_compiler(
     interface::set_thread_safe_mode(&sopts.unstable_opts);
 
     if let Some(ref code) = matches.opt_str("explain") {
-        handle_explain(&early_error_handler, diagnostics_registry(), code);
+        handle_explain(&early_error_handler, diagnostics_registry(), code, sopts.color);
         return Ok(());
     }
 
@@ -540,7 +541,7 @@ impl Compilation {
     }
 }
 
-fn handle_explain(handler: &EarlyErrorHandler, registry: Registry, code: &str) {
+fn handle_explain(handler: &EarlyErrorHandler, registry: Registry, code: &str, color: ColorConfig) {
     let upper_cased_code = code.to_ascii_uppercase();
     let normalised =
         if upper_cased_code.starts_with('E') { upper_cased_code } else { format!("E{code:0>4}") };
@@ -564,7 +565,7 @@ fn handle_explain(handler: &EarlyErrorHandler, registry: Registry, code: &str) {
                 text.push('\n');
             }
             if io::stdout().is_terminal() {
-                show_content_with_pager(&text);
+                show_md_content_with_pager(&text, color);
             } else {
                 safe_print!("{text}");
             }
@@ -575,34 +576,72 @@ fn handle_explain(handler: &EarlyErrorHandler, registry: Registry, code: &str) {
     }
 }
 
-fn show_content_with_pager(content: &str) {
+/// If color is always or auto, print formatted & colorized markdown. If color is never or
+/// if formatted printing fails, print the raw text.
+///
+/// Prefers a pager, falls back standard print
+fn show_md_content_with_pager(content: &str, color: ColorConfig) {
+    let mut fallback_to_println = false;
     let pager_name = env::var_os("PAGER").unwrap_or_else(|| {
         if cfg!(windows) { OsString::from("more.com") } else { OsString::from("less") }
     });
 
-    let mut fallback_to_println = false;
+    let mut cmd = Command::new(&pager_name);
+    // FIXME: find if other pagers accept color options
+    let mut print_formatted = if pager_name == "less" {
+        cmd.arg("-r");
+        true
+    } else if ["bat", "catbat", "delta"].iter().any(|v| *v == pager_name) {
+        true
+    } else {
+        false
+    };
 
-    match Command::new(pager_name).stdin(Stdio::piped()).spawn() {
-        Ok(mut pager) => {
-            if let Some(pipe) = pager.stdin.as_mut() {
-                if pipe.write_all(content.as_bytes()).is_err() {
-                    fallback_to_println = true;
-                }
-            }
+    if color == ColorConfig::Never {
+        print_formatted = false;
+    } else if color == ColorConfig::Always {
+        print_formatted = true;
+    }
+
+    let mdstream = markdown::MdStream::parse_str(content);
+    let bufwtr = markdown::create_stdout_bufwtr();
+    let mut mdbuf = bufwtr.buffer();
+    if mdstream.write_termcolor_buf(&mut mdbuf).is_err() {
+        print_formatted = false;
+    }
 
-            if pager.wait().is_err() {
+    if let Ok(mut pager) = cmd.stdin(Stdio::piped()).spawn() {
+        if let Some(pipe) = pager.stdin.as_mut() {
+            let res = if print_formatted {
+                pipe.write_all(mdbuf.as_slice())
+            } else {
+                pipe.write_all(content.as_bytes())
+            };
+
+            if res.is_err() {
                 fallback_to_println = true;
             }
         }
-        Err(_) => {
+
+        if pager.wait().is_err() {
             fallback_to_println = true;
         }
+    } else {
+        fallback_to_println = true;
     }
 
     // If pager fails for whatever reason, we should still print the content
     // to standard output
     if fallback_to_println {
-        safe_print!("{content}");
+        let fmt_success = match color {
+            ColorConfig::Auto => io::stdout().is_terminal() && bufwtr.print(&mdbuf).is_ok(),
+            ColorConfig::Always => bufwtr.print(&mdbuf).is_ok(),
+            ColorConfig::Never => false,
+        };
+
+        if !fmt_success {
+            safe_print!("{content}");
+        }
     }
 }
 

compiler/rustc_errors/Cargo.toml

@@ -20,7 +20,7 @@ rustc_hir = { path = "../rustc_hir" }
 rustc_lint_defs = { path = "../rustc_lint_defs" }
 rustc_type_ir = { path = "../rustc_type_ir" }
 unicode-width = "0.1.4"
-termcolor = "1.0"
+termcolor = "1.2.0"
 annotate-snippets = "0.9"
 termize = "0.1.1"
 serde = { version = "1.0.125", features = [ "derive" ] }

compiler/rustc_errors/src/emitter.rs

@@ -616,7 +616,7 @@ pub enum ColorConfig {
 }
 
 impl ColorConfig {
-    fn to_color_choice(self) -> ColorChoice {
+    pub fn to_color_choice(self) -> ColorChoice {
         match self {
             ColorConfig::Always => {
                 if io::stderr().is_terminal() {

compiler/rustc_errors/src/lib.rs

@@ -61,6 +61,7 @@ pub mod emitter;
 pub mod error;
 pub mod json;
 mod lock;
+pub mod markdown;
 pub mod registry;
 mod snippet;
 mod styled_buffer;

compiler/rustc_errors/src/markdown/mod.rs

@@ -0,0 +1,76 @@
+//! A simple markdown parser that can write formatted text to the terminal
+//!
+//! Entrypoint is `MdStream::parse_str(...)`
+use std::io;
+
+use termcolor::{Buffer, BufferWriter, ColorChoice};
+mod parse;
+mod term;
+
+/// An AST representation of a Markdown document
+#[derive(Clone, Debug, Default, PartialEq)]
+pub struct MdStream<'a>(Vec<MdTree<'a>>);
+
+impl<'a> MdStream<'a> {
+    /// Parse a markdown string to a tokenstream
+    #[must_use]
+    pub fn parse_str(s: &str) -> MdStream<'_> {
+        parse::entrypoint(s)
+    }
+
+    /// Write formatted output to a termcolor buffer
+    pub fn write_termcolor_buf(&self, buf: &mut Buffer) -> io::Result<()> {
+        term::entrypoint(self, buf)
+    }
+}
+
+/// Create a termcolor buffer with the `Always` color choice
+pub fn create_stdout_bufwtr() -> BufferWriter {
+    BufferWriter::stdout(ColorChoice::Always)
+}
+
+/// A single tokentree within a Markdown document
+#[derive(Clone, Debug, PartialEq)]
+pub enum MdTree<'a> {
+    /// Leaf types
+    Comment(&'a str),
+    CodeBlock {
+        txt: &'a str,
+        lang: Option<&'a str>,
+    },
+    CodeInline(&'a str),
+    Strong(&'a str),
+    Emphasis(&'a str),
+    Strikethrough(&'a str),
+    PlainText(&'a str),
+    /// [Foo](www.foo.com) or simple anchor <www.foo.com>
+    Link {
+        disp: &'a str,
+        link: &'a str,
+    },
+    /// `[Foo link][ref]`
+    RefLink {
+        disp: &'a str,
+        id: Option<&'a str>,
+    },
+    /// [ref]: www.foo.com
+    LinkDef {
+        id: &'a str,
+        link: &'a str,
+    },
+    /// Break bewtween two paragraphs (double `\n`), not directly parsed but
+    /// added later
+    ParagraphBreak,
+    /// Break bewtween two lines (single `\n`)
+    LineBreak,
+    HorizontalRule,
+    Heading(u8, MdStream<'a>),
+    OrderedListItem(u16, MdStream<'a>),
+    UnorderedListItem(MdStream<'a>),
+}
+
+impl<'a> From<Vec<MdTree<'a>>> for MdStream<'a> {
+    fn from(value: Vec<MdTree<'a>>) -> Self {
+        Self(value)
+    }
+}