github: support ANSI color codes in basic renderer (#63)

Co-authored-by: Joshua M. Clulow <[email protected]>

Author: sunshowers

.gitignore

@@ -1,6 +1,6 @@
-/target/
+/target
 /config.toml
 /data.sqlite3
-/cache/
-/etc/
-/var/
+/cache
+/etc
+/var

Cargo.lock

@@ -25,6 +25,7 @@ members = [
 resolver = "2"
 
 [workspace.dependencies]
+ansi-to-html = "0.2"
 anyhow = "1"
 aws-config = "1"
 aws-credential-types = "1"
@@ -73,6 +74,7 @@ slog = { version = "2.7", features = [ "release_max_level_debug" ] }
 slog-bunyan = "2.4"
 slog-term = "2.7"
 smf = { git = "https://github.com/illumos/smf-rs.git" }
+strip-ansi-escapes = "0.2"
 strum = { version = "0.25", features = [ "derive" ] }
 tempfile = "3.3"
 thiserror = "1"

Cargo.toml

@@ -14,6 +14,7 @@ buildomat-github-database = { path = "../database" }
 buildomat-github-hooktypes = { path = "../hooktypes" }
 buildomat-sse = { path = "../../sse" }
 
+ansi-to-html = { workspace = true }
 anyhow = { workspace = true }
 base64 = { workspace = true }
 chrono = { workspace = true }
@@ -30,6 +31,7 @@ schemars = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 slog = { workspace = true }
+strip-ansi-escapes = { workspace = true }
 tempfile = { workspace = true }
 thiserror = { workspace = true }
 tokio = { workspace = true }

github/server/Cargo.toml

@@ -14,6 +14,7 @@ use futures::StreamExt;
 use serde::{Deserialize, Serialize};
 #[allow(unused_imports)]
 use slog::{debug, error, info, o, trace, warn, Logger};
+use std::borrow::Cow;
 use std::collections::{HashMap, VecDeque};
 use std::sync::Arc;
 use tokio::io::{AsyncSeekExt, AsyncWriteExt};
@@ -81,7 +82,7 @@ impl JobEventEx for JobEvent {
              * Do the HTML escaping of the payload one canonical way, in the
              * server:
              */
-            html_escape::encode_safe(&self.payload),
+            encode_payload(&self.payload),
         );
 
         EventRow {
@@ -122,6 +123,37 @@ impl JobEventEx for JobEvent {
     }
 }
 
+fn encode_payload(payload: &str) -> Cow<'_, str> {
+    /*
+     * Apply ANSI formatting to the payload after escaping it (we want to
+     * transmit the corresponding HTML tags over the wire).
+     *
+     * One of the cases this does not handle is multi-line color output split
+     * across several payloads.  Doing so is quite tricky, because buildomat
+     * works with a single bash script and doesn't know when commands are
+     * completed.  Other systems like GitHub Actions (as checked on 2024-09-03)
+     * don't handle multiline color either, so it's fine to punt on that.
+     */
+    ansi_to_html::convert_with_opts(
+        payload,
+        &ansi_to_html::Opts::default()
+            .four_bit_var_prefix(Some("ansi-".to_string())),
+    )
+    .map_or_else(
+        |_| {
+            /*
+             * Invalid ANSI code: only escape HTML in case the conversion to
+             * ANSI fails.  To maintain consistency we use the same logic as
+             * ansi-to-html: do not escape "/".  (There are other differences,
+             * such as ansi-to-html using decimal escapes while html_escape uses
+             * hex, but those are immaterial.)
+             */
+            html_escape::encode_quoted_attribute(payload)
+        },
+        Cow::Owned,
+    )
+}
+
 #[derive(Debug, Serialize)]
 struct EventRow {
     task: Option<u32>,
@@ -573,7 +605,16 @@ pub(crate) async fn run(
                      */
                     let mut line =
                         if console { "|C| " } else { "| " }.to_string();
-                    let mut chars = ev.payload.chars();
+
+                    /*
+                     * We support ANSI escapes in the log renderer, which means
+                     * that tools will generate ANSI sequences.  That doesn't
+                     * work in the GitHub renderer, so we need to strip them out
+                     * entirely.
+                     */
+                    let payload = strip_ansi_escapes::strip_str(&ev.payload);
+                    let mut chars = payload.chars();
+
                     for _ in 0..MAX_LINE_LENGTH {
                         if let Some(c) = chars.next() {
                             line.push(c);
@@ -1745,6 +1786,78 @@ pub mod test {
     use super::*;
     use buildomat_github_testdata::*;
 
+    #[test]
+    fn test_encode_payload() {
+        let data = &[
+            ("Hello, world!", "Hello, world!"),
+            /*
+             * HTML escapes:
+             */
+            (
+                "2 & 3 < 4 > 5 / 6 ' 7 \" 8",
+                "2 &amp; 3 &lt; 4 &gt; 5 / 6 &#39; 7 &quot; 8",
+            ),
+            /*
+             * ANSI color codes:
+             */
+            (
+                /*
+                 * Basic 16-color example; also tests a bright color (96).
+                 * (ansi-to-html 0.2.1 claims not to support bright colors, but
+                 * it actually does.)
+                 */
+                "\x1b[31mHello, world!\x1b[0m \x1b[96mAnother message\x1b[0m",
+                "<span style='color:var(--ansi-red,#a00)'>Hello, world!</span> \
+                <span style='color:var(--ansi-bright-cyan,#5ff)'>\
+                Another message</span>",
+            ),
+            (
+                /*
+                 * Truecolor, bold, italic, underline, and also with escapes.
+                 * The second code ("another") does not have a reset, but we
+                 * want to ensure that we generate closing HTML tags anyway.
+                 */
+                "\x1b[38;2;255;0;0;1;3;4mTest message\x1b[0m and &/' \
+                \x1b[38;2;0;255;0;1;3;4manother",
+                "<span style='color:#ff0000'><b><i><u>Test message</u></i></b>\
+                </span> and &amp;/&#39; <span style='color:#00ff00'><b><i>\
+                <u>another</u></i></b></span>",
+            ),
+            (
+                /*
+                 * Invalid ANSI code "xx"; should be HTML-escaped but the
+                 * invalid ANSI code should remain as-is.  (The second ANSI code
+                 * is valid, and ansi-to-html should handle it.)
+                 */
+                "\x1b[xx;2;255;0;0;1;3;4mTest message\x1b[0m and &/' \
+                \x1b[38;2;0;255;0;1;3;4manother",
+                "\u{1b}[xx;2;255;0;0;1;3;4mTest message and &amp;/&#39; <span \
+                style='color:#00ff00'><b><i><u>another</u></i></b></span>",
+            ),
+            (
+                /*
+                 * Invalid ANSI code "9000"; should be HTML-escaped but the
+                 * invalid ANSI code should remain as-is.  (The second ANSI code
+                 * is valid, but ansi-to-html's current behavior is to error out
+                 * in this case.  This can probably be improved.)
+                 */
+                "\x1b[9000;2;255;0;0;1;3;4mTest message\x1b[0m and &/' \
+                \x1b[38;2;0;255;0;1;3;4manother",
+                "\u{1b}[9000;2;255;0;0;1;3;4mTest message\u{1b}[0m and \
+                &amp;/&#x27; \u{1b}[38;2;0;255;0;1;3;4manother",
+            )
+        ];
+
+        for (input, expected) in data {
+            let output = encode_payload(input);
+            assert_eq!(
+                output, *expected,
+                "output != expected: input: {:?}",
+                input
+            );
+        }
+    }
+
     #[test]
     fn basic_parse_basic() -> Result<()> {
         let (path, content, _) = real0();

github/server/src/variety/basic.rs

@@ -2,6 +2,32 @@
  * Copyright 2024 Oxide Computer Company
  */
 
+/*
+ * The "ansi-to-html" crate uses CSS variables when emitting text that uses the
+ * classic ANSI colour palette.  Adjust the default colours to be a little
+ * darker for more contrast against "s_stdout" and "s_stderr" backgrounds,
+ * which are both quite light.
+ */
+:root {
+    --ansi-black:           #000000;
+    --ansi-red:             #b0000f;
+    --ansi-green:           #007000;
+    --ansi-yellow:          #808000;
+    --ansi-blue:            #1d1dc9;
+    --ansi-magenta:         #7027b9;
+    --ansi-cyan:            #0a8080;
+    --ansi-white:           #ffffff;
+
+    --ansi-bright-black:    #000000;
+    --ansi-bright-red:      #b20f00;
+    --ansi-bright-green:    #557000;
+    --ansi-bright-yellow:   #b44405;
+    --ansi-bright-blue:     #5f55df;
+    --ansi-bright-magenta:  #bf2c90;
+    --ansi-bright-cyan:     #30a0a0;
+    --ansi-bright-white:    #ffffff;
+}
+
 table.table_output {
     border: none;
 }
@@ -19,7 +45,7 @@ tr.s_stdout {
 }
 
 tr.s_stderr {
-    background-color: #ffd9da;
+    background-color: #f3f3f3;
 }
 
 tr.s_task {