From d954b9579abf162181c93d1246b2f62e4ce41e2a Mon Sep 17 00:00:00 2001 From: "Henry H. Andrews" Date: Sun, 27 Apr 2025 14:42:58 -0700 Subject: [PATCH 1/3] Sequential media highlight support Improve text/event-stream support with comment support and generally better configuration (one rule wasn't doing anything). Add support for JSONL/NDJSON (basically JSON without the check for values separated only by whitespace) and JSON Text Sequences (which requires using `0x1E` as the record separator, and highlighting it separately). --- scripts/md2html/md2html.js | 64 ++++++++++++++++++++++++--- tests/md2html/fixtures/basic-new.html | 38 +++++++++++++--- tests/md2html/fixtures/basic-new.md | 34 +++++++++++++- 3 files changed, 121 insertions(+), 15 deletions(-) diff --git a/scripts/md2html/md2html.js b/scripts/md2html/md2html.js index 6abbc024a9..ad10022a93 100644 --- a/scripts/md2html/md2html.js +++ b/scripts/md2html/md2html.js @@ -52,18 +52,70 @@ hljs.registerLanguage('eventstream', function() { return { contains: [ { - scope: "attr", - begin: /^/, - end: ":", + scope: "comment", + begin: /^:/, + end: /$/, }, { - scope: "literal", - begin: /: */, - end: /$/, + scope: "attr", + match: /^[^:]+/ }, ], } }); +hljs.registerLanguage('jsonseq', function() { + return { + keywords: ["true", "false", "null"], + contains: [ + { + scope: "meta", + match: /0[xX]1[eE]/, + }, + { + scope: "attr", + begin: /"(\\.|[^\\"\r\n])*"(?=\s*:)/, + relevance: 1.01 + }, + { + scope: "punctuation", + match: /[{}[\],:]/, + relevance: 0 + }, + { + scope: "literals", + beginKeywords: ["true", "false" , "null"].join(" "), + }, + hljs.QUOTE_STRING_MODE, + hljs.C_NUMBER_MODE + ] + } + }); +hljs.registerLanguage('jsonl', function() { + return { + aliases: ["ndjson"], + keywords: ["true", "false", "null"], + contains: [ + { + scope: 'attr', + begin: /"(\\.|[^\\"\r\n])*"(?=\s*:)/, + relevance: 1.01 + }, + { + scope: "punctuation", + match: /[{}[\],:]/, + relevance: 0 + }, + { + scope: "literals", + beginKeywords: ["true", "false" , "null"].join(" "), + }, + hljs.QUOTE_STRING_MODE, + hljs.C_NUMBER_MODE + ] + } + }); + + const cheerio = require('cheerio'); let argv = require('yargs') diff --git a/tests/md2html/fixtures/basic-new.html b/tests/md2html/fixtures/basic-new.html index b1ac32d1c6..cdb4160f55 100644 --- a/tests/md2html/fixtures/basic-new.html +++ b/tests/md2html/fixtures/basic-new.html @@ -45,14 +45,38 @@ 
https://foo.com/bar{?baz*,qux}
-
data: This data is formatted
-data: across two lines
-retry: 5
+
event: addString
+data: This data is formatted
+data: across two lines
+retry: 5
 
-event: add
-data: 1234.5678
-unknown-field: this is ignored
-

+event: addNumber
+data: 1234.5678
+unknownField: this is ignored
+
+: This is a comment
+event: addJSON
+data: {"foo": 42}
+
+
{"event": "addString", "data": "This data is formatted\nacross two lines", "retry": 5}
+{"event": "addNumber", "data": "1234.5678"}
+{"event": "addJSON", "data": "{\"foo\": 42}"}
+
+
{"event": "addString", "data": "This data is formatted\nacross two lines", "retry": 5}
+{"event": "addNumber", "data": "1234.5678"}
+{"event": "addJSON", "data": "{\"foo\": 42}"}
+
+
0x1E{
+  "timestamp": "1985-04-12T23:20:50.52Z",
+  "level": 1,
+  "message": "Hi!"
+}
+0x1E{
+  "timestamp": "1985-04-12T23:20:51.37Z",
+  "level": 1,
+  "message": "Bye!"
+}
+

Appendix A: Revision History

diff --git a/tests/md2html/fixtures/basic-new.md b/tests/md2html/fixtures/basic-new.md index f5e02609bd..49e476b1e4 100644 --- a/tests/md2html/fixtures/basic-new.md +++ b/tests/md2html/fixtures/basic-new.md @@ -63,13 +63,43 @@ https://foo.com/bar{?baz*,qux} ``` ```eventstream +event: addString data: This data is formatted data: across two lines retry: 5 -event: add +event: addNumber data: 1234.5678 -unknown-field: this is ignored +unknownField: this is ignored + +: This is a comment +event: addJSON +data: {"foo": 42} +``` + +```jsonl +{"event": "addString", "data": "This data is formatted\nacross two lines", "retry": 5} +{"event": "addNumber", "data": "1234.5678"} +{"event": "addJSON", "data": "{\"foo\": 42}"} +``` + +```ndjson +{"event": "addString", "data": "This data is formatted\nacross two lines", "retry": 5} +{"event": "addNumber", "data": "1234.5678"} +{"event": "addJSON", "data": "{\"foo\": 42}"} +``` + +```jsonseq +0x1E{ + "timestamp": "1985-04-12T23:20:50.52Z", + "level": 1, + "message": "Hi!" +} +0x1E{ + "timestamp": "1985-04-12T23:20:51.37Z", + "level": 1, + "message": "Bye!" +} ``` ## Appendix A: Revision History From 795b8920fbc4a42551beed29c528b0776c611964 Mon Sep 17 00:00:00 2001 From: "Henry H. Andrews" Date: Mon, 28 Apr 2025 20:48:29 -0700 Subject: [PATCH 2/3] Add (just barely) multipart support --- scripts/md2html/md2html.js | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/scripts/md2html/md2html.js b/scripts/md2html/md2html.js index ad10022a93..a49fe7e98b 100644 --- a/scripts/md2html/md2html.js +++ b/scripts/md2html/md2html.js @@ -48,6 +48,31 @@ hljs.registerLanguage('uri', function() { ], } }); +hljs.registerLanguage('multipart', function() { + return { + // This is a very limited approach that only + // detects boundaries and headers that start + // with "Content-" + contains: [ + { + scope: "meta", + match: /^--.*$/, + }, + { + scope: "literal", + begin: /^Content-/, + end: /$/, + contains: [ + { + scope: "attr", + begin: " ", + end: /$/, + }, + ] + }, + ], + } + }); hljs.registerLanguage('eventstream', function() { return { contains: [ From a3c408a52187579a77419bcd2b68594c3407a179 Mon Sep 17 00:00:00 2001 From: "Henry H. Andrews" Date: Mon, 28 Apr 2025 20:56:59 -0700 Subject: [PATCH 3/3] Tests for multipart highlighting --- tests/md2html/fixtures/basic-new.html | 24 ++++++++++++++++++++++++ tests/md2html/fixtures/basic-new.md | 26 ++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/tests/md2html/fixtures/basic-new.html b/tests/md2html/fixtures/basic-new.html index cdb4160f55..7f04b29f3d 100644 --- a/tests/md2html/fixtures/basic-new.html +++ b/tests/md2html/fixtures/basic-new.html @@ -45,6 +45,30 @@ 
https://foo.com/bar{?baz*,qux}
+
--boundary-example
+Content-Type: application/openapi+yaml
+Content-Location: https://inaccessible-domain.com/api/openapi.yaml
+
+openapi: 3.2.0
+info:
+  title: Example API
+  version: 1.0
+  externalDocs:
+    url: docs.html
+
+--boundary-example
+Content-Type: text/html
+Content-Location: https://example.com/api/docs.html
+
+<html>
+  <head>
+    <title>API Documentation</title>
+  </head>
+  <body>
+    <p>Awesome documentation goes here</p>
+  </body>
+</html>
+
event: addString
 data: This data is formatted
 data: across two lines
diff --git a/tests/md2html/fixtures/basic-new.md b/tests/md2html/fixtures/basic-new.md
index 49e476b1e4..8c5ffc209c 100644
--- a/tests/md2html/fixtures/basic-new.md
+++ b/tests/md2html/fixtures/basic-new.md
@@ -62,6 +62,32 @@ https://foo.com/bar?baz=qux&fred=waldo#fragment
 https://foo.com/bar{?baz*,qux}
 ```
 
+```multipart
+--boundary-example
+Content-Type: application/openapi+yaml
+Content-Location: https://inaccessible-domain.com/api/openapi.yaml
+
+openapi: 3.2.0
+info:
+  title: Example API
+  version: 1.0
+  externalDocs:
+    url: docs.html
+
+--boundary-example
+Content-Type: text/html
+Content-Location: https://example.com/api/docs.html
+
+
+  
+    API Documentation
+  
+  
+    
Awesome documentation goes here

+  
+
+```
+
 ```eventstream
 event: addString
 data: This data is formatted