support fenced code block

tamlok · tamlok · commit 1429b4b66113 · 2025-03-06T20:48:18.000+08:00
diff --git a/src/data.c b/src/data.c
@@ -69,6 +69,9 @@ void mdd_node_type_data_finalize(mdd_node_type_t type, mdd_node_type_data_t *obj
     case MDD_NODE_TYPE_LINK_SIZE:
         mdd_node_type_size_data_finalize(&obj->size_data);
         break;
+    case MDD_NODE_TYPE_FENCED_CODE_BLOCK:
+        mdd_node_type_code_block_data_finalize(&obj->code_block_data);
+        break;
     default:
         break;
     }
@@ -99,6 +102,10 @@ void mdd_node_type_note_data_finalize(mdd_node_type_note_data_t *obj) {
     mdd_char_array_finalize(&obj->note);
 }
 
+void mdd_node_type_code_block_data_finalize(mdd_node_type_code_block_data_t *obj) {
+    mdd_char_array_finalize(&obj->lang);
+}
+
 void mdd_node_type_label_data_finalize(mdd_node_type_label_data_t *obj) {
     mdd_char_array_finalize(&obj->text);
 }
@@ -176,6 +183,13 @@ void mdd_node_type_note_data_new(mdd_node_data_t *obj,
     }
 }
 
+void mdd_node_type_code_block_data_new(mdd_node_data_t *obj, const char *lang) {
+    assert(!obj->type_data);
+    obj->type_data = mdd_node_type_data_new();
+    mdd_node_type_code_block_data_t *code_block_data = &obj->type_data->code_block_data;
+    mdd_char_array_assign(&obj->type_data->code_block_data.lang, lang);
+}
+
 const char *mdd_node_type_to_string(mdd_node_type_t type) {
     switch (type) {
     case MDD_NODE_TYPE_UNKNOWN: return "Unknown";
@@ -209,6 +223,7 @@ const char *mdd_node_type_to_string(mdd_node_type_t type) {
     case MDD_NODE_TYPE_INLINE_EQUATION: return "InlineEquation";
     case MDD_NODE_TYPE_MARK: return "Mark";
     case MDD_NODE_TYPE_BLOCK_QUOTE: return "BlockQuote";
+    case MDD_NODE_TYPE_FENCED_CODE_BLOCK: return "FencedCodeBlock";
     case MDD_NODE_TYPE_DUMMY: return "Dummy";
     case MDD_NODE_TYPE_H1: return "H1";
     case MDD_NODE_TYPE_H2: return "H2";
@@ -257,6 +272,12 @@ int mdd_node_type_data_dump(char *buf, size_t len, const mdd_node_data_t *obj) {
                 mdd_char_array_to_string(&data->note));
         break;
     }
+    case MDD_NODE_TYPE_FENCED_CODE_BLOCK: {
+        const mdd_node_type_code_block_data_t *data = &obj->type_data->code_block_data;
+        return snprintf(buf, len, "{%s}",
+                mdd_char_array_to_string(&data->lang));
+        break;
+    }
     default:
         break;
     }
diff --git a/src/data.h b/src/data.h
@@ -37,6 +37,7 @@ typedef enum mdd_node_type_tag {
     MDD_NODE_TYPE_INLINE_EQUATION,
     MDD_NODE_TYPE_MARK,
     MDD_NODE_TYPE_BLOCK_QUOTE,
+    MDD_NODE_TYPE_FENCED_CODE_BLOCK,
     MDD_NODE_TYPE_DUMMY,
     MDD_NODE_TYPE_H1,
     MDD_NODE_TYPE_H2,
@@ -76,6 +77,13 @@ typedef struct mdd_node_type_note_data_tag {
 
 void mdd_node_type_note_data_finalize(mdd_node_type_note_data_t *obj);
 
+// Code block specific data.
+typedef struct mdd_node_type_code_block_data_tag {
+    char_array_t lang;
+} mdd_node_type_code_block_data_t;
+
+void mdd_node_type_code_block_data_finalize(mdd_node_type_code_block_data_t *obj);
+
 // Label specific data.
 typedef struct mdd_node_type_label_data_tag {
     char_array_t text;
@@ -97,6 +105,7 @@ typedef union mdd_node_type_data_tag {
     mdd_node_type_label_data_t label_data;
     mdd_node_type_size_data_t size_data;
     mdd_node_type_note_data_t note_data;
+    mdd_node_type_code_block_data_t code_block_data;
 } mdd_node_type_data_t;
 
 void mdd_node_type_data_finalize(mdd_node_type_t type, mdd_node_type_data_t *obj);
@@ -148,6 +157,8 @@ void mdd_node_type_size_data_new(mdd_node_data_t *obj, int width, int height);
 void mdd_node_type_note_data_new(mdd_node_data_t *obj,
         char_array_t *reference, char_array_t *note);
 
+void mdd_node_type_code_block_data_new(mdd_node_data_t *obj, const char *lang);
+
 // Context data during parse.
 typedef struct mdd_data_tag {
     // Input string to parse.
diff --git a/src/markdowndown.peg b/src/markdowndown.peg
@@ -48,6 +48,8 @@ int look_back_nonspace(mddi_context_t *ctx);
 int inline_equation_pre(mddi_context_t *ctx);
 int inline_equation_post(mddi_context_t *ctx);
 
+int look_back_one_char(mddi_context_t *ctx, char *excludes, size_t len, int positive);
+
 }
 
 ######
@@ -84,11 +86,36 @@ block <-
     {
         $$ = mdd_node_create_0(MDD_NODE_TYPE_BLANK_LINE, $0s, $0e);
     }
-    / (e:block_quote / e:verbatim / e:note / e:reference / e:plain)
+    / (e:block_quote / e:verbatim / e:fenced_code_block / e:note / e:reference / e:plain)
+    {
+        $$ = e;
+    }
+
+fenced_code_block <- (e:fenced_code_block_tick / e:fenced_code_block_tidle)
     {
         $$ = e;
     }
 
+fenced_code_block_tick_close <- '```' space (new_line / end_of_file)
+
+fenced_code_block_tick <- '```' space <(!new_line !'`' .)*> new_line
+    (!fenced_code_block_tick_close (!new_line .)* new_line)*
+    fenced_code_block_tick_close
+    {
+        $$ = mdd_node_create_0(MDD_NODE_TYPE_FENCED_CODE_BLOCK, $0s, $0e);
+        mdd_node_type_code_block_data_new(&$$->custom, $1);
+    }
+
+fenced_code_block_tidle_close <- '~~~' space (new_line / end_of_file)
+
+fenced_code_block_tidle <- '~~~' space <(!new_line !'~' .)*> new_line
+    (!fenced_code_block_tidle_close (!new_line .)* new_line)*
+    fenced_code_block_tidle_close
+    {
+        $$ = mdd_node_create_0(MDD_NODE_TYPE_FENCED_CODE_BLOCK, $0s, $0e);
+        mdd_node_type_code_block_data_new(&$$->custom, $1);
+    }
+
 # TODO: for block quote, we need to re-parse it separately after the main parse.
 # For now, we will only parse the inline elements within it.
 block_quote <-
@@ -235,12 +262,15 @@ inline_equation_single <- '$' &{ @@ = inline_equation_pre(pcc_ctx); } !'$' !'\\'
 
 inline_equation_multiple <- '$' &{ @@ = inline_equation_pre(pcc_ctx); } !'$' nonspace_char (!'$' !new_line .)+ '$' &{ @@ = inline_equation_post(pcc_ctx); } ![0-9]
 
+# Define all ticks rules together
 ticks1 <- '`' !'`'
 ticks2 <- '``' !'`'
 ticks3 <- '```' !'`'
 ticks4 <- '````' !'`'
 ticks5 <- '`````' !'`'
 
+fenced_code_block_start_tick_line <- '```' (!new_line !'`' .)* new_line
+
 code <- (ticks1 space ((!'`' nonspace_char)+ / !ticks1 '`'+ / !(space ticks1) (space_char / new_line !blank_line))+ space ticks1
     / ticks2 space ((!'`' nonspace_char)+ / !ticks2 '`'+ / !(space ticks2) (space_char / new_line !blank_line))+ space ticks2
     / !fenced_code_block_start_tick_line ticks3 space ((!'`' nonspace_char)+ / !ticks3 '`'+ / !(space ticks3) (space_char / new_line !blank_line))+ space ticks3
@@ -250,10 +280,6 @@ code <- (ticks1 space ((!'`' nonspace_char)+ / !ticks1 '`'+ / !(space ticks1) (s
         $$ = mdd_node_create_0(MDD_NODE_TYPE_CODE, $0s, $0e);
     }
 
-fenced_code_block_start_tick <- '```' (!new_line !'`' .)*
-
-fenced_code_block_start_tick_line <- fenced_code_block_start_tick new_line
-
 inline_note <- '^['
     {
         // Need to fix the range at the end.
@@ -698,15 +724,22 @@ void mddi_ast_node_custom_data__finalize(mddi_ast_node_custom_data_t *obj) {
 }
 
 int look_back_nonspace(mddi_context_t *ctx) {
+    char data[] = {' ', '\t', '\n', '\r'};
+    return look_back_one_char(ctx, data, sizeof(data), 0);
+}
+
+int look_back_one_char(mddi_context_t *ctx, char *chars, size_t len, int positive) {
     const char *buf = ctx->buffer.p + ctx->cur;
     if (ctx->cur == 0) {
         return 1;
     }
     const unsigned char ch = *(buf - 1);
-    if (ch != ' ' && ch != '\t' && ch != '\n' && ch != '\r') {
-        return 1;
+    for (size_t i = 0; i < len; ++i) {
+        if (ch == chars[i]) {
+            return positive;
+        }
     }
-    return 0;
+    return !positive;
 }
 
 int inline_equation_pre(mddi_context_t *ctx) {
diff --git a/tests/parser/CMakeLists.txt b/tests/parser/CMakeLists.txt
@@ -75,6 +75,11 @@ add_executable(test_block_quote
     test_utils.h test_utils.c)
 target_link_libraries(test_block_quote PRIVATE markdowndown)
 
+add_executable(test_fenced_code_block
+    test_fenced_code_block.c
+    test_utils.h test_utils.c)
+target_link_libraries(test_fenced_code_block PRIVATE markdowndown)
+
 add_test(NAME test_document
     COMMAND $<TARGET_FILE:test_document>)
 add_test(NAME test_front_matter
@@ -105,3 +110,5 @@ add_test(NAME test_mark
     COMMAND $<TARGET_FILE:test_mark>)
 add_test(NAME test_block_quote
     COMMAND $<TARGET_FILE:test_block_quote>)
+add_test(NAME test_fenced_code_block
+    COMMAND $<TARGET_FILE:test_fenced_code_block>)
diff --git a/tests/parser/test_fenced_code_block.c b/tests/parser/test_fenced_code_block.c
@@ -0,0 +1,77 @@
+#include "test_utils.h"
+
+int main() {
+    // Basic fenced code block with backticks
+    TEST_AST("```\n"
+        "code here\n"
+        "```\n",
+        "var: Document[0,18)\n"
+        "  var: BlockList[0,18)\n"
+        "    nul: FencedCodeBlock[0,18){}\n");
+
+    // End of file.
+    TEST_AST("```\n"
+        "code here\n"
+        "```",
+        "var: Document[0,17)\n"
+        "  var: BlockList[0,17)\n"
+        "    nul: FencedCodeBlock[0,17){}\n");
+
+    // Fenced code block with language identifier
+    TEST_AST("```python\n"
+        "def hello():\n"
+        "    print('Hello')\n"
+        "```\n",
+        "var: Document[0,46)\n"
+        "  var: BlockList[0,46)\n"
+        "    nul: FencedCodeBlock[0,46){python}\n");
+
+    // Fenced code block with tildes
+    TEST_AST("~~~\n"
+        "code here\n"
+        "~~~\n",
+        "var: Document[0,18)\n"
+        "  var: BlockList[0,18)\n"
+        "    nul: FencedCodeBlock[0,18){}\n");
+
+    // Fenced code block with tildes and language
+    TEST_AST("~~~javascript\n"
+        "function hello() {\n"
+        "    console.log('Hello');\n"
+        "}\n"
+        "~~~\n",
+        "var: Document[0,65)\n"
+        "  var: BlockList[0,65)\n"
+        "    nul: FencedCodeBlock[0,65){javascript}\n");
+
+    // Multiple fenced code blocks
+    TEST_AST("```python\n"
+        "print('Hello')\n"
+        "```\n\n"
+        "```javascript\n"
+        "console.log('World')\n"
+        "```\n",
+        "var: Document[0,69)\n"
+        "  var: BlockList[0,69)\n"
+        "    nul: FencedCodeBlock[0,29){python}\n"
+        "    nul: BlankLine[29,30)\n"
+        "    nul: FencedCodeBlock[30,69){javascript}\n");
+
+    // Fenced code block with empty lines
+    TEST_AST("```\n"
+        "line 1\n"
+        "\n"
+        "line 2\n"
+        "```\n",
+        "var: Document[0,23)\n"
+        "  var: BlockList[0,23)\n"
+        "    nul: FencedCodeBlock[0,23){}\n");
+
+    TEST_AST("```\n"
+        "```\n",
+        "var: Document[0,8)\n"
+        "  var: BlockList[0,8)\n"
+        "    nul: FencedCodeBlock[0,8){}\n");
+
+    return 0;
+}
