Skip to content

Commit b9a2243

Browse files
cscheidcscheid
committed
do not let list markers interrupt paragraphs inside display math
1 parent 353614a commit b9a2243

File tree

11 files changed

+13319
-10061
lines changed

11 files changed

+13319
-10061
lines changed

‎crates/tree-sitter-qmd/tree-sitter-markdown/claude-examples/display_math.qmd‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
$$
2+
1
3+
* 2
4+
$$

‎crates/tree-sitter-qmd/tree-sitter-markdown/claude-examples/display_math_in_list.qmd‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
* item 1
2+
* item 2 with math:
3+
$$
4+
* not a list item
5+
+ also not a list item
6+
$$
7+
* item 3

‎crates/tree-sitter-qmd/tree-sitter-markdown/claude-examples/multiple_display_math.qmd‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
First paragraph with math:
2+
$$
3+
1. not a list
4+
2. still not a list
5+
$$
6+
7+
Second paragraph:
8+
$$
9+
- not a list item either
10+
* nor this
11+
$$
12+
13+
Regular list after math:
14+
* real list item 1
15+
* real list item 2

‎crates/tree-sitter-qmd/tree-sitter-markdown/claude-examples/nested_display_math.qmd‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
Outer paragraph:
2+
$$
3+
Inner content:
4+
* 1
5+
- 2
6+
+ 3
7+
1. 4
8+
2) 5
9+
$$
10+
Back to paragraph.
11+
12+
* Now a real list
13+
* With real items

‎crates/tree-sitter-qmd/tree-sitter-markdown/claude-examples/regular_list.qmd‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
* a
2+
* b
3+
* c

‎crates/tree-sitter-qmd/tree-sitter-markdown/grammar.js‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -354,15 +354,15 @@ module.exports = grammar({
354354
optional($.block_continuation)
355355
),
356356
// Some symbols get parsed as single tokens so that html blocks get detected properly
357-
_code_line: $ => prec.right(repeat1(choice($._word, $._whitespace, common.punctuation_without($, [])))),
357+
_code_line: $ => prec.right(repeat1(choice($._word, $._display_math_state_track_marker, $._inline_math_state_track_marker, $._whitespace, common.punctuation_without($, [])))),
358358

359359
// the gymnastics around `:` in _line exist to make the parser reject paragraphs that start with a colon.
360360
// Those are technically valid in Markdown, but disallowing them here makes it possible to detect an
361361
// accidentally-continued paragraph with a colon that should have been a fenced div marker.
362362
// In these cases, users can use \: to escape the first colon.
363-
_line: $ => prec.right(seq(prec.right(choice($._word, $._whitespace, common.punctuation_without($, [":"]))),
364-
prec.right(repeat(choice($._word, $._whitespace, common.punctuation_without($, [])))))),
365-
_atx_heading_line: $ => prec.right(repeat1(choice($._word, $._whitespace, common.punctuation_without($, [])))),
363+
_line: $ => prec.right(seq(prec.right(choice($._word, $._display_math_state_track_marker, $._inline_math_state_track_marker, $._whitespace, common.punctuation_without($, [":"]))),
364+
prec.right(repeat(choice($._word, $._display_math_state_track_marker, $._inline_math_state_track_marker, $._whitespace, common.punctuation_without($, [])))))),
365+
_atx_heading_line: $ => prec.right(repeat1(choice($._word, $._display_math_state_track_marker, $._inline_math_state_track_marker, $._whitespace, common.punctuation_without($, [])))),
366366
_word: $ => new RegExp('[^' + PUNCTUATION_CHARACTERS_REGEX + ' \\t\\n\\r]+'),
367367
// The external scanner emits some characters that should just be ignored.
368368
_whitespace: $ => /[ \t]+/,
@@ -442,11 +442,15 @@ module.exports = grammar({
442442
seq(
443443
choice(
444444
$._word,
445+
$._display_math_state_track_marker,
446+
$._inline_math_state_track_marker,
445447
$._backslash_escape,
446448
common.punctuation_without($, ['|']),
447449
),
448450
repeat(choice(
449451
$._word,
452+
$._display_math_state_track_marker,
453+
$._inline_math_state_track_marker,
450454
$._whitespace,
451455
$._backslash_escape,
452456
common.punctuation_without($, ['|']),
@@ -554,6 +558,10 @@ module.exports = grammar({
554558

555559
$.ref_id_specifier,
556560
$.fenced_div_note_id,
561+
562+
// special tokens to allow external scanner serialization to happen
563+
$._display_math_state_track_marker,
564+
$._inline_math_state_track_marker,
557565
],
558566
precedences: $ => [
559567
[$._setext_heading1, $._block],

‎crates/tree-sitter-qmd/tree-sitter-markdown/src/grammar.json‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3949,6 +3949,14 @@
39493949
"type": "SYMBOL",
39503950
"name": "_word"
39513951
},
3952+
{
3953+
"type": "SYMBOL",
3954+
"name": "_display_math_state_track_marker"
3955+
},
3956+
{
3957+
"type": "SYMBOL",
3958+
"name": "_inline_math_state_track_marker"
3959+
},
39523960
{
39533961
"type": "SYMBOL",
39543962
"name": "_whitespace"
@@ -4123,6 +4131,14 @@
41234131
"type": "SYMBOL",
41244132
"name": "_word"
41254133
},
4134+
{
4135+
"type": "SYMBOL",
4136+
"name": "_display_math_state_track_marker"
4137+
},
4138+
{
4139+
"type": "SYMBOL",
4140+
"name": "_inline_math_state_track_marker"
4141+
},
41264142
{
41274143
"type": "SYMBOL",
41284144
"name": "_whitespace"
@@ -4288,6 +4304,14 @@
42884304
"type": "SYMBOL",
42894305
"name": "_word"
42904306
},
4307+
{
4308+
"type": "SYMBOL",
4309+
"name": "_display_math_state_track_marker"
4310+
},
4311+
{
4312+
"type": "SYMBOL",
4313+
"name": "_inline_math_state_track_marker"
4314+
},
42914315
{
42924316
"type": "SYMBOL",
42934317
"name": "_whitespace"
@@ -4461,6 +4485,14 @@
44614485
"type": "SYMBOL",
44624486
"name": "_word"
44634487
},
4488+
{
4489+
"type": "SYMBOL",
4490+
"name": "_display_math_state_track_marker"
4491+
},
4492+
{
4493+
"type": "SYMBOL",
4494+
"name": "_inline_math_state_track_marker"
4495+
},
44644496
{
44654497
"type": "SYMBOL",
44664498
"name": "_whitespace"
@@ -5074,6 +5106,14 @@
50745106
"type": "SYMBOL",
50755107
"name": "_word"
50765108
},
5109+
{
5110+
"type": "SYMBOL",
5111+
"name": "_display_math_state_track_marker"
5112+
},
5113+
{
5114+
"type": "SYMBOL",
5115+
"name": "_inline_math_state_track_marker"
5116+
},
50775117
{
50785118
"type": "SYMBOL",
50795119
"name": "_backslash_escape"
@@ -5235,6 +5275,14 @@
52355275
"type": "SYMBOL",
52365276
"name": "_word"
52375277
},
5278+
{
5279+
"type": "SYMBOL",
5280+
"name": "_display_math_state_track_marker"
5281+
},
5282+
{
5283+
"type": "SYMBOL",
5284+
"name": "_inline_math_state_track_marker"
5285+
},
52385286
{
52395287
"type": "SYMBOL",
52405288
"name": "_whitespace"
@@ -5643,6 +5691,14 @@
56435691
{
56445692
"type": "SYMBOL",
56455693
"name": "fenced_div_note_id"
5694+
},
5695+
{
5696+
"type": "SYMBOL",
5697+
"name": "_display_math_state_track_marker"
5698+
},
5699+
{
5700+
"type": "SYMBOL",
5701+
"name": "_inline_math_state_track_marker"
56465702
}
56475703
],
56485704
"inline": [],

0 commit comments

Comments
 (0)