parse optional (frontmatter_info_string)

Author: m4rch3n1ng

grammar.js

@@ -77,6 +77,7 @@ module.exports = grammar({
     $._block_comment_content,
     $._line_doc_content,
     $._frontmatter_start,
+    $.frontmatter_info_string,
     $.frontmatter_content,
     $._frontmatter_end,
     $._error_sentinel,
@@ -1654,6 +1655,7 @@ module.exports = grammar({
 
     frontmatter: $ => seq(
       $._frontmatter_start,
+      optional($.frontmatter_info_string),
       $.frontmatter_content,
       $._frontmatter_end,
     ),

src/scanner.c

@@ -14,6 +14,7 @@ enum TokenType {
     BLOCK_COMMENT_CONTENT,
     LINE_DOC_CONTENT,
     FRONTMATTER_START,
+    FRONTMATTER_INFO_STRING,
     FRONTMATTER_CONTENT,
     FRONTMATTER_END,
     ERROR_SENTINEL
@@ -22,6 +23,7 @@ enum TokenType {
 typedef struct {
     uint8_t opening_hash_count;
     uint8_t frontmatter_dashes;
+    bool frontmatter_has_info_string;
 } Scanner;
 
 void *tree_sitter_rust_external_scanner_create() { return ts_calloc(1, sizeof(Scanner)); }
@@ -32,16 +34,19 @@ unsigned tree_sitter_rust_external_scanner_serialize(void *payload, char *buffer
     Scanner *scanner = (Scanner *)payload;
     buffer[0] = (char)scanner->opening_hash_count;
     buffer[1] = (char)scanner->frontmatter_dashes;
-    return 2;
+    buffer[2] = (char)scanner->frontmatter_has_info_string;
+    return 3;
 }
 
 void tree_sitter_rust_external_scanner_deserialize(void *payload, const char *buffer, unsigned length) {
     Scanner *scanner = (Scanner *)payload;
     scanner->opening_hash_count = 0;
     scanner->frontmatter_dashes = 0;
-    if (length == 2) {
+    scanner->frontmatter_has_info_string = false;
+    if (length == 3) {
         scanner->opening_hash_count = buffer[0];
         scanner->frontmatter_dashes = buffer[1];
+        scanner->frontmatter_has_info_string = buffer[2];
     }
 }
 
@@ -350,16 +355,38 @@ static inline bool process_frontmatter_start(TSLexer *lexer, Scanner *scanner) {
         scanner->frontmatter_dashes = amount;
         lexer->result_symbol = FRONTMATTER_START;
 
+        while (!lexer->eof(lexer) && iswspace(lexer->lookahead) && lexer->lookahead != '\n') {
+            advance(lexer);
+        }
+
         // parse optional info string after the initial fence
-        while (lexer->lookahead != '\n' && !lexer->eof(lexer)) {
+        if (lexer->eof(lexer) || lexer->lookahead == '\n') {
+            scanner->frontmatter_has_info_string = false;
             advance(lexer);
+        } else {
+            scanner->frontmatter_has_info_string = true;
         }
-        advance(lexer);
 
         return true;
     }
 }
 
+static inline bool process_frontmatter_info_string(TSLexer *lexer, Scanner *scanner) {
+    if (scanner->frontmatter_has_info_string) {
+        while (!lexer->eof(lexer) && lexer->lookahead != '\n') {
+            advance(lexer);
+        }
+
+        lexer->result_symbol = FRONTMATTER_INFO_STRING;
+        lexer->mark_end(lexer);
+
+        advance(lexer);
+        return true;
+    } else {
+        return false;
+    }
+}
+
 static inline bool process_frontmatter(TSLexer *lexer, Scanner *scanner) {
     // seperately parse empty frontmatter, as tree-sitter strips all whitespace,
     // including newlines, so i can't rely on parsing only after a newline in this case.
@@ -472,6 +499,12 @@ bool tree_sitter_rust_external_scanner_scan(void *payload, TSLexer *lexer, const
         return process_frontmatter_start(lexer, scanner);
     }
 
+    if (valid_symbols[FRONTMATTER_INFO_STRING]) {
+        if (process_frontmatter_info_string(lexer, scanner)) {
+            return true;
+        }
+    }
+
     if (valid_symbols[FRONTMATTER_CONTENT]) {
         return process_frontmatter(lexer, scanner);
     }

test/corpus/source_files.txt

@@ -287,6 +287,7 @@ fn main() {}
 
 (source_file
   (frontmatter
+    (frontmatter_info_string)
     (frontmatter_content))
   (function_item
     (identifier)