fix: correctly parse words

Author: vigoux

corpus/backtick.txt

@@ -0,0 +1,45 @@
+================================================================================
+Simple backtick
+================================================================================
+
+`foobar`
+
+--------------------------------------------------------------------------------
+
+(help_file
+  (line
+    (backtick
+      (word))))
+
+================================================================================
+Backtick in text
+================================================================================
+
+Hello `world`, I am a markup language
+
+--------------------------------------------------------------------------------
+
+(help_file
+  (line
+    (word)
+    (backtick
+      (word))
+    (word)
+    (word)
+    (word)
+    (word)
+    (word)
+    (word)))
+
+================================================================================
+Backtick with command inside
+================================================================================
+
+`:echo`
+
+--------------------------------------------------------------------------------
+
+(help_file
+  (line
+    (backtick
+      (word))))

corpus/column_heading.txt

@@ -52,8 +52,7 @@ Below
 
 (help_file
   (column_heading
-    (column_name
-      (word)))
+    (column_name))
   (line
     (word)))
 
@@ -67,8 +66,6 @@ Bleh
 
 (help_file
   (column_heading
-    (column_name
-      (word)
-      (word)))
+    (column_name))
   (line
     (word)))

grammar.js

@@ -2,12 +2,17 @@ module.exports = grammar({
   name: 'help', // The actual language name is help
 
   extras: ($) => [/[\t ]/],
-  conflicts: ($) => [[$._atom, $.column_name]],
 
-  externals: ($) => [$.code_block],
+  externals: ($) => [
+    $.word,
+    $.column_name,
+    $.uppercase_name,
+    $.code_block
+  ],
 
   rules: {
-    help_file: ($) => repeat1(choice($.line, '\n', $.column_heading, $.headline)),
+    help_file: ($) =>
+      repeat1(choice($.line, '\n', $.column_heading, $.headline)),
 
     headline: ($) =>
       seq(
@@ -16,25 +21,33 @@ module.exports = grammar({
         seq(repeat1($.word), $.tag),
         '\n',
       ),
+
     line: ($) => prec.right(seq(repeat1($._atom), optional('\n'))),
 
-    _atom: ($) => choice($.word, $.tag, $.code_block, $.tag, $.option, $.hotlink),
+    _atom: ($) =>
+      choice(
+        $.word,
+        $.tag,
+        $.code_block,
+        $.tag,
+        $.option,
+        $.hotlink,
+        $.backtick,
+      ),
 
-    uppercase_name: ($) => repeat1(/[A-Z]+/),
-    column_name: ($) => repeat1($.word),
     column_heading: ($) =>
       seq(
         choice(
           seq(field('name', $.uppercase_name), optional($.tag)),
-          seq(field('name', $.column_name), '~'),
+          field('name', $.column_name),
         ),
         '\n',
       ),
 
-    word: ($) => /[^*|'\n \t]*[^*|'\n \t~]/,
     tag: ($) => wrapped_word($, '*', 'name'),
     option: ($) => wrapped_word($, "'", 'name'),
     hotlink: ($) => wrapped_word($, '|', 'destination'),
+    backtick: ($) => wrapped_word($, '`', 'content'),
   },
 });
 

src/grammar.json

@@ -125,23 +125,13 @@
         {
           "type": "SYMBOL",
           "name": "hotlink"
+        },
+        {
+          "type": "SYMBOL",
+          "name": "backtick"
         }
       ]
     },
-    "uppercase_name": {
-      "type": "REPEAT1",
-      "content": {
-        "type": "PATTERN",
-        "value": "[A-Z]+"
-      }
-    },
-    "column_name": {
-      "type": "REPEAT1",
-      "content": {
-        "type": "SYMBOL",
-        "name": "word"
-      }
-    },
     "column_heading": {
       "type": "SEQ",
       "members": [
@@ -174,21 +164,12 @@
               ]
             },
             {
-              "type": "SEQ",
-              "members": [
-                {
-                  "type": "FIELD",
-                  "name": "name",
-                  "content": {
-                    "type": "SYMBOL",
-                    "name": "column_name"
-                  }
-                },
-                {
-                  "type": "STRING",
-                  "value": "~"
-                }
-              ]
+              "type": "FIELD",
+              "name": "name",
+              "content": {
+                "type": "SYMBOL",
+                "name": "column_name"
+              }
             }
           ]
         },
@@ -198,10 +179,6 @@
         }
       ]
     },
-    "word": {
-      "type": "PATTERN",
-      "value": "[^*|'\\n \\t]*[^*|'\\n \\t~]"
-    },
     "tag": {
       "type": "SEQ",
       "members": [
@@ -264,6 +241,27 @@
           "value": "|"
         }
       ]
+    },
+    "backtick": {
+      "type": "SEQ",
+      "members": [
+        {
+          "type": "STRING",
+          "value": "`"
+        },
+        {
+          "type": "FIELD",
+          "name": "content",
+          "content": {
+            "type": "SYMBOL",
+            "name": "word"
+          }
+        },
+        {
+          "type": "STRING",
+          "value": "`"
+        }
+      ]
     }
   },
   "extras": [
@@ -272,14 +270,21 @@
       "value": "[\\t ]"
     }
   ],
-  "conflicts": [
-    [
-      "_atom",
-      "column_name"
-    ]
-  ],
+  "conflicts": [],
   "precedences": [],
   "externals": [
+    {
+      "type": "SYMBOL",
+      "name": "word"
+    },
+    {
+      "type": "SYMBOL",
+      "name": "column_name"
+    },
+    {
+      "type": "SYMBOL",
+      "name": "uppercase_name"
+    },
     {
       "type": "SYMBOL",
       "name": "code_block"

src/node-types.json

@@ -1,4 +1,20 @@
 [
+  {
+    "type": "backtick",
+    "named": true,
+    "fields": {
+      "content": {
+        "multiple": false,
+        "required": true,
+        "types": [
+          {
+            "type": "word",
+            "named": true
+          }
+        ]
+      }
+    }
+  },
   {
     "type": "column_heading",
     "named": true,
@@ -29,21 +45,6 @@
       ]
     }
   },
-  {
-    "type": "column_name",
-    "named": true,
-    "fields": {},
-    "children": {
-      "multiple": true,
-      "required": true,
-      "types": [
-        {
-          "type": "word",
-          "named": true
-        }
-      ]
-    }
-  },
   {
     "type": "headline",
     "named": true,
@@ -110,6 +111,10 @@
       "multiple": true,
       "required": true,
       "types": [
+        {
+          "type": "backtick",
+          "named": true
+        },
         {
           "type": "code_block",
           "named": true
@@ -165,11 +170,6 @@
       }
     }
   },
-  {
-    "type": "uppercase_name",
-    "named": true,
-    "fields": {}
-  },
   {
     "type": "\n",
     "named": false
@@ -182,20 +182,28 @@
     "type": "*",
     "named": false
   },
+  {
+    "type": "`",
+    "named": false
+  },
   {
     "type": "code_block",
     "named": true
   },
   {
-    "type": "word",
+    "type": "column_name",
     "named": true
   },
   {
-    "type": "|",
-    "named": false
+    "type": "uppercase_name",
+    "named": true
+  },
+  {
+    "type": "word",
+    "named": true
   },
   {
-    "type": "~",
+    "type": "|",
     "named": false
   }
 ]