Add syntax for comments

src/end-to-end.test.ts

@@ -62,6 +62,18 @@ testCases(endToEnd, code => code)('end-to-end tests', [
   ['{ (a: A) (b: B) }', either.makeRight({ a: 'A', b: 'B' })],
   ['( { ((a): :(b)) ( ( b ): B ) } )', either.makeRight({ a: 'B', b: 'B' })],
   ['{ (a: :(")")), (")": (B)) }', either.makeRight({ a: 'B', ')': 'B' })],
+  [`/**/a/**/`, either.makeRight('a')],
+  ['hello//world', either.makeRight('hello')],
+  [`"hello//world"`, either.makeRight('hello//world')],
+  [`{a/* this works as a delimiter */b}`, either.makeRight({ 0: 'a', 1: 'b' })],
+  [
+    `/**/{/**/a:/**/b/**/,/**/c:/**/d/**/}/**/`,
+    either.makeRight({ a: 'b', c: 'd' }),
+  ],
+  [
+    `/**/(/**/a/**/=>/**/:a/**/)(/**/output/**/)/**/`,
+    either.makeRight('output'),
+  ],
   [':match({ a: A })({ tag: a, value: {} })', either.makeRight('A')],
   [':{string concatenate}(a)(b)', either.makeRight('ba')],
   [
@@ -70,6 +82,7 @@ testCases(endToEnd, code => code)('end-to-end tests', [
   ],
   [
     `{
+        // foo: bar
         "static data":"blah blah blah"
         "evaluated data": {
           0:@runtime

src/language/parsing/atom.ts

@@ -1,5 +1,6 @@
 import { parser, type Parser } from '../../parsing.js'
 import { optionallySurroundedByParentheses } from './parentheses.js'
+import { whitespace } from './trivia.js'
 
 export type Atom = string
 
@@ -9,31 +10,62 @@ export const isAtom = (value: unknown): value is Atom =>
 export const unit = '' as const
 
 export const atomParser: Parser<Atom> = optionallySurroundedByParentheses(
-  parser.map(
-    parser.lazy(() => parser.oneOf([quotedAtom, unquotedAtom])),
-    output => output.join(''),
-  ),
+  parser.lazy(() => parser.oneOf([quotedAtom, unquotedAtom])),
+)
+
+const quotedAtom = parser.map(
+  parser.sequence([
+    parser.as(parser.literal('"'), ''),
+    parser.map(
+      parser.zeroOrMore(
+        parser.oneOf([
+          parser.butNot(
+            parser.anySingleCharacter,
+            parser.oneOf([parser.literal('"'), parser.literal('\\')]),
+            '`"` or `\\`',
+          ),
+          parser.as(parser.literal('\\"'), '"'),
+          parser.as(parser.literal('\\\\'), '\\'),
+        ]),
+      ),
+      output => output.join(''),
+    ),
+    parser.as(parser.literal('"'), ''),
+  ]),
+  ([_1, contents, _2]) => contents,
 )
 
-const quotedAtom = parser.sequence([
-  parser.as(parser.literal('"'), ''),
-  parser.map(
-    parser.zeroOrMore(
+const unquotedAtom = parser.map(
+  parser.oneOrMore(
+    parser.butNot(
+      parser.anySingleCharacter,
       parser.oneOf([
-        parser.butNot(
-          parser.anySingleCharacter,
-          parser.oneOf([parser.literal('"'), parser.literal('\\')]),
-          '`"` or `\\`',
-        ),
-        parser.as(parser.literal('\\"'), '"'),
-        parser.as(parser.literal('\\\\'), '\\'),
+        whitespace,
+        parser.literal('"'),
+        parser.literal('{'),
+        parser.literal('}'),
+        parser.literal('['),
+        parser.literal(']'),
+        parser.literal('('),
+        parser.literal(')'),
+        parser.literal('<'),
+        parser.literal('>'),
+        parser.literal('#'),
+        parser.literal('&'),
+        parser.literal('|'),
+        parser.literal('\\'),
+        parser.literal('='),
+        parser.literal(':'),
+        parser.literal(';'),
+        parser.literal(','),
+        parser.literal('//'),
+        parser.literal('/*'),
+        parser.literal('*/'),
       ]),
+      'a forbidden character sequence',
     ),
-    output => output.join(''),
   ),
-  parser.as(parser.literal('"'), ''),
-])
-
-const unquotedAtom = parser.oneOrMore(
-  parser.regularExpression(/[^\s{}[\]()<>#&\|\\=:;,]+/),
+  characters => characters.join(''),
 )
+
+export { unquotedAtom as unquotedAtomParser }

src/language/parsing/molecule.ts

@@ -1,7 +1,7 @@
 import { parser, type Parser } from '../../parsing.js'
 import { atomParser, type Atom } from './atom.js'
 import { optionallySurroundedByParentheses } from './parentheses.js'
-import { whitespace } from './whitespace.js'
+import { trivia } from './trivia.js'
 
 export type Molecule = { readonly [key: Atom]: Molecule | Atom }
 
@@ -48,7 +48,14 @@ const makeIncrementingIndexer = (): Indexer => {
 
 // Language-specific parsers follow.
 
-const propertyDelimiter = parser.regularExpression(/[\s,]+/)
+const propertyDelimiter = parser.oneOf([
+  parser.sequence([
+    optional(omit(trivia)),
+    parser.literal(','),
+    optional(omit(trivia)),
+  ]),
+  trivia,
+])
 
 const sugaredLookup: Parser<PartialMolecule> =
   optionallySurroundedByParentheses(
@@ -67,9 +74,9 @@ const sugaredFunction: Parser<PartialMolecule> =
       flat(
         parser.sequence([
           parser.map(atomParser, output => [output]),
-          omit(whitespace),
+          omit(trivia),
           omit(parser.literal('=>')),
-          omit(whitespace),
+          omit(trivia),
           parser.map(
             parser.lazy(() => propertyValue),
             output => [output],
@@ -90,9 +97,9 @@ const sugaredApply: Parser<PartialMolecule> = parser.map(
     parser.oneOrMore(
       parser.sequence([
         parser.literal('('),
-        optional(omit(whitespace)),
+        optional(omit(trivia)),
         parser.lazy(() => propertyValue),
-        optional(omit(whitespace)),
+        optional(omit(trivia)),
         parser.literal(')'),
       ]),
     ),
@@ -120,7 +127,7 @@ const namedProperty = flat(
   parser.sequence([
     propertyKey,
     omit(parser.literal(':')),
-    optional(omit(whitespace)),
+    optional(omit(trivia)),
     propertyValue,
   ]),
 )

src/language/parsing/parentheses.ts

@@ -1,5 +1,5 @@
 import { parser, type Parser } from '../../parsing.js'
-import { whitespace } from './whitespace.js'
+import { trivia } from './trivia.js'
 
 const optionallySurroundedBy = <Output>(
   parser1: Parser<unknown>,
@@ -18,15 +18,15 @@ export const optionallySurroundedByParentheses = <Output>(
   theParser: Parser<Output>,
 ): Parser<Output> =>
   parser.oneOf([
-    // This allows `theParser` to greedily consume whitespace.
+    // This allows `theParser` to greedily consume trivia.
     optionallySurroundedBy(
       parser.literal('('),
       theParser,
-      parser.sequence([parser.zeroOrMore(whitespace), parser.literal(')')]),
+      parser.sequence([parser.zeroOrMore(trivia), parser.literal(')')]),
     ),
     optionallySurroundedBy(
-      parser.sequence([parser.literal('('), parser.zeroOrMore(whitespace)]),
+      parser.sequence([parser.literal('('), parser.zeroOrMore(trivia)]),
       theParser,
-      parser.sequence([parser.zeroOrMore(whitespace), parser.literal(')')]),
+      parser.sequence([parser.zeroOrMore(trivia), parser.literal(')')]),
     ),
   ])

src/language/parsing/syntax-tree.ts

@@ -10,6 +10,7 @@ import type {
 import type { KeyPath } from '../semantics.js'
 import { atomParser, type Atom } from './atom.js'
 import { moleculeParser, type Molecule } from './molecule.js'
+import { trivia } from './trivia.js'
 
 declare const _canonicalized: unique symbol
 export type Canonicalized = { readonly [_canonicalized]: true }
@@ -79,6 +80,10 @@ type JSONRecordForbiddingSymbolicKeys = {
 }>
 
 export const syntaxTreeParser: Parser<SyntaxTree> = parser.map(
-  parser.oneOf([atomParser, moleculeParser]),
-  canonicalize,
+  parser.sequence([
+    parser.zeroOrMore(trivia),
+    parser.oneOf([atomParser, moleculeParser]),
+    parser.zeroOrMore(trivia),
+  ]),
+  ([_leadingTrivia, syntaxTree, _trailingTrivia]) => canonicalize(syntaxTree),
 )

src/language/parsing/trivia.ts

@@ -0,0 +1,25 @@
+import { parser } from '../../parsing.js'
+
+const blockComment = parser.sequence([
+  parser.literal('/*'),
+  parser.zeroOrMore(
+    parser.oneOf([
+      parser.butNot(parser.anySingleCharacter, parser.literal('*'), '*'),
+      parser.lookaheadNot(parser.literal('*'), parser.literal('/'), '/'),
+    ]),
+  ),
+  parser.literal('*/'),
+])
+
+const singleLineComment = parser.sequence([
+  parser.literal('//'),
+  parser.zeroOrMore(
+    parser.butNot(parser.anySingleCharacter, parser.literal('\n'), 'newline'),
+  ),
+])
+
+export const whitespace = parser.regularExpression(/\s+/)
+
+export const trivia = parser.oneOrMore(
+  parser.oneOf([whitespace, singleLineComment, blockComment]),
+)

src/parsing/combinators.ts

@@ -37,6 +37,24 @@ export const lazy =
   input =>
     parser()(input)
 
+export const lookaheadNot =
+  <Output>(
+    parser: Parser<Output>,
+    notFollowedBy: Parser<unknown>,
+    followedByName: string,
+  ): Parser<Output> =>
+  input =>
+    either.flatMap(parser(input), success =>
+      either.match(notFollowedBy(success.remainingInput), {
+        left: _ => either.makeRight(success),
+        right: _ =>
+          either.makeLeft({
+            input,
+            message: `input was unexpectedly followed by ${followedByName}`,
+          }),
+      }),
+    )
+
 export const map =
   <Output, NewOutput>(
     parser: Parser<Output>,