wasm: Implement arity for iter nodes, + map()

Author: pdubroy

packages/miniohm-js/index.js

@@ -213,15 +213,15 @@ export class CstNode {
   }
 
   isNonterminal() {
-    return (this._type & CST_NODE_TYPE_MASK) === CstNodeType.NONTERMINAL;
+    return (this._typeAndDetails & CST_NODE_TYPE_MASK) === CstNodeType.NONTERMINAL;
   }
 
   isTerminal() {
-    return (this._type & CST_NODE_TYPE_MASK) === CstNodeType.TERMINAL;
+    return (this._typeAndDetails & CST_NODE_TYPE_MASK) === CstNodeType.TERMINAL;
   }
 
   isIter() {
-    return (this._type & CST_NODE_TYPE_MASK) === CstNodeType.ITER;
+    return (this._typeAndDetails & CST_NODE_TYPE_MASK) === CstNodeType.ITER;
   }
 
   isOptional() {
@@ -245,10 +245,14 @@ export class CstNode {
     return this._view.getUint32(this._base + 4, true);
   }
 
-  get _type() {
+  get _typeAndDetails() {
     return this._view.getInt32(this._base + 8, true);
   }
 
+  get arity() {
+    return this._typeAndDetails >>> 2;
+  }
+
   get children() {
     if (!this._children) {
       this._children = this._computeChildren();
@@ -304,6 +308,16 @@ export class CstNode {
     const {sourceString, startIdx} = this;
     return `CstNode {ctorName: ${ctorName}, sourceString: ${sourceString}, startIdx: ${startIdx} }`;
   }
+
+  map(callbackFn) {
+    const {arity, children} = this;
+    assert(callbackFn.length === arity, 'bad arity');
+    const ans = [];
+    for (let i = 0; i < children.length; i += arity) {
+      ans.push(callbackFn(...children.slice(i, i + arity)));
+    }
+    return ans;
+  }
 }
 
 export class MatchResult {

packages/wasm/src/index.js

@@ -599,11 +599,11 @@ class Assembler {
     this.emit(instr.call, w.funcidx(prebuiltFuncidx(name)));
   }
 
-  newIterNodeWithSavedPosAndBindings() {
+  newIterNodeWithSavedPosAndBindings(arity) {
     this.getSavedPos();
     this.globalGet('pos');
     this.getSavedNumBindings();
-    this.i32Const(0); // TODO: arity
+    this.i32Const(arity);
     this.callPrebuiltFunc('newIterationNode');
   }
 
@@ -1524,7 +1524,7 @@ export class Compiler {
       asm.restorePos();
       asm.restoreBindingsLength();
     });
-    asm.newIterNodeWithSavedPosAndBindings();
+    asm.newIterNodeWithSavedPosAndBindings(ir.outArity(child));
     asm.localSet('ret');
   }
 
@@ -1609,8 +1609,7 @@ export class Compiler {
     asm.restorePos();
     asm.restoreBindingsLength();
     asm.popStackFrame();
-
-    asm.newIterNodeWithSavedPosAndBindings();
+    asm.newIterNodeWithSavedPosAndBindings(ir.outArity(child));
     asm.localSet('ret');
   }
 

packages/wasm/src/ir.ts

@@ -25,9 +25,14 @@ export type Expr =
 export interface Alt {
   type: 'Alt';
   children: Expr[];
+  outArity: number;
 }
 
-export const alt = (children: Expr[]): Alt => ({type: 'Alt', children});
+export const alt = (children: Expr[]): Alt => ({
+  type: 'Alt',
+  children,
+  outArity: outArity(children[0])
+});
 
 export interface Any {
   type: 'Any';
@@ -91,17 +96,27 @@ export const end = (): End => ({type: 'End'});
 export interface Lex {
   type: 'Lex';
   child: Expr;
+  outArity: number;
 }
 
-export const lex = (child: Expr): Lex => ({type: 'Lex', child});
+export const lex = (child: Expr): Lex => ({
+  type: 'Lex',
+  child,
+  outArity: outArity(child)
+});
 
 // TODO: Eliminate this, and replace with Not(Not(...))?
 export interface Lookahead {
   type: 'Lookahead';
   child: Expr;
+  outArity: number;
 }
 
-export const lookahead = (child: Expr): Lookahead => ({type: 'Lookahead', child});
+export const lookahead = (child: Expr): Lookahead => ({
+  type: 'Lookahead',
+  child,
+  outArity: outArity(child)
+});
 
 export interface Opt {
   type: 'Opt';
@@ -142,9 +157,14 @@ export const range = (lo: number, hi: number): Range => ({type: 'Range', lo, hi}
 export interface Seq {
   type: 'Seq';
   children: Expr[];
+  outArity: number;
 }
 
-export const seq = (children: Expr[]): Seq => ({type: 'Seq', children});
+export const seq = (children: Expr[]): Seq => ({
+  type: 'Seq',
+  children,
+  outArity: children.reduce((acc, child) => acc + outArity(child), 0)
+});
 
 export interface Star {
   type: 'Star';
@@ -247,15 +267,24 @@ export function substituteParams(exp: Expr, actuals: Expr[]) {
     case 'Seq':
       return {
         type: exp.type,
-        children: exp.children.map(c => substituteParams(c, actuals))
+        children: exp.children.map(c => substituteParams(c, actuals)),
+        outArity: exp.outArity
       };
     case 'Lex':
     case 'Lookahead':
+      return {
+        type: exp.type,
+        child: substituteParams(exp.child, actuals),
+        outArity: exp.outArity
+      };
     case 'Not':
     case 'Opt':
     case 'Plus':
     case 'Star':
-      return {type: exp.type, child: substituteParams(exp.child, actuals)};
+      return {
+        type: exp.type,
+        child: substituteParams(exp.child, actuals)
+      };
     case 'Any':
     case 'ApplyGeneralized':
     case 'CaseInsensitive':
@@ -297,8 +326,9 @@ export function rewrite(exp: Expr, actions: RewriteActions): Expr {
 
   switch (exp.type) {
     case 'Alt':
+      return alt(exp.children.map((e: Expr) => rewrite(e, actions)));
     case 'Seq':
-      return {type: exp.type, children: exp.children.map((e: Expr) => rewrite(e, actions))};
+      return seq(exp.children.map((e: Expr) => rewrite(e, actions)));
     case 'Any':
     case 'Apply':
     case 'ApplyGeneralized':
@@ -311,14 +341,20 @@ export function rewrite(exp: Expr, actions: RewriteActions): Expr {
     case 'UnicodeChar':
       return exp;
     case 'Dispatch':
+      // We don't use the constructor here, to avoid type checking issues.
       return {type: exp.type, child: rewrite(exp.child, actions), patterns: exp.patterns};
     case 'Lex':
+      return lex(rewrite(exp.child, actions));
     case 'Lookahead':
+      return lookahead(rewrite(exp.child, actions));
     case 'Not':
+      return not(rewrite(exp.child, actions));
     case 'Opt':
+      return opt(rewrite(exp.child, actions));
     case 'Plus':
+      return plus(rewrite(exp.child, actions));
     case 'Star':
-      return {type: exp.type, child: rewrite(exp.child, actions)};
+      return star(rewrite(exp.child, actions));
     default:
       unreachable(exp, `not handled: ${exp}`);
   }
@@ -370,3 +406,38 @@ export function toString(exp: Expr): string {
       unreachable(exp, `not handled: ${exp}`);
   }
 }
+
+// Returns the number of bindings that `expr` produces in its parent — the
+// "out arity" or "upwards arity". Note that there is potential confusion
+// with iter nodes: they produce a single binding, but an expression like
+// `(letter digit)*` can be said to have "arity 2".
+export function outArity(exp: Expr): number {
+  console.log(JSON.stringify(exp));
+  switch (exp.type) {
+    case 'Alt':
+    case 'Seq':
+    case 'Lex':
+    case 'Lookahead':
+      return exp.outArity;
+    case 'Any':
+    case 'Apply':
+    case 'ApplyGeneralized':
+    case 'CaseInsensitive':
+    case 'End':
+    case 'LiftedTerminal':
+    case 'Param':
+    case 'Range':
+    case 'Terminal':
+    case 'UnicodeChar':
+    case 'Dispatch':
+      return 1;
+    case 'Not':
+      return 0;
+    case 'Opt':
+    case 'Plus':
+    case 'Star':
+      return 1;
+    default:
+      unreachable(exp, `not handled: ${exp}`);
+  }
+}

packages/wasm/test/test-wasm.js

@@ -46,7 +46,7 @@ test('cst returns', async t => {
   let term = root.children[0];
   t.is(term.children.length, 0);
   t.is(term.matchLength, 1);
-  t.is(term._type, 1);
+  t.is(term._typeAndDetails, 1);
   t.true(term.isTerminal());
 
   matcher = await wasmMatcherForGrammar(ohm.grammar('G { start = "a" b\nb = "b" }'));
@@ -974,13 +974,41 @@ test.failing('unicode', async t => {
   t.is(unparse(m), source);
 });
 
-// eslint-disable-next-line ava/no-skip-test
-test.skip('iter node map', async t => {
+test('iter nodes: basic map (star)', async t => {
   const m = await wasmMatcherForGrammar(ohm.grammar('G { Start = (letter digit)* }'));
+  t.is(matchWithInput(m, ''), 1, 'empty input matches');
   t.is(matchWithInput(m, 'a1 b2 c 3'), 1);
+  t.is(m.getCstRoot().children.length, 1);
   const iter = m.getCstRoot().children[0];
   t.deepEqual(
-      iter.map((letter, digit) => `${digit}${letter}`),
+      iter.map((letter, digit) => `${digit.sourceString}${letter.sourceString}`),
       ['1a', '2b', '3c'],
   );
+  t.throws(() => iter.map(() => {}), {message: /bad arity/});
+});
+
+test('iter nodes: basic map (plus)', async t => {
+  const m = await wasmMatcherForGrammar(ohm.grammar('G { Start = (letter digit)+ }'));
+  t.is(matchWithInput(m, ''), 0, 'empty input FAILS');
+  t.is(matchWithInput(m, 'a1 b2 c 3'), 1);
+  t.is(m.getCstRoot().children.length, 1);
+  const iter = m.getCstRoot().children[0];
+  t.deepEqual(
+      iter.map((letter, digit) => `${digit.sourceString}${letter.sourceString}`),
+      ['1a', '2b', '3c'],
+  );
+  t.throws(() => iter.map(() => {}), {message: /bad arity/});
+});
+
+test('iter nodes: basic map (opt)', async t => {
+  const m = await wasmMatcherForGrammar(ohm.grammar('G { Start = (letter digit)? }'));
+  t.is(matchWithInput(m, ''), 1, 'empty input matches');
+  t.is(matchWithInput(m, 'a1'), 1);
+  t.is(m.getCstRoot().children.length, 1);
+  const iter = m.getCstRoot().children[0];
+  t.deepEqual(
+      iter.map((letter, digit) => `${digit.sourceString}${letter.sourceString}`),
+      ['1a'],
+  );
+  t.throws(() => iter.map(() => {}), {message: /bad arity/});
 });