feat(parse): introduce advanced parsing (#20)

BREAKING CHANGE: ParseResult shape has changed

Author: char0n

README.md

@@ -29,6 +29,12 @@
   - [Installation](#installation)
   - [Usage](#usage)
     - [Parsing](#parsing)
+      - [Translators](#translators)
+        - [CST](#cst-translator)
+        - [AST](#ast-translator)
+        - [XML](#xml-translator)
+      - [Statistics](#statistics)
+      - [Tracing](#tracing)
     - [Validation](#validation)
     - [Escaping](#escaping)
     - [Evaluation](#evaluation)
@@ -81,95 +87,96 @@ const parseResult = parse('/foo/bar');
 
 ```
 {
-  result: {
-    success: true,
-    state: 101,
-    stateName: 'MATCH',
-    length: 8,
-    matched: 8,
-    maxMatched: 8,
-    maxTreeDepth: 8,
-    nodeHits: 49
-  },
-  ast: fnast {
-    callbacks: [
-      'json-pointer': [Function: jsonPointer],
-      'reference-token': [Function: referenceToken]
-    ],
-    init: [Function (anonymous)],
-    ruleDefined: [Function (anonymous)],
-    udtDefined: [Function (anonymous)],
-    down: [Function (anonymous)],
-    up: [Function (anonymous)],
-    translate: [Function (anonymous)],
-    setLength: [Function (anonymous)],
-    getLength: [Function (anonymous)],
-    toXml: [Function (anonymous)]
-  },
-  computed: [ 'foo', 'bar' ]
+  result: <ParseResult['result]>,
+  tree: <ParseResult['tree']>,
+  stats: <ParseResult['stats']>,
+  trace: <ParseResult['trace']>,
 }
 ```
 
-###### Evaluating AST as list of unescaped reference tokens
+[TypeScript typings](https://github.com/swaggerexpert/json-pointer/blob/main/types/index.d.ts) are available for all fields attached to parse result object returned by the `parse` function.
+
+##### Translators
+
+`@swaggerexpert/json-pointer` provides several translators to convert the parse result into different tree representations.
 
-One of the ways to interpret the parsed JSON Pointer is to evaluate it as a list of unescaped reference tokens.
+###### CST translator
+
+[Concrete Syntax Tree](https://en.wikipedia.org/wiki/Parse_tree) (Parse tree) representation is available on parse result
+when instance of `CSTTranslator` is provided via a `translator` option to the `parse` function.
+CST is suitable to be consumed by other tools like IDEs, editors, etc...
 
 ```js
-import { parse } from '@swaggerexpert/json-parse';
+import { parse, CSTTranslator } from '@swaggerexpert/json-pointer';
 
-const { computed } = parse('/foo/bar'); // computed = ['foo', 'bar']
+const { tree: CST } = parse('/foo/bar', { translator: new CSTTranslator() });
 ```
 
-###### Interpreting AST as list of entries
+CST tree has a shape documented by [TypeScript typings (CSTTree)](https://github.com/swaggerexpert/json-pointer/blob/main/types/index.d.ts).
+
+###### AST translator
+
+**Default translator**. [Abstract Syntax Tree](https://en.wikipedia.org/wiki/Abstract_syntax_tree) representation is available on parse result
+by default or when instance of `ASTTranslator` is provided via a `translator` option to the `parse` function.
+AST is suitable to be consumed by implementations that need to analyze the structure of the JSON Pointer
+or for building a custom JSON Pointer evaluation engine.
+
+AST of the parsed JSON Pointer is a list of unescaped reference tokens.
 
 ```js
-import { parse } from '@swaggerexpert/json-parse';
+import { parse } from '@swaggerexpert/json-pointer';
 
-const parseResult = parse('/foo/bar');
-const parts = [];
+const { tree: AST } = parse('/foo/bar'); // AST = ['foo', 'bar']
+```
+
+or
+
+```js
+import { parse, ASTTranslator } from '@swaggerexpert/json-poiner';
 
-parseResult.ast.translate(parts);
+const { tree: AST } = parse('/foo/bar', { translator: new ASTTranslator() }); // AST = ['foo', 'bar']
 ```
 
-After running the above code, **parts** variable has the following shape:
+###### XML translator
 
 ```js
-[
-  ['json-pointer', '/foo/bar'],
-  ['reference-token', 'foo'],
-  ['reference-token', 'bar'],
-]
+import { parse, XMLTranslator } from '@swaggerexpert/json-pointer';
+
+const { tree: XML } = parse('$.store.book[0].title', { translator: new XMLTranslator() });
 ```
 
-###### Interpreting AST as XML
+##### Statistics
+
+`parse` function returns additional statistical information about the parsing process.
+Collection of the statistics can be enabled by setting `stats` option to `true`.
 
 ```js
 import { parse } from '@swaggerexpert/json-pointer';
 
-const parseResult = parse('/foo/bar');
-const xml = parseResult.ast.toXml();
+const { stats } = parse('/foo/bar', { stats: true });
+
+stats.displayStats(); // returns operator stats
+stats.displayHits(); // returns rules grouped by hit count
 ```
 
-After running the above code, **xml** variable has the following content:
+##### Tracing
+
+`parse` function returns additional tracing information about the parsing process.
+Tracing can be enabled by setting `trace` option to `true`. Tracing is essential
+for debugging failed matches or analyzing rule execution flow.
+
+```js
+import { parse } from '@swaggerexpert/json-pointer';
 
-```xml
-<?xml version="1.0" encoding="utf-8"?>
-<root nodes="3" characters="8">
-  <!-- input string -->
-  /foo/bar
-  <node name="json-pointer" index="0" length="8">
-    /foo/bar
-    <node name="reference-token" index="1" length="3">
-      foo
-    </node><!-- name="reference-token" -->
-    <node name="reference-token" index="5" length="3">
-      bar
-    </node><!-- name="reference-token" -->
-  </node><!-- name="json-pointer" -->
-</root>
+const { result, trace } = parse('1', { trace: true });
+
+result.success; // returns false
+trace.displayTrace(); // returns trace information
 ```
 
-> NOTE: AST can also be traversed in classical way using [depth first traversal](https://www.tutorialspoint.com/data_structures_algorithms/depth_first_traversal.htm). For more information about this option please refer to [apg-js](https://github.com/ldthomas/apg-js) and [apg-js-examples](https://github.com/ldthomas/apg-js-examples).
+By combining information from `result` and `trace`, it is possible to analyze the parsing process in detail
+and generate a messages like this: `'Syntax error at position 0, expected "/"'`. Please see this
+[test file](https://github.com/swaggerexpert/json-pointer/blob/main/test/parse/trace.js) for more information how to achieve that.
 
 #### Validation
 
@@ -409,43 +416,22 @@ Before using the ApiDOM Evaluation Realm, you need to install the `@swagger-api/
 
 ```js
 import { ObjectElement } from '@swagger-api/apidom-core';
-import { InfoElement } from '@swagger-api/apidom-ns-openapi-3-0'
 import { evaluate } from '@swaggerexpert/json-pointer';
 import ApiDOMEvaluationRealm from '@swaggerexpert/json-pointer/evaluate/realms/apidom';
 
 const objectElement = new ObjectElement({
   a: ['b', 'c']
 });
-const infoElement = InfoElement.refract({
-  contact: {
-    name: 'SwaggerExpert',
-    email: 'contact@swaggerexpert.com'
-  }
-})
-
 
 evaluate(objectElement, '/a/1', { realm: new ApiDOMEvaluationRealm() }); // => StringElement('c')
-evaluate(infoElement, '/contact/name', { realm: new ApiDOMEvaluationRealm() }); // => StringElement('SwaggerExpert')
 ```
 
 ###### Custom Evaluation Realms
 
 The evaluation is designed to support **custom evaluation realms**,
 enabling JSON Pointer evaluation for **non-standard data structures**.
 
-A valid custom evaluation realm must match the structure of the `EvaluationRealm` interface.
-
-```ts
-interface EvaluationRealm {
-  readonly name: string;
-
-  isArray(node: unknown): boolean;
-  isObject(node: unknown): boolean;
-  sizeOf(node: unknown): number;
-  has(node: unknown, referenceToken: string): boolean;
-  evaluate(node: unknown, referenceToken: string): unknown;
-}
-```
+A valid custom evaluation realm must match the structure of the [EvaluationRealm interface](https://github.com/swaggerexpert/json-pointer/blob/main/types/index.d.ts).
 
 One way to create a custom realm is to extend the `EvaluationRealm` class and implement the required methods.
 
@@ -594,7 +580,7 @@ JSON Pointer is defined by the following [ABNF](https://tools.ietf.org/html/rfc5
 ```abnf
 ; JavaScript Object Notation (JSON) Pointer ABNF syntax
 ; https://datatracker.ietf.org/doc/html/rfc6901
-json-pointer    = *( "/" reference-token )
+json-pointer    = *( slash reference-token ) ; MODIFICATION: surrogate text rule used
 reference-token = *( unescaped / escaped )
 unescaped       = %x00-2E / %x30-7D / %x7F-10FFFF
                 ; %x2F ('/') and %x7E ('~') are excluded from 'unescaped'
@@ -606,6 +592,9 @@ array-location  = array-index / array-dash
 array-index     = %x30 / ( %x31-39 *(%x30-39) )
                 ; "0", or digits without a leading "0"
 array-dash      = "-"
+
+; Surrogate named rules
+slash           = "/"
 ```
 
 ## License

src/evaluate/index.js

@@ -10,10 +10,9 @@ import JSONPointerKeyError from '../errors/JSONPointerKeyError.js';
 const evaluate = (
   value,
   jsonPointer,
-  { strictArrays = true, strictObjects = true, evaluator = null, realm = new JSONRealm() } = {},
+  { strictArrays = true, strictObjects = true, realm = new JSONRealm() } = {},
 ) => {
-  const parseOptions = typeof evaluator === 'function' ? { evaluator } : undefined;
-  const { result, computed: referenceTokens } = parse(jsonPointer, parseOptions);
+  const { result, tree: referenceTokens } = parse(jsonPointer);
 
   if (!result.success) {
     throw new JSONPointerEvaluateError(`Invalid JSON Pointer: ${jsonPointer}`, {

src/grammar.bnf

@@ -1,6 +1,6 @@
 ; JavaScript Object Notation (JSON) Pointer ABNF syntax
 ; https://datatracker.ietf.org/doc/html/rfc6901
-json-pointer    = *( "/" reference-token )
+json-pointer    = *( slash reference-token ) ; MODIFICATION: surrogate text rule used
 reference-token = *( unescaped / escaped )
 unescaped       = %x00-2E / %x30-7D / %x7F-10FFFF
                 ; %x2F ('/') and %x7E ('~') are excluded from 'unescaped'
@@ -13,4 +13,5 @@ array-index     = %x30 / ( %x31-39 *(%x30-39) )
                 ; "0", or digits without a leading "0"
 array-dash      = "-"
 
-
+; Surrogate named rules
+slash           = "/"

src/grammar.js

@@ -5,14 +5,14 @@
 export default function grammar(){
   // ```
   // SUMMARY
-  //      rules = 7
+  //      rules = 8
   //       udts = 0
-  //    opcodes = 27
+  //    opcodes = 28
   //        ---   ABNF original opcodes
   //        ALT = 5
   //        CAT = 3
   //        REP = 3
-  //        RNM = 5
+  //        RNM = 6
   //        TLS = 5
   //        TBS = 1
   //        TRG = 5
@@ -34,6 +34,7 @@ export default function grammar(){
   this.rules[4] = { name: 'array-location', lower: 'array-location', index: 4, isBkr: false };
   this.rules[5] = { name: 'array-index', lower: 'array-index', index: 5, isBkr: false };
   this.rules[6] = { name: 'array-dash', lower: 'array-dash', index: 6, isBkr: false };
+  this.rules[7] = { name: 'slash', lower: 'slash', index: 7, isBkr: false };
 
   /* UDTS */
   this.udts = [];
@@ -43,7 +44,7 @@ export default function grammar(){
   this.rules[0].opcodes = [];
   this.rules[0].opcodes[0] = { type: 3, min: 0, max: Infinity };// REP
   this.rules[0].opcodes[1] = { type: 2, children: [2,3] };// CAT
-  this.rules[0].opcodes[2] = { type: 7, string: [47] };// TLS
+  this.rules[0].opcodes[2] = { type: 4, index: 7 };// RNM(slash)
   this.rules[0].opcodes[3] = { type: 4, index: 1 };// RNM(reference-token)
 
   /* reference-token */
@@ -87,12 +88,16 @@ export default function grammar(){
   this.rules[6].opcodes = [];
   this.rules[6].opcodes[0] = { type: 7, string: [45] };// TLS
 
+  /* slash */
+  this.rules[7].opcodes = [];
+  this.rules[7].opcodes[0] = { type: 7, string: [47] };// TLS
+
   // The `toString()` function will display the original grammar file(s) that produced these opcodes.
   this.toString = function toString(){
     let str = "";
     str += "; JavaScript Object Notation (JSON) Pointer ABNF syntax\n";
     str += "; https://datatracker.ietf.org/doc/html/rfc6901\n";
-    str += "json-pointer    = *( \"/\" reference-token )\n";
+    str += "json-pointer    = *( slash reference-token ) ; MODIFICATION: surrogate text rule used\n";
     str += "reference-token = *( unescaped / escaped )\n";
     str += "unescaped       = %x00-2E / %x30-7D / %x7F-10FFFF\n";
     str += "                ; %x2F ('/') and %x7E ('~') are excluded from 'unescaped'\n";
@@ -105,7 +110,8 @@ export default function grammar(){
     str += "                ; \"0\", or digits without a leading \"0\"\n";
     str += "array-dash      = \"-\"\n";
     str += "\n";
-    str += "\n";
+    str += "; Surrogate named rules\n";
+    str += "slash           = \"/\"\n";
     return str;
   }
 }

src/index.js

@@ -4,7 +4,9 @@ export { JSONString, URIFragmentIdentifier };
 
 export { default as Grammar } from './grammar.js';
 export { default as parse } from './parse/index.js';
-export { default as referenceTokenListEvaluator } from './parse/evaluators/reference-token-list.js';
+export { default as CSTTranslator } from './parse/translators/CSTTranslator.js';
+export { default as ASTTranslator } from './parse/translators/ASTTranslator.js';
+export { default as XMLTranslator } from './parse/translators/XMLTranslator.js';
 
 export { default as testJSONPointer } from './test/json-pointer.js';
 export { default as testReferenceToken } from './test/reference-token.js';

src/parse/callbacks/cst.js

@@ -0,0 +1,36 @@
+import { utilities, identifiers } from 'apg-lite';
+
+import JSONPointerParseError from '../../errors/JSONPointerParseError.js';
+
+const cst = (ruleName) => {
+  return (state, chars, phraseIndex, phraseLength, data) => {
+    if (!(typeof data === 'object' && data !== null && !Array.isArray(data))) {
+      throw new JSONPointerParseError("parser's user data must be an object");
+    }
+
+    if (state === identifiers.SEM_PRE) {
+      const node = {
+        type: ruleName,
+        text: utilities.charsToString(chars, phraseIndex, phraseLength),
+        start: phraseIndex,
+        length: phraseLength,
+        children: [],
+      };
+
+      if (data.stack.length > 0) {
+        const parent = data.stack[data.stack.length - 1];
+        parent.children.push(node);
+      } else {
+        data.root = node;
+      }
+
+      data.stack.push(node);
+    }
+
+    if (state === identifiers.SEM_POST) {
+      data.stack.pop();
+    }
+  };
+};
+
+export default cst;