From 8525360243c30434551b329869ddd145598d269e Mon Sep 17 00:00:00 2001 From: Terence Parr Date: Mon, 4 Jul 2022 12:49:34 -0700 Subject: [PATCH 01/13] initial json tree impl Signed-off-by: Terence Parr --- .../test/runtime/java/api/TestJSONTree.java | 52 +++++++++++++++ .../src/org/antlr/v4/runtime/tree/Trees.java | 65 +++++++++++++++++++ 2 files changed, 117 insertions(+) create mode 100644 runtime-testsuite/test/org/antlr/v4/test/runtime/java/api/TestJSONTree.java diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/java/api/TestJSONTree.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/java/api/TestJSONTree.java new file mode 100644 index 0000000000..888a7578ae --- /dev/null +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/java/api/TestJSONTree.java @@ -0,0 +1,52 @@ +package org.antlr.v4.test.runtime.java.api; + +import org.antlr.v4.runtime.ANTLRInputStream; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.tree.ParseTree; +import org.antlr.v4.runtime.tree.Trees; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @since 4.10.2 + */ +public class TestJSONTree { + @Test + public void testEmpty() { + } + + @Test + public void testOneToken() { + String input = "8"; + VisitorCalcLexer lexer = new VisitorCalcLexer(new ANTLRInputStream(input)); + VisitorCalcParser parser = new VisitorCalcParser(new CommonTokenStream(lexer)); + + ParseTree t = parser.expr(); + String result = Trees.toJSONTree(t, parser); + assertEquals("{\"expr\":[{\"idx\":\"0\",\"text\":\"8\"}]}", result); + } + + @Test + public void testOneRuleOneToken() { + String input = "99"; + VisitorCalcLexer lexer = new VisitorCalcLexer(new ANTLRInputStream(input)); + VisitorCalcParser parser = new VisitorCalcParser(new CommonTokenStream(lexer)); + + ParseTree t = parser.s(); + String result = Trees.toJSONTree(t, parser); + assertEquals("{\"s\":[{\"expr\":[{\"idx\":\"0\",\"text\":\"99\"}]},{\"idx\":\"1\",\"text\":\"\"}]}", result); + } + + @Test + public void testExpr() { + String input = "1 + 2"; + VisitorCalcLexer lexer = new VisitorCalcLexer(new ANTLRInputStream(input)); + VisitorCalcParser parser = new VisitorCalcParser(new CommonTokenStream(lexer)); + + ParseTree t = parser.s(); + String result = Trees.toJSONTree(t, parser); + System.out.println(result); + assertEquals("{\"s\":[{\"expr\":[{\"expr\":[{\"idx\":\"0\",\"text\":\"1\"}]},{\"idx\":\"2\",\"text\":\"+\"},{\"expr\":[{\"idx\":\"4\",\"text\":\"2\"}]}]},{\"idx\":\"5\",\"text\":\"\"}]}", result); + } +} diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/Trees.java b/runtime/Java/src/org/antlr/v4/runtime/tree/Trees.java index bf6531b7aa..6a0c24b3a7 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/Trees.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/Trees.java @@ -61,6 +61,41 @@ public static String toStringTree(final Tree t, final List ruleNames) { return buf.toString(); } + /** Print out a whole tree in JSON form. {@link #getNodeText} is used on the + * node payloads to get the text for the nodes. Detect + * parse trees and extract data appropriately. + * @since 4.10.2 + */ + public static String toJSONTree(Tree t, Parser recog) { + String[] ruleNames = recog != null ? recog.getRuleNames() : null; + List ruleNamesList = ruleNames != null ? Arrays.asList(ruleNames) : null; + return toJSONTree(t, ruleNamesList); + } + + /** Print out a whole tree in JSON form. {@link #getNodeText} is used on the + * node payloads to get the text for the nodes. Detect + * parse trees and extract data appropriately. + * @since 4.10.2 + */ + public static String toJSONTree(final Tree t, final List ruleNames) { + StringBuilder buf = new StringBuilder(); + String s = Utils.escapeWhitespace(getNodeText(t, ruleNames), false); + if ( t.getChildCount()==0 ) { + return getJSONNodeText(t, ruleNames); + } + buf.append("{"); + s = Utils.escapeWhitespace(getJSONNodeText(t, ruleNames), false); + buf.append(s); + buf.append(":["); + for (int i = 0; i0 ) buf.append(','); + buf.append(toJSONTree(t.getChild(i), ruleNames)); + } + buf.append("]"); + buf.append("}"); + return buf.toString(); + } + public static String getNodeText(Tree t, Parser recog) { String[] ruleNames = recog != null ? recog.getRuleNames() : null; List ruleNamesList = ruleNames != null ? Arrays.asList(ruleNames) : null; @@ -97,6 +132,36 @@ else if ( t instanceof TerminalNode) { return t.getPayload().toString(); } + /** @since 4.10.2 */ + public static String getJSONNodeText(Tree t, List ruleNames) { + if ( ruleNames!=null ) { + if ( t instanceof RuleContext ) { + int ruleIndex = ((RuleContext)t).getRuleContext().getRuleIndex(); + String ruleName = ruleNames.get(ruleIndex); + int altNumber = ((RuleContext) t).getAltNumber(); + if ( altNumber!=ATN.INVALID_ALT_NUMBER ) { + return ruleName+":"+altNumber; + } + return '"'+ruleName+'"'; + } + else if ( t instanceof ErrorNode) { + return t.toString(); + } + else if ( t instanceof TerminalNode) { + Token symbol = ((TerminalNode)t).getSymbol(); + if (symbol != null) { + return String.format("{\"idx\":\"%d\",\"text\":\"%s\"}", symbol.getTokenIndex(), symbol.getText()); + } + } + } + // no recog for rule names + Object payload = t.getPayload(); + if ( payload instanceof Token ) { + return ((Token)payload).getText(); + } + return t.getPayload().toString(); + } + /** Return ordered list of all children of this node */ public static List getChildren(Tree t) { List kids = new ArrayList(); From d13e826663bd589129fb52427946f09b20867a3b Mon Sep 17 00:00:00 2001 From: Terence Parr Date: Mon, 4 Jul 2022 12:59:27 -0700 Subject: [PATCH 02/13] improve literals Signed-off-by: Terence Parr --- .../v4/test/runtime/java/api/TestJSONTree.java | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/java/api/TestJSONTree.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/java/api/TestJSONTree.java index 888a7578ae..8de27a819e 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/java/api/TestJSONTree.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/java/api/TestJSONTree.java @@ -24,7 +24,9 @@ public void testOneToken() { ParseTree t = parser.expr(); String result = Trees.toJSONTree(t, parser); - assertEquals("{\"expr\":[{\"idx\":\"0\",\"text\":\"8\"}]}", result); + String expected = "{'expr':[{'idx':'0','text':'8'}]}"; + expected = expected.replace('\'', '"'); + assertEquals(expected, result); } @Test @@ -35,7 +37,9 @@ public void testOneRuleOneToken() { ParseTree t = parser.s(); String result = Trees.toJSONTree(t, parser); - assertEquals("{\"s\":[{\"expr\":[{\"idx\":\"0\",\"text\":\"99\"}]},{\"idx\":\"1\",\"text\":\"\"}]}", result); + String expected = "{'s':[{'expr':[{'idx':'0','text':'99'}]},{'idx':'1','text':''}]}"; + expected = expected.replace('\'', '"'); + assertEquals(expected, result); } @Test @@ -47,6 +51,11 @@ public void testExpr() { ParseTree t = parser.s(); String result = Trees.toJSONTree(t, parser); System.out.println(result); - assertEquals("{\"s\":[{\"expr\":[{\"expr\":[{\"idx\":\"0\",\"text\":\"1\"}]},{\"idx\":\"2\",\"text\":\"+\"},{\"expr\":[{\"idx\":\"4\",\"text\":\"2\"}]}]},{\"idx\":\"5\",\"text\":\"\"}]}", result); + String expected = + "{'s':[{'expr':[{'expr':[{'idx':'0','text':'1'}]},{'idx':'2','text':'+'},"+ + "{'expr':[{'idx':'4','text':'2'}]}]},"+ + "{'idx':'5','text':''}]}"; + expected = expected.replace('\'', '"'); + assertEquals(expected, result); } } From 82d84900b4f328f43631504a7d7f72e7bb05ab68 Mon Sep 17 00:00:00 2001 From: Terence Parr Date: Mon, 4 Jul 2022 13:03:53 -0700 Subject: [PATCH 03/13] test optional start rule Signed-off-by: Terence Parr --- .../org/antlr/v4/test/runtime/java/api/TestJSONTree.java | 9 +++++++++ .../org/antlr/v4/test/runtime/java/api/VisitorCalc.g4 | 1 + 2 files changed, 10 insertions(+) diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/java/api/TestJSONTree.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/java/api/TestJSONTree.java index 8de27a819e..533858e7d9 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/java/api/TestJSONTree.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/java/api/TestJSONTree.java @@ -14,6 +14,15 @@ public class TestJSONTree { @Test public void testEmpty() { + String input = ""; + VisitorCalcLexer lexer = new VisitorCalcLexer(new ANTLRInputStream(input)); + VisitorCalcParser parser = new VisitorCalcParser(new CommonTokenStream(lexer)); + + ParseTree t = parser.s(); // rule s can match nothing + String result = Trees.toJSONTree(t, parser); + String expected = "'s'"; + expected = expected.replace('\'', '"'); + assertEquals(expected, result); } @Test diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/java/api/VisitorCalc.g4 b/runtime-testsuite/test/org/antlr/v4/test/runtime/java/api/VisitorCalc.g4 index 6ca1c2e835..7d9cc92636 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/java/api/VisitorCalc.g4 +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/java/api/VisitorCalc.g4 @@ -2,6 +2,7 @@ grammar VisitorCalc; s : expr EOF + | ; expr From 41a46a59662a596406e2820ee6f6fa2152efbe25 Mon Sep 17 00:00:00 2001 From: Terence Parr Date: Mon, 4 Jul 2022 13:51:53 -0700 Subject: [PATCH 04/13] use token index only Signed-off-by: Terence Parr --- .../test/runtime/java/api/TestJSONTree.java | 24 ++++++++-- .../v4/test/runtime/java/api/VisitorCalc.g4 | 3 ++ .../src/org/antlr/v4/runtime/tree/Trees.java | 47 ++++++++++--------- 3 files changed, 47 insertions(+), 27 deletions(-) diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/java/api/TestJSONTree.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/java/api/TestJSONTree.java index 533858e7d9..481e03ade2 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/java/api/TestJSONTree.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/java/api/TestJSONTree.java @@ -33,7 +33,7 @@ public void testOneToken() { ParseTree t = parser.expr(); String result = Trees.toJSONTree(t, parser); - String expected = "{'expr':[{'idx':'0','text':'8'}]}"; + String expected = "{\"expr\":[0]}"; expected = expected.replace('\'', '"'); assertEquals(expected, result); } @@ -46,7 +46,7 @@ public void testOneRuleOneToken() { ParseTree t = parser.s(); String result = Trees.toJSONTree(t, parser); - String expected = "{'s':[{'expr':[{'idx':'0','text':'99'}]},{'idx':'1','text':''}]}"; + String expected = "{\"s\":[{\"expr\":[0]},1]}"; expected = expected.replace('\'', '"'); assertEquals(expected, result); } @@ -61,10 +61,24 @@ public void testExpr() { String result = Trees.toJSONTree(t, parser); System.out.println(result); String expected = - "{'s':[{'expr':[{'expr':[{'idx':'0','text':'1'}]},{'idx':'2','text':'+'},"+ - "{'expr':[{'idx':'4','text':'2'}]}]},"+ - "{'idx':'5','text':''}]}"; + "{\"s\":[{\"expr\":[{\"expr\":[0]},2,{\"expr\":[4]}]},5]}"; expected = expected.replace('\'', '"'); assertEquals(expected, result); } + + @Test + public void testMismatchedToken() { + String input = "f("; + VisitorCalcLexer lexer = new VisitorCalcLexer(new ANTLRInputStream(input)); + VisitorCalcParser parser = new VisitorCalcParser(new CommonTokenStream(lexer)); + parser.removeErrorListeners(); // Turn off error msgs. + + ParseTree t = parser.expr(); + String result = Trees.toJSONTree(t, parser); + String expected = + "{\"expr\":[0,1,{\"error\":\"\"}]}"; + expected = expected.replace('\'', '"'); + expected = expected.replace("\")\"", "')'"); // undo error msg tweak + assertEquals(expected, result); + } } diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/java/api/VisitorCalc.g4 b/runtime-testsuite/test/org/antlr/v4/test/runtime/java/api/VisitorCalc.g4 index 7d9cc92636..899287828e 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/java/api/VisitorCalc.g4 +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/java/api/VisitorCalc.g4 @@ -7,11 +7,14 @@ s expr : INT # number + | ID # var + | ID '(' ')' # func | expr (MUL | DIV) expr # multiply | expr (ADD | SUB) expr # add ; INT : [0-9]+; +ID : [a-zA-Z_]+ ; MUL : '*'; DIV : '/'; ADD : '+'; diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/Trees.java b/runtime/Java/src/org/antlr/v4/runtime/tree/Trees.java index 6a0c24b3a7..6214a57046 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/Trees.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/Trees.java @@ -134,32 +134,35 @@ else if ( t instanceof TerminalNode) { /** @since 4.10.2 */ public static String getJSONNodeText(Tree t, List ruleNames) { - if ( ruleNames!=null ) { - if ( t instanceof RuleContext ) { - int ruleIndex = ((RuleContext)t).getRuleContext().getRuleIndex(); - String ruleName = ruleNames.get(ruleIndex); - int altNumber = ((RuleContext) t).getAltNumber(); - if ( altNumber!=ATN.INVALID_ALT_NUMBER ) { - return ruleName+":"+altNumber; - } - return '"'+ruleName+'"'; - } - else if ( t instanceof ErrorNode) { - return t.toString(); + if ( ruleNames==null ) { + return null; + } + if ( t instanceof RuleContext ) { + int ruleIndex = ((RuleContext)t).getRuleContext().getRuleIndex(); + String ruleName = ruleNames.get(ruleIndex); + int altNumber = ((RuleContext) t).getAltNumber(); + if ( altNumber!=ATN.INVALID_ALT_NUMBER ) { + return '"'+ruleName+":"+altNumber+'"'; } - else if ( t instanceof TerminalNode) { - Token symbol = ((TerminalNode)t).getSymbol(); - if (symbol != null) { - return String.format("{\"idx\":\"%d\",\"text\":\"%s\"}", symbol.getTokenIndex(), symbol.getText()); - } + return '"'+ruleName+'"'; + } + else if ( t instanceof ErrorNode) { + Token symbol = ((TerminalNode)t).getSymbol(); + if (symbol != null) { + String txt = String.format("{\"idx\":\"%d\",\"text\":\"%s\"}", + symbol.getTokenIndex(), symbol.getText()); + return "{\"error\":\"" + symbol.getText() + "\"}"; } + return "{\"error\":\""+t.getPayload().toString()+"\"}"; } - // no recog for rule names - Object payload = t.getPayload(); - if ( payload instanceof Token ) { - return ((Token)payload).getText(); + else if ( t instanceof TerminalNode) { + Token symbol = ((TerminalNode)t).getSymbol(); + if (symbol != null) { + return String.valueOf(symbol.getTokenIndex()); + } + return "-1"; } - return t.getPayload().toString(); + return ""; } /** Return ordered list of all children of this node */ From 19b46df724b3586e26ba7b088a663d05f3fa42ab Mon Sep 17 00:00:00 2001 From: Terence Parr Date: Mon, 4 Jul 2022 14:37:30 -0700 Subject: [PATCH 05/13] dump it all now Signed-off-by: Terence Parr --- .../test/runtime/java/api/TestJSONTree.java | 22 +++---- .../src/org/antlr/v4/runtime/misc/Utils.java | 10 +++ .../src/org/antlr/v4/runtime/tree/Trees.java | 65 ++++++++++++++----- 3 files changed, 69 insertions(+), 28 deletions(-) diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/java/api/TestJSONTree.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/java/api/TestJSONTree.java index 481e03ade2..c9435f2e82 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/java/api/TestJSONTree.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/java/api/TestJSONTree.java @@ -19,8 +19,8 @@ public void testEmpty() { VisitorCalcParser parser = new VisitorCalcParser(new CommonTokenStream(lexer)); ParseTree t = parser.s(); // rule s can match nothing - String result = Trees.toJSONTree(t, parser); - String expected = "'s'"; + String result = Trees.toJSON(t, parser); + String expected = "{'tree':'0'}"; expected = expected.replace('\'', '"'); assertEquals(expected, result); } @@ -32,8 +32,8 @@ public void testOneToken() { VisitorCalcParser parser = new VisitorCalcParser(new CommonTokenStream(lexer)); ParseTree t = parser.expr(); - String result = Trees.toJSONTree(t, parser); - String expected = "{\"expr\":[0]}"; + String result = Trees.toJSON(t, parser); + String expected = "{'tree':{'1':[0]}}"; expected = expected.replace('\'', '"'); assertEquals(expected, result); } @@ -45,8 +45,8 @@ public void testOneRuleOneToken() { VisitorCalcParser parser = new VisitorCalcParser(new CommonTokenStream(lexer)); ParseTree t = parser.s(); - String result = Trees.toJSONTree(t, parser); - String expected = "{\"s\":[{\"expr\":[0]},1]}"; + String result = Trees.toJSON(t, parser); + String expected = "{'tree':{'0':[{'1':[0]},1]}}"; expected = expected.replace('\'', '"'); assertEquals(expected, result); } @@ -58,10 +58,10 @@ public void testExpr() { VisitorCalcParser parser = new VisitorCalcParser(new CommonTokenStream(lexer)); ParseTree t = parser.s(); - String result = Trees.toJSONTree(t, parser); + String result = Trees.toJSON(t, parser); System.out.println(result); String expected = - "{\"s\":[{\"expr\":[{\"expr\":[0]},2,{\"expr\":[4]}]},5]}"; + "{'s':[{'expr':[{'expr':[0]},2,{'expr':[4]}]},5]}"; expected = expected.replace('\'', '"'); assertEquals(expected, result); } @@ -74,11 +74,11 @@ public void testMismatchedToken() { parser.removeErrorListeners(); // Turn off error msgs. ParseTree t = parser.expr(); - String result = Trees.toJSONTree(t, parser); + String result = Trees.toJSON(t, parser); String expected = - "{\"expr\":[0,1,{\"error\":\"\"}]}"; + "{'expr':[0,1,{'error':''}]}"; expected = expected.replace('\'', '"'); - expected = expected.replace("\")\"", "')'"); // undo error msg tweak + expected = expected.replace("')'", "')'"); // undo error msg tweak assertEquals(expected, result); } } diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/Utils.java b/runtime/Java/src/org/antlr/v4/runtime/misc/Utils.java index e4d91a1910..56817d6a6b 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/Utils.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/Utils.java @@ -70,6 +70,16 @@ public static String escapeWhitespace(String s, boolean escapeSpaces) { return buf.toString(); } + /** @since 4.10.2 */ + public static String escapeJSONString(String s) { + s = s.replace("\\", "\\\\"); + s = s.replace("\"", "\\\""); + s = s.replace("\n", "\\n"); + s = s.replace("\r", "\\r"); + s = s.replace("\t", "\\t"); + return s; + } + public static void writeFile(String fileName, String content) throws IOException { writeFile(fileName, content, null); } diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/Trees.java b/runtime/Java/src/org/antlr/v4/runtime/tree/Trees.java index 6214a57046..e92d492fa2 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/Trees.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/Trees.java @@ -6,11 +6,7 @@ package org.antlr.v4.runtime.tree; -import org.antlr.v4.runtime.CommonToken; -import org.antlr.v4.runtime.Parser; -import org.antlr.v4.runtime.ParserRuleContext; -import org.antlr.v4.runtime.RuleContext; -import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.*; import org.antlr.v4.runtime.atn.ATN; import org.antlr.v4.runtime.misc.Interval; import org.antlr.v4.runtime.misc.Predicate; @@ -63,13 +59,53 @@ public static String toStringTree(final Tree t, final List ruleNames) { /** Print out a whole tree in JSON form. {@link #getNodeText} is used on the * node payloads to get the text for the nodes. Detect - * parse trees and extract data appropriately. + * parse trees and extract data appropriately. Include rulenames, input, tokens. * @since 4.10.2 */ - public static String toJSONTree(Tree t, Parser recog) { + public static String toJSON(Tree t, Parser recog) { String[] ruleNames = recog != null ? recog.getRuleNames() : null; - List ruleNamesList = ruleNames != null ? Arrays.asList(ruleNames) : null; - return toJSONTree(t, ruleNamesList); + if ( ruleNames==null ) { + return null; + } + List ruleNamesList = Arrays.asList(ruleNames); + + TokenStream tokenStream = recog.getInputStream(); + CharStream inputStream = tokenStream.getTokenSource().getInputStream(); + Interval allchar = Interval.of(0, inputStream.size() - 1); + String input = inputStream.getText(allchar); + input = Utils.escapeJSONString(input); +// List tokens; +// if ( tokenStream instanceof CommonTokenStream ) { +// tokens = ((CommonTokenStream) tokenStream).getTokens(0, tokenStream.size()-1); +// } +// else { +// } + List tokenStrings = new ArrayList<>(); + for (int i = 0; i < tokenStream.size(); i++) { + Token tok = tokenStream.get(i); + String s = String.format("{\"type\":%d,\"line\":%d,\"pos\":%d,\"channel\":%d,\"text\":\"%s\"}\n", + tok.getType(), tok.getLine(), tok.getCharPositionInLine(), tok.getChannel(), + Utils.escapeJSONString(tok.getText())); + tokenStrings.add(s); + } + String tree = toJSONTree(t, ruleNamesList); + + StringBuilder buf = new StringBuilder(); + buf.append("{"); + buf.append("\"rules\":[\""); + buf.append(String.join("\",\"", ruleNames)); + buf.append("\"],\n"); + buf.append("\"input\":\""); + buf.append(input); + buf.append("\",\n"); + buf.append("\"tokens\":["); + buf.append(String.join(",", tokenStrings)); + buf.append("],\n"); + buf.append("\"tree\":"); + buf.append(tree); + buf.append("}"); + + return buf.toString(); } /** Print out a whole tree in JSON form. {@link #getNodeText} is used on the @@ -79,13 +115,11 @@ public static String toJSONTree(Tree t, Parser recog) { */ public static String toJSONTree(final Tree t, final List ruleNames) { StringBuilder buf = new StringBuilder(); - String s = Utils.escapeWhitespace(getNodeText(t, ruleNames), false); if ( t.getChildCount()==0 ) { return getJSONNodeText(t, ruleNames); } buf.append("{"); - s = Utils.escapeWhitespace(getJSONNodeText(t, ruleNames), false); - buf.append(s); + buf.append(getJSONNodeText(t, ruleNames)); buf.append(":["); for (int i = 0; i0 ) buf.append(','); @@ -139,18 +173,15 @@ public static String getJSONNodeText(Tree t, List ruleNames) { } if ( t instanceof RuleContext ) { int ruleIndex = ((RuleContext)t).getRuleContext().getRuleIndex(); - String ruleName = ruleNames.get(ruleIndex); int altNumber = ((RuleContext) t).getAltNumber(); if ( altNumber!=ATN.INVALID_ALT_NUMBER ) { - return '"'+ruleName+":"+altNumber+'"'; + return String.format("\"%d:%d\"",ruleIndex,altNumber); } - return '"'+ruleName+'"'; + return String.format("\"%d\"",ruleIndex); } else if ( t instanceof ErrorNode) { Token symbol = ((TerminalNode)t).getSymbol(); if (symbol != null) { - String txt = String.format("{\"idx\":\"%d\",\"text\":\"%s\"}", - symbol.getTokenIndex(), symbol.getText()); return "{\"error\":\"" + symbol.getText() + "\"}"; } return "{\"error\":\""+t.getPayload().toString()+"\"}"; From ce5da1950aaab44734c21d346784e27bed766f85 Mon Sep 17 00:00:00 2001 From: Terence Parr Date: Mon, 4 Jul 2022 15:33:04 -0700 Subject: [PATCH 06/13] Move tests to descriptors. only works for java now. Signed-off-by: Terence Parr --- runtime-testsuite/pom.xml | 4 +- .../descriptors/ParseTrees/JsonEmpty.txt | 45 ++++++++++ .../descriptors/ParseTrees/JsonExpr.txt | 48 +++++++++++ .../ParseTrees/JsonMismatchedToken.txt | 51 +++++++++++ .../descriptors/ParseTrees/JsonOneToken.txt | 46 ++++++++++ .../ParseTrees/JsonOneTokenOneRule.txt | 46 ++++++++++ .../v4/test/runtime/templates/Java.test.stg | 2 + .../test/runtime/java/api/TestJSONTree.java | 84 ------------------- 8 files changed, 240 insertions(+), 86 deletions(-) create mode 100644 runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonEmpty.txt create mode 100644 runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonExpr.txt create mode 100644 runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonMismatchedToken.txt create mode 100644 runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonOneToken.txt create mode 100644 runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonOneTokenOneRule.txt delete mode 100644 runtime-testsuite/test/org/antlr/v4/test/runtime/java/api/TestJSONTree.java diff --git a/runtime-testsuite/pom.xml b/runtime-testsuite/pom.xml index 189714b33b..a7e0819c37 100644 --- a/runtime-testsuite/pom.xml +++ b/runtime-testsuite/pom.xml @@ -117,8 +117,8 @@ maven-compiler-plugin 8 - 9 - 9 + 11 + 11 diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonEmpty.txt b/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonEmpty.txt new file mode 100644 index 0000000000..c23bb05bc2 --- /dev/null +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonEmpty.txt @@ -0,0 +1,45 @@ +[type] +Parser + +[grammar] +grammar VisitorCalc; + +s +@init { + +} +@after { + +} + : expr EOF + | + ; + +expr + : INT # number + | ID # var + | ID '(' ')' # func + | expr (MUL | DIV) expr # multiply + | expr (ADD | SUB) expr # add + ; + +INT : [0-9]+; +ID : [a-zA-Z_]+ ; +MUL : '*'; +DIV : '/'; +ADD : '+'; +SUB : '-'; +WS : [ \t]+ -> channel(HIDDEN); + +[start] +s + +[input] + + +[output] +{"rules":["s","expr"], +"input":"", +"tokens":[{"type":-1,"line":1,"pos":0,"channel":0,"text":""} +], +"tree":"0"} diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonExpr.txt b/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonExpr.txt new file mode 100644 index 0000000000..df40c546f2 --- /dev/null +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonExpr.txt @@ -0,0 +1,48 @@ +[type] +Parser + +[grammar] +grammar VisitorCalc; + +s +@init { + +} +@after { + +} + : expr EOF + | + ; + +expr + : INT # number + | ID # var + | ID '(' ')' # func + | expr (MUL | DIV) expr # multiply + | expr (ADD | SUB) expr # add + ; + +INT : [0-9]+; +ID : [a-zA-Z_]+ ; +MUL : '*'; +DIV : '/'; +ADD : '+'; +SUB : '-'; +WS : [ \t]+ -> channel(HIDDEN); + +[start] +s + +[input] +1+2 + +[output] +{"rules":["s","expr"], +"input":"1+2", +"tokens":[{"type":3,"line":1,"pos":0,"channel":0,"text":"1"} +,{"type":7,"line":1,"pos":1,"channel":0,"text":"+"} +,{"type":3,"line":1,"pos":2,"channel":0,"text":"2"} +,{"type":-1,"line":1,"pos":3,"channel":0,"text":""} +], +"tree":{"0":[{"1":[{"1":[0]},1,{"1":[2]}]},3]}} \ No newline at end of file diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonMismatchedToken.txt b/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonMismatchedToken.txt new file mode 100644 index 0000000000..f7746ee308 --- /dev/null +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonMismatchedToken.txt @@ -0,0 +1,51 @@ +[type] +Parser + +[grammar] +grammar VisitorCalc; + +s + : expr EOF + | + ; + +expr +@init { + +} +@after { + +} + : INT # number + | ID # var + | ID '(' ')' # func + | expr (MUL | DIV) expr # multiply + | expr (ADD | SUB) expr # add + ; + +INT : [0-9]+; +ID : [a-zA-Z_]+ ; +MUL : '*'; +DIV : '/'; +ADD : '+'; +SUB : '-'; +WS : [ \t]+ -> channel(HIDDEN); + +[start] +expr + +[input] +f( + +[output] +{"rules":["s","expr"], +"input":"f(", +"tokens":[{"type":4,"line":1,"pos":0,"channel":0,"text":"f"} +,{"type":1,"line":1,"pos":1,"channel":0,"text":"("} +,{"type":-1,"line":1,"pos":2,"channel":0,"text":""} +], +"tree":{"1":[0,1,{"error":""}]}} + +[errors] +"""line 1:2 missing ')' at '' +""" \ No newline at end of file diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonOneToken.txt b/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonOneToken.txt new file mode 100644 index 0000000000..b656aca1b5 --- /dev/null +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonOneToken.txt @@ -0,0 +1,46 @@ +[type] +Parser + +[grammar] +grammar VisitorCalc; + +s + : expr EOF + | + ; + +expr +@init { + +} +@after { + +} + : INT # number + | ID # var + | ID '(' ')' # func + | expr (MUL | DIV) expr # multiply + | expr (ADD | SUB) expr # add + ; + +INT : [0-9]+; +ID : [a-zA-Z_]+ ; +MUL : '*'; +DIV : '/'; +ADD : '+'; +SUB : '-'; +WS : [ \t]+ -> channel(HIDDEN); + +[start] +expr + +[input] +8 + +[output] +{"rules":["s","expr"], +"input":"8", +"tokens":[{"type":3,"line":1,"pos":0,"channel":0,"text":"8"} +,{"type":-1,"line":1,"pos":1,"channel":0,"text":""} +], +"tree":{"1":[0]}} \ No newline at end of file diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonOneTokenOneRule.txt b/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonOneTokenOneRule.txt new file mode 100644 index 0000000000..7c85527f8c --- /dev/null +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonOneTokenOneRule.txt @@ -0,0 +1,46 @@ +[type] +Parser + +[grammar] +grammar VisitorCalc; + +s +@init { + +} +@after { + +} + : expr EOF + | + ; + +expr + : INT # number + | ID # var + | ID '(' ')' # func + | expr (MUL | DIV) expr # multiply + | expr (ADD | SUB) expr # add + ; + +INT : [0-9]+; +ID : [a-zA-Z_]+ ; +MUL : '*'; +DIV : '/'; +ADD : '+'; +SUB : '-'; +WS : [ \t]+ -> channel(HIDDEN); + +[start] +s + +[input] +99 + +[output] +{"rules":["s","expr"], +"input":"99", +"tokens":[{"type":3,"line":1,"pos":0,"channel":0,"text":"99"} +,{"type":-1,"line":1,"pos":2,"channel":0,"text":""} +], +"tree":{"0":[{"1":[0]},1]}} \ No newline at end of file diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Java.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Java.test.stg index b1aa46db10..a58481e402 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Java.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Java.test.stg @@ -56,6 +56,8 @@ BailErrorStrategy() ::= <%setErrorHandler(new BailErrorStrategy());%> ToStringTree(s) ::= <%.toStringTree(this)%> +ToJSON(s) ::= <%Trees.toJSON(,this)%> + Column() ::= "this.getCharPositionInLine()" Text() ::= "this.getText()" diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/java/api/TestJSONTree.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/java/api/TestJSONTree.java deleted file mode 100644 index c9435f2e82..0000000000 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/java/api/TestJSONTree.java +++ /dev/null @@ -1,84 +0,0 @@ -package org.antlr.v4.test.runtime.java.api; - -import org.antlr.v4.runtime.ANTLRInputStream; -import org.antlr.v4.runtime.CommonTokenStream; -import org.antlr.v4.runtime.tree.ParseTree; -import org.antlr.v4.runtime.tree.Trees; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * @since 4.10.2 - */ -public class TestJSONTree { - @Test - public void testEmpty() { - String input = ""; - VisitorCalcLexer lexer = new VisitorCalcLexer(new ANTLRInputStream(input)); - VisitorCalcParser parser = new VisitorCalcParser(new CommonTokenStream(lexer)); - - ParseTree t = parser.s(); // rule s can match nothing - String result = Trees.toJSON(t, parser); - String expected = "{'tree':'0'}"; - expected = expected.replace('\'', '"'); - assertEquals(expected, result); - } - - @Test - public void testOneToken() { - String input = "8"; - VisitorCalcLexer lexer = new VisitorCalcLexer(new ANTLRInputStream(input)); - VisitorCalcParser parser = new VisitorCalcParser(new CommonTokenStream(lexer)); - - ParseTree t = parser.expr(); - String result = Trees.toJSON(t, parser); - String expected = "{'tree':{'1':[0]}}"; - expected = expected.replace('\'', '"'); - assertEquals(expected, result); - } - - @Test - public void testOneRuleOneToken() { - String input = "99"; - VisitorCalcLexer lexer = new VisitorCalcLexer(new ANTLRInputStream(input)); - VisitorCalcParser parser = new VisitorCalcParser(new CommonTokenStream(lexer)); - - ParseTree t = parser.s(); - String result = Trees.toJSON(t, parser); - String expected = "{'tree':{'0':[{'1':[0]},1]}}"; - expected = expected.replace('\'', '"'); - assertEquals(expected, result); - } - - @Test - public void testExpr() { - String input = "1 + 2"; - VisitorCalcLexer lexer = new VisitorCalcLexer(new ANTLRInputStream(input)); - VisitorCalcParser parser = new VisitorCalcParser(new CommonTokenStream(lexer)); - - ParseTree t = parser.s(); - String result = Trees.toJSON(t, parser); - System.out.println(result); - String expected = - "{'s':[{'expr':[{'expr':[0]},2,{'expr':[4]}]},5]}"; - expected = expected.replace('\'', '"'); - assertEquals(expected, result); - } - - @Test - public void testMismatchedToken() { - String input = "f("; - VisitorCalcLexer lexer = new VisitorCalcLexer(new ANTLRInputStream(input)); - VisitorCalcParser parser = new VisitorCalcParser(new CommonTokenStream(lexer)); - parser.removeErrorListeners(); // Turn off error msgs. - - ParseTree t = parser.expr(); - String result = Trees.toJSON(t, parser); - String expected = - "{'expr':[0,1,{'error':''}]}"; - expected = expected.replace('\'', '"'); - expected = expected.replace("')'", "')'"); // undo error msg tweak - assertEquals(expected, result); - } -} From 02f08fcb957669064ec69ae2421e2f44d6435e28 Mon Sep 17 00:00:00 2001 From: Terence Parr Date: Mon, 4 Jul 2022 15:36:55 -0700 Subject: [PATCH 07/13] skip non-java JSON tests for now Signed-off-by: Terence Parr --- .../runtime/descriptors/ParseTrees/JsonEmpty.txt | 11 +++++++++++ .../runtime/descriptors/ParseTrees/JsonExpr.txt | 13 ++++++++++++- .../descriptors/ParseTrees/JsonMismatchedToken.txt | 13 ++++++++++++- .../runtime/descriptors/ParseTrees/JsonOneToken.txt | 13 ++++++++++++- .../descriptors/ParseTrees/JsonOneTokenOneRule.txt | 13 ++++++++++++- 5 files changed, 59 insertions(+), 4 deletions(-) diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonEmpty.txt b/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonEmpty.txt index c23bb05bc2..4f08a8d403 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonEmpty.txt +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonEmpty.txt @@ -43,3 +43,14 @@ s "tokens":[{"type":-1,"line":1,"pos":0,"channel":0,"text":""} ], "tree":"0"} + +[skip] +Cpp +CSharp +Dart +Go +JavaScript +PHP +Swift +Python2 +Python3 diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonExpr.txt b/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonExpr.txt index df40c546f2..a3a670b573 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonExpr.txt +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonExpr.txt @@ -45,4 +45,15 @@ s ,{"type":3,"line":1,"pos":2,"channel":0,"text":"2"} ,{"type":-1,"line":1,"pos":3,"channel":0,"text":""} ], -"tree":{"0":[{"1":[{"1":[0]},1,{"1":[2]}]},3]}} \ No newline at end of file +"tree":{"0":[{"1":[{"1":[0]},1,{"1":[2]}]},3]}} + +[skip] +Cpp +CSharp +Dart +Go +JavaScript +PHP +Swift +Python2 +Python3 diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonMismatchedToken.txt b/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonMismatchedToken.txt index f7746ee308..6016cb7e29 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonMismatchedToken.txt +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonMismatchedToken.txt @@ -48,4 +48,15 @@ f( [errors] """line 1:2 missing ')' at '' -""" \ No newline at end of file +""" + +[skip] +Cpp +CSharp +Dart +Go +JavaScript +PHP +Swift +Python2 +Python3 diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonOneToken.txt b/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonOneToken.txt index b656aca1b5..bce158a1e7 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonOneToken.txt +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonOneToken.txt @@ -43,4 +43,15 @@ expr "tokens":[{"type":3,"line":1,"pos":0,"channel":0,"text":"8"} ,{"type":-1,"line":1,"pos":1,"channel":0,"text":""} ], -"tree":{"1":[0]}} \ No newline at end of file +"tree":{"1":[0]}} + +[skip] +Cpp +CSharp +Dart +Go +JavaScript +PHP +Swift +Python2 +Python3 diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonOneTokenOneRule.txt b/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonOneTokenOneRule.txt index 7c85527f8c..b1e9b3a9f4 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonOneTokenOneRule.txt +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonOneTokenOneRule.txt @@ -43,4 +43,15 @@ s "tokens":[{"type":3,"line":1,"pos":0,"channel":0,"text":"99"} ,{"type":-1,"line":1,"pos":2,"channel":0,"text":""} ], -"tree":{"0":[{"1":[0]},1]}} \ No newline at end of file +"tree":{"0":[{"1":[0]},1]}} + +[skip] +Cpp +CSharp +Dart +Go +JavaScript +PHP +Swift +Python2 +Python3 From 083c87305ba9f52a02a02e5053684a7db09a3071 Mon Sep 17 00:00:00 2001 From: Terence Parr Date: Tue, 5 Jul 2022 10:51:10 -0700 Subject: [PATCH 08/13] update all target test templates for ToJSON(s) template. Signed-off-by: Terence Parr --- .../org/antlr/v4/test/runtime/templates/CSharp.test.stg | 2 ++ .../resources/org/antlr/v4/test/runtime/templates/Cpp.test.stg | 2 ++ .../resources/org/antlr/v4/test/runtime/templates/Dart.test.stg | 1 + .../resources/org/antlr/v4/test/runtime/templates/Go.test.stg | 1 + .../org/antlr/v4/test/runtime/templates/JavaScript.test.stg | 1 + .../resources/org/antlr/v4/test/runtime/templates/PHP.test.stg | 1 + .../org/antlr/v4/test/runtime/templates/Python2.test.stg | 1 + .../org/antlr/v4/test/runtime/templates/Python3.test.stg | 1 + .../org/antlr/v4/test/runtime/templates/Swift.test.stg | 1 + 9 files changed, 11 insertions(+) diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/CSharp.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/CSharp.test.stg index 14def7e03d..a0ce031103 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/CSharp.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/CSharp.test.stg @@ -56,6 +56,8 @@ BailErrorStrategy() ::= <%ErrorHandler = new BailErrorStrategy();%> ToStringTree(s) ::= <%.ToStringTree(this)%> +ToJSON(s) ::= "" + Column() ::= "this.Column" Text() ::= "this.Text" diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Cpp.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Cpp.test.stg index 61efce7a28..9d84c058b6 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Cpp.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Cpp.test.stg @@ -35,6 +35,8 @@ BuildParseTrees() ::= "setBuildParseTree(true);" BailErrorStrategy() ::= "_errHandler = std::make_shared\();" ToStringTree(s) ::= "->toStringTree(this)" +ToJSON(s) ::= "" + Column() ::= "getCharPositionInLine()" Text() ::= "getText()" ValEquals(a,b) ::= " == " diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Dart.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Dart.test.stg index a9c224fe47..94e5bbe2c7 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Dart.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Dart.test.stg @@ -55,6 +55,7 @@ BuildParseTrees() ::= "buildParseTree = true;" BailErrorStrategy() ::= <%errorHandler = new BailErrorStrategy();%> ToStringTree(s) ::= <%.toStringTree(parser: this)%> +ToJSON(s) ::= "" Column() ::= "this.charPositionInLine" diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Go.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Go.test.stg index c1f62e6f9e..cb5b4856ac 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Go.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Go.test.stg @@ -55,6 +55,7 @@ BuildParseTrees() ::= "p.BuildParseTrees = true" BailErrorStrategy() ::= <%p.SetErrorHandler(antlr.NewBailErrorStrategy())%> ToStringTree(s) ::= <%.ToStringTree(nil, p)%> +ToJSON(s) ::= "" Column() ::= "p.GetCharPositionInLine()" diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/JavaScript.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/JavaScript.test.stg index 709f13e39a..43b9352a48 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/JavaScript.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/JavaScript.test.stg @@ -55,6 +55,7 @@ BuildParseTrees() ::= "this.buildParseTrees = true;" BailErrorStrategy() ::= <%this._errHandler = new antlr4.error.BailErrorStrategy();%> ToStringTree(s) ::= <%.toStringTree(null, this)%> +ToJSON(s) ::= "" Column() ::= "this.column" diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/PHP.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/PHP.test.stg index a2ea874663..6f66e14d51 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/PHP.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/PHP.test.stg @@ -38,6 +38,7 @@ BuildParseTrees() ::= "\$this->setBuildParseTree(true);" BailErrorStrategy() ::= <%\$this->setErrorHandler(new Antlr\\Antlr4\\Runtime\\Error\\BailErrorStrategy());%> ToStringTree(s) ::= <%->toStringTree(\$this->getRuleNames())%> +ToJSON(s) ::= "" Column() ::= "\$this->getCharPositionInLine()" Text() ::= "\$this->getText()" ValEquals(a,b) ::= <%===%> diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Python2.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Python2.test.stg index 2292cafe7b..214269625f 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Python2.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Python2.test.stg @@ -55,6 +55,7 @@ BuildParseTrees() ::= "self._buildParseTrees = True" BailErrorStrategy() ::= <%self._errHandler = BailErrorStrategy()%> ToStringTree(s) ::= <%.toStringTree(recog=self)%> +ToJSON(s) ::= "" Column() ::= "self.column" diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Python3.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Python3.test.stg index 65dcdcd83a..f0059f9534 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Python3.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Python3.test.stg @@ -55,6 +55,7 @@ BuildParseTrees() ::= "self._buildParseTrees = True" BailErrorStrategy() ::= <%self._errHandler = BailErrorStrategy()%> ToStringTree(s) ::= <%.toStringTree(recog=self)%> +ToJSON(s) ::= "" Column() ::= "self.column" diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Swift.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Swift.test.stg index c2c12b63c2..0a8e03c46c 100755 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Swift.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Swift.test.stg @@ -55,6 +55,7 @@ BuildParseTrees() ::= "setBuildParseTree(true)" BailErrorStrategy() ::= <%setErrorHandler(BailErrorStrategy())%> ToStringTree(s) ::= <%.toStringTree(self)%> +ToJSON(s) ::= "" Column() ::= "self.getCharPositionInLine()" From f9d2432d12fccbc2b37734c721d649bb1233338b Mon Sep 17 00:00:00 2001 From: Terence Parr Date: Tue, 5 Jul 2022 11:05:57 -0700 Subject: [PATCH 09/13] json all on one line; add start/stop not text for tokens Signed-off-by: Terence Parr --- .../descriptors/ParseTrees/JsonEmpty.txt | 7 ++----- .../runtime/descriptors/ParseTrees/JsonExpr.txt | 10 ++-------- .../ParseTrees/JsonMismatchedToken.txt | 9 ++------- .../descriptors/ParseTrees/JsonOneToken.txt | 8 ++------ .../ParseTrees/JsonOneTokenOneRule.txt | 8 ++------ .../src/org/antlr/v4/runtime/tree/Trees.java | 17 ++++++----------- 6 files changed, 16 insertions(+), 43 deletions(-) diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonEmpty.txt b/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonEmpty.txt index 4f08a8d403..a89efbb5ac 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonEmpty.txt +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonEmpty.txt @@ -38,11 +38,8 @@ s [output] -{"rules":["s","expr"], -"input":"", -"tokens":[{"type":-1,"line":1,"pos":0,"channel":0,"text":""} -], -"tree":"0"} +"""{"rules":["s","expr"],"input":"","tokens":[{"type":-1,"line":1,"pos":0,"channel":0,"start":0,"stop":-1}],"tree":"0"} +""" [skip] Cpp diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonExpr.txt b/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonExpr.txt index a3a670b573..1aa4864b49 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonExpr.txt +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonExpr.txt @@ -38,14 +38,8 @@ s 1+2 [output] -{"rules":["s","expr"], -"input":"1+2", -"tokens":[{"type":3,"line":1,"pos":0,"channel":0,"text":"1"} -,{"type":7,"line":1,"pos":1,"channel":0,"text":"+"} -,{"type":3,"line":1,"pos":2,"channel":0,"text":"2"} -,{"type":-1,"line":1,"pos":3,"channel":0,"text":""} -], -"tree":{"0":[{"1":[{"1":[0]},1,{"1":[2]}]},3]}} +"""{"rules":["s","expr"],"input":"1+2","tokens":[{"type":3,"line":1,"pos":0,"channel":0,"start":0,"stop":0},{"type":7,"line":1,"pos":1,"channel":0,"start":1,"stop":1},{"type":3,"line":1,"pos":2,"channel":0,"start":2,"stop":2},{"type":-1,"line":1,"pos":3,"channel":0,"start":3,"stop":2}],"tree":{"0":[{"1":[{"1":[0]},1,{"1":[2]}]},3]}} +""" [skip] Cpp diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonMismatchedToken.txt b/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonMismatchedToken.txt index 6016cb7e29..ddbd7554d1 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonMismatchedToken.txt +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonMismatchedToken.txt @@ -38,13 +38,8 @@ expr f( [output] -{"rules":["s","expr"], -"input":"f(", -"tokens":[{"type":4,"line":1,"pos":0,"channel":0,"text":"f"} -,{"type":1,"line":1,"pos":1,"channel":0,"text":"("} -,{"type":-1,"line":1,"pos":2,"channel":0,"text":""} -], -"tree":{"1":[0,1,{"error":""}]}} +"""{"rules":["s","expr"],"input":"f(","tokens":[{"type":4,"line":1,"pos":0,"channel":0,"start":0,"stop":0},{"type":1,"line":1,"pos":1,"channel":0,"start":1,"stop":1},{"type":-1,"line":1,"pos":2,"channel":0,"start":2,"stop":1}],"tree":{"1":[0,1,{"error":""}]}} +""" [errors] """line 1:2 missing ')' at '' diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonOneToken.txt b/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonOneToken.txt index bce158a1e7..0cbf762df8 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonOneToken.txt +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonOneToken.txt @@ -38,12 +38,8 @@ expr 8 [output] -{"rules":["s","expr"], -"input":"8", -"tokens":[{"type":3,"line":1,"pos":0,"channel":0,"text":"8"} -,{"type":-1,"line":1,"pos":1,"channel":0,"text":""} -], -"tree":{"1":[0]}} +"""{"rules":["s","expr"],"input":"8","tokens":[{"type":3,"line":1,"pos":0,"channel":0,"start":0,"stop":0},{"type":-1,"line":1,"pos":1,"channel":0,"start":1,"stop":0}],"tree":{"1":[0]}} +""" [skip] Cpp diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonOneTokenOneRule.txt b/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonOneTokenOneRule.txt index b1e9b3a9f4..843037e079 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonOneTokenOneRule.txt +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonOneTokenOneRule.txt @@ -38,12 +38,8 @@ s 99 [output] -{"rules":["s","expr"], -"input":"99", -"tokens":[{"type":3,"line":1,"pos":0,"channel":0,"text":"99"} -,{"type":-1,"line":1,"pos":2,"channel":0,"text":""} -], -"tree":{"0":[{"1":[0]},1]}} +"""{"rules":["s","expr"],"input":"99","tokens":[{"type":3,"line":1,"pos":0,"channel":0,"start":0,"stop":1},{"type":-1,"line":1,"pos":2,"channel":0,"start":2,"stop":1}],"tree":{"0":[{"1":[0]},1]}} +""" [skip] Cpp diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/Trees.java b/runtime/Java/src/org/antlr/v4/runtime/tree/Trees.java index e92d492fa2..f53446ae82 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/Trees.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/Trees.java @@ -60,6 +60,7 @@ public static String toStringTree(final Tree t, final List ruleNames) { /** Print out a whole tree in JSON form. {@link #getNodeText} is used on the * node payloads to get the text for the nodes. Detect * parse trees and extract data appropriately. Include rulenames, input, tokens. + * * @since 4.10.2 */ public static String toJSON(Tree t, Parser recog) { @@ -74,18 +75,12 @@ public static String toJSON(Tree t, Parser recog) { Interval allchar = Interval.of(0, inputStream.size() - 1); String input = inputStream.getText(allchar); input = Utils.escapeJSONString(input); -// List tokens; -// if ( tokenStream instanceof CommonTokenStream ) { -// tokens = ((CommonTokenStream) tokenStream).getTokens(0, tokenStream.size()-1); -// } -// else { -// } List tokenStrings = new ArrayList<>(); for (int i = 0; i < tokenStream.size(); i++) { Token tok = tokenStream.get(i); - String s = String.format("{\"type\":%d,\"line\":%d,\"pos\":%d,\"channel\":%d,\"text\":\"%s\"}\n", + String s = String.format("{\"type\":%d,\"line\":%d,\"pos\":%d,\"channel\":%d,\"start\":%d,\"stop\":%d}", tok.getType(), tok.getLine(), tok.getCharPositionInLine(), tok.getChannel(), - Utils.escapeJSONString(tok.getText())); + tok.getStartIndex(), tok.getStopIndex()); tokenStrings.add(s); } String tree = toJSONTree(t, ruleNamesList); @@ -94,13 +89,13 @@ public static String toJSON(Tree t, Parser recog) { buf.append("{"); buf.append("\"rules\":[\""); buf.append(String.join("\",\"", ruleNames)); - buf.append("\"],\n"); + buf.append("\"],"); buf.append("\"input\":\""); buf.append(input); - buf.append("\",\n"); + buf.append("\","); buf.append("\"tokens\":["); buf.append(String.join(",", tokenStrings)); - buf.append("],\n"); + buf.append("],"); buf.append("\"tree\":"); buf.append(tree); buf.append("}"); From 232f5e9b8163157793660b9e499c023871eb51cc Mon Sep 17 00:00:00 2001 From: Terence Parr Date: Tue, 5 Jul 2022 11:42:56 -0700 Subject: [PATCH 10/13] move to separate class Signed-off-by: Terence Parr --- .../v4/test/runtime/templates/Java.test.stg | 2 +- .../antlr/v4/runtime/tree/JsonSerializer.java | 125 ++++++++++++++++++ .../src/org/antlr/v4/runtime/tree/Trees.java | 98 -------------- 3 files changed, 126 insertions(+), 99 deletions(-) create mode 100644 runtime/Java/src/org/antlr/v4/runtime/tree/JsonSerializer.java diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Java.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Java.test.stg index a58481e402..20a1ba7e2d 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Java.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Java.test.stg @@ -56,7 +56,7 @@ BailErrorStrategy() ::= <%setErrorHandler(new BailErrorStrategy());%> ToStringTree(s) ::= <%.toStringTree(this)%> -ToJSON(s) ::= <%Trees.toJSON(,this)%> +ToJSON(s) ::= <%JsonSerializer.toJSON(,this)%> Column() ::= "this.getCharPositionInLine()" diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/JsonSerializer.java b/runtime/Java/src/org/antlr/v4/runtime/tree/JsonSerializer.java new file mode 100644 index 0000000000..b8705513d3 --- /dev/null +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/JsonSerializer.java @@ -0,0 +1,125 @@ +package org.antlr.v4.runtime.tree; + +import org.antlr.v4.runtime.*; +import org.antlr.v4.runtime.atn.ATN; +import org.antlr.v4.runtime.misc.Interval; +import org.antlr.v4.runtime.misc.Utils; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * + * @since 4.10.2 + */ +public class JsonSerializer { + /** Print out a whole tree in JSON form. {@link #getNodeText} is used on the + * node payloads to get the text for the nodes. Detect + * parse trees and extract data appropriately. Include rulenames, input, tokens. + */ + public static String toJSON(Tree t, Parser recog) { + String[] ruleNames = recog != null ? recog.getRuleNames() : null; + if ( t==null || ruleNames==null ) { + return null; + } + TokenStream tokenStream = recog.getInputStream(); + CharStream inputStream = tokenStream.getTokenSource().getInputStream(); + return toJSON(t, Arrays.asList(ruleNames), tokenStream, inputStream); + } + + public static String toJSON(Tree t, + final List ruleNames, + final TokenStream tokenStream, + final CharStream inputStream) + { + if ( t==null || ruleNames==null ) { + return null; + } + + StringBuilder buf = new StringBuilder(); + buf.append("{"); + buf.append("\"rules\":[\""); + buf.append(String.join("\",\"", ruleNames)); + buf.append("\"],"); + + if ( inputStream!=null ) { + Interval allchar = Interval.of(0, inputStream.size() - 1); + String input = inputStream.getText(allchar); + input = Utils.escapeJSONString(input); + buf.append("\"input\":\""); + buf.append(input); + buf.append("\","); + } + + if ( tokenStream!=null ) { + List tokenStrings = new ArrayList<>(); + for (int i = 0; i < tokenStream.size(); i++) { + Token tok = tokenStream.get(i); + String s = String.format("{\"type\":%d,\"line\":%d,\"pos\":%d,\"channel\":%d,\"start\":%d,\"stop\":%d}", + tok.getType(), tok.getLine(), tok.getCharPositionInLine(), tok.getChannel(), + tok.getStartIndex(), tok.getStopIndex()); + tokenStrings.add(s); + } + buf.append("\"tokens\":["); + buf.append(String.join(",", tokenStrings)); + buf.append("],"); + } + + String tree = toJSONTree(t); + buf.append("\"tree\":"); + buf.append(tree); + buf.append("}"); + + return buf.toString(); + } + + /** Print out a whole tree in JSON form. {@link #getNodeText} is used on the + * node payloads to get the text for the nodes. Detect + * parse trees and extract data appropriately. + * @since 4.10.2 + */ + public static String toJSONTree(final Tree t) { + StringBuilder buf = new StringBuilder(); + if ( t.getChildCount()==0 ) { + return getJSONNodeText(t); + } + buf.append("{"); + buf.append(getJSONNodeText(t)); + buf.append(":["); + for (int i = 0; i0 ) buf.append(','); + buf.append(toJSONTree(t.getChild(i))); + } + buf.append("]"); + buf.append("}"); + return buf.toString(); + } + + /** @since 4.10.2 */ + public static String getJSONNodeText(Tree t) { + if ( t instanceof RuleContext) { + int ruleIndex = ((RuleContext)t).getRuleContext().getRuleIndex(); + int altNumber = ((RuleContext) t).getAltNumber(); + if ( altNumber!= ATN.INVALID_ALT_NUMBER ) { + return String.format("\"%d:%d\"",ruleIndex,altNumber); + } + return String.format("\"%d\"",ruleIndex); + } + else if ( t instanceof ErrorNode) { + Token symbol = ((TerminalNode)t).getSymbol(); + if (symbol != null) { + return "{\"error\":\"" + symbol.getText() + "\"}"; + } + return "{\"error\":\""+t.getPayload().toString()+"\"}"; + } + else if ( t instanceof TerminalNode) { + Token symbol = ((TerminalNode)t).getSymbol(); + if (symbol != null) { + return String.valueOf(symbol.getTokenIndex()); + } + return "-1"; + } + return ""; + } +} diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/Trees.java b/runtime/Java/src/org/antlr/v4/runtime/tree/Trees.java index f53446ae82..b114a6c485 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/Trees.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/Trees.java @@ -57,74 +57,6 @@ public static String toStringTree(final Tree t, final List ruleNames) { return buf.toString(); } - /** Print out a whole tree in JSON form. {@link #getNodeText} is used on the - * node payloads to get the text for the nodes. Detect - * parse trees and extract data appropriately. Include rulenames, input, tokens. - * - * @since 4.10.2 - */ - public static String toJSON(Tree t, Parser recog) { - String[] ruleNames = recog != null ? recog.getRuleNames() : null; - if ( ruleNames==null ) { - return null; - } - List ruleNamesList = Arrays.asList(ruleNames); - - TokenStream tokenStream = recog.getInputStream(); - CharStream inputStream = tokenStream.getTokenSource().getInputStream(); - Interval allchar = Interval.of(0, inputStream.size() - 1); - String input = inputStream.getText(allchar); - input = Utils.escapeJSONString(input); - List tokenStrings = new ArrayList<>(); - for (int i = 0; i < tokenStream.size(); i++) { - Token tok = tokenStream.get(i); - String s = String.format("{\"type\":%d,\"line\":%d,\"pos\":%d,\"channel\":%d,\"start\":%d,\"stop\":%d}", - tok.getType(), tok.getLine(), tok.getCharPositionInLine(), tok.getChannel(), - tok.getStartIndex(), tok.getStopIndex()); - tokenStrings.add(s); - } - String tree = toJSONTree(t, ruleNamesList); - - StringBuilder buf = new StringBuilder(); - buf.append("{"); - buf.append("\"rules\":[\""); - buf.append(String.join("\",\"", ruleNames)); - buf.append("\"],"); - buf.append("\"input\":\""); - buf.append(input); - buf.append("\","); - buf.append("\"tokens\":["); - buf.append(String.join(",", tokenStrings)); - buf.append("],"); - buf.append("\"tree\":"); - buf.append(tree); - buf.append("}"); - - return buf.toString(); - } - - /** Print out a whole tree in JSON form. {@link #getNodeText} is used on the - * node payloads to get the text for the nodes. Detect - * parse trees and extract data appropriately. - * @since 4.10.2 - */ - public static String toJSONTree(final Tree t, final List ruleNames) { - StringBuilder buf = new StringBuilder(); - if ( t.getChildCount()==0 ) { - return getJSONNodeText(t, ruleNames); - } - buf.append("{"); - buf.append(getJSONNodeText(t, ruleNames)); - buf.append(":["); - for (int i = 0; i0 ) buf.append(','); - buf.append(toJSONTree(t.getChild(i), ruleNames)); - } - buf.append("]"); - buf.append("}"); - return buf.toString(); - } - public static String getNodeText(Tree t, Parser recog) { String[] ruleNames = recog != null ? recog.getRuleNames() : null; List ruleNamesList = ruleNames != null ? Arrays.asList(ruleNames) : null; @@ -161,36 +93,6 @@ else if ( t instanceof TerminalNode) { return t.getPayload().toString(); } - /** @since 4.10.2 */ - public static String getJSONNodeText(Tree t, List ruleNames) { - if ( ruleNames==null ) { - return null; - } - if ( t instanceof RuleContext ) { - int ruleIndex = ((RuleContext)t).getRuleContext().getRuleIndex(); - int altNumber = ((RuleContext) t).getAltNumber(); - if ( altNumber!=ATN.INVALID_ALT_NUMBER ) { - return String.format("\"%d:%d\"",ruleIndex,altNumber); - } - return String.format("\"%d\"",ruleIndex); - } - else if ( t instanceof ErrorNode) { - Token symbol = ((TerminalNode)t).getSymbol(); - if (symbol != null) { - return "{\"error\":\"" + symbol.getText() + "\"}"; - } - return "{\"error\":\""+t.getPayload().toString()+"\"}"; - } - else if ( t instanceof TerminalNode) { - Token symbol = ((TerminalNode)t).getSymbol(); - if (symbol != null) { - return String.valueOf(symbol.getTokenIndex()); - } - return "-1"; - } - return ""; - } - /** Return ordered list of all children of this node */ public static List getChildren(Tree t) { List kids = new ArrayList(); From 6a1d7f9ac91aea0d87e30997e616a81f5aef8ee5 Mon Sep 17 00:00:00 2001 From: Terence Parr Date: Tue, 5 Jul 2022 11:58:31 -0700 Subject: [PATCH 11/13] add comment Signed-off-by: Terence Parr --- .../antlr/v4/runtime/tree/JsonSerializer.java | 79 ++++++++++++++++--- 1 file changed, 69 insertions(+), 10 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/JsonSerializer.java b/runtime/Java/src/org/antlr/v4/runtime/tree/JsonSerializer.java index b8705513d3..860376a017 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/JsonSerializer.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/JsonSerializer.java @@ -9,14 +9,71 @@ import java.util.Arrays; import java.util.List; -/** +/** This "class" wraps support functions that generate JSON for parse trees. + * The JSON includes everything needed to reconstruct a parse tree: * - * @since 4.10.2 + * Rule names (field: "rules") + * Input chars (field: "input") + * Tokens (field: "tokens") + * Parse tree (field: "tree"; refs rule indexes and token indexes) + * + * For example, given input "99" and a simple expression grammar giving parse tree + * "(s (expr 99) EOF)", the full JSON (formatted by jq) looks like: + * + * { + * "rules": [ + * "s", + * "expr" + * ], + * "input": "99", + * "tokens": [ + * { + * "type": 3, + * "line": 1, + * "pos": 0, + * "channel": 0, + * "start": 0, + * "stop": 1 + * }, + * { + * "type": -1, + * "line": 1, + * "pos": 2, + * "channel": 0, + * "start": 2, + * "stop": 1 + * } + * ], + * "tree": { + * "0": [ + * { + * "1": [ + * 0 + * ] + * }, + * 1 + * ] + * } + * } + * + * Notice that the tree is just a series of nested references to integers, which refer to rules + * and tokens. + * + * One potential use case: Create an ANTLR server that accepts a grammar and input as parameters then + * returns JSON for the parse tree and the tokens. This can be deserialized by JavaScript in a web browser + * to display the parse result. + * + * @since 4.10.2 */ public class JsonSerializer { - /** Print out a whole tree in JSON form. {@link #getNodeText} is used on the - * node payloads to get the text for the nodes. Detect - * parse trees and extract data appropriately. Include rulenames, input, tokens. + /** Create a JSON representation of a parse tree and include all other information necessary to reconstruct + * a printable parse tree: the rules, input, tokens, and the tree structure that refers to the rule + * and token indexes. Extract all information from the parser, which is assumed to be in a state + * post-parse and the object that created tree t. + * + * @param t The parse tree to serialize as JSON + * @param recog The parser that created the parse tree and is in the post-recognition state + * @return JSON representing the parse tree */ public static String toJSON(Tree t, Parser recog) { String[] ruleNames = recog != null ? recog.getRuleNames() : null; @@ -28,6 +85,10 @@ public static String toJSON(Tree t, Parser recog) { return toJSON(t, Arrays.asList(ruleNames), tokenStream, inputStream); } + /** Create a JSON representation of a parse tree and include all other information necessary to reconstruct + * a printable parse tree: the rules, input, tokens, and the tree structure that refers to the rule + * and token indexes. The tree and rule names are required but the token stream and input stream are optional. + */ public static String toJSON(Tree t, final List ruleNames, final TokenStream tokenStream, @@ -74,10 +135,8 @@ public static String toJSON(Tree t, return buf.toString(); } - /** Print out a whole tree in JSON form. {@link #getNodeText} is used on the - * node payloads to get the text for the nodes. Detect - * parse trees and extract data appropriately. - * @since 4.10.2 + /** Create a JSON representation of a parse tree. The tree is just a series of nested references + * to integers, which refer to rules and tokens. */ public static String toJSONTree(final Tree t) { StringBuilder buf = new StringBuilder(); @@ -96,7 +155,7 @@ public static String toJSONTree(final Tree t) { return buf.toString(); } - /** @since 4.10.2 */ + /** Create appropriate JSON text for a tree node */ public static String getJSONNodeText(Tree t) { if ( t instanceof RuleContext) { int ruleIndex = ((RuleContext)t).getRuleContext().getRuleIndex(); From f121ba4a14491ac1f0ba9170604e9739d6b1d973 Mon Sep 17 00:00:00 2001 From: Terence Parr Date: Tue, 5 Jul 2022 12:30:53 -0700 Subject: [PATCH 12/13] add more complex grammar Signed-off-by: Terence Parr --- .../descriptors/ParseTrees/JsonNotTiny.txt | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonNotTiny.txt diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonNotTiny.txt b/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonNotTiny.txt new file mode 100644 index 0000000000..469af44eb0 --- /dev/null +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/JsonNotTiny.txt @@ -0,0 +1,59 @@ +[type] +Parser + +[grammar] +grammar Expr; + +program +@init { + +} +@after { + + +} + : stat EOF + | def EOF + ; + +stat: ID '=' expr ';' + | expr ';' + ; + +def : ID '(' ID (',' ID)* ')' '{' stat* '}' ; + +expr: ID + | INT + | func + | 'not' expr + | expr 'and' expr + | expr 'or' expr + ; + +func : ID '(' expr (',' expr)* ')' ; + +INT : [0-9]+ ; +ID: [a-zA-Z_][a-zA-Z_0-9]* ; +WS: [ \t\n\r\f]+ -> skip ; + +[start] +program + +[input] +f(x, y, z) { x = x and y or z; g(30); } + +[output] +"""(program (def f ( x , y , z ) { (stat x = (expr (expr (expr x) and (expr y)) or (expr z)) ;) (stat (expr (func g ( (expr 30) ))) ;) }) ) +{"rules":["program","stat","def","expr","func"],"input":"f(x, y, z) { x = x and y or z; g(30); }","tokens":[{"type":12,"line":1,"pos":0,"channel":0,"start":0,"stop":0},{"type":3,"line":1,"pos":1,"channel":0,"start":1,"stop":1},{"type":12,"line":1,"pos":2,"channel":0,"start":2,"stop":2},{"type":4,"line":1,"pos":3,"channel":0,"start":3,"stop":3},{"type":12,"line":1,"pos":5,"channel":0,"start":5,"stop":5},{"type":4,"line":1,"pos":6,"channel":0,"start":6,"stop":6},{"type":12,"line":1,"pos":8,"channel":0,"start":8,"stop":8},{"type":5,"line":1,"pos":9,"channel":0,"start":9,"stop":9},{"type":6,"line":1,"pos":11,"channel":0,"start":11,"stop":11},{"type":12,"line":1,"pos":13,"channel":0,"start":13,"stop":13},{"type":1,"line":1,"pos":15,"channel":0,"start":15,"stop":15},{"type":12,"line":1,"pos":17,"channel":0,"start":17,"stop":17},{"type":9,"line":1,"pos":19,"channel":0,"start":19,"stop":21},{"type":12,"line":1,"pos":23,"channel":0,"start":23,"stop":23},{"type":10,"line":1,"pos":25,"channel":0,"start":25,"stop":26},{"type":12,"line":1,"pos":28,"channel":0,"start":28,"stop":28},{"type":2,"line":1,"pos":29,"channel":0,"start":29,"stop":29},{"type":12,"line":1,"pos":31,"channel":0,"start":31,"stop":31},{"type":3,"line":1,"pos":32,"channel":0,"start":32,"stop":32},{"type":11,"line":1,"pos":33,"channel":0,"start":33,"stop":34},{"type":5,"line":1,"pos":35,"channel":0,"start":35,"stop":35},{"type":2,"line":1,"pos":36,"channel":0,"start":36,"stop":36},{"type":7,"line":1,"pos":38,"channel":0,"start":38,"stop":38},{"type":-1,"line":1,"pos":39,"channel":0,"start":39,"stop":38}],"tree":{"0":[{"2":[0,1,2,3,4,5,6,7,8,{"1":[9,10,{"3":[{"3":[{"3":[11]},12,{"3":[13]}]},14,{"3":[15]}]},16]},{"1":[{"3":[{"4":[17,18,{"3":[19]},20]}]},21]},22]},23]}} +""" + +[skip] +Cpp +CSharp +Dart +Go +JavaScript +PHP +Swift +Python2 +Python3 From 7e7057bedc18386d1bc94663f795aa9e7425ab53 Mon Sep 17 00:00:00 2001 From: Terence Parr Date: Tue, 5 Jul 2022 12:45:06 -0700 Subject: [PATCH 13/13] more comment Signed-off-by: Terence Parr --- .../org/antlr/v4/runtime/tree/JsonSerializer.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/JsonSerializer.java b/runtime/Java/src/org/antlr/v4/runtime/tree/JsonSerializer.java index 860376a017..de5a8abe74 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/JsonSerializer.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/JsonSerializer.java @@ -63,6 +63,20 @@ * returns JSON for the parse tree and the tokens. This can be deserialized by JavaScript in a web browser * to display the parse result. * + * To load and dump elements from Python 3: + * + * import json + * + * with open("/tmp/t.json") as f: + * data = f.read() + * + * data = json.loads(data) + * print(data['rules']) + * print(data['input']) + * for t in data['tokens']: + * print(t) + * print(data['tree']) + * * @since 4.10.2 */ public class JsonSerializer {