support special chats in entity type definition (#358)

feich-ms · web-flow · commit 4ac2e29b6969 · 2019-11-19T09:42:02.000+08:00
diff --git a/packages/lu/src/parser/lufile/LUFileLexer.g4 b/packages/lu/src/parser/lufile/LUFileLexer.g4
@@ -186,10 +186,6 @@ NEWLINE_IN_ENTITY
   : '\r'? '\n' {this.ignoreWS = true;} -> type(NEWLINE), popMode
   ;
 
-ENTITY_IDENTIFIER
-  : (LETTER | NUMBER | '_' | '-' | '|' | '.')+ { this.ignoreWS = false;}
-  ;
-
 COMPOSITE_ENTITY
   : '[' (~[\r\n{}[()])*
   ;
@@ -198,12 +194,12 @@ REGEX_ENTITY
   : '/' (~[\r\n])*
   ;
 
-COLON_MARK
-  : ':'
+ENTITY_TEXT
+  : ~[ \t\r\n:]+ { this.ignoreWS = false;}
   ;
 
-SPECIAL_CHAR_MARK 
-  : '=' | ',' | '!'
+COLON_MARK
+  : ':'
   ;
 
 mode QNA_MODE;
diff --git a/packages/lu/src/parser/lufile/LUFileParser.g4 b/packages/lu/src/parser/lufile/LUFileParser.g4
@@ -108,11 +108,11 @@ entityLine
     ;
 
 entityName
-    : (entityIdentifier|WS)*
+    : (ENTITY_TEXT|WS)*
     ;
 
 entityType
-    : (entityIdentifier|compositeEntityIdentifier|regexEntityIdentifier|SPECIAL_CHAR_MARK|COLON_MARK|WS)*
+    : (compositeEntityIdentifier|regexEntityIdentifier|ENTITY_TEXT|COLON_MARK|WS)*
     ;
 
 compositeEntityIdentifier
@@ -123,10 +123,6 @@ regexEntityIdentifier
     : REGEX_ENTITY
     ;
 
-entityIdentifier
-    : ENTITY_IDENTIFIER
-    ;
-
 entityListBody
     : (normalItemString newline)+
     ;
diff --git a/packages/lu/src/parser/lufile/generated/LUFileLexer.interp b/packages/lu/src/parser/lufile/generated/LUFileLexer.interp
diff --git a/packages/lu/src/parser/lufile/generated/LUFileLexer.js b/packages/lu/src/parser/lufile/generated/LUFileLexer.js
diff --git a/packages/lu/src/parser/lufile/generated/LUFileLexer.tokens b/packages/lu/src/parser/lufile/generated/LUFileLexer.tokens
@@ -31,13 +31,12 @@ ESCAPE_CHARACTER=30
 EXPRESSION=31
 TEXT=32
 WS_IN_ENTITY_IGNORED=33
-ENTITY_IDENTIFIER=34
-COMPOSITE_ENTITY=35
-REGEX_ENTITY=36
+COMPOSITE_ENTITY=34
+REGEX_ENTITY=35
+ENTITY_TEXT=36
 COLON_MARK=37
-SPECIAL_CHAR_MARK=38
-WS_IN_QNA_IGNORED=39
-QNA_TEXT=40
+WS_IN_QNA_IGNORED=38
+QNA_TEXT=39
 '**Filters:**'=12
 ','=16
 '='=17
diff --git a/packages/lu/src/parser/lufile/generated/LUFileParser.interp b/packages/lu/src/parser/lufile/generated/LUFileParser.interp
diff --git a/packages/lu/src/parser/lufile/generated/LUFileParser.js b/packages/lu/src/parser/lufile/generated/LUFileParser.js
diff --git a/packages/lu/src/parser/lufile/generated/LUFileParser.tokens b/packages/lu/src/parser/lufile/generated/LUFileParser.tokens
@@ -31,13 +31,12 @@ ESCAPE_CHARACTER=30
 EXPRESSION=31
 TEXT=32
 WS_IN_ENTITY_IGNORED=33
-ENTITY_IDENTIFIER=34
-COMPOSITE_ENTITY=35
-REGEX_ENTITY=36
+COMPOSITE_ENTITY=34
+REGEX_ENTITY=35
+ENTITY_TEXT=36
 COLON_MARK=37
-SPECIAL_CHAR_MARK=38
-WS_IN_QNA_IGNORED=39
-QNA_TEXT=40
+WS_IN_QNA_IGNORED=38
+QNA_TEXT=39
 '**Filters:**'=12
 ','=16
 '='=17
diff --git a/packages/lu/src/parser/lufile/generated/LUFileParserListener.js b/packages/lu/src/parser/lufile/generated/LUFileParserListener.js
@@ -1,4 +1,4 @@
-// Generated from LUFileParser.g4 by ANTLR 4.7.1
+// Generated from ../LUFileParser.g4 by ANTLR 4.7.2
 // jshint ignore: start
 var antlr4 = require('antlr4/index');
 
@@ -263,15 +263,6 @@ LUFileParserListener.prototype.exitRegexEntityIdentifier = function(ctx) {
 };
 
 
-// Enter a parse tree produced by LUFileParser#entityIdentifier.
-LUFileParserListener.prototype.enterEntityIdentifier = function(ctx) {
-};
-
-// Exit a parse tree produced by LUFileParser#entityIdentifier.
-LUFileParserListener.prototype.exitEntityIdentifier = function(ctx) {
-};
-
-
 // Enter a parse tree produced by LUFileParser#entityListBody.
 LUFileParserListener.prototype.enterEntityListBody = function(ctx) {
 };
diff --git a/packages/lu/src/parser/lufile/generated/LUFileParserVisitor.js b/packages/lu/src/parser/lufile/generated/LUFileParserVisitor.js
@@ -1,4 +1,4 @@
-// Generated from LUFileParser.g4 by ANTLR 4.7.1
+// Generated from ../LUFileParser.g4 by ANTLR 4.7.2
 // jshint ignore: start
 var antlr4 = require('antlr4/index');
 
@@ -180,12 +180,6 @@ LUFileParserVisitor.prototype.visitRegexEntityIdentifier = function(ctx) {
 };
 
 
-// Visit a parse tree produced by LUFileParser#entityIdentifier.
-LUFileParserVisitor.prototype.visitEntityIdentifier = function(ctx) {
-  return this.visitChildren(ctx);
-};
-
-
 // Visit a parse tree produced by LUFileParser#entityListBody.
 LUFileParserVisitor.prototype.visitEntityListBody = function(ctx) {
   return this.visitChildren(ctx);
diff --git a/packages/lu/test/commands/luis/convert.test.ts b/packages/lu/test/commands/luis/convert.test.ts
@@ -186,7 +186,7 @@ describe('luis:convert', () => {
     .stderr()
     .command(['luis:convert', '--in', `${path.join(__dirname, './../../fixtures/testcases/invalid-entity-definition.lu')}`])
     .it('luis:convert writes out an error when invalid entity definition is found', async (ctx) => {
-      expect(ctx.stderr).to.contain("[ERROR] line 1:9 - line 1:10: syntax error: missing ':' at '='")
+      expect(ctx.stderr).to.contain("[ERROR] line 1:14 - line 1:16: syntax error: missing ':' at '\\r\\n'")
     })
 
     test
@@ -378,6 +378,14 @@ describe('luis:convert', () => {
       expect(ctx.stdout).to.contain(`Skipping "> !# @app = test"`)
       expect(ctx.stderr).to.contain('No LU or Luis content parsed!')
     })
+
+    test
+    .stdout()
+    .command(['luis:convert', '--in', `${path.join(__dirname, './../../fixtures/testcases/special-char-in-entity-type.lu')}`, '--out', './results/root36.json', '--log'])
+    .it('luis:convert entities with special chars in entity type line', async (ctx) => {
+      let parsedObjects = await parseJsonFiles('./../../../results/root36.json', './../../fixtures/verified/special-char-in-entity-type.json')
+      expect(parsedObjects[0]).to.deep.equal(parsedObjects[1])
+    })
 })   
 
 describe('luis:convert version 5 upgrade test', () => {
diff --git a/packages/lu/test/fixtures/testcases/special-char-in-entity-type.lu b/packages/lu/test/fixtures/testcases/special-char-in-entity-type.lu
@@ -0,0 +1,9 @@
+$entity:Foo & Sons=
+-Foo and Sons
+-F&S
+$entity:Maelström=
+-Maelstrom
+$entity:Qux (US)=
+-Qux
+-Qux USA
+-Qux US
diff --git a/packages/lu/test/fixtures/verified/special-char-in-entity-type.json b/packages/lu/test/fixtures/verified/special-char-in-entity-type.json
@@ -0,0 +1,46 @@
+{
+    "intents": [],
+    "entities": [],
+    "composites": [],
+    "closedLists": [
+      {
+        "name": "entity",
+        "subLists": [
+          {
+            "canonicalForm": "Foo & Sons",
+            "list": [
+              "Foo and Sons",
+              "F&S"
+            ]
+          },
+          {
+            "canonicalForm": "Maelström",
+            "list": [
+              "Maelstrom"
+            ]
+          },
+          {
+            "canonicalForm": "Qux (US)",
+            "list": [
+              "Qux",
+              "Qux USA",
+              "Qux US"
+            ]
+          }
+        ],
+        "roles": []
+      }
+    ],
+    "regex_entities": [],
+    "model_features": [],
+    "regex_features": [],
+    "utterances": [],
+    "patterns": [],
+    "patternAnyEntities": [],
+    "prebuiltEntities": [],
+    "luis_schema_version": "3.2.0",
+    "versionId": "0.1",
+    "name": "",
+    "desc": "",
+    "culture": "en-us"
+  }

Original file line number	Diff line number	Diff line change
`@@ -186,10 +186,6 @@ NEWLINE_IN_ENTITY`
`186`	`186`	`: '\r'? '\n' {this.ignoreWS = true;} -> type(NEWLINE), popMode`
`187`	`187`	`;`
`188`	`188`
`189`		`-ENTITY_IDENTIFIER`
`190`		`- : (LETTER \| NUMBER \| '_' \| '-' \| '\|' \| '.')+ { this.ignoreWS = false;}`
`191`		`- ;`
`192`		`-`
`193`	`189`	`COMPOSITE_ENTITY`
`194`	`190`	`: '[' (~[\r\n{}[()])*`
`195`	`191`	`;`
`@@ -198,12 +194,12 @@ REGEX_ENTITY`
`198`	`194`	`: '/' (~[\r\n])*`
`199`	`195`	`;`
`200`	`196`
`201`		`-COLON_MARK`
`202`		`- : ':'`
	`197`	`+ENTITY_TEXT`
	`198`	`+ : ~[ \t\r\n:]+ { this.ignoreWS = false;}`
`203`	`199`	`;`
`204`	`200`
`205`		`-SPECIAL_CHAR_MARK`
`206`		`- : '=' \| ',' \| '!'`
	`201`	`+COLON_MARK`
	`202`	`+ : ':'`
`207`	`203`	`;`
`208`	`204`
`209`	`205`	`mode QNA_MODE;`
Original file line number	Diff line number	Diff line change
`@@ -108,11 +108,11 @@ entityLine`
`108`	`108`	`;`
`109`	`109`
`110`	`110`	`entityName`
`111`		`- : (entityIdentifier\|WS)*`
	`111`	`+ : (ENTITY_TEXT\|WS)*`
`112`	`112`	`;`
`113`	`113`
`114`	`114`	`entityType`
`115`		`- : (entityIdentifier\|compositeEntityIdentifier\|regexEntityIdentifier\|SPECIAL_CHAR_MARK\|COLON_MARK\|WS)*`
	`115`	`+ : (compositeEntityIdentifier\|regexEntityIdentifier\|ENTITY_TEXT\|COLON_MARK\|WS)*`
`116`	`116`	`;`
`117`	`117`
`118`	`118`	`compositeEntityIdentifier`
`@@ -123,10 +123,6 @@ regexEntityIdentifier`
`123`	`123`	`: REGEX_ENTITY`
`124`	`124`	`;`
`125`	`125`
`126`		`-entityIdentifier`
`127`		`- : ENTITY_IDENTIFIER`
`128`		`- ;`
`129`		`-`
`130`	`126`	`entityListBody`
`131`	`127`	`: (normalItemString newline)+`
`132`	`128`	`;`