fix lu parser to accept quotes and interchanageable label in pharselist definition (#915)

* fix lu parser to accept doule quotes and interchanageable label in phraselist entity

* optimize lexer and parser to loose the label defintion

Author: feich-ms

packages/lu/src/parser/lufile/LUFileLexer.g4

@@ -130,6 +130,10 @@ NEW_ENTITY_TYPE_IDENTIFIER
   : S I M P L E | L I S T | R E G E X | P R E B U I L T | C O M P O S I T E | M L | P A T T E R N A N Y | P H R A S E L I S T | I N T E N T
   ;
 
+PHRASE_LIST_LABEL
+  : '(' (~[\r\n])* ')'
+  ;
+
 NEW_COMPOSITE_ENTITY
   : '[' (~[\r\n{}[\]()])* ']'
   ;

packages/lu/src/parser/lufile/LUFileParser.g4

@@ -121,11 +121,11 @@ newEntityRoleOrFeatures
     ;
 
 newEntityName
-    : NEW_ENTITY_IDENTIFIER
+    : NEW_ENTITY_IDENTIFIER (WS* PHRASE_LIST_LABEL)?
     ;
 
 newEntityNameWithWS
-    : NEW_ENTITY_IDENTIFIER_WITH_WS
+    : NEW_ENTITY_IDENTIFIER_WITH_WS (WS* PHRASE_LIST_LABEL)?
     ;
 
 entitySection

packages/lu/src/parser/lufile/generated/LUFileLexer.interp

@@ -19,26 +19,27 @@ COMMA=18
 HAS_ROLES_LABEL=19
 HAS_FEATURES_LABEL=20
 NEW_ENTITY_TYPE_IDENTIFIER=21
-NEW_COMPOSITE_ENTITY=22
-NEW_REGEX_ENTITY=23
-NEW_ENTITY_IDENTIFIER=24
-NEW_ENTITY_IDENTIFIER_WITH_WS=25
-NEWLINE_IN_NAME=26
-IDENTIFIER=27
-DOT=28
-ESCAPE_CHARACTER=29
-EXPRESSION=30
-TEXT=31
-NEWLINE_IN_ENTITY=32
-COMPOSITE_ENTITY=33
-REGEX_ENTITY=34
-ENTITY_TEXT=35
-COLON_MARK=36
-NEWLINE_IN_QNA=37
-QNA_TEXT=38
+PHRASE_LIST_LABEL=22
+NEW_COMPOSITE_ENTITY=23
+NEW_REGEX_ENTITY=24
+NEW_ENTITY_IDENTIFIER=25
+NEW_ENTITY_IDENTIFIER_WITH_WS=26
+NEWLINE_IN_NAME=27
+IDENTIFIER=28
+DOT=29
+ESCAPE_CHARACTER=30
+EXPRESSION=31
+TEXT=32
+NEWLINE_IN_ENTITY=33
+COMPOSITE_ENTITY=34
+REGEX_ENTITY=35
+ENTITY_TEXT=36
+COLON_MARK=37
+NEWLINE_IN_QNA=38
+QNA_TEXT=39
 '$'=9
 '@'=10
 '='=17
 ','=18
-'.'=28
-':'=36
+'.'=29
+':'=37

packages/lu/src/parser/lufile/generated/LUFileLexer.js

@@ -19,26 +19,27 @@ COMMA=18
 HAS_ROLES_LABEL=19
 HAS_FEATURES_LABEL=20
 NEW_ENTITY_TYPE_IDENTIFIER=21
-NEW_COMPOSITE_ENTITY=22
-NEW_REGEX_ENTITY=23
-NEW_ENTITY_IDENTIFIER=24
-NEW_ENTITY_IDENTIFIER_WITH_WS=25
-NEWLINE_IN_NAME=26
-IDENTIFIER=27
-DOT=28
-ESCAPE_CHARACTER=29
-EXPRESSION=30
-TEXT=31
-NEWLINE_IN_ENTITY=32
-COMPOSITE_ENTITY=33
-REGEX_ENTITY=34
-ENTITY_TEXT=35
-COLON_MARK=36
-NEWLINE_IN_QNA=37
-QNA_TEXT=38
+PHRASE_LIST_LABEL=22
+NEW_COMPOSITE_ENTITY=23
+NEW_REGEX_ENTITY=24
+NEW_ENTITY_IDENTIFIER=25
+NEW_ENTITY_IDENTIFIER_WITH_WS=26
+NEWLINE_IN_NAME=27
+IDENTIFIER=28
+DOT=29
+ESCAPE_CHARACTER=30
+EXPRESSION=31
+TEXT=32
+NEWLINE_IN_ENTITY=33
+COMPOSITE_ENTITY=34
+REGEX_ENTITY=35
+ENTITY_TEXT=36
+COLON_MARK=37
+NEWLINE_IN_QNA=38
+QNA_TEXT=39
 '$'=9
 '@'=10
 '='=17
 ','=18
-'.'=28
-':'=36
+'.'=29
+':'=37

packages/lu/src/parser/lufile/generated/LUFileLexer.tokens

@@ -462,7 +462,7 @@ const parseFeatureSections = function(parsedContent, featuresToProcess) {
             let intentExists = parsedContent.LUISJsonStructure.intents.find(item => item.name === section.Name);
             if (intentExists !== undefined) {
                 // verify the list of features requested have all been defined.
-                let featuresList = section.Features.split(/[,;]/g).map(item => item.trim());
+                let featuresList = section.Features.split(/[,;]/g).map(item => item.trim().replace(/^[\'\"]|[\'\"]$/g, ""));
                 (featuresList || []).forEach(feature => {
                     let entityExists = (parsedContent.LUISJsonStructure.flatListOfEntityAndRoles || []).find(item => item.name == feature || item.name == `${feature}(interchangeable)`);
                     let featureIntentExists = (parsedContent.LUISJsonStructure.intents || []).find(item => item.name == feature);
@@ -1096,7 +1096,12 @@ const parseAndHandleEntityV2 = function (parsedContent, luResource, log, locale)
     if (entities && entities.length > 0) {
         for (const entity of entities) {
             if (entity.Type !== INTENTTYPE) {
-                entity.Name = entity.Name.replace(/^[\'\"]|[\'\"]$/g, "");
+                if (entity.Name.endsWith(PLCONSTS.INTERCHANGEABLE)) {
+                    entity.Name = entity.Name.slice(0, entity.Name.length - PLCONSTS.INTERCHANGEABLE.length).trim().replace(/^[\'\"]|[\'\"]$/g, "") + PLCONSTS.INTERCHANGEABLE
+                } else {
+                    entity.Name = entity.Name.replace(/^[\'\"]|[\'\"]$/g, "");
+                }
+                
                 let entityName = entity.Name.endsWith('=') ? entity.Name.slice(0, entity.Name.length - 1) : entity.Name;
                 let entityType = !entity.Type ? getEntityType(entity.Name, entities) : entity.Type;
                 if (!entityType) {

packages/lu/src/parser/lufile/generated/LUFileParser.interp

@@ -93,6 +93,10 @@ describe('luis:convert', () => {
         await assertToJSON('./../../fixtures/examples/9a.lu', './../../fixtures/verified/9a.json', '9a')
     })
 
+    it('luis:convert phraselist entity types are parsed correctly', async () => {
+        await assertToJSON('./../../fixtures/examples/13.lu', './../../fixtures/verified/13.json', '13')
+    })
+
     it('Parse to LU instance', async () => {
         let luFile = `
         @ ml test