fix lu parser error when invalid square brackets occur in utterances (#825)

* fix lu parser error when [] in utterances

* add more corner cases

Author: feich-ms

packages/lu/src/parser/lufile/LUFileLexer.g4

@@ -77,12 +77,8 @@ AT
   : '@' -> pushMode(NEW_ENTITY_MODE)
   ;
 
-IMPORT_DESC
-  : '[' .*? ']'
-  ;
-
-IMPORT_PATH
-  : '(' .*? ')'
+IMPORT
+  : '[' ~[\r\n[\]]*? ']' '(' ~[\r\n()]*? ')'
   ;
 
 FILTER_MARK

packages/lu/src/parser/lufile/LUFileParser.g4

@@ -167,7 +167,7 @@ importSection
     ;
 
 importDefinition
-    : IMPORT_DESC IMPORT_PATH
+    : WS* IMPORT WS*
     ;
 
 qnaSection

packages/lu/src/parser/lufile/generated/LUFileLexer.interp

@@ -8,38 +8,37 @@ HASH=7
 DASH=8
 DOLLAR=9
 AT=10
-IMPORT_DESC=11
-IMPORT_PATH=12
-FILTER_MARK=13
-QNA_ID_MARK=14
-MULTI_LINE_TEXT=15
-PROMPT_MARK=16
-INVALID_TOKEN_DEFAULT_MODE=17
-EQUAL=18
-COMMA=19
-HAS_ROLES_LABEL=20
-HAS_FEATURES_LABEL=21
-NEW_ENTITY_TYPE_IDENTIFIER=22
-NEW_COMPOSITE_ENTITY=23
-NEW_REGEX_ENTITY=24
-NEW_ENTITY_IDENTIFIER=25
-NEW_ENTITY_IDENTIFIER_WITH_WS=26
-NEWLINE_IN_NAME=27
-IDENTIFIER=28
-DOT=29
-ESCAPE_CHARACTER=30
-EXPRESSION=31
-TEXT=32
-NEWLINE_IN_ENTITY=33
-COMPOSITE_ENTITY=34
-REGEX_ENTITY=35
-ENTITY_TEXT=36
-COLON_MARK=37
-NEWLINE_IN_QNA=38
-QNA_TEXT=39
+IMPORT=11
+FILTER_MARK=12
+QNA_ID_MARK=13
+MULTI_LINE_TEXT=14
+PROMPT_MARK=15
+INVALID_TOKEN_DEFAULT_MODE=16
+EQUAL=17
+COMMA=18
+HAS_ROLES_LABEL=19
+HAS_FEATURES_LABEL=20
+NEW_ENTITY_TYPE_IDENTIFIER=21
+NEW_COMPOSITE_ENTITY=22
+NEW_REGEX_ENTITY=23
+NEW_ENTITY_IDENTIFIER=24
+NEW_ENTITY_IDENTIFIER_WITH_WS=25
+NEWLINE_IN_NAME=26
+IDENTIFIER=27
+DOT=28
+ESCAPE_CHARACTER=29
+EXPRESSION=30
+TEXT=31
+NEWLINE_IN_ENTITY=32
+COMPOSITE_ENTITY=33
+REGEX_ENTITY=34
+ENTITY_TEXT=35
+COLON_MARK=36
+NEWLINE_IN_QNA=37
+QNA_TEXT=38
 '$'=9
 '@'=10
-'='=18
-','=19
-'.'=29
-':'=37
+'='=17
+','=18
+'.'=28
+':'=36

packages/lu/src/parser/lufile/generated/LUFileLexer.js

@@ -8,38 +8,37 @@ HASH=7
 DASH=8
 DOLLAR=9
 AT=10
-IMPORT_DESC=11
-IMPORT_PATH=12
-FILTER_MARK=13
-QNA_ID_MARK=14
-MULTI_LINE_TEXT=15
-PROMPT_MARK=16
-INVALID_TOKEN_DEFAULT_MODE=17
-EQUAL=18
-COMMA=19
-HAS_ROLES_LABEL=20
-HAS_FEATURES_LABEL=21
-NEW_ENTITY_TYPE_IDENTIFIER=22
-NEW_COMPOSITE_ENTITY=23
-NEW_REGEX_ENTITY=24
-NEW_ENTITY_IDENTIFIER=25
-NEW_ENTITY_IDENTIFIER_WITH_WS=26
-NEWLINE_IN_NAME=27
-IDENTIFIER=28
-DOT=29
-ESCAPE_CHARACTER=30
-EXPRESSION=31
-TEXT=32
-NEWLINE_IN_ENTITY=33
-COMPOSITE_ENTITY=34
-REGEX_ENTITY=35
-ENTITY_TEXT=36
-COLON_MARK=37
-NEWLINE_IN_QNA=38
-QNA_TEXT=39
+IMPORT=11
+FILTER_MARK=12
+QNA_ID_MARK=13
+MULTI_LINE_TEXT=14
+PROMPT_MARK=15
+INVALID_TOKEN_DEFAULT_MODE=16
+EQUAL=17
+COMMA=18
+HAS_ROLES_LABEL=19
+HAS_FEATURES_LABEL=20
+NEW_ENTITY_TYPE_IDENTIFIER=21
+NEW_COMPOSITE_ENTITY=22
+NEW_REGEX_ENTITY=23
+NEW_ENTITY_IDENTIFIER=24
+NEW_ENTITY_IDENTIFIER_WITH_WS=25
+NEWLINE_IN_NAME=26
+IDENTIFIER=27
+DOT=28
+ESCAPE_CHARACTER=29
+EXPRESSION=30
+TEXT=31
+NEWLINE_IN_ENTITY=32
+COMPOSITE_ENTITY=33
+REGEX_ENTITY=34
+ENTITY_TEXT=35
+COLON_MARK=36
+NEWLINE_IN_QNA=37
+QNA_TEXT=38
 '$'=9
 '@'=10
-'='=18
-','=19
-'.'=29
-':'=37
+'='=17
+','=18
+'.'=28
+':'=36

packages/lu/src/parser/lufile/generated/LUFileLexer.tokens

@@ -9,32 +9,37 @@ class ImportSection {
      */
     constructor(parseTree) {
         this.ParseTree = parseTree;
+        this.Errors = []
         this.SectionType = LUSectionTypes.IMPORTSECTION;
-        this.Description = this.ExtractDescription(parseTree);
-        let result = this.ExtractPath(parseTree);
-        this.Path = result.importPath;
-        this.Errors = result.errors;
+        let result = this.ExtractDescriptionAndPath(parseTree);
+        this.Description = result.description;
+        this.Path = result.path;
         this.Id = `${this.SectionType}_${this.Path}`;;
     }
 
-    ExtractDescription(parseTree) {
-        return parseTree.importDefinition().IMPORT_DESC().getText();
-    }
+    ExtractDescriptionAndPath(parseTree) {
+        let importRegex = new RegExp(/\[([^\]]*)\]\(([^\)]*)\)/);
+        let importStr = parseTree.importDefinition().IMPORT().getText();
+
+        let description
+        let path
 
-    ExtractPath(parseTree) {
-        let errors = [];
-        let importPath = parseTree.importDefinition().IMPORT_PATH().getText().replace('(', '').replace(')', '');
-        if (importPath === undefined || importPath === '') {
-            let errorMsg = `LU file reference path is empty: "${parseTree.getText()}"`;
-            let error = BuildDiagnostic({
-                message: errorMsg,
-                context: parseTree
-            })
+        let groups = importStr.match(importRegex);
+        if (groups && groups.length === 3) {
+            description = groups[1].trim();
+            path = groups[2].trim();
+            if (path === undefined || path === '') {
+                let errorMsg = `LU file reference path is empty: "${parseTree.getText()}"`;
+                let error = BuildDiagnostic({
+                    message: errorMsg,
+                    context: parseTree
+                })
 
-            errors.push(error);
+                this.Errors.push(error);
+            }
         }
 
-        return { importPath, errors };
+        return { description, path }
     }
 }
 

packages/lu/src/parser/lufile/generated/LUFileParser.interp

@@ -71,7 +71,7 @@ describe('QnA document', function() {
             luMerger.Build([new luObj(qnaContent, new luOptions('stdin'))], false, undefined, findLuFiles)
                 .then(res => done(res))
                 .catch(err => {
-                    assert(err.text.includes("line 7:18 - line 7:19"))
+                    assert(err.text.includes("line 9:12 - line 9:15"))
                     done()
                 })
         })