support special chars in intent name for lu antlr parser (#1058)

Co-authored-by: Emilio Munoz <[email protected]>

Author: feich-ms

packages/lu/src/parser/lufile/LUFileLexer.g4

@@ -28,9 +28,6 @@ fragment X: 'x' | 'X';
 fragment Y: 'y' | 'Y';
 fragment Z: 'z' | 'Z';
 
-fragment LETTER: 'a'..'z' | 'A'..'Z';
-fragment NUMBER: '0'..'9';
-
 fragment WHITESPACE
   : ' '|'\t'|'\ufeff'|'\u00a0'
   ;
@@ -169,11 +166,7 @@ NEWLINE_IN_NAME
   ;
 
 IDENTIFIER
-  : (LETTER | NUMBER | '_') (LETTER | NUMBER | '-' | '_')*
-  ;
-
-DOT
-  : '.'
+  : ~[ \t\r\n]+
   ;
 
 mode LIST_BODY_MODE;

packages/lu/src/parser/lufile/LUFileParser.g4

@@ -42,7 +42,7 @@ nestedIntentName
     ;
 
 nameIdentifier
-    : IDENTIFIER (DOT IDENTIFIER)*
+    : IDENTIFIER
     ;
 
 nestedIntentBodyDefinition

packages/lu/src/parser/lufile/generated/LUFileLexer.interp

@@ -27,20 +27,18 @@ NEW_ENTITY_IDENTIFIER=26
 NEW_ENTITY_IDENTIFIER_WITH_WS=27
 NEWLINE_IN_NAME=28
 IDENTIFIER=29
-DOT=30
-ESCAPE_CHARACTER=31
-EXPRESSION=32
-TEXT=33
-NEWLINE_IN_ENTITY=34
-COMPOSITE_ENTITY=35
-REGEX_ENTITY=36
-ENTITY_TEXT=37
-COLON_MARK=38
-NEWLINE_IN_QNA=39
-QNA_TEXT=40
+ESCAPE_CHARACTER=30
+EXPRESSION=31
+TEXT=32
+NEWLINE_IN_ENTITY=33
+COMPOSITE_ENTITY=34
+REGEX_ENTITY=35
+ENTITY_TEXT=36
+COLON_MARK=37
+NEWLINE_IN_QNA=38
+QNA_TEXT=39
 '$'=9
 '@'=10
 '='=18
 ','=19
-'.'=30
-':'=38
+':'=37

packages/lu/src/parser/lufile/generated/LUFileLexer.js

@@ -27,20 +27,18 @@ NEW_ENTITY_IDENTIFIER=26
 NEW_ENTITY_IDENTIFIER_WITH_WS=27
 NEWLINE_IN_NAME=28
 IDENTIFIER=29
-DOT=30
-ESCAPE_CHARACTER=31
-EXPRESSION=32
-TEXT=33
-NEWLINE_IN_ENTITY=34
-COMPOSITE_ENTITY=35
-REGEX_ENTITY=36
-ENTITY_TEXT=37
-COLON_MARK=38
-NEWLINE_IN_QNA=39
-QNA_TEXT=40
+ESCAPE_CHARACTER=30
+EXPRESSION=31
+TEXT=32
+NEWLINE_IN_ENTITY=33
+COMPOSITE_ENTITY=34
+REGEX_ENTITY=35
+ENTITY_TEXT=36
+COLON_MARK=37
+NEWLINE_IN_QNA=38
+QNA_TEXT=39
 '$'=9
 '@'=10
 '='=18
 ','=19
-'.'=30
-':'=38
+':'=37

packages/lu/src/parser/lufile/generated/LUFileLexer.tokens

@@ -1076,4 +1076,18 @@ describe('parseFile correctly parses utterances', function () {
                         })
                         .catch(err => done(err))
         })
+
+        it ('Correctly parses intent name with special chars', function(done){
+                let testLU = `
+# test'á
+- Hello world
+`;
+                parseFile.parseFile(testLU)
+                        .then(res => {
+                                assert.equal(res.LUISJsonStructure.intents.length, 1);
+                                assert.equal(res.LUISJsonStructure.intents[0].name, "test'á");
+                                done();
+                        })
+                        .catch(err => done(err))
+        })
 })

packages/lu/src/parser/lufile/generated/LUFileParser.interp

@@ -402,6 +402,13 @@ describe('luis:convert', () => {
       let parsedObjects = await parseJsonFiles('./../../../results/root41.json', './../../fixtures/verified/merge_intents_disabled.json')
       expect(parsedObjects[0]).to.deep.equal(parsedObjects[1])
     })
+
+    test
+    .stdout()
+    .command(['luis:convert', '--in', `${path.join(__dirname, './../../fixtures/testcases/intent_name_with_special_chars.json')}`, '--out', './results/root36.lu',])
+    .it('luis:convert lu file with special chars in intent name', async () => {
+      expect(await compareLuFiles('./../../../results/root36.lu', './../../fixtures/verified/intent_name_with_special_chars.lu')).to.be.true
+    })
 })