Enable escaping square brackets and parenthesis to avoid being recognized as pattern (#1142)

* support to excape square brackets and parenthesis

* fix logic for deduplication of utterance

Author: feich-ms

packages/lu/src/parser/lufile/parseFileContents.js

@@ -1257,7 +1257,7 @@ const parseAndHandleSimpleIntentSection = function (parsedContent, luResource, c
                         }
                     } else {
                         if(!hashTable[uttHash]) {
-                            let utteranceObject = new helperClass.utterances(utterance, intentName, []);
+                            let utteranceObject = new helperClass.utterances(utterance.replace(/\\[\[\]\(\)]/gi, match => match.slice(1)), intentName, []);
                             parsedContent.LUISJsonStructure.utterances.push(utteranceObject);
                             hashTable[uttHash] = utteranceObject;
                         }

packages/lu/src/parser/luis/luConverter.js

@@ -111,10 +111,11 @@ const parseUtterancesToLu = function(utterances, luisJSON){
         } else {
             // will not add escape char for pattern utterances since brackets are strictly used in pattern
             // so there are no exceptions that need to be handled in pattern
-            if (helpers.isUtterancePattern(utterance)) {
+            if (utterance.isPattern) {
                 updatedText = utterance.text;
             } else {
-                let tokenizedText = utterance.text.split('');
+                updatedText = utterance.text.replace(/[\[\]\(\)]/gi, match => `\\${match}`);
+                let tokenizedText = updatedText.split('');
                 tokenizedText.forEach(function (token, index) {
                     tokenizedText[index] = EscapeCharsInUtterance.includes(token) ? `\\${token}` : token;
                 });
@@ -511,9 +512,9 @@ const updateUtterancesList = function (srcCollection, tgtCollection, attribute)
             addUtteranceToCollection(attribute, srcItem, matchInTarget);
             return;
         }
-        if(!matchInTarget.utterances.find(item => item.text == srcItem[attribute])) {
+        if(!matchInTarget.utterances.find(item => item.text == srcItem[attribute] && ((item.isPattern && attribute !== 'text') || (!item.isPattern && attribute === 'text')))) {
             addUtteranceToCollection(attribute, srcItem, matchInTarget);
-            return;
+            return; 
         }
     });
 }
@@ -528,7 +529,9 @@ const addUtteranceToCollection = function (attribute, srcItem, matchInTarget) {
     if(attribute === 'text') {
         matchInTarget.utterances.push(srcItem); 
     } else {
-        matchInTarget.utterances.push(new helperClasses.utterances(srcItem.pattern.replace('{', '{@'),srcItem.intent,[]));
+        let utteranceFromPattern = new helperClasses.utterances(srcItem.pattern.replace('{', '{@'),srcItem.intent,[]);
+        utteranceFromPattern.isPattern = true;
+        matchInTarget.utterances.push(utteranceFromPattern);
     }
 }
 

packages/lu/src/parser/utils/helpers.js

@@ -187,7 +187,7 @@ const helpers = {
         if (this.isUtteranceLinkRef(utterance)) return false;
 
         // patterns must have at least one [optional] and or one (group | text)
-        let detectPatternRegex = /(\[.*?\])|(\(.*?(\|.*?)+\))/gi;
+        let detectPatternRegex = /(\[.*(?<!\\)\])|(\(.*?(\|.*?)+(?<!\\)\))/gi;
         return detectPatternRegex.test(utterance);
     },
     hashCode : function(s) {

packages/lu/test/commands/luis/convert.test.js

@@ -109,6 +109,14 @@ describe('luis:convert', () => {
         await assertToJSON('./../../fixtures/examples/newEntityIncludes.lu', './../../fixtures/verified/newEntityIncludes.json')
     })
 
+    it('luis:convert utterance escaping square brackets and parenthesis correctly to json', async () => {
+        await assertToJSON('./../../fixtures/verified/escapeSquareBrackets.lu', './../../fixtures/verified/escapeSquareBrackets.json')
+    })
+
+    it('luis:convert utterance escaping square brackets and parenthesis correctly to json to lu', async () => {
+        await assertToLu('./../../fixtures/verified/escapeSquareBrackets.json', './../../fixtures/verified/escapeSquareBrackets.lu')
+    })
+
     it('Parse to LU instance', async () => {
         let luFile = `
         @ ml test

packages/lu/test/fixtures/verified/escapeSquareBrackets.json

@@ -0,0 +1,61 @@
+{
+  "intents": [
+    {
+      "name": "None"
+    },
+    {
+      "name": "test"
+    }
+  ],
+  "entities": [],
+  "composites": [],
+  "closedLists": [],
+  "regex_entities": [],
+  "regex_features": [],
+  "utterances": [
+    {
+      "text": "who (was|is|will be) {employeelistentity}['s] manager [([in]|[on]){datetimev2}?]",
+      "intent": "test",
+      "entities": []
+    },
+    {
+      "text": "Text spacing issues [mas 1.14.12] (wcag 2.1) 10",
+      "intent": "test",
+      "entities": []
+    },
+    {
+      "text": "who will be the champion of this {eventName}?",
+      "intent": "test",
+      "entities": []
+    }
+  ],
+  "patterns": [
+    {
+      "pattern": "who (was|is|will be) the champion of that game?",
+      "intent": "test"
+    },
+    {
+      "pattern": "who \\(is\\) the champion of this {eventName}?",
+      "intent": "test"
+    },
+    {
+      "pattern": "who is the best one [in] this game",
+      "intent": "test"
+    }
+  ],
+  "patternAnyEntities": [
+    {
+      "name": "eventName",
+      "explicitList": [],
+      "roles": []
+    }
+  ],
+  "prebuiltEntities": [],
+  "luis_schema_version": "7.0.0",
+  "versionId": "0.1",
+  "name": "abcdefg",
+  "desc": "",
+  "culture": "en-us",
+  "tokenizerVersion": "1.0.0",
+  "phraselists": []
+}

packages/lu/test/fixtures/verified/escapeSquareBrackets.lu

@@ -0,0 +1,40 @@
+
+> LUIS application information
+> !# @app.name = abcdefg
+> !# @app.versionId = 0.1
+> !# @app.culture = en-us
+> !# @app.luis_schema_version = 7.0.0
+> !# @app.tokenizerVersion = 1.0.0
+
+
+> # Intent definitions
+
+# None
+
+
+# test
+- who \(was|is|will be\) \{employeelistentity\}\['s\] manager \[\(\[in\]|\[on\]\)\{datetimev2\}?\]
+- Text spacing issues \[mas 1.14.12\] \(wcag 2.1\) 10
+- who will be the champion of this \{eventName\}?
+- who (was|is|will be) the champion of that game?
+- who \(is\) the champion of this {@eventName}?
+- who is the best one [in] this game
+
+
+> # Entity definitions
+
+
+> # PREBUILT Entity definitions
+
+
+> # Phrase list definitions
+
+
+> # List entities
+
+> # RegEx entities
+
+
+> # Pattern.Any entities
+
+@ patternany eventName

packages/luis/test/fixtures/verified/luis_sorted.lu

@@ -35,7 +35,8 @@
 
 
 # DeleteAlarm
-- delete the {alarmTime} alarm
+- delete the \{alarmTime\} alarm
+- delete the {@alarmTime} alarm
 - remove the {@alarmTime} alarm