fix pattern issue in cross train (#759)

Author: feich-ms

packages/lu/src/parser/cross-train/crossTrainer.js

@@ -12,6 +12,7 @@ const DiagnosticSeverity = require('../lufile/diagnostic').DiagnosticSeverity
 const fileHelper = require('../../utils/filehelper')
 const exception = require('../utils/exception')
 const retCode = require('../utils/enums/CLI-errors')
+const prebuiltEntityTypes = require('../utils/enums/luisbuiltintypes').consolidatedList
 const NEWLINE = require('os').EOL
 const path = require('path')
 const QNA_GENERIC_SOURCE = "custom editorial"
@@ -185,10 +186,10 @@ const mergeBrothersInterruption = function (resource, result, intentName) {
     let brotherUtterances = []
     brotherSections.forEach(s => {
       if (s.SectionType === LUSectionTypes.SIMPLEINTENTSECTION) {
-        brotherUtterances = brotherUtterances.concat(s.UtteranceAndEntitiesMap.map(u => u.utterance))
+        brotherUtterances = brotherUtterances.concat(s.UtteranceAndEntitiesMap.map(u => u.utterance).filter(i => !patternWithPrebuiltEntity(i)))
       } else {
         s.SimpleIntentSections.forEach(section => {
-          brotherUtterances = brotherUtterances.concat(section.UtteranceAndEntitiesMap.map(u => u.utterance))
+          brotherUtterances = brotherUtterances.concat(section.UtteranceAndEntitiesMap.map(u => u.utterance).filter(i => !patternWithPrebuiltEntity(i)))
         })
       }
     })
@@ -300,21 +301,20 @@ const removeDupUtterances = function (resource) {
 
 const extractIntentUtterances = function(resource, intentName) {
   const intentSections = resource.Sections.filter(s => s.SectionType === LUSectionTypes.SIMPLEINTENTSECTION || s.SectionType === LUSectionTypes.NESTEDINTENTSECTION)
-  const curlyRe = /.*\{.*\}.*/
 
   let intentUtterances = []
   if (intentName && intentName !== '') {
     const specificSections = intentSections.filter(s => s.Name === intentName)
     if (specificSections.length > 0) {
-      intentUtterances = intentUtterances.concat(specificSections[0].UtteranceAndEntitiesMap.map(u => u.utterance).filter(i => curlyRe.exec(i) === null))
+      intentUtterances = intentUtterances.concat(specificSections[0].UtteranceAndEntitiesMap.map(u => u.utterance))
     }
   } else {
     intentSections.forEach(s => {
       if (s.SectionType === LUSectionTypes.SIMPLEINTENTSECTION) {
-        intentUtterances = intentUtterances.concat(s.UtteranceAndEntitiesMap.map(u => u.utterance).filter(i => curlyRe.exec(i) === null))
+        intentUtterances = intentUtterances.concat(s.UtteranceAndEntitiesMap.map(u => u.utterance))
       } else {
         s.SimpleIntentSections.forEach(section => {
-          intentUtterances = intentUtterances.concat(section.UtteranceAndEntitiesMap.map(u => u.utterance).filter(i => curlyRe.exec(i) === null))
+          intentUtterances = intentUtterances.concat(section.UtteranceAndEntitiesMap.map(u => u.utterance))
         })
       }
   })}
@@ -397,7 +397,7 @@ const qnaCrossTrainCore = function (luResource, qnaResource, fileName, interrupt
   }
 
   // construct questions content
-  dedupedQuestions = dedupedQuestions.map(q => '- '.concat(q))
+  dedupedQuestions = dedupedQuestions.map(q => '- '.concat(q)).filter(i => !patternWithPrebuiltEntity(i))
   let questionsContent = dedupedQuestions.join(NEWLINE)
 
   // cross training comments
@@ -440,8 +440,11 @@ const qnaCrossTrainCore = function (luResource, qnaResource, fileName, interrupt
     trainedQnaResource = new SectionOperator(new LUResource([], modelInforContent, [])).addSection(qnaContents)
   }
 
+  // remove utterances with curly brackets
+  const utterancesWithoutPatterns = utterances.filter(i => /{([^}]+)}/g.exec(i) === null)
+
   // remove utterances which are duplicated with local qna questions
-  const dedupedUtterances = utterances.filter(u => !questions.includes(u))
+  const dedupedUtterances = utterancesWithoutPatterns.filter(u => !questions.includes(u))
 
   // construct new question content for qna resource
   let utterancesContent = dedupedUtterances.join(NEWLINE + '- ')
@@ -494,4 +497,23 @@ const pretreatment = function (luContents, qnaContents) {
    const qnaObjectArray = fileHelper.getParsedObjects(qnaContents)
 
    return {luObjectArray, qnaObjectArray}
+}
+
+const patternWithPrebuiltEntity = function (utterance) {
+  let patternAnyEntity
+  let matchedEntity = /{([^}]+)}/g.exec(utterance)
+
+  if (matchedEntity !== null) {
+    patternAnyEntity = matchedEntity[1]
+
+    if (patternAnyEntity && patternAnyEntity.startsWith('@')) {
+      patternAnyEntity = patternAnyEntity.slice(1)
+    }
+
+    if (prebuiltEntityTypes.includes(patternAnyEntity)) {
+      return true
+    }
+  }
+
+  return false
 }

packages/lu/test/parser/cross-train/crossTrainer.test.js

@@ -541,4 +541,96 @@ describe('luis:cross training tests among lu and qna contents', () => {
     assert.equal(luResult.get('./dia2/dia2.lu').Sections[1].Name, '_Interruption')
     assert.equal(luResult.get('./dia2/dia2.lu').Sections[1].Body, `- I want to travel to Seattle${NEWLINE}- book a hotel for me`)
   })
+
+  it('luis:cross training can get expected result when handling patterns', () => {
+    let luContentArray = []
+    let qnaContentArray = []
+
+    luContentArray.push({
+      content:
+        `# dia1_trigger
+        - book a hotel for me
+        - book a hotel for {personName}
+        - book a hotel for {name}
+        
+        # dia2_trigger
+        - book a flight for me
+        - book a train ticket for me`,
+      id: 'main.lu'})
+
+    qnaContentArray.push({
+      content:
+        `# ?user guide
+
+        **Filters:**
+        - aa=bb
+        
+        \`\`\`
+            Here is the [user guide](http://contoso.com/userguide.pdf)
+        \`\`\`
+        
+        # ?tell joke
+        \`\`\`
+            tell a funny joke
+        \`\`\``,
+      id: 'main.qna'}
+    )
+
+    luContentArray.push({
+      content:
+        `> !# @app.name = my luis application
+        
+        # hotelLevel
+        - I need a four star hotel
+        
+        # hotelLocation
+        - can I book a hotel near space needle`,
+      id: 'dia1.lu'}
+    )
+
+    luContentArray.push({
+      content:
+        `# dia3_trigger
+        - book a flight from {fromLocation = Seattle} to {toLocation = Beijing}
+        
+        # dia4_trigger
+        - book a train ticket from Seattle to Portland`,
+      id: 'dia2.lu'}
+    )
+
+    let crossTrainConfig = {
+      rootIds: [
+        'main.lu'
+      ],
+      triggerRules: {
+        'main.lu': {
+          'dia1_trigger': 'dia1.lu',
+          'dia2_trigger': 'dia2.lu'
+        }
+      },
+      intentName: '_Interruption',
+      verbose: true
+    }
+
+    const trainedResult = crossTrainer.crossTrain(luContentArray, qnaContentArray, crossTrainConfig)
+    const luResult = trainedResult.luResult
+    const qnaResult = trainedResult.qnaResult
+
+    assert.equal(luResult.get('main.lu').Sections[2].Name, 'DeferToRecognizer_QnA_main')
+    assert.equal(luResult.get('main.lu').Sections[2].Body, `- user guide${NEWLINE}- tell joke`)
+
+    assert.equal(qnaResult.get('main.qna').Sections[2].Answer, 'intent=DeferToRecognizer_LUIS_main')
+    assert.equal(qnaResult.get('main.qna').Sections[2].FilterPairs[0].key, 'dialogName')
+    assert.equal(qnaResult.get('main.qna').Sections[2].FilterPairs[0].value, 'main')
+    assert.equal(qnaResult.get('main.qna').Sections[2].Questions.length, 3)
+    assert.equal(qnaResult.get('main.qna').Sections[2].Questions[0], 'book a hotel for me')
+    assert.equal(qnaResult.get('main.qna').Sections[2].Questions[1], 'book a flight for me')
+    assert.equal(qnaResult.get('main.qna').Sections[2].Questions[2], 'book a train ticket for me')
+
+    assert.equal(luResult.get('dia1.lu').Sections[3].Name, '_Interruption')
+    assert.equal(luResult.get('dia1.lu').Sections[3].Body, `- book a flight for me${NEWLINE}- book a train ticket for me${NEWLINE}- user guide${NEWLINE}- tell joke`)
+
+    assert.equal(luResult.get('dia2.lu').Sections[2].Name, '_Interruption')
+    assert.equal(luResult.get('dia2.lu').Sections[2].Body, `- book a hotel for me${NEWLINE}- book a hotel for {name}${NEWLINE}- user guide${NEWLINE}- tell joke`)
+  })
 })