fix import filtered intents not work in luis:cross-train (#1232)

Shuai Wang · Josh Gummersall · web-flow · commit 3f9ee150f39e · 2021-06-03T10:37:19.000-07:00
* fix filter intent not working

* rewrite export statement

* modify import

* add a test case

* fix a import bug

* extract code to a function

* fix issues

* fix import syntax and other minor changes

* fix deep clone

Co-authored-by: Josh Gummersall &lt;papertrail@josh.standardlabs.dev&gt;
diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml
diff --git a/packages/lu/src/parser/cross-train/cross-train.js b/packages/lu/src/parser/cross-train/cross-train.js
@@ -8,6 +8,8 @@ const fs = require('fs-extra')
 const filehelper = require('../../utils/filehelper')
 const fileExtEnum = require('../utils/helpers').FileExtTypeEnum
 const crossTrainer = require('./crossTrainer')
+const LuisBuilder = require('./../luis/luisCollate')
+const {parseUtterancesToLu} = require('./../luis/luConverter')
 
 module.exports = {
   /**
@@ -35,12 +37,32 @@ module.exports = {
     let importResolver = async function (id, idsToFind) {
       let importedContents = []
       const idWithoutExt = path.basename(id, path.extname(id))
-      const locale = /\w\.\w/.test(idWithoutExt) ? idWithoutExt.split('.').pop() : defaultLocale;
-      for (let idx = 0; idx < idsToFind.length; idx++) {
-        let file = idsToFind[idx]
+      const locale = /\w\.\w/.test(idWithoutExt) ? idWithoutExt.split('.').pop() : defaultLocale
+      const intentFilteringHandler = async (filePathOrFound, intent, isAbsolutePath) => {
+        let content = filePathOrFound
+        if (isAbsolutePath) {
+          importFile = (await filehelper.getFileContent(filePathOrFound, fileExtEnum.LUFile))[0]
+          content = importFile ? [importFile.content] : ''
+        } 
+
+        const luObj = await LuisBuilder.build(content, false, undefined, importResolver)
+
+        const matchedUtterence = luObj.utterances.find(e => e.intent === intent)
+        const fileContent = `# ${intent}\r\n${parseUtterancesToLu([matchedUtterence], luObj)}`
+        const foundItem = isAbsolutePath ? importFile : filePathOrFound[0]
+        const cloned = JSON.parse(JSON.stringify(foundItem))
+        cloned.content = fileContent
+        importedContents.push(cloned)
+      }
+
+      for (const file of idsToFind) {
         if (path.isAbsolute(file.filePath)) {
           if (file.filePath.endsWith(fileExtEnum.LUFile)) {
-            importedContents.push(...await filehelper.getFilesContent(file.filePath, fileExtEnum.LUFile))
+            if (!file.intent) {
+              importedContents.push(...await filehelper.getFilesContent(file.filePath, fileExtEnum.LUFile))
+            } else {
+              await intentFilteringHandler(file.filePath, file.intent, true)
+            }
           } else if (file.filePath.endsWith(fileExtEnum.QnAFile)) {
             importedContents.push(...await filehelper.getFilesContent(file.filePath, fileExtEnum.QnAFile))
           }
@@ -58,7 +80,11 @@ module.exports = {
             }
 
             if(found.length > 0) {
-              importedContents.push(...found)
+              if (!file.intent) {
+                importedContents.push(...found)
+              } else {
+                await intentFilteringHandler(found, file.intent, false)
+              }
             } else {
               const matchedLuisFiles = typedContents.filter(content => path.basename(content.fullPath) === id)
               for (const matchFile of matchedLuisFiles) {
diff --git a/packages/lu/src/parser/cross-train/crossTrainer.js b/packages/lu/src/parser/cross-train/crossTrainer.js
@@ -511,7 +511,7 @@ const qnaAddMetaData = function (qnaResource, fileName) {
 const parseAndValidateContent = async function (objectArray, verbose, importResolver, fileExt) {
   let fileIdToResourceMap = new Map()
   let allEmpty = true
-  for (const object of objectArray) {    
+  for (const object of objectArray) {
     let fileContent = object.content
     let objectId = object.id
     if (object.content && object.content !== '') {
diff --git a/packages/lu/src/parser/lufile/classes/filesToParse.js b/packages/lu/src/parser/lufile/classes/filesToParse.js
@@ -9,10 +9,14 @@ class FileToParse {
     /**
      * @property {Boolean} includeInCollate
      */
-    constructor(filePath, includeInCollate) {
+    /**
+     * @property {string} intent
+     */
+    constructor(filePath, includeInCollate, intent = '') {
         this.filePath = filePath?filePath:'';
         if(includeInCollate === undefined) this.includeInCollate = true;
         else this.includeInCollate = includeInCollate;
+        this.intent = intent
     }
 }
 
diff --git a/packages/lu/src/parser/lufile/parseFileContents.js b/packages/lu/src/parser/lufile/parseFileContents.js
@@ -967,7 +967,7 @@ const parseAndHandleSimpleIntentSection = function (parsedContent, luResource, c
                     }
                     let parsedLinkUriInUtterance = helpers.parseLinkURISync(utterance);
                     // examine and add these to filestoparse list.
-                    parsedContent.additionalFilesToParse.push(new fileToParse(parsedLinkUriInUtterance.fileName, false));
+                    parsedContent.additionalFilesToParse.push(new fileToParse(parsedLinkUriInUtterance.fileName, false, parsedLinkUriInUtterance.path));
                 }
                
                 (utteranceAndEntities.entities || []).forEach(entity => {
diff --git a/packages/lu/src/parser/luis/luConverter.js b/packages/lu/src/parser/luis/luConverter.js
@@ -550,4 +550,5 @@ const objectSortByStartPos = function (objectArray) {
     return ObjectByStartPos;
 }
 
-module.exports = luisToLuContent
+module.exports = luisToLuContent
+module.exports.parseUtterancesToLu = parseUtterancesToLu
diff --git a/packages/luis/bin/app/Application.lu b/packages/luis/bin/app/Application.lu
@@ -0,0 +1,5 @@
+# InternalIntent
+- Intent within application.lu
+
+# ExternalIntentShouldBeIncluded
+- [ExternalFile](External.lu#ExternalIntentShouldBeIncluded)
diff --git a/packages/luis/bin/app/Application.qna b/packages/luis/bin/app/Application.qna
@@ -0,0 +1,5 @@
+# ? Question In ExternalFile.qna
+
+```
+Answer to question in ExternalFile.qna
+```
diff --git a/packages/luis/bin/app/External.lu b/packages/luis/bin/app/External.lu
@@ -0,0 +1,5 @@
+# ExternalIntentShouldBeIncluded
+- [ExternalFile2](External2.lu#ExternalIntentShouldBeIncluded)
+
+# ExternalIntentShouldNotBeIncluded
+- Intent which should NOT be included
diff --git a/packages/luis/bin/app/External2.lu b/packages/luis/bin/app/External2.lu
@@ -0,0 +1,5 @@
+# ExternalIntentShouldBeIncluded
+- Intent which should be included
+
+# ExternalIntentShouldNotBeIncluded2
+- Intent2 which should NOT be included
diff --git a/packages/luis/bin/cross-train/Application.lu b/packages/luis/bin/cross-train/Application.lu
@@ -0,0 +1,36 @@
+
+> LUIS application information
+> !# @app.versionId = 0.1
+> !# @app.culture = en-us
+> !# @app.luis_schema_version = 3.2.0
+
+
+> # Intent definitions
+
+# InternalIntent
+- Intent within application.lu
+
+
+# ExternalIntentShouldBeIncluded
+- Intent which should be included
+
+
+> # Entity definitions
+
+
+> # PREBUILT Entity definitions
+
+
+> # Phrase list definitions
+
+
+> # List entities
+
+> # RegEx entities
+
+
+
+
+> Source: cross training. Please do not edit these directly!
+# DeferToRecognizer_QnA_Application
+- Question In ExternalFile.qna
diff --git a/packages/luis/bin/cross-train/Application.qna b/packages/luis/bin/cross-train/Application.qna
@@ -0,0 +1,21 @@
+# ? Question In ExternalFile.qna
+
+**Filters:**
+- dialogName=Application
+
+```
+Answer to question in ExternalFile.qna
+```
+
+> Source: cross training. Please do not edit these directly!
+> !# @qna.pair.source = crosstrained
+
+# ? Intent within application.lu
+- Intent which should be included
+
+**Filters:**
+- dialogName=Application
+
+```
+intent=DeferToRecognizer_LUIS_Application
+```
diff --git a/packages/luis/bin/cross-train/External.lu b/packages/luis/bin/cross-train/External.lu
@@ -0,0 +1,31 @@
+
+> LUIS application information
+> !# @app.versionId = 0.1
+> !# @app.culture = en-us
+> !# @app.luis_schema_version = 3.2.0
+
+
+> # Intent definitions
+
+# ExternalIntentShouldBeIncluded
+- Intent which should be included
+
+
+# ExternalIntentShouldNotBeIncluded
+- Intent which should NOT be included
+
+
+> # Entity definitions
+
+
+> # PREBUILT Entity definitions
+
+
+> # Phrase list definitions
+
+
+> # List entities
+
+> # RegEx entities
+
+
diff --git a/packages/luis/bin/cross-train/External2.lu b/packages/luis/bin/cross-train/External2.lu
@@ -0,0 +1,31 @@
+
+> LUIS application information
+> !# @app.versionId = 0.1
+> !# @app.culture = en-us
+> !# @app.luis_schema_version = 3.2.0
+
+
+> # Intent definitions
+
+# ExternalIntentShouldBeIncluded
+- Intent which should be included
+
+
+# ExternalIntentShouldNotBeIncluded2
+- Intent2 which should NOT be included
+
+
+> # Entity definitions
+
+
+> # PREBUILT Entity definitions
+
+
+> # Phrase list definitions
+
+
+> # List entities
+
+> # RegEx entities
+
+
diff --git a/packages/luis/test/commands/luis/crossTrain.test.ts b/packages/luis/test/commands/luis/crossTrain.test.ts
@@ -175,4 +175,16 @@ describe('luis:cross-train tests for lu and qna contents', () => {
       expect(fs.existsSync('./../../../interruptionGen/extra.en-us.lu')).to.be.false
       expect(fs.existsSync('./../../../interruptionGen/chitchat_professional.source.en-us.qna')).to.be.false
     })
+
+    test
+    .stdout()
+    .command(['luis:cross-train',
+      '--in', `${path.join(__dirname, './../../fixtures/testcases/filterIntent')}`,
+      '--config', `${path.join(__dirname, './../../fixtures/testcases/filterIntent/cross-train.config')}`,
+      '--out', './interruptionGen',
+      '--force'])
+    .it('luis:cross training should able to import files with locale and ignore files under the directory specified', async () => {
+      expect(await compareLuFiles('./../../../interruptionGen/Application.lu', './../../fixtures/verified/interruption10/Application.lu')).to.be.true
+      expect(await compareLuFiles('./../../../interruptionGen/Application.qna', './../../fixtures/verified/interruption10/Application.qna')).to.be.true
+    })
 })
diff --git a/packages/luis/test/fixtures/testcases/filterIntent/Application.lu b/packages/luis/test/fixtures/testcases/filterIntent/Application.lu
@@ -0,0 +1,5 @@
+# InternalIntent
+- Intent within application.lu
+
+# ExternalIntentShouldBeIncluded
+- [ExternalFile](External.lu#ExternalIntentShouldBeIncluded)
diff --git a/packages/luis/test/fixtures/testcases/filterIntent/Application.qna b/packages/luis/test/fixtures/testcases/filterIntent/Application.qna
@@ -0,0 +1,5 @@
+# ? Question In ExternalFile.qna
+
+```
+Answer to question in ExternalFile.qna
+```
diff --git a/packages/luis/test/fixtures/testcases/filterIntent/External.lu b/packages/luis/test/fixtures/testcases/filterIntent/External.lu
@@ -0,0 +1,5 @@
+# ExternalIntentShouldBeIncluded
+- Intent which should be included
+
+# ExternalIntentShouldNotBeIncluded
+- Intent which should NOT be included
diff --git a/packages/luis/test/fixtures/testcases/filterIntent/cross-train.config b/packages/luis/test/fixtures/testcases/filterIntent/cross-train.config
@@ -0,0 +1,5 @@
+{
+  "application": {
+    "rootDialog": true
+  }
+}
diff --git a/packages/luis/test/fixtures/verified/interruption10/Application.lu b/packages/luis/test/fixtures/verified/interruption10/Application.lu
@@ -0,0 +1,36 @@
+
+> LUIS application information
+> !# @app.versionId = 0.1
+> !# @app.culture = en-us
+> !# @app.luis_schema_version = 3.2.0
+
+
+> # Intent definitions
+
+# InternalIntent
+- Intent within application.lu
+
+
+# ExternalIntentShouldBeIncluded
+- Intent which should be included
+
+
+> # Entity definitions
+
+
+> # PREBUILT Entity definitions
+
+
+> # Phrase list definitions
+
+
+> # List entities
+
+> # RegEx entities
+
+
+
+
+> Source: cross training. Please do not edit these directly!
+# DeferToRecognizer_QnA_Application
+- Question In ExternalFile.qna
diff --git a/packages/luis/test/fixtures/verified/interruption10/Application.qna b/packages/luis/test/fixtures/verified/interruption10/Application.qna

Original file line number	Diff line number	Diff line change
`@@ -967,7 +967,7 @@ const parseAndHandleSimpleIntentSection = function (parsedContent, luResource, c`
`967`	`967`	`}`
`968`	`968`	`let parsedLinkUriInUtterance = helpers.parseLinkURISync(utterance);`
`969`	`969`	`// examine and add these to filestoparse list.`
`970`		`- parsedContent.additionalFilesToParse.push(new fileToParse(parsedLinkUriInUtterance.fileName, false));`
	`970`	`+ parsedContent.additionalFilesToParse.push(new fileToParse(parsedLinkUriInUtterance.fileName, false, parsedLinkUriInUtterance.path));`
`971`	`971`	`}`
`972`	`972`
`973`	`973`	`(utteranceAndEntities.entities \|\| []).forEach(entity => {`
Original file line number	Diff line number	Diff line change
`@@ -550,4 +550,5 @@ const objectSortByStartPos = function (objectArray) {`
`550`	`550`	`return ObjectByStartPos;`
`551`	`551`	`}`
`552`	`552`
`553`		`-module.exports = luisToLuContent`
	`553`	`+module.exports = luisToLuContent`
	`554`	`+module.exports.parseUtterancesToLu = parseUtterancesToLu`
-Original file line number
+Diff line change
@@ @@ -0,0 +1,5 @@ @@
 +# ? Question In ExternalFile.qna
++
 +```
 +Answer to question in ExternalFile.qna
 +```