fix luis cross train unable to reference external files (#1181)

* init

* add path resovle

* fix import path resolve issue

* add tests

* fix lint and unused imports

* modify config name

* retrigger

* refactor

Co-authored-by: Emilio Munoz <[email protected]>

Author: Shuai Wang

packages/lu/src/parser/cross-train/cross-train.js

@@ -4,7 +4,8 @@
  */
 
 const path = require('path')
-const file = require('../../utils/filehelper')
+const fs = require('fs-extra')
+const filehelper = require('../../utils/filehelper')
 const fileExtEnum = require('../utils/helpers').FileExtTypeEnum
 const crossTrainer = require('./crossTrainer')
 
@@ -20,26 +21,43 @@ module.exports = {
    */
   train: async function (input, intentName, config, verbose, trainingOpt) {
     // Get all related file content.
-    const luContents = await file.getFilesContent(input, fileExtEnum.LUFile)
-    const qnaContents = await file.getFilesContent(input, fileExtEnum.QnAFile)
-    const configContent = await file.getConfigContent(config)
+    const luContents = await filehelper.getFilesContent(input, fileExtEnum.LUFile)
+    const qnaContents = await filehelper.getFilesContent(input, fileExtEnum.QnAFile)
+    const configContent = await filehelper.getConfigContent(config)
 
-    let importResolver = async function (_, idsToFind) {
+    let importResolver = async function (id, idsToFind) {
       let importedContents = []
       for (let idx = 0; idx < idsToFind.length; idx++) {
         let file = idsToFind[idx]
         if (path.isAbsolute(file.filePath)) {
           if (file.filePath.endsWith(fileExtEnum.LUFile)) {
-            importedContents.push(...await file.getFilesContent(file.filePath, fileExtEnum.LUFile))
+            importedContents.push(...await filehelper.getFilesContent(file.filePath, fileExtEnum.LUFile))
           } else if (file.filePath.endsWith(fileExtEnum.QnAFile)) {
-            importedContents.push(...await file.getFilesContent(file.filePath, fileExtEnum.QnAFile))
+            importedContents.push(...await filehelper.getFilesContent(file.filePath, fileExtEnum.QnAFile))
           }
         } else {
           const fileName = path.basename(file.filePath)
+          const updateImportedContents = async function(typedContents, fileExt) {
+            const found = typedContents.filter(content => content.id === path.basename(fileName, fileExt))
+            if(found.length > 0) {
+              importedContents.push(...found)
+            } else {
+              const matchedLuisFiles = typedContents.filter(content => path.basename(content.fullPath) === id)
+              for (const matchFile of matchedLuisFiles) {
+                const sourceFileDir = path.dirname(matchFile.fullPath)
+                const targetPath = path.resolve(sourceFileDir, file.filePath)
+                if (fs.existsSync(targetPath)) {
+                  const importContent = await filehelper.getFilesContent(targetPath, fileExt)
+                  importedContents.push(...importContent)
+                }
+              }
+            }
+          }
+
           if (fileName.endsWith(fileExtEnum.LUFile)) {
-            importedContents.push(...luContents.filter(luContent => luContent.id === path.basename(fileName, fileExtEnum.LUFile)))
+            await updateImportedContents(luContents, fileExtEnum.LUFile)
           } else if (fileName.endsWith(fileExtEnum.QnAFile)) {
-            importedContents.push(...qnaContents.filter(qnaContent => qnaContent.id === path.basename(fileName, fileExtEnum.QnAFile)))
+            await updateImportedContents(qnaContents, fileExtEnum.LUFile)
           }
         }
       }
@@ -60,4 +78,4 @@ module.exports = {
 
     return trainedResult
   }
-}
+}

packages/lu/src/utils/filehelper.ts

@@ -176,7 +176,7 @@ export async function getFilesContent(input: string, extType: string) {
   if (fileStat.isFile()) {
     const filePath = path.resolve(input)
     const content = await getContentFromFile(input)
-    return [{id: path.basename(filePath, extType), content}]
+    return [{id: path.basename(filePath, extType), content, fullPath: filePath}]
   }
 
   if (!fileStat.isDirectory()) {
@@ -186,7 +186,7 @@ export async function getFilesContent(input: string, extType: string) {
   return Promise.all(paths.map(async (item: string) => {
     const itemPath = path.resolve(path.join(input, item))
     const content = await getContentFromFile(itemPath)
-    return {id: path.basename(itemPath, extType), content}
+    return {id: path.basename(itemPath, extType), content, fullPath: itemPath}
   }))
 }
 

packages/luis/test/commands/luis/crossTrain.test.ts

@@ -148,4 +148,16 @@ describe('luis:cross-train tests for lu and qna contents', () => {
       expect(await compareLuFiles('./../../../interruptionGen/dia1.lu', './../../fixtures/verified/interruption7/dia1.lu')).to.be.true
       expect(await compareLuFiles('./../../../interruptionGen/dia1.qna', './../../fixtures/verified/interruption7/dia1.qna')).to.be.true
     })
+
+    test
+    .stdout()
+    .command(['luis:cross-train',
+      '--in', `${path.join(__dirname, './../../fixtures/testcases/application')}`,
+      '--config', `${path.join(__dirname, './../../fixtures/testcases/application/cross-train.config')}`,
+      '--out', './interruptionGen',
+      '--force'])
+    .it('luis:cross training should able to import files out of current directory', async () => {
+      expect(await compareLuFiles('./../../../interruptionGen/application.lu', './../../fixtures/verified/interruption8/application.lu')).to.be.true
+      expect(await compareLuFiles('./../../../interruptionGen/application.qna', './../../fixtures/verified/interruption8/application.qna')).to.be.true
+    })
 })

packages/luis/test/fixtures/testcases/LUIS/ExternalFile.lu

@@ -0,0 +1,2 @@
+# ExternalIntent
+- utterance for ExternalIntent

packages/luis/test/fixtures/testcases/QnAMaker/ExternalFile.qna

@@ -0,0 +1,5 @@
+# ? Question In ExternalFile.qna
+
+```
+Answer to question in ExternalFile.qna
+```

packages/luis/test/fixtures/testcases/application/LUIS/application.lu

@@ -0,0 +1,4 @@
+[ExternalFile](../../LUIS/ExternalFile.lu)
+
+# LocalIntent
+- utterance for LocalIntent

packages/luis/test/fixtures/testcases/application/QnAMaker/application.qna

@@ -0,0 +1,7 @@
+[ExternalFile](../../QnaMaker/ExternalFile.qna)
+
+# ? Question In Main application.qna
+
+```
+Answer to question in application.qna
+```

packages/luis/test/fixtures/testcases/application/cross-train.config

@@ -0,0 +1,5 @@
+{
+  "application": {
+    "rootDialog": true
+  }
+}

packages/luis/test/fixtures/verified/interruption8/application.lu

@@ -0,0 +1,37 @@
+
+> LUIS application information
+> !# @app.versionId = 0.1
+> !# @app.culture = en-us
+> !# @app.luis_schema_version = 3.2.0
+
+
+> # Intent definitions
+
+# LocalIntent
+- utterance for LocalIntent
+
+
+# ExternalIntent
+- utterance for ExternalIntent
+
+
+> # Entity definitions
+
+
+> # PREBUILT Entity definitions
+
+
+> # Phrase list definitions
+
+
+> # List entities
+
+> # RegEx entities
+
+
+
+
+> Source: cross training. Please do not edit these directly!
+# DeferToRecognizer_QnA_application
+- Question In Main application.qna
+- Question In ExternalFile.qna

packages/luis/test/fixtures/verified/interruption8/application.qna

@@ -0,0 +1,30 @@
+# ? Question In Main application.qna
+
+**Filters:**
+- dialogName=application
+
+```
+Answer to question in application.qna
+```
+
+# ? Question In ExternalFile.qna
+
+**Filters:**
+- dialogName=application
+
+```
+Answer to question in ExternalFile.qna
+```
+
+> Source: cross training. Please do not edit these directly!
+> !# @qna.pair.source = crosstrained
+
+# ? utterance for LocalIntent
+- utterance for ExternalIntent
+
+**Filters:**
+- dialogName=application
+
+```
+intent=DeferToRecognizer_LUIS_application
+```