Adjust cross-train to use id as key-value instead of path in config (#1030)

* adjust cross-train to use id as config kv instead of path

* remove configId

Author: feich-ms

packages/lu/src/parser/cross-train/cross-train.js

@@ -3,7 +3,6 @@
  * Licensed under the MIT License.
  */
 
-const fs = require('fs-extra')
 const path = require('path')
 const file = require('../../utils/filehelper')
 const fileExtEnum = require('../utils/helpers').FileExtTypeEnum
@@ -24,13 +23,37 @@ module.exports = {
     const qnaContents = await file.getFilesContent(input, fileExtEnum.QnAFile)
     const configContent = await file.getConfigContent(config)
 
+    let importResolver = async function (_, idsToFind) {
+      let importedContents = []
+      for (let idx = 0; idx < idsToFind.length; idx++) {
+        let file = idsToFind[idx]
+        if (path.isAbsolute(file.filePath)) {
+          if (file.filePath.endsWith(fileExtEnum.LUFile)) {
+            importedContents.push(...await file.getFilesContent(file.filePath, fileExtEnum.LUFile))
+          } else if (file.filePath.endsWith(fileExtEnum.QnAFile)) {
+            importedContents.push(...await file.getFilesContent(file.filePath, fileExtEnum.QnAFile))
+          }
+        } else {
+          const fileName = path.basename(file.filePath)
+          if (fileName.endsWith(fileExtEnum.LUFile)) {
+            importedContents.push(...luContents.filter(luContent => luContent.id === path.basename(fileName, fileExtEnum.LUFile)))
+          } else if (fileName.endsWith(fileExtEnum.QnAFile)) {
+            importedContents.push(...qnaContents.filter(qnaContent => qnaContent.id === path.basename(fileName, fileExtEnum.QnAFile)))
+          }
+        }
+      }
+
+      return importedContents
+    }
+
     const trainedResult = await crossTrainer.crossTrain(
       luContents,
       qnaContents,
       JSON.parse(configContent.content), {
       configId: configContent.id,
       intentName,
-      verbose
+      verbose,
+      importResolver
     })
 
     return trainedResult

packages/lu/src/parser/cross-train/crossTrainer.js

@@ -3,7 +3,6 @@
  * Licensed under the MIT License.
  */
 
-const helpers = require('../utils/helpers')
 const fileExtEnum = require('../utils/helpers').FileExtTypeEnum
 const luParser = require('../lufile/luParser')
 const SectionOperator = require('../lufile/sectionOperator')
@@ -38,18 +37,17 @@ module.exports = {
 
       const crossTrainConfig = fileHelper.getConfigObject(
         configObject,
-        options.configId,
         options.intentName || '_Interruption',
         options.verbose || true)
 
       let {luObjectArray, qnaObjectArray} = pretreatment(luContents, qnaContents)
       const {rootIds, triggerRules, intentName, verbose} = crossTrainConfig
 
       // parse lu content to LUResource object
-      let {fileIdToResourceMap: luFileIdToResourceMap, allEmpty: allLuEmpty} = await parseAndValidateContent(luObjectArray, verbose, importResolver)
+      let {fileIdToResourceMap: luFileIdToResourceMap, allEmpty: allLuEmpty} = await parseAndValidateContent(luObjectArray, verbose, importResolver, fileExtEnum.LUFile)
 
       // parse qna content to LUResource object
-      let {fileIdToResourceMap: qnaFileIdToResourceMap, allEmpty: allQnAEmpty} = await parseAndValidateContent(qnaObjectArray, verbose, importResolver)
+      let {fileIdToResourceMap: qnaFileIdToResourceMap, allEmpty: allQnAEmpty} = await parseAndValidateContent(qnaObjectArray, verbose, importResolver, fileExtEnum.QnAFile)
 
       if (!allLuEmpty) {
         // construct resource tree to build the father-children relationship among lu files
@@ -181,8 +179,7 @@ const mergeRootInterruptionToLeaves = function (rootResource, result, qnaFileToR
   for (const child of rootResource.children) {
     let childResource = result.get(child.target)
     if (childResource && childResource.visited === undefined) {
-      let rootQnaFileId = rootResource.id.toLowerCase().replace(new RegExp(helpers.FileExtTypeEnum.LUFile + '$'), helpers.FileExtTypeEnum.QnAFile)
-      rootQnaFileId = Array.from(qnaFileToResourceMap.keys()).find(x => x.toLowerCase() === rootQnaFileId)
+      const rootQnaFileId = Array.from(qnaFileToResourceMap.keys()).find(x => x.toLowerCase() === rootResource.id.toLowerCase())
       const rootQnaResource = qnaFileToResourceMap.get(rootQnaFileId)
       const newChildResource = mergeFatherInterruptionToChild(rootResource, rootQnaResource, childResource, intentName)
       result.set(child.target, newChildResource)
@@ -356,12 +353,11 @@ const extractIntentUtterances = function(resource, intentName) {
 const qnaCrossTrain = function (qnaFileIdToResourceMap, luFileIdToResourceMap, interruptionIntentName, allLuEmpty) {
   try {
     for (const qnaObjectId of Array.from(qnaFileIdToResourceMap.keys())) {
-      let luObjectId = qnaObjectId.toLowerCase().replace(new RegExp(helpers.FileExtTypeEnum.QnAFile + '$'), helpers.FileExtTypeEnum.LUFile)
       let fileName = path.basename(qnaObjectId, path.extname(qnaObjectId))
       const culture = fileHelper.getCultureFromPath(qnaObjectId)
       fileName = culture ? fileName.substring(0, fileName.length - culture.length - 1) : fileName
 
-      luObjectId = Array.from(luFileIdToResourceMap.keys()).find(x => x.toLowerCase() === luObjectId)
+      const luObjectId = Array.from(luFileIdToResourceMap.keys()).find(x => x.toLowerCase() === qnaObjectId.toLowerCase())
       if (luObjectId) {
         const { luResource, qnaResource } = qnaCrossTrainCore(luFileIdToResourceMap.get(luObjectId), qnaFileIdToResourceMap.get(qnaObjectId), fileName, interruptionIntentName, allLuEmpty)
         luFileIdToResourceMap.set(luObjectId, luResource)
@@ -506,16 +502,17 @@ const qnaAddMetaData = function (qnaResource, fileName) {
  * @param {luObject[]} objectArray the lu or qna object list to be parsed
  * @param {boolean} verbose indicate to enable log messages or not
  * @param {any} importResolver import Resolver when resolving import files
+ * @param {any} fileExt file extension to indicate the file is .lu or .qna format 
  * @returns {Map<string, LUResource>} map of file id and luResource
  * @throws {exception} throws errors
  */
-const parseAndValidateContent = async function (objectArray, verbose, importResolver) {
+const parseAndValidateContent = async function (objectArray, verbose, importResolver, fileExt) {
   let fileIdToResourceMap = new Map()
   let allEmpty = true
   for (const object of objectArray) {    
     let fileContent = object.content
     if (object.content && object.content !== '') {
-      if (object.id.toLowerCase().endsWith(fileExtEnum.LUFile)) {
+      if (fileExt === fileExtEnum.LUFile) {
         let result = await LuisBuilderVerbose.build([object], verbose, undefined, importResolver)
         let luisObj = new Luis(result)
         fileContent = luisObj.parseToLuContent()

packages/lu/src/utils/filehelper.ts

@@ -172,7 +172,7 @@ export async function getFilesContent(input: string, extType: string) {
   if (fileStat.isFile()) {
     const filePath = path.resolve(input)
     const content = await getContentFromFile(input)
-    return [{id: filePath, content}]
+    return [{id: path.basename(filePath, extType), content}]
   }
 
   if (!fileStat.isDirectory()) {
@@ -182,7 +182,7 @@ export async function getFilesContent(input: string, extType: string) {
   return Promise.all(paths.map(async (item: string) => {
     const itemPath = path.resolve(path.join(input, item))
     const content = await getContentFromFile(itemPath)
-    return {id: itemPath, content}
+    return {id: path.basename(itemPath, extType), content}
   }))
 }
 
@@ -220,37 +220,34 @@ export function getParsedObjects(contents: {id: string, content: string}[]) {
   return parsedObjects
 }
 
-export function getConfigObject(configObject: any, configId: string, intentName: string, verbose: boolean) {
-  let finalLuConfigObj = Object.create(null)
-  let rootLuFiles: string[] = []
-  const configFileDir = configId && configId !== '' ? path.dirname(configId) : undefined
+export function getConfigObject(configObject: any, intentName: string, verbose: boolean) {
+  let finalConfigObj = Object.create(null)
+  let rootFileIds: string[] = []
   if (configObject) {
     try {
-      for (const rootluFilePath of Object.keys(configObject)) {
-        const rootLuFileFullPath = configFileDir ? path.resolve(configFileDir, rootluFilePath) : rootluFilePath
-        const triggerObj = configObject[rootluFilePath]
+      for (const rootFileId of Object.keys(configObject)) {
+        const triggerObj = configObject[rootFileId]
         for (const triggerObjKey of Object.keys(triggerObj)) {
           if (triggerObjKey === 'rootDialog') {
             if (triggerObj[triggerObjKey]) {
-              rootLuFiles.push(rootLuFileFullPath)
+              rootFileIds.push(rootFileId)
             }
           } else if (triggerObjKey === 'triggers') {
             const triggers = triggerObj[triggerObjKey]
             for (const triggerKey of Object.keys(triggers)) {
-              const destLuFiles = triggers[triggerKey] instanceof Array ? triggers[triggerKey] : [triggers[triggerKey]]
-              for (const destLuFile of destLuFiles) {
-                const destLuFileFullPath = configFileDir && destLuFile && destLuFile !== '' ? path.resolve(configFileDir, destLuFile) : destLuFile
-                if (rootLuFileFullPath in finalLuConfigObj) {
-                  const finalIntentToDestLuFiles = finalLuConfigObj[rootLuFileFullPath]
-                  if (finalIntentToDestLuFiles[triggerKey]) {
-                    finalIntentToDestLuFiles[triggerKey].push(destLuFileFullPath)
+              const destFileIds = triggers[triggerKey] instanceof Array ? triggers[triggerKey] : [triggers[triggerKey]]
+              for (const destFileId of destFileIds) {
+                if (rootFileId in finalConfigObj) {
+                  let finalIntentToDestFileIds = finalConfigObj[rootFileId]
+                  if (finalIntentToDestFileIds[triggerKey]) {
+                    finalIntentToDestFileIds[triggerKey].push(destFileId)
                   } else {
-                    finalIntentToDestLuFiles[triggerKey] = [destLuFileFullPath]
+                    finalIntentToDestFileIds[triggerKey] = [destFileId]
                   }
                 } else {
-                  let finalIntentToDestLuFiles = Object.create(null)
-                  finalIntentToDestLuFiles[triggerKey] = [destLuFileFullPath]
-                  finalLuConfigObj[rootLuFileFullPath] = finalIntentToDestLuFiles
+                  let finalIntentToDestFileIds = Object.create(null)
+                  finalIntentToDestFileIds[triggerKey] = [destFileId]
+                  finalConfigObj[rootFileId] = finalIntentToDestFileIds
                 }
               }
             }
@@ -263,8 +260,8 @@ export function getConfigObject(configObject: any, configId: string, intentName:
   }
 
   let crossTrainConfig = {
-    rootIds: rootLuFiles,
-    triggerRules: finalLuConfigObj,
+    rootIds: rootFileIds,
+    triggerRules: finalConfigObj,
     intentName,
     verbose
   }