bf-lu: update some function to support load file content directly (#626)

* update some function to support load file content

* update some code style

* update some ref

* optimize config parameter

* add fileHelper.ts

* fix typo

* remove semicolon

* add the object

* change the value

* make all the path the same parttern

* optimize

* adjust test cases to use file name as id

* fix typo

* fix config id issue

* move globby from dev dependency to dependency

Co-authored-by: Fei Chen <[email protected]>

Author: feich-ms

packages/lu/package.json

@@ -43,6 +43,7 @@
     "delay": "^4.3.0",
     "fs-extra": "^8.1.0",
     "get-stdin": "^6.0.0",
+    "globby": "^10.0.1",
     "intercept-stdout": "^0.1.2",
     "lodash": "^4.17.15",
     "node-fetch": "^2.1.2",
@@ -57,7 +58,6 @@
     "@types/mocha": "^5.2.7",
     "@types/node": "^10.14.15",
     "chai": "^4.2.0",
-    "globby": "^10.0.1",
     "mocha": "^6.2.2",
     "nock": "^11.7.0",
     "nyc": "^14.1.1",

packages/lu/src/parser/cross-train/cross-train.js

@@ -8,44 +8,27 @@ const path = require('path')
 const file = require('../../utils/filehelper')
 const fileExtEnum = require('../utils/helpers').FileExtTypeEnum
 const exception = require('../utils/exception')
-const retCode = require('../utils/enums/CLI-errors');
+const retCode = require('../utils/enums/CLI-errors')
 const crossTrainer = require('./crossTrainer')
 
 module.exports = {
   /**
    * Cross train lu and qna files.
    * @param {string} input input lu and qna files folder.
    * @param {string} intentName interruption intent name. Default value is _Interruption.
-   * @param {string} configPath path to config of mapping rules. If undefined, it will read config.json from input folder.
+   * @param {string} config path to config of mapping rules or mapping rules json content itself. If undefined, it will read config.json from input folder.
    * @returns {luResult: any, qnaResult: any} trainedResult of luResult and qnaResult or undefined if no results.
    */
-  train: async function (input, intentName, configPath) {
-    let trainedResult
+  train: async function (input, intentName, config) {
+    // Get all related file content.
+    const luContents = await file.getFilesContent(input, fileExtEnum.LUFile)
+    const qnaContents = await file.getFilesContent(input, fileExtEnum.QnAFile)
+    const configContent = config && !fs.existsSync(config) ? {id: path.join(input, 'config.json'), content: config} : await file.getConfigContent(config || input)
 
-    // Parse lu and qna objects
-    const luObjects = await file.getLuObjects(undefined, input, true, fileExtEnum.LUFile)
-    const qnaObjects = await file.getLuObjects(undefined, input, true, fileExtEnum.QnAFile)
-
-    let configObject
-    if (configPath && configPath !== '') {
-      configObject = await file.getConfigObject(configPath)
-    } else {
-      configObject = await file.getConfigObject(input)
-    }
-
-    if (configObject.rootIds.length > 0) {
-      let crossTrainConfig = {
-        rootIds: configObject.rootIds,
-        triggerRules: configObject.triggerRules,
-        intentName: intentName,
-        verbose: true
-      }
-
-      trainedResult = crossTrainer.crossTrain(luObjects, qnaObjects, JSON.stringify(crossTrainConfig))
-    } else {
-      throw (new exception(retCode.errorCode.INVALID_INPUT_FILE, 'rootDialog property is required in config file'))
-    }
+    const configObject = file.getConfigObject(configContent, intentName)
 
+    const trainedResult = crossTrainer.crossTrain(luContents, qnaContents, configObject)
+    
     return trainedResult
   },
 

packages/lu/src/parser/cross-train/crossTrainer.js

@@ -11,27 +11,24 @@ const LUResource = require('../lufile/luResource')
 const DiagnosticSeverity = require('../lufile/diagnostic').DiagnosticSeverity
 const fileHelper = require('../../utils/filehelper')
 const exception = require('../utils/exception')
-const retCode = require('../utils/enums/CLI-errors');
+const retCode = require('../utils/enums/CLI-errors')
 const NEWLINE = require('os').EOL
 const path = require('path')
 const QNA_GENERIC_SOURCE = "custom editorial"
 
 module.exports = {
   /**
    * Do cross training among lu files
-   * @param {luObject[]} luObjectArray the lu object list to be parsed
-   * @param {luObject[]} qnaObjectArray the qna Object list to be parsed
+   * @param {any[]} luContents the lu content array whose element includes path and content
+   * @param {any[]} qnaContents the qna content array whose element includes path and content
    * @param {any} crossTrainConfig cross train json config
    * @returns {Map<string, LUResource>} map of file id and luResource
    * @throws {exception} throws errors
    */
-  crossTrain: function (luObjectArray, qnaObjectArray, crossTrainConfig) {
+  crossTrain: function (luContents, qnaContents, crossTrainConfig) {
     try {
-      const crossTrainConfigObj = JSON.parse(crossTrainConfig)
-      const rootObjectIds = crossTrainConfigObj.rootIds
-      const triggerRules = crossTrainConfigObj.triggerRules
-      const intentName = crossTrainConfigObj.intentName
-      const verbose = crossTrainConfigObj.verbose
+      const {luObjectArray, qnaObjectArray} = pretreatment(luContents, qnaContents)
+      const {rootIds, triggerRules, intentName, verbose} = crossTrainConfig
 
       // parse lu content to LUResource object
       let luFileIdToResourceMap = parseAndValidateContent(luObjectArray, verbose)
@@ -43,7 +40,7 @@ module.exports = {
       let resources = constructResoureTree(luFileIdToResourceMap, triggerRules)
 
       // do lu cross training from roots. One root one core training
-      for (const rootObjectId of rootObjectIds) {
+      for (const rootObjectId of rootIds) {
         if (resources.some(r => r.id === rootObjectId)) {
           // do cross training for each root at top level
           const result = luCrossTrain(rootObjectId, resources, qnaFileIdToResourceMap, intentName)
@@ -466,3 +463,11 @@ const parseAndValidateContent = function (objectArray, verbose) {
 
   return fileIdToResourceMap
 }
+
+const pretreatment = function (luContents, qnaContents) {
+   // Parse lu and qna objects
+   const luObjectArray = fileHelper.getParsedObjects(luContents)
+   const qnaObjectArray = fileHelper.getParsedObjects(qnaContents)
+
+   return {luObjectArray, qnaObjectArray}
+}

packages/lu/src/utils/filehelper.ts

@@ -11,6 +11,7 @@ const path = require('path')
 const helpers = require('./../parser/utils/helpers')
 const luObject = require('./../parser/lu/lu')
 const LUOptions = require('./../parser/lu/luOptions')
+const globby = require('globby')
 
 /* tslint:disable:prefer-for-of no-unused*/
 
@@ -164,6 +165,31 @@ export async function detectLuContent(stdin: string, input: string) {
   return false
 }
 
+export async function getFilesContent(input: string, extType: string) {
+  let fileStat = await fs.stat(input)
+  if (fileStat.isFile()) {
+    const filePath = path.resolve(input)
+    const content = await getContentFromFile(input)
+    return [{id: filePath, content}]
+  }
+
+  if (!fileStat.isDirectory()) {
+    throw (new exception(retCode.errorCode.INVALID_INPUT_FILE, 'Sorry, ' + input + ' is not a folder or does not exist'))
+  }
+  const paths = await globby([`**/*${extType}`], {cwd: input, dot: true})
+  return Promise.all(paths.map(async (item: string) => {
+    const itemPath = path.resolve(path.join(input, item))
+    const content = await getContentFromFile(itemPath)
+    return {id: itemPath, content}
+  }))
+}
+
+export async function getConfigContent(input: string) {
+  const luConfigFile = await getConfigFile(input)
+  const content = await getContentFromFile(luConfigFile)
+  return {id: luConfigFile, content}
+}
+
 async function getConfigFile(input: string): Promise<string> {
   let fileStat = await fs.stat(input)
   if (fileStat.isFile()) {
@@ -183,13 +209,20 @@ async function getConfigFile(input: string): Promise<string> {
   return defaultConfigFile
 }
 
-export async function getConfigObject(input: string) {
-  const luConfigFile = await getConfigFile(input)
+export function getParsedObjects(contents: {id: string, content: string}[]) {
+  const parsedObjects = contents.map(content => {
+    const opts = new LUOptions(content.id)
+    return new luObject(content.content, opts)
+  })
+
+  return parsedObjects
+}
 
+export function getConfigObject(configContent: any, intentName: string) {
   let finalLuConfigObj = Object.create(null)
   let rootLuFiles: string[] = []
-  const configFileDir = path.dirname(luConfigFile)
-  const luConfigContent = await getContentFromFile(luConfigFile)
+  const configFileDir = path.dirname(configContent.id)
+  const luConfigContent = configContent.content
   if (luConfigContent && luConfigContent !== '') {
     try {
       const luConfigObj = JSON.parse(luConfigContent)
@@ -225,7 +258,18 @@ export async function getConfigObject(input: string) {
     }
   }
 
-  return {rootIds: rootLuFiles, triggerRules: finalLuConfigObj}
+  if (rootLuFiles.length > 0) {
+    let crossTrainConfig = {
+      rootIds: rootLuFiles,
+      triggerRules: finalLuConfigObj,
+      intentName,
+      verbose: true
+    }
+
+    return crossTrainConfig
+  } else {
+    throw (new exception(retCode.errorCode.INVALID_INPUT_FILE, 'rootDialog property is required in config file'))
+  }
 }
 
 export function parseJSON(input: string, appType: string) {