[Cherry-Pick] Orchestrator scenario for CI-CD (#1226)

* fix luis cross train unable to reference external files (#1181)

* init

* add path resovle

* fix import path resolve issue

* add tests

* fix lint and unused imports

* modify config name

* retrigger

* refactor

Co-authored-by: Emilio Munoz <[email protected]>

* fix luis:build write out version in luis setting when the directVersionPublish is not set (#1215)

* init

* fix tests

* retrigger

* remove redundent ()

* up versioned orchestrator-core to 4.13.1 (#1211)

* Update package.json

Use orchestrator-core v 4.13.1

* Update pnpm-lock.yaml

* skip empty lu file, added empty lu file to test

* Update orchestratorhelper.ts

fixed lint issues

* use forward slash for file path in orchestrator.settings.json to be compatible with non windows platform

* addressed PR comments

Co-authored-by: Emilio Munoz <[email protected]>

* fix luis-cross train does not find imported files with locale (#1209)

* fix import file with locale

* simplify test

* change folder structures

* remove only

* add cross train config

* fix spelling

* modify test

* change naming

* fix readme

* fix typo

Co-authored-by: Emilio Munoz <[email protected]>

Co-authored-by: Shuai Wang <[email protected]>
Co-authored-by: Tien Suwandy <[email protected]>

Author: 3 people

common/config/rush/pnpm-lock.yaml

@@ -891,6 +891,8 @@ OPTIONS
   --[no-]intra-dialog      Only do intra dialog cross train
 
   --log                    Writes out log messages to console
+
+  --exclude                Excludes given folders under the input directory, for example, --exclude bin,obj,lib, this will ignore the /bin, /obj, /lib folders under the input path
 ```
 
 _See code: [@microsoft/bf-luis-cli](https://github.com/microsoft/botframework-cli/tree/master/packages/luis/src/commands/luis/cross-train.ts)_

packages/cli/README.md

@@ -4,7 +4,8 @@
  */
 
 const path = require('path')
-const file = require('../../utils/filehelper')
+const fs = require('fs-extra')
+const filehelper = require('../../utils/filehelper')
 const fileExtEnum = require('../utils/helpers').FileExtTypeEnum
 const crossTrainer = require('./crossTrainer')
 
@@ -18,28 +19,63 @@ module.exports = {
    * @param {inner: boolean, intra: boolean} trainingOpt trainingOpt indicates whether you want to control do the inner or intra dialog training seperately
    * @returns {luResult: any, qnaResult: any} trainedResult of luResult and qnaResult or undefined if no results.
    */
-  train: async function (input, intentName, config, verbose, trainingOpt) {
+  train: async function (input, intentName, config, verbose, trainingOpt, exclude) {
+    // get excluded foleders
+    let excludedFolders = undefined
+    if (exclude) {
+      excludedFolders = exclude.split(',').map(e => e.trim())
+    }
+
     // Get all related file content.
-    const luContents = await file.getFilesContent(input, fileExtEnum.LUFile)
-    const qnaContents = await file.getFilesContent(input, fileExtEnum.QnAFile)
-    const configContent = await file.getConfigContent(config)
+    const luContents = await filehelper.getFilesContent(input, fileExtEnum.LUFile, excludedFolders)
+    const qnaContents = await filehelper.getFilesContent(input, fileExtEnum.QnAFile, excludedFolders)
+    const configContent = await filehelper.getConfigContent(config)
+    const defaultLocale = 'en-us'
 
-    let importResolver = async function (_, idsToFind) {
+    let importResolver = async function (id, idsToFind) {
       let importedContents = []
+      const idWithoutExt = path.basename(id, path.extname(id))
+      const locale = /\w\.\w/.test(idWithoutExt) ? idWithoutExt.split('.').pop() : defaultLocale;
       for (let idx = 0; idx < idsToFind.length; idx++) {
         let file = idsToFind[idx]
         if (path.isAbsolute(file.filePath)) {
           if (file.filePath.endsWith(fileExtEnum.LUFile)) {
-            importedContents.push(...await file.getFilesContent(file.filePath, fileExtEnum.LUFile))
+            importedContents.push(...await filehelper.getFilesContent(file.filePath, fileExtEnum.LUFile))
           } else if (file.filePath.endsWith(fileExtEnum.QnAFile)) {
-            importedContents.push(...await file.getFilesContent(file.filePath, fileExtEnum.QnAFile))
+            importedContents.push(...await filehelper.getFilesContent(file.filePath, fileExtEnum.QnAFile))
           }
         } else {
           const fileName = path.basename(file.filePath)
+          const updateImportedContents = async function(typedContents, fileExt) {
+            let found = []
+            // import resolver should be capable to find implicit import files with locale, for example '[import](b.lu)' is defined in a.en-us.lu, the resolver should find b.en-us.lu
+            const foundWithLocale = typedContents.filter(content => content.id === `${path.basename(fileName, fileExt)}.${locale}`)
+            if (foundWithLocale.length > 0) {
+              found = foundWithLocale
+            } else {
+              //if no locale specified file is found, just to check whether there is file without locale matched
+              found =  typedContents.filter(content => content.id === path.basename(fileName, fileExt))
+            }
+
+            if(found.length > 0) {
+              importedContents.push(...found)
+            } else {
+              const matchedLuisFiles = typedContents.filter(content => path.basename(content.fullPath) === id)
+              for (const matchFile of matchedLuisFiles) {
+                const sourceFileDir = path.dirname(matchFile.fullPath)
+                const targetPath = path.resolve(sourceFileDir, file.filePath)
+                if (fs.existsSync(targetPath)) {
+                  const importContent = await filehelper.getFilesContent(targetPath, fileExt)
+                  importedContents.push(...importContent)
+                }
+              }
+            }
+          }
+
           if (fileName.endsWith(fileExtEnum.LUFile)) {
-            importedContents.push(...luContents.filter(luContent => luContent.id === path.basename(fileName, fileExtEnum.LUFile)))
+            await updateImportedContents(luContents, fileExtEnum.LUFile)
           } else if (fileName.endsWith(fileExtEnum.QnAFile)) {
-            importedContents.push(...qnaContents.filter(qnaContent => qnaContent.id === path.basename(fileName, fileExtEnum.QnAFile)))
+            await updateImportedContents(qnaContents, fileExtEnum.QnAFile)
           }
         }
       }
@@ -60,4 +96,4 @@ module.exports = {
 
     return trainedResult
   }
-}
+}

packages/lu/src/parser/cross-train/cross-train.js

@@ -293,7 +293,7 @@ export class Builder {
     return settingsContent
   }
 
-  async writeDialogAssets(contents: any[], options: any = {}) {
+  async writeDialogAssets(contents: any[], options: any = {}, directVersionPublish?: boolean) {
     let force = options.force || false
     let out = options.out
     let luConfig = options.luConfig
@@ -311,7 +311,8 @@ export class Builder {
       } else {
         outPath = path.resolve(settingsContents[0].id)
       }
-      writeContents.push(this.mergeSettingsContent(outPath, settingsContents))
+
+      writeContents.push(this.mergeSettingsContent(outPath, settingsContents, directVersionPublish))
     }
 
     for (const content of writeContents) {
@@ -478,14 +479,16 @@ export class Builder {
     return contents
   }
 
-  mergeSettingsContent(settingsPath: string, contents: any[]) {
+  mergeSettingsContent(settingsPath: string, contents: any[], directVersionPublish?: boolean) {
     let settings = new Settings(settingsPath, {})
     for (const content of contents) {
       const luisAppsMap = JSON.parse(content.content).luis
       for (const appName of Object.keys(luisAppsMap)) {
-        settings.luis[appName] = {
+        settings.luis[appName] = directVersionPublish ? {
           "appId": luisAppsMap[appName]["appId"],
           "version": luisAppsMap[appName]["version"]
+        } : {
+          "appId": luisAppsMap[appName]["appId"]
         }
       }
     }

packages/lu/src/parser/lubuild/builder.ts

@@ -171,22 +171,35 @@ export async function detectLuContent(stdin: string, input: string) {
   return false
 }
 
-export async function getFilesContent(input: string, extType: string) {
+export async function getFilesContent(input: string, extType: string, ignoredFolders?: string[]) {
   let fileStat = await fs.stat(input)
   if (fileStat.isFile()) {
     const filePath = path.resolve(input)
     const content = await getContentFromFile(input)
-    return [{id: path.basename(filePath, extType), content}]
+    return [{id: path.basename(filePath, extType), content, fullPath: filePath}]
   }
 
   if (!fileStat.isDirectory()) {
     throw (new exception(retCode.errorCode.INVALID_INPUT_FILE, 'Sorry, ' + input + ' is not a folder or does not exist'))
   }
-  const paths = await globby([`**/*${extType}`], {cwd: input, dot: true})
+
+  const allPaths = (await globby([`**/*${extType}` ], {cwd: input, dot: true}))
+  let paths: string[] = []
+  if (ignoredFolders) {
+    for (const path of allPaths) {
+      const isIgnored = ignoredFolders.filter(e => path.startsWith(e)).length > 0
+      if (!isIgnored) {
+        paths.push(path)
+      }
+    }
+  } else {
+    paths = allPaths
+  }
+  
   return Promise.all(paths.map(async (item: string) => {
     const itemPath = path.resolve(path.join(input, item))
     const content = await getContentFromFile(itemPath)
-    return {id: path.basename(itemPath, extType), content}
+    return {id: path.basename(itemPath, extType), content, fullPath: itemPath}
   }))
 }
 

packages/lu/src/utils/filehelper.ts

@@ -431,6 +431,8 @@ OPTIONS
   --[no-]intra-dialog      Only do intra dialog cross train
 
   --log                    Writes out log messages to console
+
+  --exclude                Excludes given folders under the input directory, for example, --exclude bin,obj,lib, this will ignore the /bin, /obj, /lib folders under the input path
 ```
 
 _See code: [src/commands/luis/cross-train.ts](https://github.com/microsoft/botframework-cli/tree/master/packages/luis/src/commands/luis/cross-train.ts)_

packages/luis/README.md

@@ -154,13 +154,13 @@ export default class LuisBuild extends Command {
             fallbackLocale,
             schema,
             dialog
-          })
+          }, directVersionPublish)
 
           let writeDone = await builder.writeDialogAssets(dialogContents, {
             force,
             out: outputFolder,
             luConfig
-          })
+          }, directVersionPublish)
 
           if (writeDone) {
             this.log(`Successfully wrote .dialog files to ${outputFolder}\n`)
@@ -173,7 +173,7 @@ export default class LuisBuild extends Command {
           force,
           out: outputFolder,
           luConfig
-        })
+        }, directVersionPublish)
 
         if (writeDone) {
           this.log(`Successfully wrote settings file to ${outputFolder}\n`)

packages/luis/src/commands/luis/build.ts

@@ -22,7 +22,8 @@ export default class LuisCrossTrain extends Command {
     force: flags.boolean({char: 'f', description: 'If --out flag is provided with the path to an existing file, overwrites that file', default: false}),
     log: flags.boolean({description: 'Writes out log messages to console', default: false}),
     'inner-dialog': flags.boolean({description: 'Only do inner dialog cross train', default: true, allowNo: true}),
-    'intra-dialog': flags.boolean({description: 'Only do intra dialog cross train', default: true, allowNo: true})
+    'intra-dialog': flags.boolean({description: 'Only do intra dialog cross train', default: true, allowNo: true}),
+    exclude: flags.string({description: 'Excludes folders under the input directory, separated by ",". If not specified, all luis and qna files will be included in the cross-train'})
   }
 
   async run() {
@@ -46,7 +47,7 @@ export default class LuisCrossTrain extends Command {
         intra: flags['intra-dialog']
       }
 
-      const trainedResult = await crossTrain.train(flags.in, flags.intentName, flags.config, flags.log, trainingOpt)
+      const trainedResult = await crossTrain.train(flags.in, flags.intentName, flags.config, flags.log, trainingOpt, flags.exclude)
 
       if (flags.out === undefined) {
         flags.out = path.join(process.cwd(), 'cross-trained')