cross train CLI (#690)

* add qna cross training

* Revert "remove cross-train cli related changes and only keep the api"

This reverts commit ab1fb84.

* call qnaCrossTrain

* add qna cross traini

* expose function

* optimize and add more corner test cases

* rename crossTrainer class

* remove cli code pieces and optimize tests

* move cross-train.ts to lu folder

* optimize function headers and naming

* typo: dialogname to dialogName

* add more test cases

* remove unused '

* add cross-train.ts cli file for test convenience and will remove it once review done

* fix bugs

* trigger ci

* update qna cross train based on feedbacks

* add config path parameter to specify the config file

* fix posttest failure

* remove dup utterances in interuption intents

* remove dup 1

* optimize all de-dup logic

* resolve all feedbacks

* add test cases to test dedup functions

* fix tslint errors

* add more friendly description for config CLI parameter

* fix config full path issue and remove uncessary loop validation

* move cross train CLI to a new package named cross-train

* fix build error in CI validation

* add missing package in package.json

* remove unused file

* fix typo

* optimize dedep logic and resolve feedbacks

* adjust test cases

* support qna source, id, prompts reconstruction in cross training

* typo: interuption to interruption

* move CLI test cases from lu to cross-train

* adjust test cases of cross-train CLI to make it more readable

* update some function to support load file content

* support auto detect config based on root dialog and file system

* update some code style

* update some ref

* optimize config parameter

* add fileHelper.ts

* fix typo

* remove semicolon

* add the object

* change the value

* make all the path the same parttern

* optimize

* adjust test cases to use file name as id

* fix typo

* update fileHelper.ts

* fix config id issue

* update pnpm lock

* update pnpm

* fix test cases

* fix tslint

* optimzie cross train to care only trigger intents and allow empty trigger intent or dialog

* merge PR 706 and adjust test cases to cover more corner cases

* support multi trigger intents point to same lu file

* optimize config structure

* optimize config object parser

* fix bug

* adjust pnpm

* fix minor typo

* docs.

* remove cross-train from top level command and add it to luis command

* fix test

* add cross train in qna maker cli

* support to write corsstrained recognizer

* support crosstrained recognizer in qnamaker

* refine the write dialog logic to support crosstrained dialog

* update docs

* fix crosstrained recognizer configuration issue

* remove patterns from lu in crosstrained qna queations

* fix test case

* fix tslint

* fix comments of reviewer

Co-authored-by: leilzh <[email protected]>
Co-authored-by: Vishwac Sena Kannan <[email protected]>
Co-authored-by: Vishwac Sena Kannan <[email protected]>

Author: 4 people

common/config/rush/pnpm-lock.yaml

@@ -0,0 +1,108 @@
+/*!
+ * Copyright (c) Microsoft Corporation. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+const fs = require('fs-extra')
+const path = require('path')
+const exception = require('../utils/exception')
+const retCode = require('../utils/enums/CLI-errors')
+const fileHelper = require('../../utils/filehelper')
+
+const dialogExt = '.dialog'
+const luExt = '.lu'
+
+module.exports = {
+  generateConfig: async function (inputFolder, rootDialogFile) {
+    let dialogFiles = []
+    await getDialogFiles(inputFolder, dialogFiles)
+  
+    let rootDialogObject = JSON.parse(await getInputFromFile(rootDialogFile))
+    rootDialogObject.path = rootDialogFile
+    rootDialogObject.isRoot = true
+  
+    let dialogObjects = []
+    for (const dialogFile of dialogFiles) {
+      let dialogObject = JSON.parse(await getInputFromFile(dialogFile))
+      dialogObject.path = dialogFile
+      dialogObjects.push(dialogObject)
+    }
+  
+    const configObject = createConfig(rootDialogObject, dialogObjects, inputFolder)
+  
+    return JSON.stringify(configObject)
+  }
+}
+
+const getDialogFiles = async function (inputFolder, results) {
+  fs.readdirSync(inputFolder).forEach(async dirContent => {
+    dirContent = path.resolve(inputFolder, dirContent)
+    if (fs.statSync(dirContent).isDirectory()) {
+      await getDialogFiles(dirContent, results)
+    }
+
+    if (fs.statSync(dirContent).isFile()) {
+      if (dirContent.endsWith(dialogExt)) {
+        results.push(dirContent)
+      }
+    }
+  })
+}
+
+const getInputFromFile = async function (path) {
+  if (path) {
+    try {
+      return await fileHelper.getContentFromFile(path)
+    } catch (error) {
+      throw (new exception(retCode.errorCode.INVALID_INPUT, `Failed to read file: ${error}`))
+    }
+  }
+  return ''
+}
+
+const createConfig = function (rootDialog, dialogs, configPath) {
+  let result = {}
+
+  const key = createPath(rootDialog.path, configPath)
+  const rootLuPath = rootDialog.path.replace(dialogExt, luExt)
+
+  if (!fs.existsSync(rootLuPath)) {
+    throw (new exception(retCode.errorCode.INVALID_INPUT, `Failed to parse mapping rules config from file system: ${rootLuPath} does not exist. Please provide config file by --config`))
+  }
+
+  rootDialog.triggers.forEach(trigger => {
+    if (trigger.$type && trigger.$type === 'Microsoft.OnIntent') {
+      const actions = trigger.actions || []
+      for (const action of actions) {
+        if (action.$type !== 'Microsoft.BeginDialog') continue
+
+        const dialogName = action.dialog
+        const target = dialogs.find(dialog => path.basename(dialog.path, dialogExt) === dialogName)
+        
+        if (!target) continue
+
+        const relativePath = createPath(target.path, configPath)
+        if (!result[key]) result[key] = { triggers: {} }
+        if (!result[key].triggers[trigger.intent]) {
+          result[key].triggers[trigger.intent] = relativePath
+        } else if (typeof result[key].triggers[trigger.intent] === 'string') {
+          result[key].triggers[trigger.intent] = [result[key].triggers[trigger.intent], relativePath]
+        } else {
+          result[key].triggers[trigger.intent].push(relativePath)
+        }
+
+        result = { ...result, ...createConfig(target, dialogs, configPath) }
+      }
+    }
+  })
+
+  if (rootDialog.isRoot && result[key]) result[key].rootDialog = true
+
+  return result
+}
+
+const createPath = function (dialogPath, configPath) {
+  const luFilePath = dialogPath.replace('.dialog', '.lu')
+  const relativePath = path.relative(configPath, luFilePath)
+  return relativePath
+}

packages/lu/src/parser/cross-train/confighelper.js

@@ -10,11 +10,24 @@ const fileExtEnum = require('../utils/helpers').FileExtTypeEnum
 const exception = require('../utils/exception')
 const retCode = require('../utils/enums/CLI-errors')
 const crossTrainer = require('./crossTrainer')
+const confighelper = require('./confighelper')
 
 module.exports = {
+  /**
+   * Generate cross train config based on input folder and root dialog file.
+   * @param {string} inputFolder full path of input lu and qna files folder.
+   * @param {string} rootDialogFile full path of root dialog file.
+   * @returns {string} config object json string.
+   */
+  generateConfig: async function (inputFolder, rootDialogFile) {
+    const configStr = await confighelper.generateConfig(inputFolder, rootDialogFile)
+
+    return configStr
+  },
+
   /**
    * Cross train lu and qna files.
-   * @param {string} input input lu and qna files folder.
+   * @param {string} input full path of input lu and qna files folder.
    * @param {string} intentName interruption intent name. Default value is _Interruption.
    * @param {string} config path to config of mapping rules or mapping rules json content itself. If undefined, it will read config.json from input folder.
    * @returns {luResult: any, qnaResult: any} trainedResult of luResult and qnaResult or undefined if no results.

packages/lu/src/parser/cross-train/cross-train.js

@@ -300,20 +300,21 @@ const removeDupUtterances = function (resource) {
 
 const extractIntentUtterances = function(resource, intentName) {
   const intentSections = resource.Sections.filter(s => s.SectionType === LUSectionTypes.SIMPLEINTENTSECTION || s.SectionType === LUSectionTypes.NESTEDINTENTSECTION)
+  const curlyRe = /.*\{.*\}.*/
 
   let intentUtterances = []
   if (intentName && intentName !== '') {
     const specificSections = intentSections.filter(s => s.Name === intentName)
     if (specificSections.length > 0) {
-      intentUtterances = intentUtterances.concat(specificSections[0].UtteranceAndEntitiesMap.map(u => u.utterance))
+      intentUtterances = intentUtterances.concat(specificSections[0].UtteranceAndEntitiesMap.map(u => u.utterance).filter(i => curlyRe.exec(i) === null))
     }
   } else {
     intentSections.forEach(s => {
       if (s.SectionType === LUSectionTypes.SIMPLEINTENTSECTION) {
-        intentUtterances = intentUtterances.concat(s.UtteranceAndEntitiesMap.map(u => u.utterance))
+        intentUtterances = intentUtterances.concat(s.UtteranceAndEntitiesMap.map(u => u.utterance).filter(i => curlyRe.exec(i) === null))
       } else {
         s.SimpleIntentSections.forEach(section => {
-          intentUtterances = intentUtterances.concat(section.UtteranceAndEntitiesMap.map(u => u.utterance))
+          intentUtterances = intentUtterances.concat(section.UtteranceAndEntitiesMap.map(u => u.utterance).filter(i => curlyRe.exec(i) === null))
         })
       }
   })}
@@ -430,11 +431,13 @@ const qnaCrossTrainCore = function (luResource, qnaResource, fileName, interrupt
     qnaSectionContents.push(qnaSectionContent)
   }
 
-  const qnaContents = qnaSectionContents.join(NEWLINE + NEWLINE)
+  let qnaContents = qnaSectionContents.join(NEWLINE + NEWLINE)
   if (qnaContents && qnaContents !== '') {
     const modelInfoSections = qnaResource.Sections.filter(s => s.SectionType === LUSectionTypes.MODELINFOSECTION)
     const modelInforContent = modelInfoSections.map(m => m.ModelInfo).join(NEWLINE)
-    trainedQnaResource = new SectionOperator(new LUResource([], modelInforContent, [])).addSection(NEWLINE + qnaContents)
+    if (modelInforContent && modelInforContent !== '') qnaContents = NEWLINE + qnaContents
+
+    trainedQnaResource = new SectionOperator(new LUResource([], modelInforContent, [])).addSection(qnaContents)
   }
 
   // remove utterances which are duplicated with local qna questions

packages/lu/src/parser/cross-train/crossTrainer.js

@@ -7,6 +7,7 @@ import {LuBuildCore} from './core'
 import {Settings} from './settings'
 import {MultiLanguageRecognizer} from './multi-language-recognizer'
 import {Recognizer} from './recognizer'
+import {CrossTrainedRecognizer} from './cross-trained-recognizer'
 const path = require('path')
 const fs = require('fs-extra')
 const delay = require('delay')
@@ -18,6 +19,7 @@ const LuisBuilderVerbose = require('./../luis/luisCollate')
 const LuisBuilder = require('./../luis/luisBuilder')
 const LUOptions = require('./../lu/luOptions')
 const Content = require('./../lu/lu')
+const recognizerType = require('./../utils/enums/recognizertypes')
 
 export class Builder {
   private readonly handler: (input: string) => any
@@ -215,7 +217,7 @@ export class Builder {
     return dialogContents
   }
 
-  async writeDialogAssets(contents: any[], force: boolean, out: string, luconfig: string) {
+  async writeDialogAssets(contents: any[], force: boolean, out: string, dialogType: string, luconfig: string) {
     let writeDone = false
 
     let writeContents = contents.filter(c => c.id.endsWith('.dialog'))
@@ -242,7 +244,7 @@ export class Builder {
           }
 
           this.handler(`Writing to ${outFilePath}\n`)
-          await fs.writeFile(outFilePath, content.content, 'utf-8')
+          await this.writeDialog(content.content, outFilePath, dialogType)
           writeDone = true
         }
       }
@@ -254,7 +256,7 @@ export class Builder {
           }
 
           this.handler(`Writing to ${content.path}\n`)
-          await fs.writeFile(content.path, content.content, 'utf-8')
+          await this.writeDialog(content.content, content.path, dialogType)
           writeDone = true
         }
       }
@@ -404,4 +406,27 @@ export class Builder {
       app.intents = filteredIntents
     }
   }
+
+  async writeDialog(content: string, filePath: string, dialogType: string) {
+    await fs.writeFile(filePath, content, 'utf-8')
+    const contentObj = JSON.parse(content)
+    if (dialogType === recognizerType.CROSSTRAINED && contentObj.$kind === 'Microsoft.MultiLanguageRecognizer') {
+      const fileName = path.basename(filePath, '.lu.dialog')
+      const crossTrainedFileName = fileName + '.lu.qna.dialog'
+      const crossTrainedFilePath = path.join(path.dirname(filePath), crossTrainedFileName)
+      if (fs.existsSync(crossTrainedFilePath)) {
+        const existingCRDialog = JSON.parse(await fileHelper.getContentFromFile(crossTrainedFilePath))
+        if (!existingCRDialog.recognizers.includes(fileName + '.lu')) {
+          existingCRDialog.recognizers.push(fileName + '.lu')
+        }
+
+        content = JSON.stringify(existingCRDialog, null, 4)
+      } else {
+        const recognizers = [fileName + '.lu']
+        content = new CrossTrainedRecognizer(crossTrainedFilePath, recognizers).save()
+      }
+
+      await fs.writeFile(crossTrainedFilePath, content, 'utf-8')
+    }
+  }
 }

packages/lu/src/parser/lubuild/builder.ts

@@ -0,0 +1,27 @@
+/*!
+ * Copyright (c) Microsoft Corporation. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+export class CrossTrainedRecognizer {
+  public recognizers: any
+  private readonly dialogPath: string
+
+  constructor(dialogPath: string, recognizers: any) {
+    this.dialogPath = dialogPath
+    this.recognizers = recognizers
+  }
+
+  save(): string {
+    let output = {
+      $kind: 'Microsoft.CrossTrainedRecognizerSet',
+      recognizers: this.recognizers
+    }
+
+    return JSON.stringify(output, null, 4)
+  }
+
+  getDialogPath(): string {
+    return this.dialogPath
+  }
+}

packages/lu/src/parser/lubuild/cross-trained-recognizer.ts

@@ -7,6 +7,7 @@ import {QnaBuildCore} from './core'
 import {Settings} from './settings'
 import {MultiLanguageRecognizer} from './multi-language-recognizer'
 import {Recognizer} from './recognizer'
+import {CrossTrainedRecognizer} from './cross-trained-recognizer'
 const path = require('path')
 const fs = require('fs-extra')
 const delay = require('delay')
@@ -20,6 +21,7 @@ const qnaOptions = require('./../lu/qnaOptions')
 const Content = require('./../lu/qna')
 const KB = require('./../qna/qnamaker/kb')
 const NEWLINE = require('os').EOL
+const recognizerType = require('./../utils/enums/recognizertypes')
 
 export class Builder {
   private readonly handler: (input: string) => any
@@ -285,25 +287,21 @@ export class Builder {
     return kbToLuContent
   }
 
-  async writeDialogAssets(contents: any[], force: boolean, out: string) {
+  async writeDialogAssets(contents: any[], force: boolean, out: string, dialogType: string, files: string[]) {
     let writeDone = false
 
-    if (out) {
-      for (const content of contents) {
-        const outFilePath = path.join(path.resolve(out), path.basename(content.path))
-        if (force || !fs.existsSync(outFilePath)) {
-          this.handler(`Writing to ${outFilePath}\n`)
-          await fs.writeFile(outFilePath, content.content, 'utf-8')
-          writeDone = true
-        }
+    for (const content of contents) {
+      let outFilePath
+      if (out) {
+        outFilePath = path.join(path.resolve(out), path.basename(content.path))
+      } else {
+        outFilePath = content.path
       }
-    } else {
-      for (const content of contents) {
-        if (force || !fs.existsSync(content.path)) {
-          this.handler(`Writing to ${content.path}\n`)
-          await fs.writeFile(content.path, content.content, 'utf-8')
-          writeDone = true
-        }
+
+      if (force || !fs.existsSync(outFilePath)) {
+        this.handler(`Writing to ${outFilePath}\n`)
+        await this.writeDialog(content.content, outFilePath, dialogType, files)
+        writeDone = true
       }
     }
 
@@ -467,4 +465,39 @@ export class Builder {
     await qnaBuildCore.publishKB(recognizer.getKBId())
     this.handler(`Publishing finished for kb ${kbName}\n`)
   }
+
+  async writeDialog(content: string, filePath: string, dialogType: string, files: string[]) {
+    await fs.writeFile(filePath, content, 'utf-8')
+    const contentObj = JSON.parse(content)
+    if (dialogType === recognizerType.CROSSTRAINED && contentObj.$kind === 'Microsoft.MultiLanguageRecognizer') {
+      const fileName = path.basename(filePath, '.dialog')
+
+      for (const file of files) {
+        let qnafileName
+        let cultureFromPath = fileHelper.getCultureFromPath(file)
+        if (cultureFromPath) {
+          let fileNameWithCulture = path.basename(file, path.extname(file))
+          qnafileName = fileNameWithCulture.substring(0, fileNameWithCulture.length - cultureFromPath.length - 1)
+        } else {
+          qnafileName = path.basename(file, path.extname(file))
+        }
+
+        let crossTrainedFileName = `${qnafileName}.lu.qna.dialog`
+        let crossTrainedFilePath = path.join(path.dirname(filePath), crossTrainedFileName)
+        if (fs.existsSync(crossTrainedFilePath)) {
+          let existingCRDialog = JSON.parse(await fileHelper.getContentFromFile(crossTrainedFilePath))
+          if (!existingCRDialog.recognizers.includes(fileName)) {
+            existingCRDialog.recognizers.push(fileName)
+          }
+
+          content = JSON.stringify(existingCRDialog, null, 4)
+        } else {
+          let recognizers = [fileName]
+          content = new CrossTrainedRecognizer(crossTrainedFilePath, recognizers).save()
+        }
+
+        await fs.writeFile(crossTrainedFilePath, content, 'utf-8')
+      }
+    }
+  }
 }

packages/lu/src/parser/qnabuild/builder.ts

@@ -0,0 +1,27 @@
+/*!
+ * Copyright (c) Microsoft Corporation. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+export class CrossTrainedRecognizer {
+  public recognizers: any
+  private readonly dialogPath: string
+
+  constructor(dialogPath: string, recognizers: any) {
+    this.dialogPath = dialogPath
+    this.recognizers = recognizers
+  }
+
+  save(): string {
+    let output = {
+      $kind: 'Microsoft.CrossTrainedRecognizerSet',
+      recognizers: this.recognizers
+    }
+
+    return JSON.stringify(output, null, 4)
+  }
+
+  getDialogPath(): string {
+    return this.dialogPath
+  }
+}