qnamaker:build support --qnaConfig (#840)

feich-ms · web-flow · commit f95fb9875efd · 2020-06-05T12:40:29.000+08:00
* support --qnaConfig and extract subscriptionKey from system config

* optimize typo
diff --git a/packages/lu/src/parser/qnabuild/builder.ts b/packages/lu/src/parser/qnabuild/builder.ts
@@ -32,7 +32,6 @@ export class Builder {
 
   async loadContents(
     files: string[],
-    inputFolder: string,
     botName: string,
     suffix: string,
     region: string,
@@ -43,10 +42,17 @@ export class Builder {
     let qnaContents = new Map<string, string>()
 
     for (const file of files) {
-      const qnaFiles = await fileHelper.getLuObjects(undefined, file, true, fileExtEnum.QnAFile)
+      let fileCulture: string
+      let cultureFromPath = fileHelper.getCultureFromPath(file)
+      if (cultureFromPath) {
+        fileCulture = cultureFromPath
+      } else {
+        fileCulture = culture
+      }
 
       let fileContent = ''
       let result
+      const qnaFiles = await fileHelper.getLuObjects(undefined, file, true, fileExtEnum.QnAFile)
       try {
         result = await qnaBuilderVerbose.build(qnaFiles, true)
 
@@ -62,16 +68,10 @@ export class Builder {
       }
 
       this.handler(`${file} loaded\n`)
-      let fileCulture: string
-      let cultureFromPath = fileHelper.getCultureFromPath(file)
-      if (cultureFromPath) {
-        fileCulture = cultureFromPath
-      } else {
-        fileCulture = culture
-      }
 
+      const fileFolder = path.dirname(file)
       if (multiRecognizer === undefined) {
-        const multiRecognizerPath = path.join(inputFolder, `${botName}.qna.dialog`)
+        const multiRecognizerPath = path.join(fileFolder, `${botName}.qna.dialog`)
         let multiRecognizerContent = {}
         if (fs.existsSync(multiRecognizerPath)) {
           multiRecognizerContent = JSON.parse(await fileHelper.getContentFromFile(multiRecognizerPath)).recognizers
@@ -82,7 +82,7 @@ export class Builder {
       }
 
       if (settings === undefined) {
-        const settingsPath = path.join(inputFolder, `qnamaker.settings.${suffix}.${region}.json`)
+        const settingsPath = path.join(fileFolder, `qnamaker.settings.${suffix}.${region}.json`)
         let settingsContent = {}
         if (fs.existsSync(settingsPath)) {
           settingsContent = JSON.parse(await fileHelper.getContentFromFile(settingsPath)).qna
@@ -95,7 +95,7 @@ export class Builder {
       const content = new Content(fileContent, new qnaOptions(botName, true, fileCulture, file))
 
       if (!recognizers.has(content.name)) {
-        const dialogFile = path.join(inputFolder, `${content.name}.dialog`)
+        const dialogFile = path.join(fileFolder, `${content.name}.dialog`)
         let existingDialogObj: any
         if (fs.existsSync(dialogFile)) {
           existingDialogObj = JSON.parse(await fileHelper.getContentFromFile(dialogFile))
diff --git a/packages/qnamaker/README.md b/packages/qnamaker/README.md
@@ -140,6 +140,8 @@ OPTIONS
 
   --log                                  write out log messages to console
 
+  --luConfig=luConfig                    Path to config for qnamaker build which can contain switches for arguments
+
   --region=region                        [default: westus] Overrides public endpoint
                                          https://<region>.api.cognitive.microsoft.com/qnamaker/v4.0/
 
diff --git a/packages/qnamaker/src/commands/qnamaker/build.ts b/packages/qnamaker/src/commands/qnamaker/build.ts
@@ -4,7 +4,10 @@
  */
 
 import {CLIError, Command, flags} from '@microsoft/bf-cli-command'
+import {processFlags} from '../../utils/qnamakerbase'
+
 const path = require('path')
+const fs = require('fs-extra')
 const username = require('username')
 const exception = require('@microsoft/bf-lu/lib/parser/utils/exception')
 const file = require('@microsoft/bf-lu/lib/utils/filehelper')
@@ -27,59 +30,89 @@ export default class QnamakerBuild extends Command {
   static flags: any = {
     help: flags.help({char: 'h'}),
     in: flags.string({char: 'i', description: 'Source .qna file or folder'}),
-    subscriptionKey: flags.string({char: 's', description: 'QnA maker subscription key', required: true}),
-    botName: flags.string({char: 'b', description: 'Bot name', required: true}),
+    subscriptionKey: flags.string({char: 's', description: 'QnA maker subscription key'}),
+    botName: flags.string({char: 'b', description: 'Bot name'}),
     region: flags.string({description: 'Overrides public endpoint https://<region>.api.cognitive.microsoft.com/qnamaker/v4.0/', default: 'westus'}),
     out: flags.string({char: 'o', description: 'Output folder name to write out .dialog files. If not specified, knowledge base ids will be output to console'}),
     defaultCulture: flags.string({description: 'Culture code for the content. Infer from .qna if available. Defaults to en-us if not set'}),
     fallbackLocale: flags.string({description: 'Locale to be used at the fallback if no locale specific recognizer is found. Only valid if --out is set'}),
     suffix: flags.string({description: 'Environment name as a suffix identifier to include in qnamaker kb name. Defaults to current logged in user alias'}),
     dialog: flags.string({description: 'Dialog recognizer type [multiLanguage|crosstrained]', default: 'multiLanguage'}),
     force: flags.boolean({char: 'f', description: 'If --out flag is provided, overwrites relevant dialog file', default: false}),
+    qnaConfig: flags.string({description: 'Path to config for qna build which can contain switches for arguments'}),
     log: flags.boolean({description: 'write out log messages to console', default: false}),
   }
 
   async run() {
     try {
       const {flags}: any = this.parse(QnamakerBuild)
 
+      // Luconfig overrides flags
+      let files: string[] = []
+      if (flags.qnaConfig) {
+        const configFilePath = path.resolve(flags.qnaConfig)
+        if (await fs.exists(configFilePath)) {
+          const configObj = JSON.parse(await file.getContentFromFile(configFilePath))
+          for (let prop of Object.keys(configObj)) {
+            if (prop === 'models') {
+              files = configObj.models.map((m: string) => path.isAbsolute(m) ? m : path.join(path.dirname(configFilePath), m))
+            } else if (prop === 'out') {
+              flags.out = path.isAbsolute(configObj.out) ? configObj.out : path.join(path.dirname(configFilePath), configObj.out)
+            } else {
+              flags[prop] = configObj[prop]
+            }
+          }
+        }
+      }
+
+      // Flags override userConfig
+      let qnamakerBuildFlags = Object.keys(QnamakerBuild.flags)
+      qnamakerBuildFlags.push('endpoint')
+
+      let {inVal, subscriptionKey, botName, region, out, defaultCulture, fallbackLocale, suffix, dialog, force, log, endpoint}
+        = await processFlags(flags, qnamakerBuildFlags, this.config.configDir)
+
       flags.stdin = await this.readStdin()
 
-      if (!flags.stdin && !flags.in) {
+      if (!flags.stdin && !inVal && files.length === 0) {
         throw new CLIError('Missing input. Please use stdin or pass a file or folder location with --in flag')
       }
 
-      if (flags.dialog && flags.dialog !== recognizerType.MULTILANGUAGE && flags.dialog !== recognizerType.CROSSTRAINED) {
+      if (!subscriptionKey) {
+        throw new CLIError('Missing qnamaker subscription key. Please pass subscription key with --subscriptionKey flag or specify via bf config:set:qnamaker --subscriptionKey.')
+      }
+
+      if (!botName) {
+        throw new CLIError('Missing bot name. Please pass bot name with --botName flag or specify via --qnaConfig.')
+      }
+
+      if (dialog && dialog !== recognizerType.MULTILANGUAGE && dialog !== recognizerType.CROSSTRAINED) {
         throw new CLIError('Recognizer type specified by --dialog is not right. Please specify [multiLanguage|crosstrained]')
       }
 
-      flags.defaultCulture = flags.defaultCulture && flags.defaultCulture !== '' ? flags.defaultCulture : 'en-us'
-      flags.region = flags.region && flags.region !== '' ? flags.region : 'westus'
-      flags.suffix = flags.suffix && flags.suffix !== '' ? flags.suffix : await username() || 'development'
-      flags.fallbackLocale = flags.fallbackLocale && flags.fallbackLocale !== '' ? flags.fallbackLocale : 'en-us'
+      defaultCulture = defaultCulture && defaultCulture !== '' ? defaultCulture : 'en-us'
+      region = region && region !== '' ? region : 'westus'
+      suffix = suffix && suffix !== '' ? suffix : await username() || 'development'
+      fallbackLocale = fallbackLocale && fallbackLocale !== '' ? fallbackLocale : 'en-us'
 
-      const endpoint = `https://${flags.region}.api.cognitive.microsoft.com/qnamaker/v4.0`
+      endpoint = endpoint && endpoint !== '' ? endpoint : `https://${region}.api.cognitive.microsoft.com/qnamaker/v4.0`
 
       // create builder class
       const builder = new Builder((input: string) => {
-        if (flags.log) this.log(input)
+        if (log) this.log(input)
       })
 
       let qnaContents: any[] = []
       let recognizers = new Map<string, any>()
       let multiRecognizer: any
       let settings: any
-
-      const dialogFilePath = (flags.stdin || !flags.in) ? process.cwd() : flags.in.endsWith(fileExtEnum.QnAFile) ? path.dirname(path.resolve(flags.in)) : path.resolve(flags.in)
       
-      let files: string[] = []
-
-      if (flags.in && flags.in !== '') {
-        if (flags.log) this.log('Loading files...\n')
+      if ((inVal && inVal !== '') || files.length > 0) {
+        if (log) this.log('Loading files...\n')
 
         // get qna files from flags.in.
-        if (flags.in && flags.in !== '') {
-          const qnaFiles: string[] = await file.getLuFiles(flags.in, true, fileExtEnum.QnAFile)
+        if (inVal && inVal !== '') {
+          const qnaFiles: string[] = await file.getLuFiles(inVal, true, fileExtEnum.QnAFile)
           files.push(...qnaFiles)
         }
 
@@ -88,37 +121,37 @@ export default class QnamakerBuild extends Command {
 
         // load qna contents from qna files
         // load existing recognizers, multiRecogniers and settings or create default ones
-        const loadedResources = await builder.loadContents(files, dialogFilePath, flags.botName, flags.suffix, flags.region, flags.defaultCulture)
+        const loadedResources = await builder.loadContents(files, botName, suffix, region, defaultCulture)
         qnaContents = loadedResources.qnaContents
         recognizers = loadedResources.recognizers
         multiRecognizer = loadedResources.multiRecognizer
         settings = loadedResources.settings
       } else {
         // load qna content from stdin and create default recognizer, multiRecognier and settings
-        if (flags.log) this.log('Load qna content from stdin\n')
-        const content = new Content(flags.stdin, new qnaOptions(flags.botName, true, flags.defaultCulture, path.join(process.cwd(), 'stdin')))
+        if (log) this.log('Load qna content from stdin\n')
+        const content = new Content(flags.stdin, new qnaOptions(botName, true, defaultCulture, path.join(process.cwd(), 'stdin')))
         qnaContents.push(content)
-        multiRecognizer = new MultiLanguageRecognizer(path.join(process.cwd(), `${flags.botName}.qna.dialog`), {})
-        settings = new Settings(path.join(process.cwd(), `qnamaker.settings.${flags.suffix}.${flags.region}.json`), {})
+        multiRecognizer = new MultiLanguageRecognizer(path.join(process.cwd(), `${botName}.qna.dialog`), {})
+        settings = new Settings(path.join(process.cwd(), `qnamaker.settings.${suffix}.${region}.json`), {})
         const recognizer = Recognizer.load(content.path, content.name, path.join(process.cwd(), `${content.name}.dialog`), settings, {})
         recognizers.set(content.name, recognizer)
       }
 
       // update or create and then publish qnamaker kb based on loaded resources
-      if (flags.log) this.log('Handling qnamaker knowledge bases...')
-      const dialogContents = await builder.build(qnaContents, recognizers, flags.subscriptionKey, endpoint, flags.botName, flags.suffix, flags.fallbackLocale, multiRecognizer, settings)
+      if (log) this.log('Handling qnamaker knowledge bases...')
+      const dialogContents = await builder.build(qnaContents, recognizers, subscriptionKey, endpoint, botName, suffix, fallbackLocale, multiRecognizer, settings)
 
       // get endpointKeys
-      const endpointKeysInfo = await builder.getEndpointKeys(flags.subscriptionKey, endpoint)
+      const endpointKeysInfo = await builder.getEndpointKeys(subscriptionKey, endpoint)
       const endpointKeys: any = {
         "primaryEndpointKey": endpointKeysInfo.primaryEndpointKey,
         "secondaryEndpointKey": endpointKeysInfo.secondaryEndpointKey
       }
 
       // write dialog assets based on config
-      if (flags.out) {
-        const outputFolder = path.resolve(flags.out)
-        const writeDone = await builder.writeDialogAssets(dialogContents, flags.force, outputFolder, flags.dialog, files)
+      if (out) {
+        const outputFolder = path.resolve(out)
+        const writeDone = await builder.writeDialogAssets(dialogContents, force, outputFolder, dialog, files)
         if (writeDone) {
           this.log(`Successfully wrote .dialog files to ${outputFolder}\n`)
           this.log('QnA knowledge base endpointKeys:')
diff --git a/packages/qnamaker/src/utils/qnamakerbase.ts b/packages/qnamaker/src/utils/qnamakerbase.ts
@@ -11,6 +11,7 @@ const srvMan = require('./../../utils/servicemanifest')
 const {ServiceBase} = require('./../../utils/api/serviceBase')
 const file = require('@microsoft/bf-lu/lib/utils/filehelper')
 
+const configPrefix = 'qnamaker__'
 
 export async function processInputs(flags: any, payload: any, configfile: string, stdin = '') {
   let result: Inputs = {}
@@ -65,3 +66,52 @@ export async function updateQnAMakerConfig(config: any , configfile: string) {
 export interface Inputs {
   [key: string]: any
 }
+
+export async function processFlags(flags: any, flagLabels: string[], configDir: string) {
+  let config = filterByAllowedConfigValues(await getUserConfig(configDir), configPrefix)
+  config = config ? filterConfig(config, configPrefix) : config
+  const input: any = {}
+  flagLabels
+    .filter(flag => flag !== 'help')
+    .map((flag: string) => {
+      if (flag === 'in') {
+        // rename property since 'in' is a reserved keyword
+        input[`${flag}Val`] = flags[flag]
+      }
+
+      input[flag] = flags[flag] || (config ? config[configPrefix + flag] : null)
+    })
+  
+  return input
+}
+
+async function getUserConfig (configPath: string) {
+  if (fs.existsSync(path.join(configPath, 'config.json'))) {
+    return fs.readJSON(path.join(configPath, 'config.json'), {throws: false})
+  }
+
+  return {}
+}
+
+function filterByAllowedConfigValues (configObj: any, prefix: string) {
+  const allowedConfigValues = [`${prefix}kbId`, `${prefix}endpoint`, `${prefix}region`, `${prefix}subscriptionKey`]
+  const filtered = Object.keys(configObj)
+  .filter(key => allowedConfigValues.includes(key))
+  .reduce((filteredConfigObj: any, key) => {
+    filteredConfigObj[key] = configObj[key]
+    
+    return filteredConfigObj
+  }, {})
+
+  return filtered
+}
+
+function filterConfig (config: any, prefix: string) {
+  return Object.keys(config)
+    .filter((key: string) => key.startsWith(prefix))
+    .reduce((filteredConfig: any, key: string) => {
+      filteredConfig[key] = config[key]
+
+      return filteredConfig
+    }, {})
+}
diff --git a/packages/qnamaker/test/commands/qnamaker/build.test.ts b/packages/qnamaker/test/commands/qnamaker/build.test.ts
@@ -26,7 +26,7 @@ describe('qnamaker:build cli parameters test', () => {
     .stderr()
     .command(['qnamaker:build', '--in', `${path.join(__dirname, './../../fixtures/testcases/qnabuild')}`, '--botName', 'Contoso'])
     .it('displays an error if any required input parameters are missing', ctx => {
-      expect(ctx.stderr).to.contain('Missing required flag:\n -s, --subscriptionKey SUBSCRIPTIONKEY  QnA maker subscription key')
+      expect(ctx.stderr).to.contain('Missing qnamaker subscription key. Please pass subscription key with --subscriptionKey flag or specify via bf config:set:qnamaker --subscriptionKey.')
     })
 
   test
@@ -42,7 +42,7 @@ describe('qnamaker:build cli parameters test', () => {
     .stderr()
     .command(['qnamaker:build', '--subscriptionKey', uuidv1(), '--in', `${path.join(__dirname, './../../fixtures/testcases/qnabuild')}`])
     .it('displays an error if any required input parameters are missing', ctx => {
-      expect(ctx.stderr).to.contain('Missing required flag:\n -b, --botName BOTNAME  Bot name')
+      expect(ctx.stderr).to.contain('Missing bot name. Please pass bot name with --botName flag or specify via --qnaConfig.')
     })
   
   test
@@ -544,4 +544,66 @@ describe('qnamaker:build update knowledge base with multiturn successfully when
       expect(ctx.stdout).to.contain('Updating finished')
       expect(ctx.stdout).to.contain('Publishing kb')
     })
+})
+
+describe('qnamaker:build update knowledge base successfully with parameters set from qna config', () => {
+  before(async function () {
+    await fs.ensureDir(path.join(__dirname, './../../../results/'))
+
+    nock('https://westus.api.cognitive.microsoft.com')
+      .get(uri => uri.includes('qnamaker'))
+      .reply(200, {
+        knowledgebases:
+          [{
+            name: 'test(development).en-us.qna',
+            id: 'f8c64e2a-1111-3a09-8f78-39d7adc76ec5',
+            hostName: 'https://myqnamakerbot.azurewebsites.net'
+          }]
+      })
+
+    nock('https://westus.api.cognitive.microsoft.com')
+      .get(uri => uri.includes('knowledgebases'))
+      .reply(200, {
+        qnaDocuments: [{
+          id: 1,
+          source: 'custom editorial',
+          questions: ['how many sandwich types do you have'],
+          answer: '25 types',
+          metadata: []
+        }]
+      })
+
+    nock('https://westus.api.cognitive.microsoft.com')
+      .put(uri => uri.includes('knowledgebases'))
+      .reply(204)
+
+    nock('https://westus.api.cognitive.microsoft.com')
+      .post(uri => uri.includes('knowledgebases'))
+      .reply(204)
+
+    nock('https://westus.api.cognitive.microsoft.com')
+      .get(uri => uri.includes('endpointkeys'))
+      .reply(200, {
+        primaryEndpointKey: 'xxxx',
+        secondaryEndpointKey: 'yyyy'
+      })
+  })
+
+  after(async function () {
+    await fs.remove(path.join(__dirname, './../../../results/'))
+  })
+
+  test
+    .stdout()
+    .command(['qnamaker:build', '--qnaConfig', './test/fixtures/testcases/qnabuild/sandwich/qnafiles/qnaconfig.json', '--subscriptionKey', uuidv1()])
+    .it('should update a knowledge base successfully with parameters set from qna config', async ctx => {
+      expect(ctx.stdout).to.contain('Handling qnamaker knowledge bases...')
+      expect(ctx.stdout).to.contain('Updating to new version for kb test(development).en-us.qna')
+      expect(ctx.stdout).to.contain('Updating finished')
+      expect(ctx.stdout).to.contain('Publishing kb')
+
+      expect(await compareFiles('./../../../results/qnamaker.settings.development.westus.json', './../../fixtures/testcases/qnabuild/sandwich/config/qnamaker.settings.development.westus.json')).to.be.true
+      expect(await compareFiles('./../../../results/test.en-us.qna.dialog', './../../fixtures/testcases/qnabuild/sandwich/dialogs/test.en-us.qna.dialog')).to.be.true
+      expect(await compareFiles('./../../../results/test.qna.dialog', './../../fixtures/testcases/qnabuild/sandwich/dialogs/test.qna.dialog')).to.be.true
+    })
 })
diff --git a/packages/qnamaker/test/fixtures/testcases/qnabuild/sandwich/qnafiles/qnaconfig.json b/packages/qnamaker/test/fixtures/testcases/qnabuild/sandwich/qnafiles/qnaconfig.json
