[R10] fix issues #770 and #778 (#785)

* fix pattern issue in cross train

* fix bugs

* fix posttest

* refactor endpoint logic and add tests

Co-authored-by: Vishwac Sena Kannan <[email protected]>

Author: feich-ms

packages/lu/src/parser/lubuild/builder.ts

@@ -43,10 +43,20 @@ export class Builder {
       let fileName: string
       const luFiles = await fileHelper.getLuObjects(undefined, file, true, fileExtEnum.LUFile)
 
+      let cultureFromPath = fileHelper.getCultureFromPath(file)
+      if (cultureFromPath) {
+        fileCulture = cultureFromPath
+        let fileNameWithCulture = path.basename(file, path.extname(file))
+        fileName = fileNameWithCulture.substring(0, fileNameWithCulture.length - fileCulture.length - 1)
+      } else {
+        fileCulture = culture
+        fileName = path.basename(file, path.extname(file))
+      }
+
       let fileContent = ''
       let result
       try {
-        result = await LuisBuilderVerbose.build(luFiles, true, culture)
+        result = await LuisBuilderVerbose.build(luFiles, true, fileCulture)
         fileContent = result.parseToLuContent()
       } catch (err) {
         if (err.source) {
@@ -58,15 +68,6 @@ export class Builder {
       }
 
       this.handler(`${file} loaded\n`)
-      let cultureFromPath = fileHelper.getCultureFromPath(file)
-      if (cultureFromPath) {
-        fileCulture = cultureFromPath
-        let fileNameWithCulture = path.basename(file, path.extname(file))
-        fileName = fileNameWithCulture.substring(0, fileNameWithCulture.length - fileCulture.length - 1)
-      } else {
-        fileCulture = result.culture !== 'en-us' ? result.culture : culture
-        fileName = path.basename(file, path.extname(file))
-      }
 
       const fileFolder = path.dirname(file)
       const multiRecognizerPath = path.join(fileFolder, `${fileName}.lu.dialog`)
@@ -122,7 +123,7 @@ export class Builder {
     luContents: any[],
     recognizers: Map<string, Recognizer>,
     authoringKey: string,
-    region: string,
+    endpoint: string,
     botName: string,
     suffix: string,
     fallbackLocale: string,
@@ -136,7 +137,7 @@ export class Builder {
     // set luis call delay duration to 1100 millisecond because 1000 can hit corner case of rate limit
     let delayDuration = 1100
 
-    const luBuildCore = new LuBuildCore(authoringKey, `https://${region}.api.cognitive.microsoft.com`)
+    const luBuildCore = new LuBuildCore(authoringKey, endpoint)
     const apps = await luBuildCore.getApplicationList()
 
     // here we do a while loop to make full use of luis tps capacity

packages/luis/src/commands/luis/build.ts

@@ -66,8 +66,11 @@ export default class LuisBuild extends Command {
       }
 
       // Flags override userConfig
-      let {inVal, authoringKey, botName, region, out, defaultCulture, fallbackLocale, suffix, dialog, force, luConfig, deleteOldVersion, log}
-        = await utils.processInputs(flags, Object.keys(LuisBuild.flags), this.config.configDir)
+      let luisBuildFlags = Object.keys(LuisBuild.flags)
+      luisBuildFlags.push('endpoint')
+
+      let {inVal, authoringKey, botName, region, out, defaultCulture, fallbackLocale, suffix, dialog, force, luConfig, deleteOldVersion, log, endpoint}
+        = await utils.processInputs(flags, luisBuildFlags, this.config.configDir)
 
       flags.stdin = await this.readStdin()
 
@@ -92,6 +95,8 @@ export default class LuisBuild extends Command {
       suffix = suffix && suffix !== '' ? suffix : await username() || 'development'
       fallbackLocale = fallbackLocale && fallbackLocale !== '' ? fallbackLocale : 'en-us'
 
+      endpoint = endpoint && endpoint !== '' ? endpoint : `https://${region}.api.cognitive.microsoft.com`
+
       // create builder class
       const builder = new Builder((input: string) => {
         if (log) this.log(input)
@@ -134,7 +139,7 @@ export default class LuisBuild extends Command {
 
       // update or create and then train and publish luis applications based on loaded resources
       if (log) this.log('Handling applications...')
-      const dialogContents = await builder.build(luContents, recognizers, authoringKey, region, botName, flags.suffix, fallbackLocale, deleteOldVersion, multiRecognizers, settings)
+      const dialogContents = await builder.build(luContents, recognizers, authoringKey, endpoint, botName, flags.suffix, fallbackLocale, deleteOldVersion, multiRecognizers, settings)
 
       // write dialog assets based on config
       if (dialog) {

packages/luis/test/commands/luis/build.test.ts

@@ -768,4 +768,65 @@ describe('luis:build update application succeed with parameters set from luconfi
       expect(await compareFiles('./../../../results/test.en-us.lu.dialog', './../../fixtures/testcases/lubuild/luconfig/dialogs/test.en-us.lu.dialog')).to.be.true
       expect(await compareFiles('./../../../results/test.lu.dialog', './../../fixtures/testcases/lubuild/luconfig/dialogs/test.lu.dialog')).to.be.true
     })
+})
+
+describe('luis:build create a new application successfully with locale set to it-it', () => {
+  before(async function () {
+    await fs.ensureDir(path.join(__dirname, './../../../results/'))
+
+    nock('https://westus.api.cognitive.microsoft.com')
+      .get(uri => uri.includes('apps'))
+      .reply(200, [{
+        name: 'test.en-us.lu',
+        id: 'f8c64e2a-1111-3a09-8f78-39d7adc76ec5'
+      }])
+
+    nock('https://westus.api.cognitive.microsoft.com')
+      .post(uri => uri.includes('import'))
+      .reply(201, {
+        appId: 'f8c64e2a-2222-3a09-8f78-39d7adc76ec5'
+      })
+
+    nock('https://westus.api.cognitive.microsoft.com')
+      .post(uri => uri.includes('train'))
+      .reply(202, {
+        statusId: 2,
+        status: 'UpToDate'
+      })
+
+    nock('https://westus.api.cognitive.microsoft.com')
+      .get(uri => uri.includes('train'))
+      .reply(200, [{
+        modelId: '99999',
+        details: {
+          statusId: 0,
+          status: 'Success',
+          exampleCount: 0
+        }
+      }])
+
+    nock('https://westus.api.cognitive.microsoft.com')
+      .post(uri => uri.includes('publish'))
+      .reply(201, {
+        versionId: '0.2',
+        isStaging: true
+      })
+  })
+
+  after(async function () {
+    await fs.remove(path.join(__dirname, './../../../results/'))
+  })
+
+  test.only()
+    .stdout()
+    .command(['luis:build', '--in', './test/fixtures/testcases/lubuild/file-with-personName/personName.en-us.lu', '--authoringKey', uuidv1(), '--botName', 'test', '--log', '--suffix', 'development', '--out', './results', '--defaultCulture', 'it-it'])
+    .it('should create a new application successfully for prebuilt entity personName with locale set to it-it', ctx => {
+      expect(ctx.stdout).to.contain('Handling applications...')
+      expect(ctx.stdout).to.contain('Creating LUIS.ai application')
+      expect(ctx.stdout).to.contain('training version=0.1')
+      expect(ctx.stdout).to.contain('waiting for training for version=0.1')
+      expect(ctx.stdout).to.contain('publishing version=0.1')
+      expect(ctx.stdout).to.contain('publishing finished')
+      expect(ctx.stdout.includes('personName is not available for the requested locale: it-it')).to.be.false
+    })
 })

packages/luis/test/fixtures/testcases/lubuild/file-with-personName/personName.en-us.lu

@@ -0,0 +1,6 @@
+# UserProfile
+- get started
+- my name is {@userName}
+- Profile
+
+@ prebuilt personName hasRoles userName