Fix luis:cross-train generate incorrect culture info from file names (#1300)

* fix locale from file incorrectly parse to luis app

* fix qna cross train test cases

Author: Shuai Wang

packages/lu/src/parser/cross-train/crossTrainer.js

@@ -517,7 +517,7 @@ const parseAndValidateContent = async function (objectArray, verbose, importReso
     if (object.content && object.content !== '') {
       if (fileExt === fileExtEnum.LUFile) {
         if (!object.id.endsWith(fileExtEnum.LUFile)) object.id += fileExtEnum.LUFile
-        let result = await LuisBuilderVerbose.build([object], verbose, undefined, importResolver)
+        let result = await LuisBuilderVerbose.build([object], verbose, object.language, importResolver)
         let luisObj = new Luis(result)
         fileContent = luisObj.parseToLuContent()
       } else {

packages/lu/src/parser/lu/luOptions.js

@@ -1,8 +1,15 @@
+const getLuisCultureFromPath = require('../../utils/localehelper').getLuisCultureFromPath
+
 class LuOptions {
     constructor(id = '', includeInCollate = true, language = '', path = ''){
+        let fileLocale
+        if (id) {
+            fileLocale = getLuisCultureFromPath(`${id}.lu`)
+        }
+
         this.id = id ? id : get_guid()
         this.includeInCollate = includeInCollate
-        this.language = language
+        this.language = language ? language : fileLocale ? fileLocale : ''
         this.path = path
     }
 }

packages/lu/src/parser/lubuild/builder.ts

@@ -12,6 +12,7 @@ const path = require('path')
 const fs = require('fs-extra')
 const delay = require('delay')
 const fileHelper = require('./../../utils/filehelper')
+const localeHelper = require('./../../utils/localehelper')
 const fileExtEnum = require('./../utils/helpers').FileExtTypeEnum
 const retCode = require('./../utils/enums/CLI-errors')
 const exception = require('./../utils/exception')
@@ -43,7 +44,7 @@ export class Builder {
       let fileCulture: string
       let fileName: string
 
-      let cultureFromPath = fileHelper.getLuisCultureFromPath(file)
+      let cultureFromPath = localeHelper.getLuisCultureFromPath(file)
       if (cultureFromPath) {
         fileCulture = cultureFromPath
         let fileNameWithCulture = path.basename(file, path.extname(file))

packages/lu/src/parser/lufile/parseFileContents.js

@@ -284,6 +284,7 @@ const parseLuAndQnaWithAntlr = async function (parsedContent, fileContent, log,
 
     helpers.checkAndUpdateVersion(parsedContent.LUISJsonStructure);
 
+    updateCulture(parsedContent.LUISJsonStructure, locale)
 }
 
 const updateIntentAndEntityFeatures = function(luisObj) {
@@ -294,6 +295,12 @@ const updateIntentAndEntityFeatures = function(luisObj) {
     })
 }
 
+const updateCulture = function(luisObj, locale) {
+    if (!luisObj.culture) {
+        luisObj.culture = locale ? locale : 'en-us'
+    }
+}
+
 const updateFeaturesWithPL = function(collection, plName) {
     if (collection === null || collection === undefined) return;
     if (collection.hasOwnProperty("features")) {

packages/lu/src/parser/luis/luis.js

@@ -21,7 +21,7 @@ class Luis {
         this.versionId = "0.1";
         this.name = "";
         this.desc = "";
-        this.culture = "en-us";
+        this.culture = "";
 
         if (LuisJSON) {
             initialize(this, LuisJSON)

packages/lu/src/parser/utils/helpers.js

@@ -8,7 +8,6 @@ const retCode = require('./enums/CLI-errors');
 const exception = require('./exception');
 const NEWLINE = require('os').EOL;
 const ANY_NEWLINE = /\r\n|\r|\n/g;
-const url = require('url');
 const hClasses = require('../lufile/classes/hclasses');
 const helpers = {
 

packages/lu/src/utils/filehelper.ts

@@ -300,35 +300,6 @@ export function parseJSON(input: string, appType: string) {
   }
 }
 
-export function getLuisCultureFromPath(file: string): string | null {
-  let fn = path.basename(file, path.extname(file))
-  let lang = path.extname(fn).substring(1)
-  switch (lang.toLowerCase()) {
-    case 'en-us':
-    case 'ar-ar':
-    case 'zh-cn':
-    case 'nl-nl':
-    case 'fr-fr':
-    case 'fr-ca':
-    case 'de-de':
-    case 'gu-in':
-    case 'hi-in':
-    case 'it-it':
-    case 'ja-jp':
-    case 'ko-kr':
-    case 'mr-in':
-    case 'pt-br':
-    case 'es-es':
-    case 'es-mx':
-    case 'ta-in':
-    case 'te-in':
-    case 'tr-tr':
-      return lang
-    default:
-      return null
-  }
-}
-
 export function getQnACultureFromPath(file: string): string | null {
   let fn = path.basename(file, path.extname(file))
   let lang = path.extname(fn).substring(1)

packages/lu/src/utils/localehelper.ts

@@ -0,0 +1,35 @@
+/*!
+ * Copyright (c) Microsoft Corporation. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+const path = require('path')
+
+export function getLuisCultureFromPath(file: string): string | null {
+    let fn = path.basename(file, path.extname(file))
+    let lang = path.extname(fn).substring(1)
+    switch (lang.toLowerCase()) {
+      case 'en-us':
+      case 'ar-ar':
+      case 'zh-cn':
+      case 'nl-nl':
+      case 'fr-fr':
+      case 'fr-ca':
+      case 'de-de':
+      case 'gu-in':
+      case 'hi-in':
+      case 'it-it':
+      case 'ja-jp':
+      case 'ko-kr':
+      case 'mr-in':
+      case 'pt-br':
+      case 'es-es':
+      case 'es-mx':
+      case 'ta-in':
+      case 'te-in':
+      case 'tr-tr':
+        return lang
+      default:
+        return null
+    }
+  }

packages/luis/test/fixtures/testcases/interruption/dia2/dia2.lu

@@ -1,3 +1,5 @@
+> !# @app.culture = it-it
+
 # dia3_trigger
 - book a flight from {fromLocation = Seattle} to {toLocation = Beijing}
 

packages/luis/test/fixtures/verified/interruption/dia1.fr-fr.lu

@@ -1,7 +1,7 @@
 
 > LUIS application information
 > !# @app.versionId = 0.1
-> !# @app.culture = en-us
+> !# @app.culture = fr-fr
 > !# @app.luis_schema_version = 3.2.0