resolve cross train issues (#814)

* make --config relative to cwd for luis and qnamaker cross train

* fix typo

* resolve imports and only copy files sepcified in config

* support case insensitive config and cross train empty file and adjust tests

* adjust test cases for luis:cross-train and qnamaker:cross-train

* remove patterns with prebuilt entity from cross-trained _Interruption intent

* validate prebuilt entities in pattern having explicit definitions

* adjust test cases in luis package

* optimize crossTrainer dedup logic to make it compare lower case of utterances and optimize luConverter to make sure there is only one whitespace between words in utterances

* split large QA pair into smaller ones to overcome the limit of QnAMaker in cross train and fix error not thrown issue in qnamaker

* add unit tests for split large KB pair into smaller ones in cross train

* resolve file name lower casing issue when write out

Co-authored-by: Vishwac Sena Kannan <[email protected]>

Author: feich-ms

packages/lu/src/parser/cross-train/cross-train.js

@@ -36,11 +36,11 @@ module.exports = {
     // Get all related file content.
     const luContents = await file.getFilesContent(input, fileExtEnum.LUFile)
     const qnaContents = await file.getFilesContent(input, fileExtEnum.QnAFile)
-    const configContent = config && !fs.existsSync(config) ? {id: path.join(input, 'config.json'), content: config} : await file.getConfigContent(config || input)
+    const configContent = config && !fs.existsSync(config) ? {id: path.join(input, 'config.json'), content: config} : await file.getConfigContent(config)
 
     const configObject = file.getConfigObject(configContent, intentName)
 
-    const trainedResult = crossTrainer.crossTrain(luContents, qnaContents, configObject)
+    const trainedResult = await crossTrainer.crossTrain(luContents, qnaContents, configObject)
 
     return trainedResult
   },

packages/lu/src/parser/cross-train/crossTrainer.js

@@ -358,7 +358,7 @@ export class Builder {
     this.handler(`${recognizer.getLuPath()} training version=${recognizer.versionId}\n`)
     await delay(delayDuration)
     await luBuildCore.trainApplication(recognizer.getAppId(), recognizer.versionId)
-    this.handler(`${recognizer.getLuPath()} waiting for training for version=${recognizer.versionId}...`)
+    this.handler(`${recognizer.getLuPath()} waiting for training for version=${recognizer.versionId}...\n`)
     let done = true
     do {
       await delay(delayDuration)

packages/lu/src/parser/lubuild/builder.ts

@@ -791,6 +791,19 @@ const parseAndHandleSimpleIntentSection = function (parsedContent, luResource) {
                             throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString(), [error]));
                         }
 
+                        let prebuiltEntities = entitiesFound.filter(item => builtInTypes.consolidatedList.includes(item.entity));
+                        prebuiltEntities.forEach(prebuiltEntity => {
+                            if (parsedContent.LUISJsonStructure.prebuiltEntities.findIndex(e => e.name === prebuiltEntity.entity) < 0) {
+                                let errorMsg = `Pattern "${utteranceAndEntities.context.getText()}" has prebuilt entity ${prebuiltEntity.entity}. Please define it explicitly with @ prebuilt ${prebuiltEntity.entity}.`;
+                                let error = BuildDiagnostic({
+                                    message: errorMsg,
+                                    context: utteranceAndEntities.context
+                                })
+
+                                throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString(), [error]));
+                            }
+                        })
+
                         let newPattern = new helperClass.pattern(utterance, intentName);
                         if (!hashTable[uttHash]) {
                             parsedContent.LUISJsonStructure.patterns.push(newPattern);
@@ -1856,8 +1869,8 @@ const parseAndHandleModelInfoSection = function (parsedContent, luResource, log)
             if (kvPair[1] === 'enableSections') continue
 
             if (kvPair.length === 4) {
-                if (kvPair[1] === 'enableMergeIntents' && kvPair[3] === 'false') {
-                    enableMergeIntents = false;
+                if (kvPair[1] === 'enableMergeIntents') {
+                    enableMergeIntents = kvPair[3] === 'false' ? false : true;
                     continue;
                 }
 

packages/lu/src/parser/lufile/parseFileContents.js

@@ -99,7 +99,11 @@ const parseUtterancesToLu = function(utterances, luisJSON){
             getEntitiesByPositionList(sortedEntitiesList, tokenizedText);
             updatedText = tokenizedText.join('');
         }
-        if(updatedText) fileContent += '- ' + updatedText + NEWLINE;
+
+        // remove duplicated whitespaces between words inside utterance to make sure they are aligned with the luis portal
+        // as luis portal only keeps one whitespace between words even if you type multiple ones
+        // this will benefit the comparison of lu files that are converted from local and remote luis application
+        if(updatedText) fileContent += '- ' + updatedText.replace(/\s+/g, ' ') + NEWLINE;
     });
     return fileContent
 }

packages/lu/src/parser/luis/luConverter.js

@@ -7,6 +7,8 @@ import {Recognizer} from './recognizer'
 import {MultiLanguageRecognizer} from './multi-language-recognizer'
 import {Settings} from './settings'
 import * as path from 'path'
+const retCode = require('./../utils/enums/CLI-errors')
+const exception = require('./../utils/exception')
 const Content = require('./../lu/qna')
 const LUOptions = require('./../lu/luOptions')
 const {ServiceBase} = require('./serviceBase')
@@ -22,101 +24,102 @@ export class QnaBuildCore {
   public async getKBList() {
     const response = await this.service.createRequest('/knowledgebases', 'GET')
     const text = await response.text()
-    try {
-      return JSON.parse(text)
-    } catch {
-      return text
+    const kbList = JSON.parse(text)
+    if (kbList.error) {
+      throw (new exception(retCode.errorCode.LUIS_API_CALL_FAILED, text))
     }
+
+    return kbList
   }
 
   public async getKB(kbId: string) {
     const response = await this.service.createRequest(`/knowledgebases/${kbId}`, 'GET')
     const text = await response.text()
-    try {
-      return JSON.parse(text)
-    } catch {
-      return text
+    const kb = JSON.parse(text)
+    if (kb.error) {
+      throw (new exception(retCode.errorCode.LUIS_API_CALL_FAILED, text))
     }
+
+    return kb
   }
 
   public async importKB(kbPayload: any) {
     const response = await this.service.createRequest('/knowledgebases/createasync', 'POST', kbPayload)
     const text = await response.text()
-    try {
-      return JSON.parse(text)
-    } catch {
-      return text
+    const status = JSON.parse(text)
+    if (status.error) {
+      throw (new exception(retCode.errorCode.LUIS_API_CALL_FAILED, text))
     }
+
+    return status
   }
 
   public async getOperationStatus(operationId: string) {
     const response = await this.service.createRequest(`/operations/${operationId}`, 'GET')
     const text = await response.text()
-    try {
-      return JSON.parse(text)
-    } catch {
-      return text
+    const status = JSON.parse(text)
+    if (status.error) {
+      throw (new exception(retCode.errorCode.LUIS_API_CALL_FAILED, text))
     }
+
+    return status
   }
 
   public async exportKB(kbId: string, environment: string) {
     const response = await this.service.createRequest(`/knowledgebases/${kbId}/${environment}/qna`, 'GET')
     const text = await response.text()
-    try {
-      return JSON.parse(text)
-    } catch {
-      return text
+    const kb = JSON.parse(text)
+    if (kb.error) {
+      throw (new exception(retCode.errorCode.LUIS_API_CALL_FAILED, text))
     }
+
+    return kb
   }
 
   public async updateKB(kbId: string, replaceKb: any) {
     const response = await this.service.createRequest(`/knowledgebases/${kbId}`, 'PATCH', replaceKb)
     const text = await response.text()
-    try {
-      return JSON.parse(text)
-    } catch {
-      return text
+    const status = JSON.parse(text)
+    if (status.error) {
+      throw (new exception(retCode.errorCode.LUIS_API_CALL_FAILED, text))
     }
+
+    return status
   }
 
   public async replaceKB(kbId: string, replaceKb: any) {
     const response = await this.service.createRequest(`/knowledgebases/${kbId}`, 'PUT', replaceKb)
     const text = await response.text()
-    try {
-      return JSON.parse(text)
-    } catch {
-      return text
+    if (text) {
+      throw (new exception(retCode.errorCode.LUIS_API_CALL_FAILED, text))
     }
   }
 
   public async publishKB(kbId: string) {
     const response = await this.service.createRequest(`/knowledgebases/${kbId}`, 'POST')
     const text = await response.text()
-    try {
-      return JSON.parse(text)
-    } catch {
-      return text
+    if (text) {
+      throw (new exception(retCode.errorCode.LUIS_API_CALL_FAILED, text))
     }
   }
 
   public async replaceAlt(altJson: any) {
     const response = await this.service.createRequest('/alterations', 'PUT', altJson)
     const text = await response.text()
-    try {
-      return JSON.parse(text)
-    } catch {
-      return text
+    if (text) {
+      throw (new exception(retCode.errorCode.LUIS_API_CALL_FAILED, text))
     }
   }
 
   public async getEndpointKeys() {
     const response = await this.service.createRequest('/endpointkeys', 'GET')
     const text = await response.text()
-    try {
-      return JSON.parse(text)
-    } catch {
-      return text
+    const endpointKeys = JSON.parse(text)
+    if (endpointKeys.error) {
+      throw (new exception(retCode.errorCode.LUIS_API_CALL_FAILED, text))
     }
+
+    return endpointKeys
   }
 
   public generateDeclarativeAssets(recognizers: Array<Recognizer>, multiRecognizer: MultiLanguageRecognizer, settings: Settings)
@@ -148,7 +151,7 @@ export class QnaBuildCore {
         answer: qna.answer,
         source: qna.source,
         questions: qna.questions.slice(),
-        metadata: qna.metadata,
+        metadata: qna.metadata.slice(),
         context: qna.context
       }
     })
@@ -190,7 +193,7 @@ export class QnaBuildCore {
       fileContent += NEWLINE
       if (qnaItem.metadata && qnaItem.metadata.length > 0) {
         fileContent += '**Filters:**' + NEWLINE
-        qnaItem.metadata.forEach((filter: any) => {
+        qnaItem.metadata.sort((a: any, b: any) => (a.name > b.name) ? 1 : -1).forEach((filter: any) => {
           fileContent += '- ' + filter.name + ' = ' + filter.value + NEWLINE
         })
         fileContent += NEWLINE

packages/lu/src/parser/qnabuild/core.ts

@@ -220,6 +220,19 @@ describe('luis:convert negative tests', () => {
             })
 
     })
+
+    it('luis:convert should show ERR message when prebuilt entity in pattern is not explicitly defined', (done) => {
+        loadLuFile('./../../fixtures/testcases/bad4.lu')    
+            .then(res => {
+                LuisBuilder.fromLUAsync(res)
+                .then(res => done(res))
+                .catch(err => {
+                    assert.isTrue(err.text.includes(`[ERROR] line 2:0 - line 2:27: Pattern "- hi {@personName:userName}" has prebuilt entity personName. Please define it explicitly with @ prebuilt personName.`))
+                    done()
+                })
+            })
+        
+    })
   })
 
 describe('luis:convert new entity format', () => {

packages/lu/test/commands/luis/convert.test.js

@@ -0,0 +1,2 @@
+# greeting
+- hi {@personName:userName}

packages/lu/test/fixtures/testcases/bad4.lu

@@ -175,18 +175,18 @@
       ]
     },
     {
-      "text": "you can call me vishwac",
+      "text": "you  can call me vishwac",
       "intent": "AskForUserName",
       "entities": [
         {
           "entity": "userName",
-          "startPos": 16,
-          "endPos": 22
+          "startPos": 17,
+          "endPos": 23
         }
       ]
     },
     {
-      "text": "create an alarm",
+      "text": "create  an alarm",
       "intent": "CreateAlarm",
       "entities": []
     },