Optimize empty file validation and import resolver corner cases in crossTrianer (#1038)

* optimize empty file validation in crossTrianer

* adjust some corner case

Author: feich-ms

packages/lu/src/parser/cross-train/crossTrainer.js

@@ -293,7 +293,8 @@ const mergeInterruptionIntent = function (fromUtterances, toResource, intentName
 
       // add section here
       // not add the interruption intent if original file is empty
-      if (toResource.content.Content !== '') {
+      // here empty means there are no model related sections exception information section
+      if (toResource.content.Sections.filter(s => s.SectionType !== LUSectionTypes.MODELINFOSECTION).length > 0) {
         toResource.content = new SectionOperator(toResource.content).addSection(newFileContent)
       }
     }
@@ -511,12 +512,15 @@ const parseAndValidateContent = async function (objectArray, verbose, importReso
   let allEmpty = true
   for (const object of objectArray) {    
     let fileContent = object.content
+    let objectId = object.id
     if (object.content && object.content !== '') {
       if (fileExt === fileExtEnum.LUFile) {
+        if (!object.id.endsWith(fileExtEnum.LUFile)) object.id += fileExtEnum.LUFile
         let result = await LuisBuilderVerbose.build([object], verbose, undefined, importResolver)
         let luisObj = new Luis(result)
         fileContent = luisObj.parseToLuContent()
       } else {
+        if (!object.id.endsWith(fileExtEnum.QnAFile)) object.id += fileExtEnum.QnAFile
         let result = await qnaBuilderVerbose.build([object], verbose, importResolver)
         fileContent = result.parseToQnAContent()
       }
@@ -540,16 +544,16 @@ const parseAndValidateContent = async function (objectArray, verbose, importReso
       }
     }
 
-    fileIdToResourceMap.set(object.id, resource)
+    fileIdToResourceMap.set(objectId, resource)
   }
 
   return {fileIdToResourceMap, allEmpty}
 }
 
 const pretreatment = function (luContents, qnaContents) {
    // Parse lu and qna objects
-   let luObjectArray = fileHelper.getParsedObjects(luContents)
-   let qnaObjectArray = fileHelper.getParsedObjects(qnaContents)
+   let luObjectArray = fileHelper.getParsedObjects(luContents, fileExtEnum.LUFile)
+   let qnaObjectArray = fileHelper.getParsedObjects(qnaContents, fileExtEnum.QnAFile)
 
    return {luObjectArray, qnaObjectArray}
 }

packages/lu/src/utils/filehelper.ts

@@ -5,12 +5,15 @@
 
 import {readTextFile} from './textfilereader'
 import Lu = require('../parser/lu/lu')
+import QnA = require('../parser/lu/qna')
 const exception = require('./../parser/utils/exception')
 const retCode = require('./../parser/utils/enums/CLI-errors')
 const fs = require('fs-extra')
 const path = require('path')
 const helpers = require('./../parser/utils/helpers')
+const fileExtEnum = require('./../parser/utils/helpers').FileExtTypeEnum
 const LUOptions = require('./../parser/lu/luOptions')
+const QnAOptions = require('./../parser/lu/qnaOptions')
 const luParser = require('./../parser/lufile/luParser')
 const LUSectionTypes = require('./../parser/utils/enums/lusectiontypes')
 const globby = require('globby')
@@ -211,10 +214,15 @@ async function getConfigFile(input: string): Promise<string> {
   return defaultConfigFile
 }
 
-export function getParsedObjects(contents: {id: string, content: string}[]) {
+export function getParsedObjects(contents: {id: string, content: string}[], extType: string) {
   const parsedObjects = contents.map(content => {
-    const opts = new LUOptions(content.id)
-    return new Lu(content.content, opts)
+    if (extType === fileExtEnum.LUFile) {
+      const opts = new LUOptions(content.id)
+      return new Lu(content.content, opts)
+    } else {
+      const opts = new QnAOptions(content.id)
+      return new QnA(content.content, opts)
+    }
   })
 
   return parsedObjects

packages/lu/test/parser/cross-train/crossTrainer.test.js

@@ -944,4 +944,52 @@ describe('luis:cross training tests among lu and qna contents', () => {
     assert.equal(qnaResult.get(path.join(rootDir, "dia1.en-us")).Sections.filter(s => s.SectionType === sectionTypes.QNASECTION).length, 3)
     assert.isTrue(qnaResult.get(path.join(rootDir, "dia1.en-us")).Sections.filter(s => s.SectionType === sectionTypes.QNASECTION)[1].Body.includes(`# ? common_question${NEWLINE}${NEWLINE}**Filters:**${NEWLINE}- dialogName=dia1${NEWLINE}${NEWLINE}\`\`\`${NEWLINE}this is common answer${NEWLINE}\`\`\`${NEWLINE}${NEWLINE}`))
   })
+
+  it('luis:cross training can get expected result when a lu file is section emtpy', async () => {
+    let luContentArray = []
+    let qnaContentArray = []
+
+    luContentArray.push({
+      content: `> this is comments
+      # dia1_trigger
+      - trigger dial1`,
+      id: 'main'})
+
+    qnaContentArray.push({
+      content:
+        `# ?user guide
+
+        **Filters:**
+        - aa=bb
+        
+        \`\`\`
+            Here is the [user guide](http://contoso.com/userguide.pdf)
+        \`\`\`
+        
+        # ?tell joke
+        \`\`\`
+            tell a funny joke
+        \`\`\``,
+      id: 'main'}
+    )
+
+    luContentArray.push({
+      content: `> !# @app.name = my luis application`,
+      id: 'dia1'}
+    )
+
+    let crossTrainConfig = {
+      'main': {
+        'rootDialog': true,
+        'triggers': {
+          'dia1_trigger': 'dia1'
+        }
+      }
+    }
+
+    const trainedResult = await crossTrainer.crossTrain(luContentArray, qnaContentArray, crossTrainConfig)
+    const luResult = trainedResult.luResult
+
+    assert.equal(luResult.get('dia1').Sections.filter(s => s.SectionType !== sectionTypes.MODELINFOSECTION).length, 0)
+  })
 })