Merge pull request #198 from microsoft/vishwac/lu-qna-cleanup

Separate lu content to .lu file and qna content to .qna file

Author: munozemilio

packages/luis/src/commands/luis/convert.ts

@@ -4,6 +4,7 @@ const fs = require('fs-extra')
 const file = require('./../../utils/filehelper')
 const luConverter = require('./../../parser/converters/lutoluisconverter')
 const luisConverter = require('./../../parser/converters/luistoluconverter')
+const fileExtEnum = require('./../../parser/lufile/helpers').FileExtTypeEnum
 
 export default class LuisConvert extends Command {
   static description = 'Convert .lu file(s) to a LUIS application JSON model or vice versa'
@@ -34,7 +35,7 @@ export default class LuisConvert extends Command {
       // Parse the object depending on the input
       let result: any
       if (isLu) {
-        const luFiles = await file.getLuObjects(stdin, flags.in, flags.recurse)
+        const luFiles = await file.getLuObjects(stdin, flags.in, flags.recurse, fileExtEnum.LUFile)
         result = await luConverter.parseLuToLuis(luFiles, flags.log, flags.culture)
       } else {
         const luisFile = stdin ? stdin : await file.getContentFromFile(flags.in)

packages/luis/src/commands/luis/translate.ts

@@ -6,6 +6,7 @@ const exception = require('./../../parser/lufile/classes/exception')
 const luTranslator = require('./../../parser/translator/lutranslate')
 const luisConverter = require('./../../parser/converters/luistoluconverter')
 const luConverter = require('./../../parser/lufile/parseFileContents')
+const fileExtEnum = require('./../../parser/lufile/helpers').FileExtTypeEnum
 
 export default class LuisTranslate extends Command {
   static description = ' Translate given LUIS application JSON model or lu file(s)'
@@ -17,8 +18,8 @@ export default class LuisTranslate extends Command {
     srclang: flags.string({description: 'Source lang code. Auto detect if missing.'}),
     tgtlang: flags.string({description: 'Comma separated list of target languages.', required: true}),
     translatekey: flags.string({description: 'Machine translation endpoint key.', required: true}),
-    translate_comments: flags.string({description: 'When set, machine translate comments found in .lu or .qna file'}),
-    translate_link_text: flags.string({description: 'When set, machine translate link description in .lu or .qna file'}),
+    translate_comments: flags.string({description: 'When set, machine translate comments found in .lu file'}),
+    translate_link_text: flags.string({description: 'When set, machine translate link description in .lu file'}),
   }
 
   /* tslint:disable:forin no-for-in*/
@@ -36,7 +37,7 @@ export default class LuisTranslate extends Command {
       let isLu = await fileHelper.detectLuContent(stdin, flags.in)
       let result: any
       if (isLu) {
-        let luFiles = await fileHelper.getLuObjects(stdin, flags.in, flags.recurse)
+        let luFiles = await fileHelper.getLuObjects(stdin, flags.in, flags.recurse, fileExtEnum.LUFile)
         result = await luTranslator.translateLuList(luFiles, flags.translatekey, flags.tgtlang, flags.srclang, flags.translate_comments, flags.translate_link_text)
       } else {
         let json = stdin ? stdin : await fileHelper.getContentFromFile(flags.in)

packages/luis/src/commands/qnamaker/convert.ts

@@ -4,6 +4,7 @@ const fs = require('fs-extra')
 const file = require('./../../utils/filehelper')
 const luConverter = require('./../../parser/converters/qnatoqnajsonconverter')
 const qnaConverter = require('./../../parser/converters/qnajsontoqnaconverter')
+const fileExtEnum = require('./../../parser/lufile/helpers').FileExtTypeEnum
 
 export default class QnamakerConvert extends Command {
   static description = 'Convert .lu file(s) to a QnA application JSON model or vice versa'
@@ -12,7 +13,7 @@ export default class QnamakerConvert extends Command {
     in: flags.string({description: 'Source .qna file(s) or QnA KB JSON file', required: true}),
     alterations: flags.boolean({description: 'Indicates if files is QnA Alterations'}),
     log: flags.boolean({description: 'Enables log messages', default: false}),
-    sort: flags.boolean({description: 'When set, questions collections are alphabetically sorted are alphabetically sorted in .lu files', default: false}),
+    sort: flags.boolean({description: 'When set, questions collections are alphabetically sorted are alphabetically sorted in .qna files', default: false}),
     recurse: flags.boolean({description: 'Indicates if sub-folders need to be considered to file .qna file(s)'}),
     out: flags.string({description: 'Output file or folder name. If not specified stdout will be used as output'}),
     name: flags.string({description: 'Name of the QnA KB'}),
@@ -32,7 +33,7 @@ export default class QnamakerConvert extends Command {
       // Parse the object depending on the input
       let result: any
       if (isQnA) {
-        const luFiles = await file.getLuObjects(stdin, flags.in, flags.recurse)
+        let luFiles = await file.getLuObjects(stdin, flags.in, flags.recurse, fileExtEnum.QnAFile)
         result = await luConverter.parseQnaToJson(luFiles, false, flags.luis_culture)
       } else {
         const qnAFile = stdin ? stdin : await file.getContentFromFile(flags.in)
@@ -69,12 +70,12 @@ export default class QnamakerConvert extends Command {
   }
 
   private async writeOutput(convertedObject: any, flags: any, isQnA: boolean) {
-    let filePath = await file.generateNewFilePath(flags.out, flags.in, isQnA)
+    let filePath = await file.generateNewFilePath(flags.out, flags.in, isQnA, '', fileExtEnum.QnAFile)
     try {
       if (isQnA) {
         await fs.writeFile(filePath, JSON.stringify(convertedObject.finalQnAJSON, null, 2), 'utf-8')
         if (convertedObject.finalQnAAlterations) {
-          let filePathAlterations = await file.generateNewFilePath(flags.out, flags.in, isQnA, 'alterations_')
+          let filePathAlterations = await file.generateNewFilePath(flags.out, flags.in, isQnA, 'alterations_', fileExtEnum.QnAFile)
           await fs.writeFile(filePathAlterations, JSON.stringify(convertedObject.finalQnAAlterations, null, 2), 'utf-8')
         }
       } else {

packages/luis/src/commands/qnamaker/translate.ts

@@ -6,19 +6,20 @@ const exception = require('./../../parser/lufile/classes/exception')
 const luTranslator = require('./../../parser/translator/lutranslate')
 const qnaConverter = require('./../../parser/converters/qnajsontoqnaconverter')
 const luConverter = require('./../../parser/lufile/parseFileContents')
+const fileExtEnum = require('./../../parser/lufile/helpers').FileExtTypeEnum
 
 export default class QnamakerTranslate extends Command {
-  static description = 'Translate given LUIS application JSON model or lu file(s)'
+  static description = 'Translate given QnA maker application JSON model or qna file(s)'
 
   static flags: flags.Input<any> = {
-    in: flags.string({description: 'Source .lu file(s) or LUIS application JSON model', required: true}),
-    recurse: flags.boolean({description: 'Indicates if sub-folders need to be considered to file .lu file(s)'}),
+    in: flags.string({description: 'Source .qna file(s) or QnA maker application JSON model', required: true}),
+    recurse: flags.boolean({description: 'Indicates if sub-folders need to be considered to find .qna file(s)'}),
     out: flags.string({description: 'Output folder name. If not specified stdout will be used as output'}),
     srclang: flags.string({description: 'Source lang code. Auto detect if missing.'}),
     tgtlang: flags.string({description: 'Comma separated list of target languages.', required: true}),
     translatekey: flags.string({description: 'Machine translation endpoint key.', required: true}),
-    translate_comments: flags.string({description: 'When set, machine translate comments found in .lu or .qna file'}),
-    translate_link_text: flags.string({description: 'When set, machine translate link description in .lu or .qna file'}),
+    translate_comments: flags.string({description: 'When set, machine translate comments found in .qna file'}),
+    translate_link_text: flags.string({description: 'When set, machine translate link description in .qna file'}),
   }
 
   /* tslint:disable:forin no-for-in*/
@@ -36,7 +37,7 @@ export default class QnamakerTranslate extends Command {
       let isLu = await fileHelper.detectLuContent(stdin, flags.in)
       let result: any
       if (isLu) {
-        let luFiles = await fileHelper.getLuObjects(stdin, flags.in, flags.recurse)
+        let luFiles = await fileHelper.getLuObjects(stdin, flags.in, flags.recurse, fileExtEnum.QnAFile)
         result = await luTranslator.translateLuList(luFiles, flags.translatekey, flags.tgtlang, flags.srclang, flags.translate_comments, flags.translate_link_text)
       } else {
         let json = stdin ? stdin : await fileHelper.getContentFromFile(flags.in)

packages/luis/src/parser/lufile/helpers.js

@@ -20,23 +20,29 @@ const helpers = {
     sanitizeNewLines(fileContent) {
         return fileContent.replace(ANY_NEWLINE, NEWLINE);
     },
-
+    /**
+     * Enumeration of supported file extension types
+     */
+    FileExtTypeEnum: {
+        LUFile : '.lu',
+        QnAFile : '.qna'
+    },
     /**
      * Helper function to recursively get all .lu files
      * @param {string} inputfolder input folder name
      * @param {boolean} getSubFolder indicates if we should recursively look in sub-folders as well
+     * @param {FileExtTypeEnum} extType indicates if we should look for LUFile or QnAFile
      * @returns {Array} Array of .lu files found
     */
-   findLUFiles: function(inputFolder, getSubFolders) {
+   findLUFiles: function(inputFolder, getSubFolders, extType = this.FileExtTypeEnum.LUFile) {
         let results = [];
-        const luExt = '.lu';
         fs.readdirSync(inputFolder).forEach(function(dirContent) {
             dirContent = path.resolve(inputFolder,dirContent);
             if(getSubFolders && fs.statSync(dirContent).isDirectory()) {
                 results = results.concat(helpers.findLUFiles(dirContent, getSubFolders));
             }
             if(fs.statSync(dirContent).isFile()) {
-                if(dirContent.endsWith(luExt)) {
+                if(dirContent.endsWith(extType)) {
                     results.push(dirContent);
                 }
             }

packages/luis/src/utils/filehelper.ts

@@ -3,14 +3,15 @@ const fs = require('fs-extra')
 const path = require('path')
 const helpers = require('./../parser/lufile/helpers')
 const luObject = require('./../parser/lufile/classes/luObject')
+
 /* tslint:disable:prefer-for-of no-unused*/
 
-export async function getLuObjects(stdin: string, input: string | undefined, recurse = false) {
+export async function getLuObjects(stdin: string, input: string | undefined, recurse = false, extType: string | undefined) {
   let luObjects: any = []
   if (stdin) {
     luObjects.push(new luObject('stdin', stdin))
   } else {
-    let luFiles = await getLuFiles(input, recurse)
+    let luFiles = await getLuFiles(input, recurse, extType)
     for (let i = 0; i < luFiles.length; i++) {
       let luContent = await getContentFromFile(luFiles[i])
       luObjects.push(new luObject(path.resolve(luFiles[i]), luContent))
@@ -20,7 +21,7 @@ export async function getLuObjects(stdin: string, input: string | undefined, rec
   return luObjects
 }
 
-async function getLuFiles(input: string | undefined, recurse = false): Promise<Array<any>> {
+async function getLuFiles(input: string | undefined, recurse = false, extType: string | undefined): Promise<Array<any>> {
   let filesToParse = []
   let fileStat = await fs.stat(input)
   if (fileStat.isFile()) {
@@ -32,10 +33,10 @@ async function getLuFiles(input: string | undefined, recurse = false): Promise<A
     throw new CLIError('Sorry, ' + input + ' is not a folder or does not exist')
   }
 
-  filesToParse = helpers.findLUFiles(input, recurse)
+  filesToParse = helpers.findLUFiles(input, recurse, extType)
 
   if (filesToParse.length === 0) {
-    throw new CLIError('Sorry, no .lu files found in the specified folder.')
+    throw new CLIError(`Sorry, no ${extType} files found in the specified folder.`)
   }
   return filesToParse
 }
@@ -57,7 +58,7 @@ export async function getContentFromFile(file: string) {
   return fileContent
 }
 
-export async function generateNewFilePath(outFileName: string, inputfile: string, isLu: boolean, prefix = ''): Promise<string> {
+export async function generateNewFilePath(outFileName: string, inputfile: string, isLu: boolean, prefix = '', extType: string = helpers.FileExtTypeEnum.LUFile): Promise<string> {
   let base = path.resolve(outFileName)
   let extension = path.extname(base)
   if (extension) {
@@ -69,9 +70,9 @@ export async function generateNewFilePath(outFileName: string, inputfile: string
   let name = ''
   let inputStat = await fs.stat(inputfile)
   if (inputStat.isFile()) {
-    name += path.basename(inputfile, path.extname(inputfile)) + (isLu ? '.json' : '.lu')
+    name += path.basename(inputfile, path.extname(inputfile)) + (isLu ? '.json' : extType)
   } else {
-    name += isLu ? 'converted.json' : 'converted.lu'
+    name += isLu ? 'converted.json' : `converted.${extType}`
   }
   return path.join(base, prefix + name)
 }
@@ -133,7 +134,7 @@ export async function detectLuContent(stdin: string, input: string) {
     }
 
     let inputStat = await fs.stat(input)
-    return !inputStat.isFile() ? true : path.extname(input) === '.lu'
+    return !inputStat.isFile() ? true : (path.extname(input) === '.lu' || path.extname(input) === '.qna')
   }
 
   try {

packages/luis/test/commands/qnamaker/convert.test.ts

@@ -46,7 +46,7 @@ describe('qnamaker:convert', () => {
   })
 
   test
-  .command(['qnamaker:convert', '--in', `${path.join(__dirname, './../../fixtures/examples/all.lu')}`, '--out', 'qna.json', '--name', 'all-qna'])
+  .command(['qnamaker:convert', '--in', `${path.join(__dirname, './../../fixtures/examples/all.qna')}`, '--out', 'qna.json', '--name', 'all-qna'])
   .it('qnamaker:convert all concepts of lu file definition is parsed correctly  [QnA]', async () => {
     let parsedObjects = await parseJsonFiles('./../../../qna.json', './../../fixtures/verified/all-qna.json')
     expect(parsedObjects[0]).to.deep.equal(parsedObjects[1])

packages/luis/test/fixtures/examples/all.qna

@@ -0,0 +1,52 @@
+> # QnA Definitions
+> This is a QnA definition. Follows # ? Question: \<list of questions\> \```markdown \<Answer> ``` format
+
+> You can add URLs for QnA maker to ingest using the #URL reference scheme
+### ? How do I change the default message
+```markdown
+You can change the default message if you use the QnAMakerDialog. 
+See [this link](https://docs.botframework.com/en-us/azure-bot-service/templates/qnamaker/#navtitle) for details. 
+```
+
+### ? How do I programmatically update my KB?
+```markdown
+You can use our REST apis to manage your KB. 
+\#1. See here for details: https://westus.dev.cognitive.microsoft.com/docs/services/58994a073d9e04097c7ba6fe/operations/58994a073d9e041ad42d9baa
+```
+
+> You can add URLs for QnA maker to ingest using the #URL reference scheme
+
+[QnA URL - faqs](https://docs.microsoft.com/en-in/azure/cognitive-services/qnamaker/faqs)
+
+
+> You can define multilple questions for single answer as well
+### ? Who is your ceo?
+- get me your ceo info
+```markdown
+Vishwac
+```
+
+> You can define filters for QnA using the \**Filters:** \<list of name=value pairs\> format
+### ? Where can I get coffee? 
+- I need coffee
+
+**Filters:**
+- location = seattle
+```markdown
+You can get coffee in our Seattle store at 1 pike place, Seattle, WA
+```
+
+### ? Where can I get coffee? 
+- I need coffee
+
+**Filters:**
+- location = portland
+```markdown
+You can get coffee in our Portland store at 52 marine drive, Portland, OR
+```
+
+> FAQ URLs for QnA maker to ingest.
+
+[QnA maker reference](https://docs.microsoft.com/en-in/azure/cognitive-services/qnamaker/faqs)
+
+[QnA reference](./qna7.lu)

packages/luis/test/fixtures/testcases/collate/11.lu

@@ -6,7 +6,3 @@
 
 [None intent](./none.lu)
 
-[QnA1](./qna3.lu)
-
-[QnA7](./qna7.lu)
-

packages/luis/test/fixtures/testcases/collate/11.qna

@@ -0,0 +1,4 @@
+[QnA1](./qna3.qna)
+
+[QnA7](./qna7.qna)
+