Merge branch 'master' into axsuarez/luis-gen

Author: munozemilio

packages/chatdown/README.md

@@ -27,7 +27,7 @@ OPTIONS
 
   -h, --help                   Chatdown command help
 
-  -o, --out=out  Path to the directory where the output of the multiple chat file processing (-f) will be
+  -o, --out=out  Path to the directory where the output of the multiple chat file processing (-o) will be
                                placed.
 
   -p, --prefix                 Prefix stdout with package name.

packages/chatdown/src/commands/chatdown.ts

@@ -37,12 +37,11 @@ export default class Chatdown extends Command {
         })
       }
 
+      let outputDir = flags.out ? path.resolve(flags.out) : null
+
       if (inputIsDirectory) {
         let inputDir = flags.in ? flags.in.trim() : ''
-        let outputDir = (flags.out) ? flags.out.trim() : './'
-        if (outputDir.substr(0, 2) === './') {
-          outputDir = path.resolve(process.cwd(), outputDir.substr(2))
-        }
+
         const len = await this.processFiles(inputDir, outputDir)
         if (len === 0) {
           throw new CLIError('No chat files found at: ' + flags.in)
@@ -51,9 +50,10 @@ export default class Chatdown extends Command {
         return
       } else {
         const fileContents = await this.getInput(flags.in)
+        const fileName = flags.in ? this.getFileName(flags.in) : ''
         if (fileContents) {
           const activities = await chatdown(fileContents, flags)
-          const writeConfirmation = await this.writeOut(activities)
+          const writeConfirmation = await this.writeOut(activities, fileName, outputDir)
           /* tslint:disable:strict-type-predicates */
           if (typeof writeConfirmation === 'string') {
             process.stdout.write(`${chalk.green('Successfully wrote file:')}  ${writeConfirmation}\n`)
@@ -91,21 +91,20 @@ export default class Chatdown extends Command {
     }
   }
 
+  private getFileName(file: any) {
+    let fileName = path.basename(file, path.extname(file))
+    return fileName
+  }
+
   private async processFiles(inputDir: any, outputDir: any) {
     return new Promise(async (resolve, reject) => {
       let files = glob.sync(inputDir, {ignore: ['**/node_modules/**']})
       /* tslint:disable:prefer-for-of */
       for (let i = 0; i < files.length; i++) {
         try {
-          let fileName = files[i]
-          if (files[i].lastIndexOf('/') !== -1) {
-            fileName = files[i].substr(files[i].lastIndexOf('/'))
-          }
-          fileName = fileName.split('.')[0]
+          const fileName = this.getFileName(files[i])
           let activities = await chatdown(await utils.readTextFile(files[i]))
-          let writeFile = `${outputDir}/${fileName}.transcript`
-          await fs.ensureFile(writeFile)
-          await fs.writeJson(writeFile, activities, {spaces: 2})
+          await this.writeOut(activities, fileName, outputDir)
         } catch (e) {
           if (e.message.match(/no such file or directory/)) {
             reject(new CLIError(e.message))
@@ -119,7 +118,13 @@ export default class Chatdown extends Command {
     })
   }
 
-  private async writeOut(activities: any) {
+  private async writeOut(activities: any, fileName: string, outputDir: any) {
+    if (fileName && outputDir) {
+      let writeFile = path.join(outputDir, `${fileName}.transcript`)
+      await fs.ensureFile(writeFile)
+      await fs.writeJson(writeFile, activities, {spaces: 2})
+      return writeFile
+    }
     const output = JSON.stringify(activities, null, 2)
     await new Promise(done => process.stdout.write(output, 'utf-8', () => done()))
     return true

packages/chatdown/test/commands/chatdown.test.ts

@@ -44,7 +44,7 @@ describe('chatdown', () => {
     });
 
     it('should read from file when chat file is passed as an argument', done => {
-        cp.exec(`node ./bin/run chatdown --in ${path.join(__dirname, '../utils/cli.sample.chat')}`, (error, stdout) => {
+        cp.exec(`node ./bin/run chatdown --in ./test/utils/cli.sample.chat`, (error, stdout) => {
             assert.doesNotThrow(() => JSON.parse(stdout));
             done();
         });

packages/cli/package.json

@@ -9,7 +9,7 @@
   "dependencies": {
     "@microsoft/bf-chatdown": "1.0.0",
     "@microsoft/bf-cli-config": "1.0.0",
-    "@microsoft/bf-luis": "1.0.0",
+    "@microsoft/bf-lu": "1.0.0",
     "@microsoft/bf-qnamaker": "1.0.0",
     "@oclif/command": "~1.5.13",
     "@oclif/config": "~1.13.0",
@@ -64,7 +64,7 @@
       "@microsoft/bf-chatdown",
       "@microsoft/bf-cli-config",
       "@microsoft/bf-qnamaker",
-      "@microsoft/bf-luis"
+      "@microsoft/bf-lu"
     ],
     "hooks": {
       "init": "./lib/hooks/init/inithook"

packages/luis/package.json

@@ -1,9 +1,9 @@
 {
-  "name": "@microsoft/bf-luis",
+  "name": "@microsoft/bf-lu",
   "version": "1.0.0",
   "author": "Microsoft",
   "bugs": "https://github.com/microsoft/botframework-cli/issues",
-  "main": "src/parser/index.js",
+  "main": "lib/parser/index.js",
   "dependencies": {
     "@microsoft/bf-cli-command": "1.0.0",
     "@oclif/config": "~1.13.3",

packages/luis/src/commands/luis/translate.ts

@@ -18,8 +18,8 @@ export default class LuisTranslate extends Command {
     srclang: flags.string({description: 'Source lang code. Auto detect if missing.'}),
     tgtlang: flags.string({description: 'Comma separated list of target languages.', required: true}),
     translatekey: flags.string({description: 'Machine translation endpoint key.', required: true}),
-    translate_comments: flags.string({description: 'When set, machine translate comments found in .lu file'}),
-    translate_link_text: flags.string({description: 'When set, machine translate link description in .lu file'}),
+    translate_comments: flags.boolean({description: 'When set, machine translate comments found in .lu file'}),
+    translate_link_text: flags.boolean({description: 'When set, machine translate link description in .lu file'}),
   }
 
   /* tslint:disable:forin no-for-in*/

packages/luis/src/parser/converters/lumerger.js

@@ -35,7 +35,7 @@ const buildLuJsonObject = async function(luObjArray, log, luis_culture, luSearch
             filesToParse.splice(0,1)
             continue
         }
-        parsedContent = await parseLuFile(luOb.content, log, luis_culture)
+        let parsedContent = await parseLuFile(luOb.content, log, luis_culture)
         parsedFiles.push(luOb.id)
 
         if (haveLUISContent(parsedContent.LUISJsonStructure)

packages/luis/src/parser/converters/lutoluisconverter.js

@@ -258,6 +258,9 @@ const mergeResults_closedlists = function (blob, finalCollection, type) {
 const checkAndUpdateVersion = function(finalLUISJSON) {
     // Detect if there is content specific to 5.0.0 schema
     // any entity with children
+    if (!finalLUISJSON) {
+        return
+    }
     let v5DefFound = false;
     v5DefFound = (finalLUISJSON.entities || []).find(i => i.children || i.features) ||
                  (finalLUISJSON.intents || []).find(i => i.features) || 

packages/luis/src/parser/lufile/translate-helpers.js

@@ -49,7 +49,8 @@ const translateHelpers = {
                     continue;
                 }
                 if(translate_comments) {
-                    addSegment(linesToTranslate, currentLine, true);
+                    addSegment(linesToTranslate, currentLine.charAt(0), false);
+                    addSegment(linesToTranslate, currentLine.substring(1), true);
                 } else {
                     addSegment(linesToTranslate, currentLine, false);
                 }
@@ -92,77 +93,34 @@ const translateHelpers = {
                 let listSeparator = '';
                 let content = '';
                 switch (currentSectionType) {
-                case PARSERCONSTS.INTENT: {
-                    // strip line of the list separator
-                    listSeparator = currentLine.charAt(0);
-                    addSegment(linesToTranslate, listSeparator + ' ', false);
-                    content = currentLine.slice(1).trim();
-                    let entitiesList = [];
-                    // strip line off labelled entity values,mark pattern any entities as not to localize
-                    if (content.includes('{')) {
-                        const entityRegex = new RegExp(/\{(.*?)\}/g);
-                        let entitiesFound = content.match(entityRegex);
-                        let eStartIndex = -1;
-                        let eEndIndex = -1;
-                        let entity;
-                        for (var entityIdx in entitiesFound) {
-                            entity = entitiesFound[entityIdx];
-                            let lEntity = entity.replace('{', '').replace('}', '');
-                            let labelledValue = '';
-                            let updatedUtteranceLeft = content.substring(0, content.indexOf(entity));
-                            let updatedUtteranceRight = content.substring(content.indexOf(entity) + entity.length);
-                            // is this a labelled value? 
-                            if (lEntity.includes('=')) {
-                                let entitySplit = lEntity.split('=');
-                                if (entitySplit.length > 2) {
-                                    throw (new exception(retCode.errorCode.INVALID_INPUT, '[ERROR]: Nested entity references are not supported in utterance: ' + content));
-                                }
-                                lEntity = entitySplit[0].trim();
-                                labelledValue = entitySplit[1].trim();
-                                eStartIndex = content.indexOf(entity);
-                                eEndIndex = eStartIndex + labelledValue.length - 1;
-                                content = updatedUtteranceLeft + labelledValue + updatedUtteranceRight;
-                                entitiesList.push(new helperClasses.entity(lEntity, labelledValue, eStartIndex, eEndIndex));
-                            } else {
-                                // This is a pattern entity without a labelled value. Do not localize this.
-                                eStartIndex = content.indexOf(lEntity) - 1;
-                                eEndIndex = eStartIndex + lEntity.length - 1;
-                                content = updatedUtteranceLeft + lEntity + updatedUtteranceRight;
-                                entitiesList.push(new helperClasses.entity(lEntity, null, eStartIndex, eEndIndex));
-                            }
+                case PARSERCONSTS.INTENT: 
+                listSeparator = currentLine.charAt(0);
+                addSegment(linesToTranslate, listSeparator + ' ', false);
+                content = currentLine.slice(1).trim();
+                let skipChars = ['{', '}', '(', ')', '[', ']', '|', '='] 
+                for (let i = 0; i < content.length; i++) {
+                    let processedText = ''
+                    let tslt = false
+                    if (!skipChars.includes(content.charAt(i))) {  
+                        for (let j = i; j < content.length && !skipChars.includes(content.charAt(j)); j++) {
+                            processedText += content.charAt(j)
+                        } 
+                        tslt = true  
+                    } else if (content.charAt(i) == '{') {
+                        for (let j = i; j < content.length && (content.charAt(j) !== '=' && content.charAt(j) !== '}'); j++) {
+                            processedText += content.charAt(j)
                         }
+                    } else {
+                        processedText += content.charAt(i)
                     }
-                    let offset = 0;
-                    let candidateText = '';
-                    // Tokenize the input utterance.
-                    for (var idx in entitiesList) {
-                        let entity = entitiesList[idx];
-                        if (entity.start < 0) entity.start = 0;
-                        if (entity.start !== offset) {
-                            candidateText = content.substring(offset, entity.start);
-                            if (candidateText.trim() !== '') {
-                                addSegment(linesToTranslate, candidateText, true);
-                            } else {
-                                addSegment(linesToTranslate, candidateText, false);
-                            }
-                        }
-                        if (entity.value !== '') {
-                            addSegment(linesToTranslate, ' {' + entity.entity + '=', false);
-                            addSegment(linesToTranslate, content.substring(entity.start, entity.end + 1).trim(), true);
-                            addSegment(linesToTranslate, '} ', false);
-                        } else {
-                            addSegment(linesToTranslate, ' {' + entity.entity + '} ', false);
-                        }
-                        offset = entity.end + 1;
-                    }
-                    if (offset !== content.length) {
-                        candidateText = content.substring(offset);
-                        if (candidateText.trim() !== '') {
-                            addSegment(linesToTranslate, candidateText.trim(), true);
-                        } else {
-                            addSegment(linesToTranslate, candidateText, false);
-                        }
+
+                    if (processedText.charAt(0) === ' ') {
+                        addSegment(linesToTranslate, ' ', false)
                     }
+                    
+                    addSegment(linesToTranslate, processedText, tslt)
+                    content = content.slice(processedText.length)
+                    i--
                 }
                 break;
                 case PARSERCONSTS.NEWENTITY:
@@ -204,7 +162,6 @@ const translateHelpers = {
                     addSegment(linesToTranslate, '$', false);
                     addSegment(linesToTranslate, entityName.trim(), true);
                     addSegment(linesToTranslate, ' : ' + PARSERCONSTS.QNAALTERATIONS + ' = ', false);
-                    c
                 } else {
                     // we would not localize entity line but remember we are under entity section for list entities
                     // FIX for BF CLI # 121
@@ -219,7 +176,7 @@ const translateHelpers = {
                 }
             } else if(currentLine.indexOf(PARSERCONSTS.ANSWER) === 0) {
                 if (inAnswer) {
-                    answerData = '';
+                    let answerData = '';
                 }
                 addSegment(linesToTranslate, currentLine, false);
                 inAnswer = !inAnswer;

packages/luis/src/parser/translator/lutranslate.js

@@ -30,36 +30,14 @@ module.exports = {
     }
 }
 
-/**
- * Helper function to parse, translate and write out localized lu files
- * @param {string} file file name
- * @param {string} translate_key translate text API key
- * @param {string} to_lang language code to translate content to
- * @param {string} src_lang language code for source content
- * @param {boolean} translate_comments translate comments in .lu files if this is set to true
- * @param {boolean} translate_link_text translate URL or LU reference link text in .lu files if this is set to true
- * @returns {void} nothing
- * @throws {exception} Throws on errors. exception object includes errCode and text. 
- */
-async function parseFile(file) {
-    if(!fs.existsSync(path.resolve(file))) {
-        throw(new exception(retCode.errorCode.FILE_OPEN_ERROR, 'Sorry unable to open [' + file + ']'));
-    }
-    let fileContent = txtfile.readSync(file);
-    if (!fileContent) {
-        throw(new exception(retCode.errorCode.FILE_OPEN_ERROR, 'Sorry, error reading file:' + file));
-    }
-    return fileContent
-}
-
 async function translateLuObject(luObject, translate_key, to_lang, src_lang, translate_comments, translate_link_text) {
     let parsedLocContent = ''
     let result = {}
     // Support multi-language specification for targets.
     // Accepted formats are space or comma separated list of target language codes.
     // Tokenize to_lang
     let toLang = to_lang.split(/[, ]/g)
-    for (idx in toLang) {
+    for (let idx in toLang) {
         let tgt_lang = toLang[idx].trim();
         if (tgt_lang === '') continue;
         try {