Skip to content
This repository was archived by the owner on Jan 15, 2025. It is now read-only.

Commit 31299e6

Browse files
munozemiliovishwacsena
munozemilio
authored and
vishwacsena
committed
Luis commands stdin enabling (#185)
* Removing Disk I/O operatios as mandatory for lu transform * Lu Merger refactor * Removing duplicated code from filter * Adding stdin readin to convert commands * Adding object processing ability to translate commands * Changing ts files to js in parser to keep it consistent * Adding Converter to parser index * Adding tests and applying PR comments * Adding test to luis translate command * Adding JSON compare to tests * Adding doc to exposed converter * Reverting to string comparison on qnamaker translate test * Commenting test
1 parent 8f4567f commit 31299e6

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+1813
-1506
lines changed

‎packages/luis/package.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
"version": "1.0.0",
44
"author": "Microsoft",
55
"bugs": "https://github.com/microsoft/botframework-cli/issues",
6-
"main": "parser/index.js",
6+
"main": "src/parser/index.js",
77
"dependencies": {
88
"@microsoft/bf-cli-command": "1.0.0",
99
"@oclif/config": "~1.13.3",
@@ -41,7 +41,7 @@
4141
},
4242
"files": [
4343
"/lib",
44-
"/parser",
44+
"/src/parser",
4545
"/npm-shrinkwrap.json",
4646
"/oclif.manifest.json"
4747
],

‎packages/luis/src/commands/luis/convert.ts‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
import {CLIError, Command, flags} from '@microsoft/bf-cli-command'
22
const exception = require('./../../parser/lufile/classes/exception')
33
const fs = require('fs-extra')
4-
const path = require('path')
54
const file = require('./../../utils/filehelper')
65
const luConverter = require('./../../parser/converters/lutoluisconverter')
76
const luisConverter = require('./../../parser/converters/luistoluconverter')
@@ -10,7 +9,7 @@ export default class LuisConvert extends Command {
109
static description = 'Convert .lu file(s) to a LUIS application JSON model or vice versa'
1110

1211
static flags: flags.Input<any> = {
13-
in: flags.string({description: 'Source .lu file(s) or LUIS application JSON model', required: true}),
12+
in: flags.string({description: 'Source .lu file(s) or LUIS application JSON model'}),
1413
recurse: flags.boolean({description: 'Indicates if sub-folders need to be considered to file .lu file(s)', default: false}),
1514
log: flags.boolean({description: 'Enables log messages', default: false}),
1615
sort: flags.boolean({description: 'When set, intent, utterances, entities are alphabetically sorted in .lu files', default: false}),
@@ -25,18 +24,21 @@ export default class LuisConvert extends Command {
2524
async run() {
2625
try {
2726
const {flags} = this.parse(LuisConvert)
27+
// Check if data piped in stdin
28+
let stdin = await this.readStdin()
29+
2830
//Check if file or folder
2931
//if folder, only lu to luis is supported
30-
let inputStat = await fs.stat(flags.in)
31-
let isLu = !inputStat.isFile() ? true : path.extname(flags.in) === '.lu'
32+
let isLu = await file.detectLuContent(stdin, flags.in)
3233

3334
// Parse the object depending on the input
3435
let result: any
3536
if (isLu) {
36-
const luFiles = await file.getLuObjects(flags.in, flags.recurse)
37+
const luFiles = await file.getLuObjects(stdin, flags.in, flags.recurse)
3738
result = await luConverter.parseLuToLuis(luFiles, flags.log, flags.culture)
3839
} else {
39-
result = await luisConverter.parseLuisFileToLu(flags.in, flags.sort)
40+
const luisFile = stdin ? stdin : await file.getContentFromFile(flags.in)
41+
result = await luisConverter.parseLuisObjectToLu(luisFile, flags.sort, flags.in)
4042
}
4143

4244
// If result is null or undefined return

‎packages/luis/src/commands/luis/generate/cs.ts‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,8 @@ import {CLIError, Command, flags} from '@microsoft/bf-cli-command'
22
import {camelCase, upperFirst} from 'lodash'
33
import * as path from 'path'
44

5-
import {LuisToCsConverter} from '../../../parser/converters/luis-to-cs-converter'
6-
import {Utils} from '../../../utils'
7-
5+
const LuisToCsConverter = require('./../../../parser/converters/luistocsconverter')
6+
const file = require('./../../../utils/filehelper')
87
const fs = require('fs-extra')
98

109
export default class LuisGenerateCs extends Command {
@@ -52,7 +51,7 @@ export default class LuisGenerateCs extends Command {
5251
this.reorderEntities(app, 'patternAnyEntities')
5352
this.reorderEntities(app, 'composites')
5453

55-
const outputPath = Utils.validatePath(flags.out, process.cwd(), flags.className + '.cs', flags.force)
54+
const outputPath = file.validatePath(flags.out, flags.className + '.cs', flags.force)
5655

5756
this.log(
5857
`Generating file at ${outputPath || ''} that contains class ${space}.${flags.className}.`

‎packages/luis/src/commands/luis/generate/ts.ts‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,8 @@ import {CLIError, Command, flags} from '@microsoft/bf-cli-command'
22
import {camelCase, kebabCase, upperFirst} from 'lodash'
33
import * as path from 'path'
44

5-
import {LuisToTsConverter} from '../../../parser/converters/luis-to-ts-converter'
6-
import {Utils} from '../../../utils'
7-
5+
const LuisToTsConverter = require('./../../../parser/converters/luistotsconverter')
6+
const file = require('./../../../utils/filehelper')
87
const fs = require('fs-extra')
98

109
export default class LuisGenerateTs extends Command {
@@ -45,7 +44,7 @@ export default class LuisGenerateTs extends Command {
4544
this.reorderEntities(app, 'patternAnyEntities')
4645
this.reorderEntities(app, 'composites')
4746

48-
const outputPath = Utils.validatePath(flags.out, process.cwd(), kebabCase(flags.className) + '.ts', flags.force)
47+
const outputPath = file.validatePath(flags.out, kebabCase(flags.className) + '.ts', flags.force)
4948

5049
this.log(
5150
`Generating file at ${outputPath || ''} that contains class ${flags.className}.`

‎packages/luis/src/commands/luis/translate.ts‎

Lines changed: 25 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -25,31 +25,41 @@ export default class LuisTranslate extends Command {
2525
async run() {
2626
try {
2727
const {flags} = this.parse(LuisTranslate)
28-
let inputStat = await fs.stat(flags.in)
28+
// Check if data piped in stdin
29+
let stdin = await this.readStdin()
2930
let outputStat = flags.out ? await fs.stat(flags.out) : null
3031

3132
if (outputStat && outputStat.isFile()) {
3233
throw new CLIError('Output can only be writen to a folder')
3334
}
3435

35-
let isLu = !inputStat.isFile() ? true : path.extname(flags.in) === '.lu'
36+
let isLu = await fileHelper.detectLuContent(stdin, flags.in)
3637
let result: any
3738
if (isLu) {
38-
let luFiles = await fileHelper.getLuFiles(flags.in, flags.recurse)
39-
result = await luTranslator.translateLuFile(luFiles, flags.translatekey, flags.tgtlang, flags.srclang, flags.translate_comments, flags.translate_link_text)
39+
let luFiles = await fileHelper.getLuObjects(stdin, flags.in, flags.recurse)
40+
result = await luTranslator.translateLuList(luFiles, flags.translatekey, flags.tgtlang, flags.srclang, flags.translate_comments, flags.translate_link_text)
4041
} else {
41-
let translation = await luisConverter.parseLuisFileToLu(flags.in, false)
42-
translation = await luTranslator.translateLuObj(result, flags.translatekey, flags.tgtlang, flags.srclang, flags.translate_comments, flags.translate_link_text)
43-
result = {}
44-
Object.keys(translation).forEach(async idx => {
45-
result[flags.in][idx] = await luConverter.parseFile(translation[idx][0], false)
46-
})
42+
let json = stdin ? stdin : await fileHelper.getContentFromFile(flags.in)
43+
let translation = await luisConverter.parseLuisObjectToLu(json, false)
44+
translation = await luTranslator.translateLuObj(translation, flags.translatekey, flags.tgtlang, flags.srclang, flags.translate_comments, flags.translate_link_text)
45+
let key = stdin ? 'stdin' : path.basename(flags.in)
46+
result = {
47+
[key] : {}
48+
}
49+
for (let lng in translation) {
50+
let translatedJSON = await luConverter.parseFile(translation[lng], false)
51+
result[key][lng] = await translatedJSON.LUISJsonStructure
52+
}
4753
}
4854

4955
if (flags.out) {
50-
await this.writeOutput(result, flags.out)
56+
await this.writeOutput(result, flags.out, isLu)
5157
} else {
52-
this.log(result)
58+
if (isLu) {
59+
this.log(result)
60+
} else {
61+
this.log(JSON.stringify(result, null, 2))
62+
}
5363
}
5464

5565
} catch (err) {
@@ -60,13 +70,14 @@ export default class LuisTranslate extends Command {
6070
}
6171
}
6272

63-
private async writeOutput(translatedObject: any, out: string) {
73+
private async writeOutput(translatedObject: any, out: string, isLu: boolean) {
6474
let filePath = ''
6575
try {
6676
for (let file in translatedObject) {
6777
for (let lng in translatedObject[file]) {
6878
filePath = await fileHelper.generateNewTranslatedFilePath(file, lng, out)
69-
await fs.writeFile(filePath, translatedObject[path.basename(file)][lng][0], 'utf-8')
79+
let content = isLu ? translatedObject[file][lng] : JSON.stringify(translatedObject[file][lng], null, 2)
80+
await fs.writeFile(filePath, content, 'utf-8')
7081
}
7182
}
7283
} catch (err) {

‎packages/luis/src/commands/qnamaker/convert.ts‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
import {CLIError, Command, flags} from '@microsoft/bf-cli-command'
22
const exception = require('./../../parser/lufile/classes/exception')
33
const fs = require('fs-extra')
4-
const path = require('path')
54
const file = require('./../../utils/filehelper')
65
const luConverter = require('./../../parser/converters/qnatoqnajsonconverter')
76
const qnaConverter = require('./../../parser/converters/qnajsontoqnaconverter')
@@ -23,18 +22,21 @@ export default class QnamakerConvert extends Command {
2322
try {
2423
const {flags} = this.parse(QnamakerConvert)
2524

26-
// Check if file or folder
27-
// If folder, only lu to luis is supported
28-
let inputStat = await fs.stat(flags.in)
29-
let isQnA = !inputStat.isFile() ? true : path.extname(flags.in) === '.lu'
25+
// Check if data piped in stdin
26+
let stdin = await this.readStdin()
27+
28+
//Check if file or folder
29+
//if folder, only lu to luis is supported
30+
let isQnA = await file.detectLuContent(stdin, flags.in)
3031

3132
// Parse the object depending on the input
3233
let result: any
3334
if (isQnA) {
34-
const luFiles = await file.getLuObjects(flags.in, flags.recurse)
35+
const luFiles = await file.getLuObjects(stdin, flags.in, flags.recurse)
3536
result = await luConverter.parseQnaToJson(luFiles, false, flags.luis_culture)
3637
} else {
37-
result = await qnaConverter.parseQnAFileToLu(flags.in, flags.sort, flags.alterations)
38+
const qnAFile = stdin ? stdin : await file.getContentFromFile(flags.in)
39+
result = await qnaConverter.parseQnAObjectToLu(qnAFile, flags.sort, flags.alterations)
3840
}
3941

4042
// If result is null or undefined return

‎packages/luis/src/commands/qnamaker/translate.ts‎

Lines changed: 24 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -25,31 +25,41 @@ export default class QnamakerTranslate extends Command {
2525
async run() {
2626
try {
2727
const {flags} = this.parse(QnamakerTranslate)
28-
let inputStat = await fs.stat(flags.in)
28+
// Check if data piped in stdin
29+
let stdin = await this.readStdin()
2930
let outputStat = flags.out ? await fs.stat(flags.out) : null
3031

3132
if (outputStat && outputStat.isFile()) {
3233
throw new CLIError('Output can only be writen to a folder')
3334
}
3435

35-
let isLu = !inputStat.isFile() ? true : path.extname(flags.in) === '.lu'
36+
let isLu = await fileHelper.detectLuContent(stdin, flags.in)
3637
let result: any
3738
if (isLu) {
38-
const luFiles = await fileHelper.getLuFiles(flags.in, flags.recurse)
39-
result = await luTranslator.translateLuFile(luFiles, flags.translatekey, flags.tgtlang, flags.srclang, flags.translate_comments, flags.translate_link_text)
39+
let luFiles = await fileHelper.getLuObjects(stdin, flags.in, flags.recurse)
40+
result = await luTranslator.translateLuList(luFiles, flags.translatekey, flags.tgtlang, flags.srclang, flags.translate_comments, flags.translate_link_text)
4041
} else {
41-
let translation = await qnaConverter.parseQnAFileToLu(flags.in, false, false)
42+
let json = stdin ? stdin : await fileHelper.getContentFromFile(flags.in)
43+
let translation = await qnaConverter.parseQnAObjectToLu(json, false, false)
4244
translation = await luTranslator.translateLuObj(translation, flags.translatekey, flags.tgtlang, flags.srclang, flags.translate_comments, flags.translate_link_text)
43-
result = {}
44-
Object.keys(translation).forEach(async idx => {
45-
result[flags.in][idx] = await luConverter.parseFile(translation[idx][0], false)
46-
})
45+
let key = stdin ? 'stdin' : path.basename(flags.in)
46+
result = {
47+
[key] : {}
48+
}
49+
for (let lng in translation) {
50+
let translatedJSON = await luConverter.parseFile(translation[lng], false)
51+
result[key][lng] = await translatedJSON.qnaJsonStructure
52+
}
4753
}
4854

4955
if (flags.out) {
50-
await this.writeOutput(result, flags.out)
56+
await this.writeOutput(result, flags.out, isLu)
5157
} else {
52-
this.log(JSON.stringify(result, null, 2))
58+
if (isLu) {
59+
this.log(result)
60+
} else {
61+
this.log(JSON.stringify(result, null, 2))
62+
}
5363
}
5464

5565
} catch (err) {
@@ -60,13 +70,14 @@ export default class QnamakerTranslate extends Command {
6070
}
6171
}
6272

63-
private async writeOutput(translatedObject: any, out: string) {
73+
private async writeOutput(translatedObject: any, out: string, isLu: boolean) {
6474
let filePath = ''
6575
try {
6676
for (let file in translatedObject) {
6777
for (let lng in translatedObject[file]) {
6878
filePath = await fileHelper.generateNewTranslatedFilePath(file, lng, out)
69-
await fs.writeFile(filePath, translatedObject[path.basename(file)][lng], 'utf-8')
79+
let content = isLu ? translatedObject[file][lng] : JSON.stringify(translatedObject[file][lng], null, 2)
80+
await fs.writeFile(filePath, content, 'utf-8')
7081
}
7182
}
7283
} catch (err) {

‎packages/luis/src/parser/converters/helpers/writer.js‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
const fs = require("fs");
2+
class Writer {
3+
constructor() {
4+
this.indentSize = 4;
5+
this.indentLevel = 0;
6+
this.outputStream = undefined;
7+
}
8+
async setOutputStream(outputPath) {
9+
const ConsoleStream = require('console-stream');
10+
const stream = outputPath ? fs.createWriteStream(outputPath) : ConsoleStream();
11+
const streamPromise = new Promise((resolve) => {
12+
if (stream instanceof fs.WriteStream) {
13+
stream.once('ready', (_fd) => {
14+
this.outputStream = stream;
15+
resolve();
16+
});
17+
}
18+
else {
19+
this.outputStream = stream;
20+
resolve();
21+
}
22+
});
23+
const timeoutPromise = new Promise((resolve) => {
24+
setTimeout(resolve, 2000);
25+
this.outputStream = stream;
26+
});
27+
return Promise.race([streamPromise, timeoutPromise]).then(() => {
28+
});
29+
}
30+
increaseIndentation() {
31+
this.indentLevel += this.indentSize;
32+
}
33+
decreaseIndentation() {
34+
this.indentLevel -= this.indentSize;
35+
}
36+
write(str) {
37+
this.outputStream.write(str);
38+
}
39+
writeLine(str = '') {
40+
if (typeof str === 'string') {
41+
this.write(str + '\n');
42+
}
43+
else {
44+
str.forEach(line => {
45+
this.write(line + '\n');
46+
});
47+
}
48+
}
49+
writeIndented(str) {
50+
let writeFunction = (text) => {
51+
for (let index = 0; index < this.indentLevel; index++) {
52+
this.write(' ');
53+
}
54+
this.write(text);
55+
};
56+
writeFunction.bind(this);
57+
if (typeof str === 'string') {
58+
writeFunction(str);
59+
}
60+
else {
61+
str.forEach(line => {
62+
writeFunction(line);
63+
});
64+
}
65+
}
66+
writeLineIndented(lines) {
67+
if (typeof lines === 'string') {
68+
this.writeIndented(lines + '\n');
69+
}
70+
else {
71+
lines.forEach(line => {
72+
this.writeIndented(line + '\n');
73+
});
74+
}
75+
}
76+
async closeOutputStream() {
77+
this.outputStream.end();
78+
const streamPromise = new Promise((resolve) => {
79+
if (this.outputStream instanceof fs.WriteStream) {
80+
this.outputStream.on('finish', (_fd) => {
81+
resolve();
82+
});
83+
}
84+
else {
85+
resolve();
86+
}
87+
});
88+
const timeoutPromise = new Promise((resolve) => {
89+
setTimeout(resolve, 1000);
90+
});
91+
return Promise.race([streamPromise, timeoutPromise]).then(() => {
92+
this.outputStream = undefined;
93+
});
94+
}
95+
}
96+
module.exports = Writer;

0 commit comments

Comments
 (0)