Skip to content
This repository was archived by the owner on Jan 15, 2025. It is now read-only.

Commit a130cb9

Browse files
munozemiliomunozemilio
authored
Lu parser re architecture to be OO (#298)
* Initial draft for Parser Rearchitecture * Moving folders to original parser * Fixing PR comments * Moving classes to the correct folder * Unifying LuisGen * Split of Entity parsing * Inverting conditions in luConverter to improve readability * Updating index * Refactor to translator to retrun a list of markdown objects * Commenting test due to error in the build server * Adding Documentation references to exposed functions * Fixing missing references in LuisGen * Removing debug lines * Fixing comments
1 parent 48b85c6 commit a130cb9

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

69 files changed

+1805
-1669
lines changed

‎packages/lu/src/commands/luis/convert.ts‎

Lines changed: 19 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,12 @@
44
*/
55

66
import {CLIError, Command, flags, utils} from '@microsoft/bf-cli-command'
7-
const exception = require('./../../parser/lufile/classes/exception')
7+
const exception = require('./../../parser/utils/exception')
88
const fs = require('fs-extra')
99
const file = require('./../../utils/filehelper')
10-
const luConverter = require('./../../parser/converters/lutoluisconverter')
11-
const luisConverter = require('./../../parser/converters/luistoluconverter')
12-
const fileExtEnum = require('./../../parser/lufile/helpers').FileExtTypeEnum
10+
const fileExtEnum = require('./../../parser/utils/helpers').FileExtTypeEnum
11+
const Luis = require('./../../parser/luis/luis')
12+
const LuisBuilder = require('./../../parser/luis/luisBuilder')
1313

1414
export default class LuisConvert extends Command {
1515
static description = 'Convert .lu file(s) to a LUIS application JSON model or vice versa'
@@ -33,25 +33,31 @@ export default class LuisConvert extends Command {
3333
try {
3434
const {flags} = this.parse(LuisConvert)
3535
// Check if data piped in stdin
36-
let stdin = await this.readStdin()
36+
const stdin = await this.readStdin()
3737

3838
//Check if file or folder
3939
//if folder, only lu to luis is supported
40-
let isLu = await file.detectLuContent(stdin, flags.in)
40+
const isLu = await file.detectLuContent(stdin, flags.in)
4141

4242
// Parse the object depending on the input
4343
let result: any
4444
if (isLu) {
4545
const luFiles = await file.getLuObjects(stdin, flags.in, flags.recurse, fileExtEnum.LUFile)
46-
result = await luConverter.parseLuToLuis(luFiles, flags.log, flags.culture)
46+
result = await LuisBuilder.build(luFiles, flags.log, flags.culture)
47+
if (!result.hasContent()) {
48+
throw new CLIError('No LU or Luis content parsed!')
49+
}
4750
} else {
48-
const luisFile = stdin ? stdin : await file.getContentFromFile(flags.in)
49-
result = await luisConverter.parseLuisObjectToLu(luisFile, flags.sort, flags.in)
50-
}
51+
const luisContent = stdin ? stdin : await file.getContentFromFile(flags.in)
52+
const luisObject = new Luis(file.parseJSON(luisContent, 'Luis'))
53+
if (flags.sort) {
54+
luisObject.sort()
55+
}
5156

52-
// If result is null or undefined return
53-
if (!result) {
54-
throw new CLIError('No LU or Luis content parsed!')
57+
result = luisObject.parseToLuContent()
58+
if (!result) {
59+
throw new CLIError('No LU or Luis content parsed!')
60+
}
5561
}
5662

5763
// Add headers to Luis Json

‎packages/lu/src/commands/luis/translate.ts‎

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,11 @@ import {CLIError, Command, flags, utils} from '@microsoft/bf-cli-command'
77
const fs = require('fs-extra')
88
const path = require('path')
99
const fileHelper = require('./../../utils/filehelper')
10-
const exception = require('./../../parser/lufile/classes/exception')
10+
const exception = require('./../../parser/utils/exception')
1111
const luTranslator = require('./../../parser/translator/lutranslate')
12-
const luisConverter = require('./../../parser/converters/luistoluconverter')
13-
const luConverter = require('./../../parser/lufile/parseFileContents')
14-
const fileExtEnum = require('./../../parser/lufile/helpers').FileExtTypeEnum
12+
const fileExtEnum = require('./../../parser/utils/helpers').FileExtTypeEnum
13+
const Lu = require('./../../parser/lu/lu')
14+
const Luis = require('./../../parser/luis/luis')
1515

1616
export default class LuisTranslate extends Command {
1717
static description = ' Translate given LUIS application JSON model or lu file(s)'
@@ -36,22 +36,30 @@ export default class LuisTranslate extends Command {
3636
// Check if data piped in stdin
3737
let stdin = await this.readStdin()
3838
let isLu = await fileHelper.detectLuContent(stdin, flags.in)
39-
let result: any
39+
let result: any = {}
4040

4141
if (isLu) {
4242
let luFiles = await fileHelper.getLuObjects(stdin, flags.in, flags.recurse, fileExtEnum.LUFile)
43-
result = await luTranslator.translateLuList(luFiles, flags.translatekey, flags.tgtlang, flags.srclang, flags.translate_comments, flags.translate_link_text)
43+
let translatedLuFiles = await luTranslator.translateLuList(luFiles, flags.translatekey, flags.tgtlang, flags.srclang, flags.translate_comments, flags.translate_link_text)
44+
luFiles.forEach((lu: any) => {
45+
if (!result[lu.id]) {
46+
result[lu.id] = {}
47+
}
48+
translatedLuFiles[lu.id].forEach((t: any) => {
49+
result[t.id][t.language] = t.content
50+
})
51+
})
4452
} else {
4553
let json = stdin ? stdin : await fileHelper.getContentFromFile(flags.in)
46-
let translation = await luisConverter.parseLuisObjectToLu(json, false)
47-
translation = await luTranslator.translateLuObj(translation, flags.translatekey, flags.tgtlang, flags.srclang, flags.translate_comments, flags.translate_link_text)
54+
let luisObject = new Luis(fileHelper.parseJSON(json, 'Luis'))
4855
let key = stdin ? 'stdin' : path.basename(flags.in)
56+
let translation = new Lu(luisObject.parseToLuContent(), key)
57+
let translatedLuis = await luTranslator.translateLu(translation, flags.translatekey, flags.tgtlang, flags.srclang, flags.translate_comments, flags.translate_link_text)
4958
result = {
5059
[key] : {}
5160
}
52-
for (let lng in translation) {
53-
let translatedJSON = await luConverter.parseFile(translation[lng], false)
54-
result[key][lng] = await translatedJSON.LUISJsonStructure
61+
for (let lu of translatedLuis) {
62+
result[key][lu.language] = await lu.parseToLuis()
5563
}
5664
}
5765

‎packages/lu/src/commands/qnamaker/convert.ts‎

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,15 @@
44
*/
55

66
import {CLIError, Command, flags, utils} from '@microsoft/bf-cli-command'
7-
const exception = require('./../../parser/lufile/classes/exception')
7+
const exception = require('./../../parser/utils/exception')
88
const fs = require('fs-extra')
99
const file = require('./../../utils/filehelper')
10-
const luConverter = require('./../../parser/converters/qnatoqnajsonconverter')
11-
const qnaConverter = require('./../../parser/converters/qnajsontoqnaconverter')
12-
const fileExtEnum = require('./../../parser/lufile/helpers').FileExtTypeEnum
10+
const fileExtEnum = require('./../../parser/utils/helpers').FileExtTypeEnum
11+
12+
const QnAMaker = require('./../../parser/qna/qnamaker/qnamaker')
13+
const Alterations = require('./../../parser/qna/alterations/alterations')
14+
const QnAMakerBuilder = require('./../../parser/qna/qnamaker/qnaMakerBuilder')
15+
const alterationsBuilder = require('./../../parser/qna/alterations/alterationsBuilder')
1316

1417
export default class QnamakerConvert extends Command {
1518
static description = 'Converts .lu file(s) to QnA application JSON models or vice versa.'
@@ -31,20 +34,26 @@ export default class QnamakerConvert extends Command {
3134
const {flags} = this.parse(QnamakerConvert)
3235

3336
// Check if data piped in stdin
34-
let stdin = await this.readStdin()
37+
const stdin = await this.readStdin()
3538

3639
//Check if file or folder
3740
//if folder, only lu to luis is supported
38-
let isQnA = await file.detectLuContent(stdin, flags.in)
41+
const isQnA = await file.detectLuContent(stdin, flags.in)
3942

4043
// Parse the object depending on the input
4144
let result: any
4245
if (isQnA) {
43-
let luFiles = await file.getLuObjects(stdin, flags.in, flags.recurse, fileExtEnum.QnAFile)
44-
result = await luConverter.parseQnaToJson(luFiles, false, flags.luis_culture)
46+
const luFiles = await file.getLuObjects(stdin, flags.in, flags.recurse, fileExtEnum.QnAFile)
47+
result = {}
48+
result.finalQnAJSON = await QnAMakerBuilder.build(luFiles, false, flags.luis_culture)
49+
result.finalQnAAlterations = await alterationsBuilder.build(luFiles, false, flags.luis_culture)
4550
} else {
46-
const qnAFile = stdin ? stdin : await file.getContentFromFile(flags.in)
47-
result = await qnaConverter.parseQnAObjectToLu(qnAFile, flags.sort, flags.alterations)
51+
const qnaContent = stdin ? stdin : await file.getContentFromFile(flags.in)
52+
const QnA = flags.alterations ? new Alterations(file.parseJSON(qnaContent, 'QnA Alterations')) : new QnAMaker(file.parseJSON(qnaContent, 'QnA'))
53+
if (flags.sort) {
54+
QnA.sort()
55+
}
56+
result = QnA.parseToLuContent()
4857
}
4958

5059
// If result is null or undefined return

‎packages/lu/src/commands/qnamaker/translate.ts‎

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,11 @@ import {CLIError, Command, flags, utils} from '@microsoft/bf-cli-command'
77
const fs = require('fs-extra')
88
const path = require('path')
99
const fileHelper = require('./../../utils/filehelper')
10-
const exception = require('./../../parser/lufile/classes/exception')
10+
const exception = require('./../../parser/utils/exception')
1111
const luTranslator = require('./../../parser/translator/lutranslate')
12-
const qnaConverter = require('./../../parser/converters/qnajsontoqnaconverter')
13-
const luConverter = require('./../../parser/lufile/parseFileContents')
14-
const fileExtEnum = require('./../../parser/lufile/helpers').FileExtTypeEnum
12+
const qnaMaker = require('./../../parser/qna/qnamaker/qnamaker')
13+
const QnA = require('./../../parser/lu/qna')
14+
const fileExtEnum = require('./../../parser/utils/helpers').FileExtTypeEnum
1515

1616
export default class QnamakerTranslate extends Command {
1717
static description = 'Translate given QnA maker application JSON model or qna file(s)'
@@ -37,21 +37,29 @@ export default class QnamakerTranslate extends Command {
3737
let stdin = await this.readStdin()
3838

3939
let isLu = await fileHelper.detectLuContent(stdin, flags.in)
40-
let result: any
40+
let result: any = {}
4141
if (isLu) {
4242
let luFiles = await fileHelper.getLuObjects(stdin, flags.in, flags.recurse, fileExtEnum.QnAFile)
43-
result = await luTranslator.translateLuList(luFiles, flags.translatekey, flags.tgtlang, flags.srclang, flags.translate_comments, flags.translate_link_text)
43+
let translatedLuFiles = await luTranslator.translateQnAList(luFiles, flags.translatekey, flags.tgtlang, flags.srclang, flags.translate_comments, flags.translate_link_text)
44+
luFiles.forEach((lu: any) => {
45+
if (!result[lu.id]) {
46+
result[lu.id] = {}
47+
}
48+
translatedLuFiles[lu.id].forEach((t: any) => {
49+
result[t.id][t.language] = t.content
50+
})
51+
})
4452
} else {
4553
let json = stdin ? stdin : await fileHelper.getContentFromFile(flags.in)
46-
let translation = await qnaConverter.parseQnAObjectToLu(json, false, false)
47-
translation = await luTranslator.translateLuObj(translation, flags.translatekey, flags.tgtlang, flags.srclang, flags.translate_comments, flags.translate_link_text)
54+
let qnaM = new qnaMaker(fileHelper.parseJSON(json, 'QnA'))
55+
let qna = new QnA(qnaM.parseToLuContent())
56+
let qnaTranslation = await luTranslator.translateQnA(qna, flags.translatekey, flags.tgtlang, flags.srclang, flags.translate_comments, flags.translate_link_text)
4857
let key = stdin ? 'stdin' : path.basename(flags.in)
4958
result = {
5059
[key] : {}
5160
}
52-
for (let lng in translation) {
53-
let translatedJSON = await luConverter.parseFile(translation[lng], false)
54-
result[key][lng] = await translatedJSON.qnaJsonStructure
61+
for (let q of qnaTranslation) {
62+
result[key][q.language] = await q.parseToQna()
5563
}
5664
}
5765

‎packages/lu/src/parser/converters/luistocsconverter.js‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
1-
const parse_multi_platform_luis_1 = require("./../luisfile/parseMultiPlatform");
1+
const parse_multi_platform_luis_1 = require("./../luis/propertyHelper");
2+
const LuisGenBuilder = require('./../luis/luisGenBuilder')
23
const Writer = require("./helpers/writer");
34

45
module.exports = {
56
writeFromLuisJson: async function(luisJson, className, space, outPath) {
6-
const app = parse_multi_platform_luis_1.fromLuisApp(luisJson);
7+
const app = LuisGenBuilder.build(luisJson);
78
let writer = new Writer();
89
await writer.setOutputStream(outPath);
910
this.header(space, className, writer);
@@ -80,22 +81,22 @@ module.exports = {
8081
'{'
8182
]);
8283
writer.increaseIndentation();
83-
this.writeEntityBlock(app.simpleEntities, 'Simple entities', (entity) => {
84+
this.writeEntityBlock(app.entities, 'Simple entities', (entity) => {
8485
writer.writeLineIndented(this.getEntityWithType(entity));
8586
}, writer);
86-
this.writeEntityBlock(app.builtInEntities, 'Built-in entities', (entities) => {
87+
this.writeEntityBlock(app.prebuiltEntities, 'Built-in entities', (entities) => {
8788
const entityType = entities[0];
8889
entities.forEach(entity => {
8990
writer.writeLineIndented(this.getEntityWithType(entity, entityType));
9091
});
9192
}, writer);
92-
this.writeEntityBlock(app.listEntities, 'Lists', (entity) => {
93+
this.writeEntityBlock(app.closedLists, 'Lists', (entity) => {
9394
writer.writeLineIndented(this.getEntityWithType(entity, 'list'));
9495
}, writer);
95-
this.writeEntityBlock(app.regexEntities, 'Regex entities', (entity) => {
96+
this.writeEntityBlock(app.regex_entities, 'Regex entities', (entity) => {
9697
writer.writeLineIndented(this.getEntityWithType(entity));
9798
}, writer);
98-
this.writeEntityBlock(app.patternEntities, 'Pattern.any', (entity) => {
99+
this.writeEntityBlock(app.patternAnyEntities, 'Pattern.any', (entity) => {
99100
writer.writeLineIndented(this.getEntityWithType(entity));
100101
}, writer);
101102
// Composites
@@ -128,7 +129,8 @@ module.exports = {
128129
]);
129130
writer.increaseIndentation();
130131
composite.attributes.forEach(attr => {
131-
writer.writeLineIndented(this.getEntityWithType(attr, app.listEntities.includes(attr) ? 'list' : attr));
132+
writer.writeLineIndented(this.getEntityWithType(attr, app.closedLists.includes(attr) ? 'list' : attr));
133+
132134
});
133135
writer.writeLineIndented([
134136
'[JsonProperty("$instance")]',

0 commit comments

Comments
 (0)