Skip to content
This repository was archived by the owner on Jan 15, 2025. It is now read-only.

Commit a104eca

Browse files
munozemiliomunozemilio
authored
Merge pull request #208 from microsoft/emilio/luis
Translate test cases and parser added to lib
2 parents 8fbed97 + 2789137 commit a104eca

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+798
-276
lines changed

‎packages/cli/package.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
"dependencies": {
1010
"@microsoft/bf-chatdown": "1.0.0",
1111
"@microsoft/bf-cli-config": "1.0.0",
12-
"@microsoft/bf-luis": "1.0.0",
12+
"@microsoft/bf-lu": "1.0.0",
1313
"@microsoft/bf-qnamaker": "1.0.0",
1414
"@oclif/command": "~1.5.13",
1515
"@oclif/config": "~1.13.0",
@@ -64,7 +64,7 @@
6464
"@microsoft/bf-chatdown",
6565
"@microsoft/bf-cli-config",
6666
"@microsoft/bf-qnamaker",
67-
"@microsoft/bf-luis"
67+
"@microsoft/bf-lu"
6868
],
6969
"hooks": {
7070
"init": "./lib/hooks/init/inithook"

‎packages/luis/package.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
{
2-
"name": "@microsoft/bf-luis",
2+
"name": "@microsoft/bf-lu",
33
"version": "1.0.0",
44
"author": "Microsoft",
55
"bugs": "https://github.com/microsoft/botframework-cli/issues",
6-
"main": "src/parser/index.js",
6+
"main": "lib/parser/index.js",
77
"dependencies": {
88
"@microsoft/bf-cli-command": "1.0.0",
99
"@oclif/config": "~1.13.3",

‎packages/luis/src/commands/luis/translate.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@ export default class LuisTranslate extends Command {
1818
srclang: flags.string({description: 'Source lang code. Auto detect if missing.'}),
1919
tgtlang: flags.string({description: 'Comma separated list of target languages.', required: true}),
2020
translatekey: flags.string({description: 'Machine translation endpoint key.', required: true}),
21-
translate_comments: flags.string({description: 'When set, machine translate comments found in .lu file'}),
22-
translate_link_text: flags.string({description: 'When set, machine translate link description in .lu file'}),
21+
translate_comments: flags.boolean({description: 'When set, machine translate comments found in .lu file'}),
22+
translate_link_text: flags.boolean({description: 'When set, machine translate link description in .lu file'}),
2323
}
2424

2525
/* tslint:disable:forin no-for-in*/

‎packages/luis/src/parser/converters/lumerger.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ const buildLuJsonObject = async function(luObjArray, log, luis_culture, luSearch
3535
filesToParse.splice(0,1)
3636
continue
3737
}
38-
parsedContent = await parseLuFile(luOb.content, log, luis_culture)
38+
let parsedContent = await parseLuFile(luOb.content, log, luis_culture)
3939
parsedFiles.push(luOb.id)
4040

4141
if (haveLUISContent(parsedContent.LUISJsonStructure)

‎packages/luis/src/parser/converters/lutoluisconverter.js‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -258,6 +258,9 @@ const mergeResults_closedlists = function (blob, finalCollection, type) {
258258
const checkAndUpdateVersion = function(finalLUISJSON) {
259259
// Detect if there is content specific to 5.0.0 schema
260260
// any entity with children
261+
if (!finalLUISJSON) {
262+
return
263+
}
261264
let v5DefFound = false;
262265
v5DefFound = (finalLUISJSON.entities || []).find(i => i.children || i.features) ||
263266
(finalLUISJSON.intents || []).find(i => i.features) ||

‎packages/luis/src/parser/lufile/translate-helpers.js‎

Lines changed: 28 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,8 @@ const translateHelpers = {
4949
continue;
5050
}
5151
if(translate_comments) {
52-
addSegment(linesToTranslate, currentLine, true);
52+
addSegment(linesToTranslate, currentLine.charAt(0), false);
53+
addSegment(linesToTranslate, currentLine.substring(1), true);
5354
} else {
5455
addSegment(linesToTranslate, currentLine, false);
5556
}
@@ -92,77 +93,34 @@ const translateHelpers = {
9293
let listSeparator = '';
9394
let content = '';
9495
switch (currentSectionType) {
95-
case PARSERCONSTS.INTENT: {
96-
// strip line of the list separator
97-
listSeparator = currentLine.charAt(0);
98-
addSegment(linesToTranslate, listSeparator + ' ', false);
99-
content = currentLine.slice(1).trim();
100-
let entitiesList = [];
101-
// strip line off labelled entity values,mark pattern any entities as not to localize
102-
if (content.includes('{')) {
103-
const entityRegex = new RegExp(/\{(.*?)\}/g);
104-
let entitiesFound = content.match(entityRegex);
105-
let eStartIndex = -1;
106-
let eEndIndex = -1;
107-
let entity;
108-
for (var entityIdx in entitiesFound) {
109-
entity = entitiesFound[entityIdx];
110-
let lEntity = entity.replace('{', '').replace('}', '');
111-
let labelledValue = '';
112-
let updatedUtteranceLeft = content.substring(0, content.indexOf(entity));
113-
let updatedUtteranceRight = content.substring(content.indexOf(entity) + entity.length);
114-
// is this a labelled value?
115-
if (lEntity.includes('=')) {
116-
let entitySplit = lEntity.split('=');
117-
if (entitySplit.length > 2) {
118-
throw (new exception(retCode.errorCode.INVALID_INPUT, '[ERROR]: Nested entity references are not supported in utterance: ' + content));
119-
}
120-
lEntity = entitySplit[0].trim();
121-
labelledValue = entitySplit[1].trim();
122-
eStartIndex = content.indexOf(entity);
123-
eEndIndex = eStartIndex + labelledValue.length - 1;
124-
content = updatedUtteranceLeft + labelledValue + updatedUtteranceRight;
125-
entitiesList.push(new helperClasses.entity(lEntity, labelledValue, eStartIndex, eEndIndex));
126-
} else {
127-
// This is a pattern entity without a labelled value. Do not localize this.
128-
eStartIndex = content.indexOf(lEntity) - 1;
129-
eEndIndex = eStartIndex + lEntity.length - 1;
130-
content = updatedUtteranceLeft + lEntity + updatedUtteranceRight;
131-
entitiesList.push(new helperClasses.entity(lEntity, null, eStartIndex, eEndIndex));
132-
}
96+
case PARSERCONSTS.INTENT:
97+
listSeparator = currentLine.charAt(0);
98+
addSegment(linesToTranslate, listSeparator + ' ', false);
99+
content = currentLine.slice(1).trim();
100+
let skipChars = ['{', '}', '(', ')', '[', ']', '|', '=']
101+
for (let i = 0; i < content.length; i++) {
102+
let processedText = ''
103+
let tslt = false
104+
if (!skipChars.includes(content.charAt(i))) {
105+
for (let j = i; j < content.length && !skipChars.includes(content.charAt(j)); j++) {
106+
processedText += content.charAt(j)
107+
}
108+
tslt = true
109+
} else if (content.charAt(i) == '{') {
110+
for (let j = i; j < content.length && (content.charAt(j) !== '=' && content.charAt(j) !== '}'); j++) {
111+
processedText += content.charAt(j)
133112
}
113+
} else {
114+
processedText += content.charAt(i)
134115
}
135-
let offset = 0;
136-
let candidateText = '';
137-
// Tokenize the input utterance.
138-
for (var idx in entitiesList) {
139-
let entity = entitiesList[idx];
140-
if (entity.start < 0) entity.start = 0;
141-
if (entity.start !== offset) {
142-
candidateText = content.substring(offset, entity.start);
143-
if (candidateText.trim() !== '') {
144-
addSegment(linesToTranslate, candidateText, true);
145-
} else {
146-
addSegment(linesToTranslate, candidateText, false);
147-
}
148-
}
149-
if (entity.value !== '') {
150-
addSegment(linesToTranslate, ' {' + entity.entity + '=', false);
151-
addSegment(linesToTranslate, content.substring(entity.start, entity.end + 1).trim(), true);
152-
addSegment(linesToTranslate, '} ', false);
153-
} else {
154-
addSegment(linesToTranslate, ' {' + entity.entity + '} ', false);
155-
}
156-
offset = entity.end + 1;
157-
}
158-
if (offset !== content.length) {
159-
candidateText = content.substring(offset);
160-
if (candidateText.trim() !== '') {
161-
addSegment(linesToTranslate, candidateText.trim(), true);
162-
} else {
163-
addSegment(linesToTranslate, candidateText, false);
164-
}
116+
117+
if (processedText.charAt(0) === ' ') {
118+
addSegment(linesToTranslate, ' ', false)
165119
}
120+
121+
addSegment(linesToTranslate, processedText, tslt)
122+
content = content.slice(processedText.length)
123+
i--
166124
}
167125
break;
168126
case PARSERCONSTS.NEWENTITY:
@@ -204,7 +162,6 @@ const translateHelpers = {
204162
addSegment(linesToTranslate, '$', false);
205163
addSegment(linesToTranslate, entityName.trim(), true);
206164
addSegment(linesToTranslate, ' : ' + PARSERCONSTS.QNAALTERATIONS + ' = ', false);
207-
c
208165
} else {
209166
// we would not localize entity line but remember we are under entity section for list entities
210167
// FIX for BF CLI # 121
@@ -219,7 +176,7 @@ const translateHelpers = {
219176
}
220177
} else if(currentLine.indexOf(PARSERCONSTS.ANSWER) === 0) {
221178
if (inAnswer) {
222-
answerData = '';
179+
let answerData = '';
223180
}
224181
addSegment(linesToTranslate, currentLine, false);
225182
inAnswer = !inAnswer;

‎packages/luis/src/parser/translator/lutranslate.js‎

Lines changed: 1 addition & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -30,36 +30,14 @@ module.exports = {
3030
}
3131
}
3232

33-
/**
34-
* Helper function to parse, translate and write out localized lu files
35-
* @param {string} file file name
36-
* @param {string} translate_key translate text API key
37-
* @param {string} to_lang language code to translate content to
38-
* @param {string} src_lang language code for source content
39-
* @param {boolean} translate_comments translate comments in .lu files if this is set to true
40-
* @param {boolean} translate_link_text translate URL or LU reference link text in .lu files if this is set to true
41-
* @returns {void} nothing
42-
* @throws {exception} Throws on errors. exception object includes errCode and text.
43-
*/
44-
async function parseFile(file) {
45-
if(!fs.existsSync(path.resolve(file))) {
46-
throw(new exception(retCode.errorCode.FILE_OPEN_ERROR, 'Sorry unable to open [' + file + ']'));
47-
}
48-
let fileContent = txtfile.readSync(file);
49-
if (!fileContent) {
50-
throw(new exception(retCode.errorCode.FILE_OPEN_ERROR, 'Sorry, error reading file:' + file));
51-
}
52-
return fileContent
53-
}
54-
5533
async function translateLuObject(luObject, translate_key, to_lang, src_lang, translate_comments, translate_link_text) {
5634
let parsedLocContent = ''
5735
let result = {}
5836
// Support multi-language specification for targets.
5937
// Accepted formats are space or comma separated list of target language codes.
6038
// Tokenize to_lang
6139
let toLang = to_lang.split(/[, ]/g)
62-
for (idx in toLang) {
40+
for (let idx in toLang) {
6341
let tgt_lang = toLang[idx].trim();
6442
if (tgt_lang === '') continue;
6543
try {

‎packages/luis/test/commands/luis/convert.test.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -332,37 +332,47 @@ describe('luis:convert', () => {
332332

333333
test
334334
.stdout()
335+
.stderr()
335336
.command(['luis:convert', '--in', `${path.join(__dirname, './../../fixtures/testcases/invalid_prebuilt_2.lu')}`])
336337
.it('luis:convert Invalid entity inherits information is skipped (prebuilt 2)', async (ctx) => {
337338
expect(ctx.stdout).to.contain(`Skipping "> !# @entity.inherits = name : Web.WebSearch"`)
339+
expect(ctx.stderr).to.contain('No LU or Luis content parsed!')
338340
})
339341

340342
test
341343
.stdout()
344+
.stderr()
342345
.command(['luis:convert', '--in', `${path.join(__dirname, './../../fixtures/testcases/invalid_prebuilt_1.lu')}`])
343346
.it('luis:convert Invalid intent inherits information is skipped (prebuilt 1)', async (ctx) => {
344347
expect(ctx.stdout).to.contain(`Skipping "> !# @intent.inherits = name : Web.WebSearch"`)
348+
expect(ctx.stderr).to.contain('No LU or Luis content parsed!')
345349
})
346350

347351
test
348352
.stdout()
353+
.stderr()
349354
.command(['luis:convert', '--in', `${path.join(__dirname, './../../fixtures/testcases/invalid_prebuilt_3.lu')}`, '--log'])
350355
.it('luis:convert Invalid entity inherits information is skipped (prebuilt 3)', async (ctx) => {
351356
expect(ctx.stdout).to.contain(`Skipping "> !# @entity.inherits2 = name : Web.WebSearch"`)
357+
expect(ctx.stderr).to.contain('No LU or Luis content parsed!')
352358
})
353359

354360
test
355361
.stdout()
362+
.stderr()
356363
.command(['luis:convert', '--in', `${path.join(__dirname, './../../fixtures/testcases/invalid_prebuilt_4.lu')}`, '--log'])
357364
.it('luis:convert Invalid intent inherits information is skipped (prebuilt 4)', async (ctx) => {
358365
expect(ctx.stdout).to.contain(`Skipping "> !# @intent.inherits2 = name : Web.WebSearch"`)
366+
expect(ctx.stderr).to.contain('No LU or Luis content parsed!')
359367
})
360368

361369
test
362370
.stdout()
371+
.stderr()
363372
.command(['luis:convert', '--in', `${path.join(__dirname, './../../fixtures/testcases/invalid_model.lu')}`, '--log'])
364373
.it('luis:convert Invalid intent inherits information is skipped (invalid model)', async (ctx) => {
365374
expect(ctx.stdout).to.contain(`Skipping "> !# @app = test"`)
375+
expect(ctx.stderr).to.contain('No LU or Luis content parsed!')
366376
})
367377
})
368378

0 commit comments

Comments
 (0)