Skip to content
This repository was archived by the owner on Jan 15, 2025. It is now read-only.

Commit 5c5c684

Browse files
munozemiliomunozemilio
authored
Adding fix for translation commands (#1150)
* Adding fix for translation commands * Checking for headers in http request * Adding translation settings object to LU object translate * Fixing Docs
1 parent 5220b63 commit 5c5c684

File tree

7 files changed

+99
-56
lines changed

7 files changed

+99
-56
lines changed

‎packages/lu/src/parser/index.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ const modules = {
1313
constructMdFromQnAAlteration: require('./qna/alterations/qnaConverter')
1414
},
1515
translate: {
16+
translationSettings: require('./lufile/translate-helpers').translationSettings,
1617
parseAndTranslate: require('./lufile/translate-helpers').parseAndTranslate,
1718
translateText: require('./lufile/translate-helpers').translateText
1819
},

‎packages/lu/src/parser/lu/lu.js‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
*/
55

66
const translateHelpers = require('./../lufile/translate-helpers')
7+
78
const luOptions = require('./luOptions')
89

910
class Lu {
@@ -21,8 +22,14 @@ class Lu {
2122
}
2223
}
2324

24-
async translate(translate_key, tgt_lang, translate_comments = false, translate_link_text = false){
25-
this.content = await translateHelpers.parseAndTranslate(this.content, translate_key, tgt_lang, '', translate_comments, translate_link_text)
25+
async translate(translate_key, tgt_lang, translate_comments = false, translate_link_text = false, region = ''){
26+
const translateSettings = new translateHelpers.translationSettings();
27+
translateSettings.subscriptionKey = translate_key
28+
translateSettings.translate_link_text = translate_link_text
29+
translateSettings.translate_comments = translate_comments
30+
translateSettings.to_lang = tgt_lang
31+
translateSettings.region = region
32+
this.content = await translateHelpers.parseAndTranslate(this.content, translateSettings)
2633
}
2734
}
2835

‎packages/lu/src/parser/lufile/translate-helpers.js‎

Lines changed: 38 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -14,21 +14,27 @@ const MAX_TRANSLATE_BATCH_SIZE = 25;
1414
const MAX_CHAR_IN_REQUEST = 4990;
1515

1616
const translateHelpers = {
17+
translationSettings : class {
18+
subscriptionKey = '';
19+
to_lang = '';
20+
src_lang = '';
21+
translate_comments = false;
22+
translate_link_text = false;
23+
log = false;
24+
batch_translate = MAX_TRANSLATE_BATCH_SIZE;
25+
region = '';
26+
},
1727
/**
1828
* Helper function to parseAndTranslate lu file content
1929
* @param {string} fileContent file content
20-
* @param {string} subscriptionKey translate text API key
21-
* @param {string} to_lang language code to translate content to
22-
* @param {string} src_lang language code for source content
23-
* @param {boolean} translate_comments translate comments in .lu files if this is set to true
24-
* @param {boolean} translate_link_text translate URL or LU reference link text in .lu files if this is set to true
25-
* @param {boolean} log indicates if this function should write verbose messages to process.stdout
26-
* @param {number} batch_translate indicates number of input lines to batch up before calling translation API
30+
* @param {translationSettings} ts translation settings
2731
* @returns {string} Localized file content
2832
* @throws {exception} Throws on errors. exception object includes errCode and text.
2933
*/
30-
parseAndTranslate : async function(fileContent, subscriptionKey, to_lang, src_lang, translate_comments, translate_link_text, log, batch_translate) {
31-
let batch_translate_size = batch_translate ? parseInt(batch_translate) : MAX_TRANSLATE_BATCH_SIZE;
34+
parseAndTranslate : async function(fileContent, ts) {
35+
const initializeTS = new this.translationSettings();
36+
const translationSettings = {...initializeTS, ...ts};
37+
let batch_translate_size = translationSettings.batch_translate;
3238
fileContent = helpers.sanitizeNewLines(fileContent);
3339
let linesInFile = fileContent.split(NEWLINE);
3440
let linesToTranslate = [];
@@ -46,7 +52,7 @@ const translateHelpers = {
4652
addSegment(linesToTranslate, NEWLINE, false);
4753
continue;
4854
}
49-
if(translate_comments) {
55+
if(translationSettings.translate_comments) {
5056
addSegment(linesToTranslate, currentLine.charAt(0), false);
5157
addSegment(linesToTranslate, currentLine.substring(1), true);
5258
} else {
@@ -186,7 +192,7 @@ const translateHelpers = {
186192
continue;
187193
}
188194
currentSectionType = PARSERCONSTS.URLORFILEREF;
189-
if(translate_link_text) {
195+
if(translationSettings.translate_link_text) {
190196
const linkValueRegEx = new RegExp(/\(.*?\)/g);
191197
let linkValueList = currentLine.trim().match(linkValueRegEx);
192198
let linkValue = linkValueList[0].replace('(','').replace(')','');
@@ -222,7 +228,7 @@ const translateHelpers = {
222228
// do we have any payload to localize? and have we hit the batch size limit?
223229
if ((linesToTranslate.length !== 0) && (lineCtr % batch_translate_size === 0)) {
224230
try {
225-
localizedContent += await batchTranslateText(linesToTranslate, subscriptionKey, to_lang, src_lang, log);
231+
localizedContent += await batchTranslateText(linesToTranslate, translationSettings);
226232
linesToTranslate = [];
227233
} catch (err) {
228234
throw (err)
@@ -231,7 +237,7 @@ const translateHelpers = {
231237
}
232238
if (linesToTranslate.length !== 0) {
233239
try {
234-
localizedContent += await batchTranslateText(linesToTranslate, subscriptionKey, to_lang, src_lang, log);
240+
localizedContent += await batchTranslateText(linesToTranslate, translationSettings);
235241
linesToTranslate = [];
236242
} catch (err) {
237243
throw (err)
@@ -245,25 +251,28 @@ const translateHelpers = {
245251
/**
246252
* Helper function to call MT rest API to translate content
247253
* @param {string} text Text to translate
248-
* @param {string} subscriptionKey user provided subscription to text translation API
249-
* @param {string} to_lang target language to localize to
250-
* @param {string} from_lang source language of text
254+
* @param {translationSettings} translationSettings translation settings
251255
* @returns {object} response from MT call.
252256
* @throws {exception} Throws on errors. exception object includes errCode and text.
253257
*/
254-
translateText: async function(text, subscriptionKey, to_lang, from_lang) {
258+
translateText: async function(text, translationSettings) {
255259
let payload = Array.isArray(text) ? text : [{'Text' : text}];
256-
let tUri = 'https://api.cognitive.microsofttranslator.com/translate?api-version=3.0&to=' + to_lang + '&includeAlignment=true';
257-
if(from_lang) tUri += '&from=' + from_lang;
260+
let tUri = 'https://api.cognitive.microsofttranslator.com/translate?api-version=3.0&to=' + translationSettings.to_lang + '&includeAlignment=true';
261+
if(translationSettings.src_lang) tUri += '&from=' + translationSettings.src_lang;
258262
const options = {
259263
method: 'POST',
260264
body: JSON.stringify (payload),
261265
headers: {
262266
'Content-Type': 'application/json',
263-
'Ocp-Apim-Subscription-Key' : subscriptionKey,
267+
'Ocp-Apim-Subscription-Key' : translationSettings.subscriptionKey,
264268
'X-ClientTraceId' : get_guid (),
265269
}
266270
};
271+
272+
if (translationSettings.region) {
273+
options.headers['Ocp-Apim-Subscription-Region'] = translationSettings.region
274+
}
275+
267276
const res = await fetch(tUri, options);
268277
if (!res.ok) {
269278
throw(new exception(retCode.errorCode.TRANSLATE_SERVICE_FAIL,'Text translator service call failed with [' + res.status + '] : ' + res.statusText + '.\nPlease check key & language code validity'));
@@ -294,14 +303,11 @@ const addSegment = function (linesToTranslate, text, localize) {
294303
/**
295304
* Helper function to batch calls to translate API
296305
* @param {translateLine []} linesToTranslate Array of translateLine objects
297-
* @param {string} subscriptionKey translate text API key
298-
* @param {string} to_lang language code to translate content to
299-
* @param {string} src_lang language code for source content
300-
* @param {boolean} log indicates if this function should write verbose messages to process.stdout
306+
* @param {translationSettings} translationSettings translation settings
301307
* @returns {string} translated content
302308
* @throws {exception} Throws on errors. exception object includes errCode and text.
303309
*/
304-
const batchTranslateText = async function(linesToTranslate, subscriptionKey, to_lang, src_lang, log) {
310+
const batchTranslateText = async function(linesToTranslate, translationSettings) {
305311
// responsible for breaking localizable text into chunks that are
306312
// - not more than 5000 characters in combined length
307313
// - not more than 25 segments in one chunk
@@ -312,13 +318,13 @@ const batchTranslateText = async function(linesToTranslate, subscriptionKey, to_
312318
for (var idx in linesToTranslate) {
313319
let item = linesToTranslate[idx];
314320
if (item.text.length + charCountInChunk >= MAX_CHAR_IN_REQUEST) {
315-
await translateAndMap(batchTranslate, subscriptionKey, to_lang, src_lang, linesToTranslate);
321+
await translateAndMap(batchTranslate, linesToTranslate, translationSettings);
316322
batchTranslate = [];
317323
charCountInChunk = 0;
318324
}
319325
let currentBatchSize = batchTranslate.length > 0 ? batchTranslate.length : 1;
320326
if (currentBatchSize % MAX_TRANSLATE_BATCH_SIZE === 0) {
321-
await translateAndMap(batchTranslate, subscriptionKey, to_lang, src_lang, linesToTranslate);
327+
await translateAndMap(batchTranslate, linesToTranslate, translationSettings);
322328
batchTranslate = [];
323329
charCountInChunk = 0;
324330
}
@@ -329,28 +335,26 @@ const batchTranslateText = async function(linesToTranslate, subscriptionKey, to_
329335
}
330336
}
331337
if (batchTranslate.length !== 0) {
332-
await translateAndMap(batchTranslate, subscriptionKey, to_lang, src_lang, linesToTranslate);
338+
await translateAndMap(batchTranslate, linesToTranslate, translationSettings);
333339
batchTranslate = [];
334340
charCountInChunk = 0;
335341
}
336342
linesToTranslate.forEach(item => retValue += item.text);
337-
if(log) process.stdout.write(chalk.default.gray(retValue));
343+
if(translationSettings.log) process.stdout.write(chalk.default.gray(retValue));
338344
return retValue;
339345
};
340346

341347
/**
342348
* Helper function to call translate and update text with localized result
343349
* @param {object []} batchRequest Array of {'Text':'value'} objects
344-
* @param {string} subscriptionKey translate text API key
345-
* @param {string} to_lang language code to translate content to
346-
* @param {string} src_lang language code for source content
350+
* @param {translationSettings} translationSettings translation settings
347351
* @param {translateLine []} linesToTranslateCopy Array of translateLine objects
348352
* @returns {void}
349353
*/
350-
const translateAndMap = async function (batchRequest, subscriptionKey, to_lang, src_lang, linesToTranslateCopy) {
354+
const translateAndMap = async function (batchRequest, linesToTranslateCopy, translationSettings) {
351355
if (batchRequest.length === 0) return;
352356
let data;
353-
data = await translateHelpers.translateText(batchRequest, subscriptionKey, to_lang, src_lang);
357+
data = await translateHelpers.translateText(batchRequest, translationSettings);
354358
data.forEach((item, idx) => {
355359
// find the correponding item in linesToTranslate
356360
let itemInLine = linesToTranslateCopy.find(item => item.idx === idx);

‎packages/lu/src/parser/translator/lutranslate.js‎

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -7,47 +7,48 @@ const Qna = require('./../lu/qna')
77
const QNAOptions = require('./../lu/qnaOptions')
88

99
module.exports = {
10-
translateLuList: async function(files, translate_key, to_lang, src_lang, translate_comments, translate_link_text) {
11-
return await translateMarkDownList(files, translate_key, to_lang, src_lang, translate_comments, translate_link_text, true)
10+
translateLuList: async function(files, translationSettings) {
11+
return await translateMarkDownList(files, translationSettings, true)
1212
},
13-
translateLu: async function(luObject, translate_key, to_lang, src_lang, translate_comments, translate_link_text) {
14-
return await translateMarkDown(luObject, translate_key, to_lang, src_lang, translate_comments, translate_link_text, true)
13+
translateLu: async function(luObject, translationSettings) {
14+
return await translateMarkDown(luObject, translationSettings, true)
1515
},
16-
translateQnAList: async function(files, translate_key, to_lang, src_lang, translate_comments, translate_link_text) {
17-
return await translateMarkDownList(files, translate_key, to_lang, src_lang, translate_comments, translate_link_text, false)
16+
translateQnAList: async function(files, translationSettings) {
17+
return await translateMarkDownList(files, translationSettings, false)
1818
},
19-
translateQnA: async function(qnaObject, translate_key, to_lang, src_lang, translate_comments, translate_link_text) {
20-
return await translateMarkDown(qnaObject, translate_key, to_lang, src_lang, translate_comments, translate_link_text, false)
19+
translateQnA: async function(qnaObject, translationSettings) {
20+
return await translateMarkDown(qnaObject, translationSettings, false)
2121
}
2222
}
2323

2424

25-
const translateMarkDownList = async function(files, translate_key, to_lang, src_lang, translate_comments, translate_link_text, isLu) {
25+
const translateMarkDownList = async function(files, translationSettings, isLu) {
2626
let translation = {}
2727
let i = 0
2828
while(files.length > i) {
2929
let luObject = files[i++]
3030
try {
31-
translation[luObject.id] = await translateMarkDown(luObject, translate_key, to_lang, src_lang, translate_comments, translate_link_text, isLu)
31+
translation[luObject.id] = await translateMarkDown(luObject, translationSettings, isLu)
3232
} catch (err) {
3333
throw(err);
3434
}
3535
}
3636
return translation
3737
}
3838

39-
const translateMarkDown = async function(luObject, translate_key, to_lang, src_lang, translate_comments, translate_link_text, isLu) {
39+
const translateMarkDown = async function(luObject, translationSettings, isLu) {
4040
let parsedLocContent = ''
4141
let result = []
4242
// Support multi-language specification for targets.
4343
// Accepted formats are space or comma separated list of target language codes.
4444
// Tokenize to_lang
45-
let toLang = to_lang.split(/[, ]/g)
45+
46+
let toLang = translationSettings.to_lang.split(/[, ]/g)
4647
for (let idx in toLang) {
4748
let tgt_lang = toLang[idx].trim();
4849
if (tgt_lang === '') continue;
4950
try {
50-
parsedLocContent = await translateHelpers.parseAndTranslate(luObject.content, translate_key, tgt_lang, src_lang, translate_comments, translate_link_text, false)
51+
parsedLocContent = await translateHelpers.parseAndTranslate(luObject.content, translationSettings)
5152
} catch (err) {
5253
throw(err);
5354
}

‎packages/luis/src/commands/luis/translate.ts‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ const LuisBuilder = require('@microsoft/bf-lu').V2.LuisBuilder
1212
const exception = require('@microsoft/bf-lu').V2.Exception
1313
const fileHelper = require('@microsoft/bf-lu/lib/utils/filehelper')
1414
const luTranslator = require('@microsoft/bf-lu/lib/parser/translator/lutranslate')
15+
const translate = require('@microsoft/bf-lu').translate.translationSettings
1516
const fileExtEnum = require('@microsoft/bf-lu/lib/parser/utils/helpers').FileExtTypeEnum
1617

1718
export default class LuisTranslate extends Command {
@@ -26,6 +27,7 @@ export default class LuisTranslate extends Command {
2627
translatekey: flags.string({description: 'Machine translation endpoint key.', required: true}),
2728
translate_comments: flags.boolean({description: 'When set, machine translate comments found in .lu file'}),
2829
translate_link_text: flags.boolean({description: 'When set, machine translate link description in .lu file'}),
30+
subscription_region: flags.string({description: 'Required request header if using a Cognitive Services Resource. Optional if using a Translator Resource.'}),
2931
force: flags.boolean({char: 'f', description: 'If --out flag is provided with the path to an existing file, overwrites that file', default: false}),
3032
help: flags.help({char: 'h', description: 'luis:translate help'}),
3133
}
@@ -39,9 +41,18 @@ export default class LuisTranslate extends Command {
3941
const isLu = await fileHelper.detectLuContent(stdin, flags.in)
4042
let result: any = {}
4143

44+
let translationSettings: InstanceType<typeof translate> = {
45+
subscriptionKey : flags.translatekey,
46+
to_lang : flags.tgtlang,
47+
src_lang : flags.srclang,
48+
translate_comments : flags.translate_comments,
49+
translate_link_text : flags.translate_link_text,
50+
region : flags.subscription_region
51+
}
52+
4253
if (isLu) {
4354
const luFiles = await fileHelper.getLuObjects(stdin, flags.in, flags.recurse, fileExtEnum.LUFile)
44-
const translatedLuFiles = await luTranslator.translateLuList(luFiles, flags.translatekey, flags.tgtlang, flags.srclang, flags.translate_comments, flags.translate_link_text)
55+
const translatedLuFiles = await luTranslator.translateLuList(luFiles, translationSettings)
4556
luFiles.forEach((lu: any) => {
4657
if (!result[lu.id]) {
4758
result[lu.id] = {}
@@ -55,7 +66,7 @@ export default class LuisTranslate extends Command {
5566
const luisObject = new Luis(fileHelper.parseJSON(json, 'Luis'))
5667
const key = flags.in ? path.basename(flags.in) : 'stdin'
5768
const translation = new Lu(luisObject.parseToLuContent(), key)
58-
const translatedLuis = await luTranslator.translateLu(translation, flags.translatekey, flags.tgtlang, flags.srclang, flags.translate_comments, flags.translate_link_text)
69+
const translatedLuis = await luTranslator.translateLu(translation, translationSettings)
5970
result = {
6071
[key]: {},
6172
}

‎packages/luis/test/commands/luis/translate.test.ts‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,17 +19,25 @@ describe('luis:translate lu file', async () => {
1919
})
2020

2121
before(function(){
22-
nock('https://api.cognitive.microsofttranslator.com')
22+
nock('https://api.cognitive.microsofttranslator.com', {
23+
reqheaders: {
24+
'Ocp-Apim-Subscription-Region': () => true,
25+
},
26+
})
2327
.post(/.*/)
2428
.reply(200, response)
2529

26-
nock('https://api.cognitive.microsofttranslator.com')
30+
nock('https://api.cognitive.microsofttranslator.com', {
31+
reqheaders: {
32+
'Ocp-Apim-Subscription-Region': () => true,
33+
},
34+
})
2735
.post(/.*/)
2836
.reply(200, response2)
2937
})
3038

3139
test
32-
.command(['luis:translate', '--translatekey','xxxxxxx', '--in', `${path.join(__dirname, './../../fixtures/file.lu')}`, '--tgtlang', 'fr', '--out', './'])
40+
.command(['luis:translate', '--translatekey','xxxxxxx', '--in', `${path.join(__dirname, './../../fixtures/file.lu')}`, '--tgtlang', 'fr', '--out', './', '--subscription_region', 'westus2'])
3341
.it('', async () => {
3442
await compareLuFiles('./../../../fr/file.lu', './../../fixtures/fr/file.lu')
3543
})

0 commit comments

Comments
 (0)