Skip to content
This repository was archived by the owner on Jan 15, 2025. It is now read-only.

Commit 6b714c2

Browse files
feich-msvishwacsena
feich-ms
and
vishwacsena
authored
add --force and --log arg for cross-train cli (#847)
* add --force and --log arg for cross-train cli * support --force and --log for qnamaker:cross-train * fix tslint Co-authored-by: Vishwac Sena Kannan <[email protected]>
1 parent 6536d7c commit 6b714c2

File tree

16 files changed

+184
-86
lines changed

16 files changed

+184
-86
lines changed

‎packages/lu/src/parser/cross-train/cross-train.js‎

Lines changed: 3 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,6 @@ const fs = require('fs-extra')
77
const path = require('path')
88
const file = require('../../utils/filehelper')
99
const fileExtEnum = require('../utils/helpers').FileExtTypeEnum
10-
const exception = require('../utils/exception')
11-
const retCode = require('../utils/enums/CLI-errors')
1210
const crossTrainer = require('./crossTrainer')
1311
const confighelper = require('./confighelper')
1412

@@ -30,54 +28,19 @@ module.exports = {
3028
* @param {string} input full path of input lu and qna files folder.
3129
* @param {string} intentName interruption intent name. Default value is _Interruption.
3230
* @param {string} config path to config of mapping rules or mapping rules json content itself. If undefined, it will read config.json from input folder.
31+
* @param {boolean} verbose verbose to indicate whether log warnings and errors or not when parsing cross-train files.
3332
* @returns {luResult: any, qnaResult: any} trainedResult of luResult and qnaResult or undefined if no results.
3433
*/
35-
train: async function (input, intentName, config) {
34+
train: async function (input, intentName, config, verbose) {
3635
// Get all related file content.
3736
const luContents = await file.getFilesContent(input, fileExtEnum.LUFile)
3837
const qnaContents = await file.getFilesContent(input, fileExtEnum.QnAFile)
3938
const configContent = config && !fs.existsSync(config) ? {id: path.join(input, 'config.json'), content: config} : await file.getConfigContent(config)
4039

41-
const configObject = file.getConfigObject(configContent, intentName)
40+
const configObject = file.getConfigObject(configContent, intentName, verbose)
4241

4342
const trainedResult = await crossTrainer.crossTrain(luContents, qnaContents, configObject)
4443

4544
return trainedResult
46-
},
47-
48-
/**
49-
* Write lu and qna files
50-
* @param {Map<string, any>} fileIdToLuResourceMap lu or qna file id to lu resource map.
51-
* @param {string} out output folder name. If not specified, source lu and qna files will be updated.
52-
* @throws {exception} Throws on errors.
53-
*/
54-
writeFiles: async function (fileIdToLuResourceMap, out) {
55-
if (fileIdToLuResourceMap) {
56-
let newFolder
57-
if (out) {
58-
newFolder = out
59-
if (!path.isAbsolute(out)) {
60-
newFolder = path.resolve(out)
61-
}
62-
63-
if (!fs.existsSync(newFolder)) {
64-
fs.mkdirSync(newFolder)
65-
}
66-
}
67-
68-
for (const fileId of fileIdToLuResourceMap.keys()) {
69-
try {
70-
if (newFolder) {
71-
const fileName = path.basename(fileId)
72-
const newFileId = path.join(newFolder, fileName)
73-
await fs.writeFile(newFileId, fileIdToLuResourceMap.get(fileId).Content, 'utf-8')
74-
} else {
75-
await fs.writeFile(fileId, fileIdToLuResourceMap.get(fileId).Content, 'utf-8')
76-
}
77-
} catch (err) {
78-
throw (new exception(retCode.errorCode.OUTPUT_FOLDER_INVALID, `Unable to write to file ${fileId}. Error: ${err.message}`))
79-
}
80-
}
81-
}
8245
}
8346
}

‎packages/lu/src/parser/cross-train/crossTrainer.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -494,11 +494,11 @@ const parseAndValidateContent = async function (objectArray, verbose) {
494494
let fileContent = object.content
495495
if (object.content && object.content !== '') {
496496
if (object.id.toLowerCase().endsWith(fileExtEnum.LUFile)) {
497-
let result = await LuisBuilderVerbose.build([object], true)
497+
let result = await LuisBuilderVerbose.build([object], verbose)
498498
let luisObj = new Luis(result)
499499
fileContent = luisObj.parseToLuContent()
500500
} else {
501-
let result = await qnaBuilderVerbose.build([object], true)
501+
let result = await qnaBuilderVerbose.build([object], verbose)
502502
fileContent = result.parseToQnAContent()
503503
}
504504
}

‎packages/lu/src/utils/filehelper.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -218,7 +218,7 @@ export function getParsedObjects(contents: {id: string, content: string}[]) {
218218
return parsedObjects
219219
}
220220

221-
export function getConfigObject(configContent: any, intentName: string) {
221+
export function getConfigObject(configContent: any, intentName: string, verbose: boolean) {
222222
let finalLuConfigObj = Object.create(null)
223223
let rootLuFiles: string[] = []
224224
const configFileDir = path.dirname(configContent.id)
@@ -267,7 +267,7 @@ export function getConfigObject(configContent: any, intentName: string) {
267267
rootIds: rootLuFiles,
268268
triggerRules: finalLuConfigObj,
269269
intentName,
270-
verbose: true
270+
verbose
271271
}
272272

273273
return crossTrainConfig

‎packages/lu/test/utils/filehelper.test.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ describe('utils/filehelper test', () => {
5252
}
5353
}
5454

55-
let configObject = fileHelper.getConfigObject({ id: path.join(__dirname, 'config.json'), content: JSON.stringify(configContent) }, '_Interruption')
55+
let configObject = fileHelper.getConfigObject({ id: path.join(__dirname, 'config.json'), content: JSON.stringify(configContent) }, '_Interruption', true)
5656
assert.equal(configObject.rootIds[0].includes('main.lu'), true)
5757
assert.equal(configObject.rootIds[1].includes('main.fr-fr.lu'), true)
5858

‎packages/luis/README.md‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -414,18 +414,22 @@ USAGE
414414
$ bf luis:cross-train
415415
416416
OPTIONS
417-
-h, --help luis:cross-train help
418-
-i, --in=in source lu and qna files folder
417+
-h, --help Luis:cross-train help
418+
-i, --in=in Source lu and qna files folder
419419
420-
-o, --out=out output folder name. If not specified, the cross trained files will be written to cross-trained
420+
-o, --out=out Output folder name. If not specified, the cross trained files will be written to cross-trained
421421
folder under folder of current command
422422
423-
--config=config path to config file of mapping rules
423+
--config=config Path to config file of mapping rules
424424
425425
--intentName=intentName [default: _Interruption] Interruption intent name
426426
427-
--rootDialog=rootDialog rootDialog file path. If --config not specified,
427+
--rootDialog=rootDialog RootDialog file path. If --config not specified,
428428
cross-trian will automatically construct the config from file system based on root dialog file
429+
430+
-f, --force [default: false] If --out flag is provided with the path to an existing file, overwrites that file
431+
432+
--log [default: false] Write out log messages to console
429433
```
430434

431435
_See code: [src/commands/luis/cross-train.ts](https://github.com/microsoft/botframework-cli/tree/master/packages/luis/src/commands/luis/cross-train.ts)_

‎packages/luis/src/commands/luis/cross-train.ts‎

Lines changed: 46 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3,21 +3,24 @@
33
* Licensed under the MIT License.
44
*/
55

6-
import {CLIError, Command, flags} from '@microsoft/bf-cli-command'
6+
import {CLIError, Command, flags, utils} from '@microsoft/bf-cli-command'
7+
const fs = require('fs-extra')
8+
const path = require('path')
79
const crossTrain = require('@microsoft/bf-lu/lib/parser/cross-train/cross-train')
810
const exception = require('@microsoft/bf-lu/lib/parser/utils/exception')
9-
const path = require('path')
1011

1112
export default class LuisCrossTrain extends Command {
1213
static description = 'Lu and Qna cross train tool'
1314

1415
static flags: flags.Input<any> = {
15-
help: flags.help({char: 'h', description: 'luis:cross-train help'}),
16-
in: flags.string({char: 'i', description: 'source lu and qna files folder'}),
17-
out: flags.string({char: 'o', description: 'output folder name. If not specified, the cross trained files will be written to cross-trained folder under folder of current command'}),
18-
config: flags.string({description: 'path to config file of mapping rules'}),
16+
help: flags.help({char: 'h', description: 'Luis:cross-train help'}),
17+
in: flags.string({char: 'i', description: 'Source lu and qna files folder'}),
18+
out: flags.string({char: 'o', description: 'Output folder name. If not specified, the cross trained files will be written to cross-trained folder under folder of current command'}),
19+
config: flags.string({description: 'Path to config file of mapping rules'}),
1920
intentName: flags.string({description: 'Interruption intent name', default: '_Interruption'}),
20-
rootDialog: flags.string({description: 'rootDialog file path. If --config not specified, cross-trian will automatically construct the config from file system based on root dialog file'})
21+
rootDialog: flags.string({description: 'RootDialog file path. If --config not specified, cross-trian will automatically construct the config from file system based on root dialog file'}),
22+
force: flags.boolean({char: 'f', description: 'If --out flag is provided with the path to an existing file, overwrites that file', default: false}),
23+
log: flags.boolean({description: 'Write out log messages to console', default: false})
2124
}
2225

2326
async run() {
@@ -39,19 +42,52 @@ export default class LuisCrossTrain extends Command {
3942
throw new CLIError('Missing cross train config. Please provide config by --config or automatically construct config with --rootDialog.')
4043
}
4144

42-
const trainedResult = await crossTrain.train(flags.in, flags.intentName, flags.config)
45+
const trainedResult = await crossTrain.train(flags.in, flags.intentName, flags.config, flags.log)
4346

4447
if (flags.out === undefined) {
4548
flags.out = path.join(process.cwd(), 'cross-trained')
4649
}
4750

48-
await crossTrain.writeFiles(trainedResult.luResult, flags.out)
49-
await crossTrain.writeFiles(trainedResult.qnaResult, flags.out)
51+
await this.writeFiles(trainedResult.luResult, flags.out, flags.force)
52+
await this.writeFiles(trainedResult.qnaResult, flags.out, flags.force)
5053
} catch (err) {
5154
if (err instanceof exception) {
5255
throw new CLIError(err.text)
5356
}
5457
throw err
5558
}
5659
}
60+
61+
async writeFiles(fileIdToLuResourceMap: any, out: string, force: boolean) {
62+
if (fileIdToLuResourceMap) {
63+
let newFolder
64+
if (out) {
65+
newFolder = out
66+
if (!path.isAbsolute(out)) {
67+
newFolder = path.resolve(out)
68+
}
69+
70+
if (!fs.existsSync(newFolder)) {
71+
fs.mkdirSync(newFolder)
72+
}
73+
}
74+
75+
for (const fileId of fileIdToLuResourceMap.keys()) {
76+
try {
77+
let validatedPath
78+
if (newFolder) {
79+
const fileName = path.basename(fileId)
80+
const newFileId = path.join(newFolder, fileName)
81+
validatedPath = utils.validatePath(newFileId, '', force)
82+
} else {
83+
validatedPath = utils.validatePath(fileId, '', force)
84+
}
85+
86+
await fs.writeFile(validatedPath, fileIdToLuResourceMap.get(fileId).Content, 'utf-8')
87+
} catch (err) {
88+
throw new CLIError(`Unable to write to file ${fileId}. Error: ${err.message}`)
89+
}
90+
}
91+
}
92+
}
5793
}

‎packages/luis/test/commands/luis/crossTrain.test.ts‎

Lines changed: 25 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,8 @@ describe('luis:cross-train tests for lu and qna contents', () => {
3939
'--in', `${path.join(__dirname, './../../fixtures/testcases/interruption')}`,
4040
'--intentName', '_Interruption',
4141
'--config', `${path.join(__dirname, './../../fixtures/testcases/interruption/mapping_rules.json')}`,
42-
'--out', './interruptionGen'])
42+
'--out', './interruptionGen',
43+
'--force'])
4344
.it('luis:cross training can get expected result when handling multi locales and duplications', async () => {
4445
expect(await compareLuFiles('./../../../interruptionGen/Main.lu', './../../fixtures/verified/interruption/Main.lu')).to.be.true
4546
expect(await compareLuFiles('./../../../interruptionGen/Main.qna', './../../fixtures/verified/interruption/Main.qna')).to.be.true
@@ -62,7 +63,8 @@ describe('luis:cross-train tests for lu and qna contents', () => {
6263
'--in', `${path.join(__dirname, './../../fixtures/testcases/interruption2')}`,
6364
'--intentName', '_Interruption',
6465
'--config', `${path.join(__dirname, './../../fixtures/testcases/interruption2/config.json')}`,
65-
'--out', './interruptionGen'])
66+
'--out', './interruptionGen',
67+
'--force'])
6668
.it('luis:cross training can get expected result when nestedIntentSection is enabled', async () => {
6769
expect(await compareLuFiles('./../../../interruptionGen/main.lu', './../../fixtures/verified/interruption2/main.lu')).to.be.true
6870
expect(await compareLuFiles('./../../../interruptionGen/dia1.lu', './../../fixtures/verified/interruption2/dia1.lu')).to.be.true
@@ -75,7 +77,8 @@ describe('luis:cross-train tests for lu and qna contents', () => {
7577
'--in', `${path.join(__dirname, './../../fixtures/testcases/interruption3')}`,
7678
'--intentName', '_Interruption',
7779
'--config', `${path.join(__dirname, './../../fixtures/testcases/interruption3/config.json')}`,
78-
'--out', './interruptionGen'])
80+
'--out', './interruptionGen',
81+
'--force'])
7982
.it('luis:cross training can get expected result when multiple dialog invocations occur in same trigger', async () => {
8083
expect(await compareLuFiles('./../../../interruptionGen/main.lu', './../../fixtures/verified/interruption3/main.lu')).to.be.true
8184
expect(await compareLuFiles('./../../../interruptionGen/dia1.lu', './../../fixtures/verified/interruption3/dia1.lu')).to.be.true
@@ -89,7 +92,8 @@ describe('luis:cross-train tests for lu and qna contents', () => {
8992
'--in', './test/fixtures/testcases/interruption4',
9093
'--intentName', '_Interruption',
9194
'--out', './interruptionGen',
92-
'--rootDialog', './test/fixtures/testcases/interruption4/main/main.dialog'])
95+
'--rootDialog', './test/fixtures/testcases/interruption4/main/main.dialog',
96+
'--force'])
9397
.it('luis:cross training can get expected result when automatically detecting config based on rootdialog and file system', async () => {
9498
expect(await compareLuFiles('./../../../interruptionGen/main.lu', './../../fixtures/verified/interruption4/main.lu')).to.be.true
9599
expect(await compareLuFiles('./../../../interruptionGen/dia1.lu', './../../fixtures/verified/interruption4/dia1.lu')).to.be.true
@@ -106,9 +110,22 @@ describe('luis:cross-train tests for lu and qna contents', () => {
106110
'--config', `${path.join(__dirname, './../../fixtures/testcases/interruption5/mapping_rules.json')}`,
107111
'--out', './interruptionGen'])
108112
.it('luis:cross training can split large DeferToLUIS QA pair into smaller ones when it has more than 1000 questions', async () => {
109-
expect(await compareLuFiles('./../../../interruptionGen/main.lu', './../../fixtures/verified/interruption5/main.lu')).to.be.true
110-
expect(await compareLuFiles('./../../../interruptionGen/main.qna', './../../fixtures/verified/interruption5/main.qna')).to.be.true
111-
expect(await compareLuFiles('./../../../interruptionGen/dia1.lu', './../../fixtures/verified/interruption5/dia1.lu')).to.be.true
112-
expect(await compareLuFiles('./../../../interruptionGen/dia1.qna', './../../fixtures/verified/interruption5/dia1.qna')).to.be.true
113+
expect(await compareLuFiles('./../../../interruptionGen/main(1).lu', './../../fixtures/verified/interruption5/main.lu')).to.be.true
114+
expect(await compareLuFiles('./../../../interruptionGen/main(1).qna', './../../fixtures/verified/interruption5/main.qna')).to.be.true
115+
expect(await compareLuFiles('./../../../interruptionGen/dia1(1).lu', './../../fixtures/verified/interruption5/dia1.lu')).to.be.true
116+
expect(await compareLuFiles('./../../../interruptionGen/dia1(1).qna', './../../fixtures/verified/interruption5/dia1.qna')).to.be.true
117+
})
118+
119+
test
120+
.stdout()
121+
.stderr()
122+
.command(['luis:cross-train',
123+
'--in', `${path.join(__dirname, './../../fixtures/testcases/interruption6')}`,
124+
'--intentName', '_Interruption',
125+
'--config', `${path.join(__dirname, './../../fixtures/testcases/interruption6/mapping_rules.json')}`,
126+
'--out', './interruptionGen',
127+
'--log'])
128+
.it('displays a warning if log is set true', ctx => {
129+
expect(ctx.stdout).to.contain('[WARN] line 1:0 - line 1:15: no utterances found for intent definition: "# hotelLocation"')
113130
})
114131
})

‎packages/luis/test/fixtures/testcases/interruption6/dia1/dia1.lu‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
# hotelLocation

‎packages/luis/test/fixtures/testcases/interruption6/main/main.lu‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# dia1_trigger
2+
- book a hotel for me

‎packages/luis/test/fixtures/testcases/interruption6/mapping_rules.json‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
{
2+
"./main/Main.lu": {
3+
"rootDialog": true,
4+
"triggers": {
5+
"dia1_trigger": "./Dia1/dia1.lu"
6+
}
7+
}
8+
}

0 commit comments

Comments
 (0)