Skip to content
This repository was archived by the owner on Jan 15, 2025. It is now read-only.

Commit 9695b25

Browse files
feich-msfeich-ms
authored
optimize logic when handling empty files (#925)
1 parent 789391b commit 9695b25

File tree

18 files changed

+227
-9
lines changed

18 files changed

+227
-9
lines changed

‎packages/lu/src/parser/lu/luMerger.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -547,7 +547,7 @@ const parseLuFile = async function(luOb, log, luis_culture) {
547547
let parsedContent = ''
548548
if (!luOb.content) {
549549
let error = BuildDiagnostic({ message: `Cannot parse empty ${luOb.id}. Please add content to the file or remove it.` })
550-
throw(new exception(retCode.errorCode.EMPTY_CONTENT, error.toString()));
550+
throw(new exception(retCode.errorCode.INVALID_INPUT_FILE, error.toString()));
551551
}
552552
try {
553553
parsedContent = await parseFileContents.parseFile(luOb.content, log, luis_culture);

‎packages/lu/src/parser/lubuild/builder.ts‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -74,15 +74,18 @@ export class Builder {
7474
let fileContent = ''
7575
let result
7676
let luisObj
77-
const luFiles = await fileHelper.getLuObjects(undefined, file, true, fileExtEnum.LUFile)
77+
let luFiles = await fileHelper.getLuObjects(undefined, file, true, fileExtEnum.LUFile)
78+
this.handler(`${file} loaded\n`)
79+
80+
// filter empty lu files
81+
luFiles = luFiles.filter((file: any) => file.content !== '')
82+
if (luFiles.length <= 0) continue
83+
7884
try {
7985
result = await LuisBuilderVerbose.build(luFiles, true, fileCulture)
8086
luisObj = new Luis(result)
8187
fileContent = luisObj.parseToLuContent()
82-
this.handler(`${file} loaded\n`)
8388
} catch (err) {
84-
if (err.errCode === retCode.errorCode.EMPTY_CONTENT) continue
85-
8689
if (err.source) {
8790
err.text = `Invalid LU file ${err.source}: ${err.text}`
8891
} else {

‎packages/lu/src/parser/qnabuild/builder.ts‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -73,9 +73,13 @@ export class Builder {
7373
crosstrainedRecognizers.set(fileName, new CrossTrainedRecognizer(crossTrainedRecognizerPath, crosstrainedRecognizerContent, crosstrainedRecognizerSchema as string))
7474
}
7575

76-
const qnaFiles = await fileHelper.getLuObjects(undefined, file, true, fileExtEnum.QnAFile)
76+
let qnaFiles = await fileHelper.getLuObjects(undefined, file, true, fileExtEnum.QnAFile)
7777
this.handler(`${file} loaded\n`)
7878

79+
// filter empty qna files
80+
qnaFiles = qnaFiles.filter((file: any) => file.content !== '')
81+
if (qnaFiles.length <= 0) continue
82+
7983
const multiRecognizerPath = path.join(fileFolder, `${fileName}.qna.dialog`)
8084
if (!multiRecognizers.has(fileName)) {
8185
let multiRecognizerContent = {}
@@ -588,8 +592,6 @@ export class Builder {
588592
content.content = mergedContent
589593
contents.set(name, content)
590594
} catch (err) {
591-
if (err.errCode === retCode.errorCode.EMPTY_CONTENT) continue
592-
593595
if (err.source) {
594596
err.text = `Invalid QnA file ${err.source}: ${err.text}`
595597
} else {

‎packages/lu/src/parser/utils/enums/CLI-errors.js‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@ module.exports = {
2929
INVALID_REGEX_ENTITY: 22,
3030
INVALID_COMPOSITE_ENTITY: 23,
3131
LUIS_API_CALL_FAILED: 24,
32-
EMPTY_CONTENT: 25,
3332
UNKNOWN_ERROR: 99,
3433
BOUNDARY_INTENTS: 501,
3534
BOUNDARY_PATTERNANYENTITY: 502,

‎packages/luis/test/commands/luis/build.test.ts‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -980,4 +980,74 @@ describe('luis:build write dialog assets successfully if schema is specified', (
980980
expect(await compareFiles('./../../../results/sandwich.lu.dialog', './../../fixtures/testcases/lubuild/sandwich/dialogs-with-schema/sandwich.lu.dialog')).to.be.true
981981
expect(await compareFiles('./../../../results/sandwich.lu.qna.dialog', './../../fixtures/testcases/lubuild/sandwich/dialogs-with-schema/sandwich.lu.qna.dialog')).to.be.true
982982
})
983+
})
984+
985+
describe('luis:build write dialog assets successfully when empty files exist', () => {
986+
before(async function () {
987+
await fs.ensureDir(path.join(__dirname, './../../../results/'))
988+
989+
nock('https://westus.api.cognitive.microsoft.com')
990+
.get(uri => uri.includes('apps'))
991+
.reply(200, [{
992+
name: 'test.en-us.lu',
993+
id: 'f8c64e2a-2222-3a09-8f78-39d7adc76ec5'
994+
}])
995+
996+
nock('https://westus.api.cognitive.microsoft.com')
997+
.post(uri => uri.includes('import'))
998+
.reply(201, {
999+
appId: 'f8c64e2a-1111-3a09-8f78-39d7adc76ec5'
1000+
})
1001+
1002+
nock('https://westus.api.cognitive.microsoft.com')
1003+
.post(uri => uri.includes('train'))
1004+
.reply(202, {
1005+
statusId: 2,
1006+
status: 'UpToDate'
1007+
})
1008+
1009+
nock('https://westus.api.cognitive.microsoft.com')
1010+
.get(uri => uri.includes('train'))
1011+
.reply(200, [{
1012+
modelId: '99999',
1013+
details: {
1014+
statusId: 0,
1015+
status: 'Success',
1016+
exampleCount: 0
1017+
}
1018+
}])
1019+
1020+
nock('https://westus.api.cognitive.microsoft.com')
1021+
.post(uri => uri.includes('publish'))
1022+
.reply(201, {
1023+
versionId: '0.1',
1024+
isStaging: false
1025+
})
1026+
})
1027+
1028+
after(async function () {
1029+
await fs.remove(path.join(__dirname, './../../../results/'))
1030+
})
1031+
1032+
test
1033+
.stdout()
1034+
.command(['luis:build', '--in', './test/fixtures/testcases/lubuild/empty-file/lufiles', '--authoringKey', uuidv1(), '--botName', 'test', '--dialog', 'crosstrained', '--out', './results', '--log', '--suffix', 'development'])
1035+
.it('should write dialog assets successfully when empty files exist', async ctx => {
1036+
expect(ctx.stdout).to.contain('empty.lu loaded')
1037+
expect(ctx.stdout).to.contain('non-empty.lu loaded')
1038+
1039+
expect(ctx.stdout).to.contain('Handling applications...')
1040+
expect(ctx.stdout).to.contain('Creating LUIS.ai application')
1041+
expect(ctx.stdout).to.contain('training version=0.1')
1042+
expect(ctx.stdout).to.contain('waiting for training for version=0.1')
1043+
expect(ctx.stdout).to.contain('publishing version=0.1')
1044+
1045+
expect(ctx.stdout).to.contain('non-empty.lu publishing finished')
1046+
1047+
expect(await compareFiles('./../../../results/empty.lu.qna.dialog', './../../fixtures/testcases/lubuild/empty-file/dialogs/empty.lu.qna.dialog')).to.be.true
1048+
expect(await compareFiles('./../../../results/luis.settings.development.westus.json', './../../fixtures/testcases/lubuild/empty-file/config/luis.settings.development.westus.json')).to.be.true
1049+
expect(await compareFiles('./../../../results/non-empty.lu.dialog', './../../fixtures/testcases/lubuild/empty-file/dialogs/non-empty.lu.dialog')).to.be.true
1050+
expect(await compareFiles('./../../../results/non-empty.en-us.lu.dialog', './../../fixtures/testcases/lubuild/empty-file/dialogs/non-empty.en-us.lu.dialog')).to.be.true
1051+
expect(await compareFiles('./../../../results/non-empty.lu.qna.dialog', './../../fixtures/testcases/lubuild/empty-file/dialogs/non-empty.lu.qna.dialog')).to.be.true
1052+
})
9831053
})

‎packages/luis/test/fixtures/testcases/lubuild/empty-file/config/luis.settings.development.westus.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"luis": {
3+
"non_empty_en_us_lu": "f8c64e2a-1111-3a09-8f78-39d7adc76ec5"
4+
}
5+
}

‎packages/luis/test/fixtures/testcases/lubuild/empty-file/dialogs/non-empty.en-us.lu.dialog‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"$kind": "Microsoft.LuisRecognizer",
3+
"id": "LUIS_non-empty",
4+
"applicationId": "=settings.luis.non_empty_en_us_lu",
5+
"endpoint": "=settings.luis.endpoint",
6+
"endpointKey": "=settings.luis.endpointKey"
7+
}

‎packages/luis/test/fixtures/testcases/lubuild/empty-file/dialogs/non-empty.lu.dialog‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
{
2+
"$kind": "Microsoft.MultiLanguageRecognizer",
3+
"id": "LUIS_non-empty",
4+
"recognizers": {
5+
"en-us": "non-empty.en-us.lu",
6+
"": "non-empty.en-us.lu"
7+
}
8+
}

‎packages/luis/test/fixtures/testcases/lubuild/empty-file/dialogs/non-empty.lu.qna.dialog‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"$kind": "Microsoft.CrossTrainedRecognizerSet",
3+
"recognizers": [
4+
"non-empty.lu"
5+
]
6+
}

‎packages/luis/test/fixtures/testcases/lubuild/empty-file/lufiles/non-empty.lu‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# greeting
2+
- hello

0 commit comments

Comments
 (0)