Skip to content
This repository was archived by the owner on Jan 15, 2025. It is now read-only.

Commit e631072

Browse files
vishwacsenavishwacsena
authored
Composite entity definitions can now be split across different .lu files (#622)
* Fix composite merge behavior * Fix for #620 * fix typo
1 parent bc3e4b2 commit e631072

File tree

5 files changed

+78
-41
lines changed

5 files changed

+78
-41
lines changed

‎packages/lu/src/parser/lu/luMerger.js‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -378,8 +378,10 @@ const buildLuJsonObject = async function(luObjArray, log, luis_culture, luSearch
378378
let parsedContent = await parseLuFile(luOb, log, luis_culture)
379379
parsedFiles.push(luOb.id)
380380

381-
if (haveLUISContent(parsedContent.LUISJsonStructure)
382-
&& parsedContent.LUISJsonStructure.validate()) {
381+
// Fix for BF-CLI #620
382+
// We do not perform validation here. for parseFile V1 API route,
383+
// the recommendation is to call validate() after parse.
384+
if (haveLUISContent(parsedContent.LUISJsonStructure)) {
383385
allParsedLUISContent.push(parserObject.create(parsedContent.LUISJsonStructure, undefined, undefined, luOb.id, luOb.includeInCollate))
384386
}
385387

‎packages/lu/src/parser/luis/luisBuilder.js‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,7 @@ class LuisBuilder {
2626
* @throws {exception} Throws on errors. exception object includes errCode and text.
2727
*/
2828
static async fromContent(luContent) {
29-
let parsedContent = await parseFileContents(luContent, false, '')
30-
return new Luis(parsedContent.LUISJsonStructure)
29+
return await parseAndValidateLuFile(luContent)
3130
}
3231

3332
/**
@@ -43,8 +42,7 @@ class LuisBuilder {
4342
}
4443

4544
if(luArray.length === 1){
46-
let parsedContent = await parseFileContents(luArray[0].content, false, luArray[0].language)
47-
return new Luis(parsedContent.LUISJsonStructure)
45+
return await parseAndValidateLuFile(luArray[0].content, false, luArray[0].language)
4846
}
4947

5048
return build(luArray, false, '', luSearchFn)
@@ -53,3 +51,10 @@ class LuisBuilder {
5351
}
5452

5553
module.exports = LuisBuilder
54+
55+
const parseAndValidateLuFile = async function(luContent, log = undefined, culture = undefined) {
56+
let parsedContent = await parseFileContents(luContent, log, culture)
57+
let LUISObj = new Luis(parsedContent.LUISJsonStructure)
58+
LUISObj.validate()
59+
return LUISObj
60+
}

‎packages/lu/src/parser/luis/luisCollate.js‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ const LUISObjNameEnum = require('./../utils/enums/luisobjenum')
44
const Luis = require('./luis')
55
const helpers = require('./../utils/helpers')
66
const mergeLuFiles = require('./../lu/luMerger').Build
7+
const exception = require('./../utils/exception')
78

89
/**
910
* Builds a Luis instance from a Lu list.
@@ -259,7 +260,9 @@ const buildComposites = function(blob, FinalLUISJSON){
259260
FinalLUISJSON.composites.push(composite);
260261
} else {
261262
if (JSON.stringify(composite.children.sort()) !== JSON.stringify(compositeInMaster[0].children.sort())) {
262-
throw (new exception(retCode.errorCode.INVALID_COMPOSITE_ENTITY, `[ERROR]: Composite entity: ${composite.name} has multiple definition with different children. \n 1. ${compositeInMaster[0].children.join(', ')}\n 2. ${composite.children.join(', ')}`));
263+
composite.children.forEach(child => {
264+
if (!compositeInMaster[0].children.includes(child)) compositeInMaster[0].children.push(child)
265+
})
263266
} else {
264267
// merge roles
265268
(composite.roles || []).forEach(blobRole => {

‎packages/lu/test/parser/lufile/luis.boundary.test.js‎

Lines changed: 29 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,11 @@ const luMerger = require('./../../../src/parser/lu/luMerger');
88
const luObj = require('../../../src/parser/lu/lu');
99
const retCode = require('./../../../src/parser/utils/enums/CLI-errors');
1010
const POSSIBLE_CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz 0123456789 ";
11+
const LuisBuilder = require('./../../../src/parser/luis/luisBuilder');
12+
1113
describe('Validations for LU content (based on LUIS boundaries)', function () {
1214
it (`At most ${retCode.boundaryLimits.MAX_NUM_INTENTS} intents in LU content`, function(done) {
13-
luMerger.Build(new Array(new luObj(getMaxIntentTestData())))
15+
LuisBuilder.fromLU(new Array(new luObj(getMaxIntentTestData())))
1416
.then(res => done(res))
1517
.catch(err => {
1618
assert.equal(err.errCode, retCode.errorCode.BOUNDARY_INTENTS);
@@ -20,7 +22,7 @@ describe('Validations for LU content (based on LUIS boundaries)', function () {
2022
})
2123

2224
it (`At most ${retCode.boundaryLimits.MAX_NUM_UTTERANCES} utterances in LU content`, function(done) {
23-
luMerger.Build(new Array(new luObj(getMaxUtteranceTestData())))
25+
LuisBuilder.fromLU(new Array(new luObj(getMaxUtteranceTestData())))
2426
.then(res => done(res))
2527
.catch(err => {
2628
assert.equal(err.errCode, retCode.errorCode.BOUNDARY_UTTERANCES);
@@ -30,7 +32,7 @@ describe('Validations for LU content (based on LUIS boundaries)', function () {
3032
})
3133

3234
it (`At most ${retCode.boundaryLimits.MAX_NUM_PATTERNANY_ENTITIES} pattern.any entities in LU content`, function(done) {
33-
luMerger.Build(new Array(new luObj(getMaxPatternAnyEntities())))
35+
LuisBuilder.fromLU(new Array(new luObj(getMaxPatternAnyEntities())))
3436
.then(res => done(res))
3537
.catch(err => {
3638
assert.equal(err.errCode, retCode.errorCode.BOUNDARY_PATTERNANYENTITY);
@@ -40,7 +42,7 @@ describe('Validations for LU content (based on LUIS boundaries)', function () {
4042
})
4143

4244
it (`At most ${retCode.boundaryLimits.MAX_CHAR_IN_UTTERANCE} characters in any utterance`, function(done) {
43-
luMerger.Build(new Array(new luObj(getMaxUtteranceCharLimit(), 'stdin', true)))
45+
LuisBuilder.fromLU(new Array(new luObj(getMaxUtteranceCharLimit(), 'stdin', true)))
4446
.then(res => done(res))
4547
.catch(err => {
4648
assert.equal(err.errCode, retCode.errorCode.BOUNDARY_UTTERANCE_CHAR_LENGTH);
@@ -50,7 +52,7 @@ describe('Validations for LU content (based on LUIS boundaries)', function () {
5052
})
5153

5254
it (`At most ${retCode.boundaryLimits.MAX_NUM_PATTERNS} patterns in LU content`, function(done) {
53-
luMerger.Build(new Array(new luObj(getMaxPatterns(), 'stdin', true)))
55+
LuisBuilder.fromLU(new Array(new luObj(getMaxPatterns(), 'stdin', true)))
5456
.then(res => done(res))
5557
.catch(err => {
5658
assert.equal(err.errCode, retCode.errorCode.BOUNDARY_PATTERNS);
@@ -60,7 +62,7 @@ describe('Validations for LU content (based on LUIS boundaries)', function () {
6062
})
6163

6264
it (`At most ${retCode.boundaryLimits.MAX_CHAR_IN_PATTERNS} characters in any pattern`, function(done) {
63-
luMerger.Build(new Array(new luObj(getMaxPatternCharLimit(), 'stdin', true)))
65+
LuisBuilder.fromLU(new Array(new luObj(getMaxPatternCharLimit(), 'stdin', true)))
6466
.then(res => done(res))
6567
.catch(err => {
6668
assert.equal(err.errCode, retCode.errorCode.BOUNDARY_PATTERN_CHAR_LIMIT);
@@ -70,7 +72,7 @@ describe('Validations for LU content (based on LUIS boundaries)', function () {
7072
})
7173

7274
it (`At most ${retCode.boundaryLimits.MAX_NUM_REGEX_ENTITIES} regex entities`, function(done) {
73-
luMerger.Build(new Array(new luObj(getMaxRegeExEntityDefinition(retCode.boundaryLimits.MAX_NUM_REGEX_ENTITIES), 'stdin', true)))
75+
LuisBuilder.fromLU(new Array(new luObj(getMaxRegeExEntityDefinition(retCode.boundaryLimits.MAX_NUM_REGEX_ENTITIES), 'stdin', true)))
7476
.then(res => done(res))
7577
.catch(err => {
7678
assert.equal(err.errCode, retCode.errorCode.BOUNDARY_REGEX_ENTITY);
@@ -80,7 +82,7 @@ describe('Validations for LU content (based on LUIS boundaries)', function () {
8082
})
8183

8284
it (`At most ${retCode.boundaryLimits.MAX_CHAR_REGEX_ENTITY_PATTERN} characters in regex entity pattern`, function(done) {
83-
luMerger.Build(new Array(new luObj(getMaxRegeExEntityDefinition(1, retCode.boundaryLimits.MAX_CHAR_REGEX_ENTITY_PATTERN), 'stdin', true)))
85+
LuisBuilder.fromLU(new Array(new luObj(getMaxRegeExEntityDefinition(1, retCode.boundaryLimits.MAX_CHAR_REGEX_ENTITY_PATTERN), 'stdin', true)))
8486
.then(res => done(res))
8587
.catch(err => {
8688
assert.equal(err.errCode, retCode.errorCode.BOUNDARY_REGEX_CHAR_LIMIT);
@@ -90,7 +92,7 @@ describe('Validations for LU content (based on LUIS boundaries)', function () {
9092
})
9193

9294
it (`At most ${retCode.boundaryLimits.MAX_LIST_ENTITY_SYNONYMS} synonyms under any parent for a list entity`, function(done) {
93-
luMerger.Build(new Array(new luObj(getListEntity(0, retCode.boundaryLimits.MAX_LIST_ENTITY_SYNONYMS), 'stdin', true)))
95+
LuisBuilder.fromLU(new Array(new luObj(getListEntity(0, retCode.boundaryLimits.MAX_LIST_ENTITY_SYNONYMS), 'stdin', true)))
9496
.then(res => done(res))
9597
.catch(err => {
9698
assert.equal(err.errCode, retCode.errorCode.BOUNDARY_SYNONYMS_LENGTH);
@@ -100,7 +102,7 @@ describe('Validations for LU content (based on LUIS boundaries)', function () {
100102
})
101103

102104
it (`At most ${retCode.boundaryLimits.MAX_NUM_PHRASE_LISTS} phrase lists`, function(done) {
103-
luMerger.Build(new Array(new luObj(getMaxPhraseLists(), 'stdin', true)))
105+
LuisBuilder.fromLU(new Array(new luObj(getMaxPhraseLists(), 'stdin', true)))
104106
.then(res => done(res))
105107
.catch(err => {
106108
assert.equal(err.errCode, retCode.errorCode.BOUNDARY_PHRASE_LIST_LIMIT);
@@ -110,7 +112,7 @@ describe('Validations for LU content (based on LUIS boundaries)', function () {
110112
})
111113

112114
it (`At most ${retCode.boundaryLimits.MAX_INTERCHANGEABLE_PHRASES} phrases across all interchangeable phrase lists`, function(done) {
113-
luMerger.Build(new Array(new luObj(getMaxPhraseLists(0, retCode.boundaryLimits.MAX_INTERCHANGEABLE_PHRASES, true), 'stdin', true)))
115+
LuisBuilder.fromLU(new Array(new luObj(getMaxPhraseLists(0, retCode.boundaryLimits.MAX_INTERCHANGEABLE_PHRASES, true), 'stdin', true)))
114116
.then(res => done(res))
115117
.catch(err => {
116118
assert.equal(err.errCode, retCode.errorCode.BOUNDARY_INTC_PHRASES_LIMIT);
@@ -120,7 +122,7 @@ describe('Validations for LU content (based on LUIS boundaries)', function () {
120122
})
121123

122124
it (`At most ${retCode.boundaryLimits.MAX_NON_INTERCHANGEABLE_PHRASES} phrases across all non-interchangeable phrase lists`, function(done) {
123-
luMerger.Build(new Array(new luObj(getMaxPhraseLists(0, retCode.boundaryLimits.MAX_NON_INTERCHANGEABLE_PHRASES), 'stdin', true)))
125+
LuisBuilder.fromLU(new Array(new luObj(getMaxPhraseLists(0, retCode.boundaryLimits.MAX_NON_INTERCHANGEABLE_PHRASES), 'stdin', true)))
124126
.then(res => done(res))
125127
.catch(err => {
126128
assert.equal(err.errCode, retCode.errorCode.BOUNDARY_NINTC_PHRASES_LIMIT);
@@ -130,7 +132,7 @@ describe('Validations for LU content (based on LUIS boundaries)', function () {
130132
})
131133

132134
it (`At most ${retCode.boundaryLimits.MAX_ROLES_PER_ENTITY} roles per entity`, function(done) {
133-
luMerger.Build(new Array(new luObj(getEntityWithRoles())))
135+
LuisBuilder.fromLU(new Array(new luObj(getEntityWithRoles())))
134136
.then(res => done(res))
135137
.catch(err => {
136138
assert.equal(err.errCode, retCode.errorCode.BOUNDARY_ROLES_PER_ENTITY);
@@ -140,7 +142,7 @@ describe('Validations for LU content (based on LUIS boundaries)', function () {
140142
})
141143

142144
it (`At most ${retCode.boundaryLimits.MAX_NUM_ROLES} roles across all entities per LU content`, function(done) {
143-
luMerger.Build(new Array(new luObj(getEntityWithRoles(51, 6))))
145+
LuisBuilder.fromLU(new Array(new luObj(getEntityWithRoles(51, 6))))
144146
.then(res => done(res))
145147
.catch(err => {
146148
assert.equal(err.errCode, retCode.errorCode.BOUNDARY_TOTAL_ROLES);
@@ -150,7 +152,7 @@ describe('Validations for LU content (based on LUIS boundaries)', function () {
150152
})
151153

152154
it (`At most ${retCode.boundaryLimits.MAX_NUM_DESCRIPTORS_PER_MODEL} descriptors per model`, function(done) {
153-
luMerger.Build(new Array(new luObj(getEntityWithFeatures())))
155+
LuisBuilder.fromLU(new Array(new luObj(getEntityWithFeatures())))
154156
.then(res => done(res))
155157
.catch(err => {
156158
assert.equal(err.errCode, retCode.errorCode.BOUNDARY_FEATURE_PER_MODEL);
@@ -160,7 +162,7 @@ describe('Validations for LU content (based on LUIS boundaries)', function () {
160162
})
161163

162164
it (`At most ${retCode.boundaryLimits.MAX_NUM_PARENT_ENTITIES} parent nodes in an ML entitiy`, function(done) {
163-
luMerger.Build(new Array(new luObj(getMLEntity())))
165+
LuisBuilder.fromLU(new Array(new luObj(getMLEntity())))
164166
.then(res => done(res))
165167
.catch(err => {
166168
assert.equal(err.errCode, retCode.errorCode.BOUNDARY_PARENT_ENTITY_LIMIT);
@@ -170,7 +172,7 @@ describe('Validations for LU content (based on LUIS boundaries)', function () {
170172
})
171173

172174
it (`At most ${retCode.boundaryLimits.MAX_TOTAL_ENTITES_AND_ROLES} total entities and roles in given LU content`, function(done) {
173-
luMerger.Build(new Array(new luObj(getMaxEntityAndRoles())))
175+
LuisBuilder.fromLU(new Array(new luObj(getMaxEntityAndRoles())))
174176
.then(res => done(res))
175177
.catch(err => {
176178
assert.equal(err.errCode, retCode.errorCode.BOUNDARY_TOTAL_ENTITIES_AND_ROLES);
@@ -180,7 +182,7 @@ describe('Validations for LU content (based on LUIS boundaries)', function () {
180182
})
181183

182184
it (`At most ${retCode.boundaryLimits.MAX_NUM_CLOSED_LISTS} list entities`, function(done) {
183-
luMerger.Build(new Array(new luObj(getMaxListEntity())))
185+
LuisBuilder.fromLU(new Array(new luObj(getMaxListEntity())))
184186
.then(res => done(res))
185187
.catch(err => {
186188
assert.equal(err.errCode, retCode.errorCode.BOUNDARY_TOTAL_CLOSED_LISTS);
@@ -189,16 +191,15 @@ describe('Validations for LU content (based on LUIS boundaries)', function () {
189191
})
190192
})
191193

192-
// This test is commented out because it takes > 60s to complete.
193-
// it (`At most ${retCode.boundaryLimits.MAX_NUM_PHRASES_IN_ALL_PHRASE_LIST} phrases across all phrase lists`, function(done) {
194-
// luMerger.Build(new Array(new luObj(getMaxPhraseLists(0, retCode.boundaryLimits.MAX_NUM_PHRASES_IN_ALL_PHRASE_LIST), 'stdin', true)))
195-
// .then(res => done(res))
196-
// .catch(err => {
197-
// assert.equal(err.errCode, retCode.errorCode.BOUNDARY_TOTAL_PHRASES);
198-
// assert(err.text.includes(`At most ${retCode.boundaryLimits.MAX_NUM_PHRASES_IN_ALL_PHRASE_LIST} is allowed.`));
199-
// done();
200-
// })
201-
// })
194+
it (`At most ${retCode.boundaryLimits.MAX_NUM_PHRASES_IN_ALL_PHRASE_LIST} phrases across all phrase lists`, function(done) {
195+
LuisBuilder.fromLU(new Array(new luObj(getMaxPhraseLists(0, retCode.boundaryLimits.MAX_NUM_PHRASES_IN_ALL_PHRASE_LIST), 'stdin', true)))
196+
.then(res => done(res))
197+
.catch(err => {
198+
assert.equal(err.errCode, retCode.errorCode.BOUNDARY_TOTAL_PHRASES);
199+
assert(err.text.includes(`At most ${retCode.boundaryLimits.MAX_NUM_PHRASES_IN_ALL_PHRASE_LIST} is allowed.`));
200+
done();
201+
})
202+
})
202203

203204
})
204205

‎packages/lu/test/parser/lufile/parseFileContents.composeEntity.test.js‎

Lines changed: 32 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -117,26 +117,52 @@ $deviceTemperature:simple`;
117117
.catch(err => done(`Test failed - ${JSON.stringify(err)}`))
118118
});
119119

120-
it('Parser throws when a composite entity has different definition across two different .lu file', function(done){
121-
let luFile1Content = `$deviceTemperature : [child1; child2]`;
122-
let luFile2Content = `$deviceTemperature : [child3]`;
120+
it('Merge composite entity definition split across .lu files', function(done){
121+
let luFile1Content = `@ ml userDOB
122+
@ composite fooBar = [userDOB]`;
123+
let luFile2Content = `@ ml username
124+
@ composite fooBar = [username]`;
123125
parseFile(luFile1Content, false)
124126
.then(res1 => {
125127
parseFile(luFile2Content, false)
126128
.then(res2 => {
127129
try {
128130
let luisList = [res1.LUISJsonStructure, res2.LUISJsonStructure]
129-
collate(luisList)
130-
done(`Test fail! Did not throw when expected`)
131+
let finalConten = collate(luisList)
132+
assert.equal(finalConten.composites.length, 1)
133+
assert.equal(finalConten.composites[0].children.length, 2)
134+
done()
131135
} catch (error) {
132-
done()
136+
done(error)
133137
}
134138
})
135139
.catch(err => done(`Test failed - ${err}`))
136140
})
137141
.catch(err => done(`Test failed - ${err}`))
138142
});
139143

144+
it('composite entity definition can be split across .lu files', function(done) {
145+
let luFile1Content = `@ ml username`;
146+
let luFile2Content = `@ composite fooBar = [username]`;
147+
parseFile(luFile1Content, false)
148+
.then(res1 => {
149+
parseFile(luFile2Content, false)
150+
.then(res2 => {
151+
try {
152+
let luisList = [res1.LUISJsonStructure, res2.LUISJsonStructure]
153+
let finalConten = collate(luisList)
154+
assert.equal(finalConten.composites.length, 1)
155+
assert.equal(finalConten.composites[0].children.length, 1)
156+
done()
157+
} catch (error) {
158+
done(error)
159+
}
160+
})
161+
.catch(err => done(`Test failed - ${err}`))
162+
})
163+
.catch(err => done(`Test failed - ${err}`))
164+
})
165+
140166
it('Parser throws and correctly identifies a child without an explicit or implicit child entity definition [across .lu files]', function(done){
141167
let luFile1Content = `$deviceTemperature : [p1; child2]`;
142168
let luFile2Content = `# test

0 commit comments

Comments
 (0)