Skip to content
This repository was archived by the owner on Jan 15, 2025. It is now read-only.

Commit a020a62

Browse files
vishwacsenaChris McConnell
vishwacsena
and
Chris McConnell
authored
Fixes to LU format for phraselist handling as well introduction of new parser flags for collision handling for labels (#979)
* Fix phraselist import issue * Fixing overlap handling. * finalizing fix for collision detection. Co-authored-by: Chris McConnell <[email protected]>
1 parent 4f7c793 commit a020a62

File tree

5 files changed

+192
-7
lines changed

5 files changed

+192
-7
lines changed

‎packages/lu/src/parser/lufile/parseFileContents.js‎

Lines changed: 61 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1010,7 +1010,52 @@ const parseAndHandleSimpleIntentSection = async function (parsedContent, luResou
10101010
if (item.role && item.role !== '') {
10111011
utteranceEntity.role = item.role.trim();
10121012
}
1013-
if (!utteranceObject.entities.find(item => deepEqual(item, utteranceEntity))) utteranceObject.entities.push(utteranceEntity)
1013+
// detect overlap and respect OnAmbiguousLabels
1014+
let priorLabelFound = utteranceObject.entities.find(item => {
1015+
if (item.entity === utteranceEntity.entity) {
1016+
if (utteranceEntity.role === undefined || utteranceEntity.role === '') {
1017+
if (item.role === undefined || item.role === '') {
1018+
// do the labels overlap?
1019+
if ((item.startPos >= utteranceEntity.startPos && item.endPos <= utteranceEntity.endPos) ||
1020+
(utteranceEntity.startPos >= item.startPos && utteranceEntity.endPos <= item.endPos)) {
1021+
return true;
1022+
}
1023+
}
1024+
}
1025+
}
1026+
return false;
1027+
});
1028+
if (priorLabelFound === undefined) {
1029+
utteranceObject.entities.push(utteranceEntity)
1030+
} else {
1031+
if (!utteranceObject.entities.find(item => deepEqual(item, utteranceEntity)))
1032+
{
1033+
let overlapHandling = parsedContent.LUISJsonStructure.onAmbiguousLabels || 'takeLongestLabel';
1034+
switch (overlapHandling.toLowerCase()) {
1035+
case 'takefirst':
1036+
break;
1037+
case 'takelast':
1038+
priorLabelFound.startPos = utteranceEntity.startPos;
1039+
priorLabelFound.endPos = utteranceEntity.endPos;
1040+
break;
1041+
case 'throwanerror':
1042+
let oldUtterance = expandUtterance(utterance, priorLabelFound);
1043+
let newUtterance = expandUtterance(utterance, utteranceEntity);
1044+
let errorMsg = `[Error] Duplicate overlapping labels found for entity '${priorLabelFound.name}' for Intent '${priorLabelFound.intent}'.\n 1. ${oldUtterance}\n 2. ${newUtterance}`;
1045+
let error = BuildDiagnostic({
1046+
message: errorMsg,
1047+
range: utteranceAndEntities.range
1048+
})
1049+
throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString(), [error]));
1050+
default:
1051+
// take the longest label
1052+
if ((utteranceEntity.startPos >= priorLabelFound.startPos) && (utteranceEntity.endPos >= priorLabelFound.endPos)) {
1053+
priorLabelFound.startPos = utteranceEntity.startPos;
1054+
priorLabelFound.endPos = utteranceEntity.endPos;
1055+
}
1056+
}
1057+
}
1058+
}
10141059
});
10151060
}
10161061

@@ -1035,6 +1080,14 @@ const parseAndHandleSimpleIntentSection = async function (parsedContent, luResou
10351080
}
10361081
}
10371082

1083+
const expandUtterance = function(utterance, entity)
1084+
{
1085+
let utteranceSplit = (utterance || '').split('');
1086+
utteranceSplit[entity.startPos] = `{@${entity.entity}=${utteranceSplit[entity.startPos]}`;
1087+
utteranceSplit[entity.endPos] = `${utteranceSplit[entity.endPos]}}`;
1088+
return utteranceSplit.join('');
1089+
}
1090+
10381091
const isChildEntity = function(entity, entitiesFound) {
10391092
// return true if the current entity is contained within a parent bounding label
10401093
return (entitiesFound || []).find(item => item.entity !== entity.entity && entity.startPos >= item.startPos && entity.endPos <= item.endPos) === undefined ? false : true;
@@ -1902,7 +1955,7 @@ const parseAndHandleModelInfoSection = function (parsedContent, luResource, log)
19021955
if (modelInfos && modelInfos.length > 0) {
19031956
for (const modelInfo of modelInfos) {
19041957
let line = modelInfo.ModelInfo
1905-
let kvPair = line.split(/@(app|kb|intent|entity|enableSections|enableMergeIntents|patternAnyEntity).(.*)=/g).map(item => item.trim());
1958+
let kvPair = line.split(/@(app|kb|intent|entity|enableSections|enableMergeIntents|patternAnyEntity|parser).(.*)=/g).map(item => item.trim());
19061959

19071960
// avoid to throw invalid model info when meeting enableSections info which is handled in luParser.js
19081961
if (kvPair[1] === 'enableSections') continue
@@ -1927,7 +1980,12 @@ const parseAndHandleModelInfoSection = function (parsedContent, luResource, log)
19271980
if(hasError) {
19281981
continue;
19291982
}
1930-
1983+
if (kvPair[1].toLowerCase() === 'parser') {
1984+
if (kvPair[2].toLowerCase().startsWith('onambiguouslabels')) {
1985+
parsedContent.LUISJsonStructure.onAmbiguousLabels = kvPair[3].toLowerCase() || 'takeLongestLabel';
1986+
continue;
1987+
}
1988+
}
19311989
if (kvPair[1].toLowerCase() === 'app') {
19321990
if (kvPair[2].toLowerCase().startsWith('settings')) {
19331991
let settingsRegExp = /^settings.(?<property>.*?$)/gmi;

‎packages/lu/src/parser/luis/luisCollate.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ const cleanupEntities = function(luisObject) {
7575
if (consolidatedList.find(e => e.name == item.name) !== undefined) idxToRemove.push(idx);
7676
})
7777
idxToRemove.sort((a, b) => a-b).forEach(idx => luisObject.entities.splice(idx, 1))
78+
delete luisObject.onAmbiguousLabels;
7879
}
7980

8081
const mergeResultsWithHash = function (blob, finalCollection, type, hashTable) {

‎packages/lu/src/parser/utils/helpers.js‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -239,7 +239,11 @@ const updateToV7 = function(finalLUISJSON) {
239239
}
240240
});
241241
});
242-
(finalLUISJSON.entities || []).forEach(entity => transformAllEntityConstraintsToFeatures(entity));
242+
let phraseListsNamesInFinal = [];
243+
(finalLUISJSON.phraselists || []).forEach((item) => {
244+
if (!phraseListsNamesInFinal.includes(item.name)) phraseListsNamesInFinal.push(item.name);
245+
});
246+
(finalLUISJSON.entities || []).forEach(entity => transformAllEntityConstraintsToFeatures(entity, phraseListsNamesInFinal));
243247
(finalLUISJSON.intents || []).forEach(intent => addIsRequiredProperty(intent));
244248
// do we have nDepthEntities?
245249
let nDepthEntityExists = (finalLUISJSON.entities || []).find(x => x.children !== undefined && Array.isArray(x.children) && x.children.length !== 0);
@@ -489,8 +493,8 @@ const objectSortByStartPos = function (objectArray) {
489493
return ObjectByStartPos
490494
}
491495

492-
const transformAllEntityConstraintsToFeatures = function(entity) {
493-
addIsRequiredProperty(entity);
496+
const transformAllEntityConstraintsToFeatures = function(entity, phraseListsInFinal) {
497+
addIsRequiredProperty(entity, phraseListsInFinal);
494498
if (entity.hasOwnProperty("instanceOf") && entity.instanceOf !== null) {
495499
if (entity.hasOwnProperty("features") && Array.isArray(entity.features)) {
496500
let featureFound = (entity.features || []).find(i => i.modelName == entity.instanceOf);
@@ -513,10 +517,15 @@ const transformAllEntityConstraintsToFeatures = function(entity) {
513517
entity.children.forEach(c => transformAllEntityConstraintsToFeatures(c));
514518
};
515519

516-
const addIsRequiredProperty = function(item) {
520+
const addIsRequiredProperty = function(item, phraseListInFinal = []) {
517521
(item.features || []).forEach(feature => {
518522
if (feature.isRequired === undefined)
519523
feature.isRequired = false;
524+
if (feature.modelName !== undefined && phraseListInFinal.includes(feature.modelName))
525+
{
526+
feature.featureName = feature.modelName;
527+
delete feature.modelName;
528+
}
520529
delete feature.featureType;
521530
delete feature.modelType;
522531
});

‎packages/lu/test/parser/lufile/appKbMetaData.test.js‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,5 +121,97 @@ describe('App/ Kb meta data information', function () {
121121
.catch(err => done(err))
122122
})
123123

124+
it ('Parser instruction to handle OnAmbiguousLabels=takeFirst is handled correctly', function(done) {
125+
let testLU = `
126+
> !# @parser.OnAmbiguousLabels = takeFirst
124127
128+
# test
129+
- {@cuisineEntity=chinese cuisine}
130+
131+
# test
132+
- {@cuisineEntity=chinese} cuisine
133+
`;
134+
parseFile.parseFile(testLU)
135+
.then(res => {
136+
let parsedLuis = new luis(res.LUISJsonStructure);
137+
assert.equal(parsedLuis.utterances[0].entities[0].startPos, 0);
138+
assert.equal(parsedLuis.utterances[0].entities[0].endPos, 14);
139+
done();
140+
})
141+
.catch(err => done(err))
142+
})
143+
144+
it ('Parser instruction to handle OnAmbiguousLabels=takeLast is handled correctly', function(done) {
145+
let testLU = `
146+
> !# @parser.OnAmbiguousLabels = takeLast
147+
148+
# test
149+
- {@cuisineEntity=chinese cuisine}
150+
151+
# test
152+
- {@cuisineEntity=chinese} cuisine
153+
`;
154+
parseFile.parseFile(testLU)
155+
.then(res => {
156+
let parsedLuis = new luis(res.LUISJsonStructure);
157+
assert.equal(parsedLuis.utterances[0].entities[0].startPos, 0);
158+
assert.equal(parsedLuis.utterances[0].entities[0].endPos, 6);
159+
done();
160+
})
161+
.catch(err => done(err))
162+
})
163+
164+
165+
it ('Parser instruction to handle OnAmbiguousLabels=throwAnError is handled correctly', function(done) {
166+
let testLU = `
167+
> !# @parser.OnAmbiguousLabels = throwAnError
168+
169+
# test
170+
- {@cuisineEntity=chinese cuisine}
171+
172+
# test
173+
- {@cuisineEntity=chinese} cuisine
174+
`;
175+
parseFile.parseFile(testLU)
176+
.then(res => done(res))
177+
.catch(err => done())
178+
})
179+
180+
it ('Parser instruction to handle OnAmbiguousLabels=takeLongestLabel is handled correctly', function(done) {
181+
let testLU = `
182+
> !# @parser.OnAmbiguousLabels = takeLongestLabel
183+
184+
# test
185+
- {@cuisineEntity=chinese cuisine}
186+
187+
# test
188+
- {@cuisineEntity=chinese} cuisine
189+
`;
190+
parseFile.parseFile(testLU)
191+
.then(res => {
192+
let parsedLuis = new luis(res.LUISJsonStructure);
193+
assert.equal(parsedLuis.utterances[0].entities[0].startPos, 0);
194+
assert.equal(parsedLuis.utterances[0].entities[0].endPos, 14);
195+
done();
196+
})
197+
.catch(err => done(err))
198+
})
199+
200+
it ('Parser defaults to OnAmbiguousLabels=takeLongestLabel', function(done) {
201+
let testLU = `
202+
# test
203+
- {@cuisineEntity=chinese cuisine}
204+
205+
# test
206+
- {@cuisineEntity=chinese} cuisine
207+
`;
208+
parseFile.parseFile(testLU)
209+
.then(res => {
210+
let parsedLuis = new luis(res.LUISJsonStructure);
211+
assert.equal(parsedLuis.utterances[0].entities[0].startPos, 0);
212+
assert.equal(parsedLuis.utterances[0].entities[0].endPos, 14);
213+
done();
214+
})
215+
.catch(err => done(err))
216+
})
125217
});

‎packages/lu/test/parser/lufile/luQnAFileReference.test.js‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -169,6 +169,23 @@ describe('Deep reference tests', function() {
169169
})
170170
})
171171

172+
it('Phrase lists defined as feautre to an entity are handled correctly when the phrase list definition is imported', function(done) {
173+
let luContent = `
174+
[import phraselist](phraselists)
175+
176+
# test
177+
- utterance
178+
179+
@ ml test1 usesFeature phraseList1
180+
`;
181+
luisBuilder.fromLUAsync([new luObj(luContent, new luOptions('main.lu', true))], findLuFiles)
182+
.then(res => {
183+
assert.equal(res.entities[0].features[0].featureName, "phraseList1");
184+
done()
185+
})
186+
.catch(err => done(err))
187+
})
188+
172189
it('Fix for BF-CLI #797 - deep references to phrase lists are handled correctly', function(done) {
173190
let luContent = `
174191
@ phraselist pl_1(interchangeable) =
@@ -318,6 +335,14 @@ const findLuFiles = async function(srcId, idsToFind){
318335
let retPayload = [];
319336
idsToFind.forEach(ask => {
320337
switch(ask.filePath) {
338+
case 'phraselists':
339+
retPayload.push(new luObj(`
340+
@ phraselist phraseList1 =
341+
- one
342+
- two
343+
- three
344+
`, new luOptions(ask.filePath)));
345+
break;
321346
case './3nDepth.lu':
322347
retPayload.push(new luObj(`
323348
@ ml AddToName r2 =

0 commit comments

Comments
 (0)