Skip to content
This repository was archived by the owner on Jan 15, 2025. It is now read-only.

Commit c0e8483

Browse files
vishwacsenaVishwac Sena KannanVishwac Sena KannanVishwac Sena Kannan
authored
Various fixes to bf-lu (#813)
* various fixes. * fixes * updates * updates * fix for luis:build issue. * updates for nDepth reference resolver. * Update luis:build to create result using luis() Co-authored-by: Vishwac Sena Kannan <[email protected]> Co-authored-by: Vishwac Sena Kannan <[email protected]> Co-authored-by: Vishwac Sena Kannan <[email protected]>
1 parent dd36356 commit c0e8483

File tree

10 files changed

+686
-175
lines changed

10 files changed

+686
-175
lines changed

‎packages/lu/src/parser/lubuild/builder.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ const retCode = require('./../utils/enums/CLI-errors')
1717
const exception = require('./../utils/exception')
1818
const LuisBuilderVerbose = require('./../luis/luisCollate')
1919
const LuisBuilder = require('./../luis/luisBuilder')
20+
const Luis = require('./../luis/luis')
2021
const LUOptions = require('./../lu/luOptions')
2122
const Content = require('./../lu/lu')
2223
const recognizerType = require('./../utils/enums/recognizertypes')
@@ -55,9 +56,11 @@ export class Builder {
5556

5657
let fileContent = ''
5758
let result
59+
let luisObj
5860
try {
5961
result = await LuisBuilderVerbose.build(luFiles, true, fileCulture)
60-
fileContent = result.parseToLuContent()
62+
luisObj = new Luis(result)
63+
fileContent = luisObj.parseToLuContent()
6164
} catch (err) {
6265
if (err.source) {
6366
err.text = `Invalid LU file ${err.source}: ${err.text}`

‎packages/lu/src/parser/lufile/parseFileContents.js‎

Lines changed: 53 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -184,10 +184,6 @@ const parseLuAndQnaWithAntlr = async function (parsedContent, fileContent, log,
184184

185185
validateNDepthEntities(parsedContent.LUISJsonStructure.entities, parsedContent.LUISJsonStructure.flatListOfEntityAndRoles, parsedContent.LUISJsonStructure.intents);
186186

187-
// This nDepth child might have been labelled, if so, remove the duplicate simple entity.
188-
// If utterances have this child, then all parents must be included in the label
189-
updateModelBasedOnNDepthEntities(parsedContent.LUISJsonStructure.utterances, parsedContent.LUISJsonStructure.entities);
190-
191187
// Update intent and entities with phrase lists if any
192188
updateIntentAndEntityFeatures(parsedContent.LUISJsonStructure);
193189

@@ -234,23 +230,30 @@ const updateModelBasedOnNDepthEntities = function(utterances, entities) {
234230
entityFoundInMaster.push({id: idx, entityRoot: entity, path: entityPath});
235231
}
236232
});
233+
let isParentLabelled = false;
237234
entityFoundInMaster.forEach(entityInMaster => {
238235
let splitPath = entityInMaster.path.split("/").filter(item => item.trim() !== "");
239236
if (entityFoundInMaster.length > 1 && splitPath.length === 0 && (!entityInMaster.entityRoot.children || entityInMaster.entityRoot.children.length === 0)) {
240237
// this child needs to be removed. Note: There can only be at most one more entity due to utterance validation rules.
241238
entities.splice(entityInMaster.id, 1);
242239
} else {
243-
splitPath.reverse().forEach(parent => {
244-
// Ensure each parent is also labelled in this utterance
245-
let parentLabelled = utterance.entities.find(entityUtt => entityUtt.entity == parent);
246-
if (!parentLabelled) {
247-
const errorMsg = `Every child entity labelled in an utterance must have its parent labelled in that utterance. Parent "${parent}" for child "${entityInUtterance.entity}" is not labelled in utterance "${utterance.text}" for intent "${utterance.intent}".`;
248-
const error = BuildDiagnostic({
249-
message: errorMsg
250-
});
251-
throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString(), [error]));
252-
}
253-
})
240+
if (isParentLabelled === false) {
241+
let rSplitPath = splitPath.reverse();
242+
rSplitPath.splice(0, 1);
243+
rSplitPath.forEach(parent => {
244+
// Ensure each parent is also labelled in this utterance
245+
let parentLabelled = utterance.entities.find(entityUtt => entityUtt.entity == parent);
246+
if (!parentLabelled) {
247+
const errorMsg = `Every child entity labelled in an utterance must have its parent labelled in that utterance. Parent "${parent}" for child "${entityInUtterance.entity}" is not labelled in utterance "${utterance.text}" for intent "${utterance.intent}".`;
248+
const error = BuildDiagnostic({
249+
message: errorMsg
250+
});
251+
throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString(), [error]));
252+
} else {
253+
isParentLabelled = true;
254+
}
255+
})
256+
}
254257
}
255258
})
256259
})
@@ -876,37 +879,43 @@ const parseAndHandleSimpleIntentSection = function (parsedContent, luResource) {
876879
if (entity.role) {
877880
addItemOrRoleIfNotPresent(parsedContent.LUISJsonStructure, LUISObjNameEnum.CLOSEDLISTS, entity.entity, [entity.role.trim()]);
878881
} else {
879-
let errorMsg = `${entity.entity} has been defined as a LIST entity type. It cannot be explicitly included in a labelled utterance unless the label includes a role.`;
880-
let error = BuildDiagnostic({
881-
message: errorMsg,
882-
context: utteranceAndEntities.context
883-
});
884-
885-
throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString(), [error]));
882+
if (!isChildEntity(entity, entitiesFound)) {
883+
let errorMsg = `${entity.entity} has been defined as a LIST entity type. It cannot be explicitly included in a labelled utterance unless the label includes a role.`;
884+
let error = BuildDiagnostic({
885+
message: errorMsg,
886+
context: utteranceAndEntities.context
887+
});
888+
889+
throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString(), [error]));
890+
}
886891
}
887892
} else if (prebuiltExists !== undefined) {
888893
if (entity.role) {
889894
addItemOrRoleIfNotPresent(parsedContent.LUISJsonStructure, LUISObjNameEnum.PREBUILT, entity.entity, [entity.role.trim()]);
890895
} else {
891-
let errorMsg = `${entity.entity} has been defined as a PREBUILT entity type. It cannot be explicitly included in a labelled utterance unless the label includes a role.`;
892-
let error = BuildDiagnostic({
893-
message: errorMsg,
894-
context: utteranceAndEntities.context
895-
});
896-
897-
throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString(), [error]));
896+
if (!isChildEntity(entity, entitiesFound)) {
897+
let errorMsg = `${entity.entity} has been defined as a PREBUILT entity type. It cannot be explicitly included in a labelled utterance unless the label includes a role.`;
898+
let error = BuildDiagnostic({
899+
message: errorMsg,
900+
context: utteranceAndEntities.context
901+
});
902+
903+
throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString(), [error]));
904+
}
898905
}
899906
} else if (regexExists !== undefined) {
900907
if (entity.role) {
901908
addItemOrRoleIfNotPresent(parsedContent.LUISJsonStructure, LUISObjNameEnum.REGEX, entity.entity, [entity.role.trim()]);
902909
} else {
903-
let errorMsg = `${entity.entity} has been defined as a Regex entity type. It cannot be explicitly included in a labelled utterance unless the label includes a role.`;
904-
let error = BuildDiagnostic({
905-
message: errorMsg,
906-
context: utteranceAndEntities.context
907-
});
908-
909-
throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString(), [error]));
910+
if (!isChildEntity(entity, entitiesFound)) {
911+
let errorMsg = `${entity.entity} has been defined as a Regex entity type. It cannot be explicitly included in a labelled utterance unless the label includes a role.`;
912+
let error = BuildDiagnostic({
913+
message: errorMsg,
914+
context: utteranceAndEntities.context
915+
});
916+
917+
throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString(), [error]));
918+
}
910919
}
911920
} else if (patternAnyExists !== undefined) {
912921
if (entity.value != '') {
@@ -981,6 +990,11 @@ const parseAndHandleSimpleIntentSection = function (parsedContent, luResource) {
981990
}
982991
}
983992

993+
const isChildEntity = function(entity, entitiesFound) {
994+
// return true if the current entity is contained within a parent bounding label
995+
return (entitiesFound || []).find(item => item.entity !== entity.entity && entity.startPos >= item.startPos && entity.endPos <= item.endPos) === undefined ? false : true;
996+
}
997+
984998
/**
985999
* Helper function to get entity type based on a name match
9861000
* @param {String} entityName name of entity to look up type for.
@@ -1161,6 +1175,7 @@ const handleNDepthEntity = function(parsedContent, entityName, entityRoles, enti
11611175
const SPACEASTABS = 4;
11621176
addItemOrRoleIfNotPresent(parsedContent.LUISJsonStructure, LUISObjNameEnum.ENTITIES, entityName, entityRoles);
11631177
let rootEntity = parsedContent.LUISJsonStructure.entities.find(item => item.name == entityName);
1178+
rootEntity.explicitlyAdded = true;
11641179
let defLine = line.start.line;
11651180
let baseTabLevel = 0;
11661181
let entityIdxByLevel = [];
@@ -1640,6 +1655,8 @@ const parseAndHandleEntitySection = function (parsedContent, luResource, log, lo
16401655
} else if (entityType.toLowerCase() === 'simple') {
16411656
// add this to entities if it doesnt exist
16421657
addItemOrRoleIfNotPresent(parsedContent.LUISJsonStructure, LUISObjNameEnum.ENTITIES, entityName, entityRoles);
1658+
let rootEntity = parsedContent.LUISJsonStructure.entities.find(item => item.name == entityName);
1659+
rootEntity.explicitlyAdded = true;
16431660
} else if (entityType.endsWith('=')) {
16441661
// is this qna maker alterations list?
16451662
if (entityType.includes(PARSERCONSTS.QNAALTERATIONS)) {

‎packages/lu/src/parser/lufile/simpleIntentSection.js‎

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -50,13 +50,24 @@ class SimpleIntentSection {
5050
}
5151

5252
for (const normalIntentStr of parseTree.intentDefinition().intentBody().normalIntentBody().normalIntentString()) {
53-
let utteranceAndEntities = visitor.visitNormalIntentStringContext(normalIntentStr);
54-
utteranceAndEntities.context = normalIntentStr;
55-
utteranceAndEntitiesMap.push(utteranceAndEntities);
56-
utteranceAndEntities.errorMsgs.forEach(errorMsg => errors.push(BuildDiagnostic({
57-
message: errorMsg,
58-
context: normalIntentStr
59-
})))
53+
let utteranceAndEntities;
54+
try {
55+
utteranceAndEntities = visitor.visitNormalIntentStringContext(normalIntentStr);
56+
}
57+
catch (err) {
58+
errors.push(BuildDiagnostic({
59+
message: "Invalid utterance definition found. Did you miss a '{' or '}'?",
60+
context: normalIntentStr
61+
}))
62+
};
63+
if (utteranceAndEntities !== undefined) {
64+
utteranceAndEntities.context = normalIntentStr;
65+
utteranceAndEntitiesMap.push(utteranceAndEntities);
66+
utteranceAndEntities.errorMsgs.forEach(errorMsg => errors.push(BuildDiagnostic({
67+
message: errorMsg,
68+
context: normalIntentStr
69+
})));
70+
}
6071
}
6172
}
6273

‎packages/lu/src/parser/luis/luis.js‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ class Luis {
3030

3131
parseToLuContent(){
3232
helpers.checkAndUpdateVersion(this)
33+
helpers.cleanUpExplicitEntityProperty(this)
3334
return luConverter(this)
3435
}
3536

@@ -54,6 +55,12 @@ const initialize = function(instance, LuisJSON) {
5455
if (Object.keys(LuisJSON).includes(regexEntities)) instance[regexEntities] = LuisJSON[regexEntities];
5556

5657
settingsAndTokenizerCheck(instance, LuisJSON)
58+
59+
initializeEntities(instance)
60+
}
61+
62+
const initializeEntities = function (instance) {
63+
(instance.entities || []).forEach(e => e.explicitlyAdded = true)
5764
}
5865

5966
const settingsAndTokenizerCheck = function(instance, LuisJSON) {

‎packages/lu/src/parser/luis/luisCollate.js‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ const Luis = require('./luis')
55
const helpers = require('./../utils/helpers')
66
const mergeLuFiles = require('./../lu/luMerger').Build
77
const exception = require('./../utils/exception')
8+
const retCode = require('../utils/enums/CLI-errors')
89

910
/**
1011
* Builds a Luis instance from a Lu list.
@@ -41,6 +42,7 @@ const collate = function(luisList) {
4142
let blob = luisList[i]
4243
mergeResults(blob, luisObject, LUISObjNameEnum.INTENT);
4344
mergeResults(blob, luisObject, LUISObjNameEnum.ENTITIES);
45+
mergeNDepthEntities(blob.entities, luisObject.entities);
4446
mergeResults_closedlists(blob, luisObject, LUISObjNameEnum.CLOSEDLISTS);
4547
mergeResults(blob, luisObject, LUISObjNameEnum.PATTERNANYENTITY);
4648
mergeResultsWithHash(blob, luisObject, LUISObjNameEnum.UTTERANCE, hashTable);
@@ -52,6 +54,7 @@ const collate = function(luisList) {
5254
buildPatternAny(blob, luisObject)
5355
}
5456
helpers.checkAndUpdateVersion(luisObject)
57+
helpers.cleanUpExplicitEntityProperty(luisObject)
5558
cleanupEntities(luisObject)
5659
return luisObject
5760
}
@@ -102,7 +105,77 @@ const mergeResultsWithHash = function (blob, finalCollection, type, hashTable) {
102105
}
103106
});
104107
}
108+
const mergeNDepthEntities = function (blob, finalCollection) {
109+
let nDepthInBlob = (blob || []).filter(x => x.children !== undefined && Array.isArray(x.children) && x.children.length !== 0);
110+
if (nDepthInBlob === undefined) return;
111+
nDepthInBlob.forEach(item => {
112+
let itemExistsInFinal = (finalCollection || []).find(x => x.name == item.name);
113+
if (itemExistsInFinal === undefined) {
114+
finalCollection.push(item);
115+
} else {
116+
// de-dupe and merge roles
117+
(item.roles || []).forEach(r => {
118+
if (itemExistsInFinal.roles === undefined) {
119+
itemExistsInFinal.roles = [r];
120+
} else {
121+
if (!itemExistsInFinal.roles.includes(r)) {
122+
itemExistsInFinal.roles.push(r);
123+
}
124+
}
125+
})
126+
// de-dupe and merge children
127+
if (item.children !== undefined && Array.isArray(item.children) && item.children.length !== 0) {
128+
recursivelyMergeChildrenAndFeatures(item.children, itemExistsInFinal.children)
129+
}
130+
}
131+
})
132+
}
105133

134+
const recursivelyMergeChildrenAndFeatures = function(srcChildren, tgtChildren) {
135+
if (tgtChildren === undefined || !Array.isArray(tgtChildren) || tgtChildren.length === 0) {
136+
tgtChildren = srcChildren;
137+
return;
138+
}
139+
(srcChildren || []).forEach(item => {
140+
// find child in tgt
141+
let itemExistsInFinal = (tgtChildren || []).find(x => x.name == item.name);
142+
if (itemExistsInFinal === undefined) {
143+
tgtChildren.push(item);
144+
} else {
145+
// merge features
146+
if (item.features !== undefined && item.features.length !== 0) {
147+
// merge and verify type
148+
let typeForFinalItem = (itemExistsInFinal.features || []).find(t => t.isRequired == true);
149+
let typeForItem = (item.features || []).find(t1 => t1.isRequired == true);
150+
if (typeForFinalItem !== undefined) {
151+
if (typeForItem !== undefined) {
152+
if (typeForFinalItem.modelName !== typeForItem.modelName) {
153+
throw new exception(retCode.errorCode.INVALID_REGEX_ENTITY, `Child entity ${item.name} does not have consistent type definition. Please verify all definitions for this entity.`)
154+
}
155+
}
156+
}
157+
item.features.forEach(f => {
158+
let featureInFinal = (itemExistsInFinal.features || []).find(itFea => {
159+
return ((itFea.featureName !== undefined && itFea.featureName == f.featureName) ||
160+
(itFea.modelName !== undefined && itFea.modelName == f.modelName))
161+
});
162+
if (featureInFinal === undefined) {
163+
itemExistsInFinal.features.push(f);
164+
} else {
165+
// throw if isRequired is not the same.
166+
if (featureInFinal.isRequired !== f.isRequired) {
167+
throw new exception(retCode.errorCode.INVALID_REGEX_ENTITY, `Feature ${f.featureName} does not have consistent definition for entity ${item.name}. Please verify all definitions for this feature for this entity.`)
168+
}
169+
}
170+
})
171+
}
172+
// de-dupe and merge children
173+
if (item.children !== undefined && Array.isArray(item.children) && item.children.length !== 0) {
174+
recursivelyMergeChildrenAndFeatures(item.children, itemExistsInFinal.children)
175+
}
176+
}
177+
})
178+
}
106179
/**
107180
* Helper function to merge item if it does not already exist
108181
*

0 commit comments

Comments
 (0)