Skip to content
This repository was archived by the owner on Jan 15, 2025. It is now read-only.

Commit c90e87e

Browse files
vishwacsenavishwacsena
authored
nDepth entity with features (#219)
* initial * updates * FI from master * delete from local * nDepth entity fixup * Update for PR comment
1 parent ecb6257 commit c90e87e

File tree

21 files changed

+612
-60
lines changed

21 files changed

+612
-60
lines changed

‎packages/lu/docs/lu-file-format.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,7 @@ The following LUIS prebuilt entity types are supported -
133133
- ordinalV2
134134
- percentage
135135
- personName
136-
- phoneNumber
136+
- phonenumber
137137
- temperature
138138
- url
139139
- datetime

‎packages/lu/src/commands/luis/convert.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,9 +50,9 @@ export default class LuisConvert extends Command {
5050
// Add headers to Luis Json
5151
if (isLu) {
5252
result.luis_schema_version = flags.schemaversion || result.luis_schema_version || '3.2.0'
53-
result.versionId = flags.versionId || result.versionId || '0.1'
53+
result.versionId = flags.versionid || result.versionId || '0.1'
5454
result.name = flags.name || result.name || ''
55-
result.desc = flags.desc || result.desc || ''
55+
result.desc = flags.description || result.desc || ''
5656
result.culture = flags.culture || result.culture || 'en-us'
5757
result.culture = result.culture.toLowerCase()
5858
result = JSON.stringify(result, null, 2)

‎packages/lu/src/parser/converters/luistoluconverter.js‎

Lines changed: 25 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -126,16 +126,13 @@ module.exports = {
126126
}
127127

128128
if(LUISJSON.model_features && LUISJSON.model_features.length >= 0) {
129-
fileContent += '> # Phrase list definitions' + NEWLINE + NEWLINE;
130-
LUISJSON.model_features.forEach(function(entity) {
131-
fileContent += `@ phraselist ${entity.name}${(entity.mode ? `(interchangeable)` : ``)}`;
132-
if (entity.words && entity.words !== '') {
133-
fileContent += ` = ${NEWLINE}\t- ${entity.words}`;
134-
}
135-
fileContent += NEWLINE + NEWLINE;
136-
});
137-
fileContent += NEWLINE;
129+
fileContent += handlePhraseLists(LUISJSON.model_features);
138130
}
131+
132+
if(LUISJSON.phraselists && LUISJSON.phraselists.length >= 0) {
133+
fileContent += handlePhraseLists(LUISJSON.phraselists);
134+
}
135+
139136
if(LUISJSON.closedLists && LUISJSON.closedLists.length >= 0){
140137
fileContent += '> # List entities' + NEWLINE + NEWLINE;
141138
LUISJSON.closedLists.forEach(function(ListItem) {
@@ -188,6 +185,24 @@ module.exports = {
188185
return fileContent;
189186
}
190187
}
188+
189+
/**
190+
* Helper to handle phrase lists both in the new and old property.
191+
* @param {Object[]} collection
192+
*/
193+
const handlePhraseLists = function(collection) {
194+
let fileContent = '> # Phrase list definitions' + NEWLINE + NEWLINE;
195+
collection.forEach(function(entity) {
196+
fileContent += `@ phraselist ${entity.name}${(entity.mode ? `(interchangeable)` : ``)}`;
197+
if (entity.words && entity.words !== '') {
198+
fileContent += ` = ${NEWLINE}\t- ${entity.words}`;
199+
}
200+
fileContent += NEWLINE + NEWLINE;
201+
});
202+
fileContent += NEWLINE;
203+
204+
return fileContent;
205+
}
191206
/**
192207
* Helper to add application inforamtion metadata
193208
* @param {Object} LUISJSON
@@ -234,7 +249,7 @@ const addNDepthChildDefinitions = function(childCollection, tabStop, fileContent
234249
myFileContent += addRolesAndFeatures(child);
235250
myFileContent += NEWLINE;
236251
if (child.children && child.children.length !== 0) {
237-
myFileContent += addNDepthChildDefinitions(child.children, ++tabStop, myFileContent);
252+
myFileContent += addNDepthChildDefinitions(child.children, tabStop + 1, myFileContent);
238253
}
239254
});
240255
return myFileContent;

‎packages/lu/src/parser/converters/lutoluisconverter.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -266,7 +266,7 @@ const checkAndUpdateVersion = function(finalLUISJSON) {
266266
(finalLUISJSON.intents || []).find(i => i.features) ||
267267
(finalLUISJSON.composites || []).find(i => i.features);
268268
if (v5DefFound) {
269-
finalLUISJSON.luis_schema_version = "5.0.0";
269+
finalLUISJSON.luis_schema_version = "6.0.0";
270270
if (finalLUISJSON.model_features && finalLUISJSON.model_features.length !== 0) {
271271
finalLUISJSON.phraselists = [];
272272
finalLUISJSON.model_features.forEach(item => finalLUISJSON.phraselists.push(Object.assign({}, item)));

‎packages/lu/src/parser/lufile/classes/filesToParse.js‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,4 @@ class FileToParse {
1616
}
1717
}
1818

19-
FileToParse.stringArrayToFileToParseList = function(obj) {
20-
if(Array.isArray(obj)) return obj.map(item => new FileToParse(item));
21-
}
22-
2319
module.exports = FileToParse;

‎packages/lu/src/parser/lufile/classes/hclasses.js‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -132,19 +132,21 @@ const readerObj = {
132132
}
133133
},
134134
featureToModel: class {
135-
constructor(name) {
135+
constructor(name, featureType) {
136136
this.featureName = name ? name : '';
137+
this.featureType = featureType ? featureType : '';
137138
}
138139
},
139140
modelToFeature: class {
140-
constructor(name) {
141+
constructor(name, featureType) {
141142
this.modelName = name ? name : '';
143+
this.modelType = featureType ? featureType : '';
142144
}
143145
},
144146
childEntity: class {
145147
constructor(name, instanceOf, context, children, features) {
146148
this.name = name ? name : '';
147-
this.instanceOf = instanceOf ? instanceOf : '';
149+
this.instanceOf = instanceOf ? instanceOf : null;
148150
this.children = children ? children : [];
149151
this.features = features ? features : '';
150152
this.context = context ? context : '';

‎packages/lu/src/parser/lufile/enums/luisbuiltintypes.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ module.exports = {
1616
"ordinalV2",
1717
"percentage",
1818
"personName",
19-
"phoneNumber",
19+
"phonenumber",
2020
"temperature",
2121
"url",
2222
"datetime"
@@ -114,7 +114,7 @@ module.exports = {
114114
"ordinalV2": null,
115115
"percentage": null,
116116
"personName": null,
117-
"phoneNumber": null,
117+
"phonenumber": null,
118118
"temperature": null,
119119
"url": null,
120120
"datetime": null

‎packages/lu/src/parser/lufile/parseFileContents.js‎

Lines changed: 94 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,18 @@ const featureTypeEnum = {
3131
featureToModel: 'modelName',
3232
modelToFeature: 'featureName'
3333
};
34+
const featureProperties = {
35+
entityFeatureToModel : {
36+
'simple': 'Entity Extractor',
37+
'list' : 'Closed List Entity Extractor',
38+
'prebuilt': 'Prebuilt Entity Extractor',
39+
'composite': 'Composite Entity Extractor',
40+
'regex': 'Regex Entity Extractor',
41+
'ml': 'Entity Extractor'
42+
},
43+
intentFeatureToModel: 'Intent Classifier',
44+
phraseListFeature: 'phraselist'
45+
}
3446
const INTENTTYPE = 'intent';
3547
const parseFileContentsModule = {
3648
/**
@@ -154,9 +166,73 @@ const parseLuAndQnaWithAntlr = async function (parsedContent, fileContent, log,
154166
}
155167

156168
validateNDepthEntities(parsedContent.LUISJsonStructure.entities, parsedContent.LUISJsonStructure.flatListOfEntityAndRoles, parsedContent.LUISJsonStructure.intents);
169+
170+
// This nDepth child might have been labelled, if so, remove the duplicate simple entity.
171+
// If utterances have this child, then all parents must be included in the label
172+
updateModelBasedOnNDepthEntities(parsedContent.LUISJsonStructure.utterances, parsedContent.LUISJsonStructure.entities);
173+
157174
if (parsedContent.LUISJsonStructure.flatListOfEntityAndRoles) delete parsedContent.LUISJsonStructure.flatListOfEntityAndRoles
158175

159176
}
177+
178+
/**
179+
* Helper to update final LUIS model based on labelled nDepth entities.
180+
* @param {Object []} utterances
181+
* @param {Object []} entities
182+
*/
183+
const updateModelBasedOnNDepthEntities = function(utterances, entities) {
184+
// filter to all utterances that have a labelled entity
185+
let utterancesWithLabels = utterances.filter(utterance => utterance.entities && utterance.entities.length !== 0);
186+
utterancesWithLabels.forEach(utterance => {
187+
utterance.entities.forEach(entityInUtterance => {
188+
// find this entity's root. There can be multiple and if there are, we need to delete the one that does not have children.
189+
let entityFoundInMaster = [];
190+
entities.forEach((entity, idx) => {
191+
if (entity.name == entityInUtterance.entity) {
192+
entityFoundInMaster.push({id: idx, entityRoot: entity, path: '/'});
193+
}
194+
let entityPath = findEntityPath(entity, entityInUtterance.entity, "");
195+
if (entityPath !== "") {
196+
entityFoundInMaster.push({id: idx, entityRoot: entity, path: entityPath});
197+
}
198+
});
199+
entityFoundInMaster.forEach(entityInMaster => {
200+
let splitPath = entityInMaster.path.split("/").filter(item => item.trim() !== "");
201+
if (entityFoundInMaster.length > 1 && splitPath.length === 0 && (!entityInMaster.entityRoot.children || entityInMaster.entityRoot.children.length === 0)) {
202+
// this child needs to be removed. Note: There can only be at most one more entity due to utterance validation rules.
203+
entities.splice(entityInMaster.id, 1);
204+
} else {
205+
splitPath.reverse().forEach(parent => {
206+
// Ensure each parent is also labelled in this utterance
207+
let parentLabelled = utterance.entities.find(entityUtt => entityUtt.entity == parent);
208+
if (!parentLabelled) {
209+
let errorMsg = `[ERROR]: Every child entity labelled in an utterance must have its parent labelled in that utterance. Parent "${parent}" for child "${entityInUtterance.entity}" is not labelled in utterance "${utterance.text}" for intent "${utterance.intent}".`;
210+
throw (new exception(retCode.errorCode.INVALID_INPUT, errorMsg));
211+
}
212+
})
213+
}
214+
})
215+
})
216+
})
217+
}
218+
/**
219+
* Helper function to recursively find the path to a child entity
220+
* @param {Object} obj
221+
* @param {String} entityName
222+
* @param {String} path
223+
*/
224+
const findEntityPath = function(obj, entityName, path) {
225+
path = path || "";
226+
var fullpath = "";
227+
if (obj.name === entityName) {
228+
return path;
229+
} else if (obj.children && obj.children.length !== 0) {
230+
obj.children.forEach(child => {
231+
fullpath = findEntityPath(child, entityName, path + "/" + obj.name + "/" + child.name) || fullpath;
232+
})
233+
}
234+
return fullpath;
235+
}
160236
/**
161237
* Helper function to validate and update nDepth entities
162238
* @param {Object[]} collection
@@ -188,21 +264,21 @@ const validateNDepthEntities = function(collection, entitiesAndRoles, intentsCol
188264
if (featureExists) {
189265
// is feature phrase list?
190266
if (featureExists.type == EntityTypeEnum.PHRASELIST) {
191-
child.features[idx] = new helperClass.featureToModel(feature);
267+
child.features[idx] = new helperClass.featureToModel(feature, featureProperties.phraseListFeature);
192268
featureHandled = true;
193269
} else if (featureExists.type == EntityTypeEnum.PATTERNANY) {
194270
let errorMsg = `[Error] line ${child.context.line}: Invalid child entity definition found. "${feature}" is of type "${EntityTypeEnum.PATTERNANY}" in child entity definition "${child.context.definition}". Child cannot include a usesFeature of type "${EntityTypeEnum.PATTERNANY}".`;
195271
throw (new exception(retCode.errorCode.INVALID_INPUT, errorMsg));
196272
} else {
197-
child.features[idx] = new helperClass.modelToFeature(feature);
273+
child.features[idx] = new helperClass.modelToFeature(feature, featureProperties.entityFeatureToModel[featureExists.type]);
198274
featureHandled = true;
199275
}
200276
}
201277
if (!featureHandled) {
202278
// find feature as intent
203279
let intentFeatureExists = intentsCollection.find(i => i.name == feature);
204280
if (intentFeatureExists) {
205-
child.features[idx] = new helperClass.modelToFeature(feature);
281+
child.features[idx] = new helperClass.modelToFeature(feature, featureProperties.intentFeatureToModel);
206282
featureHandled = true;
207283
} else {
208284
let errorMsg = `[Error] line ${child.context.line}: Invalid child entity definition found. No definition found for "${feature}" in child entity definition "${child.context.definition}". Features must be defined before they can be added to a child.`;
@@ -219,6 +295,10 @@ const validateNDepthEntities = function(collection, entitiesAndRoles, intentsCol
219295
if (child.context) {
220296
delete child.context
221297
}
298+
299+
if (child.features === "") {
300+
delete child.features
301+
}
222302
})
223303
};
224304
/**
@@ -263,7 +343,7 @@ const validateFeatureAssignment = function(srcItemType, srcItemName, tgtFeatureT
263343
* @param {String} featureType
264344
* @param {Object} line
265345
*/
266-
const addFeatures = function(tgtItem, feature, featureType, line) {
346+
const addFeatures = function(tgtItem, feature, featureType, line, featureProperties) {
267347
// target item cannot have the same name as the feature name
268348
if (tgtItem.name === feature) {
269349
// Item must be defined before being added as a feature.
@@ -278,17 +358,17 @@ const addFeatures = function(tgtItem, feature, featureType, line) {
278358
switch (featureType) {
279359
case featureTypeEnum.featureToModel: {
280360
if (tgtItem.features) {
281-
if (!featureAlreadyDefined) tgtItem.features.push(new helperClass.featureToModel(feature));
361+
if (!featureAlreadyDefined) tgtItem.features.push(new helperClass.featureToModel(feature, featureProperties));
282362
} else {
283-
tgtItem.features = new Array(new helperClass.featureToModel(feature));
363+
tgtItem.features = new Array(new helperClass.featureToModel(feature, featureProperties));
284364
}
285365
break;
286366
}
287367
case featureTypeEnum.modelToFeature: {
288368
if (tgtItem.features) {
289-
if (!featureAlreadyDefined) tgtItem.features.push(new helperClass.modelToFeature(feature));
369+
if (!featureAlreadyDefined) tgtItem.features.push(new helperClass.modelToFeature(feature, featureProperties));
290370
} else {
291-
tgtItem.features = new Array(new helperClass.modelToFeature(feature));
371+
tgtItem.features = new Array(new helperClass.modelToFeature(feature, featureProperties));
292372
}
293373
break;
294374
}
@@ -327,19 +407,19 @@ const parseFeatureSections = function(parsedContent, featuresToProcess) {
327407
if (entityExists.type === EntityTypeEnum.PHRASELIST) {
328408
// de-dupe and add features to intent.
329409
validateFeatureAssignment(section.Type, section.Name, entityExists.type, feature, section.ParseTree.newEntityLine());
330-
addFeatures(intentExists, feature, featureTypeEnum.featureToModel, section.ParseTree.newEntityLine());
410+
addFeatures(intentExists, feature, featureTypeEnum.featureToModel, section.ParseTree.newEntityLine(), featureProperties.phraseListFeature);
331411
// set enabledForAllModels on this phrase list
332412
let plEnity = parsedContent.LUISJsonStructure.model_features.find(item => item.name == feature);
333413
plEnity.enabledForAllModels = false;
334414
} else {
335415
// de-dupe and add model to intent.
336416
validateFeatureAssignment(section.Type, section.Name, entityExists.type, feature, section.ParseTree.newEntityLine());
337-
addFeatures(intentExists, feature, featureTypeEnum.modelToFeature, section.ParseTree.newEntityLine());
417+
addFeatures(intentExists, feature, featureTypeEnum.modelToFeature, section.ParseTree.newEntityLine(), featureProperties.entityFeatureToModel[entityExists.type]);
338418
}
339419
} else if (featureIntentExists) {
340420
// Add intent as a feature to another intent
341421
validateFeatureAssignment(section.Type, section.Name, INTENTTYPE, feature, section.ParseTree.newEntityLine());
342-
addFeatures(intentExists, feature, featureTypeEnum.modelToFeature, section.ParseTree.newEntityLine());
422+
addFeatures(intentExists, feature, featureTypeEnum.modelToFeature, section.ParseTree.newEntityLine(), featureProperties.intentFeatureToModel);
343423
} else {
344424
// Item must be defined before being added as a feature.
345425
let errorMsg = `Features must be defined before assigned to an intent. No definition found for feature "${feature}" in usesFeature definition for intent "${section.Name}"`;
@@ -374,19 +454,19 @@ const parseFeatureSections = function(parsedContent, featuresToProcess) {
374454
if (featureExists.type === EntityTypeEnum.PHRASELIST) {
375455
// de-dupe and add features to intent.
376456
validateFeatureAssignment(entityType, section.Name, featureExists.type, feature, section.ParseTree.newEntityLine());
377-
addFeatures(srcEntity, feature, featureTypeEnum.featureToModel, section.ParseTree.newEntityLine());
457+
addFeatures(srcEntity, feature, featureTypeEnum.featureToModel, section.ParseTree.newEntityLine(), featureProperties.phraseListFeature);
378458
// set enabledForAllModels on this phrase list
379459
let plEnity = parsedContent.LUISJsonStructure.model_features.find(item => item.name == feature);
380460
plEnity.enabledForAllModels = false;
381461
} else {
382462
// de-dupe and add model to intent.
383463
validateFeatureAssignment(entityType, section.Name, featureExists.type, feature, section.ParseTree.newEntityLine());
384-
addFeatures(srcEntity, feature, featureTypeEnum.modelToFeature, section.ParseTree.newEntityLine());
464+
addFeatures(srcEntity, feature, featureTypeEnum.modelToFeature, section.ParseTree.newEntityLine(), featureProperties.entityFeatureToModel[featureExists.type]);
385465
}
386466
} else if (featureIntentExists) {
387467
// Add intent as a feature to another intent
388468
validateFeatureAssignment(entityType, section.Name, INTENTTYPE, feature, section.ParseTree.newEntityLine());
389-
addFeatures(srcEntity, feature, featureTypeEnum.modelToFeature, section.ParseTree.newEntityLine());
469+
addFeatures(srcEntity, feature, featureTypeEnum.modelToFeature, section.ParseTree.newEntityLine(), featureProperties.intentFeatureToModel);
390470
} else {
391471
// Item must be defined before being added as a feature.
392472
let errorMsg = `Features must be defined before assigned to an entity. No definition found for feature "${feature}" in usesFeature definition for entity "${section.Name}"`;

0 commit comments

Comments
 (0)