Skip to content
This repository was archived by the owner on Jan 15, 2025. It is now read-only.

Commit b822399

Browse files
vishwacsenaVishwac Sena Kannan
vishwacsena
and
Vishwac Sena Kannan
authored
Updates to LU round tripping with luis_schema_version 7.0.0 (#730)
* Base functionality * Updates. * lu tests passing * luis tests updated * updates to functionality * all tests are passing for core changes * ability for prebuilt defs via @type name * resetting launch.json * removing bad file * Updates to tests. Co-authored-by: Vishwac Sena Kannan <vishwacsenakannan@ALLISONT6-PROBK.northamerica.corp.microsoft.com>
1 parent 5392092 commit b822399

File tree

51 files changed

+1423
-380
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

51 files changed

+1423
-380
lines changed

‎packages/lu/src/parser/lufile/classes/hclasses.js‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,18 @@ const readerObj = {
137137
this.features = features ? features : '';
138138
this.context = context ? context : '';
139139
}
140+
},
141+
entityFeature: class {
142+
constructor(name, isRequired) {
143+
this.modelName = name ? name : '';
144+
this.isRequired = isRequired ? isRequired : false;
145+
}
146+
},
147+
plFeature: class {
148+
constructor(name, isRequired) {
149+
this.featureName = name ? name : '';
150+
this.isRequired = isRequired ? isRequired : false;
151+
}
140152
}
141153
};
142154

‎packages/lu/src/parser/lufile/parseFileContents.js‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -288,7 +288,7 @@ const validateNDepthEntities = function(collection, entitiesAndRoles, intentsCol
288288
throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString(), [error]));
289289
}
290290
// base type can only be a list or regex or prebuilt.
291-
if (![EntityTypeEnum.LIST, EntityTypeEnum.REGEX, EntityTypeEnum.PREBUILT].includes(baseEntityFound.type)) {
291+
if (![EntityTypeEnum.LIST, EntityTypeEnum.REGEX, EntityTypeEnum.PREBUILT, EntityTypeEnum.ML].includes(baseEntityFound.type)) {
292292
let errorMsg = `Invalid child entity definition found. "${child.instanceOf}" is of type "${baseEntityFound.type}" in child entity definition "${child.context.definition}". Child cannot be only be an instance of "${EntityTypeEnum.LIST}, ${EntityTypeEnum.REGEX} or ${EntityTypeEnum.PREBUILT}.`;
293293
const error = BuildDiagnostic({
294294
message: errorMsg,
@@ -982,7 +982,10 @@ const parseAndHandleSimpleIntentSection = function (parsedContent, luResource) {
982982
*/
983983
const getEntityType = function(entityName, entities) {
984984
let entityFound = (entities || []).find(item => item.Name == entityName || item.Name == `${entityName}(interchangeable)`);
985-
return entityFound ? entityFound.Type : undefined;
985+
if (entityFound && entityFound.Type !== undefined) return entityFound.Type
986+
// see if this one of the prebuilt type
987+
if (builtInTypes.consolidatedList.includes(entityName)) return EntityTypeEnum.PREBUILT
988+
return undefined;
986989
};
987990

988991
/**

‎packages/lu/src/parser/luis/luConverter.js‎

Lines changed: 38 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -94,29 +94,34 @@ const parseUtterancesToLu = function(utterances, luisJSON){
9494
let text = utterance.text;
9595
let sortedEntitiesList = objectSortByStartPos(utterance.entities);
9696
let tokenizedText = text.split('');
97-
let nonCompositesInUtterance = sortedEntitiesList.filter(entity => luisJSON.composites.find(composite => composite.name == entity.entity) == undefined);
98-
nonCompositesInUtterance.forEach(entity => {
99-
if (entity.role !== undefined) {
100-
tokenizedText[parseInt(entity.startPos)] = `{@${entity.role}=${tokenizedText[parseInt(entity.startPos)]}`;
101-
} else {
102-
tokenizedText[parseInt(entity.startPos)] = `{@${entity.entity}=${tokenizedText[parseInt(entity.startPos)]}`;
103-
}
104-
tokenizedText[parseInt(entity.endPos)] += `}`;
105-
})
106-
let compositeEntitiesInUtterance = sortedEntitiesList.filter(entity => luisJSON.composites.find(composite => composite.name == entity.entity) != undefined);
107-
compositeEntitiesInUtterance.forEach(entity => {
108-
if (entity.role !== undefined) {
109-
tokenizedText[parseInt(entity.startPos)] = `{@${entity.role}=${tokenizedText[parseInt(entity.startPos)]}`;
110-
} else {
111-
tokenizedText[parseInt(entity.startPos)] = `{@${entity.entity}=${tokenizedText[parseInt(entity.startPos)]}`;
112-
}
113-
tokenizedText[parseInt(entity.endPos)] += `}`;
114-
})
115-
updatedText = tokenizedText.join('');
97+
// handle cases where we have both child as well as cases where more than one entity can have the same start position
98+
// if there are multiple entities in the same start position, then order them by composite, nDepth, regular entity
99+
getEntitiesByPositionList(sortedEntitiesList, tokenizedText);
100+
updatedText = tokenizedText.join('');
116101
}
117102
if(updatedText) fileContent += '- ' + updatedText + NEWLINE;
118-
});
119-
return fileContent
103+
});
104+
return fileContent
105+
}
106+
107+
const getEntitiesByPositionList = function(entitiesList, tokenizedText) {
108+
(entitiesList || []).forEach(entity => {
109+
// does this entity have child labels?
110+
(entity.children || []).forEach(child => {
111+
getEntitiesByPositionList(child.children, tokenizedText);
112+
updateTokenizedTextByEntity(tokenizedText, child);
113+
})
114+
updateTokenizedTextByEntity(tokenizedText, entity);
115+
})
116+
};
117+
118+
const updateTokenizedTextByEntity = function(tokenizedText, entity) {
119+
if (entity.role !== undefined) {
120+
tokenizedText[parseInt(entity.startPos)] = `{@${entity.role}=${tokenizedText[parseInt(entity.startPos)]}`;
121+
} else {
122+
tokenizedText[parseInt(entity.startPos)] = `{@${entity.entity}=${tokenizedText[parseInt(entity.startPos)]}`;
123+
}
124+
tokenizedText[parseInt(entity.endPos)] = tokenizedText[parseInt(entity.endPos)] + '}';
120125
}
121126

122127
const parsePredictedResultToLu = function(utterance, luisJSON){
@@ -379,8 +384,10 @@ const addNDepthChildDefinitions = function(childCollection, tabStop, fileContent
379384
(childCollection || []).forEach(child => {
380385
myFileContent += "".padStart(tabStop * 4, ' ');
381386
myFileContent += '- @ ';
382-
if (child.instanceOf) {
383-
myFileContent += child.instanceOf;
387+
// find constraint
388+
let constraint = (child.features || []).find(feature => feature.isRequired == true);
389+
if (constraint !== undefined) {
390+
myFileContent += constraint.modelName;
384391
} else {
385392
myFileContent += EntityTypeEnum.ML;
386393
}
@@ -414,7 +421,14 @@ const addRolesAndFeatures = function(entity) {
414421
let featuresList = new Array();
415422
entity.features.forEach(item => {
416423
if (item.featureName) featuresList.push(item.featureName);
417-
if (item.modelName) featuresList.push(item.modelName);
424+
if (item.modelName) {
425+
if (item.isRequired !== undefined) {
426+
if (item.isRequired !== true)
427+
featuresList.push(item.modelName);
428+
} else {
429+
featuresList.push(item.modelName);
430+
}
431+
}
418432
})
419433
if (featuresList.length > 0) {
420434
roleAndFeatureContent += ` ${featuresList.length > 1 ? `usesFeatures` : `usesFeature`} `;

‎packages/lu/src/parser/utils/helpers.js‎

Lines changed: 205 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ const exception = require('./exception');
99
const NEWLINE = require('os').EOL;
1010
const ANY_NEWLINE = /\r\n|\r|\n/g;
1111
const url = require('url');
12+
const hClasses = require('../lufile/classes/hclasses');
1213
const helpers = {
1314

1415
/**
@@ -163,36 +164,213 @@ const helpers = {
163164
if (!finalLUISJSON) {
164165
return
165166
}
166-
let v5DefFound = false;
167-
v5DefFound = (finalLUISJSON.entities || []).find(i => i.children || i.features) ||
168-
(finalLUISJSON.intents || []).find(i => i.features) ||
169-
(finalLUISJSON.composites || []).find(i => i.features) ||
170-
(finalLUISJSON.luis_schema_version === '6.0.0');
171-
if (v5DefFound) {
172-
finalLUISJSON.luis_schema_version = "6.0.0";
173-
if (finalLUISJSON.hasOwnProperty("model_features")) {
174-
if (finalLUISJSON.model_features !== undefined) {
175-
finalLUISJSON.phraselists = finalLUISJSON.phraselists || [];
176-
finalLUISJSON.model_features.forEach(item => {
177-
if (item.enabledForAllModels === undefined) item.enabledForAllModels = true
178-
finalLUISJSON.phraselists.push(Object.assign({}, item))
179-
});
180-
}
181-
delete finalLUISJSON.model_features;
182-
}
183-
(finalLUISJSON.composites || []).forEach(composite => {
184-
let children = composite.children;
185-
composite.children = [];
186-
children.forEach(c => {
187-
if (c.name === undefined) {
188-
composite.children.push({name : c})
189-
} else {
190-
composite.children.push(c)
191-
}
167+
updateToV7(finalLUISJSON);
168+
}
169+
};
170+
171+
module.exports = helpers;
172+
173+
const updateToV7 = function(finalLUISJSON) {
174+
let v7DefFound = false;
175+
v7DefFound = (finalLUISJSON.entities || []).find(i => i.children || i.features) ||
176+
(finalLUISJSON.intents || []).find(i => i.features) ||
177+
(finalLUISJSON.composites || []).find(i => i.features) ||
178+
(finalLUISJSON.luis_schema_version === '6.0.0' ||
179+
(finalLUISJSON.luis_schema_version === '7.0.0'));
180+
if (v7DefFound) {
181+
finalLUISJSON.luis_schema_version = "7.0.0";
182+
if (finalLUISJSON.hasOwnProperty("model_features")) {
183+
if (finalLUISJSON.model_features !== undefined) {
184+
finalLUISJSON.phraselists = finalLUISJSON.phraselists || [];
185+
finalLUISJSON.model_features.forEach(item => {
186+
if (item.enabledForAllModels === undefined)
187+
item.enabledForAllModels = true;
188+
finalLUISJSON.phraselists.push(Object.assign({}, item));
192189
});
190+
}
191+
delete finalLUISJSON.model_features;
192+
}
193+
(finalLUISJSON.composites || []).forEach(composite => {
194+
let children = composite.children;
195+
composite.children = [];
196+
children.forEach(c => {
197+
if (c.name === undefined) {
198+
composite.children.push({ name: c });
199+
}
200+
else {
201+
composite.children.push(c);
202+
}
203+
});
204+
});
205+
(finalLUISJSON.entities || []).forEach(entity => transformAllEntityConstraintsToFeatures(entity));
206+
(finalLUISJSON.intents || []).forEach(intent => addIsRequiredProperty(intent));
207+
transformUtterancesWithNDepthEntities(finalLUISJSON)
208+
}
209+
}
210+
211+
const constructEntityParentTree = function(entityCollection, entityParentTree, curPath)
212+
{
213+
entityCollection.forEach(entity => {
214+
if (entity.children !== undefined && Array.isArray(entity.children) && entity.children.length !== 0) {
215+
constructEntityParentTree(entity.children, entityParentTree, curPath.concat(entity.name));
216+
}
217+
updateTreeWithNode(curPath, entity.name, entityParentTree)
218+
})
219+
}
220+
const updateTreeWithNode = function(curPath, entityName, entityParentTree) {
221+
let revPath = JSON.parse(JSON.stringify(curPath.reverse()));
222+
if (entityParentTree[entityName] === undefined) {
223+
entityParentTree[entityName] = [revPath];
224+
}
225+
else {
226+
if (entityParentTree[entityName].find(item => item.join('->') == revPath.join('->')) === undefined)
227+
entityParentTree[entityName].push(revPath);
228+
}
229+
curPath.reverse();
230+
}
231+
232+
const transformUtterancesWithNDepthEntities = function (finalLUISJSON) {
233+
let entityParentTree = {};
234+
const curPath = ["$root$"];
235+
constructEntityParentTree(finalLUISJSON.entities, entityParentTree, curPath);
236+
finalLUISJSON.utterances.forEach(utt => {
237+
if (utt.entities !== undefined && Array.isArray(utt.entities) && utt.entities.length !== 0) {
238+
// sort all entities by start and end position
239+
utt.entities = objectSortByStartPos(utt.entities)
240+
let entityIdsToRemove = [];
241+
utt.entities.forEach((item, idx) => {
242+
// find the immediate parents of this entity
243+
// if the enity has a role, find by that
244+
let entityToFind = item.role || item.entity;
245+
if (isRootEntity(entityToFind, finalLUISJSON.entities)) return;
246+
247+
if (entityParentTree[entityToFind] === undefined) {
248+
return;
249+
}
250+
let parentPathsForEntity = entityParentTree[entityToFind];
251+
let parentIdx = [];
252+
parentPathsForEntity.forEach(path => {
253+
utt.entities.find((i, id) => {
254+
if (i.entity === path[0] && i.startPos <= item.startPos && i.endPos >= item.endPos) {
255+
parentIdx.push(id);
256+
}
257+
})
258+
})
259+
if (parentIdx.length !== 0) {
260+
parentIdx.forEach(id => {
261+
if (item.role !== undefined) {
262+
item.entity = item.role;
263+
delete item.role;
264+
}
265+
if (utt.entities[id].children === undefined) {
266+
utt.entities[id].children = [item]
267+
} else {
268+
utt.entities[id].children.push(item);
269+
}
270+
})
271+
entityIdsToRemove.push(idx);
272+
}
193273
})
274+
if (entityIdsToRemove.length !== 0) {
275+
entityIdsToRemove.sort((a, b) => b - a).forEach(id => {
276+
utt.entities.splice(id, 1);
277+
})
278+
}
279+
// remove any children that are not a root entity
280+
removeNonRootChildren(utt.entities, finalLUISJSON.entities)
194281
}
282+
})
283+
}
284+
285+
const removeNonRootChildren = function(entitiesList, allEntitiesList) {
286+
let idxToRemove = [];
287+
entitiesList.forEach((entity, idx) => {
288+
if (!isRootEntity(entity.entity, allEntitiesList)) {
289+
idxToRemove.push(idx)
290+
}
291+
})
292+
if (idxToRemove.length !== 0) {
293+
idxToRemove.sort((a,b) => b-a).forEach(id => entitiesList.splice(id, 1));
294+
idxToRemove = [];
195295
}
296+
// de-dupe children
297+
deDupeChildren(entitiesList);
196298
}
197299

198-
module.exports = helpers;
300+
const deDupeChildren = function(collection) {
301+
collection.forEach(entity => {
302+
if (entity.children !== undefined && Array.isArray(entity.children) && entity.children.length !== 0) {
303+
let childAsStr = entity.children.map(item => JSON.stringify(item));
304+
var newList = [];
305+
childAsStr.forEach(item => {
306+
if (newList.indexOf(item) === -1) newList.push(item)
307+
})
308+
entity.children = newList.map(item => JSON.parse(item))
309+
deDupeChildren(entity.children)
310+
}
311+
})
312+
}
313+
314+
const isRootEntity = function (entityName, entitiesCollection) {
315+
if ((entitiesCollection || []).find(ecEntity => ecEntity.name == entityName) !== undefined)
316+
return true
317+
return false
318+
}
319+
320+
const findParent = function (entityInUtt, entityCollection, parentTree, curParent) {
321+
let numOfParentsFound = 0;
322+
entityCollection.forEach(childEntity => {
323+
if (childEntity.name == entityInUtt.entity) {
324+
parentTree[curParent] = childEntity.name;
325+
numOfParentsFound++;
326+
}
327+
if (childEntity.children !== undefined && Array.isArray(childEntity.children) && childEntity.children.length !== 0) {
328+
parentTree[childEntity.name] = {};
329+
numOfParentsFound += findParent(entityInUtt, childEntity.children, parentTree[childEntity.name], childEntity.name)
330+
}
331+
});
332+
return numOfParentsFound;
333+
}
334+
335+
const objectSortByStartPos = function (objectArray) {
336+
let ObjectByStartPos = objectArray.slice(0);
337+
ObjectByStartPos.sort(function (a, b) {
338+
if (a.startPos === b.startPos)
339+
return a.endPos - b.endPos;
340+
return a.startPos - b.startPos;
341+
})
342+
return ObjectByStartPos
343+
}
344+
345+
const transformAllEntityConstraintsToFeatures = function(entity) {
346+
addIsRequiredProperty(entity);
347+
if (entity.hasOwnProperty("instanceOf") && entity.instanceOf !== null) {
348+
if (entity.hasOwnProperty("features") && Array.isArray(entity.features)) {
349+
let featureFound = (entity.features || []).find(i => i.modelName == entity.instanceOf);
350+
if (featureFound !== undefined) {
351+
if (featureFound.featureType === undefined)
352+
featureFound.isRequired = true;
353+
}
354+
else {
355+
entity.features.push(new hClasses.entityFeature(entity.instanceOf, true));
356+
}
357+
}
358+
else {
359+
if (entity.instanceOf !== "" && entity.instanceOf !== null)
360+
entity.features = [new hClasses.entityFeature(entity.instanceOf, true)];
361+
}
362+
}
363+
delete entity.instanceOf;
364+
if (!entity.children || entity.children.length === 0)
365+
return;
366+
entity.children.forEach(c => transformAllEntityConstraintsToFeatures(c));
367+
};
368+
369+
const addIsRequiredProperty = function(item) {
370+
(item.features || []).forEach(feature => {
371+
if (feature.isRequired === undefined)
372+
feature.isRequired = false;
373+
delete feature.featureType;
374+
delete feature.modelType;
375+
});
376+
}

‎packages/lu/test/commands/luis/convert.test.js‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,23 @@ describe('luis:convert', () => {
135135
assert.equal(sanitize(newLU.content), result);
136136
});
137137
})
138-
describe('luis:convert version 5 upgrade test', () => {
138+
describe('luis:convert version 7 upgrade test', () => {
139+
it('luis:convert successfully converts a lu file with depth information preserved in entity and utterances', async () => {
140+
await assertToJSON('./../../fixtures/examples/v7UpgradeTest.lu', './../../fixtures/verified/v7UpgradeTest.json')
141+
})
142+
143+
it('luis:convert successfully converts a JSON file with depth information preserved in entity and utterances', async () => {
144+
await assertToLu('./../../fixtures/verified/v7UpgradeTest.json', './../../fixtures/verified/v7UpgradeTest.lu')
145+
})
146+
147+
it('V7 json from LUIS team converts correctly to lu format', async () => {
148+
await assertToLu('./../../fixtures/testcases/v7app.json', './../../fixtures/verified/v7app.lu')
149+
})
150+
151+
it('V7 LU (from LUIS team) converts correctly to JSON format', async () => {
152+
await assertToJSON('./../../fixtures/verified/v7app.lu', './../../fixtures/verified/v7app_c.json')
153+
})
154+
139155
it('luis:convert successfully reconstructs a markdown file from a LUIS input file with v5 constructs', async () => {
140156
await assertToJSON('./../../fixtures/verified/v5UpgradeTest.lu', './../../fixtures/verified/v5Upgrade.json')
141157
})

0 commit comments

Comments
 (0)