Skip to content
This repository was archived by the owner on Jan 15, 2025. It is now read-only.

Commit fba06a0

Browse files
vishwacsenamunozemilio
vishwacsena
authored and
munozemilio
committed
Fix to update LU parser to support phrase list flags - enabledForAllModels, disabled. (#366)
* fix * Fix enabledForAllModels * Fix
1 parent 104f81a commit fba06a0

File tree

12 files changed

+221
-22
lines changed

12 files changed

+221
-22
lines changed

‎packages/lu/docs/lu-file-format.md‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -358,6 +358,25 @@ By default synonyms are set to be **not interchangeable** (matches with the port
358358
- you are
359359
```
360360

361+
Phrase lists can be marked as `disabled` using this notation:
362+
363+
```markdown
364+
@ phraselist abc disabled
365+
366+
> also same as this
367+
@ phraselist question(interchangeable) =
368+
- are you
369+
- you are
370+
371+
@ question disabled
372+
```
373+
374+
Phrase lists by default are enabled for all models. However when you explicitly start assigning phrase list as a feature (descriptor) to other models, then the specific phrase lists is not enabled for all models. To explicitly make a phrase list always available to all models, you can do so via:
375+
376+
```markdown
377+
@ phraselist abc enabledForAllModels
378+
```
379+
361380
## Model as feature
362381

363382
Here's how you add a feature to a ml entity or an intent - with `usesFeature`.

‎packages/lu/src/parser/index.js‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,9 @@ const modules = {
88
validateLUISBlob: require('./luis/luisValidator').validateLUIS
99
},
1010
refresh: {
11-
constructMdFromLUIS: require('./luis/luConverter').luisToLuContent,
12-
constructMdFromQnA: require('./qna/qnamaker/qnaConverter').qnaToLuContent,
13-
constructMdFromQnAAlteration: require('./qna/alterations/qnaConverter').qnaAlterationsToLuContent
11+
constructMdFromLUIS: require('./luis/luConverter'),
12+
constructMdFromQnA: require('./qna/qnamaker/qnaConverter'),
13+
constructMdFromQnAAlteration: require('./qna/alterations/qnaConverter')
1414
},
1515
translate: {
1616
parseAndTranslate: require('./lufile/translate-helpers').parseAndTranslate,

‎packages/lu/src/parser/lu/luMerger.js‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,8 @@ const haveLUISContent = function(blob) {
136136
(blob[LUISObjNameEnum.UTTERANCE].length > 0) ||
137137
(blob.prebuiltEntities.length > 0) ||
138138
(blob[LUISObjNameEnum.REGEX].length > 0) ||
139-
(blob.model_features.length > 0) ||
139+
(blob.model_features && blob.model_features.length > 0) ||
140+
(blob.phraselists && blob.phraselists.length > 0) ||
140141
(blob.composites.length > 0));
141142
}
142143

‎packages/lu/src/parser/lufile/parseFileContents.js‎

Lines changed: 23 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -409,7 +409,7 @@ const parseFeatureSections = function(parsedContent, featuresToProcess) {
409409
addFeatures(intentExists, feature, featureTypeEnum.featureToModel, section.ParseTree.newEntityLine(), featureProperties.phraseListFeature);
410410
// set enabledForAllModels on this phrase list
411411
let plEnity = parsedContent.LUISJsonStructure.model_features.find(item => item.name == feature);
412-
plEnity.enabledForAllModels = false;
412+
if (plEnity.enabledForAllModels === undefined) plEnity.enabledForAllModels = false;
413413
} else {
414414
// de-dupe and add model to intent.
415415
validateFeatureAssignment(section.Type, section.Name, entityExists.type, feature, section.ParseTree.newEntityLine());
@@ -456,7 +456,7 @@ const parseFeatureSections = function(parsedContent, featuresToProcess) {
456456
addFeatures(srcEntity, feature, featureTypeEnum.featureToModel, section.ParseTree.newEntityLine(), featureProperties.phraseListFeature);
457457
// set enabledForAllModels on this phrase list
458458
let plEnity = parsedContent.LUISJsonStructure.model_features.find(item => item.name == feature);
459-
plEnity.enabledForAllModels = false;
459+
if (plEnity.enabledForAllModels === undefined) plEnity.enabledForAllModels = false;
460460
} else {
461461
// de-dupe and add model to intent.
462462
validateFeatureAssignment(entityType, section.Name, featureExists.type, feature, section.ParseTree.newEntityLine());
@@ -1207,14 +1207,25 @@ const RemoveDuplicatePatternAnyEntity = function(parsedContent, pEntityName, ent
12071207
* @param {String []} valuesList Array of individual lines to be processed and added to phrase list.
12081208
*/
12091209
const handlePhraseList = function(parsedContent, entityName, entityType, entityRoles, valuesList, currentLine) {
1210+
let isPLEnabledForAllModels = undefined;
1211+
let isPLEnabled = undefined;
12101212
if (entityRoles.length !== 0) {
1211-
let errorMsg = `Phrase list entity ${entityName} has invalid role definition with roles = ${entityRoles.join(', ')}. Roles are not supported for Phrase Lists`;
1212-
let error = BuildDiagnostic({
1213-
message: errorMsg,
1214-
context: currentLine
1213+
// Phrase lists cannot have roles; however we will allow inline definition of enabledForAllModels as well as disabled as a property on phrase list.
1214+
entityRoles.forEach(item => {
1215+
if (item.toLowerCase() === 'disabled') {
1216+
isPLEnabled = false;
1217+
} else if (item.toLowerCase() === 'enabledforallmodels') {
1218+
isPLEnabledForAllModels = true;
1219+
} else {
1220+
let errorMsg = `Phrase list entity ${entityName} has invalid role definition with roles = ${entityRoles.join(', ')}. Roles are not supported for Phrase Lists`;
1221+
let error = BuildDiagnostic({
1222+
message: errorMsg,
1223+
context: currentLine
1224+
})
1225+
1226+
throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString()));
1227+
}
12151228
})
1216-
1217-
throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString()));
12181229
}
12191230
// check if this phraselist entity is already labelled in an utterance and or added as a simple entity. if so, throw an error.
12201231
try {
@@ -1255,8 +1266,11 @@ const handlePhraseList = function(parsedContent, entityName, entityType, entityR
12551266
if (!wordsSplit.includes(plValueItem)) pLEntityExists.words += (pLEntityExists.words !== '' ? ',' : '') + plValueItem;
12561267
})
12571268
} else {
1258-
parsedContent.LUISJsonStructure.model_features.push(new helperClass.modelObj(entityName, intc, pLValues.join(','), true));
1269+
pLEntityExists = new helperClass.modelObj(entityName, intc, pLValues.join(','), true);
1270+
parsedContent.LUISJsonStructure.model_features.push(pLEntityExists);
12591271
}
1272+
if (isPLEnabled !== undefined) pLEntityExists.activated = isPLEnabled;
1273+
if (isPLEnabledForAllModels !== undefined) pLEntityExists.enabledForAllModels = isPLEnabledForAllModels;
12601274
}
12611275

12621276
/**

‎packages/lu/src/parser/luis/luConverter.js‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -221,7 +221,13 @@ const handlePhraseLists = function(collection) {
221221
}
222222
fileContent = '> # Phrase list definitions' + NEWLINE + NEWLINE;
223223
collection.forEach(function(entity) {
224+
let flags = '';
224225
fileContent += `@ phraselist ${entity.name}${(entity.mode ? `(interchangeable)` : ``)}`;
226+
if (entity.activated !== undefined && !entity.activated) flags += `disabled`;
227+
if (entity.enabledForAllModels !== undefined && entity.enabledForAllModels) {
228+
flags += (flags !== '') ? `, enabledForAllModels` : `enabledForAllModels`;
229+
}
230+
if (flags !== '') fileContent += ` ${flags}`;
225231
if (entity.words && entity.words !== '') {
226232
fileContent += ` = ${NEWLINE}\t- ${entity.words}`;
227233
}

‎packages/lu/src/parser/utils/helpers.js‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -165,12 +165,16 @@ const helpers = {
165165
let v5DefFound = false;
166166
v5DefFound = (finalLUISJSON.entities || []).find(i => i.children || i.features) ||
167167
(finalLUISJSON.intents || []).find(i => i.features) ||
168-
(finalLUISJSON.composites || []).find(i => i.features);
168+
(finalLUISJSON.composites || []).find(i => i.features) ||
169+
(finalLUISJSON.luis_schema_version === '6.0.0');
169170
if (v5DefFound) {
170171
finalLUISJSON.luis_schema_version = "6.0.0";
171172
if (finalLUISJSON.model_features) {
172173
finalLUISJSON.phraselists = [];
173-
finalLUISJSON.model_features.forEach(item => finalLUISJSON.phraselists.push(Object.assign({}, item)));
174+
finalLUISJSON.model_features.forEach(item => {
175+
if (item.enabledForAllModels === undefined) item.enabledForAllModels = true
176+
finalLUISJSON.phraselists.push(Object.assign({}, item))
177+
});
174178
delete finalLUISJSON.model_features;
175179
}
176180
(finalLUISJSON.composites || []).forEach(composite => {

‎packages/lu/test/commands/luis/convert.test.ts‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -186,7 +186,7 @@ describe('luis:convert', () => {
186186
.stderr()
187187
.command(['luis:convert', '--in', `${path.join(__dirname, './../../fixtures/testcases/invalid-entity-definition.lu')}`])
188188
.it('luis:convert writes out an error when invalid entity definition is found', async (ctx) => {
189-
expect(ctx.stderr).to.contain("[ERROR] line 1:14 - line 1:16: syntax error: missing ':' at '\\r\\n'")
189+
expect(ctx.stderr).to.contain("syntax error: missing ':'")
190190
})
191191

192192
test
@@ -424,6 +424,13 @@ describe('luis:convert version 5 upgrade test', () => {
424424
.it('luis:convert successfully converts LUIS JSON model with no phrase lists (output must have phraselists if any v6 concepts are present in the .lu file)', async () => {
425425
expect(await compareLuFiles('./../../../results/root38.json', './../../fixtures/verified/v6WithoutPhraseLists.json')).to.be.true
426426
})
427+
428+
test
429+
.stdout()
430+
.command(['luis:convert', '--in', `${path.join(__dirname, './../../fixtures/testcases/plWithFlags.lu')}`, '--out', './results/root39.json'])
431+
.it('luis:convert successfully converts LUIS JSON model with no phrase lists (output must have phraselists if any v6 concepts are present in the .lu file)', async () => {
432+
expect(await compareLuFiles('./../../../results/root39.json', './../../fixtures/verified/plWithFlags.json')).to.be.true
433+
})
427434
})
428435

429436
describe('luis:convert negative tests', () => {

‎packages/lu/test/fixtures/testcases/plWithFlags.lu‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
> phrase list as feature to intent (also applicable to entities)
2+
@ intent getUserProfileIntent usesFeature city
3+
4+
# getUserProfileIntent
5+
- test
6+
7+
@ phraselist city
8+
@ city =
9+
- Seattle
10+
- SEATAC
11+
- SEA
12+
13+
@ city enabledForAllModels
14+
@ city disabled

‎packages/lu/test/fixtures/verified/plWithFlags.json‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
{
2+
"intents": [
3+
{
4+
"name": "getUserProfileIntent",
5+
"features": [
6+
{
7+
"featureName": "city",
8+
"featureType": "phraselist"
9+
}
10+
]
11+
}
12+
],
13+
"entities": [],
14+
"composites": [],
15+
"closedLists": [],
16+
"regex_entities": [],
17+
"regex_features": [],
18+
"utterances": [
19+
{
20+
"text": "test",
21+
"intent": "getUserProfileIntent",
22+
"entities": []
23+
}
24+
],
25+
"patterns": [],
26+
"patternAnyEntities": [],
27+
"prebuiltEntities": [],
28+
"luis_schema_version": "6.0.0",
29+
"phraselists": [
30+
{
31+
"name": "city",
32+
"words": "Seattle,SEATAC,SEA",
33+
"mode": false,
34+
"activated": false,
35+
"enabledForAllModels": true
36+
}
37+
],
38+
"versionId": "0.1",
39+
"name": "",
40+
"desc": "",
41+
"culture": "en-us"
42+
}

‎packages/lu/test/fixtures/verified/v5Upgrade.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,8 @@
4444
"name": "abc",
4545
"words": "one,two,three",
4646
"mode": false,
47-
"activated": true
47+
"activated": true,
48+
"enabledForAllModels": true
4849
}
4950
],
5051
"versionId": "0.1",

0 commit comments

Comments
 (0)