Skip to content
This repository was archived by the owner on Jan 15, 2025. It is now read-only.

Commit 5d27394

Browse files
feich-msboydc2014
feich-ms
and
boydc2014
authored
Expose import resolver interface for cross-train/luis:build/qnamaker:build (#935)
* add fileResolver for lu and qna build API * add file resolver in crosstrain * optimize file resolver interface * adjust test case * add tests for file resolver * refactor naming * add more import resolver unit tests for luis and qna loadContents function Co-authored-by: Dong Lei <[email protected]>
1 parent 1c4ddb8 commit 5d27394

File tree

20 files changed

+306
-92
lines changed

20 files changed

+306
-92
lines changed

‎packages/lu/src/parser/cross-train/crossTrainer.js‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -28,19 +28,20 @@ module.exports = {
2828
* @param {any[]} luContents the lu content array whose element includes path and content
2929
* @param {any[]} qnaContents the qna content array whose element includes path and content
3030
* @param {any} crossTrainConfig cross train json config
31+
* @param {any} importResolver import Resolver when resolving import files
3132
* @returns {Map<string, LUResource>} map of file id and luResource
3233
* @throws {exception} throws errors
3334
*/
34-
crossTrain: async function (luContents, qnaContents, crossTrainConfig) {
35+
crossTrain: async function (luContents, qnaContents, crossTrainConfig, importResolver) {
3536
try {
3637
let {luObjectArray, qnaObjectArray} = pretreatment(luContents, qnaContents)
3738
const {rootIds, triggerRules, intentName, verbose} = crossTrainConfig
3839

3940
// parse lu content to LUResource object
40-
let {fileIdToResourceMap: luFileIdToResourceMap, allEmpty: allLuEmpty} = await parseAndValidateContent(luObjectArray, verbose)
41+
let {fileIdToResourceMap: luFileIdToResourceMap, allEmpty: allLuEmpty} = await parseAndValidateContent(luObjectArray, verbose, importResolver)
4142

4243
// parse qna content to LUResource object
43-
let {fileIdToResourceMap: qnaFileIdToResourceMap, allEmpty: allQnAEmpty} = await parseAndValidateContent(qnaObjectArray, verbose)
44+
let {fileIdToResourceMap: qnaFileIdToResourceMap, allEmpty: allQnAEmpty} = await parseAndValidateContent(qnaObjectArray, verbose, importResolver)
4445

4546
if (!allLuEmpty) {
4647
// construct resource tree to build the father-children relationship among lu files
@@ -496,21 +497,22 @@ const qnaAddMetaData = function (qnaResource, fileName) {
496497
* Parse and validate lu or qna object array to convert to LUResource object dict
497498
* @param {luObject[]} objectArray the lu or qna object list to be parsed
498499
* @param {boolean} verbose indicate to enable log messages or not
500+
* @param {any} importResolver import Resolver when resolving import files
499501
* @returns {Map<string, LUResource>} map of file id and luResource
500502
* @throws {exception} throws errors
501503
*/
502-
const parseAndValidateContent = async function (objectArray, verbose) {
504+
const parseAndValidateContent = async function (objectArray, verbose, importResolver) {
503505
let fileIdToResourceMap = new Map()
504506
let allEmpty = true
505507
for (const object of objectArray) {
506508
let fileContent = object.content
507509
if (object.content && object.content !== '') {
508510
if (object.id.toLowerCase().endsWith(fileExtEnum.LUFile)) {
509-
let result = await LuisBuilderVerbose.build([object], verbose)
511+
let result = await LuisBuilderVerbose.build([object], verbose, undefined, importResolver)
510512
let luisObj = new Luis(result)
511513
fileContent = luisObj.parseToLuContent()
512514
} else {
513-
let result = await qnaBuilderVerbose.build([object], verbose)
515+
let result = await qnaBuilderVerbose.build([object], verbose, importResolver)
514516
fileContent = result.parseToQnAContent()
515517
}
516518
}

‎packages/lu/src/parser/lu/luMerger.js‎

Lines changed: 24 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -30,15 +30,15 @@ module.exports = {
3030
*/
3131
Build: async function(luObjArray, verbose, luis_culture, luSearchFn){
3232
let allParsedContent = await buildLuJsonObject(luObjArray, verbose, luis_culture, luSearchFn)
33-
let refTree = buildRefTree(allParsedContent)
33+
let refTree = await buildRefTree(allParsedContent, luSearchFn)
3434
resolveTreeRefs(refTree, luObjArray);
3535
return allParsedContent
3636
}
3737
}
3838

39-
const buildRefTree = function(allParsedContent) {
39+
const buildRefTree = async function(allParsedContent, luSearchFn) {
4040
let refs = {};
41-
allParsedContent.LUISContent.forEach((parserObj, objIdx) => {
41+
await Promise.all(allParsedContent.LUISContent.map(async (parserObj, objIdx) => {
4242
let luObj = {
4343
obj : parserObj.LUISJsonStructure,
4444
srcFile : parserObj.srcFile,
@@ -54,23 +54,23 @@ const buildRefTree = function(allParsedContent) {
5454
}
5555
}
5656
parserObj.LUISJsonStructure.uttHash = {};
57-
(parserObj.LUISJsonStructure.utterances || []).forEach((utterance, uttIdx) => {
57+
(parserObj.LUISJsonStructure.utterances || []).forEach(async (utterance, uttIdx) => {
5858
parserObj.LUISJsonStructure.uttHash[utterance.text] = '';
5959
if (helpers.isUtteranceLinkRef(utterance.text)) {
60-
let parsedLinkUri = helpers.parseLinkURI(utterance.text);
60+
let parsedLinkUri = await helpers.parseLinkURI(utterance.text, parserObj.srcFile, luSearchFn);
6161
refs[parserObj.srcFile].luis.refs.push({
62-
refId : parsedLinkUri.fileName,
63-
uttId : uttIdx,
64-
parsedLink : parsedLinkUri,
65-
uttObj : utterance,
66-
text : utterance.text,
67-
type : 'luis'
62+
refId: parsedLinkUri.fileName,
63+
uttId: uttIdx,
64+
parsedLink: parsedLinkUri,
65+
uttObj: utterance,
66+
text: utterance.text,
67+
type: 'luis'
6868
})
6969
}
7070
})
71-
})
71+
}))
7272

73-
allParsedContent.QnAContent.forEach((parserObj, objIdx) => {
73+
await Promise.all(allParsedContent.QnAContent.map(async (parserObj, objIdx) => {
7474
let qnaObj = {
7575
obj : parserObj.qnaJsonStructure,
7676
alt : allParsedContent.QnAAlterations[objIdx].qnaAlterations,
@@ -84,22 +84,23 @@ const buildRefTree = function(allParsedContent) {
8484
refs[parserObj.srcFile].qna = qnaObj;
8585
}
8686
}
87-
(parserObj.qnaJsonStructure.qnaList.forEach(qnaPair => {
88-
qnaPair.questions.forEach((question, qIdx) => {
87+
(parserObj.qnaJsonStructure.qnaList.forEach(async qnaPair => {
88+
qnaPair.questions.forEach(async (question, qIdx) => {
8989
if (helpers.isUtteranceLinkRef(question)) {
90-
let parsedLinkUri = helpers.parseLinkURI(question);
90+
let parsedLinkUri = await helpers.parseLinkURI(question)
9191
refs[parserObj.srcFile].qna.refs.push({
92-
refId : parsedLinkUri.fileName,
93-
qId : qIdx,
94-
text : question,
95-
qObj : qnaPair,
96-
parsedLink : parsedLinkUri,
97-
type : 'qna'
92+
refId: parsedLinkUri.fileName,
93+
qId: qIdx,
94+
text: question,
95+
qObj: qnaPair,
96+
parsedLink: parsedLinkUri,
97+
type: 'qna'
9898
})
9999
}
100100
})
101101
}))
102-
});
102+
}));
103+
103104
return refs;
104105
}
105106

‎packages/lu/src/parser/lubuild/builder.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,8 @@ export class Builder {
3434
culture: string,
3535
suffix: string,
3636
region: string,
37-
schema?: string) {
37+
schema?: string,
38+
importResolver?: object) {
3839
let multiRecognizers = new Map<string, MultiLanguageRecognizer>()
3940
let settings: any
4041
let recognizers = new Map<string, Recognizer>()
@@ -82,7 +83,7 @@ export class Builder {
8283
if (luFiles.length <= 0) continue
8384

8485
try {
85-
result = await LuisBuilderVerbose.build(luFiles, true, fileCulture)
86+
result = await LuisBuilderVerbose.build(luFiles, true, fileCulture, importResolver)
8687
luisObj = new Luis(result)
8788
fileContent = luisObj.parseToLuContent()
8889
} catch (err) {

‎packages/lu/src/parser/lufile/parseFileContents.js‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -174,10 +174,10 @@ const parseLuAndQnaWithAntlr = async function (parsedContent, fileContent, log,
174174
parseAndHandleEntitySection(parsedContent, luResource, log, locale);
175175

176176
// parse simple intent section
177-
parseAndHandleSimpleIntentSection(parsedContent, luResource);
177+
await parseAndHandleSimpleIntentSection(parsedContent, luResource);
178178

179179
// parse qna section
180-
parseAndHandleQnaSection(parsedContent, luResource);
180+
await parseAndHandleQnaSection(parsedContent, luResource);
181181

182182
if (featuresToProcess && featuresToProcess.length > 0) {
183183
parseFeatureSections(parsedContent, featuresToProcess);
@@ -753,7 +753,7 @@ const parseAndHandleNestedIntentSection = function (luResource, enableMergeInten
753753
* @param {LUResouce} luResource resources extracted from lu file content
754754
* @throws {exception} Throws on errors. exception object includes errCode and text.
755755
*/
756-
const parseAndHandleSimpleIntentSection = function (parsedContent, luResource) {
756+
const parseAndHandleSimpleIntentSection = async function (parsedContent, luResource) {
757757
// handle intent
758758
let intents = luResource.Sections.filter(s => s.SectionType === SectionType.SIMPLEINTENTSECTION);
759759
let hashTable = {}
@@ -769,7 +769,7 @@ const parseAndHandleSimpleIntentSection = function (parsedContent, luResource) {
769769
// Fix for BF-CLI #122.
770770
// Ensure only links are detected and passed on to be parsed.
771771
if (helpers.isUtteranceLinkRef(utterance || '')) {
772-
let parsedLinkUriInUtterance = helpers.parseLinkURI(utterance);
772+
let parsedLinkUriInUtterance = await helpers.parseLinkURI(utterance);
773773
// examine and add these to filestoparse list.
774774
parsedContent.additionalFilesToParse.push(new fileToParse(parsedLinkUriInUtterance.fileName, false));
775775
}
@@ -1818,7 +1818,7 @@ const handleRegExEntity = function(parsedContent, entityName, entityType, entity
18181818
* @param {LUResouce} luResource resources extracted from lu file content
18191819
* @throws {exception} Throws on errors. exception object includes errCode and text.
18201820
*/
1821-
const parseAndHandleQnaSection = function (parsedContent, luResource) {
1821+
const parseAndHandleQnaSection = async function (parsedContent, luResource) {
18221822
// handle QNA
18231823
let qnas = luResource.Sections.filter(s => s.SectionType === SectionType.QNASECTION);
18241824
if (qnas && qnas.length > 0) {
@@ -1828,14 +1828,14 @@ const parseAndHandleQnaSection = function (parsedContent, luResource) {
18281828
}
18291829
let questions = qna.Questions;
18301830
// detect if any question is a reference
1831-
(questions || []).forEach(question => {
1831+
await Promise.all((questions || []).map(async question => {
18321832
// Ensure only links are detected and passed on to be parsed.
18331833
if (helpers.isUtteranceLinkRef(question || '')) {
1834-
let parsedLinkUriInUtterance = helpers.parseLinkURI(question);
1834+
let parsedLinkUriInUtterance = await helpers.parseLinkURI(question);
18351835
// examine and add these to filestoparse list.
18361836
parsedContent.additionalFilesToParse.push(new fileToParse(parsedLinkUriInUtterance.fileName, false));
18371837
}
1838-
})
1838+
}))
18391839
let filterPairs = qna.FilterPairs;
18401840
let metadata = [];
18411841
if (filterPairs && filterPairs.length > 0) {

‎packages/lu/src/parser/qnabuild/builder.ts‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,8 @@ export class Builder {
3636
suffix: string,
3737
region: string,
3838
culture: string,
39-
schema?: string) {
39+
schema?: string,
40+
importResolver?: object) {
4041
let multiRecognizers = new Map<string, MultiLanguageRecognizer>()
4142
let settings: any
4243
let recognizers = new Map<string, Recognizer>()
@@ -133,7 +134,7 @@ export class Builder {
133134
}
134135
}
135136

136-
await this.resolveMergedQnAContentIds(qnaContents, qnaObjects)
137+
await this.resolveMergedQnAContentIds(qnaContents, qnaObjects, importResolver)
137138

138139
return {qnaContents: [...qnaContents.values()], recognizers, multiRecognizers, settings, crosstrainedRecognizers}
139140
}
@@ -583,11 +584,11 @@ export class Builder {
583584
this.handler(`Publishing finished for kb ${kbName}\n`)
584585
}
585586

586-
async resolveMergedQnAContentIds(contents: Map<string, any>, objects: Map<string, any[]>) {
587+
async resolveMergedQnAContentIds(contents: Map<string, any>, objects: Map<string, any[]>, importResolver?: object) {
587588
for (const [name, content] of contents) {
588589
let qnaObjects = objects.get(name)
589590
try {
590-
let result = await qnaBuilderVerbose.build(qnaObjects, true)
591+
let result = await qnaBuilderVerbose.build(qnaObjects, true, importResolver)
591592
let mergedContent = result.parseToQnAContent()
592593
content.content = mergedContent
593594
contents.set(name, content)

‎packages/lu/src/parser/utils/helpers.js‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,18 +67,23 @@ const helpers = {
6767
/**
6868
* Helper function to parse link URIs in utterances
6969
* @param {String} utterance
70-
* @param {String} srcPath
70+
* @param {String} srcId
71+
* @param {Object} luSearchFn
7172
* @returns {Object} Object that contains luFile and ref. ref can be Intent-Name or ? or * or **
7273
* @throws {exception} Throws on errors. exception object includes errCode and text.
7374
*/
74-
parseLinkURI: function (utterance, srcPath = null) {
75+
parseLinkURI: async function (utterance, srcId = null, luSearchFn = null) {
7576
let linkValueList = utterance.trim().match(new RegExp(/\(.*?\)/g));
7677
let linkValue = linkValueList[0].replace('(', '').replace(')', '');
7778
if (linkValue === '') throw (new exception(retCode.errorCode.INVALID_LU_FILE_REF, `[ERROR]: Invalid LU File Ref: "${utterance}"`));
7879
// reference can either be #<Intent-Name> or #? or /*#? or /**#? or #*utterance* or #<Intent-Name>*patterns*
7980
let splitRegExp = new RegExp(/^(?<fileName>.*?)(?<segment>#|\*+)(?<path>.*?)$/gim);
8081
let splitReference = splitRegExp.exec(linkValue);
8182
if (!splitReference) throw (new exception(retCode.errorCode.INVALID_LU_FILE_REF, `[ERROR]: Invalid LU File Ref: "${utterance}".\n Reference needs a qualifier - either a #Intent-Name or #? or *#? or **#? or #*utterances* etc.`));
83+
if (splitReference.groups.fileName && srcId && luSearchFn) {
84+
let luObjects = await luSearchFn(srcId, [{filePath: splitReference.groups.fileName, includeInCollate: false}])
85+
if (luObjects && luObjects.length > 0) splitReference.groups.fileName = luObjects[0].id
86+
}
8287
if (splitReference.groups.segment.includes('*')) {
8388
if (splitReference.groups.path === '') {
8489
throw (new exception(retCode.errorCode.INVALID_LU_FILE_REF, `[ERROR]: Invalid LU File Ref: "${utterance}".\n '*' and '**' can only be used with QnA qualitifier. e.g. *#? and **#?`));

‎packages/lu/test/fixtures/testcases/import-resolver/lu-import-resolver/common.en-us.lu‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
[import](welcome.lu)
2+
3+
# help
4+
- could you help
5+
- [import help](help.lu#help)
6+
7+
# cancel
8+
- cancel that
9+
- [import cancel](terminate.lu#*utterances*)

‎packages/lu/test/fixtures/testcases/import-resolver/lu-import-resolver/help.en-us.lu‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# help
2+
- can you help me

‎packages/lu/test/fixtures/testcases/import-resolver/lu-import-resolver/terminate.en-us.lu‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# cancel
2+
- cancel the task
3+
4+
# stop
5+
- stop that

‎packages/lu/test/fixtures/testcases/import-resolver/lu-import-resolver/welcome.en-us.lu‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# welcome
2+
- welcome here

0 commit comments

Comments
 (0)