[bf-lu] fix section operator issues and adjust file and url import interface (#965)

feich-ms · boydc2014 · web-flow · commit 4f7c7933f73d · 2020-09-16T14:07:25.000+08:00
* fix some bugs and add tests

* fix tsc issue

* fix whitespace issue in import section

* support multi turn extraction

Co-authored-by: Dong Lei &lt;donglei@microsoft.com&gt;
diff --git a/packages/lu/src/parser/lubuild/builder.ts b/packages/lu/src/parser/lubuild/builder.ts
@@ -229,7 +229,8 @@ export class Builder {
           // otherwise create a new application
           if (recognizer.getAppId() && recognizer.getAppId() !== '') {
             // To see if need update the model
-            needTrainAndPublish = await this.updateApplication(currentApp, luBuildCore, recognizer, timeBucket, keptVersionCount ?? maxVersionCount)
+            const versionCount = keptVersionCount && keptVersionCount > 0 ? keptVersionCount : maxVersionCount
+            needTrainAndPublish = await this.updateApplication(currentApp, luBuildCore, recognizer, timeBucket, versionCount)
           } else {
             // create a new application
             needTrainAndPublish = await this.createApplication(currentApp, luBuildCore, recognizer, timeBucket)
diff --git a/packages/lu/src/parser/lufile/LUFileLexer.g4 b/packages/lu/src/parser/lufile/LUFileLexer.g4
@@ -78,11 +78,11 @@ AT
   ;
 
 IMPORT
-  : '[' ~[\r\n[\]]*? ']' ('(' ~[\r\n()]*? ')' | '[' ~[\r\n[\]]*? ']')
+  : WS* '[' ~[\r\n[\]]*? ']' WS* ('(' ~[\r\n()]*? ')' | '[' ~[\r\n[\]]*? ']')
   ;
 
 REFERENCE
-  : '[' ~[\r\n[\]]*? ']' WS* ':' WS* ~[\r\n]*
+  : WS* '[' ~[\r\n[\]]*? ']' WS* ':' WS* ~[\r\n]*
   ;
 
 FILTER_MARK
diff --git a/packages/lu/src/parser/lufile/generated/LUFileLexer.interp b/packages/lu/src/parser/lufile/generated/LUFileLexer.interp
diff --git a/packages/lu/src/parser/lufile/generated/LUFileLexer.js b/packages/lu/src/parser/lufile/generated/LUFileLexer.js
diff --git a/packages/lu/src/parser/lufile/importSection.js b/packages/lu/src/parser/lufile/importSection.js
@@ -26,7 +26,7 @@ class ImportSection extends BaseSection {
     ExtractDescriptionAndPath(parseTree) {
         let importPathRegex = new RegExp(/\[([^\]]*)\]\(([^\)]*)\)/);
         let importReferenceRegex = new RegExp(/\[([^\]]*)\]\[([^\]]*)\]/);
-        let importStr = parseTree.importDefinition().IMPORT().getText();
+        let importStr = parseTree.importDefinition().IMPORT().getText().trim();
         let description;
         let path;
         let groups = importStr.match(importPathRegex);
diff --git a/packages/lu/src/parser/lufile/referenceSection.js b/packages/lu/src/parser/lufile/referenceSection.js
@@ -25,7 +25,7 @@ class ReferenceSection extends BaseSection {
 
     ExtractIdAndPath(parseTree) {
         let referenceRegex = new RegExp(/\[([^\]]*)\]\s*:\s*([^\)]*)/);
-        let referenceStr = parseTree.referenceDefinition().REFERENCE().getText();
+        let referenceStr = parseTree.referenceDefinition().REFERENCE().getText().trim();
 
         let referenceId
         let path
diff --git a/packages/lu/src/parser/lufile/sectionOperator.js b/packages/lu/src/parser/lufile/sectionOperator.js
@@ -66,7 +66,7 @@ class SectionOperator {
     this.adjustRangeForErrors(this.Luresource.Errors, newLineRange - originalRange, endLine);
 
     // adjust updated sections' errors
-    const offset = oldSection.Range.Start.Line - newResource.Sections[0].Range.Start.Line;
+    const offset = oldSection.Range.Start.Line - 1;
     this.adjustRangeForErrors(newResource.Errors, offset);
     this.Luresource.Errors.push(...newResource.Errors);
 
@@ -80,7 +80,7 @@ class SectionOperator {
   deleteSection(id) {
     const sectionIndex = this.Luresource.Sections.findIndex(u => u.Id === id);
     if (sectionIndex < 0) {
-      return this;
+      return this.Luresource;
     }
 
     const oldSection = this.Luresource.Sections[sectionIndex];
@@ -122,7 +122,7 @@ class SectionOperator {
 
     // adjust original errors
     const newLineRange = sectionContent.split(/\r?\n/).length;
-    const startLine = sectionIndex <= 0 ? 1 : this.Luresource.Sections[sectionIndex].Range.Start.Line;
+    const startLine = sectionIndex < 0 ? 1 : this.Luresource.Sections[sectionIndex].Range.Start.Line;
     this.adjustRangeForErrors(this.Luresource.Errors, newLineRange, startLine);
 
     // adjust the insert errors of section
@@ -208,20 +208,19 @@ class SectionOperator {
     const sectionsSize = newSections.length;
     const oldStartLine = this.Luresource.Sections[oldIndex].Range.Start.Line;
     const oldEndLine = this.Luresource.Sections[oldIndex].Range.End.Line;
-    const newStartLine = newSections[0].Range.Start.Line;
     const newEndLine = newSections[newSections.length - 1].Range.End.Line;
 
     this.Luresource.Sections.splice(oldIndex, 1, ...newSections);
 
     // adjust updated sections' range
-    const updateOffset = oldStartLine - this.Luresource.Sections[oldIndex].Range.Start.Line;
+    const updateOffset = oldStartLine - 1;
     for (let i = oldIndex; i < oldIndex + sectionsSize; i++) {
       const section = this.Luresource.Sections[i];
       this.adjustSectionRange(section, updateOffset);
     }
 
     // adjust remaining sections' range
-    const remainingOffset = (newEndLine - newStartLine) - (oldEndLine - oldStartLine);
+    const remainingOffset = (newEndLine - 1) - (oldEndLine - oldStartLine);
     for (let i = oldIndex + sectionsSize; i < this.Luresource.Sections.length; i++) {
       const section = this.Luresource.Sections[i];
       this.adjustSectionRange(section, remainingOffset);
@@ -230,8 +229,7 @@ class SectionOperator {
 
   adjustRangeForInsertSection(postIndex, newSections) {
     const sectionsSize = newSections.length;
-    const insertOffset = postIndex <= 0 ? 0 : this.Luresource.Sections[postIndex].Range.Start.Line - 1;
-    const newStartLine = newSections[0].Range.Start.Line;
+    const insertOffset = postIndex < 0 ? 0 : this.Luresource.Sections[postIndex].Range.Start.Line - 1;
     const newEndLine = newSections[newSections.length - 1].Range.End.Line;
 
     this.Luresource.Sections.splice(postIndex, 0, ...newSections);
@@ -243,7 +241,7 @@ class SectionOperator {
     }
 
     // adjust remaining sections' range
-    const remainingOffset = newEndLine - newStartLine + 1;
+    const remainingOffset = newEndLine;
     for (let i = postIndex + sectionsSize; i < this.Luresource.Sections.length; i++) {
       const section = this.Luresource.Sections[i];
       this.adjustSectionRange(section, remainingOffset);
diff --git a/packages/lu/src/parser/qnabuild/builder.ts b/packages/lu/src/parser/qnabuild/builder.ts
@@ -300,7 +300,9 @@ export class Builder {
     url: string,
     subscriptionkey: string,
     endpoint: string,
-    kbName: string) {
+    kbName: string,
+    enableHierarchicalExtraction: boolean = false,
+    defaultAnswerUsedForExtraction: string = 'More Answers') {
     const qnaBuildCore = new QnaBuildCore(subscriptionkey, endpoint)
     const kbs = (await qnaBuildCore.getKBList()).knowledgebases
 
@@ -319,7 +321,7 @@ export class Builder {
     }
 
     // create a new kb
-    kbId = await this.createUrlKB(qnaBuildCore, url, kbName)
+    kbId = await this.createUrlKB(qnaBuildCore, url, kbName, enableHierarchicalExtraction, defaultAnswerUsedForExtraction)
 
     const kbJson = await qnaBuildCore.exportKB(kbId, 'Test')
     const kb = new KB(kbJson)
@@ -334,7 +336,9 @@ export class Builder {
     fileUri: string,
     subscriptionkey: string,
     endpoint: string,
-    kbName: string) {
+    kbName: string,
+    enableHierarchicalExtraction: boolean = false,
+    defaultAnswerUsedForExtraction: string = 'More Answers') {
     const qnaBuildCore = new QnaBuildCore(subscriptionkey, endpoint)
     const kbs = (await qnaBuildCore.getKBList()).knowledgebases
 
@@ -353,7 +357,7 @@ export class Builder {
     }
 
     // create a new kb
-    kbId = await this.createFileKB(qnaBuildCore, fileName, fileUri, kbName)
+    kbId = await this.createFileKB(qnaBuildCore, fileName, fileUri, kbName, enableHierarchicalExtraction, defaultAnswerUsedForExtraction)
 
     const kbJson = await qnaBuildCore.exportKB(kbId, 'Test')
     const kb = new KB(kbJson)
@@ -481,12 +485,17 @@ export class Builder {
     return true
   }
 
-  async createUrlKB(qnaBuildCore: QnaBuildCore, url: string, kbName: string) {
-    const kbJson = {
+  async createUrlKB(qnaBuildCore: QnaBuildCore, url: string, kbName: string, enableHierarchicalExtraction: boolean, defaultAnswerUsedForExtraction: string) {
+    let kbJson: any = {
       name: kbName,
       qnaList: [],
       urls: [url],
-      files: []
+      files: [],
+    }
+
+    if (enableHierarchicalExtraction) {
+      kbJson.enableHierarchicalExtraction = true
+      kbJson.defaultAnswerUsedForExtraction = defaultAnswerUsedForExtraction
     }
 
     let response = await qnaBuildCore.importKB(kbJson)
@@ -497,8 +506,8 @@ export class Builder {
     return kbId
   }
 
-  async createFileKB(qnaBuildCore: QnaBuildCore, fileName: string, fileUri: string, kbName: string) {
-    let kbJson = {
+  async createFileKB(qnaBuildCore: QnaBuildCore, fileName: string, fileUri: string, kbName: string, enableHierarchicalExtraction: boolean, defaultAnswerUsedForExtraction: string) {
+    let kbJson: any = {
       name: kbName,
       qnaList: [],
       urls: [],
@@ -508,6 +517,11 @@ export class Builder {
       }]
     }
 
+    if (enableHierarchicalExtraction) {
+      kbJson.enableHierarchicalExtraction = true
+      kbJson.defaultAnswerUsedForExtraction = defaultAnswerUsedForExtraction
+    }
+
     let response = await qnaBuildCore.importKB(kbJson)
     let operationId = response.operationId
     const opResult = await this.getKBOperationStatus(qnaBuildCore, operationId, 1000)
diff --git a/packages/lu/test/parser/lufile/sectionapi.test.js b/packages/lu/test/parser/lufile/sectionapi.test.js
@@ -497,4 +497,148 @@ describe('Section range tests', () => {
         assert.equal(luresource.Sections[1].Range.End.Line, 11)
         assert.equal(luresource.Sections[1].Range.End.Character, 7)
     });
-})
+})
+
+describe('Section CRUD tests for insert and update sections with newline', () => {
+    let luresource = undefined;
+
+    let fileContent =
+`# ? who is CEO of Microsoft
+- Microsoft CEO
+
+\`\`\`
+Satya Nadella
+\`\`\``;
+
+    let updatedQnAConent =
+`# ? who is CEO of Facebook
+- Facebook CEO
+
+\`\`\`
+Mark Zuckerberg
+\`\`\``;
+
+    let insertQnAContent =
+`# ? how to greet
+
+\`\`\`
+hello
+\`\`\``
+
+    let insertQnAContent2 =
+`# ? how to cancel
+
+\`\`\`
+cancel that
+\`\`\``
+
+    let insertLuContent =
+`# welcome
+- welcome here`
+
+    let insertLuContent2 =
+`# stop
+- stop that`
+
+    it('update qna section test', () => {
+        luresource = luparser.parse(fileContent);
+
+        luresource = new SectionOperator(luresource).updateSection(luresource.Sections[0].Id, `${NEWLINE}${updatedQnAConent}${NEWLINE}`);
+
+        assert.equal(luresource.Errors.length, 0);
+        assert.equal(luresource.Sections.length, 1);
+        assert.equal(luresource.Sections[0].SectionType, LUSectionTypes.QNASECTION);
+        assert.equal(luresource.Sections[0].Body.replace(/\r\n/g, "\n"), `${updatedQnAConent}\n`);
+    });
+
+    it('insert qna section at begining test', () => {
+        luresource = new SectionOperator(luresource).insertSection(luresource.Sections[0].Id, `${NEWLINE}${insertQnAContent}`);
+
+        assert.equal(luresource.Errors.length, 0);
+        assert.equal(luresource.Sections.length, 2);
+        assert.equal(luresource.Sections[0].SectionType, LUSectionTypes.QNASECTION);
+        assert.equal(luresource.Sections[1].SectionType, LUSectionTypes.QNASECTION);
+        assert.equal(luresource.Sections[0].Body.replace(/\r\n/g, "\n"), `${insertQnAContent}`);
+        assert.equal(luresource.Sections[1].Body.replace(/\r\n/g, "\n"), `${updatedQnAConent}\n`);
+    });
+
+    it('insert qna section at middle test', () => {
+        luresource = new SectionOperator(luresource).insertSection(luresource.Sections[1].Id, `${NEWLINE}${NEWLINE}${insertQnAContent2}${NEWLINE}`);
+
+        assert.equal(luresource.Errors.length, 0);
+        assert.equal(luresource.Sections.length, 3);
+        assert.equal(luresource.Sections[0].SectionType, LUSectionTypes.QNASECTION);
+        assert.equal(luresource.Sections[1].SectionType, LUSectionTypes.QNASECTION);
+        assert.equal(luresource.Sections[2].SectionType, LUSectionTypes.QNASECTION);
+        assert.equal(luresource.Sections[0].Body.replace(/\r\n/g, "\n"), `${insertQnAContent}\n\n`);
+        assert.equal(luresource.Sections[1].Body.replace(/\r\n/g, "\n"), `${insertQnAContent2}\n`);
+        assert.equal(luresource.Sections[2].Body.replace(/\r\n/g, "\n"), `${updatedQnAConent}\n`);
+        assert.equal(luresource.Content.replace(/\r\n/g, "\n"), `\n\n${insertQnAContent}\n\n\n${insertQnAContent2}\n\n${updatedQnAConent}\n`);
+    });
+
+    it('insert lu section at begining test', () => {
+        luresource = new SectionOperator(luresource).insertSection(luresource.Sections[0].Id, `${NEWLINE}${insertLuContent}${NEWLINE}${NEWLINE}`);
+
+        assert.equal(luresource.Errors.length, 0);
+        assert.equal(luresource.Sections.length, 4);
+        assert.equal(luresource.Sections[0].SectionType, LUSectionTypes.SIMPLEINTENTSECTION);
+        assert.equal(`# welcome\n${luresource.Sections[0].Body.replace(/\r\n/g, "\n")}`, `${insertLuContent}\n\n`);
+    });
+
+    it('update qna section with lu section test', () => {
+        luresource = new SectionOperator(luresource).updateSection(luresource.Sections[3].Id, `${NEWLINE}${insertLuContent2}${NEWLINE}`);
+
+        assert.equal(luresource.Errors.length, 0);
+        assert.equal(luresource.Sections.length, 4);
+        assert.equal(luresource.Sections[0].SectionType, LUSectionTypes.SIMPLEINTENTSECTION);
+        assert.equal(luresource.Sections[1].SectionType, LUSectionTypes.QNASECTION);
+        assert.equal(luresource.Sections[2].SectionType, LUSectionTypes.QNASECTION);
+        assert.equal(luresource.Sections[3].SectionType, LUSectionTypes.SIMPLEINTENTSECTION);
+        assert.equal(`# welcome\n${luresource.Sections[0].Body.replace(/\r\n/g, "\n")}`, `${insertLuContent}\n\n`);
+        assert.equal(luresource.Sections[1].Body.replace(/\r\n/g, "\n"), `${insertQnAContent}\n\n`);
+        assert.equal(luresource.Sections[2].Body.replace(/\r\n/g, "\n"), `${insertQnAContent2}\n\n`);
+        assert.equal(`# stop\n${luresource.Sections[3].Body.replace(/\r\n/g, "\n")}`, `${insertLuContent2}\n`);
+        assert.equal(luresource.Content.replace(/\r\n/g, "\n"), `\n\n\n${insertLuContent}\n\n\n${insertQnAContent}\n\n\n${insertQnAContent2}\n\n\n${insertLuContent2}\n`);
+    });
+});
+
+describe('Section CRUD tests for import section', () => {
+    let luresource = undefined;
+
+    let importContent1 = `[import](howto.source.qna)`;
+    let importContent2 = ` [import](guide.source.qna)`;
+
+    let qnaContent = 
+`# ? who is CEO of Microsoft
+- Microsoft CEO
+
+\`\`\`
+Satya Nadella
+\`\`\``;
+
+    let insertImportConent = ` [import](windows.source.qna)`;
+
+    it('insert qna section test', () => {
+        luresource = luparser.parse(`${importContent1}${NEWLINE}${importContent2}${NEWLINE}${NEWLINE}${qnaContent}`);
+        luresource = new SectionOperator(luresource).insertSection(luresource.Sections[2].Id, `${NEWLINE}${insertImportConent}`);
+
+        assert.equal(luresource.Errors.length, 0);
+        assert.equal(luresource.Sections.length, 4);
+        assert.equal(luresource.Sections[0].SectionType, LUSectionTypes.IMPORTSECTION);
+        assert.equal(luresource.Sections[1].SectionType, LUSectionTypes.IMPORTSECTION);
+        assert.equal(luresource.Sections[2].SectionType, LUSectionTypes.IMPORTSECTION);
+        assert.equal(luresource.Sections[3].SectionType, LUSectionTypes.QNASECTION);
+        assert.equal(luresource.Content.replace(/\r\n/g, "\n"), `${importContent1}\n${importContent2}\n\n\n${insertImportConent}\n${qnaContent}`);
+    });
+
+    it('delete qna section test', () => {
+        luresource = new SectionOperator(luresource).deleteSection(luresource.Sections[1].Id);
+
+        assert.equal(luresource.Errors.length, 0);
+        assert.equal(luresource.Sections.length, 3);
+        assert.equal(luresource.Sections[0].SectionType, LUSectionTypes.IMPORTSECTION);
+        assert.equal(luresource.Sections[1].SectionType, LUSectionTypes.IMPORTSECTION);
+        assert.equal(luresource.Sections[2].SectionType, LUSectionTypes.QNASECTION);
+        assert.equal(luresource.Content.replace(/\r\n/g, "\n"), `${importContent1}\n\n\n${insertImportConent}\n${qnaContent}`);
+    });
+});

Original file line number	Diff line number	Diff line change
`@@ -78,11 +78,11 @@ AT`
`78`	`78`	`;`
`79`	`79`
`80`	`80`	`IMPORT`
`81`		`- : '[' ~[\r\n[\]]? ']' ('(' ~[\r\n()]? ')' \| '[' ~[\r\n[\]]*? ']')`
	`81`	`+ : WS* '[' ~[\r\n[\]]? ']' WS ('(' ~[\r\n()]? ')' \| '[' ~[\r\n[\]]? ']')`
`82`	`82`	`;`
`83`	`83`
`84`	`84`	`REFERENCE`
`85`		`- : '[' ~[\r\n[\]]? ']' WS ':' WS* ~[\r\n]*`
	`85`	`+ : WS* '[' ~[\r\n[\]]? ']' WS ':' WS* ~[\r\n]*`
`86`	`86`	`;`
`87`	`87`
`88`	`88`	`FILTER_MARK`