Skip to content
This repository was archived by the owner on Jan 15, 2025. It is now read-only.

Commit 7816875

Browse files
Danieladufeich-ms
Danieladu
and
feich-ms
authored
Improve the CRUD performance of LU sections (#901)
* init * init * init * refine * init * init * fix test * fix test errors * fix error * fix error * fix comments * fix comments * refine * clone deep lu resource * use cloneDeepWith wo clone luResource * remove antlr parser context from UtteranceAndEntitiesMap Co-authored-by: Fei Chen <[email protected]>
1 parent 9de7f4c commit 7816875

15 files changed

+451
-155
lines changed

‎packages/lu/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@
4747
"get-stdin": "^6.0.0",
4848
"globby": "^10.0.1",
4949
"intercept-stdout": "^0.1.2",
50-
"lodash": "^4.17.15",
50+
"lodash": "^4.17.19",
5151
"node-fetch": "~2.6.0",
5252
"semver": "^5.5.1",
5353
"tslib": "^1.10.0"

‎packages/lu/src/parser/lufile/baseSection.js‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
class BaseSection {
2+
constructor(parameters) {
3+
this.Errors = [];
4+
this.SectionType = '';
5+
this.Id = '';
6+
this.Body = '';
7+
this.Range;
8+
9+
if (parameters) {
10+
this.Errors = parameters.Errors || [];
11+
this.SectionType = parameters.SectionType || '';
12+
this.Id = parameters.Id || '';
13+
this.Body = parameters.Body || '';
14+
this.Range = parameters.Range;
15+
}
16+
}
17+
}
18+
19+
module.exports = BaseSection;

‎packages/lu/src/parser/lufile/diagnostic.js‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ class Diagnostic {
99
}
1010

1111
toString() {
12-
// ignore error range if source is "inline"
1312
if (this.Range === undefined) {
1413
return `[${DiagnosticSeverity[this.Severity]}] ${this.Message.toString()}`;
1514
}
@@ -78,8 +77,13 @@ const BuildDiagnostic =function(parameter) {
7877
const severity = parameter.severity === undefined ? DiagnosticSeverity.ERROR : parameter.severity;
7978

8079
let range;
80+
const rangeInput = parameter.range;
8181
const context = parameter.context;
82-
if (context !== undefined) {
82+
if (rangeInput !== undefined) {
83+
const startPosition = new Position(rangeInput.Start.Line, rangeInput.Start.Character);
84+
const stopPosition = new Position(rangeInput.End.Line, rangeInput.End.Character);
85+
range = new Range(startPosition, stopPosition);
86+
} else if (context !== undefined) {
8387
const startPosition = new Position(context.start.line, context.start.column);
8488
const stopPosition = new Position(context.stop.line, context.stop.column + context.stop.text.length);
8589
range = new Range(startPosition, stopPosition);

‎packages/lu/src/parser/lufile/entitySection.js‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,20 +3,25 @@ const DiagnosticSeverity = require('./diagnostic').DiagnosticSeverity;
33
const BuildDiagnostic = require('./diagnostic').BuildDiagnostic;
44
const LUSectionTypes = require('./../utils/enums/lusectiontypes');
55
const InvalidCharsInIntentOrEntityName = require('./../utils/enums/invalidchars').InvalidCharsInIntentOrEntityName;
6+
const BaseSection = require('./baseSection');
7+
const Range = require('./diagnostic').Range;
8+
const Position = require('./diagnostic').Position;
69

7-
class EntitySection {
10+
class EntitySection extends BaseSection {
811
/**
912
*
1013
* @param {EntitySectionContext} parseTree
1114
*/
1215
constructor(parseTree) {
13-
this.ParseTree = parseTree;
16+
super();
1417
this.SectionType = LUSectionTypes.ENTITYSECTION;
15-
this.Errors = []
1618
this.Name = this.ExtractName(parseTree);
1719
this.Type = this.ExtractType(parseTree);
1820
this.SynonymsOrPhraseList = this.ExtractSynonymsOrPhraseList(parseTree);
1921
this.Id = `${this.SectionType}_${this.Name}`;
22+
const startPosition = new Position(parseTree.start.line, parseTree.start.column);
23+
const stopPosition = new Position(parseTree.stop.line, parseTree.stop.column + parseTree.stop.text.length);
24+
this.Range = new Range(startPosition, stopPosition);
2025
}
2126

2227
ExtractName(parseTree) {

‎packages/lu/src/parser/lufile/importSection.js‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,26 @@
11
const ImportSectionContext = require('./generated/LUFileParser').LUFileParser.ImportSectionContext;
22
const BuildDiagnostic = require('./diagnostic').BuildDiagnostic;
33
const LUSectionTypes = require('./../utils/enums/lusectiontypes');
4+
const BaseSection = require('./baseSection');
5+
const Range = require('./diagnostic').Range;
6+
const Position = require('./diagnostic').Position;
47

5-
class ImportSection {
8+
class ImportSection extends BaseSection {
69
/**
710
*
811
* @param {ImportSectionContext} parseTree
912
*/
1013
constructor(parseTree) {
11-
this.ParseTree = parseTree;
14+
super();
1215
this.Errors = []
1316
this.SectionType = LUSectionTypes.IMPORTSECTION;
1417
let result = this.ExtractDescriptionAndPath(parseTree);
1518
this.Description = result.description;
1619
this.Path = result.path;
17-
this.Id = `${this.SectionType}_${this.Path}`;;
20+
this.Id = `${this.SectionType}_${this.Path}`;
21+
const startPosition = new Position(parseTree.start.line, parseTree.start.column);
22+
const stopPosition = new Position(parseTree.stop.line, parseTree.stop.column + parseTree.stop.text.length);
23+
this.Range = new Range(startPosition, stopPosition);
1824
}
1925

2026
ExtractDescriptionAndPath(parseTree) {

‎packages/lu/src/parser/lufile/luParser.js‎

Lines changed: 41 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -14,22 +14,42 @@ const LUErrorListener = require('./luErrorListener');
1414
const SectionType = require('./../utils/enums/lusectiontypes');
1515
const DiagnosticSeverity = require('./diagnostic').DiagnosticSeverity;
1616
const BuildDiagnostic = require('./diagnostic').BuildDiagnostic;
17+
const Range = require('./diagnostic').Range;
18+
const Position = require('./diagnostic').Position;
1719
const NEWLINE = require('os').EOL;
1820

1921
class LUParser {
22+
2023
/**
21-
* @param {string} text
24+
*
25+
* @param {string} text
26+
* @param {LUResource} luResource
2227
*/
23-
static parse(text) {
28+
static parseWithRef(text, luResource) {
2429
if (text === undefined || text === '') {
2530
return new LUResource([], '', []);
2631
}
2732

28-
let sections = [];
29-
let content = text;
33+
const sectionEnabled = luResource ? this.isSectionEnabled(luResource.Sections) : undefined;
34+
35+
return this.parse(text, sectionEnabled);
36+
}
37+
38+
/**
39+
* @param {string} text
40+
*/
41+
static parse(text, sectionEnabled) {
42+
if (text === undefined || text === '') {
43+
return new LUResource([], '', []);
44+
}
3045

3146
let {fileContent, errors} = this.getFileContent(text);
3247

48+
return this.extractFileContent(fileContent, text, errors, sectionEnabled);
49+
}
50+
51+
static extractFileContent(fileContent, content, errors, sectionEnabled) {
52+
let sections = [];
3353
try {
3454
let modelInfoSections = this.extractModelInfoSections(fileContent);
3555
modelInfoSections.forEach(section => errors = errors.concat(section.Errors));
@@ -41,7 +61,7 @@ class LUParser {
4161
}
4262

4363
try {
44-
let isSectionEnabled = this.isSectionEnabled(sections);
64+
let isSectionEnabled = sectionEnabled === undefined ? this.isSectionEnabled(sections) : sectionEnabled;
4565

4666
let nestedIntentSections = this.extractNestedIntentSections(fileContent, content);
4767
nestedIntentSections.forEach(section => errors = errors.concat(section.Errors));
@@ -50,12 +70,21 @@ class LUParser {
5070
} else {
5171
nestedIntentSections.forEach(section => {
5272
let emptyIntentSection = new SimpleIntentSection();
53-
emptyIntentSection.ParseTree = section.ParseTree.nestedIntentNameLine();
5473
emptyIntentSection.Name = section.Name;
74+
emptyIntentSection.Id = `${emptyIntentSection.SectionType}_${emptyIntentSection.Name}`
75+
76+
// get the end character index
77+
const firstLine = content.split(/\r?\n/)[0];
78+
let endCharacter = section.Name.length + 2;
79+
if (firstLine.includes(section.Name)) {
80+
endCharacter = firstLine.length;
81+
}
82+
const range = new Range(section.Range.Start, new Position(section.Range.Start.Line, endCharacter))
83+
emptyIntentSection.Range = range;
5584
let errorMsg = `no utterances found for intent definition: "# ${emptyIntentSection.Name}"`
5685
let error = BuildDiagnostic({
5786
message: errorMsg,
58-
context: emptyIntentSection.ParseTree,
87+
range: emptyIntentSection.Range,
5988
severity: DiagnosticSeverity.WARN
6089
})
6190

@@ -301,23 +330,24 @@ class LUParser {
301330
* @param {string} content
302331
*/
303332
static extractSectionBody(sections, content) {
304-
sections.sort((a, b) => a.ParseTree.start.line - b.ParseTree.start.line)
333+
sections.sort((a, b) => a.Range.Start.Line - b.Range.Start.Line)
305334
const originList = content.split(/\r?\n/)
306335
let qnaSectionIndex = 0
307336
sections.forEach(function (section, index) {
308337
if (section.SectionType === SectionType.SIMPLEINTENTSECTION
309338
|| section.SectionType === SectionType.NESTEDINTENTSECTION
310339
|| section.SectionType === SectionType.QNASECTION) {
311-
const startLine = section.ParseTree.start.line - 1
340+
const startLine = section.Range.Start.Line - 1;
312341
let stopLine
313342
if (index + 1 < sections.length) {
314-
stopLine = sections[index + 1].ParseTree.start.line - 1
315-
if (isNaN(startLine) || isNaN(stopLine) || startLine < 0 || startLine >= stopLine || originList.Length <= stopLine) {
343+
stopLine = sections[index + 1].Range.Start.Line - 1
344+
if (isNaN(startLine) || isNaN(stopLine) || startLine < 0 || startLine > stopLine) {
316345
throw new Error("index out of range.")
317346
}
318347
} else {
319348
stopLine = originList.length
320349
}
350+
section.Range.End.Line = stopLine;
321351

322352
let destList
323353
if (section.SectionType === SectionType.QNASECTION) {
@@ -329,15 +359,10 @@ class LUParser {
329359
}
330360

331361
section.Body = destList.join(NEWLINE)
332-
section.StartLine = startLine
333-
section.StopLine = stopLine - 1
334362

335363
if (section.SectionType === SectionType.NESTEDINTENTSECTION) {
336364
LUParser.extractSectionBody(section.SimpleIntentSections, originList.slice(0, stopLine).join(NEWLINE))
337365
}
338-
} else {
339-
section.StartLine = section.ParseTree.start.line
340-
section.StopLine = section.ParseTree.stop.line - 1
341366
}
342367
})
343368
}

‎packages/lu/src/parser/lufile/luResource.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,9 @@ class LUResource {
55
* @param {any[]} errors
66
*/
77
constructor(sections, content, errors) {
8-
this.Sections = sections;
8+
this.Sections = sections || [];
99
this.Content = content;
10-
this.Errors = errors;
10+
this.Errors = errors || [];
1111
}
1212
}
1313

‎packages/lu/src/parser/lufile/modelInfoSection.js‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,23 @@
11
const ModelInfoSectionContext = require('./generated/LUFileParser').LUFileParser.ModelInfoSectionContext;
22
const LUSectionTypes = require('./../utils/enums/lusectiontypes');
3+
const BaseSection = require('./baseSection');
4+
const Range = require('./diagnostic').Range;
5+
const Position = require('./diagnostic').Position;
36

4-
class LUModelInfo {
7+
class LUModelInfo extends BaseSection {
58
/**
69
*
710
* @param {ModelInfoSectionContext} parseTree
811
*/
912
constructor(parseTree) {
10-
this.ParseTree = parseTree;
13+
super();
1114
this.SectionType = LUSectionTypes.MODELINFOSECTION;
1215
this.ModelInfo = parseTree.modelInfoDefinition().getText();
1316
this.Errors = [];
1417
this.Id = `${this.SectionType}_${this.ModelInfo}`;
18+
const startPosition = new Position(parseTree.start.line, parseTree.start.column);
19+
const stopPosition = new Position(parseTree.stop.line, parseTree.stop.column + parseTree.stop.text.length);
20+
this.Range = new Range(startPosition, stopPosition);
1521
}
1622
}
1723

‎packages/lu/src/parser/lufile/nestedIntentSection.js‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,17 @@ const NestedIntentSectionContext = require('./generated/LUFileParser').LUFilePar
22
const SimpleIntentSection = require('./simpleIntentSection');
33
const LUSectionTypes = require('./../utils/enums/lusectiontypes');
44
const NEWLINE = require('os').EOL;
5+
const BaseSection = require('./baseSection');
6+
const Range = require('./diagnostic').Range;
7+
const Position = require('./diagnostic').Position;
58

6-
class NestedIntentSection {
9+
class NestedIntentSection extends BaseSection {
710
/**
811
*
912
* @param {NestedIntentSectionContext} parseTree
1013
*/
1114
constructor(parseTree, content) {
12-
this.ParseTree = parseTree;
15+
super();
1316
this.SectionType = LUSectionTypes.NESTEDINTENTSECTION;
1417
this.Name = this.ExtractName(parseTree);
1518
this.Body = this.ExtractBody(parseTree, content);
@@ -22,15 +25,18 @@ class NestedIntentSection {
2225
}
2326

2427
this.Id = `${this.SectionType}_${this.Name}`;
28+
const startPosition = new Position(parseTree.start.line, parseTree.start.column);
29+
const stopPosition = new Position(parseTree.stop.line, parseTree.stop.column + parseTree.stop.text.length);
30+
this.Range = new Range(startPosition, stopPosition);
2531
}
2632

2733
ExtractName(parseTree) {
2834
return parseTree.nestedIntentNameLine().nestedIntentName().getText().trim();
2935
}
3036

3137
ExtractBody(parseTree, content) {
32-
const startLine = parseTree.start.line - 1
33-
const stopLine = parseTree.stop.line - 1
38+
const startLine = parseTree.start.line - 1;
39+
const stopLine = parseTree.stop.line - 1;
3440
const originList = content.split(/\r?\n/)
3541
if (isNaN(startLine) || isNaN(stopLine) || startLine < 0 || startLine > stopLine || originList.Length <= stopLine) {
3642
throw new Error("index out of range.")

‎packages/lu/src/parser/lufile/newEntitySection.js‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,17 @@ const DiagnosticSeverity = require('./diagnostic').DiagnosticSeverity;
33
const BuildDiagnostic = require('./diagnostic').BuildDiagnostic;
44
const LUSectionTypes = require('./../utils/enums/lusectiontypes');
55
const InvalidCharsInIntentOrEntityName = require('./../utils/enums/invalidchars').InvalidCharsInIntentOrEntityName;
6+
const BaseSection = require('./baseSection');
7+
const Range = require('./diagnostic').Range;
8+
const Position = require('./diagnostic').Position;
69

7-
class NewEntitySection {
10+
class NewEntitySection extends BaseSection {
811
/**
912
*
1013
* @param {NewEntitySectionContext} parseTree
1114
*/
1215
constructor(parseTree) {
13-
this.ParseTree = parseTree;
16+
super();
1417
this.SectionType = LUSectionTypes.NEWENTITYSECTION;
1518
this.Errors = []
1619
this.Name = this.ExtractName(parseTree);
@@ -21,6 +24,9 @@ class NewEntitySection {
2124
this.RegexDefinition = this.ExtractRegexDefinition(parseTree);
2225
this.ListBody = this.ExtractSynonymsOrPhraseList(parseTree);
2326
this.Id = `${this.SectionType}_${this.Name}`;
27+
const startPosition = new Position(parseTree.start.line, parseTree.start.column);
28+
const stopPosition = new Position(parseTree.stop.line, parseTree.stop.column + parseTree.stop.text.length);
29+
this.Range = new Range(startPosition, stopPosition);
2430
}
2531

2632
ExtractName(parseTree) {

0 commit comments

Comments
 (0)