Phrase list as features to intents and entities

Vishwac Sena Kannan · Vishwac Sena Kannan · commit c959b47bb4f2 · 2019-09-30T19:08:07.000-07:00
diff --git a/packages/luis/.gitignore b/packages/luis/.gitignore
@@ -6,3 +6,4 @@
 /tmp
 /yarn.lock
 node_modules
+**/.antlr
diff --git a/packages/luis/src/parser/lufile/LUFileLexer.g4 b/packages/luis/src/parser/lufile/LUFileLexer.g4
@@ -100,7 +100,7 @@ HAS_FEATURES_LABEL
   ;
 
 NEW_ENTITY_TYPE_IDENTIFIER
-  : 'simple'|'list'|'regex'|'prebuilt'|'composite'|'machine-learned'|'patternany'|'phraselist'
+  : 'simple'|'list'|'regex'|'prebuilt'|'composite'|'machine-learned'|'patternany'|'phraselist'|'intent'
   ;
 
 NEW_ENTITY_IDENTIFIER
diff --git a/packages/luis/src/parser/lufile/classes/hclasses.js b/packages/luis/src/parser/lufile/classes/hclasses.js
@@ -130,6 +130,11 @@ const readerObj = {
         hasRole(value) {
             return this.roles.includes(value);
         }
+    },
+    intentFeature: class {
+        constructor(name) {
+            this.featureName = name ? name : '';
+        }
     }
 };
 
diff --git a/packages/luis/src/parser/lufile/enums/luisEntityTypeNameMap.js b/packages/luis/src/parser/lufile/enums/luisEntityTypeNameMap.js
@@ -0,0 +1,14 @@
+/**
+ * Copyright (c) Microsoft Corporation. All rights reserved.
+ * Licensed under the MIT License.
+ */
+module.exports = {
+    'simple': 'entities',
+    'prebuilt': 'prebuiltEntities',
+    'list': 'closedLists',
+    'regex': 'regex_entities',
+    'composite': 'composites',
+    'machine-learned': 'entities',
+    'patternany':'patternAnyEntities',
+    'phraselist': 'model_features'
+};
diff --git a/packages/luis/src/parser/lufile/generated/LUFileLexer.interp b/packages/luis/src/parser/lufile/generated/LUFileLexer.interp
diff --git a/packages/luis/src/parser/lufile/generated/LUFileLexer.js b/packages/luis/src/parser/lufile/generated/LUFileLexer.js
diff --git a/packages/luis/src/parser/lufile/generated/LUFileParser.js b/packages/luis/src/parser/lufile/generated/LUFileParser.js
@@ -1,4 +1,4 @@
-// Generated from ../LUFileParser.g4 by ANTLR 4.7.2
+// Generated from LUFileParser.g4 by ANTLR 4.7.2
 // jshint ignore: start
 var antlr4 = require('antlr4/index');
 var LUFileParserListener = require('./LUFileParserListener').LUFileParserListener;
diff --git a/packages/luis/src/parser/lufile/generated/LUFileParserListener.js b/packages/luis/src/parser/lufile/generated/LUFileParserListener.js
@@ -1,4 +1,4 @@
-// Generated from ../LUFileParser.g4 by ANTLR 4.7.2
+// Generated from LUFileParser.g4 by ANTLR 4.7.2
 // jshint ignore: start
 var antlr4 = require('antlr4/index');
 
diff --git a/packages/luis/src/parser/lufile/generated/LUFileParserVisitor.js b/packages/luis/src/parser/lufile/generated/LUFileParserVisitor.js
@@ -1,4 +1,4 @@
-// Generated from ../LUFileParser.g4 by ANTLR 4.7.2
+// Generated from LUFileParser.g4 by ANTLR 4.7.2
 // jshint ignore: start
 var antlr4 = require('antlr4/index');
 
diff --git a/packages/luis/src/parser/lufile/parseFileContents.js b/packages/luis/src/parser/lufile/parseFileContents.js
diff --git a/packages/luis/test/parser/lufile/parseFileContent.modelAsFeature.test.js b/packages/luis/test/parser/lufile/parseFileContent.modelAsFeature.test.js
diff --git a/packages/luis/test/parser/lufile/parseFileContents.@EntityDefinition.test.js b/packages/luis/test/parser/lufile/parseFileContents.@EntityDefinition.test.js
@@ -652,6 +652,26 @@ describe('V2 Entity definitions using @ notation', function () {
                 .catch(err => done(err))
         });
 
+        it('Interchangeable phrase list definition can be separated', function(done) {
+            let luFile = `
+                @phraselist xyz(interchangeable)
+                @xyz =
+                    - one;two
+                @xyz =
+                    - three, four
+            `;
+
+            parseFile.parseFile(luFile)
+                .then(res => {
+                    assert.equal(res.LUISJsonStructure.model_features.length, 1);
+                    assert.equal(res.LUISJsonStructure.model_features[0].name, 'xyz');
+                    assert.equal(res.LUISJsonStructure.model_features[0].mode, true);
+                    assert.equal(res.LUISJsonStructure.model_features[0].words, 'one,two,three,four');
+                    done();
+                })
+                .catch(err => done(err))
+        })
+
         it('missing entity name for phrase list throws', function(done) {
             let luFile = `
                 @xyz = 
@@ -910,6 +930,20 @@ describe('V2 Entity definitions using @ notation', function () {
                 })
                 .catch(err => done(err))
         });
+
+        it('Duplicate composite entity definition throws', function(done){
+            let luFile = `
+                @composite x1 = [s1, number]
+                @simple s1
+                @prebuilt number
+                @composite x1 = [s1, age]
+                @prebuilt age
+            `;
+
+            parseFile.parseFile(luFile)
+                .then(res => done(res))
+                .catch(err => done())
+        })
     });
 
     describe('Closed list definitions', function() {

Original file line number	Diff line number	Diff line change
`@@ -100,7 +100,7 @@ HAS_FEATURES_LABEL`
`100`	`100`	`;`
`101`	`101`
`102`	`102`	`NEW_ENTITY_TYPE_IDENTIFIER`
`103`		`- : 'simple'\|'list'\|'regex'\|'prebuilt'\|'composite'\|'machine-learned'\|'patternany'\|'phraselist'`
	`103`	`+ : 'simple'\|'list'\|'regex'\|'prebuilt'\|'composite'\|'machine-learned'\|'patternany'\|'phraselist'\|'intent'`
`104`	`104`	`;`
`105`	`105`
`106`	`106`	`NEW_ENTITY_IDENTIFIER`
Original file line number	Diff line number	Diff line change
`@@ -130,6 +130,11 @@ const readerObj = {`
`130`	`130`	`hasRole(value) {`
`131`	`131`	`return this.roles.includes(value);`
`132`	`132`	`}`
	`133`	`+ },`
	`134`	`+ intentFeature: class {`
	`135`	`+ constructor(name) {`
	`136`	`+ this.featureName = name ? name : '';`
	`137`	`+ }`
`133`	`138`	`}`
`134`	`139`	`};`
`135`	`140`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-// Generated from ../LUFileParser.g4 by ANTLR 4.7.2`
	`1`	`+// Generated from LUFileParser.g4 by ANTLR 4.7.2`
`2`	`2`	`// jshint ignore: start`
`3`	`3`	`var antlr4 = require('antlr4/index');`
`4`	`4`	`var LUFileParserListener = require('./LUFileParserListener').LUFileParserListener;`