Merge branch 'master' into emimunoz/qnamaker

Author: munozemilio

packages/luis/.gitignore

@@ -7,3 +7,4 @@
 /yarn.lock
 node_modules
 **/.antlr
+**/java_generated

packages/luis/src/commands/luis/convert.ts

@@ -54,7 +54,6 @@ export default class LuisConvert extends Command {
         result.desc = flags.desc || result.desc || ''
         result.culture = flags.culture || result.culture || 'en-us'
         result.culture = result.culture.toLowerCase()
-        if (result.flatListOfEntityAndRoles) delete result.flatListOfEntityAndRoles
         result = JSON.stringify(result, null, 2)
       }
 

packages/luis/src/parser/converters/luistoluconverter.js

@@ -6,7 +6,7 @@ const luisFile = require('./../luisfile/parseLuisFile')
 const helperClasses = require('./../lufile/classes/hclasses')
 const exception = require('./../lufile/classes/exception')
 const retCode = require('./../lufile/enums/CLI-errors')
-
+const EntityTypeEnum = require('./../lufile/enums/luisEntityTypes');
 module.exports = {
     parseLuisFileToLu: async function(file, sort) {
         let LUISFileContent = await openFileAndReadContent(file)
@@ -33,6 +33,7 @@ module.exports = {
         updateUtterancesList(LUISJSON.patterns, luisObj.intents, 'pattern');
         if(luisObj.intents.length >= 0) {
             fileContent += NEWLINE;
+            fileContent += addAppMetaData(LUISJSON);
             fileContent += '> # Intent definitions' + NEWLINE + NEWLINE;
             // write out intents and utterances..
             luisObj.intents.forEach(function(intent) {
@@ -90,21 +91,26 @@ module.exports = {
         if(LUISJSON.entities && LUISJSON.entities.length >= 0) {
             fileContent += '> # Entity definitions' + NEWLINE + NEWLINE;
             LUISJSON.entities.forEach(function(entity) {
-                // Add inherits information if any
-                if (entity.inherits !== undefined) {
-                    // > !# @intent.inherits = {name = Web.WebSearch; domain_name = Web; model_name = WebSearch}
-                    fileContent += '> !# @entity.inherits = name : ' + entity.name;
-                    if (entity.inherits.domain_name !== undefined) {
-                        fileContent += '; domain_name : ' + entity.inherits.domain_name;
-                    }
-                    if (entity.inherits.model_name !== undefined) {
-                        fileContent += '; model_name : ' + entity.inherits.model_name;
+                if (!entity.children || entity.children.length === 0) {
+                    // Add inherits information if any
+                    if (entity.inherits !== undefined) {
+                        // > !# @intent.inherits = {name = Web.WebSearch; domain_name = Web; model_name = WebSearch}
+                        fileContent += '> !# @entity.inherits = name : ' + entity.name;
+                        if (entity.inherits.domain_name !== undefined) {
+                            fileContent += '; domain_name : ' + entity.inherits.domain_name;
+                        }
+                        if (entity.inherits.model_name !== undefined) {
+                            fileContent += '; model_name : ' + entity.inherits.model_name;
+                        }
+                        fileContent += NEWLINE + NEWLINE;
                     }
+                    fileContent += `@ ml ${entity.name}`;
+                    fileContent += addRolesAndFeatures(entity);
                     fileContent += NEWLINE + NEWLINE;
+                } else {
+                    // handle n-depth entity
+                    fileContent += handleNDepthEntity(entity);
                 }
-                fileContent += `@ simple ${entity.name}`;
-                fileContent += addRolesAndFeatures(entity);
-                fileContent += NEWLINE + NEWLINE;
             });
             fileContent += NEWLINE;
         }
@@ -170,15 +176,69 @@ module.exports = {
                 fileContent += `@ composite ${composite.name}`;
                 fileContent += addRolesAndFeatures(composite);
                 if (composite.children.length > 0) {
-                    fileContent += ` = [${composite.children.join(', ')}]`;
+                    if (typeof composite.children[0] == "object") {
+                        fileContent += ` = [${composite.children.map(item => item.name).join(', ')}]`;
+                    } else {
+                        fileContent += ` = [${composite.children.join(', ')}]`;
+                    }
                 }
                 fileContent += NEWLINE;
             })
         }
         return fileContent;
     }
 }
-
+/**
+ * Helper to add application inforamtion metadata
+ * @param {Object} LUISJSON 
+ */
+const addAppMetaData = function(LUISJSON) {
+    let fileContent = '';
+    if (LUISJSON.name) fileContent += `> !# @app.name = ${LUISJSON.name}` + NEWLINE;
+    if (LUISJSON.desc) fileContent += `> !# @app.desc = ${LUISJSON.desc}` + NEWLINE;
+    if (LUISJSON.versionId) fileContent += `> !# @app.versionId = ${LUISJSON.versionId}` + NEWLINE;
+    if (LUISJSON.culture) fileContent += `> !# @app.culture = ${LUISJSON.culture}` + NEWLINE;
+    if (LUISJSON.luis_schema_version) fileContent += `> !# @app.luis_schema_version = ${LUISJSON.luis_schema_version}` + NEWLINE;
+    return fileContent === '' ? fileContent : `> LUIS application information` + NEWLINE + fileContent + NEWLINE + NEWLINE;
+}
+/**
+ * Helper function to handle nDepth entity definition
+ * @param {Object} entity 
+ */
+const handleNDepthEntity = function(entity) {
+    let fileContent = '';
+    const BASE_TAB_STOP = 1;
+    fileContent += `@ ${EntityTypeEnum.ML} ${entity.name}`;
+    fileContent += addRolesAndFeatures(entity);
+    fileContent += NEWLINE;
+    fileContent += addNDepthChildDefinitions(entity.children, BASE_TAB_STOP, fileContent) + NEWLINE + NEWLINE
+    return fileContent;
+}
+/**
+ * Recursive helper function to add child definitions. 
+ * @param {Object[]} childCollection 
+ * @param {number} tabStop 
+ * @param {string} fileContent 
+ */
+const addNDepthChildDefinitions = function(childCollection, tabStop, fileContent) {
+    let myFileContent = '';
+    (childCollection || []).forEach(child => {
+        myFileContent += "".padStart(tabStop * 4, ' ');
+        myFileContent += '- @ ';
+        if (child.instanceOf) {
+            myFileContent += child.instanceOf;
+        } else {
+            myFileContent += EntityTypeEnum.ML;
+        }
+        myFileContent += ` ${child.name}`;
+        myFileContent += addRolesAndFeatures(child);
+        myFileContent += NEWLINE;
+        if (child.children && child.children.length !== 0) {
+            myFileContent += addNDepthChildDefinitions(child.children, ++tabStop, myFileContent);
+        }
+    });
+    return myFileContent;
+}
 /**
  * Helper to construt role and features list for an entity
  * @param {Object} entity 
@@ -278,17 +338,6 @@ const objectSortByStartPos = function (objectArray) {
     return ObjectByStartPos;
 }
 
-constructModelDescFromLUISJSON = async function(LUISJSON) {
-    let modelDesc = NEWLINE;
-    modelDesc += '> LUIS application information' + NEWLINE;
-    modelDesc += '> !# @app.name = ' + LUISJSON.name + NEWLINE;
-    modelDesc += '> !# @app.desc = ' + LUISJSON.desc + NEWLINE;
-    modelDesc += '> !# @app.culture = ' + LUISJSON.culture + NEWLINE;
-    modelDesc += '> !# @app.versionId = ' + LUISJSON.versionId + NEWLINE;
-    modelDesc += '> !# @app.luis_schema_version = ' + LUISJSON.luis_schema_version + NEWLINE;
-    return modelDesc;
-}
-
 /**
  * Helper function to return sorted LUIS JSON model
  * @param {Object} LUISJSON 

packages/luis/src/parser/converters/lutoluisconverter.js

@@ -25,6 +25,7 @@ module.exports = {
             if (haveLUISContent(finalLUISJSON)) {
                 await luisJSON.validateLUISBlob(finalLUISJSON)
             }
+            checkAndUpdateVersion(finalLUISJSON)
             return finalLUISJSON
         } catch (err) {
             throw(err)
@@ -250,7 +251,31 @@ const mergeResults_closedlists = function (blob, finalCollection, type) {
         });
     }
 }
-
+/**
+ * Helper to detect luis schema version based on content and update the final payload as needed.
+ * @param {LUIS} finalLUISJSON 
+ */
+const checkAndUpdateVersion = function(finalLUISJSON) {
+    // Detect if there is content specific to 5.0.0 schema
+    // any entity with children
+    let v5DefFound = false;
+    v5DefFound = (finalLUISJSON.entities || []).find(i => i.children || i.features) ||
+                 (finalLUISJSON.intents || []).find(i => i.features) || 
+                 (finalLUISJSON.composites || []).find(i => i.features);
+    if (v5DefFound) {
+        finalLUISJSON.luis_schema_version = "5.0.0";
+        if (finalLUISJSON.model_features && finalLUISJSON.model_features.length !== 0) {
+            finalLUISJSON.phraselists = [];
+            finalLUISJSON.model_features.forEach(item => finalLUISJSON.phraselists.push(Object.assign({}, item)));
+            delete finalLUISJSON.model_features;
+        }
+        (finalLUISJSON.composites || []).forEach(composite => {
+            let children = composite.children;
+            composite.children = [];
+            children.forEach(c => composite.children.push({name : c}));
+        })
+    }
+}
 /**
  * Helper function to see if we have any luis content in the blob
  * @param {object} blob Contents of parsed luis blob

packages/luis/src/parser/index.js

@@ -4,7 +4,6 @@
  */
 const modules = {
     parser: {
-        parseLuList: require('./converters/lutoluisconverter').parseLuToLuis,
         parseFile: require('./lufile/parseFileContents').parseFile,
         validateLUISBlob: require('./luisfile/parseLuisFile').validateLUISBlob
     },

packages/luis/src/parser/lufile/LUFileLexer.g4

@@ -14,15 +14,15 @@ fragment WHITESPACE
 fragment UTTERANCE_MARK: '-' | '*' | '+';
 
 MODEL_INFO
-  : '>' WHITESPACE* '!#' ~('\r'|'\n')+
+  : WS* '>' WHITESPACE* '!#' ~('\r'|'\n')+
   ;
 
 COMMENT
-  : '>' ~('\r'|'\n')+ -> skip
+  : WS* '>' ~('\r'|'\n')+ -> skip
   ;
 
 WS
-  : WHITESPACE+ -> skip
+  : WHITESPACE+
   ;
 
 NEWLINE
@@ -100,7 +100,7 @@ HAS_FEATURES_LABEL
   ;
 
 NEW_ENTITY_TYPE_IDENTIFIER
-  : 'simple'|'list'|'regex'|'prebuilt'|'composite'|'machine-learned'|'patternany'|'phraselist'|'intent'
+  : 'simple'|'list'|'regex'|'prebuilt'|'composite'|'ml'|'patternany'|'phraselist'|'intent'
   ;
 
 NEW_ENTITY_IDENTIFIER

packages/luis/src/parser/lufile/LUFileParser.g4

@@ -20,16 +20,15 @@ paragraph
 // It's possible that parser doesn't even have to handle NEWLINE, 
 // but before the syntax is finalized, we still keep the NEWLINE in grammer 
 newline
-    : NEWLINE
-    | EOF
+    : WS* (NEWLINE | EOF)
     ;
 
 intentDefinition
 	: intentNameLine newline intentBody?
 	;
 
 intentNameLine
-	: HASH intentName
+	: WS* HASH intentName
 	;
 
 intentName
@@ -41,15 +40,15 @@ intentNameIdentifier
     ;
 
 intentBody
-	: normalIntentBody
+	: WS* normalIntentBody
 	;
 
 normalIntentBody
-    : (normalIntentString newline)+
+    : WS* (normalIntentString newline)+
     ;
 
 normalIntentString
-	: DASH (WS|TEXT|EXPRESSION|ESCAPE_CHARACTER)*
+	: WS* DASH (WS|TEXT|EXPRESSION|ESCAPE_CHARACTER)*
 	;
 
 newEntityDefinition
@@ -61,7 +60,7 @@ newEntityListbody
     ;
 
 newEntityLine
-    : AT newEntityType? (newEntityName|newEntityNameWithWS) newEntityRoles? newEntityUsesFeatures? NEW_EQUAL? (newCompositeDefinition|newRegexDefinition)?
+    : WS* AT newEntityType? (newEntityName|newEntityNameWithWS) newEntityRoles? newEntityUsesFeatures? NEW_EQUAL? (newCompositeDefinition|newRegexDefinition)?
     ;
 
 newCompositeDefinition
@@ -105,7 +104,7 @@ entityDefinition
     ;
 
 entityLine
-    : DOLLAR entityName COLON_MARK entityType
+    : WS* DOLLAR entityName COLON_MARK entityType
     ;
 
 entityName
@@ -133,7 +132,7 @@ entityListBody
     ;
 
 normalItemString
-    : DASH (WS|TEXT|EXPRESSION)*
+    : WS* DASH (WS|TEXT|EXPRESSION)*
     ;
 
 importDefinition
@@ -145,15 +144,15 @@ qnaDefinition
     ;
 
 qnaQuestion
-    : QNA questionText newline
+    : WS* QNA questionText newline
     ;
 
 questionText
     : (WS|QNA_TEXT)*
     ;
 
 moreQuestionsBody
-    : (moreQuestion newline)*
+    : WS* (moreQuestion newline)*
     ;
 
 moreQuestion
@@ -165,17 +164,17 @@ qnaAnswerBody
     ;
 
 filterSection
-    : FILTER_MARK filterLine+
+    : WS* FILTER_MARK filterLine+
     ;
 
 filterLine
-    : DASH (WS|TEXT)* newline
+    : WS* DASH (WS|TEXT)* newline
     ;
 
 multiLineAnswer
-    : MULTI_LINE_TEXT
+    : WS* MULTI_LINE_TEXT
     ;
 
 modelInfoDefinition
-    : MODEL_INFO
+    : WS* MODEL_INFO
     ;

packages/luis/src/parser/lufile/classes/hclasses.js

@@ -140,6 +140,15 @@ const readerObj = {
         constructor(name) {
             this.modelName = name ? name : '';
         }
+    },
+    childEntity: class {
+        constructor(name, instanceOf, context, children, features) {
+            this.name = name ? name : '';
+            this.instanceOf = instanceOf ? instanceOf : '';
+            this.children = children ? children : [];
+            this.features = features ? features : '';
+            this.context = context ? context : '';
+        }
     }
 };
 

packages/luis/src/parser/lufile/enums/luisEntityTypeNameMap.js

@@ -8,7 +8,7 @@ module.exports = {
     'list': 'closedLists',
     'regex': 'regex_entities',
     'composite': 'composites',
-    'machine-learned': 'entities',
+    'ml': 'entities',
     'patternany':'patternAnyEntities',
     'phraselist': 'model_features'
 };

packages/luis/src/parser/lufile/enums/luisEntityTypes.js

@@ -8,7 +8,7 @@ module.exports = {
     REGEX: 'regex',
     PREBUILT: 'prebuilt',
     COMPOSITE: 'composite',
-    ML: 'machine-learned',
+    ML: 'ml',
     PATTERNANY:'patternany',
     PHRASELIST: 'phraselist'
 };