decouple config validation from parseFile (#1002)

* add validate function

* reactor to sync

* fix resource is changed in memory issue

Author: feich-ms

packages/lu/src/parser/composerindex.js

@@ -1,7 +1,8 @@
 module.exports = {
     parser: {
         parseFile: require('./lufile/parseFileContents').parseFile,
-        validateLUISBlob: require('./luis/luisValidator')
+        validateLUISBlob: require('./luis/luisValidator'),
+        validateResource: require('./lufile/parseFileContents').validateResource
     },
     sectionHandler: {
         luParser: require('./lufile/luParser'),

packages/lu/src/parser/lufile/parseFileContents.js

@@ -21,13 +21,13 @@ const fetch = require('node-fetch');
 const qnaFile = require('./../qna/qnamaker/qnaFiles');
 const fileToParse = require('./classes/filesToParse');
 const luParser = require('./luParser');
-const DiagnosticSeverity = require('./diagnostic').DiagnosticSeverity;
-const BuildDiagnostic = require('./diagnostic').BuildDiagnostic;
+const {BuildDiagnostic, DiagnosticSeverity} = require('./diagnostic');
 const EntityTypeEnum = require('./../utils/enums/luisEntityTypes');
 const luisEntityTypeMap = require('./../utils/enums/luisEntityTypeNameMap');
 const qnaContext = require('../qna/qnamaker/qnaContext');
 const qnaPrompt = require('../qna/qnamaker/qnaPrompt');
-const { config } = require('process');
+const LUResource = require('./luResource');
+
 const plAllowedTypes = ["composite", "ml"];
 const featureTypeEnum = {
     featureToModel: 'modelName',
@@ -91,6 +91,65 @@ const parseFileContentsModule = {
 
         return parsedContent;
     },
+    /**
+     * Validate resource based on config.
+     * @param {LUResource} originalResource Original parsed lu or qna resource
+     * @param {any} config Features config
+     * @returns {any[]} Diagnostic errors returned
+     */
+    validateResource: function (originalResource, config) {
+        config = config || {};
+        config = {...defaultConfig, ...config};
+
+        let resource = JSON.parse(JSON.stringify(originalResource));
+        if (resource.Errors.filter(error => (error && error.Severity && error.Severity === DiagnosticSeverity.ERROR)).length > 0) {
+            return []
+        }
+
+        let errors = []
+
+        try {
+            let parsedContent = new parserObj();
+
+            // parse model info section
+            let enableMergeIntents = parseAndHandleModelInfoSection(parsedContent, resource, false, config);
+
+            // validate reference section
+            validateImportSection(resource, config);
+
+            // parse nested intent section
+            parseAndHandleNestedIntentSection(resource, enableMergeIntents);
+
+            GetEntitySectionsFromSimpleIntentSections(resource);
+
+            // parse entity definition v2 section
+            let featuresToProcess = parseAndHandleEntityV2(parsedContent, resource, false, undefined, config);
+
+            // parse entity section
+            parseAndHandleEntitySection(parsedContent, resource, false, undefined, config);
+
+            // parse entity section
+            parseAndHandleEntitySection(parsedContent, resource, false, undefined, config);
+
+            // validate simple intent section
+            parseAndHandleSimpleIntentSection(parsedContent, resource, config)
+
+            if (featuresToProcess && featuresToProcess.length > 0) {
+                parseFeatureSections(parsedContent, featuresToProcess, config);
+            }
+
+        } catch(e) {
+            if (e instanceof exception) {
+                errors.push(...e.diagnostics)
+            } else {
+                errors.push(BuildDiagnostic({
+                    message: e.message
+                }))
+            }
+        }
+
+        return errors
+    },
     /**
      * Helper function to add an item to collection if it does not exist
      * @param {object} collection contents of the current collection
@@ -175,7 +234,7 @@ const parseLuAndQnaWithAntlr = async function (parsedContent, fileContent, log,
     }
 
     // parse model info section
-    let enableMergeIntents = parseAndHandleModelInfoSection(parsedContent, luResource, log);
+    let enableMergeIntents = parseAndHandleModelInfoSection(parsedContent, luResource, log, config);
 
     // parse reference section
     await parseAndHandleImportSection(parsedContent, luResource, config);
@@ -192,7 +251,7 @@ const parseLuAndQnaWithAntlr = async function (parsedContent, fileContent, log,
     parseAndHandleEntitySection(parsedContent, luResource, log, locale, config);
 
     // parse simple intent section
-    await parseAndHandleSimpleIntentSection(parsedContent, luResource, config);
+    parseAndHandleSimpleIntentSection(parsedContent, luResource, config);
 
     // parse qna section
     await parseAndHandleQnaSection(parsedContent, luResource);
@@ -715,6 +774,23 @@ const parseAndHandleImportSection = async function (parsedContent, luResource, c
         }
     }
 }
+/**
+ * Reference parser code to parse reference section.
+ * @param {LUResouce} luResource resources extracted from lu file content
+ * @throws {exception} Throws on errors. exception object includes errCode and text.
+ */
+const validateImportSection = function (luResource, config) {
+    // handle reference
+    let luImports = luResource.Sections.filter(s => s.SectionType === SectionType.IMPORTSECTION);
+    if (luImports && luImports.length > 0) {
+        if (!config.enableExternalReferences) {
+          const error = BuildDiagnostic({
+            message: 'Do not support External References. Please make sure enableExternalReferences is set to true.'
+          });
+          throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString(), [error]));
+        }
+    }
+}
 /**
  * Helper function to handle @ reference in patterns
  * @param {String} utterance
@@ -800,7 +876,7 @@ const parseAndHandleNestedIntentSection = function (luResource, enableMergeInten
  * @param {LUResouce} luResource resources extracted from lu file content
  * @throws {exception} Throws on errors. exception object includes errCode and text.
  */
-const parseAndHandleSimpleIntentSection = async function (parsedContent, luResource, config) {
+const parseAndHandleSimpleIntentSection = function (parsedContent, luResource, config) {
     // handle intent
     let intents = luResource.Sections.filter(s => s.SectionType === SectionType.SIMPLEINTENTSECTION);
     let hashTable = {}
@@ -833,7 +909,7 @@ const parseAndHandleSimpleIntentSection = async function (parsedContent, luResou
 
                         utterance = `${utterance.slice(0, index)}(${reference.Path})`
                     }
-                    let parsedLinkUriInUtterance = await helpers.parseLinkURI(utterance);
+                    let parsedLinkUriInUtterance = helpers.parseLinkURISync(utterance);
                     // examine and add these to filestoparse list.
                     parsedContent.additionalFilesToParse.push(new fileToParse(parsedLinkUriInUtterance.fileName, false));
                 }
@@ -2042,13 +2118,21 @@ const parseAndHandleQnaSection = async function (parsedContent, luResource) {
  * @param {parserObj} Object with that contains list of additional files to parse, parsed LUIS object and parsed QnA object
  * @param {LUResouce} luResource resources extracted from lu file content
  * @param {boolean} log indicates if we need verbose logging.
+ * @param {any} config config to indicate which features are enabled
  * @throws {exception} Throws on errors. exception object includes errCode and text.
  */
-const parseAndHandleModelInfoSection = function (parsedContent, luResource, log) {
+const parseAndHandleModelInfoSection = function (parsedContent, luResource, log, config) {
     // handle model info
     let enableMergeIntents = true;
     let modelInfos = luResource.Sections.filter(s => s.SectionType === SectionType.MODELINFOSECTION);
     if (modelInfos && modelInfos.length > 0) {
+        if (!config.enableModelDescription) {
+            const error = BuildDiagnostic({
+                message: `Do not support Model Description. Please make sure enableModelDescription is set to true.`
+            })
+            throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString(), [error]));
+        }
+
         for (const modelInfo of modelInfos) {
             let line = modelInfo.ModelInfo
             let kvPair = line.split(/@(app|kb|intent|entity|enableSections|enableMergeIntents|patternAnyEntity|parser).(.*)=/g).map(item => item.trim());

packages/lu/src/parser/utils/helpers.js

@@ -92,6 +92,28 @@ const helpers = {
         }
         return splitReference.groups;
     },
+    /**
+     * Helper function to parse link URIs in utterances
+     * @param {String} utterance
+     * @returns {Object} Object that contains luFile and ref. ref can be Intent-Name or ? or * or **
+     * @throws {exception} Throws on errors. exception object includes errCode and text. 
+     */
+    parseLinkURISync: function (utterance) {
+        let linkValueList = utterance.trim().match(new RegExp(/\(.*?\)/g));
+        let linkValue = linkValueList[0].replace('(', '').replace(')', '');
+        if (linkValue === '') throw (new exception(retCode.errorCode.INVALID_LU_FILE_REF, `[ERROR]: Invalid LU File Ref: "${utterance}"`));
+        // reference can either be #<Intent-Name> or #? or /*#? or /**#? or #*utterance* or #<Intent-Name>*patterns*
+        let splitRegExp = new RegExp(/^(?<fileName>.*?)(?<segment>#|\*+)(?<path>.*?)$/gim);
+        let splitReference = splitRegExp.exec(linkValue);
+        if (!splitReference) throw (new exception(retCode.errorCode.INVALID_LU_FILE_REF, `[ERROR]: Invalid LU File Ref: "${utterance}".\n Reference needs a qualifier - either a #Intent-Name or #? or *#? or **#? or #*utterances* etc.`));
+        if (splitReference.groups.segment.includes('*')) {
+            if (splitReference.groups.path === '') {
+                throw (new exception(retCode.errorCode.INVALID_LU_FILE_REF, `[ERROR]: Invalid LU File Ref: "${utterance}".\n '*' and '**' can only be used with QnA qualitifier. e.g. *#? and **#?`));
+            }
+            splitReference.groups.fileName += '*';
+        }
+        return splitReference.groups;
+    },
     /**
      * Helper function to do a filter operation based search over an Array
      * @param {Array} srcList Object to filter on