Changes for R14 (#1249)

* orchestrator:remove

orchestrator:remove - take 1

* remove data source

* use @microsoft scoped package of orchestrator-core

* Update labelresolver.ts

* refactored add data source

* add missing qna file

* unit tests fixes

* Update settings.test.ts

* fix orchestrator tests

* Update basemodel.ts

update download progress

* Update basemodel.ts

* updated basemodel:get log

* log model download status

* create - incremental update if existing blu file already exists

* added tests for remove

* Update pnpm-lock.yaml

* Update datasourcehelper.ts

fixed linting

* updated to release package of orchestrator-core

updated to release package of orchestrator-core

* Update package.json

* addressed PR comments from Josh

* Update create.ts

PR comments from Josh G

* Addressed PR comments per Josh G

Removed "strictNullChecks": false,

* Update build.ts

more PR comment

* Addressed PR comments from Emilio

* Update settings.ts

fixed validation build errors

Author: tsuwandy

common/config/rush/pnpm-lock.yaml

@@ -5,7 +5,7 @@
 
 import * as path from 'path';
 import {Command, CLIError, flags} from '@microsoft/bf-cli-command';
-import {DataSourceHelper, Orchestrator, OrchestratorDataSource, OrchestratorHelper, OrchestratorSettings, Utility} from '@microsoft/bf-orchestrator';
+import {Orchestrator, OrchestratorDataSource, OrchestratorHelper, OrchestratorSettings} from '@microsoft/bf-orchestrator';
 
 export default class OrchestratorAdd extends Command {
   static description: string = 'Add examples from .lu/.qna/.json/.blu files, LUIS app(s) and QnaMaker kb(s) to Orchestrator snapshot file.';
@@ -57,33 +57,32 @@ export default class OrchestratorAdd extends Command {
         fullEmbeddings = true;
       }
 
+      const settings: OrchestratorSettings = OrchestratorSettings.getCurrent();
       if (type.length > 0) {
-        if (type === 'luis' && Utility.isEmptyString(endpoint)) {
-          throw new CLIError('LUIS endpoint required, ie --endpoint https://westus.api.cognitive.microsoft.com');
-        }
-        OrchestratorSettings.init(cwd, baseModelPath, entityBaseModelPath, output, true, true);
-        const dataSource: OrchestratorDataSource = new OrchestratorDataSource(id, key, version, endpoint, type, routingName, OrchestratorSettings.DataSources.path);
-        await DataSourceHelper.ensureDataSourceAsync(dataSource, OrchestratorSettings.DataSources.path);
+        settings.init(cwd, baseModelPath, entityBaseModelPath, output, true, true);
+        const dataSource: OrchestratorDataSource =
+          new OrchestratorDataSource(id, key, version, endpoint, type, routingName, inputPath);
+        await Orchestrator.addDataSource(dataSource);
 
         if (dataSource.Type === 'file') {
           this.log(`Added ${dataSource.Type} source  ${dataSource.FilePath}`);
         } else {
           this.log(`Added ${dataSource.Type} source with id ${dataSource.Id}`);
         }
       } else {
-        OrchestratorSettings.init(cwd, baseModelPath, entityBaseModelPath, output);
-        const snapshot: Uint8Array = OrchestratorHelper.getSnapshotFromFile(path.resolve(OrchestratorSettings.SnapshotPath));
+        settings.init(cwd, baseModelPath, entityBaseModelPath, output);
+        const snapshot: Uint8Array = OrchestratorHelper.getSnapshotFromFile(path.resolve(settings.SnapshotPath));
         const inputs: any = OrchestratorHelper.getLuInputs(inputPath);
         await Orchestrator.addAsync(
-          OrchestratorSettings.ModelPath,
+          settings.ModelPath,
           snapshot,
           inputs,
           isDialog,
           entityBaseModelPath,
           fullEmbeddings);
       }
 
-      OrchestratorSettings.persist();
+      settings.persist();
     } catch (error) {
       throw (new CLIError(error));
     }

packages/orchestrator/src/commands/orchestrator/add.ts

@@ -30,7 +30,9 @@ export default class OrchestratorBaseModelGet extends Command {
       if (!OrchestratorHelper.exists(output)) {
         OrchestratorHelper.mkDir(output);
       }
-      OrchestratorSettings.init(cwd, output, '', '');
+
+      const settings: OrchestratorSettings = OrchestratorSettings.getCurrent();
+      settings.init(cwd, output, '', '');
       const models: any[] = [];
 
       let versions: any;
@@ -64,10 +66,13 @@ export default class OrchestratorBaseModelGet extends Command {
           await OrchestratorBaseModel.getAsync(
             modelInfo.modelFolder,
             modelInfo.versionId,
-            (message: any) => {
-              if (flags.verbose) {
+            (message: any, pct: number) => {
+              if (message) {
                 this.log(message);
               }
+              if (flags.verbose && pct) {
+                this.log(`Downloaded ${pct}%...`);
+              }
             },
             (message: any) => {
               this.log(`Model ${modelInfo.versionId} downloaded to ${modelInfo.modelFolder}`);
@@ -78,7 +83,7 @@ export default class OrchestratorBaseModelGet extends Command {
         })
       );
 
-      OrchestratorSettings.persist();
+      settings.persist();
     } catch (error) {
       throw (new CLIError(error));
     }

packages/orchestrator/src/commands/orchestrator/basemodel/get.ts

@@ -45,19 +45,21 @@ export default class OrchestratorBuild extends Command {
       output = path.dirname(output);
     }
 
+    const settings: OrchestratorSettings = OrchestratorSettings.getCurrent();
     const luInputs: any[] = OrchestratorHelper.getLuInputs(input);
     const snapshots: Map<string, Uint8Array> = OrchestratorHelper.getSnapshots(output);
-    const labelResolvers: Map<string, LabelResolver> = await Orchestrator.getLabelResolversAsync(OrchestratorSettings.ModelPath, entityBaseModelPath, snapshots);
+    const labelResolvers: Map<string, LabelResolver> = await Orchestrator.getLabelResolversAsync(settings.ModelPath, entityBaseModelPath, snapshots);
 
     try {
       let fullEmbeddings: boolean = false;
       if (process.env.fullEmbeddings) {
         fullEmbeddings = true;
       }
       Utility.toPrintDebuggingLogToConsole = flags.debug;
-      OrchestratorSettings.init(cwd, flags.model, output, cwd);
+
+      settings.init(cwd, flags.model, output, cwd);
       const retPayload: any = await Orchestrator.buildAsync(
-        OrchestratorSettings.ModelPath,
+        settings.ModelPath,
         luInputs,
         labelResolvers,
         isDialog,

packages/orchestrator/src/commands/orchestrator/build.ts

@@ -5,7 +5,7 @@
 
 import * as path from 'path';
 import {Command, CLIError, flags} from '@microsoft/bf-cli-command';
-import {DataSourceHelper, Orchestrator, OrchestratorHelper, OrchestratorDataSourceSettings, OrchestratorSettings, Utility} from '@microsoft/bf-orchestrator';
+import {DataSourceHelper, Orchestrator, OrchestratorHelper, OrchestratorSettings, OrchestratorDataSourceSettings, Utility} from '@microsoft/bf-orchestrator';
 
 export default class OrchestratorCreate extends Command {
   static description: string = 'Create orchestrator snapshot (.blu) file from .lu/.qna/.json/.tsv/.dispatch files, which represent bot modules';
@@ -44,7 +44,8 @@ export default class OrchestratorCreate extends Command {
     Utility.resetFlagToPrintDebuggingLogToConsole(flags.debug);
 
     try {
-      OrchestratorSettings.init(cwd, baseModelPath, entityBaseModelPath, output, flags.hierarchical);
+      const settings: OrchestratorSettings = OrchestratorSettings.getCurrent();
+      settings.init(cwd, baseModelPath, entityBaseModelPath, output, flags.hierarchical);
 
       let fullEmbeddings: boolean = false;
       if (process.env.fullEmbeddings) {
@@ -55,30 +56,30 @@ export default class OrchestratorCreate extends Command {
 
       if (DataSourceHelper.isDispatchInput(input)) {
         const dispatchJson: any = JSON.parse(OrchestratorHelper.readFile(input));
-        DataSourceHelper.convertDispatchInputs(dispatchJson, OrchestratorSettings.DataSources);
+        DataSourceHelper.convertDispatchInputs(dispatchJson, settings.DataSources);
         refresh = true;
       }
 
-      const hasDataSources: boolean = OrchestratorSettings.DataSources && OrchestratorSettings.DataSources.inputs.length > 0;
+      const hasDataSources: boolean = settings.DataSources?.inputs?.length > 0 ?? false;
       if (hasDataSources) {
-        input = OrchestratorSettings.DataSources.path;
+        input = settings.DataSources.path;
         hierarchical = true;
       }
 
       if (refresh) {
-        await this.refreshLuisQnAInputs(OrchestratorSettings.DataSources);
+        await this.refreshLuisQnAInputs(settings.DataSources);
       }
 
       const snapshotFilePath: string = await Orchestrator.createAsync(
-        OrchestratorSettings.ModelPath,
+        settings.ModelPath,
         input,
-        OrchestratorSettings.SnapshotPath,
-        OrchestratorSettings.EntityModelPath,
+        settings.SnapshotPath,
+        settings.EntityModelPath,
         hierarchical,
         fullEmbeddings);
 
-      OrchestratorSettings.SnapshotPath = snapshotFilePath;
-      OrchestratorSettings.persist();
+      settings.SnapshotPath = snapshotFilePath;
+      settings.persist();
     } catch (error) {
       throw (new CLIError(error));
     }
@@ -93,7 +94,7 @@ export default class OrchestratorCreate extends Command {
         this.log(`Refreshing ${dataSource.Type} data - id ${dataSource.Id}...`);
       }
       // eslint-disable-next-line no-await-in-loop
-      await DataSourceHelper.ensureDataSourceAsync(dataSource, OrchestratorSettings.DataSources.path, false);
+      await DataSourceHelper.ensureDataSourceAsync(dataSource, dataSources.path, false);
     }
   }
 }

packages/orchestrator/src/commands/orchestrator/create.ts

@@ -0,0 +1,58 @@
+/*!
+ * Copyright (c) Microsoft Corporation. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import * as path from 'path';
+import {Command, CLIError, flags} from '@microsoft/bf-cli-command';
+import {Orchestrator, OrchestratorDataSource, OrchestratorSettings} from '@microsoft/bf-orchestrator';
+
+export default class OrchestratorRemove extends Command {
+  static description: string = 'Remove examples from LUIS app(s), QnaMaker kb(s) or .lu/.qna/.json files from Orchestrator snapshot file.';
+
+  static examples: Array<string> = [`	
+    $ bf orchestrator:remove 	
+    $ bf orchestrator:remove -t luis --id LUIS_APP_ID 
+    $ bf orchestrator:remove -t qna --id QNA_KB 
+    $ bf orchestrator:remove -t file -i FILE_PATH 
+    `]
+
+  static flags: flags.Input<any> = {
+    in: flags.string({char: 'i', description: 'Path to example file (.lu/.qna/.json/.blu).'}),
+    id: flags.string({description: 'LUIS app id or QnAMaker kb id if type = luis/qna.'}),
+    type: flags.string({char: 't', description: 'Type of input (luis/qna/file).'}),
+    debug: flags.boolean({char: 'd'}),
+    help: flags.help({char: 'h', description: 'Orchestrator remove command help'}),
+  }
+
+  async run() {
+    const {flags}: flags.Output = this.parse(OrchestratorRemove);
+    const cwd: string = process.cwd();
+    const filePath: string = path.resolve(flags.in || cwd);
+    const type: string = (flags.type.toLowerCase() || '');
+    const id: string = (flags.id || '');
+
+    if (!type) {
+      throw new CLIError('Type parameter required');
+    }
+
+    try {
+      const settings: OrchestratorSettings = OrchestratorSettings.getCurrent();
+      settings.init(cwd, '', '', '', true, true);
+      const dataSource: OrchestratorDataSource = new OrchestratorDataSource(id, '', '', '', type, '', filePath);
+      if (Orchestrator.removeDataSource(dataSource)) {
+        if (dataSource.Type === 'file') {
+          this.log(`Removed ${dataSource.Type} source  ${dataSource.FilePath}`);
+        } else {
+          this.log(`Removed ${dataSource.Type} source with id ${dataSource.Id}`);
+        }
+      } else {
+        this.log('Data source not found');
+      }
+
+      settings.persist();
+    } catch (error) {
+      throw (new CLIError(error));
+    }
+  }
+}

packages/orchestrator/src/commands/orchestrator/remove.ts

@@ -0,0 +1,15 @@
+/*!
+ * Copyright (c) Microsoft Corporation. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import {expect, test} from '@oclif/test';
+describe('orchestrator:remove tests', () => {
+  test
+  .stdout()
+  .command(['orchestrator:remove', '--help'])
+  .exit(1)
+  .it('should print the help contents when --help is passed as an argument', (ctx: any) => {
+    expect(ctx.stdout).to.contain('Remove examples from LUIS app(s), QnaMaker kb(s) or .lu/.qna/.json files');
+  });
+});

packages/orchestrator/test/commands/orchestrator/remove.test.ts

@@ -40,7 +40,7 @@
     "axios":"~0.21.1",
     "https-proxy-agent": "^5.0.0",
     "tslib": "^2.0.3",
-    "@microsoft/orchestrator-core": "4.14.0-preview",
+    "@microsoft/orchestrator-core": "~4.14.0",
     "@types/fs-extra": "~8.1.0",
     "fs-extra": "~9.0.0",
     "read-text-file": "~1.1.0",

packages/orchestratorlib/package.json

@@ -62,7 +62,7 @@ export class OrchestratorBaseModel {
       const modelZipPath: string = path.join(baseModelPath, fileName);
       Utility.debuggingLog(`OrchestratorBaseModel.getModelAsync(): finished calling  fileName=${fileName}`);
       Utility.debuggingLog(`OrchestratorBaseModel.getModelAsync(): finished calling  modelZipPath=${modelZipPath}`);
-      await this.downloadModel(modelUrl, modelZipPath);
+      await this.downloadModel(modelUrl, modelZipPath, onProgress);
       if (onProgress) {
         onProgress('OrchestratorBaseModel.getModelAsync(): model downloaded...');
       }
@@ -100,13 +100,25 @@ export class OrchestratorBaseModel {
     return response.data;
   }
 
-  public static async downloadModel(modelUrl: string, modelZipPath: string): Promise<(resolve: any) => void> {
+  public static async downloadModel(modelUrl: string, modelZipPath: string, onProgress: any = OrchestratorBaseModel.defaultHandler): Promise<(resolve: any) => void> {
     const response: AxiosResponse<Stream> = await axios({
       method: 'GET',
       url: modelUrl,
       responseType: 'stream',
     });
+    const totalLength: number = response.headers['content-length'];
+    onProgress(`Total to download: ${totalLength} bytes...`);
+    let totalCompleted: number = 0;
+    let totalCompletedPct: number = 0;
     Utility.debuggingLog('OrchestratorBaseModel.getModelAsync(): calling download zipped model files');
+    response.data.on('data', (chunk: any) => {
+      totalCompleted += chunk.length;
+      const newPct: number = Math.round(totalCompleted / totalLength * 100);
+      if (newPct > totalCompletedPct) {
+        totalCompletedPct = newPct;
+        onProgress('', totalCompletedPct);
+      }
+    });
     response.data.pipe(fs.createWriteStream(modelZipPath));
 
     return new Promise((resolve: any) => {