Skip to content

Commit 7c03e6a

Browse files
ijdocijdoc
committed
March 14, 2022 release
1 parent b600c11 commit 7c03e6a

File tree

99 files changed

+5191
-358
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

99 files changed

+5191
-358
lines changed

‎cli/blocks.ts‎

Lines changed: 184 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,10 @@ import { Config, EdgeImpulseConfig } from './config';
88
import checkNewVersions from './check-new-version';
99
import inquirer from 'inquirer';
1010
import {
11+
AddOrganizationTransferLearningBlockRequest,
1112
AddOrganizationTransformationBlockRequest,
1213
OrganizationJobsApi,
14+
OrganizationTransformationBlockOperatesOnEnum,
1315
UploadCustomBlockRequestTypeEnum,
1416
UploadCustomBlockRequestTypeEnumValues
1517
} from '../sdk/studio/api';
@@ -21,18 +23,37 @@ import WebSocket, { OPEN } from 'ws';
2123
import dockerignore from '@zeit/dockerignore';
2224
import { getCliVersion } from './init-cli-app';
2325
import util from 'util';
26+
import {
27+
OrganizationTransferLearningBlockObjectDetectionLastLayerEnum,
28+
OrganizationTransferLearningBlockOperatesOnEnum
29+
} from '../sdk/studio/model/organizationTransferLearningBlock';
2430

2531
const version = getCliVersion();
2632

2733
type BlockConfigItem = {
2834
name: string,
2935
description: string,
30-
type: UploadCustomBlockRequestTypeEnum,
3136
id?: number,
32-
port?: number,
3337
organizationId: number,
34-
operatesOn: 'file' | 'dataitem' | 'standalone' | undefined
35-
};
38+
type: UploadCustomBlockRequestTypeEnum,
39+
} & ({
40+
type: 'transform',
41+
operatesOn: 'file' | 'dataitem' | 'standalone' | undefined,
42+
transformMountpoints: {
43+
bucketId: number;
44+
mountPoint: string;
45+
}[] | undefined,
46+
} | {
47+
type: 'transferLearning',
48+
tlOperatesOn?: OrganizationTransferLearningBlockOperatesOnEnum,
49+
tlObjectDetectionLastLayer?: OrganizationTransferLearningBlockObjectDetectionLastLayerEnum,
50+
} | {
51+
type: 'deploy',
52+
deployCategory?: 'library' | 'firmware',
53+
} | {
54+
type: 'dsp',
55+
port?: number,
56+
});
3657

3758
type BlockConfigV1 = {
3859
version: 1,
@@ -200,6 +221,10 @@ let globalCurrentBlockConfig: BlockConfigV1 | undefined;
200221
{
201222
name: 'DSP block',
202223
value: 'dsp'
224+
},
225+
{
226+
name: 'Transfer learning block',
227+
value: 'transferLearning'
203228
}
204229
],
205230
name: 'type',
@@ -212,13 +237,21 @@ let globalCurrentBlockConfig: BlockConfigV1 | undefined;
212237
let blockName: string | undefined;
213238
let blockDescription: string | undefined;
214239
let blockOperatesOn: 'file' | 'dataitem' | 'standalone' | undefined;
240+
let blockTlOperatesOn: OrganizationTransferLearningBlockOperatesOnEnum | undefined;
241+
let blockTlObjectDetectionLastLayer: OrganizationTransferLearningBlockObjectDetectionLastLayerEnum | undefined;
242+
let transformMountpoints: {
243+
bucketId: number;
244+
mountPoint: string;
245+
}[] | undefined;
246+
247+
let deployCategory: 'library' | 'firmware' | undefined;
215248

216249
// Fetch all relevant existing blocks so the user can select an existing block to update
217250
let existingBlocks: {
218251
name: string, value: number, block: {
219252
description: string, name: string, operatesOn: 'file' | 'dataitem' | 'standalone' | undefined }
220253
}[] = [];
221-
if (blockTypeInqRes.type === 'transform') {
254+
if (blockType === 'transform') {
222255
let blocks = await config.api.organizationBlocks.listOrganizationTransformationBlocks(organizationId);
223256
if (blocks.body && blocks.body.transformationBlocks && blocks.body.transformationBlocks.length > 0) {
224257
existingBlocks = blocks.body.transformationBlocks.map(p => (
@@ -230,7 +263,7 @@ let globalCurrentBlockConfig: BlockConfigV1 | undefined;
230263
));
231264
}
232265
}
233-
else if (blockTypeInqRes.type === 'deploy') {
266+
else if (blockType === 'deploy') {
234267
let blocks = await config.api.organizationBlocks.listOrganizationDeployBlocks(organizationId);
235268
if (blocks.body && blocks.body.deployBlocks && blocks.body.deployBlocks.length > 0) {
236269
existingBlocks = blocks.body.deployBlocks.map(p => (
@@ -242,7 +275,7 @@ let globalCurrentBlockConfig: BlockConfigV1 | undefined;
242275
));
243276
}
244277
}
245-
else if (blockTypeInqRes.type === 'dsp') {
278+
else if (blockType === 'dsp') {
246279
let blocks = await config.api.organizationBlocks.listOrganizationDspBlocks(organizationId);
247280
if (blocks.body && blocks.body.dspBlocks && blocks.body.dspBlocks.length > 0) {
248281
existingBlocks = blocks.body.dspBlocks.map(p => (
@@ -254,6 +287,18 @@ let globalCurrentBlockConfig: BlockConfigV1 | undefined;
254287
));
255288
}
256289
}
290+
else if (blockType === 'transferLearning') {
291+
let blocks = await config.api.organizationBlocks.listOrganizationTransferLearningBlocks(organizationId);
292+
if (blocks.body && blocks.body.transferLearningBlocks && blocks.body.transferLearningBlocks.length > 0) {
293+
existingBlocks = blocks.body.transferLearningBlocks.map(p => (
294+
{
295+
name: p.name,
296+
value: p.id,
297+
block: { description: p.description, name: p.name, operatesOn: undefined }
298+
}
299+
));
300+
}
301+
}
257302
else {
258303
console.error(`Invalid block type: ${blockTypeInqRes.type}`);
259304
process.exit(1);
@@ -299,7 +344,7 @@ let globalCurrentBlockConfig: BlockConfigV1 | undefined;
299344
let defaultName = 'My new block';
300345
let defaultDescription: string | undefined;
301346

302-
if (blockTypeInqRes.type === 'dsp') {
347+
if (blockType === 'dsp') {
303348
let paramsFile = Path.join(process.cwd(), 'parameters.json');
304349
if (paramsFile) {
305350
try {
@@ -343,8 +388,7 @@ let globalCurrentBlockConfig: BlockConfigV1 | undefined;
343388
if (blockDescription === '') blockDescription = blockName;
344389
}
345390

346-
347-
if (createOrUpdateInqRes === 'create' && blockTypeInqRes.type === 'transform') {
391+
if (createOrUpdateInqRes === 'create' && blockType === 'transform') {
348392
blockOperatesOn = <'file' | 'dataitem' | 'standalone'>(await inquirer.prompt([{
349393
type: 'list',
350394
name: 'operatesOn',
@@ -364,6 +408,100 @@ let globalCurrentBlockConfig: BlockConfigV1 | undefined;
364408
],
365409
message: 'What type of data does this block operate on?',
366410
}])).operatesOn;
411+
412+
let buckets = await config.api.organizationData.listOrganizationBuckets(organizationId);
413+
if (buckets.body && buckets.body.buckets && buckets.body.buckets.length > 0) {
414+
transformMountpoints = (<string[]>(await inquirer.prompt([{
415+
type: 'checkbox',
416+
name: 'buckets',
417+
choices: buckets.body.buckets.map(x => {
418+
return {
419+
name: x.name,
420+
value: x.id.toString()
421+
};
422+
}),
423+
message: 'Which buckets do you want to mount into this block ' +
424+
'(will be mounted under /mnt/s3fs/BUCKET_NAME, you can change these mount points in the Studio)?',
425+
}])).buckets).map(y => {
426+
let b = buckets.body.buckets?.find(z => z.id === Number(y));
427+
return {
428+
bucketId: Number(y),
429+
mountPoint: b ? ('/mnt/s3fs/' + b?.name) : '',
430+
};
431+
}).filter(x => !!x.mountPoint && !isNaN(x.bucketId));
432+
}
433+
}
434+
435+
if (createOrUpdateInqRes === 'create' && blockType === 'transferLearning') {
436+
blockTlOperatesOn = <OrganizationTransferLearningBlockOperatesOnEnum>(await inquirer.prompt([{
437+
type: 'list',
438+
name: 'operatesOn',
439+
choices: [
440+
{
441+
name: 'Object Detection',
442+
value: 'object_detection'
443+
},
444+
{
445+
name: 'Image classification',
446+
value: 'image'
447+
},
448+
{
449+
name: 'Audio classification',
450+
value: 'audio'
451+
},
452+
{
453+
name: 'Other (classification)',
454+
value: 'other'
455+
},
456+
{
457+
name: 'Other (regression)',
458+
value: 'regression'
459+
},
460+
],
461+
message: 'What type of data does this model operate on?',
462+
}])).operatesOn;
463+
464+
if (blockTlOperatesOn === 'object_detection') {
465+
blockTlObjectDetectionLastLayer = <OrganizationTransferLearningBlockObjectDetectionLastLayerEnum>
466+
(await inquirer.prompt([{
467+
type: 'list',
468+
name: 'lastLayer',
469+
choices: [
470+
{
471+
name: 'MobileNet SSD',
472+
value: 'mobilenet_ssd'
473+
},
474+
{
475+
name: 'Edge Impulse FOMO',
476+
value: 'fomo'
477+
},
478+
{
479+
name: 'YOLOv5',
480+
value: 'yolov5'
481+
},
482+
],
483+
message: `What's the last layer of this object detection model?`,
484+
}])).lastLayer;
485+
}
486+
}
487+
488+
489+
if (createOrUpdateInqRes === 'create' && blockType === 'deploy') {
490+
deployCategory = <'library' | 'firmware'>(await inquirer.prompt([{
491+
type: 'list',
492+
name: 'category',
493+
choices: [
494+
{
495+
name: 'Library',
496+
value: 'library'
497+
},
498+
{
499+
name: 'Firmware',
500+
value: 'firmware'
501+
}
502+
],
503+
message: 'Where to show this deployment block in the UI?',
504+
}])).category;
367505
}
368506

369507
// Create & write the config
@@ -375,15 +513,23 @@ let globalCurrentBlockConfig: BlockConfigV1 | undefined;
375513
description: blockDescription,
376514
organizationId,
377515
operatesOn: blockOperatesOn,
516+
tlObjectDetectionLastLayer: blockTlObjectDetectionLastLayer,
517+
tlOperatesOn: blockTlOperatesOn,
518+
deployCategory: deployCategory,
519+
transformMountpoints: transformMountpoints,
378520
} : {
379521
name: blockName,
380522
type: blockType,
381523
description: blockDescription,
382524
organizationId,
383525
operatesOn: blockOperatesOn,
526+
tlObjectDetectionLastLayer: blockTlObjectDetectionLastLayer,
527+
tlOperatesOn: blockTlOperatesOn,
528+
deployCategory: deployCategory,
529+
transformMountpoints: transformMountpoints,
384530
};
385531

386-
console.log('Creating block with config:', globalCurrentBlockConfig);
532+
// console.log('Creating block with config:', globalCurrentBlockConfig);
387533
await writeConfigFile();
388534

389535
const hasDockerFile = await exists(dockerfilePath);
@@ -407,10 +553,12 @@ let globalCurrentBlockConfig: BlockConfigV1 | undefined;
407553
templateSourcePath =
408554
'https://github.com/edgeimpulse/template-transformation-block-python/archive/main.zip';
409555
directoryRoot = 'template-transformation-block-python-main/';
410-
} else if (blockType === 'deploy') {
556+
}
557+
else if (blockType === 'deploy') {
411558
templateSourcePath = 'https://github.com/edgeimpulse/template-deployment-block/archive/main.zip';
412559
directoryRoot = 'template-deployment-block-main/';
413-
} else {
560+
}
561+
else {
414562
console.error(`Invalid block type: ${blockType}`);
415563
process.exit(1);
416564
}
@@ -501,14 +649,22 @@ let globalCurrentBlockConfig: BlockConfigV1 | undefined;
501649
indMetadata: true,
502650
cliArguments: '',
503651
operatesOn: currentBlockConfig.operatesOn || 'file',
504-
additionalMountPoints: []
652+
additionalMountPoints: (currentBlockConfig.transformMountpoints || []).map(x => {
653+
return {
654+
type: 'bucket',
655+
bucketId: x.bucketId,
656+
mountPoint: x.mountPoint,
657+
};
658+
}),
505659
};
506660
newResponse = await config.api.organizationBlocks.addOrganizationTransformationBlock(
507661
organizationId, newBlockObject);
508662
}
509663
else if (currentBlockConfig.type === 'deploy') {
510664
newResponse = await config.api.organizationBlocks.addOrganizationDeployBlock(
511-
organizationId, currentBlockConfig.name, '', currentBlockConfig.description, '');
665+
organizationId, currentBlockConfig.name, '', currentBlockConfig.description, '',
666+
undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined,
667+
undefined, undefined, currentBlockConfig.deployCategory);
512668
}
513669
else if (currentBlockConfig.type === 'dsp') {
514670
if (currentBlockConfig.type === 'dsp' && typeof currentBlockConfig.port !== 'number') {
@@ -556,8 +712,20 @@ let globalCurrentBlockConfig: BlockConfigV1 | undefined;
556712
port: currentBlockConfig.port || 80,
557713
});
558714
}
715+
else if (currentBlockConfig.type === 'transferLearning') {
716+
const newBlockObject: AddOrganizationTransferLearningBlockRequest = {
717+
name: currentBlockConfig.name,
718+
description: currentBlockConfig.description,
719+
dockerContainer: '',
720+
objectDetectionLastLayer: currentBlockConfig.tlObjectDetectionLastLayer,
721+
operatesOn: currentBlockConfig.tlOperatesOn || 'image',
722+
};
723+
newResponse = await config.api.organizationBlocks.addOrganizationTransferLearningBlock(
724+
organizationId, newBlockObject);
725+
}
559726
else {
560-
console.error(`Unable to upload your block - unknown block type: ${currentBlockConfig.type}`);
727+
console.error(`Unable to upload your block - unknown block type: ` +
728+
`${(<any>currentBlockConfig).type}`);
561729
process.exit(1);
562730
}
563731
if (!newResponse.body.success) {

‎cli/config.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,8 @@ import {
1212
OrganizationCreateProjectApi, OrganizationCreateProjectApiApiKeys,
1313
OrganizationJobsApi, OrganizationJobsApiApiKeys, OrganizationBlocksApi,
1414
OrganizationBlocksApiApiKeys, DeploymentApi, ImpulseApi, LearnApi,
15-
JobsApi, ImpulseApiApiKeys, LearnApiApiKeys, JobsApiApiKeys, DeploymentApiApiKeys
15+
JobsApi, ImpulseApiApiKeys, LearnApiApiKeys, JobsApiApiKeys, DeploymentApiApiKeys,
16+
OrganizationDataApi, OrganizationDataApiApiKeys
1617
} from '../sdk/studio/api';
1718

1819
const PREFIX = '\x1b[34m[CFG]\x1b[0m';
@@ -54,6 +55,7 @@ export interface EdgeImpulseAPI {
5455
organizationCreateProject: OrganizationCreateProjectApi;
5556
organizationBlocks: OrganizationBlocksApi;
5657
organizationJobs: OrganizationJobsApi;
58+
organizationData: OrganizationDataApi;
5759
}
5860

5961
export interface EdgeImpulseEndpoints {
@@ -248,6 +250,7 @@ export class Config {
248250
organizationCreateProject: new OrganizationCreateProjectApi(apiEndpointInternal),
249251
organizationBlocks: new OrganizationBlocksApi(apiEndpointInternal),
250252
organizationJobs: new OrganizationJobsApi(apiEndpointInternal),
253+
organizationData: new OrganizationDataApi(apiEndpointInternal),
251254
};
252255

253256
this._endpoints = {
@@ -277,6 +280,7 @@ export class Config {
277280
apiKey);
278281
this._api.organizationBlocks.setApiKey(OrganizationBlocksApiApiKeys.ApiKeyAuthentication, apiKey);
279282
this._api.organizationJobs.setApiKey(OrganizationJobsApiApiKeys.ApiKeyAuthentication, apiKey);
283+
this._api.organizationData.setApiKey(OrganizationDataApiApiKeys.ApiKeyAuthentication, apiKey);
280284
config.apiKey = apiKey;
281285
}
282286
else {
@@ -316,6 +320,7 @@ export class Config {
316320
this._api.organizationBlocks.setApiKey(OrganizationBlocksApiApiKeys.JWTAuthentication,
317321
config.jwtToken);
318322
this._api.organizationJobs.setApiKey(OrganizationJobsApiApiKeys.JWTAuthentication, config.jwtToken);
323+
this._api.organizationData.setApiKey(OrganizationDataApiApiKeys.JWTAuthentication, config.jwtToken);
319324
}
320325

321326
if (verifyType === 'project') {

0 commit comments

Comments
 (0)