Skip to content

Commit 6004484

Browse files
ijdocijdoc
committed
Release v1.16.0
1 parent b477a58 commit 6004484

File tree

71 files changed

+3566
-534
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

71 files changed

+3566
-534
lines changed

‎cli/block-runner.ts‎

Lines changed: 133 additions & 130 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@ import { spawn, SpawnOptions } from "child_process";
77
import { Config, EdgeImpulseConfig } from "./config";
88
import { BlockConfigItem, exists } from "./blocks";
99
import inquirer from "inquirer";
10-
import { FSHelpers } from "./fs-helpers";
1110
import { split as argvSplit } from './argv-split';
1211
import http from 'http';
1312
import {
@@ -85,15 +84,15 @@ export type RunnerOptions = {
8584
learningRate?: string;
8685
validationSetSize?: string;
8786
inputShape?: string;
88-
downloadData?: boolean;
87+
downloadData?: string | boolean;
8988
}
9089
| {
9190
type: "dsp";
9291
port?: string;
9392
}
9493
| {
9594
type: "deploy";
96-
downloadData?: boolean;
95+
downloadData?: string | boolean;
9796
}
9897
);
9998

@@ -609,10 +608,6 @@ export class BlockRunnerTransform extends BlockRunner {
609608
this._dataItem.id
610609
);
611610

612-
if (await fileExists(path)) {
613-
await FSHelpers.rmDir(path);
614-
}
615-
616611
await fs.promises.mkdir(path);
617612

618613
let targetFilePath = Path.join(
@@ -843,13 +838,15 @@ export class BlockRunnerTransferLearning extends BlockRunner {
843838
let learnBlockId = -1;
844839

845840
if (!runner.blockId) {
846-
if (impulseRes.impulse.learnBlocks.length === 1) {
847-
learnBlockId = impulseRes.impulse.learnBlocks[0].id;
841+
let learnBlocks = impulseRes.impulse.learnBlocks.filter(x => x.primaryVersion);
842+
843+
if (learnBlocks.length === 1) {
844+
learnBlockId = learnBlocks[0].id;
848845
} else {
849846
let learnInq = await inquirer.prompt([
850847
{
851848
type: "list",
852-
choices: (impulseRes.impulse.learnBlocks || []).map((b) => ({
849+
choices: (learnBlocks || []).map((b) => ({
853850
name: b.title,
854851
value: b.id
855852
})),
@@ -879,9 +876,13 @@ export class BlockRunnerTransferLearning extends BlockRunner {
879876
"if you want to fetch new data run edge-impulse-blocks runner --download-data");
880877
}
881878
else {
879+
let targetDir = typeof this._runnerOpts.downloadData === 'string' ?
880+
this._runnerOpts.downloadData :
881+
Path.join(process.cwd(), 'ei-block-data', this._projectId.toString());
882+
882883
console.log(CON_PREFIX, "Downloading files...");
883-
await this.downloadFiles(this._projectId, learnBlockId);
884-
console.log(CON_PREFIX, `Training files downloaded to ./ei-block-data/${this._projectId}`);
884+
await this.downloadFiles(this._projectId, learnBlockId, targetDir);
885+
console.log(CON_PREFIX, `Training files downloaded to`, targetDir);
885886

886887
if (this._runnerOpts.downloadData) {
887888
process.exit(0);
@@ -1003,28 +1004,34 @@ export class BlockRunnerTransferLearning extends BlockRunner {
10031004

10041005
private async downloadFiles(
10051006
projectId: number,
1006-
learnId: number
1007+
learnId: number,
1008+
targetDir: string,
10071009
): Promise<void> {
10081010

1009-
let targetDir = Path.join(process.cwd(), 'ei-block-data', projectId.toString());
1010-
1011-
if (await fileExists(targetDir)) {
1012-
await FSHelpers.rmDir(Path.join(targetDir));
1013-
}
1014-
10151011
await fs.promises.mkdir(targetDir, { recursive: true });
10161012

1017-
let trainXRes = (await this._eiConfig.api.learn.getLearnXData(projectId, learnId));
1018-
await fs.promises.writeFile(
1019-
Path.join(targetDir, "X_train_features.npy"),
1020-
trainXRes
1021-
);
1013+
console.log(CON_PREFIX, 'Creating download job...');
1014+
let job = await this._eiConfig.api.jobs.exportKerasBlockData(projectId, learnId);
1015+
console.log(CON_PREFIX, 'Creating download job OK', job.id);
1016+
await this._eiConfig.api.runJobUntilCompletion({
1017+
type: 'project',
1018+
projectId: projectId,
1019+
jobId: job.id,
1020+
}, x => {
1021+
process.stdout.write(x);
1022+
});
1023+
console.log(CON_PREFIX, 'Download job completed, downloading data...');
10221024

1023-
let trainYRes = (await this._eiConfig.api.learn.getLearnYData(projectId, learnId));
1025+
let zipFile = Path.join(targetDir, "data.zip");
1026+
let data = await this._eiConfig.api.learn.downloadKerasData(projectId, learnId);
10241027
await fs.promises.writeFile(
1025-
Path.join(targetDir, "y_train.npy"),
1026-
trainYRes
1028+
zipFile,
1029+
data
10271030
);
1031+
1032+
console.log(CON_PREFIX, 'Download completed, unzipping...');
1033+
await extractFiles(zipFile, targetDir);
1034+
await fs.promises.unlink(zipFile);
10281035
}
10291036

10301037
private async checkFilesPresent(projectId: number): Promise<boolean> {
@@ -1125,15 +1132,17 @@ export class BlockRunnerDeploy extends BlockRunner {
11251132
console.log(CON_PREFIX, "Finished building");
11261133
}
11271134

1128-
let targetDir = Path.join(process.cwd(), "ei-block-data", this._projectId.toString(), "input");
1135+
let targetDir = (this._runnerOpts.type === 'deploy' && typeof this._runnerOpts.downloadData === 'string') ?
1136+
this._runnerOpts.downloadData :
1137+
Path.join(process.cwd(), 'ei-block-data', this._projectId.toString(), 'input');
11291138
let zipDir = Path.join(process.cwd(), "ei-block-data", "download");
11301139

11311140
console.log(CON_PREFIX, `Downloading build artifacts ZIP into ${zipDir}...`);
11321141
await this.downloadFile(zipDir, "build-data.zip", this._projectId);
11331142
console.log(CON_PREFIX, "Downloaded file");
11341143

11351144
console.log(CON_PREFIX, `Unzipping into ${targetDir}...`);
1136-
await this.extractFiles(Path.join(zipDir, "build-data.zip"), targetDir);
1145+
await extractFiles(Path.join(zipDir, "build-data.zip"), targetDir);
11371146
console.log(CON_PREFIX, `Extracted into ${targetDir}`);
11381147

11391148
if (this._runnerOpts.type === 'deploy' && this._runnerOpts.downloadData) {
@@ -1220,107 +1229,6 @@ export class BlockRunnerDeploy extends BlockRunner {
12201229

12211230
await fs.promises.writeFile(downloadPath, buildArtifacts);
12221231
}
1223-
1224-
private async extractFiles(
1225-
zipFileName: string,
1226-
targetFolder: string
1227-
): Promise<void> {
1228-
if (!(await fileExists(zipFileName))) {
1229-
throw new Error(`Unable to find file ${zipFileName} for unzipping`);
1230-
}
1231-
1232-
if (await fileExists(targetFolder)) {
1233-
await FSHelpers.rmDir(targetFolder);
1234-
}
1235-
1236-
await fs.promises.mkdir(targetFolder, { recursive: true });
1237-
1238-
return new Promise<void>((resolve, reject) => {
1239-
1240-
1241-
yauzl.open(zipFileName, { lazyEntries: true }, (err, zipFile) => {
1242-
if (err) reject(new Error(err.toString()));
1243-
if (!zipFile) reject(new Error("File to unzip is undefined"));
1244-
1245-
if (zipFile) {
1246-
zipFile.readEntry();
1247-
1248-
zipFile.on("entry", async (entry: Entry) => {
1249-
try {
1250-
if (/\/$/.test(entry.fileName)) {
1251-
// A directory
1252-
await fs.promises.mkdir(
1253-
Path.join(targetFolder, entry.fileName)
1254-
);
1255-
zipFile.readEntry();
1256-
}
1257-
else {
1258-
zipFile.openReadStream(
1259-
entry,
1260-
(error2, readStream) => {
1261-
if (error2) {
1262-
reject(error2);
1263-
}
1264-
if (!readStream) {
1265-
reject(
1266-
new Error(
1267-
"Unable to open file stream"
1268-
)
1269-
);
1270-
}
1271-
1272-
let writeStream = fs.createWriteStream(
1273-
Path.join(
1274-
targetFolder,
1275-
entry.fileName
1276-
),
1277-
{ flags: "w" }
1278-
);
1279-
1280-
if (readStream) {
1281-
readStream.pipe(writeStream);
1282-
}
1283-
1284-
writeStream.on("finish", () => {
1285-
// @ts-ignore
1286-
writeStream.close(() => {
1287-
zipFile.readEntry();
1288-
});
1289-
1290-
writeStream.on("error", (error) => {
1291-
zipFile.close();
1292-
reject(
1293-
new Error(
1294-
"Error reading ZIP entry: " + error
1295-
)
1296-
);
1297-
});
1298-
});
1299-
}
1300-
);
1301-
}
1302-
} catch (ex) {
1303-
zipFile.close();
1304-
reject(new Error("Unable to read ZIP file"));
1305-
}
1306-
});
1307-
1308-
zipFile.on("end", () => {
1309-
resolve();
1310-
});
1311-
1312-
zipFile.on("error", (error) => {
1313-
zipFile.close();
1314-
reject(
1315-
new Error(
1316-
"Error reading ZIP file: " + error
1317-
)
1318-
);
1319-
});
1320-
}
1321-
});
1322-
});
1323-
}
13241232
}
13251233

13261234
export class BlockRunnerDSP extends BlockRunner {
@@ -1526,3 +1434,98 @@ export class BlockRunnerDSP extends BlockRunner {
15261434
});
15271435
}
15281436
}
1437+
1438+
async function extractFiles(
1439+
zipFileName: string,
1440+
targetFolder: string
1441+
): Promise<void> {
1442+
if (!(await fileExists(zipFileName))) {
1443+
throw new Error(`Unable to find file ${zipFileName} for unzipping`);
1444+
}
1445+
1446+
await fs.promises.mkdir(targetFolder, { recursive: true });
1447+
1448+
return new Promise<void>((resolve, reject) => {
1449+
yauzl.open(zipFileName, { lazyEntries: true }, (err, zipFile) => {
1450+
if (err) reject(new Error(err.toString()));
1451+
if (!zipFile) reject(new Error("File to unzip is undefined"));
1452+
1453+
if (zipFile) {
1454+
zipFile.readEntry();
1455+
1456+
zipFile.on("entry", async (entry: Entry) => {
1457+
try {
1458+
if (/\/$/.test(entry.fileName)) {
1459+
// A directory
1460+
await fs.promises.mkdir(
1461+
Path.join(targetFolder, entry.fileName)
1462+
);
1463+
zipFile.readEntry();
1464+
}
1465+
else {
1466+
zipFile.openReadStream(
1467+
entry,
1468+
(error2, readStream) => {
1469+
if (error2) {
1470+
reject(error2);
1471+
}
1472+
if (!readStream) {
1473+
reject(
1474+
new Error(
1475+
"Unable to open file stream"
1476+
)
1477+
);
1478+
}
1479+
1480+
let writeStream = fs.createWriteStream(
1481+
Path.join(
1482+
targetFolder,
1483+
entry.fileName
1484+
),
1485+
{ flags: "w" }
1486+
);
1487+
1488+
if (readStream) {
1489+
readStream.pipe(writeStream);
1490+
}
1491+
1492+
writeStream.on("finish", () => {
1493+
// @ts-ignore
1494+
writeStream.close(() => {
1495+
zipFile.readEntry();
1496+
});
1497+
1498+
writeStream.on("error", (error) => {
1499+
zipFile.close();
1500+
reject(
1501+
new Error(
1502+
"Error reading ZIP entry: " + error
1503+
)
1504+
);
1505+
});
1506+
});
1507+
}
1508+
);
1509+
}
1510+
} catch (ex) {
1511+
zipFile.close();
1512+
reject(new Error("Unable to read ZIP file"));
1513+
}
1514+
});
1515+
1516+
zipFile.on("end", () => {
1517+
resolve();
1518+
});
1519+
1520+
zipFile.on("error", (error) => {
1521+
zipFile.close();
1522+
reject(
1523+
new Error(
1524+
"Error reading ZIP file: " + error
1525+
)
1526+
);
1527+
});
1528+
}
1529+
});
1530+
});
1531+
}

0 commit comments

Comments
 (0)