Skip to content

Commit 2914011

Browse files
OmkarPhOmkarPh
authored
Merge pull request #624 from nexB/support/only-findings-option
Build filetree with missing directories to support --only-findings scans
2 parents b5ea739 + 83ee678 commit 2914011

File tree

19 files changed

+405
-85
lines changed

19 files changed

+405
-85
lines changed

‎src/App.tsx‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import "react-toastify/dist/ReactToastify.css";
2929
import "bootstrap/dist/css/bootstrap.min.css";
3030
import "react-tooltip/dist/react-tooltip.css";
3131
import "./styles/app.css";
32+
import "./styles/toastify.css";
3233
import "./styles/dashStyles.css";
3334
import "./styles/customFaColors.css";
3435

‎src/constants/errors.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
export const NO_RESOURCES_ERROR = "No files found in the scan"

‎src/contexts/dbContext.tsx‎

Lines changed: 35 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import {
2020
UTIL_CHANNEL,
2121
} from "../constants/IpcConnection";
2222
import { DEFAULT_ROUTE_ON_IMPORT, ROUTES } from "../constants/routes";
23+
import { NO_RESOURCES_ERROR } from "../constants/errors";
2324
import { AddEntry, GetHistory, RemoveEntry } from "../services/historyStore";
2425
import { WorkbenchDB } from "../services/workbenchDB";
2526
import { isSqliteSchemaOutdated } from "../utils/checks";
@@ -183,12 +184,8 @@ export const WorkbenchDBProvider = (
183184
// Check that the database has the correct header information.
184185
if (!infoHeader) {
185186
const errTitle = "Invalid SQLite file";
186-
const errMessage =
187-
"Invalid SQLite file: " +
188-
sqliteFilePath +
189-
"\n" +
190-
"The SQLite file is invalid. Try re-importing the ScanCode JSON " +
191-
"file and creating a new SQLite file.";
187+
const errMessage = `Invalid SQLite file: ${sqliteFilePath}
188+
The SQLite file is invalid. Try re-importing the ScanCode JSON file and creating a new SQLite file.`;
192189

193190
console.error("Handled invalid sqlite import", {
194191
title: errTitle,
@@ -239,8 +236,7 @@ export const WorkbenchDBProvider = (
239236
.then((db) => db.File.findOne({ where: { parent: "#" } }))
240237
.then(async (root) => {
241238
if (!root) {
242-
console.error("Root path not found !!!!", root);
243-
return;
239+
throw new Error("Root path not found !!");
244240
}
245241

246242
const defaultPath = root.getDataValue("path");
@@ -271,35 +267,20 @@ export const WorkbenchDBProvider = (
271267
);
272268

273269
if (!preventNavigation) changeRouteOnImport();
274-
})
275-
.catch((err) => {
276-
const foundInvalidHistoryItem = GetHistory().find(
277-
(historyItem) => historyItem.sqlite_path === sqliteFilePath
278-
);
279-
if (foundInvalidHistoryItem) {
280-
RemoveEntry(foundInvalidHistoryItem);
281-
}
282-
console.error("Err trying to import sqlite:");
283-
console.error(err);
284-
toast.error(
285-
`Unexpected error while importing json \nPlease check console for more info`
286-
);
287-
abortImport();
288270
});
289271
})
290-
.catch((err) => {
272+
.catch((err: Error) => {
273+
abortImport();
291274
const foundInvalidHistoryItem = GetHistory().find(
292275
(historyItem) => historyItem.sqlite_path === sqliteFilePath
293276
);
294277
if (foundInvalidHistoryItem) {
295278
RemoveEntry(foundInvalidHistoryItem);
296279
}
297-
console.error("Err trying to import sqlite:");
298-
console.error(err);
280+
console.error("Err trying to import sqlite:", err);
299281
toast.error(
300-
`Unexpected error while finalising json import \nPlease check console for more info`
282+
`Sqlite file is outdated or corrupt\nPlease try importing json file again`
301283
);
302-
abortImport();
303284
});
304285
}
305286

@@ -341,11 +322,10 @@ export const WorkbenchDBProvider = (
341322
.then((db) => db.File.findOne({ where: { parent: "#" } }))
342323
.then(async (root) => {
343324
if (!root) {
344-
console.error("Root path not found !!!!");
345325
console.error("Root:", root);
346-
abortImport();
347-
return;
326+
throw new Error("Root path not found !!!!");
348327
}
328+
349329
const defaultPath = root.getDataValue("path");
350330

351331
await updateWorkbenchDB(newWorkbenchDB, sqliteFilePath);
@@ -369,17 +349,34 @@ export const WorkbenchDBProvider = (
369349
);
370350

371351
if (!preventNavigation) changeRouteOnImport();
352+
})
353+
.catch((err) => {
354+
abortImport();
355+
const foundInvalidHistoryItem = GetHistory().find(
356+
(historyItem) => historyItem.sqlite_path === sqliteFilePath
357+
);
358+
if (foundInvalidHistoryItem) {
359+
RemoveEntry(foundInvalidHistoryItem);
360+
}
361+
console.error(err);
362+
toast.error(
363+
`Can't resolve root directory \nPlease check console for more info`
364+
);
372365
});
373366
})
374-
.catch((err) => {
367+
.catch((err: Error) => {
375368
abortImport();
376-
console.error(
377-
"Some error parsing data (caught in workbenchContext) !!",
378-
err
379-
);
380-
toast.error(
381-
"Some error parsing data !! \nPlease check console for more info"
382-
);
369+
if (err.message === NO_RESOURCES_ERROR) {
370+
toast.error("No resources found in scan\nAborting import");
371+
} else {
372+
console.error(
373+
"Some error parsing json data (caught in dbContext)",
374+
err
375+
);
376+
toast.error(
377+
"Some error parsing json data !! \nPlease check console for more info"
378+
);
379+
}
383380
});
384381
}
385382

‎src/pages/Licenses/Licenses.tsx‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -171,7 +171,13 @@ const LicenseDetections = () => {
171171
activateLicenseClue(newLicenseClues[0]);
172172
}
173173
}
174-
})().then(endProcessing);
174+
})()
175+
.then(endProcessing)
176+
.catch((err) => {
177+
endProcessing();
178+
console.error("Error getting license contents", err);
179+
toast.error("Error loading license data.");
180+
});
175181
}, []);
176182

177183
const handleItemToggle = (

‎src/pages/TableView/TableView.tsx‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { Op } from "sequelize";
2+
import { toast } from "react-toastify";
23
import React, { useEffect, useState } from "react";
34
import { ColDef, ColumnApi, GridApi, GridReadyEvent } from "ag-grid-community";
45

@@ -102,6 +103,11 @@ const TableView = () => {
102103
if (prevColDefs.length > 0) return prevColDefs; // Don't mutate cols, if already set
103104
return [...COLUMN_GROUPS.DEFAULT];
104105
});
106+
})
107+
.catch((err) => {
108+
endProcessing();
109+
console.error("Error getting tableview contents", err);
110+
toast.error("Error loading table data.");
105111
});
106112

107113
// Update set filters whenever new db is loaded or path is changed

‎src/services/models/header.ts‎

Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -53,15 +53,33 @@ export default function headerModel(sequelize: Sequelize) {
5353
primaryKey: true,
5454
type: DataTypes.INTEGER,
5555
},
56-
json_file_name: DataTypes.STRING,
57-
tool_name: DataTypes.STRING,
58-
tool_version: DataTypes.STRING,
59-
notice: DataTypes.STRING,
60-
duration: DataTypes.DOUBLE,
56+
json_file_name: {
57+
type: DataTypes.STRING,
58+
defaultValue: null,
59+
},
60+
tool_name: {
61+
type: DataTypes.STRING,
62+
defaultValue: null,
63+
},
64+
tool_version: {
65+
type: DataTypes.STRING,
66+
defaultValue: null,
67+
},
68+
notice: {
69+
type: DataTypes.STRING,
70+
defaultValue: null,
71+
},
72+
duration: {
73+
type: DataTypes.DOUBLE,
74+
defaultValue: null,
75+
},
6176
options: jsonDataType("options", {}),
6277
input: jsonDataType("input", []),
6378
header_content: DataTypes.STRING,
64-
files_count: DataTypes.INTEGER,
79+
files_count: {
80+
type: DataTypes.INTEGER,
81+
defaultValue: 0,
82+
},
6583
output_format_version: {
6684
type: DataTypes.STRING,
6785
defaultValue: null,

‎src/services/workbenchDB.ts‎

Lines changed: 62 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ import { TodoAttributes } from "./models/todo";
5555
import { HeaderAttributes } from "./models/header";
5656
import { LicenseReferenceAttributes } from "./models/licenseReference";
5757
import { LicenseRuleReferenceAttributes } from "./models/licenseRuleReference";
58+
import { NO_RESOURCES_ERROR } from "../constants/errors";
5859

5960
const { version: workbenchVersion } = packageJson;
6061

@@ -282,12 +283,12 @@ export class WorkbenchDB {
282283
fileList.forEach((file) => {
283284
const fileParentPath = file.getDataValue("parent");
284285
const fileNode = pathToNodeMap.get(file.getDataValue("path"));
285-
if (Number(file.getDataValue("id")) !== 0) {
286+
if (file.getDataValue("parent") === "#") {
287+
roots.push(fileNode);
288+
} else {
286289
if (pathToNodeMap.has(fileParentPath)) {
287290
pathToNodeMap.get(fileParentPath).children?.push(fileNode);
288291
}
289-
} else {
290-
roots.push(fileNode);
291292
}
292293
});
293294

@@ -309,11 +310,10 @@ export class WorkbenchDB {
309310
const stream = fs.createReadStream(jsonFilePath, { encoding: "utf8" });
310311
let files_count = 0;
311312
let dirs_count = 0;
312-
let index = 0;
313-
let rootPath: string | null = null;
314-
let hasRootPath = false;
315313
const batchSize = 1000;
316314
let files: Resource[] = [];
315+
const parsedFilePaths = new Set<string>();
316+
317317
let progress = 0;
318318
let promiseChain: Promise<unknown> = this.sync;
319319

@@ -379,38 +379,40 @@ export class WorkbenchDB {
379379
.on("data", function (file?: Resource) {
380380
if (!file) return;
381381

382-
if (!rootPath) {
383-
rootPath = file.path.split("/")[0];
384-
}
385-
if (rootPath === file.path) {
386-
hasRootPath = true;
387-
}
388382
// @TODO: When/if scancode reports directories in its header, this needs
389383
// to be replaced.
390-
if (index === 0) {
384+
if (parsedFilePaths.size === 0) {
391385
dirs_count = file.dirs_count;
392386
}
393-
file.id = index++;
387+
file.id = parsedFilePaths.size;
394388

395389
primaryPromise._parseLicenseDetections(file, TopLevelData);
396390
primaryPromise._parseLicenseClues(file, TopLevelData);
397391

398392
files.push(file);
393+
parsedFilePaths.add(file.path);
394+
399395
if (files.length >= batchSize) {
400396
// Need to set a new variable before handing to promise
401397
this.pause();
402398

403399
promiseChain = promiseChain
400+
.then(() =>
401+
primaryPromise._imputeIntermediateDirectories(
402+
files,
403+
parsedFilePaths
404+
)
405+
)
404406
.then(() => primaryPromise._batchCreateFiles(files))
405407
.then(() => {
406408
const currentProgress = Math.round(
407-
(index / (files_count + dirs_count)) * 100
409+
(parsedFilePaths.size / (files_count + dirs_count)) * 100
408410
);
409411
if (currentProgress > progress) {
410412
progress = currentProgress;
411413
console.info(
412414
`Batch-${++batchCount} completed, \n`,
413-
`JSON Import progress @ ${progress} % -- ${index}/${files_count}+${dirs_count}`
415+
`JSON Import progress @ ${progress} % -- ${parsedFilePaths.size}/${files_count}+${dirs_count}`
414416
);
415417
onProgressUpdate(progress);
416418
}
@@ -426,22 +428,20 @@ export class WorkbenchDB {
426428
// Add root directory into data
427429
// See https://github.com/nexB/scancode-toolkit/issues/543
428430
promiseChain
431+
.then(() =>
432+
this._imputeIntermediateDirectories(files, parsedFilePaths)
433+
)
429434
.then(() => {
430-
if (rootPath && !hasRootPath) {
431-
files.push({
432-
path: rootPath,
433-
name: rootPath,
434-
type: "directory",
435-
files_count: files_count,
436-
});
435+
if (files.length === 0) {
436+
throw new Error(NO_RESOURCES_ERROR);
437437
}
438438
})
439439
.then(() => this._batchCreateFiles(files))
440440
.then(() => this.db.Header.create(TopLevelData.parsedHeader))
441441
.then(() => {
442442
console.info(
443443
`Batch-${++batchCount} completed, \n`,
444-
`JSON Import progress @ ${progress} % -- ${index}/${files_count}+${dirs_count}`
444+
`JSON Import progress @ ${progress} % -- ${parsedFilePaths.size}/${files_count}+${dirs_count}`
445445
);
446446
onProgressUpdate(90);
447447
})
@@ -456,10 +456,12 @@ export class WorkbenchDB {
456456
.then(() => {
457457
onProgressUpdate(100);
458458
console.info("JSON parse completed (final step)");
459-
console.timeEnd("json-parse-time");
460459
resolve();
461460
})
462-
.catch((e: unknown) => reject(e));
461+
.catch((e: unknown) => reject(e))
462+
.finally(() => {
463+
console.timeEnd("json-parse-time");
464+
});
463465
})
464466
.on("error", (err: unknown) => {
465467
console.error(
@@ -600,7 +602,7 @@ export class WorkbenchDB {
600602
duration: header.duration,
601603
options: header?.options || {},
602604
input,
603-
files_count: header.extra_data?.files_count,
605+
files_count: header.extra_data?.files_count || 0,
604606
output_format_version: header.output_format_version,
605607
spdx_license_list_version: header.extra_data?.spdx_license_list_version,
606608
operating_system: header.extra_data?.system_environment?.operating_system,
@@ -880,6 +882,39 @@ export class WorkbenchDB {
880882
});
881883
}
882884

885+
// Adds & modifies files array in place, adding missing intermediate directories
886+
_imputeIntermediateDirectories(
887+
files: Resource[],
888+
parsedFilePaths: Set<string>
889+
) {
890+
const intermediateDirectories: Resource[] = [];
891+
892+
files.forEach((file) => {
893+
file.parent = parentPath(file.path);
894+
895+
// Add intermediate directories if parent not available in files
896+
if (!parsedFilePaths.has(file.parent)) {
897+
for (
898+
let currentDir = file.parent;
899+
currentDir !== parentPath(currentDir) &&
900+
!parsedFilePaths.has(currentDir);
901+
currentDir = parentPath(currentDir)
902+
) {
903+
intermediateDirectories.push({
904+
id: parsedFilePaths.size,
905+
path: currentDir,
906+
parent: parentPath(currentDir),
907+
name: path.basename(currentDir),
908+
type: "directory",
909+
files_count: 0,
910+
});
911+
parsedFilePaths.add(currentDir);
912+
}
913+
}
914+
});
915+
files.push(...intermediateDirectories);
916+
}
917+
883918
_batchCreateFiles(files: Resource[]) {
884919
// Add batched files to the DB
885920
return this._addFlattenedFiles(files).then(() => this._addFiles(files));

0 commit comments

Comments
 (0)