Build filetree with missing directories to support --only-findings scans

OmkarPh · OmkarPh · commit 19d3d6a42f35 · 2024-01-01T23:50:49.000+05:30
Signed-off-by: Omkar Phansopkar &lt;omkarphansopkar@gmail.com&gt;
diff --git a/src/App.tsx b/src/App.tsx
@@ -29,6 +29,7 @@ import "react-toastify/dist/ReactToastify.css";
 import "bootstrap/dist/css/bootstrap.min.css";
 import "react-tooltip/dist/react-tooltip.css";
 import "./styles/app.css";
+import "./styles/toastify.css";
 import "./styles/dashStyles.css";
 import "./styles/customFaColors.css";
 
diff --git a/src/contexts/dbContext.tsx b/src/contexts/dbContext.tsx
@@ -183,12 +183,8 @@ export const WorkbenchDBProvider = (
         // Check that the database has the correct header information.
         if (!infoHeader) {
           const errTitle = "Invalid SQLite file";
-          const errMessage =
-            "Invalid SQLite file: " +
-            sqliteFilePath +
-            "\n" +
-            "The SQLite file is invalid. Try re-importing the ScanCode JSON " +
-            "file and creating a new SQLite file.";
+          const errMessage = `Invalid SQLite file: ${sqliteFilePath}
+The SQLite file is invalid. Try re-importing the ScanCode JSON file and creating a new SQLite file.`;
 
           console.error("Handled invalid sqlite import", {
             title: errTitle,
@@ -239,8 +235,7 @@ export const WorkbenchDBProvider = (
           .then((db) => db.File.findOne({ where: { parent: "#" } }))
           .then(async (root) => {
             if (!root) {
-              console.error("Root path not found !!!!", root);
-              return;
+              throw new Error("Root path not found !!");
             }
 
             const defaultPath = root.getDataValue("path");
@@ -271,35 +266,20 @@ export const WorkbenchDBProvider = (
             );
 
             if (!preventNavigation) changeRouteOnImport();
-          })
-          .catch((err) => {
-            const foundInvalidHistoryItem = GetHistory().find(
-              (historyItem) => historyItem.sqlite_path === sqliteFilePath
-            );
-            if (foundInvalidHistoryItem) {
-              RemoveEntry(foundInvalidHistoryItem);
-            }
-            console.error("Err trying to import sqlite:");
-            console.error(err);
-            toast.error(
-              `Unexpected error while importing json \nPlease check console for more info`
-            );
-            abortImport();
           });
       })
-      .catch((err) => {
+      .catch((err: Error) => {
+        abortImport();
         const foundInvalidHistoryItem = GetHistory().find(
           (historyItem) => historyItem.sqlite_path === sqliteFilePath
         );
         if (foundInvalidHistoryItem) {
           RemoveEntry(foundInvalidHistoryItem);
         }
-        console.error("Err trying to import sqlite:");
-        console.error(err);
+        console.error("Err trying to import sqlite:", err);
         toast.error(
-          `Unexpected error while finalising json import \nPlease check console for more info`
+          `Sqlite file is outdated or corrupt\nPlease try importing json file again`
         );
-        abortImport();
       });
   }
 
@@ -341,11 +321,10 @@ export const WorkbenchDBProvider = (
           .then((db) => db.File.findOne({ where: { parent: "#" } }))
           .then(async (root) => {
             if (!root) {
-              console.error("Root path not found !!!!");
               console.error("Root:", root);
-              abortImport();
-              return;
+              throw new Error("Root path not found !!!!");
             }
+
             const defaultPath = root.getDataValue("path");
 
             await updateWorkbenchDB(newWorkbenchDB, sqliteFilePath);
@@ -369,16 +348,29 @@ export const WorkbenchDBProvider = (
             );
 
             if (!preventNavigation) changeRouteOnImport();
+          })
+          .catch((err) => {
+            abortImport();
+            const foundInvalidHistoryItem = GetHistory().find(
+              (historyItem) => historyItem.sqlite_path === sqliteFilePath
+            );
+            if (foundInvalidHistoryItem) {
+              RemoveEntry(foundInvalidHistoryItem);
+            }
+            console.error(err);
+            toast.error(
+              `Can't resolve root directory \nPlease check console for more info`
+            );
           });
       })
       .catch((err) => {
         abortImport();
         console.error(
-          "Some error parsing data (caught in workbenchContext) !!",
+          "Some error parsing json data (caught in dbContext)",
           err
         );
         toast.error(
-          "Some error parsing data !! \nPlease check console for more info"
+          "Some error parsing json data !! \nPlease check console for more info"
         );
       });
   }
diff --git a/src/pages/Licenses/Licenses.tsx b/src/pages/Licenses/Licenses.tsx
@@ -171,7 +171,13 @@ const LicenseDetections = () => {
           activateLicenseClue(newLicenseClues[0]);
         }
       }
-    })().then(endProcessing);
+    })()
+      .then(endProcessing)
+      .catch((err) => {
+        endProcessing();
+        console.error("Error getting license contents", err);
+        toast.error("Error loading license data.");
+      });
   }, []);
 
   const handleItemToggle = (
diff --git a/src/pages/TableView/TableView.tsx b/src/pages/TableView/TableView.tsx
@@ -1,4 +1,5 @@
 import { Op } from "sequelize";
+import { toast } from "react-toastify";
 import React, { useEffect, useState } from "react";
 import { ColDef, ColumnApi, GridApi, GridReadyEvent } from "ag-grid-community";
 
@@ -101,6 +102,11 @@ const TableView = () => {
           if (prevColDefs.length > 0) return prevColDefs; // Don't mutate cols, if already set
           return [...COLUMN_GROUPS.DEFAULT];
         });
+      })
+      .catch((err) => {
+        endProcessing();
+        console.error("Error getting tableview contents", err);
+        toast.error("Error loading table data.");
       });
 
     // Update set filters whenever new db is loaded or path is changed
diff --git a/src/services/models/header.ts b/src/services/models/header.ts
@@ -53,15 +53,33 @@ export default function headerModel(sequelize: Sequelize) {
         primaryKey: true,
         type: DataTypes.INTEGER,
       },
-      json_file_name: DataTypes.STRING,
-      tool_name: DataTypes.STRING,
-      tool_version: DataTypes.STRING,
-      notice: DataTypes.STRING,
-      duration: DataTypes.DOUBLE,
+      json_file_name: {
+        type: DataTypes.STRING,
+        defaultValue: null,
+      },
+      tool_name: {
+        type: DataTypes.STRING,
+        defaultValue: null,
+      },
+      tool_version: {
+        type: DataTypes.STRING,
+        defaultValue: null,
+      },
+      notice: {
+        type: DataTypes.STRING,
+        defaultValue: null,
+      },
+      duration: {
+        type: DataTypes.DOUBLE,
+        defaultValue: null,
+      },
       options: jsonDataType("options", {}),
       input: jsonDataType("input", []),
       header_content: DataTypes.STRING,
-      files_count: DataTypes.INTEGER,
+      files_count: {
+        type: DataTypes.INTEGER,
+        defaultValue: 0,
+      },
       output_format_version: {
         type: DataTypes.STRING,
         defaultValue: null,
diff --git a/src/services/workbenchDB.ts b/src/services/workbenchDB.ts
@@ -282,12 +282,12 @@ export class WorkbenchDB {
     fileList.forEach((file) => {
       const fileParentPath = file.getDataValue("parent");
       const fileNode = pathToNodeMap.get(file.getDataValue("path"));
-      if (Number(file.getDataValue("id")) !== 0) {
+      if (file.getDataValue("parent") === "#") {
+        roots.push(fileNode);
+      } else {
         if (pathToNodeMap.has(fileParentPath)) {
           pathToNodeMap.get(fileParentPath).children?.push(fileNode);
         }
-      } else {
-        roots.push(fileNode);
       }
     });
 
@@ -401,6 +401,7 @@ export class WorkbenchDB {
                 this.pause();
 
                 promiseChain = promiseChain
+                  .then(() => this._imputeMissingIntermediateDirectories(files))
                   .then(() => primaryPromise._batchCreateFiles(files))
                   .then(() => {
                     const currentProgress = Math.round(
@@ -427,7 +428,8 @@ export class WorkbenchDB {
               // See https://github.com/nexB/scancode-toolkit/issues/543
               promiseChain
                 .then(() => {
-                  if (rootPath && !hasRootPath) {
+                  if (!hasRootPath) {
+                    rootPath = rootPath || "no-files";
                     files.push({
                       path: rootPath,
                       name: rootPath,
@@ -436,6 +438,7 @@ export class WorkbenchDB {
                     });
                   }
                 })
+                .then(() => this._imputeIntermediateDirectories(files))
                 .then(() => this._batchCreateFiles(files))
                 .then(() => this.db.Header.create(TopLevelData.parsedHeader))
                 .then(() => {
@@ -600,7 +603,7 @@ export class WorkbenchDB {
       duration: header.duration,
       options: header?.options || {},
       input,
-      files_count: header.extra_data?.files_count,
+      files_count: header.extra_data?.files_count || 0,
       output_format_version: header.output_format_version,
       spdx_license_list_version: header.extra_data?.spdx_license_list_version,
       operating_system: header.extra_data?.system_environment?.operating_system,
@@ -880,6 +883,37 @@ export class WorkbenchDB {
     });
   }
 
+  // Adds & modifies files array in place, adding missing intermediate directories
+  _imputeIntermediateDirectories(files: Resource[]) {
+    const availableFiles = new Set(files.map((file) => file.path));
+    const intermediateDirectories: Resource[] = [];
+
+    files.forEach((file) => {
+      file.parent = parentPath(file.path);
+
+      // Add intermediate directories if parent not available in files
+      if (!availableFiles.has(file.parent)) {
+        for (
+          let currentDir = file.parent;
+          currentDir !== parentPath(currentDir) &&
+          !availableFiles.has(currentDir);
+          currentDir = parentPath(currentDir)
+        ) {
+          availableFiles.add(currentDir);
+          intermediateDirectories.push({
+            path: currentDir,
+            parent: parentPath(currentDir),
+            name: path.basename(currentDir),
+            type: "directory",
+            files_count: 0,
+          });
+        }
+        availableFiles.add(file.parent);
+      }
+    });
+    files.push(...intermediateDirectories);
+  }
+
   _batchCreateFiles(files: Resource[]) {
     // Add batched files to the DB
     return this._addFlattenedFiles(files).then(() => this._addFiles(files));
diff --git a/src/styles/toastify.css b/src/styles/toastify.css
@@ -0,0 +1,10 @@
+.Toastify__toast-body {
+  white-space: pre;
+  width: fit-content;
+  padding-right: 10px;
+}
+
+.Toastify__toast-container,
+.Toastify__toast-body>div:last-child {
+  width: max-content;
+}
diff --git a/tests/test-scans/fileTree/expectedFileTree.ts b/tests/test-scans/fileTree/expectedFileTree.ts
diff --git a/tests/test-scans/fileTree/onlyFindings.json b/tests/test-scans/fileTree/onlyFindings.json
diff --git a/tests/test-scans/licenses/expectedLicenses.ts b/tests/test-scans/licenses/expectedLicenses.ts

Original file line number	Diff line number	Diff line change
`@@ -171,7 +171,13 @@ const LicenseDetections = () => {`
`171`	`171`	`activateLicenseClue(newLicenseClues[0]);`
`172`	`172`	`}`
`173`	`173`	`}`
`174`		`- })().then(endProcessing);`
	`174`	`+ })()`
	`175`	`+ .then(endProcessing)`
	`176`	`+ .catch((err) => {`
	`177`	`+ endProcessing();`
	`178`	`+ console.error("Error getting license contents", err);`
	`179`	`+ toast.error("Error loading license data.");`
	`180`	`+ });`
`175`	`181`	`}, []);`
`176`	`182`
`177`	`183`	`const handleItemToggle = (`