Merge pull request #619 from nexB/fix/missing-license-expressions_spdx

Fixed compund SPDX expression resolution in detection & clue matches

Author: OmkarPh

src/components/LicenseEntity/LicenseEntity.tsx

@@ -9,7 +9,6 @@ import {
 } from "./FileRegionTableCols";
 import { ScanOptionKeys } from "../../utils/parsers";
 import LicenseMatchesTable from "./LicenseMatchesTable";
-import { LicenseTypes } from "../../services/workbenchDB.types";
 import { useWorkbenchDB } from "../../contexts/dbContext";
 import { TodoAttributes } from "../../services/models/todo";
 
@@ -105,13 +104,7 @@ const LicenseEntity = (props: LicenseDetectionEntityProps) => {
       </div>
       <b>Matches</b>
       <LicenseMatchesTable
-        matchesInfo={{
-          licenseType:
-            activeLicenseEntity.type === "detection"
-              ? LicenseTypes.DETECTION
-              : LicenseTypes.CLUE,
-          matches: matches,
-        }}
+        matches={matches}
         showLIcenseText={
           Boolean(scanInfo.optionsMap.get(ScanOptionKeys.LICENSE_TEXT)) ||
           matches[0]?.matched_text?.length > 0 ||

src/components/LicenseEntity/LicenseMatchCells/MatchLicenseExpression.tsx

@@ -3,7 +3,7 @@ import CoreLink from "../../CoreLink/CoreLink";
 import {
   LICENSE_EXPRESSIONS_CONJUNCTIONS,
   parseTokensFromExpression,
-} from "../../../services/models/databaseUtils";
+} from "../../../utils/expressions";
 import {
   LicenseClueMatch,
   LicenseDetectionMatch,

src/components/LicenseEntity/LicenseMatchesTable.tsx

@@ -6,7 +6,6 @@ import {
 import { Button, OverlayTrigger, Popover, Table } from "react-bootstrap";
 import MatchedTextRenderer from "./LicenseMatchCells/MatchedText";
 import MatchLicenseExpressionRenderer from "./LicenseMatchCells/MatchLicenseExpression";
-import { LicenseTypes } from "../../services/workbenchDB.types";
 import { MatchedTextProvider } from "./MatchedTextContext";
 import MatchRuleDetails from "./MatchRuleDetails";
 import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
@@ -15,23 +14,15 @@ import CoreLink from "../CoreLink/CoreLink";
 
 interface LicenseMatchProps {
   showLIcenseText?: boolean;
-  matchesInfo:
-    | {
-        licenseType: LicenseTypes.DETECTION;
-        matches: LicenseDetectionMatch[];
-      }
-    | {
-        licenseType: LicenseTypes.CLUE;
-        matches: LicenseClueMatch[];
-      };
+  matches: LicenseClueMatch[] | LicenseDetectionMatch[];
 }
 const LicenseMatchesTable = (props: LicenseMatchProps) => {
-  const { matchesInfo, showLIcenseText } = props;
+  const { matches, showLIcenseText } = props;
 
   return (
     <MatchedTextProvider>
       <div>
-        {matchesInfo.matches.map((match, idx) => (
+        {matches.map((match, idx) => (
           <div
             className="matches-table-container"
             key={match.license_expression + "_" + idx}
@@ -49,20 +40,19 @@ const LicenseMatchesTable = (props: LicenseMatchProps) => {
                     />
                   </td>
                 </tr>
-                {matchesInfo.licenseType === LicenseTypes.DETECTION &&
-                  (match as LicenseDetectionMatch)?.license_expression_spdx && (
-                    <tr>
-                      <td colSpan={2}>License Expression SPDX</td>
-                      <td colSpan={7}>
-                        <MatchLicenseExpressionRenderer
-                          matchInfo={{
-                            match: match,
-                            spdxLicense: true,
-                          }}
-                        />
-                      </td>
-                    </tr>
-                  )}
+                {match.license_expression_spdx && (
+                  <tr>
+                    <td colSpan={2}>License Expression SPDX</td>
+                    <td colSpan={7}>
+                      <MatchLicenseExpressionRenderer
+                        matchInfo={{
+                          match: match,
+                          spdxLicense: true,
+                        }}
+                      />
+                    </td>
+                  </tr>
+                )}
                 {showLIcenseText && (
                   <tr className="matched-text-row">
                     <td colSpan={2}>Matched Text</td>

src/services/importedJsonTypes.ts

@@ -18,18 +18,17 @@ export interface LicenseMatch {
   match_coverage: number;
   matcher: string;
   license_expression: string;
+  license_expression_spdx?: string;
   rule_identifier: string;
   rule_relevance: number;
   rule_url: string;
 
   // Parser-added fields
   path?: string;
   license_expression_keys?: LicenseExpressionKey[];
-}
-export interface LicenseDetectionMatch extends LicenseMatch {
-  license_expression_spdx?: string;
   license_expression_spdx_keys?: LicenseExpressionSpdxKey[];
 }
+export type LicenseDetectionMatch = LicenseMatch;
 export type LicenseClueMatch = LicenseMatch;
 
 export interface LicenseFileRegion {
@@ -57,6 +56,7 @@ export interface LicenseClue {
   fileClueIdx: number;
   matches?: LicenseClueMatch[];
   file_regions?: LicenseFileRegion[];
+  license_expression_spdx?: string;
 }
 export interface TopLevelLicenseDetection {
   identifier: string;

src/services/models/databaseUtils.ts

@@ -37,76 +37,3 @@ export function parentPath(path: string) {
   const splits = path.split("/");
   return splits.length === 1 ? "#" : splits.slice(0, -1).join("/");
 }
-
-export const LICENSE_EXPRESSIONS_CONJUNCTIONS = ["AND", "OR", "WITH"];
-
-// @TODO - Needs more testing
-export const parseSubExpressions = (expression: string) => {
-  if (!expression || !expression.length) return [];
-  const tokens = expression.split(/( |\(|\))/);
-  const result = [];
-  let currSubExpression = "";
-  let popTokens = 0;
-  for (const token of tokens) {
-    if (token === "(") {
-      if (popTokens) currSubExpression += "(";
-      popTokens++;
-    } else if (token === ")") {
-      popTokens--;
-      if (popTokens) {
-        currSubExpression += ")";
-      } else {
-        result.push(currSubExpression);
-        currSubExpression = "";
-      }
-    } else {
-      if (popTokens) currSubExpression += token;
-      else {
-        if (token.trim().length) result.push(token);
-      }
-    }
-  }
-
-  return result.filter(
-    (subExpression) =>
-      subExpression.trim().length &&
-      !LICENSE_EXPRESSIONS_CONJUNCTIONS.includes(subExpression.trim())
-  );
-};
-
-// Test parseSubExpressions
-// [
-//   'apache-2.0 AND (mit AND json) AND (apache-2.0 AND bsd-simplified AND bsd-new AND cc0-1.0 AND cddl-1.0) AND (cddl-1.0 AND bsd-new) AND (bsd-new AND epl-2.0 AND elastic-license-v2) AND (bsd-new AND json AND lgpl-2.0 AND mit AND gpl-2.0 AND universal-foss-exception-1.0)',
-//   'apache-2.0 AND cc0-1.0 AND mit AND (lgpl-2.1 AND bsd-new AND unknown-license-reference) AND bsd-new AND (mit AND apache-2.0 AND bsd-new) AND (ofl-1.1 AND mit AND cc-by-3.0) AND (mit AND cc0-1.0) AND (mit AND apache-2.0) AND (mit AND gpl-3.0) AND ((mit OR gpl-3.0) AND mit AND gpl-3.0) AND ofl-1.1 AND (ofl-1.1 AND proprietary-license) AND isc AND (bsd-new AND bsd-simplified) AND unknown AND (apache-2.0 AND isc) AND (apache-2.0 AND mit) AND ((gpl-2.0 WITH font-exception-gpl OR ofl-1.1) AND apache-2.0) AND (apache-2.0 AND bsd-new AND bsd-simplified AND cc-by-3.0 AND cc0-1.0 AND gpl-3.0 AND isc AND lgpl-2.1 AND mit AND ofl-1.1 AND unknown-license-reference AND other-copyleft AND other-permissive AND unknown)',
-//   '(gpl-2.0 WITH font-exception-gpl OR ofl-1.1) AND apache-2.0',
-//   'apache OR apache-2.0',
-//   '(mit OR gpl-3.0) AND mit AND gpl-3.0',
-//   'apache-2.0 AND (mit AND json)'
-// ].map(key => {
-//   console.log(key, parseSubExpressions(key), "\n");
-// })
-
-export function parseTokensFromExpression(expression: string) {
-  if (!expression) expression = "";
-  const tokens = expression.split(/( |\(|\))/);
-  return tokens;
-}
-
-export function parseTokenKeysFromExpression(expression: string) {
-  if (!expression) expression = "";
-  const AVOID_KEYWORDS = new Set(["WITH", "OR", "AND", "(", ")"]);
-  const tokens = parseTokensFromExpression(expression);
-  return tokens.filter(
-    (token) =>
-      token.trim().length && token.length && !AVOID_KEYWORDS.has(token.trim())
-  );
-}
-export function filterSpdxKeys(keys: string[]) {
-  const ignoredPrefixes = ["License-scancode-", "LicenseRef-scancode-"];
-  return keys.filter((key) => {
-    for (const prefix of ignoredPrefixes) {
-      if (key.includes(prefix)) return false;
-    }
-    return true;
-  });
-}