Merge branch 'main' into store-all-downloaded-packages

Author: VarshaUN

CHANGELOG.rst

@@ -1,6 +1,21 @@
 Changelog
 =========
 
+v35.1.0 (unreleased)
+--------------------
+
+- Add a ``--fail-on-vulnerabilities`` option in ``check-compliance`` management command.
+  When this option is enabled, the command will exit with a non-zero status if known
+  vulnerabilities are detected in discovered packages and dependencies.
+  Requires the ``find_vulnerabilities`` pipeline to be executed beforehand.
+  https://github.com/aboutcode-org/scancode.io/pull/1702
+
+- Enable ``--license-references`` scan option in the ``scan_single_package`` pipeline.
+  The ``license_references`` and ``license_rule_references`` attributes will now be
+  available in the scan results, including the details about detected licenses and
+  license rules used during the scan.
+  https://github.com/aboutcode-org/scancode.io/issues/1657
+
 v35.0.0 (2025-06-23)
 --------------------
 
@@ -36,6 +51,11 @@ v35.0.0 (2025-06-23)
 - Add "Package Compliance Alert" chart in the Policies section.
   https://github.com/aboutcode-org/scancode.io/pull/1699
 
+- Update univers to v31.0.0, catch ``NotImplementedError`` in
+  ``get_unique_unresolved_purls``, and properly log error in project.
+  https://github.com/aboutcode-org/scancode.io/pull/1700
+  https://github.com/aboutcode-org/scancode.io/pull/1701
+
 v34.11.0 (2025-05-02)
 ---------------------
 

docs/command-line-interface.rst

@@ -497,6 +497,10 @@ Optional arguments:
 - ``--fail-level {ERROR,WARNING,MISSING}`` Compliance alert level that will cause the
   command to exit with a non-zero status. Default is ERROR.
 
+- ``--fail-on-vulnerabilities`` Exit with a non-zero status if known vulnerabilities
+  are detected in discovered packages and dependencies.
+  Requires the ``find_vulnerabilities`` pipeline to be executed beforehand.
+
 `$ scanpipe archive-project --project PROJECT`
 ----------------------------------------------
 

scancodeio/static/add-inputs.js

@@ -21,15 +21,41 @@
 // Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
 const fileInput = document.querySelector("#id_input_files");
+let selectedFiles = []; // Store selected files
 fileInput.onchange = updateFiles;
 
 // Update the list of files to be uploaded in the UI
 function updateFiles() {
   if (fileInput.files.length > 0) {
     const fileName = document.querySelector("#inputs_file_name");
     fileName.innerHTML = "";
-    for (let file of fileInput.files) {
-      fileName.innerHTML += `<span class="is-block">${file.name}</span>`;
+
+    // Update the selectedFiles array
+    const newFiles = Array.from(fileInput.files);
+    // Create a Set to track unique file names
+    const uniqueFileNames = new Set(selectedFiles.map(file => file.name));
+    // Filter out files with the same name
+    const filteredNewFiles = newFiles.filter(file => !uniqueFileNames.has(file.name));
+    // Concatenate the unique files to the existing selectedFiles array
+    selectedFiles = selectedFiles.concat(filteredNewFiles);
+
+    for (let file of selectedFiles) {
+      const fileNameWithoutSpaces = file.name.replace(/\s/g, '');
+      fileName.innerHTML += `
+      <span class="is-flex is-justify-content-space-between is-block" id="file-name-${fileNameWithoutSpaces}">
+        <span class="is-block">${file.name}</span>
+        <a href="#" onclick="removeFile('${fileNameWithoutSpaces}')" class="model-button" id="file-delete-btn-${fileNameWithoutSpaces}">
+          <i class="fa-solid fa-trash-can"></i>
+        </a>
+      </span>
+      `;
+      document.getElementById("file-delete-btn-"+ fileNameWithoutSpaces).addEventListener("click", function(event){
+        disableEvent(event);
+        removeFile(fileNameWithoutSpaces);
+        if(selectedFiles.length == 0){
+          fileName.innerHTML ="<i>No files selected</i>"
+        }
+      });
     }
   }
 }
@@ -40,15 +66,37 @@ function disableEvent(event) {
   event.preventDefault();
 }
 
+function removeFile(fileName) {
+  selectedFiles = selectedFiles.filter(file => {
+    const fileNameWithoutSpaces = file.name.replace(/\s/g, '');
+    return fileNameWithoutSpaces !== fileName;
+  });
+
+  const fileNameElement = document.getElementById(`file-name-${fileName}`);
+  if (fileNameElement) {
+    fileNameElement.remove();
+  }
+
+  const dataTransfer = new DataTransfer();
+  for (let file of selectedFiles) {
+    dataTransfer.items.add(file);
+  }
+
+  fileInput.files = dataTransfer.files;
+}
+
 function dropHandler(event) {
   disableEvent(event);
   const droppedFiles = event.dataTransfer.files;
-  const updatedFiles = Array.from(fileInput.files);
+  const updatedFilesSet = new Set(Array.from(fileInput.files));
 
   for (let file of droppedFiles) {
-    updatedFiles.push(file);
+    updatedFilesSet.add(file);
   }
 
+  // Convert the Set back to an array if needed
+  const updatedFiles = Array.from(updatedFilesSet);
+
   const dataTransfer = new DataTransfer();
   for (let file of updatedFiles) {
     dataTransfer.items.add(file);

scanpipe/filters.py

@@ -483,6 +483,7 @@ def filter(self, qs, value):
     ("about_file", "about file"),
     ("java_to_class", "java to class"),
     ("jar_to_source", "jar to source"),
+    ("javascript_strings", "js strings"),
     ("javascript_symbols", "js symbols"),
     ("js_compiled", "js compiled"),
     ("js_colocation", "js colocation"),

scanpipe/management/commands/check-compliance.py

@@ -45,31 +45,64 @@ def add_arguments(self, parser):
                 "non-zero status. Default is ERROR."
             ),
         )
+        parser.add_argument(
+            "--fail-on-vulnerabilities",
+            action="store_true",
+            help=(
+                "Exit with a non-zero status if known vulnerabilities are detected in "
+                "discovered packages and dependencies. "
+                "Requires the `find_vulnerabilities` pipeline to be executed "
+                "beforehand."
+            ),
+        )
 
     def handle(self, *args, **options):
         super().handle(*args, **options)
-        fail_level = options["fail_level"]
-        compliance_alerts = get_project_compliance_alerts(self.project, fail_level)
+        exit_code = 0
+
+        if self.check_compliance(options["fail_level"]):
+            exit_code = 1
+
+        if options["fail_on_vulnerabilities"] and self.check_vulnerabilities():
+            exit_code = 1
 
-        compliance_alerts_count = sum(
-            len(issues_by_severity)
-            for model_alerts in compliance_alerts.values()
-            for issues_by_severity in model_alerts.values()
+        sys.exit(exit_code)
+
+    def check_compliance(self, fail_level):
+        alerts = get_project_compliance_alerts(self.project, fail_level)
+        count = sum(
+            len(issues) for model in alerts.values() for issues in model.values()
         )
-        if not compliance_alerts_count:
-            sys.exit(0)
 
-        if self.verbosity > 0:
-            msg = [
-                f"{compliance_alerts_count} compliance issues detected on this project."
-            ]
-            for label, issues in compliance_alerts.items():
-                msg.append(f"[{label}]")
-                for severity, entries in issues.items():
-                    msg.append(f" > {severity.upper()}: {len(entries)}")
+        if count and self.verbosity > 0:
+            self.stderr.write(f"{count} compliance issues detected.")
+            for label, model in alerts.items():
+                self.stderr.write(f"[{label}]")
+                for severity, entries in model.items():
+                    self.stderr.write(f" > {severity.upper()}: {len(entries)}")
                     if self.verbosity > 1:
-                        msg.append("   " + "\n   ".join(entries))
+                        self.stderr.write("   " + "\n   ".join(entries))
+
+        return count > 0
 
-            self.stderr.write("\n".join(msg))
+    def check_vulnerabilities(self):
+        packages = self.project.discoveredpackages.vulnerable_ordered()
+        dependencies = self.project.discovereddependencies.vulnerable_ordered()
+
+        vulnerable_records = list(packages) + list(dependencies)
+        count = len(vulnerable_records)
+
+        if self.verbosity > 0:
+            if count:
+                self.stderr.write(f"{count} vulnerable records found:")
+                for entry in vulnerable_records:
+                    self.stderr.write(str(entry))
+                    vulnerability_ids = [
+                        vulnerability.get("vulnerability_id")
+                        for vulnerability in entry.affected_by_vulnerabilities
+                    ]
+                    self.stderr.write(" > " + ", ".join(vulnerability_ids))
+            else:
+                self.stdout.write("No vulnerabilities found")
 
-        sys.exit(1)
+        return count > 0

scanpipe/models.py

@@ -3155,6 +3155,13 @@ class VulnerabilityQuerySetMixin:
     def vulnerable(self):
         return self.filter(~Q(affected_by_vulnerabilities__in=EMPTY_VALUES))
 
+    def vulnerable_ordered(self):
+        return (
+            self.vulnerable()
+            .only_package_url_fields(extra=["affected_by_vulnerabilities"])
+            .order_by_package_url()
+        )
+
 
 class DiscoveredPackageQuerySet(
     VulnerabilityQuerySetMixin,

scanpipe/pipelines/__init__.py

@@ -78,7 +78,10 @@ def flag_ignored_resources(self):
             ignored_patterns = ignored_patterns.splitlines()
         ignored_patterns.extend(flag.DEFAULT_IGNORED_PATTERNS)
 
-        flag.flag_ignored_patterns(self.project, patterns=ignored_patterns)
+        flag.flag_ignored_patterns(
+            codebaseresources=self.project.codebaseresources.no_status(),
+            patterns=ignored_patterns,
+        )
 
     def extract_archive(self, location, target):
         """Extract archive at `location` to `target`. Save errors as messages."""
@@ -179,6 +182,8 @@ def __init__(self, run_instance):
         self.selected_groups = run_instance.selected_groups
         self.selected_steps = run_instance.selected_steps
 
+        self.ecosystem_config = None
+
     @classmethod
     def get_initial_steps(cls):
         """Add the ``download_inputs`` step as an initial step if enabled."""

scanpipe/pipelines/deploy_to_develop.py

@@ -21,14 +21,11 @@
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
 import logging
-<<<<<<< HEAD
-=======
-from pathlib import Path
->>>>>>> ca3a1ac0c0147a6f3f59999a67bf586eab9b8a36
 from aboutcode.pipeline import optional_step
 from scanpipe import pipes
 from scanpipe.pipelines import Pipeline
 from scanpipe.pipes import d2d
+from scanpipe.pipes import d2d_config
 from scanpipe.pipes import flag
 from scanpipe.pipes import input
 from scanpipe.pipes import matchcode
@@ -72,6 +69,8 @@ def steps(cls):
             cls.flag_empty_files,
             cls.flag_whitespace_files,
             cls.flag_ignored_resources,
+            cls.load_ecosystem_config,
+            cls.map_ruby,
             cls.map_about_files,
             cls.map_checksum,
             cls.match_archives_to_purldb,
@@ -80,6 +79,7 @@ def steps(cls):
             cls.map_jar_to_source,
             cls.map_javascript,
             cls.map_javascript_symbols,
+            cls.map_javascript_strings,
             cls.map_elf,
             cls.map_macho,
             cls.map_winpe,
@@ -102,6 +102,7 @@ def steps(cls):
             cls.create_local_files_packages,
         )
 
+
     purldb_package_extensions = [".jar", ".war", ".zip"]
     purldb_resource_extensions = [
         ".map",
@@ -197,6 +198,15 @@ def flag_whitespace_files(self):
         """Flag whitespace files with size less than or equal to 100 byte as ignored."""
         d2d.flag_whitespace_files(project=self.project)
 
+    def load_ecosystem_config(self):
+        """Load ecosystem specific configurations for d2d steps for selected options."""
+        d2d_config.load_ecosystem_config(pipeline=self, options=self.selected_groups)
+
+    @optional_step("Ruby")
+    def map_ruby(self):
+        """Load Ruby specific configurations for d2d steps."""
+        pass
+
     def map_about_files(self):
         """Map ``from/`` .ABOUT files to their related ``to/`` resources."""
         d2d.map_about_files(project=self.project, logger=self.log)
@@ -213,7 +223,7 @@ def match_archives_to_purldb(self):
 
         d2d.match_purldb_resources(
             project=self.project,
-            extensions=self.purldb_package_extensions,
+            extensions=self.matchable_package_extensions,
             matcher_func=d2d.match_purldb_package,
             logger=self.log,
         )
@@ -246,6 +256,11 @@ def map_javascript_symbols(self):
         """Map deployed JavaScript, TypeScript to its sources using symbols."""
         d2d.map_javascript_symbols(project=self.project, logger=self.log)
 
+    @optional_step("JavaScript")
+    def map_javascript_strings(self):
+        """Map deployed JavaScript, TypeScript to its sources using string literals."""
+        d2d.map_javascript_strings(project=self.project, logger=self.log)
+
     @optional_step("Elf")
     def map_elf(self):
         """Map ELF binaries to their sources using dwarf paths and symbols."""
@@ -291,7 +306,7 @@ def match_resources_to_purldb(self):
 
         d2d.match_purldb_resources(
             project=self.project,
-            extensions=self.purldb_resource_extensions,
+            extensions=self.matchable_resource_extensions,
             matcher_func=d2d.match_purldb_resource,
             logger=self.log,
         )
@@ -329,6 +344,7 @@ def flag_mapped_resources_archives_and_ignored_directories(self):
     def perform_house_keeping_tasks(self):
         """
         On deployed side
+            - Ignore specific files based on ecosystem based configurations.
             - PurlDB match files with ``no-java-source`` and empty status,
                 if no match is found update status to ``requires-review``.
             - Update status for uninteresting files.
@@ -339,9 +355,14 @@ def perform_house_keeping_tasks(self):
         """
         d2d.match_resources_with_no_java_source(project=self.project, logger=self.log)
         d2d.handle_dangling_deployed_legal_files(project=self.project, logger=self.log)
+        d2d.ignore_unmapped_resources_from_config(
+            project=self.project,
+            patterns_to_ignore=self.ecosystem_config.deployed_resource_path_exclusions,
+            logger=self.log,
+        )
         d2d.match_unmapped_resources(
             project=self.project,
-            matched_extensions=self.purldb_resource_extensions,
+            matched_extensions=self.ecosystem_config.matchable_resource_extensions,
             logger=self.log,
         )
         d2d.flag_undeployed_resources(project=self.project)
@@ -377,5 +398,5 @@ def flag_deployed_from_resources_with_missing_license(self):
         """Update the status for deployed from files with missing license."""
         d2d.flag_deployed_from_resources_with_missing_license(
             self.project,
-            doc_extensions=self.doc_extensions,
+            doc_extensions=self.ecosystem_config.doc_extensions,
         )

scanpipe/pipelines/scan_single_package.py

@@ -66,6 +66,7 @@ def steps(cls):
         "info": True,
         "license": True,
         "license_text": True,
+        "license_references": True,
         "package": True,
         "url": True,
         "classify": True,