Capture errors during the inspect_elf_binaries pipeline execution (#1123)

Signed-off-by: tdruez <[email protected]>

Author: tdruez

CHANGELOG.rst

@@ -15,6 +15,14 @@ v34.1.0 (unreleased)
   Universal Ctags.
   https://github.com/nexB/scancode.io/pull/1116
 
+- Capture errors during the `inspect_elf_binaries` pipeline execution.
+  Errors on resource inspection are stored as project error message instead of global
+  pipeline failure.
+  The problematic resource path is stored in the message details and displayed in the
+  message list UI as a link to the resource details view.
+  https://github.com/nexB/scancode.io/issues/1121
+  https://github.com/nexB/scancode.io/issues/1122
+
 v34.0.0 (2024-03-04)
 --------------------
 

docs/built-in-pipelines.rst

@@ -45,7 +45,7 @@ Analyse Docker Windows Image
 .. _pipeline_collect_symbols:
 
 Collect Codebase Symbols (addon)
----------------------------------
+--------------------------------
 .. autoclass:: scanpipe.pipelines.collect_symbols.CollectSymbols()
     :members:
     :member-order: bysource

docs/scanpipe-pipes.rst

@@ -33,28 +33,53 @@ Docker
 .. automodule:: scanpipe.pipes.docker
     :members:
 
+ELF
+---
+.. automodule:: scanpipe.pipes.elf
+    :members:
+
 Fetch
 -----
 .. automodule:: scanpipe.pipes.fetch
     :members:
     :exclude-members: Download
 
+Flag
+----
+.. automodule:: scanpipe.pipes.flag
+    :members:
+
 Input
 -----
 .. automodule:: scanpipe.pipes.input
     :members:
 
+JS
+--
+.. automodule:: scanpipe.pipes.js
+    :members:
+
 JVM
 ---
 .. automodule:: scanpipe.pipes.jvm
     :members:
 
+MatchCode
+---------
+.. automodule:: scanpipe.pipes.matchcode
+    :members:
+
 Output
 ------
 .. automodule:: scanpipe.pipes.output
     :members:
     :exclude-members: JSONResultsGenerator
 
+PathMap
+-------
+.. automodule:: scanpipe.pipes.pathmap
+    :members:
+
 PurlDB
 ------
 .. automodule:: scanpipe.pipes.purldb
@@ -80,9 +105,9 @@ SPDX
 .. automodule:: scanpipe.pipes.spdx
     :members:
 
-Flag
-----
-.. automodule:: scanpipe.pipes.flag
+Symbols
+-------
+.. automodule:: scanpipe.pipes.symbols
     :members:
 
 VulnerableCode

scanpipe/models.py

@@ -1097,12 +1097,20 @@ def get_next_run(self):
             return self.runs.not_started().earliest("created_date")
 
     def add_message(
-        self, severity, description="", model="", details=None, exception=None
+        self,
+        severity,
+        description="",
+        model="",
+        details=None,
+        exception=None,
+        resource=None,
     ):
         """
         Create a ProjectMessage record for this Project.
 
         The ``model`` attribute can be provided as a string or as a Model class.
+        A ``resource`` can be provided to keep track of the codebase resource that was
+        analyzed when the error occurred.
         """
         if inspect.isclass(model):
             model = model.__name__
@@ -1114,29 +1122,61 @@ def add_message(
         if exception and not description:
             description = str(exception)
 
+        details = details or {}
+        if resource:
+            # Do not change this field name as it has special behavior in templates.
+            details["resource_path"] = resource.path
+
         return ProjectMessage.objects.create(
             project=self,
             severity=severity,
             description=description,
             model=model,
-            details=details or {},
+            details=details,
             traceback=traceback,
         )
 
-    def add_info(self, description="", model="", details=None, exception=None):
+    def add_info(
+        self,
+        description="",
+        model="",
+        details=None,
+        exception=None,
+        resource=None,
+    ):
         """Create an INFO ProjectMessage record for this project."""
         severity = ProjectMessage.Severity.INFO
-        return self.add_message(severity, description, model, details, exception)
+        return self.add_message(
+            severity, description, model, details, exception, resource
+        )
 
-    def add_warning(self, description="", model="", details=None, exception=None):
+    def add_warning(
+        self,
+        description="",
+        model="",
+        details=None,
+        exception=None,
+        resource=None,
+    ):
         """Create a WARNING ProjectMessage record for this project."""
         severity = ProjectMessage.Severity.WARNING
-        return self.add_message(severity, description, model, details, exception)
+        return self.add_message(
+            severity, description, model, details, exception, resource
+        )
 
-    def add_error(self, description="", model="", details=None, exception=None):
+    def add_error(
+        self,
+        description="",
+        model="",
+        details=None,
+        exception=None,
+        resource=None,
+    ):
         """Create an ERROR ProjectMessage record using for this project."""
         severity = ProjectMessage.Severity.ERROR
-        return self.add_message(severity, description, model, details, exception)
+        return self.add_message(
+            severity, description, model, details, exception, resource
+        )
 
     def get_absolute_url(self):
         """Return this project's details URL."""
@@ -1409,10 +1449,15 @@ def add_error(self, exception):
         Create a ProjectMessage record using the provided ``exception`` Exception
         instance.
         """
+        resource = None
+        if isinstance(self, CodebaseResource):
+            resource = self
+
         return self.project.add_error(
             model=self.__class__,
             details=model_to_dict(self),
             exception=exception,
+            resource=resource,
         )
 
     def add_errors(self, exceptions):
@@ -2538,12 +2583,9 @@ def create_and_add_package(self, package_data):
         except Exception as exception:
             self.project.add_warning(
                 model=DiscoveredPackage,
-                details={
-                    "codebase_resource_path": self.path,
-                    "codebase_resource_pk": self.pk,
-                    **package_data,
-                },
+                details=package_data,
                 exception=exception,
+                resource=self,
             )
         else:
             self.add_package(package)

scanpipe/pipelines/__init__.py

@@ -230,25 +230,35 @@ def download_missing_inputs(self):
         if errors:
             raise InputFileError(errors)
 
-    def add_error(self, exception):
+    def add_error(self, exception, resource=None):
         """Create a ``ProjectMessage`` ERROR record on the current `project`."""
-        self.project.add_error(model=self.pipeline_name, exception=exception)
+        self.project.add_error(
+            model=self.pipeline_name,
+            exception=exception,
+            resource=resource,
+        )
 
     @contextmanager
-    def save_errors(self, *exceptions):
+    def save_errors(self, *exceptions, **kwargs):
         """
         Context manager to save specified exceptions as ``ProjectMessage`` in the
         database.
 
-        Example in a Pipeline step:
+        - Example in a Pipeline step::
 
         with self.save_errors(rootfs.DistroNotFound):
             rootfs.scan_rootfs_for_system_packages(self.project, rfs)
+
+        - Example when iterating over resources::
+
+        for resource in self.project.codebaseresources.all():
+            with self.save_errors(Exception, resource=resource):
+                analyse(resource)
         """
         try:
             yield
         except exceptions as error:
-            self.add_error(exception=error)
+            self.add_error(exception=error, **kwargs)
 
 
 class Pipeline(BasePipeline):

scanpipe/pipelines/inspect_elf_binaries.py

@@ -20,11 +20,8 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/nexB/scancode.io for support and download.
 
-from pathlib import Path
-
-from elf_inspector.dwarf import get_dwarf_paths
-
 from scanpipe.pipelines import Pipeline
+from scanpipe.pipes.elf import collect_dwarf_source_path_references
 
 
 class InspectELFBinaries(Pipeline):
@@ -38,10 +35,7 @@ def steps(cls):
         return (cls.collect_dwarf_source_path_references,)
 
     def collect_dwarf_source_path_references(self):
-        """
-        Update ``extra_data`` of ELF files with
-        dwarf data extracted from ELF files.
-        """
-        for elf in self.project.codebaseresources.elfs():
-            dwarf_paths = get_dwarf_paths(Path(self.project.codebase_path / elf.path))
-            elf.update_extra_data(dwarf_paths)
+        """Collect DWARF paths from ELF files and set values on the extra_data field."""
+        for elf_resource in self.project.codebaseresources.elfs():
+            with self.save_errors(Exception, resource=elf_resource):
+                collect_dwarf_source_path_references(elf_resource)

scanpipe/pipes/d2d.py

@@ -826,15 +826,13 @@ def create_indexes(cls, project, from_about_files, logger=None):
             package_data = resolve.resolve_about_package(
                 input_location=str(about_file_resource.location_path)
             )
-            error_message_details = {
-                "path": about_file_resource.path,
-                "package_data": package_data,
-            }
+            error_message_details = {"package_data": package_data}
             if not package_data:
                 project.add_error(
                     description="Cannot create package from ABOUT file",
                     model="map_about_files",
                     details=error_message_details,
+                    resource=about_file_resource,
                 )
                 continue
 
@@ -846,6 +844,7 @@ def create_indexes(cls, project, from_about_files, logger=None):
                     description="ABOUT file does not have about_resource",
                     model="map_about_files",
                     details=error_message_details,
+                    resource=about_file_resource,
                 )
                 continue
             else:

scanpipe/pipes/elf.py

@@ -0,0 +1,30 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# http://nexb.com and https://github.com/nexB/scancode.io
+# The ScanCode.io software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode.io is provided as-is without warranties.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+# ScanCode.io should be considered or used as legal advice. Consult an Attorney
+# for any legal advice.
+#
+# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
+# Visit https://github.com/nexB/scancode.io for support and download.
+
+from elf_inspector.dwarf import get_dwarf_paths
+
+
+def collect_dwarf_source_path_references(resource):
+    """Collect and store the DWARF debug paths of the provided ELF ``resource``."""
+    dwarf_paths = get_dwarf_paths(resource.location_path)
+    resource.update_extra_data(dwarf_paths)
+    return dwarf_paths

scanpipe/pipes/resolve.py

@@ -57,16 +57,13 @@ def get_packages(project, package_registry, manifest_resources, model=None):
         return
 
     for resource in manifest_resources:
-        if packages := get_packages_from_manifest(
-            input_location=resource.location,
-            package_registry=package_registry,
-        ):
+        if packages := get_packages_from_manifest(resource.location, package_registry):
             resolved_packages.extend(packages)
         else:
             project.add_error(
                 description="No packages could be resolved for",
                 model=model,
-                details={"path": resource.path},
+                resource=resource,
             )
 
     return resolved_packages

scanpipe/pipes/scancode.py

@@ -391,11 +391,10 @@ def add_resource_to_package(package_uid, resource, project):
     try:
         package = project.discoveredpackages.get(package_uid=package_uid)
     except ObjectDoesNotExist as error:
-        details = {
-            "package_uid": str(package_uid),
-            "resource": str(resource),
-        }
-        project.add_error(error, model="assemble_package", details=details)
+        details = {"package_uid": str(package_uid)}
+        project.add_error(
+            error, model="assemble_package", details=details, resource=resource
+        )
         return
 
     resource.discovered_packages.add(package)