Use matchcode-toolkit branch

    * there is a bug in the code stemming function

Signed-off-by: Jono Yang <[email protected]>

Author: JonoYang

scanpipe/pipes/matchcode.py

@@ -28,6 +28,7 @@
 import requests
 from matchcode_toolkit.fingerprinting import compute_codebase_directory_fingerprints
 from matchcode_toolkit.fingerprinting import get_file_fingerprint_hashes
+from matchcode_toolkit.fingerprinting import get_line_by_pos
 from matchcode_toolkit.fingerprinting import get_stemmed_file_fingerprint_hashes
 from scancode import Scanner
 
@@ -405,3 +406,14 @@ def create_packages_from_match_results(project, match_results):
             package_data=matched_package,
             status=flag.MATCHED_TO_PURLDB_PACKAGE,
         )
+    match_resources = match_results.get("files", [])
+    for match_resource in match_resources:
+        match_resource_extra_data = match_resource["extra_data"]
+        if match_resource_extra_data:
+            resource = project.codebaseresources.get(path=match_resource["path"])
+            # compute line_by_pos for displaying matches in CodebaseResource detail view
+            with open(resource.location) as f:
+                content = f.read()
+                line_by_pos = get_line_by_pos(content)
+            match_resource_extra_data["line_by_pos"] = line_by_pos
+            resource.update_extra_data(match_resource_extra_data)

scanpipe/views.py

@@ -61,6 +61,7 @@
 import saneyaml
 import xlsxwriter
 from django_filters.views import FilterView
+from licensedcode.spans import Span
 from packageurl.contrib.django.models import PACKAGE_URL_FIELDS
 
 from scancodeio.auth import ConditionalLoginRequired
@@ -1948,6 +1949,26 @@ def get_license_annotations(self, field_name):
 
         return annotations
 
+    def get_matched_snippet_annotations(self, resource):
+        # convert qspan from list of ints to Spans
+        matched_snippet_annotations = []
+        matched_snippets = resource.extra_data.get("matched_snippets")
+        if matched_snippets:
+            line_by_pos = resource.extra_data.get("line_by_pos")
+            for matched_snippet in matched_snippets:
+                match_detections = matched_snippet["match_detections"]
+                qspan = Span(match_detections)
+                for span in qspan.subspans():
+                    # line_by_pos is stored as JSON and keys in JSON are always
+                    # strings
+                    matched_snippet_annotations.append(
+                        {
+                            "start_line": line_by_pos[str(span.start)],
+                            "end_line": line_by_pos[str(span.end)],
+                        }
+                    )
+        return matched_snippet_annotations
+
     def get_context_data(self, **kwargs):
         context = super().get_context_data(**kwargs)
         resource = self.object
@@ -1964,6 +1985,9 @@ def get_context_data(self, **kwargs):
             "licenses": license_annotations,
         }
 
+        matched_snippet_annotations = self.get_matched_snippet_annotations(resource)
+        context["detected_values"]["matched_snippets"] = matched_snippet_annotations
+
         fields = [
             ("copyrights", "copyright"),
             ("holders", "holder"),

setup.cfg

@@ -97,7 +97,7 @@ install_requires =
     cyclonedx-python-lib==9.1.0
     jsonschema==4.23.0
     # MatchCode-toolkit
-    matchcode-toolkit==7.2.1
+    matchcode-toolkit @ git+https://github.com/aboutcode-org/matchcode-toolkit.git@bf9fd9afd6ce1dd25dce4935135ea4118fc80b06
     # Univers
     univers==30.12.1
     # Markdown