Merge branch 'main' into 1145-load-spdx-dependencies

Author: tdruez

.github/workflows/publish-docker.yml

@@ -59,5 +59,5 @@ jobs:
           push: true
           tags: |
             ${{ steps.meta.outputs.tags }}
-            ${{ env.REGISTRY }}/nexb/scancode.io:latest
+            ${{ env.REGISTRY }}/aboutcode-org/scancode.io:latest
           labels: ${{ steps.meta.outputs.labels }}

CHANGELOG.rst

@@ -1,7 +1,7 @@
 Changelog
 =========
 
-v34.7.2 (unreleased)
+v34.8.0 (2024-08-15)
 --------------------
 
 - Add a new ``enrich_with_purldb`` add-on pipeline to enrich the discovered packages
@@ -51,6 +51,9 @@ v34.7.2 (unreleased)
   a project.
   https://github.com/nexB/scancode.io/issues/1182
 
+- Fix issues in ``match_to_matchcode`` where the incorrect polling function was
+  used and match results were not properly collected.
+
 v34.7.1 (2024-07-15)
 --------------------
 

docs/faq.rst

@@ -23,7 +23,7 @@ you may use the Docker image to run pipelines as a single command:
 
   docker run --rm \
     -v "$(pwd)":/codedrop \
-    ghcr.io/nexb/scancode.io:latest \
+    ghcr.io/aboutcode-org/scancode.io:latest \
     sh -c "run scan_codebase /codedrop" \
     > results.json
 

docs/tutorial_cli_end_to_end_scanning_to_dejacode.rst

@@ -153,7 +153,7 @@ on the codebase in our current directory.
 
     docker run --rm \
       -v "$(pwd)":/code \
-      ghcr.io/nexb/scancode.io:latest \
+      ghcr.io/aboutcode-org/scancode.io:latest \
       sh -c "run inspect_packages:StaticResolver /code" \
       > results.json
 

scancodeio/__init__.py

@@ -28,7 +28,7 @@
 
 import git
 
-VERSION = "34.7.1"
+VERSION = "34.8.0"
 
 PROJECT_DIR = Path(__file__).resolve().parent
 ROOT_DIR = PROJECT_DIR.parent

scanpipe/pipelines/match_to_matchcode.py

@@ -70,13 +70,15 @@ def check_matchcode_service_availability(self):
 
     def send_project_json_to_matchcode(self):
         """Create a JSON scan of the project Codebase and send it to MatchCode.io."""
-        self.run_url = matchcode.send_project_json_to_matchcode(self.project)
+        self.match_url, self.run_url = matchcode.send_project_json_to_matchcode(
+            self.project
+        )
 
     def poll_matching_results(self):
         """Wait until the match results are ready by polling the match run status."""
-        matchcode.poll_until_success(self.run_url)
+        matchcode.poll_run_url_status(self.run_url)
 
     def create_packages_from_match_results(self):
         """Create DiscoveredPackages from match results."""
-        match_results = matchcode.get_match_results(self.run_url)
+        match_results = matchcode.get_match_results(self.match_url)
         matchcode.create_packages_from_match_results(self.project, match_results)

scanpipe/pipes/__init__.py

@@ -441,27 +441,18 @@ def poll_until_success(check, sleep=10, **kwargs):
     function.
     """
     run_status = AbstractTaskFieldsModel.Status
-    # Continue looping if the run instance has the following statuses
-    CONTINUE_STATUSES = [
-        run_status.NOT_STARTED,
-        run_status.QUEUED,
-        run_status.RUNNING,
-    ]
     # Return False if the run instance has the following statuses
     FAIL_STATUSES = [
         run_status.FAILURE,
         run_status.STOPPED,
         run_status.STALE,
     ]
-
     while True:
         status = check(**kwargs)
+
         if status == run_status.SUCCESS:
             return True
 
-        if status in CONTINUE_STATUSES:
-            continue
-
         if status in FAIL_STATUSES:
             return False
 

scanpipe/pipes/matchcode.py

@@ -266,8 +266,9 @@ def send_project_json_to_matchcode(
             timeout=timeout,
             files=files,
         )
+    match_url = response["url"]
     run_url = response["runs"][0]["url"]
-    return run_url
+    return match_url, run_url
 
 
 def get_run_url_status(run_url, **kwargs):
@@ -299,19 +300,26 @@ def poll_run_url_status(run_url, sleep=10):
         raise MatchCodeIOException(msg)
 
 
-def get_match_results(run_url):
+def create_match_results_url(match_url):
     """
-    Given the `run_url` for a pipeline running the matchcode matching pipeline,
-    return the match results for that run.
+    Given the `match_url` for a project running the matchcode matching pipeline,
+    return the match results URL from `match_url`.
     """
-    response = request_get(run_url)
-    project_url = response["project"]
     # `project_url` can have params, such as "?format=json"
-    if "?" in project_url:
-        project_url, _ = project_url.split("?")
-    project_url = project_url.rstrip("/")
-    results_url = project_url + "/results/"
-    return request_get(results_url)
+    if "?" in match_url:
+        match_url, _ = match_url.split("?")
+    match_url = match_url.rstrip("/")
+    match_url = match_url + "/results/"
+    return match_url
+
+
+def get_match_results(match_url):
+    """
+    Given the `match_url` for a project running the matchcode matching pipeline,
+    return the match results.
+    """
+    match_results_url = create_match_results_url(match_url)
+    return request_get(match_results_url)
 
 
 def map_match_results(match_results):

scanpipe/tests/pipes/test_matchcode.py

@@ -76,10 +76,14 @@ def mock_request_post_return(url, files, timeout):
 
         mock_request_post.side_effect = mock_request_post_return
 
-        run_url = matchcode.send_project_json_to_matchcode(self.project1)
+        match_url, run_url = matchcode.send_project_json_to_matchcode(self.project1)
+        expected_match_url = (
+            "http://192.168.1.12/api/matching/65bf1e6d-6bff-4841-9c9b-db5cf25edfa7/"
+        )
         expected_run_url = (
             "http://192.168.1.12/api/runs/52b2930d-6e85-4b3e-ba3e-17dd9a618650/"
         )
+        self.assertEqual(expected_match_url, match_url)
         self.assertEqual(expected_run_url, run_url)
 
     @mock.patch("scanpipe.pipes.matchcode.request_get")
@@ -280,22 +284,21 @@ def test_scanpipe_pipes_matchcode_create_packages_from_match_results(self):
         # This resource should not have a Package match
         self.assertFalse(0, len(r2.for_packages))
 
+    def test_scanpipe_pipes_matchcode_create_match_results_url(self):
+        match_url = (
+            "http://192.168.1.12/api/matching/65bf1e6d-6bff-4841-9c9b-db5cf25edfa7/"
+        )
+        expected_match_url = "http://192.168.1.12/api/matching/65bf1e6d-6bff-4841-9c9b-db5cf25edfa7/results/"
+        self.assertEqual(
+            expected_match_url, matchcode.create_match_results_url(match_url)
+        )
+
     @mock.patch("scanpipe.pipes.matchcode.request_get")
     @mock.patch("scanpipe.pipes.matchcode.is_available")
     def test_scanpipe_pipes_matchcode_get_match_results(
         self, mock_is_available, mock_request_get
     ):
         mock_is_available.return_value = True
-
-        request_get_check_response_loc = (
-            self.data
-            / "matchcode"
-            / "match_to_matchcode"
-            / "request_get_check_response.json"
-        )
-        with open(request_get_check_response_loc) as f:
-            mock_request_get_check_return = json.load(f)
-
         request_get_results_response_loc = (
             self.data
             / "matchcode"
@@ -305,13 +308,13 @@ def test_scanpipe_pipes_matchcode_get_match_results(
         with open(request_get_results_response_loc) as f:
             mock_request_get_results_return = json.load(f)
         mock_request_get.side_effect = [
-            mock_request_get_check_return,
             mock_request_get_results_return,
         ]
 
-        run_url = "http://192.168.1.12/api/runs/52b2930d-6e85-4b3e-ba3e-17dd9a618650/"
-        match_results = matchcode.get_match_results(run_url)
-
+        match_url = (
+            "http://192.168.1.12/api/matching/65bf1e6d-6bff-4841-9c9b-db5cf25edfa7/"
+        )
+        match_results = matchcode.get_match_results(match_url)
         self.assertEqual(mock_request_get_results_return, match_results)
 
     def test_scanpipe_pipes_matchcode_fingerprint_codebase_resources(self):

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = scancodeio
-version = 34.7.1
+version = 34.8.0
 license = Apache-2.0
 description = Automate software composition analysis pipelines
 long_description = file:README.rst
@@ -150,7 +150,7 @@ scancodeio_pipelines =
 
 [bumpver]
 version_pattern = "MAJOR.MINOR.PATCH"
-current_version = "34.7.1"
+current_version = "34.8.0"
 
 [bumpver:file_patterns]
 setup.cfg =