Move MatchCode-related code (#1077)

* Rename match_to_purld to match_to_matchcode

    * Move matchcode related code from pipes/purldb.py to pipes/matchcode.py
    * Update scancode.io settings to get matchcode API settings
    * This is done since matchcode and purldb are technically two seperate services with their own users, etc.

Signed-off-by: Jono Yang <[email protected]>

* Update docstring in MatchToMatchCode pipeline

Signed-off-by: Jono Yang <[email protected]>

* Update test_matchcode.py

Signed-off-by: Jono Yang <[email protected]>

* Update test file directory name

Signed-off-by: Jono Yang <[email protected]>

* Create migration for match_to_matchcode pipeline renaming

Signed-off-by: Jono Yang <[email protected]>

* Add MatchCode.io integration docs

    * Update references to match_to_purldb to match_to_matchcode in docs

Signed-off-by: Jono Yang <[email protected]>

* Fix the documentation regarding MatchCode #1077

Signed-off-by: tdruez <[email protected]>

---------

Signed-off-by: Jono Yang <[email protected]>
Signed-off-by: tdruez <[email protected]>
Co-authored-by: tdruez <[email protected]>

Author: JonoYang

CHANGELOG.rst

@@ -23,6 +23,9 @@ v34.1.0 (unreleased)
   https://github.com/nexB/scancode.io/issues/1121
   https://github.com/nexB/scancode.io/issues/1122
 
+- Rename the ``match_to_purldb`` pipeline to ``match_to_matchcode``, and add
+  MatchCode.io API settings to ScanCode.io settings.
+
 v34.0.0 (2024-03-04)
 --------------------
 

docs/application-settings.rst

@@ -314,6 +314,26 @@ you can provide the API key using ``VULNERABLECODE_API_KEY``::
 
     VULNERABLECODE_API_KEY=insert_your_api_key_here
 
+.. _scancodeio_settings_matchcodeio:
+
+MATCHCODE.IO
+^^^^^^^^^^^^
+
+There is currently no public instance of MatchCode.io.
+
+Alternatively, you can deploy your own instance of MatchCode.io by
+following the instructions provided in the documentation at
+https://purldb.readthedocs.io/.
+
+To configure your local environment, set the ``MATCHCODEIO_URL`` in your ``.env`` file::
+
+    MATCHCODEIO_URL=https://<Address to MatchCode.io host>/
+
+If authentication is enabled on your MatchCode.io instance, you can provide the
+API key using ``MATCHCODEIO_API_KEY``::
+
+    MATCHCODEIO_API_KEY=insert_your_api_key_here
+
 .. _scancodeio_settings_fetch_authentication:
 
 Fetch Authentication

docs/built-in-pipelines.rst

@@ -122,17 +122,17 @@ Map Deploy To Develop
     :members:
     :member-order: bysource
 
-.. _pipeline_match_to_purldb:
+.. _pipeline_match_to_matchcode:
 
-Match to PurlDB (addon)
------------------------
+Match to MatchCode (addon)
+--------------------------
 
 .. warning::
-    This pipeline requires access to a PurlDB service.
-    Refer to :ref:`scancodeio_settings_purldb` to configure access to PurlDB in your
-    ScanCode.io instance.
+    This pipeline requires access to a MatchCode.io service.
+    Refer to :ref:`scancodeio_settings_matchcodeio` to configure access to
+    MatchCode.io in your ScanCode.io instance.
 
-.. autoclass:: scanpipe.pipelines.match_to_purldb.MatchToPurlDB()
+.. autoclass:: scanpipe.pipelines.match_to_matchcode.MatchToMatchCode()
     :members:
     :member-order: bysource
 

docs/faq.rst

@@ -70,10 +70,10 @@ existing data, allowing for more comprehensive analysis and insights.
   Before executing this pipeline, make sure to set up
   :ref:`PurlDB <scancodeio_settings_purldb>`.
 
-- To **match your project codebase resources to PurlDB for Package matches**,
-  utilize the :ref:`match_to_purldb <pipeline_match_to_purldb>` pipeline.
-  It's essential to set up :ref:`PurlDB <scancodeio_settings_purldb>` before executing
-  this pipeline.
+- To **match your project codebase resources to MatchCode.io for Package matches**,
+  utilize the :ref:`match_to_matchcode <pipeline_match_to_matchcode>` pipeline.
+  It's essential to set up :ref:`MatchCode.io <scancodeio_settings_matchcodeio>` before
+  executing this pipeline.
 
 What is the difference between scan_codebase and scan_single_package pipelines?
 -------------------------------------------------------------------------------

scancodeio/settings.py

@@ -403,3 +403,10 @@
 PURLDB_USER = env.str("PURLDB_USER", default="")
 PURLDB_PASSWORD = env.str("PURLDB_PASSWORD", default="")
 PURLDB_API_KEY = env.str("PURLDB_API_KEY", default="")
+
+# MatchCode.io integration
+
+MATCHCODEIO_URL = env.str("MATCHCODEIO_URL", default="")
+MATCHCODEIO_USER = env.str("MATCHCODEIO_USER", default="")
+MATCHCODEIO_PASSWORD = env.str("MATCHCODEIO_PASSWORD", default="")
+MATCHCODEIO_API_KEY = env.str("MATCHCODEIO_API_KEY", default="")

scanpipe/migrations/0054_rename_pipeline.py

@@ -0,0 +1,33 @@
+# Generated by Django 5.0.3 on 2024-03-20 22:52
+
+from django.db import migrations
+
+
+pipeline_old_names_mapping = {
+    "match_to_purldb": "match_to_matchcode",
+}
+
+
+def rename_pipelines_data(apps, schema_editor):
+    Run = apps.get_model("scanpipe", "Run")
+    for old_name, new_name in pipeline_old_names_mapping.items():
+        Run.objects.filter(pipeline_name=old_name).update(pipeline_name=new_name)
+
+
+def reverse_rename_pipelines_data(apps, schema_editor):
+    Run = apps.get_model("scanpipe", "Run")
+    for old_name, new_name in pipeline_old_names_mapping.items():
+        Run.objects.filter(pipeline_name=new_name).update(pipeline_name=old_name)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("scanpipe", "0053_restructure_pipelines_data"),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            rename_pipelines_data,
+            reverse_code=reverse_rename_pipelines_data,
+        ),
+    ]

scanpipe/pipelines/match_to_purldb.py renamed to scanpipe/pipelines/match_to_matchcode.py

@@ -21,19 +21,26 @@
 # Visit https://github.com/nexB/scancode.io for support and download.
 
 from scanpipe.pipelines import Pipeline
-from scanpipe.pipes import purldb
+from scanpipe.pipes import matchcode
 
 
-class MatchToPurlDB(Pipeline):
+class MatchToMatchCode(Pipeline):
     """
-    Match the codebase resources of a project against PurlDB to identify packages.
+    Match the codebase resources of a project against MatchCode.io to identify packages.
 
     This process involves:
 
-    1. generating a JSON scan of the project codebase
-    2. transmitting it to MatchCode on PurlDB and awaiting match results
-    3. creating discovered packages from the package data obtained
-    4. associating the codebase resources with those discovered packages
+    1. Generating a JSON scan of the project codebase
+    2. Transmitting it to MatchCode.io and awaiting match results
+    3. Creating discovered packages from the package data obtained
+    4. Associating the codebase resources with those discovered packages
+
+    Currently, MatchCode.io can only match for archives, directories, and files
+    from Maven and npm Packages.
+
+    This pipeline requires a MatchCode.io instance to be configured and available.
+    There is currently no public instance of MatchCode.io. Reach out to nexB, Inc.
+    for other arrangements.
     """
 
     download_inputs = False
@@ -42,29 +49,34 @@ class MatchToPurlDB(Pipeline):
     @classmethod
     def steps(cls):
         return (
-            cls.check_purldb_service_availability,
+            cls.check_matchcode_service_availability,
             cls.send_project_json_to_matchcode,
             cls.poll_matching_results,
             cls.create_packages_from_match_results,
         )
 
-    def check_purldb_service_availability(self):
-        """Check if the PurlDB service if configured and available."""
-        if not purldb.is_configured():
-            raise Exception("PurlDB is not configured.")
+    def check_matchcode_service_availability(self):
+        """Check if the MatchCode.io service if configured and available."""
+        if not matchcode.is_configured():
+            msg = (
+                "MatchCode.io is not configured. Set the MatchCode.io "
+                "related settings to a MatchCode.io instance or reach out "
+                "to the maintainers for other arrangements."
+            )
+            raise Exception(msg)
 
-        if not purldb.is_available():
-            raise Exception("PurlDB is not available.")
+        if not matchcode.is_available():
+            raise Exception("MatchCode.io is not available.")
 
     def send_project_json_to_matchcode(self):
-        """Create a JSON scan of the project Codebase and send it to MatchCode."""
-        self.run_url = purldb.send_project_json_to_matchcode(self.project)
+        """Create a JSON scan of the project Codebase and send it to MatchCode.io."""
+        self.run_url = matchcode.send_project_json_to_matchcode(self.project)
 
     def poll_matching_results(self):
         """Wait until the match results are ready by polling the match run status."""
-        purldb.poll_until_success(self.run_url)
+        matchcode.poll_until_success(self.run_url)
 
     def create_packages_from_match_results(self):
         """Create DiscoveredPackages from match results."""
-        match_results = purldb.get_match_results(self.run_url)
-        purldb.create_packages_from_match_results(self.project, match_results)
+        match_results = matchcode.get_match_results(self.run_url)
+        matchcode.create_packages_from_match_results(self.project, match_results)