Refactor policies implementation to support more than licenses

Signed-off-by: tdruez <[email protected]>

Author: tdruez

scancodeio/settings.py

@@ -292,6 +292,7 @@
     SCANCODEIO_WORKSPACE_LOCATION = tempfile.mkdtemp()
     SCANCODEIO_REQUIRE_AUTHENTICATION = True
     SCANCODEIO_SCAN_FILE_TIMEOUT = 120
+    SCANCODEIO_POLICIES_FILE = None
     # The default password hasher is rather slow by design.
     # Using a faster hashing algorithm in the testing context to speed up the run.
     PASSWORD_HASHERS = ["django.contrib.auth.hashers.MD5PasswordHasher"]

scanpipe/apps.py

@@ -39,7 +39,6 @@
 from licensedcode.models import load_licenses
 
 from scanpipe.policies import load_policies_file
-from scanpipe.policies import make_license_policy_index
 
 try:
     from importlib import metadata as importlib_metadata
@@ -61,7 +60,7 @@ def __init__(self, app_name, app_module):
 
         # Mapping of registered pipeline names to pipeline classes.
         self._pipelines = {}
-        self.license_policies_index = {}
+        self.policies = {}
 
         workspace_location = settings.SCANCODEIO_WORKSPACE_LOCATION
         self.workspace_path = Path(workspace_location).expanduser().resolve()
@@ -226,7 +225,7 @@ def get_scancode_licenses(self):
 
     def set_policies(self):
         """
-        Compute and sets the `license_policies` on the app instance.
+        Set the global app policies on the app instance.
 
         If the policies file is available but not formatted properly or doesn't
         include the proper content, we want to raise an exception while the app
@@ -240,7 +239,7 @@ def set_policies(self):
         if policies_file.exists():
             policies = load_policies_file(policies_file)
             logger.debug(style.SUCCESS(f"Loaded policies from {policies_file}"))
-            self.license_policies_index = make_license_policy_index(policies)
+            self.policies = policies
         else:
             logger.debug(style.WARNING("Policies file not found."))
 

scanpipe/models.py

@@ -1495,32 +1495,35 @@ def has_single_resource(self):
         """
         return self.resource_count == 1
 
-    def get_policy_index(self):
+    def get_policies_dict(self):
         """
-        Return the policy index for this project instance.
+        Load and return the policies from the following locations in that order:
 
-        The policies are loaded from the following locations in that order:
-        1. the project local settings
-        2. the "policies.yml" file in the project input/ directory
-        3. the global app settings license policies
+        1. project local settings (stored in the database)
+        2. "policies.yml" file in the project ``input/`` directory
+        3. global app settings policies, from SCANCODEIO_POLICIES_FILE setting
         """
         if policies_from_settings := self.get_env("policies"):
             policies_dict = policies_from_settings
             if isinstance(policies_from_settings, str):
                 policies_dict = policies.load_policies_yaml(policies_from_settings)
-            return policies.make_license_policy_index(policies_dict)
+            return policies_dict
 
-        elif policies_file := self.get_input_policies_file():
-            policies_dict = policies.load_policies_file(policies_file)
-            return policies.make_license_policy_index(policies_dict)
+        elif project_input_policies_file := self.get_input_policies_file():
+            return policies.load_policies_file(project_input_policies_file)
 
-        else:
-            return scanpipe_app.license_policies_index
+        return scanpipe_app.policies
+
+    def get_license_policy_index(self):
+        """Return the policy license index for this project instance."""
+        if policies_dict := self.get_policies_dict():
+            return policies.make_license_policy_index(policies_dict)
+        return {}
 
     @cached_property
     def policy_index(self):
-        """Return the cached policy index for this project instance."""
-        return self.get_policy_index()
+        """Return the cached license policy index for this project instance."""
+        return self.get_license_policy_index()
 
     @property
     def policies_enabled(self):

scanpipe/tests/pipes/test_scancode.py

@@ -467,8 +467,8 @@ def test_scanpipe_max_file_size_works(self):
             self.assertEqual(resource1.status, flag.IGNORED_BY_MAX_FILE_SIZE)
 
     def test_scanpipe_pipes_scancode_make_results_summary(self, regen=FIXTURES_REGEN):
-        # Ensure the policies index is empty to avoid any side effect on results
-        scanpipe_app.license_policies_index = None
+        # Ensure the policies are empty to avoid any side effect on results
+        scanpipe_app.policies = None
         # Run the scan_single_package pipeline to have a proper DB and local files setup
         pipeline_name = "scan_single_package"
         project1 = Project.objects.create(name="Analysis")

scanpipe/tests/test_apps.py

@@ -23,6 +23,7 @@
 import uuid
 from pathlib import Path
 from unittest import mock
+from unittest.mock import patch
 
 from django.apps import apps
 from django.core.exceptions import ImproperlyConfigured
@@ -33,7 +34,7 @@
 from scanpipe.models import Project
 from scanpipe.models import Run
 from scanpipe.tests import filter_warnings
-from scanpipe.tests import license_policies_index
+from scanpipe.tests import global_policies
 from scanpipe.tests.pipelines.register_from_file import RegisterFromFile
 
 scanpipe_app = apps.get_app_config("scanpipe")
@@ -43,26 +44,23 @@ class ScanPipeAppsTest(TestCase):
     data = Path(__file__).parent / "data"
     pipelines_location = Path(__file__).parent / "pipelines"
 
-    def test_scanpipe_apps_set_policies(self):
-        scanpipe_app.license_policies_index = {}
-        policies_files = None
-        with override_settings(SCANCODEIO_POLICIES_FILE=policies_files):
+    @patch.object(scanpipe_app, "policies", new_callable=dict)
+    def test_scanpipe_apps_set_policies(self, mock_policies):
+        # Case 1: No file set
+        with override_settings(SCANCODEIO_POLICIES_FILE=None):
             scanpipe_app.set_policies()
-            self.assertEqual({}, scanpipe_app.license_policies_index)
+            self.assertEqual({}, scanpipe_app.policies)
 
-        scanpipe_app.license_policies_index = {}
-        policies_files = "not_existing"
-        with override_settings(SCANCODEIO_POLICIES_FILE=policies_files):
+        # Case 2: Non-existing file
+        with override_settings(SCANCODEIO_POLICIES_FILE="not_existing"):
             scanpipe_app.set_policies()
-            self.assertEqual({}, scanpipe_app.license_policies_index)
+            self.assertEqual({}, scanpipe_app.policies)
 
-        scanpipe_app.license_policies_index = {}
+        # Case 3: Valid file
         policies_files = self.data / "policies" / "policies.yml"
         with override_settings(SCANCODEIO_POLICIES_FILE=str(policies_files)):
             scanpipe_app.set_policies()
-            self.assertEqual(
-                license_policies_index, scanpipe_app.license_policies_index
-            )
+            self.assertEqual(global_policies, scanpipe_app.policies)
 
     def test_scanpipe_apps_register_pipeline_from_file(self):
         path = self.pipelines_location / "do_nothing.py"

scanpipe/tests/test_forms.py

@@ -230,7 +230,7 @@ def test_scanpipe_forms_project_settings_form_policies(self):
         self.assertTrue(form.is_valid())
         project = form.save()
         self.assertEqual(policies_as_yaml.strip(), project.settings["policies"])
-        self.assertEqual(license_policies_index, project.get_policy_index())
+        self.assertEqual(license_policies_index, project.get_license_policy_index())
 
     def test_scanpipe_forms_project_settings_form_purl(self):
         data_invalid_purl = {

scanpipe/tests/test_models.py

@@ -75,6 +75,7 @@
 from scanpipe.pipes.input import copy_input
 from scanpipe.tests import dependency_data1
 from scanpipe.tests import dependency_data2
+from scanpipe.tests import global_policies
 from scanpipe.tests import license_policies_index
 from scanpipe.tests import make_dependency
 from scanpipe.tests import make_message
@@ -697,21 +698,21 @@ def test_scanpipe_project_get_input_policies_file(self):
         policies_file_location = str(self.project1.get_input_policies_file())
         self.assertTrue(policies_file_location.endswith("input/policies.yml"))
 
-    def test_scanpipe_project_model_get_policy_index(self):
-        scanpipe_app.license_policies_index = None
-        self.assertFalse(self.project1.policies_enabled)
-
-        policies_from_app_settings = {"from": "scanpipe_app"}
-        scanpipe_app.license_policies_index = policies_from_app_settings
-        self.assertEqual(policies_from_app_settings, self.project1.get_policy_index())
+    @patch.object(scanpipe_app, "policies", new=global_policies)
+    def test_scanpipe_project_model_get_license_policy_index(self):
+        self.assertEqual(
+            license_policies_index, self.project1.get_license_policy_index()
+        )
 
         policies_from_input_dir = {"license_policies": [{"license_key": "input_dir"}]}
         policies_file = self.project1.input_path / "policies.yml"
         policies_file.touch()
         policies_as_yaml = saneyaml.dump(policies_from_input_dir)
         policies_file.write_text(policies_as_yaml)
         expected_index_from_input = {"input_dir": {"license_key": "input_dir"}}
-        self.assertEqual(expected_index_from_input, self.project1.get_policy_index())
+        self.assertEqual(
+            expected_index_from_input, self.project1.get_license_policy_index()
+        )
         # Refresh the instance to bypass the cached_property cache.
         self.project1 = Project.objects.get(uuid=self.project1.uuid)
         self.assertTrue(self.project1.policies_enabled)
@@ -723,10 +724,9 @@ def test_scanpipe_project_model_get_policy_index(self):
         self.project1.settings = config
         self.project1.save()
         expected_index_from_env = {"project_env": {"license_key": "project_env"}}
-        self.assertEqual(expected_index_from_env, self.project1.get_policy_index())
-
-        # Reset the index value
-        scanpipe_app.license_policies_index = None
+        self.assertEqual(
+            expected_index_from_env, self.project1.get_license_policy_index()
+        )
 
     def test_scanpipe_project_get_settings_as_yml(self):
         self.assertEqual("{}\n", self.project1.get_settings_as_yml())
@@ -1547,28 +1547,31 @@ def test_scanpipe_codebase_resource_model_commoncode_methods_extracted_to_from(
         self.assertEqual(extracted_dir_resource, archive_resource.extracted_to())
         self.assertEqual(archive_resource, extracted_dir_resource.extracted_from())
 
+    @patch.object(scanpipe_app, "policies", new=global_policies)
     def test_scanpipe_codebase_resource_model_compliance_alert(self):
-        scanpipe_app.license_policies_index = license_policies_index
+        project_license_policies_index = self.project1.policy_index
+        self.assertEqual(license_policies_index, project_license_policies_index)
+
         resource = CodebaseResource.objects.create(project=self.project1, path="file")
         self.assertEqual("", resource.compliance_alert)
 
         license_expression = "bsd-new"
-        self.assertNotIn(license_expression, scanpipe_app.license_policies_index)
+        self.assertNotIn(license_expression, project_license_policies_index)
         resource.update(detected_license_expression=license_expression)
         self.assertEqual("missing", resource.compliance_alert)
 
         license_expression = "apache-2.0"
-        self.assertIn(license_expression, scanpipe_app.license_policies_index)
+        self.assertIn(license_expression, project_license_policies_index)
         resource.update(detected_license_expression=license_expression)
         self.assertEqual("ok", resource.compliance_alert)
 
         license_expression = "mpl-2.0"
-        self.assertIn(license_expression, scanpipe_app.license_policies_index)
+        self.assertIn(license_expression, project_license_policies_index)
         resource.update(detected_license_expression=license_expression)
         self.assertEqual("warning", resource.compliance_alert)
 
         license_expression = "gpl-3.0"
-        self.assertIn(license_expression, scanpipe_app.license_policies_index)
+        self.assertIn(license_expression, project_license_policies_index)
         resource.update(detected_license_expression=license_expression)
         self.assertEqual("error", resource.compliance_alert)
 
@@ -1584,11 +1587,8 @@ def test_scanpipe_codebase_resource_model_compliance_alert(self):
         resource.update(detected_license_expression=license_expression)
         self.assertEqual("warning", resource.compliance_alert)
 
-        # Reset the index value
-        scanpipe_app.license_policies_index = None
-
+    @patch.object(scanpipe_app, "policies", new=global_policies)
     def test_scanpipe_codebase_resource_model_compliance_alert_update_fields(self):
-        scanpipe_app.license_policies_index = license_policies_index
         resource = CodebaseResource.objects.create(project=self.project1, path="file")
         self.assertEqual("", resource.compliance_alert)
 
@@ -1598,9 +1598,6 @@ def test_scanpipe_codebase_resource_model_compliance_alert_update_fields(self):
         resource.refresh_from_db()
         self.assertEqual("ok", resource.compliance_alert)
 
-        # Reset the index value
-        scanpipe_app.license_policies_index = None
-
     def test_scanpipe_scan_fields_model_mixin_methods(self):
         expected = [
             "detected_license_expression",
@@ -2422,30 +2419,27 @@ def test_scanpipe_discovered_package_model_as_cyclonedx(self):
             cyclonedx_component.evidence.licenses[0].value,
         )
 
+    @patch.object(scanpipe_app, "policies", new=global_policies)
     def test_scanpipe_discovered_package_model_compliance_alert(self):
-        scanpipe_app.license_policies_index = license_policies_index
         package_data = package_data1.copy()
         package_data["declared_license_expression"] = ""
         package = DiscoveredPackage.create_from_data(self.project1, package_data)
         self.assertEqual("", package.compliance_alert)
 
         license_expression = "bsd-new"
-        self.assertNotIn(license_expression, scanpipe_app.license_policies_index)
+        self.assertNotIn(license_expression, self.project1.policy_index)
         package.update(declared_license_expression=license_expression)
         self.assertEqual("missing", package.compliance_alert)
 
         license_expression = "apache-2.0"
-        self.assertIn(license_expression, scanpipe_app.license_policies_index)
+        self.assertIn(license_expression, self.project1.policy_index)
         package.update(declared_license_expression=license_expression)
         self.assertEqual("ok", package.compliance_alert)
 
         license_expression = "apache-2.0 AND mpl-2.0 OR gpl-3.0"
         package.update(declared_license_expression=license_expression)
         self.assertEqual("error", package.compliance_alert)
 
-        # Reset the index value
-        scanpipe_app.license_policies_index = None
-
     def test_scanpipe_discovered_package_model_spdx_id(self):
         package1 = make_package(self.project1, "pkg:type/a")
         expected = f"SPDXRef-scancodeio-discoveredpackage-{package1.uuid}"

scanpipe/tests/test_policies.py

@@ -98,10 +98,12 @@ def test_scanpipe_policies_scan_codebase_pipeline_integration(self):
         pipeline = run.make_pipeline_instance()
 
         # Capture the real method's return value
-        real_get_policy_index = project1.get_policy_index
+        real_get_license_policy_index = project1.get_license_policy_index
 
-        with mock.patch("scanpipe.models.Project.get_policy_index") as mock_get_index:
-            mock_get_index.side_effect = real_get_policy_index
+        with mock.patch(
+            "scanpipe.models.Project.get_license_policy_index"
+        ) as mock_get_index:
+            mock_get_index.side_effect = real_get_license_policy_index
             exitcode, out = pipeline.execute()
         mock_get_index.assert_called_once()
 
@@ -132,7 +134,7 @@ def test_scanpipe_policies_scan_codebase_pipeline_integration(self):
             "apache-2.0": {"license_key": "apache-2.0", "compliance_alert": ""},
             "gpl-2.0": {"license_key": "gpl-2.0", "compliance_alert": "error"},
         }
-        self.assertEqual(expected_index, project1.get_policy_index())
+        self.assertEqual(expected_index, project1.get_license_policy_index())
 
     def test_scanpipe_policies_through_scancode_config_file(self):
         project1 = make_project()