Merge branch 'main' into fix-d2d-config-bugs

Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

Author: AyanSinhaMahapatra

.github/workflows/publish-docker.yml renamed to .github/workflows/publish-docker-image.yml

@@ -1,4 +1,4 @@
-name: Publish Docker image on GHCR
+name: Publish Docker image on GitHub Container Registry
 # https://docs.github.com/en/packages/managing-github-packages-using-github-actions-workflows/publishing-and-installing-a-package-with-github-actions
 
 on:
@@ -15,8 +15,8 @@ env:
   IMAGE_NAME: ${{ github.repository }}
 
 jobs:
-  build-and-push-image:
-    runs-on: ubuntu-22.04
+  build-and-publish-image:
+    runs-on: ubuntu-24.04
 
     # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
     permissions:

.github/workflows/pypi-release-aboutcode-pipeline.yml renamed to .github/workflows/publish-pypi-release-aboutcode-pipeline.yml

@@ -3,13 +3,13 @@ name: Build aboutcode.pipeline Python distributions and publish on PyPI
 on:
   workflow_dispatch:
   push:
-   tags:
-     - "aboutcode.pipeline/*"
+    tags:
+      - "aboutcode.pipeline/*"
 
 jobs:
   build-and-publish:
     name: Build and publish library to PyPI
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
 
     steps:
       - uses: actions/checkout@v4

.github/workflows/pypi-release.yml renamed to .github/workflows/publish-pypi-release.yml

@@ -9,7 +9,7 @@ on:
 jobs:
   build-and-publish:
     name: Build and publish library to PyPI
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
 
     steps:
       - uses: actions/checkout@v4

.github/workflows/ci-docker.yml renamed to .github/workflows/run-unit-tests-docker.yml

@@ -1,10 +1,18 @@
-name: Test on Docker CI
+name: Run unit tests on Docker container
 
-on: [push, pull_request]
+on:
+  workflow_dispatch:
+  pull_request:
+  push:
+    branches:
+      - main
+
+permissions:
+  contents: read
 
 jobs:
-  build:
-    runs-on: ubuntu-22.04
+  run-unit-tests:
+    runs-on: ubuntu-24.04
 
     steps:
       - name: Checkout code

.github/workflows/run-unit-tests-macos.yml

@@ -0,0 +1,53 @@
+name: Run unit tests on macOS
+
+on:
+  workflow_dispatch:
+  pull_request:
+  push:
+    branches:
+      - main
+
+permissions:
+  contents: read
+
+env:
+  POSTGRES_DB: scancodeio
+  POSTGRES_USER: scancodeio
+  POSTGRES_PASSWORD: scancodeio
+
+jobs:
+  run-unit-tests:
+    runs-on: macos-13
+
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: ikalnytskyi/action-setup-postgres@v7
+        id: postgres
+        with:
+          postgres-version: "14"  # 13 is not supported.
+          database: ${{ env.POSTGRES_DB }}
+          username: ${{ env.POSTGRES_USER }}
+          password: ${{ env.POSTGRES_PASSWORD }}
+          port: 5432
+
+      - name: Install Python dependencies
+        run: make dev envfile
+
+      - name: Run Django tests
+        run: .venv/bin/python manage.py test --verbosity=2 --noinput
+        env:
+          SCANCODEIO_DB_NAME: ${{ env.POSTGRES_DB }}
+          SCANCODEIO_DB_USER: ${{ env.POSTGRES_USER }}
+          SCANCODEIO_DB_PASSWORD: ${{ env.POSTGRES_PASSWORD }}

.github/workflows/ci.yml renamed to .github/workflows/run-unit-tests.yml

@@ -1,6 +1,14 @@
-name: Test CI
+name: Run unit tests
 
-on: [push, pull_request]
+on:
+  workflow_dispatch:
+  pull_request:
+  push:
+    branches:
+      - main
+
+permissions:
+  contents: read
 
 env:
   POSTGRES_DB: scancodeio
@@ -9,8 +17,8 @@ env:
   POSTGRES_INITDB_ARGS: --encoding=UTF-8 --lc-collate=en_US.UTF-8 --lc-ctype=en_US.UTF-8
 
 jobs:
-  build:
-    runs-on: ubuntu-22.04
+  run-unit-tests:
+    runs-on: ubuntu-24.04
 
     services:
       postgres:

CHANGELOG.rst

@@ -4,6 +4,16 @@ Changelog
 v35.2.0 (unreleased)
 --------------------
 
+- Refactor policies implementation to support more than licenses.
+  The entire ``policies`` data is now stored on the ``ScanPipeConfig`` in place of the
+  ``license_policy_index``.
+  Also, a new method ``get_policies_dict`` methods is now available on the ``Project``
+  model to easily retrieve all the policies data as a dictionary.
+  Renamed for clarity:
+  * ``policy_index`` to ``license_policy_index``
+  * ``policies_enabled`` to ``license_policies_enabled``
+  https://github.com/aboutcode-org/scancode.io/pull/1718
+
 - Add support for SPDX license identifiers as ``license_key`` in license policies
   ``policies.yml`` file.
   https://github.com/aboutcode-org/scancode.io/issues/1348

pyproject.toml

@@ -164,9 +164,12 @@ select = [
     "D",  # pydocstyle
     "F",  # Pyflakes
     "UP", # pyupgrade
+    "DJ", # flake8-django
     "S",  # flake8-bandit
     "I",  # isort
     "C9", # McCabe complexity
+    "FIX", # flake8-fix
+    "FURB", # refurb
 ]
 ignore = ["D1", "D203", "D205", "D212", "D400", "D415"]
 
@@ -188,6 +191,5 @@ max-complexity = 10
 [tool.ruff.lint.per-file-ignores]
 # Allow the usage of assert in the test_spdx file.
 "**/test_spdx.py*" = ["S101"]
-"scanpipe/pipes/spdx.py" = ["UP006", "UP035"]
 # Allow complexity in management commands
 "scanpipe/management/commands/*" = ["C901"]

scancodeio/settings.py

@@ -292,6 +292,7 @@
     SCANCODEIO_WORKSPACE_LOCATION = tempfile.mkdtemp()
     SCANCODEIO_REQUIRE_AUTHENTICATION = True
     SCANCODEIO_SCAN_FILE_TIMEOUT = 120
+    SCANCODEIO_POLICIES_FILE = None
     # The default password hasher is rather slow by design.
     # Using a faster hashing algorithm in the testing context to speed up the run.
     PASSWORD_HASHERS = ["django.contrib.auth.hashers.MD5PasswordHasher"]

scanpipe/apps.py

@@ -39,7 +39,6 @@
 from licensedcode.models import load_licenses
 
 from scanpipe.policies import load_policies_file
-from scanpipe.policies import make_license_policy_index
 
 try:
     from importlib import metadata as importlib_metadata
@@ -61,7 +60,7 @@ def __init__(self, app_name, app_module):
 
         # Mapping of registered pipeline names to pipeline classes.
         self._pipelines = {}
-        self.license_policies_index = {}
+        self.policies = {}
 
         workspace_location = settings.SCANCODEIO_WORKSPACE_LOCATION
         self.workspace_path = Path(workspace_location).expanduser().resolve()
@@ -226,7 +225,7 @@ def get_scancode_licenses(self):
 
     def set_policies(self):
         """
-        Compute and sets the `license_policies` on the app instance.
+        Set the global app policies on the app instance.
 
         If the policies file is available but not formatted properly or doesn't
         include the proper content, we want to raise an exception while the app
@@ -240,7 +239,7 @@ def set_policies(self):
         if policies_file.exists():
             policies = load_policies_file(policies_file)
             logger.debug(style.SUCCESS(f"Loaded policies from {policies_file}"))
-            self.license_policies_index = make_license_policy_index(policies)
+            self.policies = policies
         else:
             logger.debug(style.WARNING("Policies file not found."))