Merge branch 'main' into update-federatedcode-pipes

Author: AyanSinhaMahapatra

.github/workflows/sca-integration-ort.yml

@@ -0,0 +1,50 @@
+name: Generate SBOM with ORT and load into ScanCode.io
+
+# This workflow:
+#  1. Generates a CycloneDX SBOM for a requirement.txt file using ORT.
+#  2. Uploads the SBOM as a GitHub artifact for future inspection.
+#  3. Loads the SBOM into ScanCode.io for further analysis.
+#  4. Runs assertions to verify that the SBOM was properly processed in ScanCode.io.
+#
+# It runs on demand, and once a week (scheduled).
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Run once a week (every 7 days) at 00:00 UTC on Sunday
+    - cron: "0 0 * * 0"
+  pull_request:
+  push:
+    branches:
+      - main
+
+permissions:
+  contents: read
+
+jobs:
+  generate-and-load-sbom:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Create a Python requirements.txt
+        run: |
+          cat << 'EOF' > requirements.txt
+          amqp==5.1.1
+          appdirs==1.4.4
+          asgiref==3.5.2
+          urllib3==1.26.0
+          EOF
+
+      - name: Run GitHub Action for ORT
+        uses: oss-review-toolkit/ort-ci-github-action@v1
+
+      - name: Import SBOM into ScanCode.io
+        uses: aboutcode-org/scancode-action@main
+        with:
+          pipelines: "load_sbom"
+          inputs-path: "${{ env.ORT_RESULTS_PATH }}/bom.cyclonedx.json"
+          scancodeio-repo-branch: "main"
+
+      - name: Verify SBOM Analysis Results in ScanCode.io
+        shell: bash
+        run: |
+          scanpipe shell --command "from scanpipe.models import DiscoveredPackage, DiscoveredDependency; package_manager = DiscoveredPackage.objects; assert package_manager.count() >= 5; assert package_manager.vulnerable().count() >= 1; assert DiscoveredDependency.objects.count() >= 1"

CHANGELOG.rst

@@ -4,15 +4,38 @@ Changelog
 v35.4.0 (unreleased)
 --------------------
 
+- Use deterministic UID/GID in Dockerfile.
+  A temporary ``chown`` service is now started in the ``docker-compose`` stack
+  to fix the permissions. This process is only fully run once.
+  You may manually run this process using the following:
+  ``$ chown -R 1000:1000 /var/scancodeio/``
+  https://github.com/aboutcode-org/scancode.io/issues/1555
+
 - Resolve and load dependencies from SPDX SBOMs.
   https://github.com/aboutcode-org/scancode.io/issues/1145
 
+- Display the optional steps in the Pipelines autodoc.
+  https://github.com/aboutcode-org/scancode.io/issues/1822
+
+- Add new ``benchmark_purls`` pipeline.
+  https://github.com/aboutcode-org/scancode.io/issues/1804
+
+- Add a Resources tree view.
+  https://github.com/aboutcode-org/scancode.io/issues/1682
+
+- Improve CycloneDX SBOM support.
+  * Upgrade the cyclonedx-python-lib to 11.0.0
+  * Fix the validate_document following library upgrade.
+  * Add support when the "components" entry is missing.
+  https://github.com/aboutcode-org/scancode.io/issues/1727
+
 - Split the functionality of
   ``scanpipe.pipes.federatedcode.commit_and_push_changes`` into
   ``scanpipe.pipes.federatedcode.commit_changes`` and
   ``scanpipe.pipes.federatedcode.push_changes``. Add
   ``scanpipe.pipes.federatedcode.write_data_as_yaml``.
 
+
 v35.3.0 (2025-08-20)
 --------------------
 

Dockerfile

@@ -26,17 +26,23 @@ LABEL org.opencontainers.image.source="https://github.com/aboutcode-org/scancode
 LABEL org.opencontainers.image.description="ScanCode.io"
 LABEL org.opencontainers.image.licenses="Apache-2.0"
 
-ENV APP_NAME scancodeio
-ENV APP_USER app
-ENV APP_DIR /opt/$APP_NAME
-ENV VENV_LOCATION /opt/$APP_NAME/.venv
+# Set default values for APP_UID and APP_GID at build-time
+ARG APP_UID=1000
+ARG APP_GID=1000
+
+ENV APP_NAME=scancodeio
+ENV APP_USER=app
+ENV APP_UID=${APP_UID}
+ENV APP_GID=${APP_GID}
+ENV APP_DIR=/opt/$APP_NAME
+ENV VENV_LOCATION=/opt/$APP_NAME/.venv
 
 # Force Python unbuffered stdout and stderr (they are flushed to terminal immediately)
-ENV PYTHONUNBUFFERED 1
+ENV PYTHONUNBUFFERED=1
 # Do not write Python .pyc files
-ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONDONTWRITEBYTECODE=1
 # Add the app dir in the Python path for entry points availability
-ENV PYTHONPATH $PYTHONPATH:$APP_DIR
+ENV PYTHONPATH=$PYTHONPATH:$APP_DIR
 
 # OS requirements as per
 # https://scancode-toolkit.readthedocs.io/en/latest/getting-started/install.html
@@ -64,27 +70,24 @@ RUN apt-get update \
  && apt-get clean \
  && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
-# Create the APP_USER group and user
-RUN addgroup --system $APP_USER \
- && adduser --system --group --home=$APP_DIR $APP_USER \
- && chown $APP_USER:$APP_USER $APP_DIR
-
-# Create the /var/APP_NAME directory with proper permission for APP_USER
-RUN mkdir -p /var/$APP_NAME \
+# Create the APP_USER group, user, and directory with specific UID and GID
+RUN groupadd --gid $APP_GID --system $APP_USER \
+ && useradd --uid $APP_UID --gid $APP_GID --home-dir $APP_DIR --system --create-home $APP_USER \
+ && chown $APP_USER:$APP_USER $APP_DIR \
+ && mkdir -p /var/$APP_NAME \
  && chown $APP_USER:$APP_USER /var/$APP_NAME
 
 # Setup the work directory and the user as APP_USER for the remaining stages
 WORKDIR $APP_DIR
 USER $APP_USER
 
+# Create static/ and workspace/ directories
+RUN mkdir -p /var/$APP_NAME/static/ /var/$APP_NAME/workspace/
+
 # Create the virtualenv
 RUN python -m venv $VENV_LOCATION
 # Enable the virtualenv, similar effect as "source activate"
-ENV PATH $VENV_LOCATION/bin:$PATH
-
-# Create static/ and workspace/ directories
-RUN mkdir -p /var/$APP_NAME/static/ \
- && mkdir -p /var/$APP_NAME/workspace/
+ENV PATH=$VENV_LOCATION/bin:$PATH
 
 # Install the dependencies before the codebase COPY for proper Docker layer caching
 COPY --chown=$APP_USER:$APP_USER pyproject.toml $APP_DIR/

docker-compose.yml

@@ -8,6 +8,11 @@ services:
       - db_data:/var/lib/postgresql/data/
     shm_size: "1gb"
     restart: always
+    healthcheck:
+      test: [ "CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}" ]
+      interval: 10s
+      timeout: 5s
+      retries: 5
 
   redis:
     image: docker.io/library/redis:latest
@@ -18,12 +23,45 @@ services:
       - redis_data:/data
     restart: always
 
+  # This service is responsible for ensuring the correct ownership of files
+  # in the shared volumes used by the application (static and workspace).
+  # It ensures that all files inside the `/var/scancodeio/` directory are owned
+  # by the user and group with the UID and GID defined in the environment variables
+  # APP_UID and APP_GID, which default to 1000 if not set.
+  #
+  # The service runs only once (due to "restart: no") and performs a `chown` operation
+  # to change the ownership of the static and workspace directories, ensuring proper
+  # file access rights for the running application containers.
+  #
+  # Volumes mounted:
+  # - static: Ensures the ownership of static files in the /var/scancodeio/static directory
+  # - media: Ensures the ownership of media files in the /var/scancodeio/workspace directory
+  #
+  # Notes: This service can be removed in future ScanCode.io release.
+  chown:
+    image: docker.io/library/alpine:latest
+    restart: "no"
+    command: sh -c "
+      if [ ! -f /var/scancodeio/workspace/.chown_done ]; then
+        chown -R ${APP_UID:-1000}:${APP_GID:-1000} /var/scancodeio/ &&
+        touch /var/scancodeio/workspace/.chown_done;
+        echo 'Chown applied!';
+      else
+        echo 'Chown already applied, skipping...';
+      fi"
+    env_file:
+      - docker.env
+    volumes:
+      - static:/var/scancodeio/static/
+      - workspace:/var/scancodeio/workspace/
+
   web:
     build: .
-    command: wait-for-it --strict --timeout=60 db:5432 -- sh -c "
+    command: sh -c "
         ./manage.py migrate &&
         ./manage.py collectstatic --no-input --verbosity 0 --clear &&
-        gunicorn scancodeio.wsgi:application --bind :8000 --timeout 600 --workers 8 ${GUNICORN_RELOAD_FLAG:-}"
+        gunicorn scancodeio.wsgi:application --bind :8000 --timeout 600 \
+          --workers 8 ${GUNICORN_RELOAD_FLAG:-}"
     env_file:
       - docker.env
     expose:
@@ -34,12 +72,17 @@ services:
       - workspace:/var/scancodeio/workspace/
       - static:/var/scancodeio/static/
     depends_on:
-      - db
+      db:
+        condition: service_healthy
+      redis:
+        condition: service_started
+      chown:
+        condition: service_completed_successfully
 
   worker:
     build: .
     # Ensure that potential db migrations run first by waiting until "web" is up
-    command: wait-for-it --strict --timeout=120 web:8000 -- sh -c "
+    command: wait-for-it --strict --timeout=600 web:8000 -- sh -c "
         ./manage.py rqworker --worker-class scancodeio.worker.ScanCodeIOWorker
                              --queue-class scancodeio.worker.ScanCodeIOQueue
                              --verbosity 1"
@@ -53,6 +96,7 @@ services:
       - redis
       - db
       - web
+      - chown
 
   nginx:
     image: docker.io/library/nginx:alpine

docs/built-in-pipelines.rst

@@ -46,6 +46,47 @@ Analyse Docker Windows Image
     :members:
     :member-order: bysource
 
+.. _pipeline_benchmark_purls:
+
+Benchmark PURLs (addon)
+-----------------------
+
+To check an **SBOM against a list of expected Package URLs (PURLs)**:
+
+1. **Create a new project** and provide two inputs:
+
+   * The SBOM file you want to check.
+   * A list of expected PURLs in a ``*-purls.txt`` file with one PURL per line.
+
+     .. tip:: You may also flag any filename using the ``purls`` input tag.
+
+2. **Run the pipelines**:
+
+   * Select and run the ``load_sbom`` pipeline to load the SBOM.
+   * Run the ``benchmark_purls`` pipeline to validate against the expected PURLs.
+
+3. **Download the results** from the "output" section of the project.
+
+The output file contains only the differences between the discovered PURLs and
+the expected PURLs:
+
+* Lines starting with ``-`` are missing from the project.
+* Lines starting with ``+`` are unexpected in the project.
+
+.. note::
+  The ``load_sbom`` pipeline is provided as an example to benchmark external
+  tools using SBOMs as inputs. You can also run ``benchmark_purls`` directly
+  after any ScanCode.io pipeline to validate the discovered PURLs.
+
+.. tip::
+  You can provide multiple expected PURLs files.
+
+
+.. autoclass:: scanpipe.pipelines.benchmark_purls.BenchmarkPurls()
+    :members:
+    :member-order: bysource
+
+
 .. _pipeline_collect_strings_gettext:
 
 Collect string with Xgettext (addon)

docs/conf.py

@@ -81,3 +81,23 @@
 
 # user starts in light mode (Default Mode)
 default_dark_mode = False
+
+
+# Display the optional steps in the Pipelines autodoc.
+def autodoc_process_docstring(app, what, name, obj, options, lines):
+    """
+    Sphinx autodoc extension hook to insert `@optional_step` groups
+    into the generated documentation.
+
+    If a function or method has been decorated with ``@optional_step``,
+    the decorator attaches a ``.groups`` attribute to the object.
+    This hook inspects that attribute during autodoc processing and prepends
+    a short note at the top of the function’s docstring.
+    """
+    if hasattr(obj, "groups"):
+        groups_str = " ".join(f":guilabel:`{group}`" for group in sorted(obj.groups))
+        lines[:0] = [f"**Optional step:** {groups_str}", ""]
+
+
+def setup(app):
+    app.connect("autodoc-process-docstring", autodoc_process_docstring)

docs/faq.rst

@@ -376,9 +376,10 @@ are actively supported and tested::
   - Anchore: https://anchore.com/sbom/
   - CycloneDX cdxgen: https://cyclonedx.github.io/cdxgen/
   - OWASP dep-scan: https://owasp.org/www-project-dep-scan/
+  - OSS Review Toolkit (ORT): https://oss-review-toolkit.org/ort/
+  - OSV-Scanner: https://osv.dev/
   - SBOM tool: https://github.com/microsoft/sbom-tool/
   - Trivy: https://trivy.dev/
-  - OSV-Scanner: https://osv.dev/
 
 .. note:: Imported SBOMs must follow the SPDX or CycloneDX standards, in JSON format.
    You can use the ``load_sbom`` pipeline to process and enhance these SBOMs in your

docs/scanpipe-pipes.rst

@@ -8,6 +8,11 @@ Generic
 .. automodule:: scanpipe.pipes
     :members:
 
+Benchmark
+---------
+.. automodule:: scanpipe.pipes.benchmark
+    :members:
+
 ClamAV
 ------
 .. automodule:: scanpipe.pipes.clamav

pyproject.toml

@@ -82,8 +82,8 @@ dependencies = [
   # Profiling
   "pyinstrument==5.1.1",
   # CycloneDX
-  "cyclonedx-python-lib==10.2.0",
-  "jsonschema==4.24.0",
+  "cyclonedx-python-lib==11.0.0",
+  "jsonschema==4.25.1",
   # MatchCode-toolkit
   "matchcode-toolkit==7.2.2",
   # Univers
@@ -135,6 +135,7 @@ run = "scancodeio:combined_run"
 analyze_docker_image = "scanpipe.pipelines.analyze_docker:Docker"
 analyze_root_filesystem_or_vm_image = "scanpipe.pipelines.analyze_root_filesystem:RootFS"
 analyze_windows_docker_image = "scanpipe.pipelines.analyze_docker_windows:DockerWindows"
+benchmark_purls = "scanpipe.pipelines.benchmark_purls:BenchmarkPurls"
 collect_strings_gettext = "scanpipe.pipelines.collect_strings_gettext:CollectStringsGettext"
 collect_symbols_ctags = "scanpipe.pipelines.collect_symbols_ctags:CollectSymbolsCtags"
 collect_symbols_pygments = "scanpipe.pipelines.collect_symbols_pygments:CollectSymbolsPygments"