clean: Improve build process for OpenSSF malicious packages index

Author: afsmeira

.circleci/config.yml

@@ -12,13 +12,13 @@ references:
       curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b . v0.66.0
       mkdir cache
       ./trivy --cache-dir ./cache image --download-db-only
-      # Download OpenSSF malicious packages database
-      mkdir -p openssf-cache
-      echo "Downloading OpenSSF malicious packages database..."
-      curl -sfL https://api.github.com/repos/ossf/malicious-packages/tarball/main | tar -xz --strip-components=1 -C openssf-cache
-      # Copy OSV files to the expected cache location
-      mkdir -p openssf-cache/osv
-      find openssf-cache -name "*.json" -path "*/osv/*" -exec cp {} openssf-cache/osv/ \;
+
+  build_openssf_malicious_package_index: &build_openssf_malicious_package_index
+    persist_to_workspace: true
+    cmd: |
+      mkdir openssf-malicious-packages
+      curl -sfL https://api.github.com/repos/ossf/malicious-packages/tarball/main | tar -xz --strip-components=1 -C openssf-malicious-packages
+      python3 scripts/build_openssf_index.py
 
   build_and_publish_docker: &build_and_publish_docker
     persist_to_workspace: true
@@ -43,11 +43,17 @@ workflows:
           name: install_trivy_and_download_dbs
           requires:
             - generate_and_test
+      - codacy/shell:
+          <<: *build_openssf_malicious_package_index
+          name: build_openssf_malicious_package_index
+          requires:
+            - generate_and_test
       - codacy/shell:
           <<: *build_and_publish_docker
           name: publish_docker_local
           requires:
             - install_trivy_and_download_dbs
+            - build_openssf_malicious_package_index
       - codacy_plugins_test/run:
           name: plugins_test
           run_multiple_tests: true
@@ -91,11 +97,17 @@ workflows:
           name: install_trivy_and_download_dbs
           requires:
             - generate_and_test
+      - codacy/shell:
+          <<: *build_openssf_malicious_package_index
+          name: build_openssf_malicious_package_index
+          requires:
+            - generate_and_test
       - codacy/shell:
           <<: *build_and_publish_docker
           name: publish_docker_local
           requires:
             - install_trivy_and_download_dbs
+            - build_openssf_malicious_package_index
       - codacy/publish_docker:
           name: publish_dockerhub
           context: CodacyDocker

.gitignore

@@ -11,21 +11,21 @@ project
 target
 bin
 cache
+openssf-malicious-packages
 *.gen.go
 .codacyrc
 trivy
 
 
-#Ignore vscode AI rules
+# Ignore vscode AI rules
 .github/copilot-instructions.md
 
-
-#Ignore cursor AI rules
+# Ignore cursor AI rules
 .cursor/rules/codacy.mdc
 
-#Ignore codacy stuff
+# Ignore codacy stuff
 .codacy/cli.sh
 .codacy/codacy.yaml
 
-#Ignore patterns.json
+# Ignore patterns.json
 docs/patterns.json

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.25-alpine as builder
+FROM golang:1.25-alpine AS builder
 
 ARG TRIVY_VERSION=dev
 ENV TRIVY_VERSION=$TRIVY_VERSION
@@ -24,21 +24,13 @@ COPY docs docs
 RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/go/pkg/mod \
     go run ./cmd/docgen
 
-# Generate the OpenSSF index during build
-COPY scripts/ scripts/
-COPY openssf-cache/osv/ openssf-cache/osv/
-RUN apk add --no-cache python3 && \
-    python3 scripts/build_openssf_index.py
-
 FROM busybox
 
 RUN adduser -u 2004 -D docker
 
 COPY --from=builder --chown=docker:docker /src/bin /dist/bin
 COPY --from=builder --chown=docker:docker /src/docs /docs 
-COPY --from=builder --chown=docker:docker /src/openssf-index.json.gz /dist/cache/openssf-index.json.gz
 COPY --chown=docker:docker cache/ /dist/cache/codacy-trivy
-
-USER docker
+COPY --chown=docker:docker openssf-malicious-packages/openssf-malicious-packages-index.json.gz /dist/cache/codacy-trivy/openssf-malicious-packages-index.json.gz
 
 CMD [ "/dist/bin/codacy-trivy" ]

scripts/build_openssf_index.py

@@ -36,16 +36,18 @@
 import os, json, gzip
 from concurrent.futures import ThreadPoolExecutor, as_completed
 
-BASE = os.environ.get('OPENSSF_OSV_DIR', 'openssf-cache/osv')
-OUT = os.environ.get('OPENSSF_INDEX_OUT', 'openssf-index.json.gz')
+# We are ignoring withdrawn packages.
+# See https://github.com/ossf/malicious-packages/tree/main/osv/withdrawn
+BASE = os.environ.get('OPENSSF_OSV_MALICIOUS_DIR', 'openssf-malicious-packages/osv/malicious')
+OUT = os.environ.get('OPENSSF_INDEX_OUT', 'openssf-malicious-packages/openssf-malicious-packages-index.json.gz')
 
 def read_json_file(path):
     with open(path, 'r', encoding='utf-8') as fh:
         return json.load(fh)
 
 
 def extract_package_info(pkg):
-    """Extract and validate package information."""
+    """Extract package information."""
     eco = (pkg.get('ecosystem') or '').lower()
     name = (pkg.get('name') or '').lower()
     return eco, name
@@ -85,6 +87,7 @@ def process_file(path):
         return []
 
 
+# Get all malicious package files to work on them in parallel.
 files = []
 for root, _, fns in os.walk(BASE):
     for fn in fns:
@@ -94,7 +97,7 @@ def process_file(path):
 index = {}
 workers = min(32, os.cpu_count() or 8)
 with ThreadPoolExecutor(max_workers=workers) as ex:
-    futs = [ex.submit(process_file, p) for p in files]
+    futs = [ex.submit(process_file, f) for f in files]
     for fut in as_completed(futs):
         for eco, name, entry in fut.result():
             eco_map = index.setdefault(eco, {})