diff --git a/scanpipe/pipes/benchmark.py b/scanpipe/pipes/benchmark.py index 52a405cf8a..b522b58f69 100644 --- a/scanpipe/pipes/benchmark.py +++ b/scanpipe/pipes/benchmark.py @@ -49,6 +49,18 @@ def get_expected_purls(project): return sorted(set(expected_purls)) +def get_unique_project_purls(project): + """ + Return the sorted list of unique Package URLs (PURLs) discovered in the project. + + Extracts the ``purl`` field from all discovered packages, removes duplicates, + and sorts the result to provide a deterministic list of project PURLs. + """ + project_packages = project.discoveredpackages.only_package_url_fields() + sorted_unique_purls = sorted({package.purl for package in project_packages}) + return sorted_unique_purls + + def compare_purls(project, expected_purls): """ Compare discovered project PURLs against the expected PURLs. @@ -57,10 +69,10 @@ def compare_purls(project, expected_purls): - Lines starting with '-' are missing from the project. - Lines starting with '+' are unexpected in the project. """ - project_packages = project.discoveredpackages.only_package_url_fields() - sorted_unique_purls = sorted({package.purl for package in project_packages}) + sorted_project_purls = get_unique_project_purls(project) + print(sorted_project_purls) - diff_result = difflib.ndiff(sorted_unique_purls, expected_purls) + diff_result = difflib.ndiff(sorted_project_purls, expected_purls) # Keep only lines that are diffs (- or +) filtered_diff = [line for line in diff_result if line.startswith(("-", "+"))] diff --git a/scanpipe/tests/test_sca_integrations.py b/scanpipe/tests/test_sca_integrations.py index dd15d7b074..01d3311136 100644 --- a/scanpipe/tests/test_sca_integrations.py +++ b/scanpipe/tests/test_sca_integrations.py @@ -20,168 +20,393 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/scancode.io for support and download. -from pathlib import Path +""" +Test suite for validating ScanCode.io integrations with third-party SCA SBOM tools. -from django.test import TestCase +Each test ensures that SBOM files generated by tools such as Anchore Grype, +CycloneDX (cdxgen), osv-scanner, Trivy, SBOM Tool, and more. -from scanpipe.tests import make_project +Adding new test data +==================== +1. Generate an SBOM with the tool you want to test. For example with Trivy: -class ScanPipeSCAIntegrationsTest(TestCase): - data = Path(__file__).parent / "data" + trivy image --scanners vuln,license --format cyclonedx \ + --output trivy-alpine-3.17-sbom.json alpine:3.17.0 - def test_scanpipe_scan_integrations_load_sbom_trivy(self): - # Input file generated with: - # $ trivy image --scanners vuln,license --format cyclonedx \ - # --output trivy-alpine-3.17-sbom.json alpine:3.17.0 - input_location = self.data / "sca-integrations" / "trivy-alpine-3.17-sbom.json" +2. Save the SBOM file under: + tests/data/sca-integrations/ - pipeline_name = "load_sbom" - project1 = make_project() - project1.copy_input_from(input_location) +3. Add expected counts for that SBOM to ``SCA_INTEGRATIONS_TEST_DATA`` below. - run = project1.add_pipeline(pipeline_name) - pipeline = run.make_pipeline_instance() +Example: + "trivy-alpine-3.17-sbom.json": { + "resources": 1, + "packages": 16, + "packages_vulnerable": 7, + "dependencies": 25, + "purls": [ + "pkg:unknown/alpine@3.17.0", + ], + }, - exitcode, out = pipeline.execute() - self.assertEqual(0, exitcode, msg=out) + Tip: run the test once without expected values, check the counts from the + failure messages, then update the dictionary accordingly. - self.assertEqual(1, project1.codebaseresources.count()) - self.assertEqual(16, project1.discoveredpackages.count()) - self.assertEqual(7, project1.discoveredpackages.vulnerable().count()) - self.assertEqual(25, project1.discovereddependencies.count()) - - def test_scanpipe_scan_integrations_load_sbom_anchore(self): - # Input file generated with: - # $ grype -v -o cyclonedx-json \ - # --file anchore-alpine-3.17-sbom.json alpine:3.17.0 - input_location = ( - self.data / "sca-integrations" / "anchore-alpine-3.17-sbom.json" - ) +4. Run the test suite: - pipeline_name = "load_sbom" - project1 = make_project() - project1.copy_input_from(input_location) + ./manage.py test scanpipe.tests.test_sca_integrations - run = project1.add_pipeline(pipeline_name) - pipeline = run.make_pipeline_instance() +5. Commit both the SBOM file and dictionary entry. - exitcode, out = pipeline.execute() - self.assertEqual(0, exitcode, msg=out) - - self.assertEqual(1, project1.codebaseresources.count()) - self.assertEqual(94, project1.discoveredpackages.count()) - self.assertEqual(7, project1.discoveredpackages.vulnerable().count()) - self.assertEqual(20, project1.discovereddependencies.count()) +""" - def test_scanpipe_scan_integrations_load_sbom_cdxgen(self): - # Input file generated with: - # $ cdxgen alpine:3.17.0 --type docker --spec-version 1.6 --json-pretty \ - # --output cdxgen-alpine-3.17-sbom.json - input_location = self.data / "sca-integrations" / "cdxgen-alpine-3.17-sbom.json" +from pathlib import Path - pipeline_name = "load_sbom" - project1 = make_project() - project1.copy_input_from(input_location) +from django.test import TestCase - run = project1.add_pipeline(pipeline_name) - pipeline = run.make_pipeline_instance() +from scanpipe.pipes import benchmark +from scanpipe.tests import make_project - exitcode, out = pipeline.execute() - self.assertEqual(0, exitcode, msg=out) +# Mapping of SBOM filenames to expected counts. +# Each SBOM is stored under ``tests/data/sca-integrations/``. +# Keys are filenames, values are dicts with expected numbers of: +# - ``resources``: CodebaseResource +# - ``packages``: DiscoveredPackages +# - ``packages_vulnerable``: Vulnerable DiscoveredPackages +# - ``dependencies``: DiscoveredDependencies +# - ``purls``: The list of PURLs present in the SBOM +SCA_INTEGRATIONS_TEST_DATA = { + ### Anchore Grype + # $ grype -v -o cyclonedx-json \ + # --file anchore-alpine-3.17-sbom.json alpine:3.17.0 + "anchore-alpine-3.17-sbom.json": { + "resources": 1, + "packages": 94, + "packages_vulnerable": 7, + "dependencies": 20, + "purls": [ + "pkg:apk/alpine/alpine-baselayout-data@3.4.0-r0?arch=x86_64&distro=alpine-3.17.0&upstream=alpine-baselayout", + "pkg:apk/alpine/alpine-baselayout@3.4.0-r0?arch=x86_64&distro=alpine-3.17.0", + "pkg:apk/alpine/alpine-keys@2.4-r1?arch=x86_64&distro=alpine-3.17.0", + "pkg:apk/alpine/apk-tools@2.12.10-r1?arch=x86_64&distro=alpine-3.17.0", + "pkg:apk/alpine/busybox-binsh@1.35.0-r29?arch=x86_64&distro=alpine-3.17.0&upstream=busybox", + "pkg:apk/alpine/busybox@1.35.0-r29?arch=x86_64&distro=alpine-3.17.0", + "pkg:apk/alpine/ca-certificates-bundle@20220614-r2?arch=x86_64&distro=alpine-3.17.0&upstream=ca-certificates", + "pkg:apk/alpine/libc-utils@0.7.2-r3?arch=x86_64&distro=alpine-3.17.0&upstream=libc-dev", + "pkg:apk/alpine/libcrypto3@3.0.7-r0?arch=x86_64&distro=alpine-3.17.0&upstream=openssl", + "pkg:apk/alpine/libssl3@3.0.7-r0?arch=x86_64&distro=alpine-3.17.0&upstream=openssl", + "pkg:apk/alpine/musl-utils@1.2.3-r4?arch=x86_64&distro=alpine-3.17.0&upstream=musl", + "pkg:apk/alpine/musl@1.2.3-r4?arch=x86_64&distro=alpine-3.17.0", + "pkg:apk/alpine/scanelf@1.3.5-r1?arch=x86_64&distro=alpine-3.17.0&upstream=pax-utils", + "pkg:apk/alpine/ssl_client@1.35.0-r29?arch=x86_64&distro=alpine-3.17.0&upstream=busybox", + "pkg:apk/alpine/zlib@1.2.13-r0?arch=x86_64&distro=alpine-3.17.0", + "pkg:unknown/alpine@3.17.0", + "pkg:unknown/bin/busybox", + "pkg:unknown/etc/apk/keys/alpine-devel%40lists.alpinelinux.org-4a6a0840.rsa.pub", + "pkg:unknown/etc/apk/keys/alpine-devel%40lists.alpinelinux.org-5243ef4b.rsa.pub", + "pkg:unknown/etc/apk/keys/alpine-devel%40lists.alpinelinux.org-5261cecb.rsa.pub", + "pkg:unknown/etc/apk/keys/alpine-devel%40lists.alpinelinux.org-6165ee59.rsa.pub", + "pkg:unknown/etc/apk/keys/alpine-devel%40lists.alpinelinux.org-61666e3f.rsa.pub", + "pkg:unknown/etc/crontabs/root", + "pkg:unknown/etc/fstab", + "pkg:unknown/etc/group", + "pkg:unknown/etc/hostname", + "pkg:unknown/etc/hosts", + "pkg:unknown/etc/inittab", + "pkg:unknown/etc/logrotate.d/acpid", + "pkg:unknown/etc/modprobe.d/aliases.conf", + "pkg:unknown/etc/modprobe.d/blacklist.conf", + "pkg:unknown/etc/modprobe.d/i386.conf", + "pkg:unknown/etc/modprobe.d/kms.conf", + "pkg:unknown/etc/modules", + "pkg:unknown/etc/motd", + "pkg:unknown/etc/network/if-up.d/dad", + "pkg:unknown/etc/nsswitch.conf", + "pkg:unknown/etc/passwd", + "pkg:unknown/etc/profile", + "pkg:unknown/etc/profile.d/README", + "pkg:unknown/etc/profile.d/color_prompt.sh.disabled", + "pkg:unknown/etc/profile.d/locale.sh", + "pkg:unknown/etc/protocols", + "pkg:unknown/etc/securetty", + "pkg:unknown/etc/services", + "pkg:unknown/etc/shadow", + "pkg:unknown/etc/shells", + "pkg:unknown/etc/ssl/certs/ca-certificates.crt", + "pkg:unknown/etc/ssl/ct_log_list.cnf", + "pkg:unknown/etc/ssl/ct_log_list.cnf.dist", + "pkg:unknown/etc/ssl/misc/CA.pl", + "pkg:unknown/etc/ssl/misc/tsget.pl", + "pkg:unknown/etc/ssl/openssl.cnf", + "pkg:unknown/etc/ssl/openssl.cnf.dist", + "pkg:unknown/etc/sysctl.conf", + "pkg:unknown/etc/udhcpd.conf", + "pkg:unknown/lib/apk/db/installed", + "pkg:unknown/lib/ld-musl-x86_64.so.1", + "pkg:unknown/lib/libapk.so.3.12.0", + "pkg:unknown/lib/libcrypto.so.3", + "pkg:unknown/lib/libssl.so.3", + "pkg:unknown/lib/libz.so.1.2.13", + "pkg:unknown/lib/sysctl.d/00-alpine.conf", + "pkg:unknown/sbin/apk", + "pkg:unknown/sbin/ldconfig", + "pkg:unknown/usr/bin/getconf", + "pkg:unknown/usr/bin/getent", + "pkg:unknown/usr/bin/iconv", + "pkg:unknown/usr/bin/ldd", + "pkg:unknown/usr/bin/scanelf", + "pkg:unknown/usr/bin/ssl_client", + "pkg:unknown/usr/lib/engines-3/afalg.so", + "pkg:unknown/usr/lib/engines-3/capi.so", + "pkg:unknown/usr/lib/engines-3/loader_attic.so", + "pkg:unknown/usr/lib/engines-3/padlock.so", + "pkg:unknown/usr/lib/ossl-modules/legacy.so", + "pkg:unknown/usr/share/apk/keys/alpine-devel%40lists.alpinelinux.org-4a6a0840.rsa.pub", + "pkg:unknown/usr/share/apk/keys/alpine-devel%40lists.alpinelinux.org-5243ef4b.rsa.pub", + "pkg:unknown/usr/share/apk/keys/alpine-devel%40lists.alpinelinux.org-524d27bb.rsa.pub", + "pkg:unknown/usr/share/apk/keys/alpine-devel%40lists.alpinelinux.org-5261cecb.rsa.pub", + "pkg:unknown/usr/share/apk/keys/alpine-devel%40lists.alpinelinux.org-58199dcc.rsa.pub", + "pkg:unknown/usr/share/apk/keys/alpine-devel%40lists.alpinelinux.org-58cbb476.rsa.pub", + "pkg:unknown/usr/share/apk/keys/alpine-devel%40lists.alpinelinux.org-58e4f17d.rsa.pub", + "pkg:unknown/usr/share/apk/keys/alpine-devel%40lists.alpinelinux.org-5e69ca50.rsa.pub", + "pkg:unknown/usr/share/apk/keys/alpine-devel%40lists.alpinelinux.org-60ac2099.rsa.pub", + "pkg:unknown/usr/share/apk/keys/alpine-devel%40lists.alpinelinux.org-6165ee59.rsa.pub", + "pkg:unknown/usr/share/apk/keys/alpine-devel%40lists.alpinelinux.org-61666e3f.rsa.pub", + "pkg:unknown/usr/share/apk/keys/alpine-devel%40lists.alpinelinux.org-616a9724.rsa.pub", + "pkg:unknown/usr/share/apk/keys/alpine-devel%40lists.alpinelinux.org-616abc23.rsa.pub", + "pkg:unknown/usr/share/apk/keys/alpine-devel%40lists.alpinelinux.org-616ac3bc.rsa.pub", + "pkg:unknown/usr/share/apk/keys/alpine-devel%40lists.alpinelinux.org-616adfeb.rsa.pub", + "pkg:unknown/usr/share/apk/keys/alpine-devel%40lists.alpinelinux.org-616ae350.rsa.pub", + "pkg:unknown/usr/share/apk/keys/alpine-devel%40lists.alpinelinux.org-616db30d.rsa.pub", + "pkg:unknown/usr/share/udhcpc/default.script", + ], + }, + ### CycloneDX cdxgen + # $ cdxgen alpine:3.17.0 --type docker --spec-version 1.6 --json-pretty \ + # --output cdxgen-alpine-3.17-sbom.json + "cdxgen-alpine-3.17-sbom.json": { + "resources": 1, + "packages": 14, + "packages_vulnerable": 0, + "dependencies": 0, + "purls": [ + "pkg:generic/apk", + "pkg:generic/busybox", + "pkg:generic/getconf", + "pkg:generic/getent", + "pkg:generic/iconv", + "pkg:generic/ld-musl-x86_64.so.1", + "pkg:generic/ldconfig", + "pkg:generic/ldd", + "pkg:generic/libapk.so.3.12.0", + "pkg:generic/libcrypto.so.3", + "pkg:generic/libssl.so.3", + "pkg:generic/libz.so.1.2.13", + "pkg:generic/scanelf", + "pkg:generic/ssl_client", + ], + }, + ### OWASP dep-scan + # $ depscan --src alpine:3.17.0 --type docker + "depscan-alpine-3.17-sbom.json": { + "resources": 1, + "packages": 33, + "packages_vulnerable": 3, + "dependencies": 20, + "purls": [ + "pkg:apk/alpine/alpine-baselayout-data@3.4.0-r0?arch=x86_64&distro=alpine-3.17.0&distro_name=alpine-3.17", + "pkg:apk/alpine/alpine-baselayout@3.4.0-r0?arch=x86_64&distro=alpine-3.17.0&distro_name=alpine-3.17", + "pkg:apk/alpine/alpine-keys@2.4-r1?arch=x86_64&distro=alpine-3.17.0&distro_name=alpine-3.17", + "pkg:apk/alpine/apk-tools@2.12.10-r1?arch=x86_64&distro=alpine-3.17.0&distro_name=alpine-3.17", + "pkg:apk/alpine/busybox-binsh@1.35.0-r29?arch=x86_64&distro=alpine-3.17.0&distro_name=alpine-3.17", + "pkg:apk/alpine/busybox@1.35.0-r29?arch=x86_64&distro=alpine-3.17.0&distro_name=alpine-3.17", + "pkg:apk/alpine/ca-certificates-bundle@20220614-r2?arch=x86_64&distro=alpine-3.17.0&distro_name=alpine-3.17", + "pkg:apk/alpine/ca-certificates@20220614-r2?arch=x86_64&distro=alpine-3.17.0&distro_name=alpine-3.17", + "pkg:apk/alpine/libc-dev@0.7.2-r3?arch=x86_64&distro=alpine-3.17.0&distro_name=alpine-3.17", + "pkg:apk/alpine/libc-utils@0.7.2-r3?arch=x86_64&distro=alpine-3.17.0&distro_name=alpine-3.17", + "pkg:apk/alpine/libcrypto3@3.0.7-r0?arch=x86_64&distro=alpine-3.17.0&distro_name=alpine-3.17", + "pkg:apk/alpine/libssl3@3.0.7-r0?arch=x86_64&distro=alpine-3.17.0&distro_name=alpine-3.17", + "pkg:apk/alpine/musl-utils@1.2.3-r4?arch=x86_64&distro=alpine-3.17.0&distro_name=alpine-3.17", + "pkg:apk/alpine/musl@1.2.3-r4?arch=x86_64&distro=alpine-3.17.0&distro_name=alpine-3.17", + "pkg:apk/alpine/openssl@3.0.7-r0?arch=x86_64&distro=alpine-3.17.0&distro_name=alpine-3.17", + "pkg:apk/alpine/pax-utils@1.3.5-r1?arch=x86_64&distro=alpine-3.17.0&distro_name=alpine-3.17", + "pkg:apk/alpine/scanelf@1.3.5-r1?arch=x86_64&distro=alpine-3.17.0&distro_name=alpine-3.17", + "pkg:apk/alpine/ssl_client@1.35.0-r29?arch=x86_64&distro=alpine-3.17.0&distro_name=alpine-3.17", + "pkg:apk/alpine/zlib@1.2.13-r0?arch=x86_64&distro=alpine-3.17.0&distro_name=alpine-3.17", + "pkg:generic/apk", + "pkg:generic/busybox", + "pkg:generic/getconf", + "pkg:generic/getent", + "pkg:generic/iconv", + "pkg:generic/ld-musl-x86_64.so.1", + "pkg:generic/ldconfig", + "pkg:generic/ldd", + "pkg:generic/libapk.so.3.12.0", + "pkg:generic/libcrypto.so.3", + "pkg:generic/libssl.so.3", + "pkg:generic/libz.so.1.2.13", + "pkg:generic/scanelf", + "pkg:generic/ssl_client", + ], + }, + ### OSV-Scanner + # $ osv-scanner scan image alpine:3.17.0 \ + # --all-packages \ + # --format spdx-2-3 \ + # --output osv-scanner-alpine-3.17-sbom.spdx.json + "osv-scanner-alpine-3.17-sbom.spdx.json": { + "resources": 1, + "packages": 16, + "packages_vulnerable": 0, + "dependencies": 15, + "purls": [ + "pkg:apk/alpine/alpine-baselayout-data@3.4.0-r0?arch=x86_64&distro=3.17.0&origin=alpine-baselayout", + "pkg:apk/alpine/alpine-baselayout@3.4.0-r0?arch=x86_64&distro=3.17.0&origin=alpine-baselayout", + "pkg:apk/alpine/alpine-keys@2.4-r1?arch=x86_64&distro=3.17.0&origin=alpine-keys", + "pkg:apk/alpine/apk-tools@2.12.10-r1?arch=x86_64&distro=3.17.0&origin=apk-tools", + "pkg:apk/alpine/busybox-binsh@1.35.0-r29?arch=x86_64&distro=3.17.0&origin=busybox", + "pkg:apk/alpine/busybox@1.35.0-r29?arch=x86_64&distro=3.17.0&origin=busybox", + "pkg:apk/alpine/ca-certificates-bundle@20220614-r2?arch=x86_64&distro=3.17.0&origin=ca-certificates", + "pkg:apk/alpine/libc-utils@0.7.2-r3?arch=x86_64&distro=3.17.0&origin=libc-dev", + "pkg:apk/alpine/libcrypto3@3.0.7-r0?arch=x86_64&distro=3.17.0&origin=openssl", + "pkg:apk/alpine/libssl3@3.0.7-r0?arch=x86_64&distro=3.17.0&origin=openssl", + "pkg:apk/alpine/musl-utils@1.2.3-r4?arch=x86_64&distro=3.17.0&origin=musl", + "pkg:apk/alpine/musl@1.2.3-r4?arch=x86_64&distro=3.17.0&origin=musl", + "pkg:apk/alpine/scanelf@1.3.5-r1?arch=x86_64&distro=3.17.0&origin=pax-utils", + "pkg:apk/alpine/ssl_client@1.35.0-r29?arch=x86_64&distro=3.17.0&origin=busybox", + "pkg:apk/alpine/zlib@1.2.13-r0?arch=x86_64&distro=3.17.0&origin=zlib", + "pkg:unknown/main@0", + ], + }, + # Example file from osv-scanner documentation: + # https://google.github.io/osv-scanner/output/#cyclonedx + "osv-scanner-vulns-sbom.cdx.json": { + "resources": 1, + "packages": 3, + "packages_vulnerable": 1, + "dependencies": 0, + "purls": [ + "pkg:composer/league/flysystem@1.0.8", + "pkg:npm/has-flag@4.0.0", + "pkg:npm/wrappy@1.0.2", + ], + }, + ### SBOM Tool + # $ sbom-tool generate -di alpine:3.17.0 \ + # -pn DockerImage -pv 1.0.0 -ps Company -nsb https://sbom.company.com + "sbom-tool-alpine-3.17-sbom.spdx.json": { + "resources": 1, + "packages": 16, + "packages_vulnerable": 0, + "dependencies": 15, + "purls": [ + "pkg:swid/Company/sbom.company.com/DockerImage@1.0.0?tag_id=60e3f440-f9a8-449e-b516-da3049700fff", + "pkg:unknown/alpine-baselayout-data@3.4.0-r0", + "pkg:unknown/alpine-baselayout@3.4.0-r0", + "pkg:unknown/alpine-keys@2.4-r1", + "pkg:unknown/apk-tools@2.12.10-r1", + "pkg:unknown/busybox-binsh@1.35.0-r29", + "pkg:unknown/busybox@1.35.0-r29", + "pkg:unknown/ca-certificates-bundle@20220614-r2", + "pkg:unknown/libc-utils@0.7.2-r3", + "pkg:unknown/libcrypto3@3.0.7-r0", + "pkg:unknown/libssl3@3.0.7-r0", + "pkg:unknown/musl-utils@1.2.3-r4", + "pkg:unknown/musl@1.2.3-r4", + "pkg:unknown/scanelf@1.3.5-r1", + "pkg:unknown/ssl_client@1.35.0-r29", + "pkg:unknown/zlib@1.2.13-r0", + ], + }, + ### Trivy + # $ trivy image --scanners vuln,license --format cyclonedx \ + # --output trivy-alpine-3.17-sbom.json alpine:3.17.0 + "trivy-alpine-3.17-sbom.json": { + "resources": 1, + "packages": 16, + "packages_vulnerable": 7, + "dependencies": 25, + "purls": [ + "pkg:apk/alpine/alpine-baselayout-data@3.4.0-r0?arch=x86_64&distro=3.17.0", + "pkg:apk/alpine/alpine-baselayout@3.4.0-r0?arch=x86_64&distro=3.17.0", + "pkg:apk/alpine/alpine-keys@2.4-r1?arch=x86_64&distro=3.17.0", + "pkg:apk/alpine/apk-tools@2.12.10-r1?arch=x86_64&distro=3.17.0", + "pkg:apk/alpine/busybox-binsh@1.35.0-r29?arch=x86_64&distro=3.17.0", + "pkg:apk/alpine/busybox@1.35.0-r29?arch=x86_64&distro=3.17.0", + "pkg:apk/alpine/ca-certificates-bundle@20220614-r2?arch=x86_64&distro=3.17.0", + "pkg:apk/alpine/libc-utils@0.7.2-r3?arch=x86_64&distro=3.17.0", + "pkg:apk/alpine/libcrypto3@3.0.7-r0?arch=x86_64&distro=3.17.0", + "pkg:apk/alpine/libssl3@3.0.7-r0?arch=x86_64&distro=3.17.0", + "pkg:apk/alpine/musl-utils@1.2.3-r4?arch=x86_64&distro=3.17.0", + "pkg:apk/alpine/musl@1.2.3-r4?arch=x86_64&distro=3.17.0", + "pkg:apk/alpine/scanelf@1.3.5-r1?arch=x86_64&distro=3.17.0", + "pkg:apk/alpine/ssl_client@1.35.0-r29?arch=x86_64&distro=3.17.0", + "pkg:apk/alpine/zlib@1.2.13-r0?arch=x86_64&distro=3.17.0", + "pkg:unknown/alpine@3.17.0", + ], + }, +} - self.assertEqual(1, project1.codebaseresources.count()) - self.assertEqual(14, project1.discoveredpackages.count()) - self.assertEqual(0, project1.discoveredpackages.vulnerable().count()) - self.assertEqual(0, project1.discovereddependencies.count()) - def test_scanpipe_scan_integrations_load_sbom_depscan(self): - # Input file generated with: - # $ depscan --src alpine:3.17.0 --type docker - input_location = ( - self.data / "sca-integrations" / "depscan-alpine-3.17-sbom.json" - ) +class ScanPipeSCAIntegrationsTest(TestCase): + """ + Run consistency checks across all SBOM integration test files. - pipeline_name = "load_sbom" - project1 = make_project() - project1.copy_input_from(input_location) + For each SBOM listed in ``SCA_INTEGRATIONS_TEST_DATA``, this test: + - Loads the SBOM into a temporary ScanCode.io project. + - Executes the ``load_sbom`` pipeline. + - Verifies that the number of resources, packages, vulnerable packages, + and dependencies match the expected values. + """ - run = project1.add_pipeline(pipeline_name) - pipeline = run.make_pipeline_instance() + data = Path(__file__).parent / "data" - exitcode, out = pipeline.execute() - self.assertEqual(0, exitcode, msg=out) + def test_scanpipe_sca_integrations_tools(self): + """Loop through all SBOM files and run integration checks.""" + for sbom_filename, expected_results in SCA_INTEGRATIONS_TEST_DATA.items(): + self._test_scanpipe_sca_integrations_tool(sbom_filename, expected_results) - self.assertEqual(1, project1.codebaseresources.count()) - self.assertEqual(33, project1.discoveredpackages.count()) - self.assertEqual(3, project1.discoveredpackages.vulnerable().count()) - self.assertEqual(20, project1.discovereddependencies.count()) - - def test_scanpipe_scan_integrations_load_sbom_sbomtool(self): - # Input file generated with: - # $ sbom-tool generate -di alpine:3.17.0 \ - # -pn DockerImage -pv 1.0.0 -ps Company -nsb https://sbom.company.com - input_location = ( - self.data / "sca-integrations" / "sbom-tool-alpine-3.17-sbom.spdx.json" - ) + def _test_scanpipe_sca_integrations_tool(self, sbom_filename, expected_results): + """Run a single SBOM integration test.""" + input_location = self.data / "sca-integrations" / sbom_filename - pipeline_name = "load_sbom" - project1 = make_project() - project1.copy_input_from(input_location) + # Create a fresh project and load the SBOM into it + project = make_project() + project.copy_input_from(input_location) - run = project1.add_pipeline(pipeline_name) + run = project.add_pipeline(pipeline_name="load_sbom") pipeline = run.make_pipeline_instance() exitcode, out = pipeline.execute() + # Ensure the SBOM is properly loaded self.assertEqual(0, exitcode, msg=out) - self.assertEqual(1, project1.codebaseresources.count()) - self.assertEqual(16, project1.discoveredpackages.count()) - self.assertEqual(0, project1.discoveredpackages.vulnerable().count()) - self.assertEqual(15, project1.discovereddependencies.count()) - - def test_scanpipe_scan_integrations_load_sbom_osv_scanner(self): - # Input file generated with: - # $ osv-scanner scan image alpine:3.17.0 \ - # --all-packages \ - # --format spdx-2-3 \ - # --output osv-scanner-alpine-3.17-sbom.spdx.json - input_location = ( - self.data / "sca-integrations" / "osv-scanner-alpine-3.17-sbom.spdx.json" + # Verify resource, package, vulnerability, and dependency counts + self.assertEqual( + expected_results["resources"], + project.codebaseresources.count(), + msg=sbom_filename, ) - - pipeline_name = "load_sbom" - project1 = make_project() - project1.copy_input_from(input_location) - - run = project1.add_pipeline(pipeline_name) - pipeline = run.make_pipeline_instance() - - exitcode, out = pipeline.execute() - self.assertEqual(0, exitcode, msg=out) - - self.assertEqual(1, project1.codebaseresources.count()) - self.assertEqual(16, project1.discoveredpackages.count()) - self.assertEqual(0, project1.discoveredpackages.vulnerable().count()) - self.assertEqual(15, project1.discovereddependencies.count()) - - def test_scanpipe_scan_integrations_load_sbom_osv_scanner_cdx_vulnerabilities(self): - # Input file taken from: https://google.github.io/osv-scanner/output/#cyclonedx - input_location = ( - self.data / "sca-integrations" / "osv-scanner-vulns-sbom.cdx.json" + self.assertEqual( + expected_results["packages"], + project.discoveredpackages.count(), + msg=sbom_filename, + ) + self.assertEqual( + expected_results["packages_vulnerable"], + project.discoveredpackages.vulnerable().count(), + msg=sbom_filename, + ) + self.assertEqual( + expected_results["dependencies"], + project.discovereddependencies.count(), + msg=sbom_filename, ) - pipeline_name = "load_sbom" - project1 = make_project() - project1.copy_input_from(input_location) - - run = project1.add_pipeline(pipeline_name) - pipeline = run.make_pipeline_instance() - - exitcode, out = pipeline.execute() - self.assertEqual(0, exitcode, msg=out) - - self.assertEqual(1, project1.codebaseresources.count()) - self.assertEqual(3, project1.discoveredpackages.count()) - self.assertEqual(1, project1.discoveredpackages.vulnerable().count()) - self.assertEqual(0, project1.discovereddependencies.count()) + # Verify that all the expected PURLs are present in the project + expected_purls = expected_results.get("purls", []) + if expected_purls: + purls_diff = benchmark.compare_purls(project, expected_purls) + formatted_diff = "\n".join(purls_diff) + self.assertFalse(purls_diff, msg=f"\n{sbom_filename}\n{formatted_diff}")