Skip to content

Commit 56b12bb

Browse files
tdrueztdruez
committed
Refactor the SCA test module to simplify addition of new tools #1726
Signed-off-by: tdruez <[email protected]>
1 parent 3d0e1bd commit 56b12bb

File tree

1 file changed

+141
-134
lines changed

1 file changed

+141
-134
lines changed

‎scanpipe/tests/test_sca_integrations.py‎

Lines changed: 141 additions & 134 deletions
Original file line numberDiff line numberDiff line change
@@ -20,168 +20,175 @@
2020
# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
2121
# Visit https://github.com/nexB/scancode.io for support and download.
2222

23-
from pathlib import Path
23+
"""
24+
Test suite for validating ScanCode.io integrations with third-party SCA SBOM tools.
2425
25-
from django.test import TestCase
26+
Each test ensures that SBOM files generated by tools such as Anchore Grype,
27+
CycloneDX (cdxgen), osv-scanner, Trivy, SBOM Tool, and more.
2628
27-
from scanpipe.tests import make_project
29+
Adding new test data
30+
====================
2831
32+
1. Generate an SBOM with the tool you want to test. For example with Trivy:
2933
30-
class ScanPipeSCAIntegrationsTest(TestCase):
31-
data = Path(__file__).parent / "data"
34+
trivy image --scanners vuln,license --format cyclonedx \
35+
--output trivy-alpine-3.17-sbom.json alpine:3.17.0
3236
33-
def test_scanpipe_scan_integrations_load_sbom_trivy(self):
34-
# Input file generated with:
35-
# $ trivy image --scanners vuln,license --format cyclonedx \
36-
# --output trivy-alpine-3.17-sbom.json alpine:3.17.0
37-
input_location = self.data / "sca-integrations" / "trivy-alpine-3.17-sbom.json"
37+
2. Save the SBOM file under:
38+
tests/data/sca-integrations/
3839
39-
pipeline_name = "load_sbom"
40-
project1 = make_project()
41-
project1.copy_input_from(input_location)
40+
3. Add expected counts for that SBOM to ``SCA_INTEGRATIONS_TEST_DATA`` below.
4241
43-
run = project1.add_pipeline(pipeline_name)
44-
pipeline = run.make_pipeline_instance()
42+
Example:
43+
"trivy-alpine-3.17-sbom.json": {
44+
"resources": 1,
45+
"packages": 16,
46+
"packages_vulnerable": 7,
47+
"dependencies": 25,
48+
},
4549
46-
exitcode, out = pipeline.execute()
47-
self.assertEqual(0, exitcode, msg=out)
50+
Tip: run the test once without expected values, check the counts from the
51+
failure messages, then update the dictionary accordingly.
4852
49-
self.assertEqual(1, project1.codebaseresources.count())
50-
self.assertEqual(16, project1.discoveredpackages.count())
51-
self.assertEqual(7, project1.discoveredpackages.vulnerable().count())
52-
self.assertEqual(25, project1.discovereddependencies.count())
53-
54-
def test_scanpipe_scan_integrations_load_sbom_anchore(self):
55-
# Input file generated with:
56-
# $ grype -v -o cyclonedx-json \
57-
# --file anchore-alpine-3.17-sbom.json alpine:3.17.0
58-
input_location = (
59-
self.data / "sca-integrations" / "anchore-alpine-3.17-sbom.json"
60-
)
53+
4. Run the test suite:
6154
62-
pipeline_name = "load_sbom"
63-
project1 = make_project()
64-
project1.copy_input_from(input_location)
65-
66-
run = project1.add_pipeline(pipeline_name)
67-
pipeline = run.make_pipeline_instance()
55+
./manage.py test scanpipe.tests.test_sca_integrations
6856
69-
exitcode, out = pipeline.execute()
70-
self.assertEqual(0, exitcode, msg=out)
57+
5. Commit both the SBOM file and dictionary entry.
7158
72-
self.assertEqual(1, project1.codebaseresources.count())
73-
self.assertEqual(94, project1.discoveredpackages.count())
74-
self.assertEqual(7, project1.discoveredpackages.vulnerable().count())
75-
self.assertEqual(20, project1.discovereddependencies.count())
59+
"""
7660

77-
def test_scanpipe_scan_integrations_load_sbom_cdxgen(self):
78-
# Input file generated with:
79-
# $ cdxgen alpine:3.17.0 --type docker --spec-version 1.6 --json-pretty \
80-
# --output cdxgen-alpine-3.17-sbom.json
81-
input_location = self.data / "sca-integrations" / "cdxgen-alpine-3.17-sbom.json"
61+
from pathlib import Path
8262

83-
pipeline_name = "load_sbom"
84-
project1 = make_project()
85-
project1.copy_input_from(input_location)
63+
from django.test import TestCase
8664

87-
run = project1.add_pipeline(pipeline_name)
88-
pipeline = run.make_pipeline_instance()
65+
from scanpipe.tests import make_project
8966

90-
exitcode, out = pipeline.execute()
91-
self.assertEqual(0, exitcode, msg=out)
67+
# Mapping of SBOM filenames to expected counts.
68+
# Each SBOM is stored under ``tests/data/sca-integrations/``.
69+
# Keys are filenames, values are dicts with expected numbers of:
70+
# - ``resources``: CodebaseResource
71+
# - ``packages``: DiscoveredPackages
72+
# - ``packages_vulnerable``: Vulnerable DiscoveredPackages
73+
# - ``dependencies``: DiscoveredDependencies
74+
SCA_INTEGRATIONS_TEST_DATA = {
75+
### Anchore Grype
76+
# $ grype -v -o cyclonedx-json \
77+
# --file anchore-alpine-3.17-sbom.json alpine:3.17.0
78+
"anchore-alpine-3.17-sbom.json": {
79+
"resources": 1,
80+
"packages": 94,
81+
"packages_vulnerable": 7,
82+
"dependencies": 20,
83+
},
84+
### CycloneDX cdxgen
85+
# $ cdxgen alpine:3.17.0 --type docker --spec-version 1.6 --json-pretty \
86+
# --output cdxgen-alpine-3.17-sbom.json
87+
"cdxgen-alpine-3.17-sbom.json": {
88+
"resources": 1,
89+
"packages": 14,
90+
"packages_vulnerable": 0,
91+
"dependencies": 0,
92+
},
93+
### OWASP dep-scan
94+
# $ depscan --src alpine:3.17.0 --type docker
95+
"depscan-alpine-3.17-sbom.json": {
96+
"resources": 1,
97+
"packages": 33,
98+
"packages_vulnerable": 3,
99+
"dependencies": 20,
100+
},
101+
### OSV-Scanner
102+
# $ osv-scanner scan image alpine:3.17.0 \
103+
# --all-packages \
104+
# --format spdx-2-3 \
105+
# --output osv-scanner-alpine-3.17-sbom.spdx.json
106+
"osv-scanner-alpine-3.17-sbom.spdx.json": {
107+
"resources": 1,
108+
"packages": 16,
109+
"packages_vulnerable": 0,
110+
"dependencies": 15,
111+
},
112+
# Example file from osv-scanner documentation:
113+
# https://google.github.io/osv-scanner/output/#cyclonedx
114+
"osv-scanner-vulns-sbom.cdx.json": {
115+
"resources": 1,
116+
"packages": 3,
117+
"packages_vulnerable": 1,
118+
"dependencies": 0,
119+
},
120+
### SBOM Tool
121+
# $ sbom-tool generate -di alpine:3.17.0 \
122+
# -pn DockerImage -pv 1.0.0 -ps Company -nsb https://sbom.company.com
123+
"sbom-tool-alpine-3.17-sbom.spdx.json": {
124+
"resources": 1,
125+
"packages": 16,
126+
"packages_vulnerable": 0,
127+
"dependencies": 15,
128+
},
129+
### Trivy
130+
# $ trivy image --scanners vuln,license --format cyclonedx \
131+
# --output trivy-alpine-3.17-sbom.json alpine:3.17.0
132+
"trivy-alpine-3.17-sbom.json": {
133+
"resources": 1,
134+
"packages": 16,
135+
"packages_vulnerable": 7,
136+
"dependencies": 25,
137+
},
138+
}
92139

93-
self.assertEqual(1, project1.codebaseresources.count())
94-
self.assertEqual(14, project1.discoveredpackages.count())
95-
self.assertEqual(0, project1.discoveredpackages.vulnerable().count())
96-
self.assertEqual(0, project1.discovereddependencies.count())
97140

98-
def test_scanpipe_scan_integrations_load_sbom_depscan(self):
99-
# Input file generated with:
100-
# $ depscan --src alpine:3.17.0 --type docker
101-
input_location = (
102-
self.data / "sca-integrations" / "depscan-alpine-3.17-sbom.json"
103-
)
141+
class ScanPipeSCAIntegrationsTest(TestCase):
142+
"""
143+
Run consistency checks across all SBOM integration test files.
104144
105-
pipeline_name = "load_sbom"
106-
project1 = make_project()
107-
project1.copy_input_from(input_location)
145+
For each SBOM listed in ``SCA_INTEGRATIONS_TEST_DATA``, this test:
146+
- Loads the SBOM into a temporary ScanCode.io project.
147+
- Executes the ``load_sbom`` pipeline.
148+
- Verifies that the number of resources, packages, vulnerable packages,
149+
and dependencies match the expected values.
150+
"""
108151

109-
run = project1.add_pipeline(pipeline_name)
110-
pipeline = run.make_pipeline_instance()
152+
data = Path(__file__).parent / "data"
111153

112-
exitcode, out = pipeline.execute()
113-
self.assertEqual(0, exitcode, msg=out)
154+
def test_scanpipe_sca_integrations_tools(self):
155+
"""Loop through all SBOM files and run integration checks."""
156+
for sbom_filename, expected_results in SCA_INTEGRATIONS_TEST_DATA.items():
157+
self._test_scanpipe_sca_integrations_tool(sbom_filename, expected_results)
114158

115-
self.assertEqual(1, project1.codebaseresources.count())
116-
self.assertEqual(33, project1.discoveredpackages.count())
117-
self.assertEqual(3, project1.discoveredpackages.vulnerable().count())
118-
self.assertEqual(20, project1.discovereddependencies.count())
119-
120-
def test_scanpipe_scan_integrations_load_sbom_sbomtool(self):
121-
# Input file generated with:
122-
# $ sbom-tool generate -di alpine:3.17.0 \
123-
# -pn DockerImage -pv 1.0.0 -ps Company -nsb https://sbom.company.com
124-
input_location = (
125-
self.data / "sca-integrations" / "sbom-tool-alpine-3.17-sbom.spdx.json"
126-
)
159+
def _test_scanpipe_sca_integrations_tool(self, sbom_filename, expected_results):
160+
"""Run a single SBOM integration test."""
161+
input_location = self.data / "sca-integrations" / sbom_filename
127162

128-
pipeline_name = "load_sbom"
129-
project1 = make_project()
130-
project1.copy_input_from(input_location)
163+
# Create a fresh project and load the SBOM into it
164+
project = make_project()
165+
project.copy_input_from(input_location)
131166

132-
run = project1.add_pipeline(pipeline_name)
167+
run = project.add_pipeline(pipeline_name="load_sbom")
133168
pipeline = run.make_pipeline_instance()
134169

135170
exitcode, out = pipeline.execute()
171+
# Ensure the SBOM is properly loaded
136172
self.assertEqual(0, exitcode, msg=out)
137173

138-
self.assertEqual(1, project1.codebaseresources.count())
139-
self.assertEqual(16, project1.discoveredpackages.count())
140-
self.assertEqual(0, project1.discoveredpackages.vulnerable().count())
141-
self.assertEqual(15, project1.discovereddependencies.count())
142-
143-
def test_scanpipe_scan_integrations_load_sbom_osv_scanner(self):
144-
# Input file generated with:
145-
# $ osv-scanner scan image alpine:3.17.0 \
146-
# --all-packages \
147-
# --format spdx-2-3 \
148-
# --output osv-scanner-alpine-3.17-sbom.spdx.json
149-
input_location = (
150-
self.data / "sca-integrations" / "osv-scanner-alpine-3.17-sbom.spdx.json"
174+
# Verify resource, package, vulnerability, and dependency counts
175+
self.assertEqual(
176+
expected_results["resources"],
177+
project.codebaseresources.count(),
178+
msg=sbom_filename,
151179
)
152-
153-
pipeline_name = "load_sbom"
154-
project1 = make_project()
155-
project1.copy_input_from(input_location)
156-
157-
run = project1.add_pipeline(pipeline_name)
158-
pipeline = run.make_pipeline_instance()
159-
160-
exitcode, out = pipeline.execute()
161-
self.assertEqual(0, exitcode, msg=out)
162-
163-
self.assertEqual(1, project1.codebaseresources.count())
164-
self.assertEqual(16, project1.discoveredpackages.count())
165-
self.assertEqual(0, project1.discoveredpackages.vulnerable().count())
166-
self.assertEqual(15, project1.discovereddependencies.count())
167-
168-
def test_scanpipe_scan_integrations_load_sbom_osv_scanner_cdx_vulnerabilities(self):
169-
# Input file taken from: https://google.github.io/osv-scanner/output/#cyclonedx
170-
input_location = (
171-
self.data / "sca-integrations" / "osv-scanner-vulns-sbom.cdx.json"
180+
self.assertEqual(
181+
expected_results["packages"],
182+
project.discoveredpackages.count(),
183+
msg=sbom_filename,
184+
)
185+
self.assertEqual(
186+
expected_results["packages_vulnerable"],
187+
project.discoveredpackages.vulnerable().count(),
188+
msg=sbom_filename,
189+
)
190+
self.assertEqual(
191+
expected_results["dependencies"],
192+
project.discovereddependencies.count(),
193+
msg=sbom_filename,
172194
)
173-
174-
pipeline_name = "load_sbom"
175-
project1 = make_project()
176-
project1.copy_input_from(input_location)
177-
178-
run = project1.add_pipeline(pipeline_name)
179-
pipeline = run.make_pipeline_instance()
180-
181-
exitcode, out = pipeline.execute()
182-
self.assertEqual(0, exitcode, msg=out)
183-
184-
self.assertEqual(1, project1.codebaseresources.count())
185-
self.assertEqual(3, project1.discoveredpackages.count())
186-
self.assertEqual(1, project1.discoveredpackages.vulnerable().count())
187-
self.assertEqual(0, project1.discovereddependencies.count())

0 commit comments

Comments
 (0)