diff --git a/.github/workflows/sca-integration-cyclonedx-gomod.yml b/.github/workflows/sca-integration-cyclonedx-gomod.yml index 3dd873ac36..bbbf724f7c 100644 --- a/.github/workflows/sca-integration-cyclonedx-gomod.yml +++ b/.github/workflows/sca-integration-cyclonedx-gomod.yml @@ -13,7 +13,6 @@ on: schedule: # Run once a week (every 7 days) at 00:00 UTC on Sunday - cron: "0 0 * * 0" - pull_request: permissions: contents: read diff --git a/scanpipe/pipes/docker.py b/scanpipe/pipes/docker.py index 075500d465..aacfe44bd6 100644 --- a/scanpipe/pipes/docker.py +++ b/scanpipe/pipes/docker.py @@ -169,8 +169,24 @@ def get_layer_tag(image_id, layer_id, layer_index, id_length=6): return f"img-{short_image_id}-layer-{layer_index:02}-{short_layer_id}" -def create_codebase_resources(project, image): - """Create the CodebaseResource for an `image` in a `project`.""" +def create_codebase_resources(project, image: Image) -> None: + """ + Create codebase resources for the provided image and its layers. + + Creates a codebase resource for the extracted image root directory and each + extracted layer directory, ensuring the structure is properly indexed for tree + rendering. + + Args: + project: The project instance. + image: The image object with the extracted_location attribute. + + """ + pipes.make_codebase_resource( + project=project, + location=str(project.codebase_path / Path(image.extracted_location).name), + ) + for layer_index, layer in enumerate(image.layers, start=1): layer_tag = get_layer_tag(image.image_id, layer.layer_id, layer_index) @@ -182,6 +198,17 @@ def create_codebase_resources(project, image): tag=layer_tag, ) + layer_data = layer.to_dict() + layer_data.pop("extracted_location", None) + layer_data.pop("archive_location", None) + pipes.make_codebase_resource( + project=project, + location=str(layer.extracted_location), + tag=layer_tag, + # Store the layer data in the extra_data for display in the UI + extra_data={"layer": layer_data}, + ) + def create_system_package(project, purl, package, layer, layer_tag): """Create system package and related resources.""" diff --git a/scanpipe/tests/data/docker/alpine_3_15_4_scan_codebase.json b/scanpipe/tests/data/docker/alpine_3_15_4_scan_codebase.json index 9236a5ff16..39990fd614 100644 --- a/scanpipe/tests/data/docker/alpine_3_15_4_scan_codebase.json +++ b/scanpipe/tests/data/docker/alpine_3_15_4_scan_codebase.json @@ -1574,6 +1574,94 @@ ], "dependencies": [], "files": [ + { + "path": "alpine_3_15_4.tar.gz-extract", + "type": "directory", + "name": "alpine_3_15_4.tar.gz-extract", + "status": "scanned", + "for_packages": [], + "tag": "", + "extension": ".tar.gz-extract", + "programming_language": "", + "detected_license_expression": "", + "detected_license_expression_spdx": "", + "license_detections": [], + "license_clues": [], + "percentage_of_license_text": null, + "copyrights": [], + "holders": [], + "authors": [], + "package_data": [], + "emails": [], + "urls": [], + "md5": "", + "sha1": "", + "sha256": "", + "sha512": "", + "sha1_git": "", + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_legal": false, + "is_manifest": false, + "is_readme": false, + "is_top_level": false, + "is_key_file": false, + "extra_data": {} + }, + { + "path": "alpine_3_15_4.tar.gz-extract/40e48c8ef2450e6a9e8d50b846a58ede43f1b01dd351d2bdd7dca14c5c033f20", + "type": "directory", + "name": "40e48c8ef2450e6a9e8d50b846a58ede43f1b01dd351d2bdd7dca14c5c033f20", + "status": "scanned", + "for_packages": [], + "tag": "img-06c7c4-layer-01-40e48c", + "extension": "", + "programming_language": "", + "detected_license_expression": "", + "detected_license_expression_spdx": "", + "license_detections": [], + "license_clues": [], + "percentage_of_license_text": null, + "copyrights": [], + "holders": [], + "authors": [], + "package_data": [], + "emails": [], + "urls": [], + "md5": "", + "sha1": "", + "sha256": "", + "sha512": "", + "sha1_git": "", + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_legal": false, + "is_manifest": false, + "is_readme": false, + "is_top_level": false, + "is_key_file": false, + "extra_data": { + "layer": { + "os": null, + "author": null, + "labels": [], + "sha256": "40e48c8ef2450e6a9e8d50b846a58ede43f1b01dd351d2bdd7dca14c5c033f20", + "comment": null, + "created": "2022-04-05T00:19:59.790636867Z", + "variant": null, + "layer_id": "40e48c8ef2450e6a9e8d50b846a58ede43f1b01dd351d2bdd7dca14c5c033f20", + "created_by": "/bin/sh -c #(nop) ADD file:5d673d25da3a14ce1f6cf66e4c7fd4f4b85a3759a9d93efb3fd9ff852b5b56e4 in / ", + "os_version": null, + "architecture": null, + "docker_version": null, + "is_empty_layer": false + } + } + }, { "path": "alpine_3_15_4.tar.gz-extract/40e48c8ef2450e6a9e8d50b846a58ede43f1b01dd351d2bdd7dca14c5c033f20/bin", "type": "directory", diff --git a/scanpipe/tests/data/docker/centos_scan_codebase.json b/scanpipe/tests/data/docker/centos_scan_codebase.json index 2e7225d8b5..770d4cf5cd 100644 --- a/scanpipe/tests/data/docker/centos_scan_codebase.json +++ b/scanpipe/tests/data/docker/centos_scan_codebase.json @@ -191354,6 +191354,94 @@ ], "dependencies": [], "files": [ + { + "path": "centos.tar.gz-extract", + "type": "directory", + "name": "centos.tar.gz-extract", + "status": "scanned", + "for_packages": [], + "tag": "", + "extension": ".tar.gz-extract", + "programming_language": "", + "detected_license_expression": "", + "detected_license_expression_spdx": "", + "license_detections": [], + "license_clues": [], + "percentage_of_license_text": null, + "copyrights": [], + "holders": [], + "authors": [], + "package_data": [], + "emails": [], + "urls": [], + "md5": "", + "sha1": "", + "sha256": "", + "sha512": "", + "sha1_git": "", + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_legal": false, + "is_manifest": false, + "is_readme": false, + "is_top_level": false, + "is_key_file": false, + "extra_data": {} + }, + { + "path": "centos.tar.gz-extract/a10cf747c363a52be048f884c084a25e03280d54a7ac02e17dbd8c5ad160e9bd", + "type": "directory", + "name": "a10cf747c363a52be048f884c084a25e03280d54a7ac02e17dbd8c5ad160e9bd", + "status": "scanned", + "for_packages": [], + "tag": "img-c967b7-layer-01-a10cf7", + "extension": "", + "programming_language": "", + "detected_license_expression": "", + "detected_license_expression_spdx": "", + "license_detections": [], + "license_clues": [], + "percentage_of_license_text": null, + "copyrights": [], + "holders": [], + "authors": [], + "package_data": [], + "emails": [], + "urls": [], + "md5": "", + "sha1": "", + "sha256": "", + "sha512": "", + "sha1_git": "", + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_legal": false, + "is_manifest": false, + "is_readme": false, + "is_top_level": false, + "is_key_file": false, + "extra_data": { + "layer": { + "os": null, + "author": null, + "labels": [], + "sha256": "a10cf747c363a52be048f884c084a25e03280d54a7ac02e17dbd8c5ad160e9bd", + "comment": null, + "created": null, + "variant": null, + "layer_id": "a10cf747c363a52be048f884c084a25e03280d54a7ac02e17dbd8c5ad160e9bd", + "created_by": null, + "os_version": null, + "architecture": null, + "docker_version": null, + "is_empty_layer": false + } + } + }, { "path": "centos.tar.gz-extract/a10cf747c363a52be048f884c084a25e03280d54a7ac02e17dbd8c5ad160e9bd/etc", "type": "directory", diff --git a/scanpipe/tests/data/docker/debian_scan_codebase.json b/scanpipe/tests/data/docker/debian_scan_codebase.json index 321bd8ccab..209e1ae05d 100644 --- a/scanpipe/tests/data/docker/debian_scan_codebase.json +++ b/scanpipe/tests/data/docker/debian_scan_codebase.json @@ -439,6 +439,94 @@ ], "dependencies": [], "files": [ + { + "path": "debian.tar.gz-extract", + "type": "directory", + "name": "debian.tar.gz-extract", + "status": "scanned", + "for_packages": [], + "tag": "", + "extension": ".tar.gz-extract", + "programming_language": "", + "detected_license_expression": "", + "detected_license_expression_spdx": "", + "license_detections": [], + "license_clues": [], + "percentage_of_license_text": null, + "copyrights": [], + "holders": [], + "authors": [], + "package_data": [], + "emails": [], + "urls": [], + "md5": "", + "sha1": "", + "sha256": "", + "sha512": "", + "sha1_git": "", + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_legal": false, + "is_manifest": false, + "is_readme": false, + "is_top_level": false, + "is_key_file": false, + "extra_data": {} + }, + { + "path": "debian.tar.gz-extract/8a63761caf6d45e65b8e6cdc2e0c03c55625fd142ec3356b80a9ea4a34b11b66", + "type": "directory", + "name": "8a63761caf6d45e65b8e6cdc2e0c03c55625fd142ec3356b80a9ea4a34b11b66", + "status": "scanned", + "for_packages": [], + "tag": "img-c19c05-layer-01-8a6376", + "extension": "", + "programming_language": "", + "detected_license_expression": "", + "detected_license_expression_spdx": "", + "license_detections": [], + "license_clues": [], + "percentage_of_license_text": null, + "copyrights": [], + "holders": [], + "authors": [], + "package_data": [], + "emails": [], + "urls": [], + "md5": "", + "sha1": "", + "sha256": "", + "sha512": "", + "sha1_git": "", + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_legal": false, + "is_manifest": false, + "is_readme": false, + "is_top_level": false, + "is_key_file": false, + "extra_data": { + "layer": { + "os": null, + "author": null, + "labels": [], + "sha256": "8a63761caf6d45e65b8e6cdc2e0c03c55625fd142ec3356b80a9ea4a34b11b66", + "comment": null, + "created": "2022-04-29T23:21:15.290486282Z", + "variant": null, + "layer_id": "8a63761caf6d45e65b8e6cdc2e0c03c55625fd142ec3356b80a9ea4a34b11b66", + "created_by": "/bin/sh -c #(nop) ADD file:37744639836b248c88f6e126619829290b45c233309538310e8fffb82e98eaf8 in / ", + "os_version": null, + "architecture": null, + "docker_version": null, + "is_empty_layer": false + } + } + }, { "path": "debian.tar.gz-extract/8a63761caf6d45e65b8e6cdc2e0c03c55625fd142ec3356b80a9ea4a34b11b66/etc", "type": "directory", diff --git a/scanpipe/tests/data/docker/gcr_io_distroless_base_scan_codebase.json b/scanpipe/tests/data/docker/gcr_io_distroless_base_scan_codebase.json index 088335db65..ef3dffcd29 100644 --- a/scanpipe/tests/data/docker/gcr_io_distroless_base_scan_codebase.json +++ b/scanpipe/tests/data/docker/gcr_io_distroless_base_scan_codebase.json @@ -507,6 +507,94 @@ ], "dependencies": [], "files": [ + { + "path": "gcr_io_distroless_base.tar.gz-extract", + "type": "directory", + "name": "gcr_io_distroless_base.tar.gz-extract", + "status": "scanned", + "for_packages": [], + "tag": "", + "extension": ".tar.gz-extract", + "programming_language": "", + "detected_license_expression": "", + "detected_license_expression_spdx": "", + "license_detections": [], + "license_clues": [], + "percentage_of_license_text": null, + "copyrights": [], + "holders": [], + "authors": [], + "package_data": [], + "emails": [], + "urls": [], + "md5": "", + "sha1": "", + "sha256": "", + "sha512": "", + "sha1_git": "", + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_legal": false, + "is_manifest": false, + "is_readme": false, + "is_top_level": false, + "is_key_file": false, + "extra_data": {} + }, + { + "path": "gcr_io_distroless_base.tar.gz-extract/cb3279093e638ddfd56bff4d3d89c5a3ed6dd59dbcfbc2f3107045635996b822", + "type": "directory", + "name": "cb3279093e638ddfd56bff4d3d89c5a3ed6dd59dbcfbc2f3107045635996b822", + "status": "scanned", + "for_packages": [], + "tag": "img-f596dc-layer-02-cb3279", + "extension": "", + "programming_language": "", + "detected_license_expression": "", + "detected_license_expression_spdx": "", + "license_detections": [], + "license_clues": [], + "percentage_of_license_text": null, + "copyrights": [], + "holders": [], + "authors": [], + "package_data": [], + "emails": [], + "urls": [], + "md5": "", + "sha1": "", + "sha256": "", + "sha512": "", + "sha1_git": "", + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_legal": false, + "is_manifest": false, + "is_readme": false, + "is_top_level": false, + "is_key_file": false, + "extra_data": { + "layer": { + "os": null, + "author": "Bazel", + "labels": [], + "sha256": "cb3279093e638ddfd56bff4d3d89c5a3ed6dd59dbcfbc2f3107045635996b822", + "comment": null, + "created": "1970-01-01T00:00:00Z", + "variant": null, + "layer_id": "cb3279093e638ddfd56bff4d3d89c5a3ed6dd59dbcfbc2f3107045635996b822", + "created_by": "bazel build ...", + "os_version": null, + "architecture": null, + "docker_version": null, + "is_empty_layer": false + } + } + }, { "path": "gcr_io_distroless_base.tar.gz-extract/cb3279093e638ddfd56bff4d3d89c5a3ed6dd59dbcfbc2f3107045635996b822/etc", "type": "directory", @@ -15597,6 +15685,58 @@ "is_key_file": false, "extra_data": {} }, + { + "path": "gcr_io_distroless_base.tar.gz-extract/d92879194ba1c23b840306b007bce6568f71f0e954d63625d48504d533749e30", + "type": "directory", + "name": "d92879194ba1c23b840306b007bce6568f71f0e954d63625d48504d533749e30", + "status": "scanned", + "for_packages": [], + "tag": "img-f596dc-layer-01-d92879", + "extension": "", + "programming_language": "", + "detected_license_expression": "", + "detected_license_expression_spdx": "", + "license_detections": [], + "license_clues": [], + "percentage_of_license_text": null, + "copyrights": [], + "holders": [], + "authors": [], + "package_data": [], + "emails": [], + "urls": [], + "md5": "", + "sha1": "", + "sha256": "", + "sha512": "", + "sha1_git": "", + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_legal": false, + "is_manifest": false, + "is_readme": false, + "is_top_level": false, + "is_key_file": false, + "extra_data": { + "layer": { + "os": null, + "author": "Bazel", + "labels": [], + "sha256": "d92879194ba1c23b840306b007bce6568f71f0e954d63625d48504d533749e30", + "comment": null, + "created": "1970-01-01T00:00:00Z", + "variant": null, + "layer_id": "d92879194ba1c23b840306b007bce6568f71f0e954d63625d48504d533749e30", + "created_by": "bazel build ...", + "os_version": null, + "architecture": null, + "docker_version": null, + "is_empty_layer": false + } + } + }, { "path": "gcr_io_distroless_base.tar.gz-extract/d92879194ba1c23b840306b007bce6568f71f0e954d63625d48504d533749e30/bin", "type": "directory", diff --git a/scanpipe/tests/data/image-with-symlinks/minitag.tar-expected-scan.json b/scanpipe/tests/data/image-with-symlinks/minitag.tar-expected-scan.json index 4d6246675e..48f80e14fa 100644 --- a/scanpipe/tests/data/image-with-symlinks/minitag.tar-expected-scan.json +++ b/scanpipe/tests/data/image-with-symlinks/minitag.tar-expected-scan.json @@ -88,6 +88,94 @@ "packages": [], "dependencies": [], "files": [ + { + "path": "minitag.tar-extract", + "type": "directory", + "name": "minitag.tar-extract", + "status": "scanned", + "for_packages": [], + "tag": "", + "extension": ".tar-extract", + "programming_language": "", + "detected_license_expression": "", + "detected_license_expression_spdx": "", + "license_detections": [], + "license_clues": [], + "percentage_of_license_text": null, + "copyrights": [], + "holders": [], + "authors": [], + "package_data": [], + "emails": [], + "urls": [], + "md5": "", + "sha1": "", + "sha256": "", + "sha512": "", + "sha1_git": "", + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_legal": false, + "is_manifest": false, + "is_readme": false, + "is_top_level": false, + "is_key_file": false, + "extra_data": {} + }, + { + "path": "minitag.tar-extract/887ac3b73f9a48d1324380c8d1ef033873a6cb882a5a9170fba51fa69fd5e5f0", + "type": "directory", + "name": "887ac3b73f9a48d1324380c8d1ef033873a6cb882a5a9170fba51fa69fd5e5f0", + "status": "scanned", + "for_packages": [], + "tag": "img-592e8d-layer-01-887ac3", + "extension": "", + "programming_language": "", + "detected_license_expression": "", + "detected_license_expression_spdx": "", + "license_detections": [], + "license_clues": [], + "percentage_of_license_text": null, + "copyrights": [], + "holders": [], + "authors": [], + "package_data": [], + "emails": [], + "urls": [], + "md5": "", + "sha1": "", + "sha256": "", + "sha512": "", + "sha1_git": "", + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_legal": false, + "is_manifest": false, + "is_readme": false, + "is_top_level": false, + "is_key_file": false, + "extra_data": { + "layer": { + "os": null, + "author": null, + "labels": [], + "sha256": "887ac3b73f9a48d1324380c8d1ef033873a6cb882a5a9170fba51fa69fd5e5f0", + "comment": null, + "created": "2022-07-30T14:18:12.008793231Z", + "variant": null, + "layer_id": "887ac3b73f9a48d1324380c8d1ef033873a6cb882a5a9170fba51fa69fd5e5f0", + "created_by": "/bin/sh -c #(nop) ADD file:783bda715f24a7fcabfe87ca07ae47e5cca3d35b99dd78e3155748e41477acb2 in / ", + "os_version": null, + "architecture": null, + "docker_version": null, + "is_empty_layer": false + } + } + }, { "path": "minitag.tar-extract/887ac3b73f9a48d1324380c8d1ef033873a6cb882a5a9170fba51fa69fd5e5f0/lib", "type": "directory", diff --git a/scanpipe/tests/test_pipelines.py b/scanpipe/tests/test_pipelines.py index 3acfcf28f3..9ce88b0ff7 100644 --- a/scanpipe/tests/test_pipelines.py +++ b/scanpipe/tests/test_pipelines.py @@ -1145,7 +1145,7 @@ def test_scanpipe_docker_pipeline_alpine_integration(self): exitcode, out = pipeline.execute() self.assertEqual(0, exitcode, msg=out) - self.assertEqual(510, project1.codebaseresources.count()) + self.assertEqual(512, project1.codebaseresources.count()) self.assertEqual(14, project1.discoveredpackages.count()) self.assertEqual(0, project1.discovereddependencies.count()) @@ -1195,7 +1195,7 @@ def test_scanpipe_docker_pipeline_rpm_integration(self): exitcode, out = pipeline.execute() self.assertEqual(0, exitcode, msg=out) - self.assertEqual(29, project1.codebaseresources.count()) + self.assertEqual(31, project1.codebaseresources.count()) self.assertEqual(101, project1.discoveredpackages.count()) self.assertEqual(0, project1.discovereddependencies.count()) @@ -1218,10 +1218,14 @@ def test_scanpipe_docker_pipeline_debian_integration(self): exitcode, out = pipeline.execute() self.assertEqual(0, exitcode, msg=out) - self.assertEqual(16, project1.codebaseresources.count()) + self.assertEqual(18, project1.codebaseresources.count()) self.assertEqual(2, project1.discoveredpackages.count()) self.assertEqual(0, project1.discovereddependencies.count()) + # Ensure all extracted resources exists as CodebaseResource + fs_resource_count = sum(1 for _ in project1.codebase_path.rglob("*")) + self.assertEqual(18, fs_resource_count) + result_file = output.to_json(project1) expected_file = self.data / "docker" / "debian_scan_codebase.json" self.assertPipelineResultEqual(expected_file, result_file) @@ -1241,7 +1245,7 @@ def test_scanpipe_docker_pipeline_distroless_debian_integration(self): exitcode, out = pipeline.execute() self.assertEqual(0, exitcode, msg=out) - self.assertEqual(2458, project1.codebaseresources.count()) + self.assertEqual(2461, project1.codebaseresources.count()) self.assertEqual(6, project1.discoveredpackages.count()) self.assertEqual(0, project1.discovereddependencies.count())