diff --git a/src/confcom/HISTORY.rst b/src/confcom/HISTORY.rst index 1c00993a147..70f0f95515c 100644 --- a/src/confcom/HISTORY.rst +++ b/src/confcom/HISTORY.rst @@ -3,6 +3,14 @@ Release History =============== +1.2.7 +++++++ +* bugfix making it so that oras discover function doesn't error when no fragments are found in the remote repository + +1.2.6 +++++++ +* bugfix making it so the fields in the --input format are case-insensitive + 1.2.5 ++++++ * consolidating functions for --input policygen diff --git a/src/confcom/azext_confcom/data/internal_config.json b/src/confcom/azext_confcom/data/internal_config.json index 598e47d5f32..84bb5c2ddc4 100644 --- a/src/confcom/azext_confcom/data/internal_config.json +++ b/src/confcom/azext_confcom/data/internal_config.json @@ -1,5 +1,5 @@ { - "version": "1.2.5", + "version": "1.2.6", "hcsshim_config": { "maxVersion": "1.0.0", "minVersion": "0.0.1" diff --git a/src/confcom/azext_confcom/oras_proxy.py b/src/confcom/azext_confcom/oras_proxy.py index 28e06e6f118..6f39e51a520 100644 --- a/src/confcom/azext_confcom/oras_proxy.py +++ b/src/confcom/azext_confcom/oras_proxy.py @@ -79,7 +79,10 @@ def discover( f"Error pulling the policy fragment from {image}.\n\n" + "Please log into the registry and try again.\n\n" ) - eprint(f"Error retrieving fragments from remote repo: {item.stderr.decode('utf-8')}", exit_code=item.returncode) + elif "not found" in item.stderr.decode("utf-8"): + logger.warning("No policy fragments found for image %s", image) + else: + eprint(f"Error retrieving fragments from remote repo: {item.stderr.decode('utf-8')}", exit_code=item.returncode) return hashes diff --git a/src/confcom/azext_confcom/template_util.py b/src/confcom/azext_confcom/template_util.py index 49614e7e4f8..cbaade26e8a 100644 --- a/src/confcom/azext_confcom/template_util.py +++ b/src/confcom/azext_confcom/template_util.py @@ -44,7 +44,7 @@ def __exit__(self, exc_type, exc_value, traceback) -> None: self._client.close() -def case_insensitive_dict_get(dictionary, search_key) -> Any: +def case_insensitive_dict_get(dictionary, search_key, default_value=None) -> Any: if not isinstance(dictionary, dict): return None # if the cases happen to match, immediately return .get() result @@ -55,7 +55,7 @@ def case_insensitive_dict_get(dictionary, search_key) -> Any: for key in dictionary.keys(): if key.lower() == search_key.lower(): return dictionary[key] - return None + return default_value def deep_dict_update(source: dict, destination: dict): @@ -1572,25 +1572,31 @@ def convert_config_v0_to_v1(old_data): # Prepare the structure of the new JSON new_data = { - config.ACI_FIELD_VERSION: old_data.get(config.ACI_FIELD_VERSION, "1.0"), # default if missing - config.ACI_FIELD_CONTAINERS_REGO_FRAGMENTS: [], # empty by default in your example + config.ACI_FIELD_VERSION: case_insensitive_dict_get( + old_data, config.ACI_FIELD_VERSION, "1.0" + ), # default if missing + config.ACI_FIELD_CONTAINERS_REGO_FRAGMENTS: [], config.ACI_FIELD_CONTAINERS: [] } - old_containers = old_data.get(config.ACI_FIELD_CONTAINERS, []) + old_containers = case_insensitive_dict_get(old_data, config.ACI_FIELD_CONTAINERS, []) for old_container in old_containers: # Build the 'environmentVariables' section in the new format new_envs = [] - for env_var in old_container.get(config.ACI_FIELD_CONTAINERS_ENVS) or []: + for env_var in case_insensitive_dict_get(old_container, config.ACI_FIELD_CONTAINERS_ENVS) or []: # Decide if we need 'regex' or not, based on 'strategy' or your custom logic # Here we'll assume "strategy"=="re2" means 'regex' = True # If strategy is missing or 'string', omit 'regex' or set it to False env_entry = { - config.ACI_FIELD_CONTAINERS_ENVS_NAME: env_var.get(config.ACI_FIELD_CONTAINERS_ENVS_NAME), - config.ACI_FIELD_CONTAINERS_ENVS_VALUE: env_var.get(config.ACI_FIELD_CONTAINERS_ENVS_VALUE, "") + config.ACI_FIELD_CONTAINERS_ENVS_NAME: case_insensitive_dict_get( + env_var, config.ACI_FIELD_CONTAINERS_ENVS_NAME + ), + config.ACI_FIELD_CONTAINERS_ENVS_VALUE: case_insensitive_dict_get( + env_var, config.ACI_FIELD_CONTAINERS_ENVS_VALUE, "" + ) } - strategy = env_var.get(config.ACI_FIELD_CONTAINERS_ENVS_STRATEGY) + strategy = case_insensitive_dict_get(env_var, config.ACI_FIELD_CONTAINERS_ENVS_STRATEGY) if strategy == "re2": env_entry["regex"] = True @@ -1598,25 +1604,25 @@ def convert_config_v0_to_v1(old_data): # Build the 'execProcesses' from the old 'command' exec_processes = [] - old_command_list = old_container.get(config.ACI_FIELD_CONTAINERS_EXEC_PROCESSES, []) + old_command_list = case_insensitive_dict_get(old_container, config.ACI_FIELD_CONTAINERS_EXEC_PROCESSES, []) if old_command_list: exec_processes.append({config.ACI_FIELD_CONTAINERS_COMMAND: old_command_list}) - command = old_container.get(config.ACI_FIELD_CONTAINERS_COMMAND) + command = case_insensitive_dict_get(old_container, config.ACI_FIELD_CONTAINERS_COMMAND) # Liveness probe => exec process - liveness_probe = old_container.get(config.ACI_FIELD_CONTAINERS_LIVENESS_PROBE, {}) - liveness_exec = liveness_probe.get(config.ACI_FIELD_CONTAINERS_PROBE_ACTION, {}) - liveness_command = liveness_exec.get(config.ACI_FIELD_CONTAINERS_COMMAND, []) + liveness_probe = case_insensitive_dict_get(old_container, config.ACI_FIELD_CONTAINERS_LIVENESS_PROBE, {}) + liveness_exec = case_insensitive_dict_get(liveness_probe, config.ACI_FIELD_CONTAINERS_PROBE_ACTION, {}) + liveness_command = case_insensitive_dict_get(liveness_exec, config.ACI_FIELD_CONTAINERS_COMMAND, []) if liveness_command: exec_processes.append({ config.ACI_FIELD_CONTAINERS_COMMAND: liveness_command }) # Readiness probe => exec process - readiness_probe = old_container.get(config.ACI_FIELD_CONTAINERS_READINESS_PROBE, {}) - readiness_exec = readiness_probe.get(config.ACI_FIELD_CONTAINERS_PROBE_ACTION, {}) - readiness_command = readiness_exec.get(config.ACI_FIELD_CONTAINERS_COMMAND, []) + readiness_probe = case_insensitive_dict_get(old_container, config.ACI_FIELD_CONTAINERS_READINESS_PROBE, {}) + readiness_exec = case_insensitive_dict_get(readiness_probe, config.ACI_FIELD_CONTAINERS_PROBE_ACTION, {}) + readiness_command = case_insensitive_dict_get(readiness_exec, config.ACI_FIELD_CONTAINERS_COMMAND, []) if readiness_command: exec_processes.append({ config.ACI_FIELD_CONTAINERS_COMMAND: readiness_command @@ -1624,47 +1630,61 @@ def convert_config_v0_to_v1(old_data): # Build the 'volumeMounts' section volume_mounts = [] - for mount in old_container.get(config.ACI_FIELD_CONTAINERS_MOUNTS) or []: + for mount in case_insensitive_dict_get(old_container, config.ACI_FIELD_CONTAINERS_MOUNTS) or []: # For 'name', we can take the mountType or generate something else: # e.g. if mountType is "azureFile", name "azurefile" - mount_name = mount.get(config.ACI_FIELD_CONTAINERS_MOUNTS_TYPE, "defaultName").lower() + mount_name = case_insensitive_dict_get( + mount, config.ACI_FIELD_CONTAINERS_MOUNTS_TYPE, "defaultName" + ).lower() volume_mount = { config.ACI_FIELD_CONTAINERS_ENVS_NAME: mount_name, - config.ACI_FIELD_TEMPLATE_MOUNTS_PATH: mount.get(config.ACI_FIELD_CONTAINERS_MOUNTS_PATH), - config.ACI_FIELD_TEMPLATE_MOUNTS_TYPE: mount.get(config.ACI_FIELD_CONTAINERS_MOUNTS_TYPE), - config.ACI_FIELD_TEMPLATE_MOUNTS_READONLY: mount.get(config.ACI_FIELD_CONTAINERS_MOUNTS_READONLY, True), + config.ACI_FIELD_TEMPLATE_MOUNTS_PATH: case_insensitive_dict_get( + mount, config.ACI_FIELD_CONTAINERS_MOUNTS_PATH + ), + config.ACI_FIELD_TEMPLATE_MOUNTS_TYPE: case_insensitive_dict_get( + mount, config.ACI_FIELD_CONTAINERS_MOUNTS_TYPE + ), + config.ACI_FIELD_TEMPLATE_MOUNTS_READONLY: case_insensitive_dict_get( + mount, config.ACI_FIELD_CONTAINERS_MOUNTS_READONLY, True + ), } volume_mounts.append(volume_mount) # Create the container's "properties" object container_properties = { - config.ACI_FIELD_TEMPLATE_IMAGE: old_container.get(config.ACI_FIELD_CONTAINERS_CONTAINERIMAGE), + config.ACI_FIELD_TEMPLATE_IMAGE: case_insensitive_dict_get( + old_container, config.ACI_FIELD_CONTAINERS_CONTAINERIMAGE + ), config.ACI_FIELD_CONTAINERS_EXEC_PROCESSES: exec_processes, config.ACI_FIELD_TEMPLATE_VOLUME_MOUNTS: volume_mounts, config.ACI_FIELD_CONTAINERS_ENVS: new_envs, config.ACI_FIELD_CONTAINERS_COMMAND: command, } - if old_container.get(config.ACI_FIELD_TEMPLATE_SECURITY_CONTEXT) is not None: + if case_insensitive_dict_get(old_container, config.ACI_FIELD_TEMPLATE_SECURITY_CONTEXT) is not None: container_properties[ config.ACI_FIELD_TEMPLATE_SECURITY_CONTEXT ] = old_container[config.ACI_FIELD_TEMPLATE_SECURITY_CONTEXT] - if old_container.get(config.ACI_FIELD_CONTAINERS_ALLOW_ELEVATED) is not None: + if case_insensitive_dict_get(old_container, config.ACI_FIELD_CONTAINERS_ALLOW_ELEVATED) is not None: if config.ACI_FIELD_TEMPLATE_SECURITY_CONTEXT not in container_properties: container_properties[config.ACI_FIELD_TEMPLATE_SECURITY_CONTEXT] = {} container_properties[ config.ACI_FIELD_TEMPLATE_SECURITY_CONTEXT - ][config.ACI_FIELD_CONTAINERS_PRIVILEGED] = old_container.get(config.ACI_FIELD_CONTAINERS_ALLOW_ELEVATED) + ][config.ACI_FIELD_CONTAINERS_PRIVILEGED] = case_insensitive_dict_get( + old_container, config.ACI_FIELD_CONTAINERS_ALLOW_ELEVATED + ) - if old_container.get(config.ACI_FIELD_CONTAINERS_WORKINGDIR) is not None: + if case_insensitive_dict_get(old_container, config.ACI_FIELD_CONTAINERS_WORKINGDIR) is not None: container_properties[ config.ACI_FIELD_CONTAINERS_WORKINGDIR - ] = old_container.get(config.ACI_FIELD_CONTAINERS_WORKINGDIR) + ] = case_insensitive_dict_get(old_container, config.ACI_FIELD_CONTAINERS_WORKINGDIR) # Finally, assemble the new container dict new_container = { - config.ACI_FIELD_CONTAINERS_NAME: old_container.get(config.ACI_FIELD_CONTAINERS_NAME), + config.ACI_FIELD_CONTAINERS_NAME: case_insensitive_dict_get( + old_container, config.ACI_FIELD_CONTAINERS_NAME + ), config.ACI_FIELD_TEMPLATE_PROPERTIES: container_properties } @@ -1677,8 +1697,10 @@ def detect_old_format(old_data): # we want to encourage customers to transition to the new format. The best way to check for the old format is # to see if the json is flattened. This is an appropriate check since the image name is required # and they are located in different places in the two formats - old_containers = old_data.get(config.ACI_FIELD_CONTAINERS, []) - if len(old_containers) > 0 and old_containers[0].get(config.ACI_FIELD_CONTAINERS_CONTAINERIMAGE) is not None: + old_containers = case_insensitive_dict_get(old_data, config.ACI_FIELD_CONTAINERS, []) + if len(old_containers) > 0 and case_insensitive_dict_get( + old_containers[0], config.ACI_FIELD_CONTAINERS_CONTAINERIMAGE + ) is not None: logger.warning( "%s %s %s", "(Deprecation Warning) The input format used is deprecated.", diff --git a/src/confcom/azext_confcom/tests/latest/README.md b/src/confcom/azext_confcom/tests/latest/README.md index 469edd7f905..7128cf98bab 100644 --- a/src/confcom/azext_confcom/tests/latest/README.md +++ b/src/confcom/azext_confcom/tests/latest/README.md @@ -160,6 +160,7 @@ test_fragment_user_container_customized_mounts | mcr.microsoft.com/azurelinux/di test_fragment_user_container_mount_injected_dns | mcr.microsoft.com/azurelinux/distroless/base:3.0 | See if the resolvconf mount works properly test_fragment_omit_id | mcr.microsoft.com/aci/msi-atlas-adapter:master_20201203.1 | Check that the id field is omitted from the policy test_fragment_injected_sidecar_container_msi | mcr.microsoft.com/aci/msi-atlas-adapter:master_20201203.1 | Make sure User mounts and env vars aren't added to sidecar containers, using JSON output format +test_tar_file_fragment | mcr.microsoft.com/aks/e2e/library-busybox:master.220314.1-linux-amd64 | Make sure fragment generation doesn't fail for image tarball test_debug_processes | mcr.microsoft.com/azurelinux/distroless/base:3.0 | Enable exec_processes via debug_mode test_fragment_sidecar | mcr.microsoft.com/aci/msi-atlas-adapter:master_20201210.1 | See if sidecar fragments can be created by a given policy.json test_fragment_sidecar_stdio_access_default | mcr.microsoft.com/aci/msi-atlas-adapter:master_20201210.1 | Check that sidecar containers have std I/O access by default diff --git a/src/confcom/azext_confcom/tests/latest/test_confcom_fragment.py b/src/confcom/azext_confcom/tests/latest/test_confcom_fragment.py index 7a4a06ca873..dbd8ffd0da5 100644 --- a/src/confcom/azext_confcom/tests/latest/test_confcom_fragment.py +++ b/src/confcom/azext_confcom/tests/latest/test_confcom_fragment.py @@ -4,7 +4,10 @@ # -------------------------------------------------------------------------------------------- import os +from tarfile import TarFile +import tempfile import unittest +import deepdiff import json import subprocess from knack.util import CLIError @@ -12,11 +15,14 @@ from azext_confcom.security_policy import ( UserContainerImage, OutputType, + load_policy_from_arm_template_str, load_policy_from_json ) - +from azext_confcom.errors import ( + AccContainerError, +) from azext_confcom.cose_proxy import CoseSignToolProxy - +from azext_confcom.rootfs_proxy import SecurityPolicyProxy import azext_confcom.config as config from azext_confcom.template_util import ( case_insensitive_dict_get, @@ -36,6 +42,11 @@ from azext_confcom.custom import acifragmentgen_confcom from azure.cli.testsdk import ScenarioTest +from test_confcom_tar import ( + create_tar_file, + remove_tar_file, +) + TEST_DIR = os.path.abspath(os.path.join(os.path.abspath(__file__), "..")) class FragmentMountEnforcement(unittest.TestCase): @@ -465,6 +476,173 @@ def test_fragment_injected_sidecar_container_msi(self): self.assertEqual(image._workingDir, expected_workingdir) +class FragmentPolicyGeneratingTarfile(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + path = os.path.dirname(__file__) + cls.path = path + + def test_tar_file_fragment(self): + custom_arm_json_default_value = """ + { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + + + "parameters": { + "containergroupname": { + "type": "string", + "metadata": { + "description": "Name for the container group" + }, + "defaultValue":"simple-container-group" + }, + "image": { + "type": "string", + "metadata": { + "description": "Name for the container group" + }, + "defaultValue":"mcr.microsoft.com/aks/e2e/library-busybox:master.220314.1-linux-amd64" + }, + "containername": { + "type": "string", + "metadata": { + "description": "Name for the container" + }, + "defaultValue":"simple-container" + }, + + "port": { + "type": "string", + "metadata": { + "description": "Port to open on the container and the public IP address." + }, + "defaultValue": "8080" + }, + "cpuCores": { + "type": "string", + "metadata": { + "description": "The number of CPU cores to allocate to the container." + }, + "defaultValue": "1.0" + }, + "memoryInGb": { + "type": "string", + "metadata": { + "description": "The amount of memory to allocate to the container in gigabytes." + }, + "defaultValue": "1.5" + }, + "location": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "Location for all resources." + } + } + }, + "resources": [ + { + "name": "[parameters('containergroupname')]", + "type": "Microsoft.ContainerInstance/containerGroups", + "apiVersion": "2023-05-01", + "location": "[parameters('location')]", + + "properties": { + "containers": [ + { + "name": "[parameters('containername')]", + "properties": { + "image": "[parameters('image')]", + "environmentVariables": [ + { + "name": "PORT", + "value": "80" + } + ], + + "ports": [ + { + "port": "[parameters('port')]" + } + ], + "command": [ + "/bin/bash", + "-c", + "while sleep 5; do cat /mnt/input/access.log; done" + ], + "resources": { + "requests": { + "cpu": "[parameters('cpuCores')]", + "memoryInGb": "[parameters('memoryInGb')]" + } + } + } + } + ], + + "osType": "Linux", + "restartPolicy": "OnFailure", + "confidentialComputeProperties": { + "IsolationType": "SevSnp" + }, + "ipAddress": { + "type": "Public", + "ports": [ + { + "protocol": "Tcp", + "port": "[parameters( 'port' )]" + } + ] + } + } + } + ], + "outputs": { + "containerIPv4Address": { + "type": "string", + "value": "[reference(resourceId('Microsoft.ContainerInstance/containerGroups/', parameters('containergroupname'))).ipAddress.ip]" + } + } + } + """ + + SecurityPolicyProxy.layer_cache = {} + clean_room_image = load_policy_from_arm_template_str( + custom_arm_json_default_value, "" + )[0] + + try: + with tempfile.TemporaryDirectory() as folder: + filename = os.path.join(folder, "oci.tar") + + tar_mapping_file = {"mcr.microsoft.com/aks/e2e/library-busybox:master.220314.1-linux-amd64": os.path.join(self.path, "oci2.tar")} + create_tar_file(filename) + with TarFile(f"{folder}/oci.tar", "r") as tar: + tar.extractall(path=folder) + + os.remove(os.path.join(folder, "manifest.json")) + os.remove(os.path.join(folder, "oci.tar")) + + with TarFile.open(os.path.join(self.path, "oci2.tar"), mode="w") as out_tar: + out_tar.add(os.path.join(folder, "index.json"), "index.json") + out_tar.add(os.path.join(folder, "blobs"), "blobs", recursive=True) + + clean_room_image.populate_policy_content_for_all_images( + tar_mapping=tar_mapping_file + ) + + clean_room_fragment_text = clean_room_image.generate_fragment("payload", "1", OutputType.RAW) + except Exception as e: + print(e) + raise AccContainerError("Could not get image from tar file") + finally: + remove_tar_file(filename) + remove_tar_file(os.path.join(self.path, "oci2.tar")) + + self.assertIsNotNone(clean_room_fragment_text) + + class FragmentPolicyGeneratingDebugMode(unittest.TestCase): custom_json = """ { diff --git a/src/confcom/azext_confcom/tests/latest/test_confcom_policy_conversion.py b/src/confcom/azext_confcom/tests/latest/test_confcom_policy_conversion.py index 94a1b50cbee..3feeaf58da7 100644 --- a/src/confcom/azext_confcom/tests/latest/test_confcom_policy_conversion.py +++ b/src/confcom/azext_confcom/tests/latest/test_confcom_policy_conversion.py @@ -48,6 +48,11 @@ def setUpClass(cls) -> None: "mountType": "azureFile", "mountPath": "/mnt/af", "readonly": true + }, + { + "mountType": "emptyDir", + "mountPath": "/mnt/empty", + "readOnly": false }] }] } @@ -106,6 +111,13 @@ def test_volume_mount_basic_fields(self) -> None: self.assertEqual(vm[cfg.ACI_FIELD_TEMPLATE_MOUNTS_TYPE], "azureFile") self.assertTrue(vm[cfg.ACI_FIELD_TEMPLATE_MOUNTS_READONLY]) + vm = props[cfg.ACI_FIELD_TEMPLATE_VOLUME_MOUNTS][1] + + self.assertEqual(vm[cfg.ACI_FIELD_CONTAINERS_ENVS_NAME], "emptydir") + self.assertEqual(vm[cfg.ACI_FIELD_TEMPLATE_MOUNTS_PATH], "/mnt/empty") + self.assertEqual(vm[cfg.ACI_FIELD_TEMPLATE_MOUNTS_TYPE], "emptyDir") + self.assertFalse(vm[cfg.ACI_FIELD_TEMPLATE_MOUNTS_READONLY]) + def test_workingdir_and_allow_elevated_migrated(self) -> None: props = self._new_cfg[cfg.ACI_FIELD_CONTAINERS][0][ cfg.ACI_FIELD_TEMPLATE_PROPERTIES diff --git a/src/confcom/azext_confcom/tests/latest/test_confcom_scenario.py b/src/confcom/azext_confcom/tests/latest/test_confcom_scenario.py index a6ccb14db2c..7616d40705d 100644 --- a/src/confcom/azext_confcom/tests/latest/test_confcom_scenario.py +++ b/src/confcom/azext_confcom/tests/latest/test_confcom_scenario.py @@ -586,9 +586,9 @@ def test_image_layers_python(self): aci_policy.populate_policy_content_for_all_images() layers = aci_policy.get_images()[0]._layers expected_layers = [ - "1824eb49720f59202136bd38681f32e57239676c9a423fb1e025aa210aaa22aa", - "8d68e2b0c2c32fa7fab2a5543d94e0b70b5bea400ca7f89f4c925a02ae15f453", - "44e8e0480dc3c0d04564ba87b3ba851cfab008717abdf6be83baf47963599614" + "4b17d51a118cfa6405698048bbb9f258f70c44235cf54dab8977e689d4422c1d", + "374b10f7af01a18c4408738ec38b302f4766ab62c033208c4b86eb7434ed8217", + "c9d8b0df7e0ab9ff83672dd67f154f28fdee0ae0b62c82a3451a44c8e2e29838" ] self.assertEqual(len(layers), len(expected_layers)) for i in range(len(expected_layers)): diff --git a/src/confcom/setup.py b/src/confcom/setup.py index b3c6f2b7f33..3ce7907b25a 100644 --- a/src/confcom/setup.py +++ b/src/confcom/setup.py @@ -19,7 +19,7 @@ logger.warn("Wheel is not available, disabling bdist_wheel hook") -VERSION = "1.2.5" +VERSION = "1.2.7" # The full list of classifiers is available at # https://pypi.python.org/pypi?%3Aaction=list_classifiers