diff --git a/samtranslator/parser/parser.py b/samtranslator/parser/parser.py index 243cedefe6..9baac51ba5 100644 --- a/samtranslator/parser/parser.py +++ b/samtranslator/parser/parser.py @@ -9,6 +9,7 @@ from samtranslator.plugins import LifeCycleEvents from samtranslator.plugins.sam_plugins import SamPlugins from samtranslator.public.sdk.template import SamTemplate +from samtranslator.utils.utils import safe_dict from samtranslator.validator.value_validator import sam_expect LOG = logging.getLogger(__name__) @@ -32,7 +33,7 @@ def validate_datatypes(sam_template): # type: ignore[no-untyped-def] ): raise InvalidDocumentException([InvalidTemplateException("'Resources' section is required")]) - if not all(isinstance(sam_resource, dict) for sam_resource in sam_template["Resources"].values()): + if not all(isinstance(sam_resource, dict) for sam_resource in safe_dict(sam_template["Resources"]).values()): raise InvalidDocumentException( [ InvalidTemplateException( diff --git a/samtranslator/sdk/template.py b/samtranslator/sdk/template.py index 0b90bc97de..e8fe2f6f28 100644 --- a/samtranslator/sdk/template.py +++ b/samtranslator/sdk/template.py @@ -5,6 +5,7 @@ from typing import Any, Dict, Iterator, Optional, Set, Tuple, Union from samtranslator.sdk.resource import SamResource +from samtranslator.utils.utils import safe_dict class SamTemplate: @@ -30,7 +31,7 @@ def iterate(self, resource_types: Optional[Set[str]] = None) -> Iterator[Tuple[s """ if resource_types is None: resource_types = set() - for logicalId, resource_dict in self.resources.items(): + for logicalId, resource_dict in safe_dict(self.resources).items(): resource = SamResource(resource_dict) needs_filter = resource.valid() if resource_types: diff --git a/samtranslator/utils/utils.py b/samtranslator/utils/utils.py index ab952efdfc..5b932d7f72 100644 --- a/samtranslator/utils/utils.py +++ b/samtranslator/utils/utils.py @@ -71,3 +71,39 @@ def dict_deep_set(d: Any, path: str, value: Any) -> None: if not isinstance(d, dict): raise InvalidValueType(relative_path) d[_path_nodes[0]] = value + +def namespace_prefix(prefix: str, string: str): + """ + Joins `prefix` and `string` separated by `"::"` if neither is empty. + + Returns the non empty one if only one is empty + Returns `""` if both are empty + """ + return "::".join(filter(None, [prefix, string])) + +def safe_dict(input_dict, namespace = None): + """ + Manipulates entries to support usage of `Fn::ForEach` intrinsic function in + resources dicts. + + Recursively searches for array entries with keys starting with + `Fn::ForEach::` and replaces them with the provided resource fragments. + + To support embedded usage of `Fn::ForEach` intrinsic function, resource + fragment keys are prefixed with provided unique loop name + """ + output_dict = {} + for_each_function = "Fn::ForEach::" + + for k, v in input_dict.items(): + recurse = False + if isinstance(k, str) and k.startswith(for_each_function): + if isinstance(v, list) and len(v) == 3: + recurse = True + + if recurse: + output_dict = output_dict | safe_dict(v[2], namespace_prefix(namespace, k.removeprefix(for_each_function))) + else: + output_dict[namespace_prefix(namespace, str(k))] = v + + return output_dict diff --git a/tests/schema/test_validate_schema.py b/tests/schema/test_validate_schema.py index 2c1a716ed7..b218b46f34 100644 --- a/tests/schema/test_validate_schema.py +++ b/tests/schema/test_validate_schema.py @@ -58,6 +58,7 @@ # TODO: Support globals (e.g. somehow make all fields of a model optional only for Globals) "api_with_custom_base_path", "function_with_tracing", # TODO: intentionally skip this tests to cover incorrect scenarios + "intrinsic_for_each_resource", # intrinsic forEach loop is not supported ] diff --git a/tests/sdk/test_template.py b/tests/sdk/test_template.py index 59ff88487f..fe87bcf047 100644 --- a/tests/sdk/test_template.py +++ b/tests/sdk/test_template.py @@ -15,6 +15,8 @@ def setUp(self): "Api": {"Type": "AWS::Serverless::Api"}, "Layer": {"Type": "AWS::Serverless::LayerVersion"}, "NonSam": {"Type": "AWS::Lambda::Function"}, + "Fn::ForEach::LambdaFunctions": [ "FunctionName", ["1", "2"], { "${FunctioName}": {"Type": "AWS::Lambda::Function", "a": "b"}}], + "Fn::ForEach::ServerlessFunctions": ["FunctionName", ["3", "4"], {"${FunctionName}": {"Type": "AWS::Serverless::Function", "a": "b"}}], }, } @@ -26,6 +28,7 @@ def test_iterate_must_yield_sam_resources_only(self): ("Function2", {"Type": "AWS::Serverless::Function", "a": "b", "Properties": {}}), ("Api", {"Type": "AWS::Serverless::Api", "Properties": {}}), ("Layer", {"Type": "AWS::Serverless::LayerVersion", "Properties": {}}), + ("ServerlessFunctions::${FunctionName}", {"Type": "AWS::Serverless::Function", "a": "b", "Properties": {}}), ] actual = [(id, resource.to_dict()) for id, resource in template.iterate()] @@ -38,6 +41,7 @@ def test_iterate_must_filter_by_resource_type(self): expected = [ ("Function1", {"Type": "AWS::Serverless::Function", "DependsOn": "SomeOtherResource", "Properties": {}}), ("Function2", {"Type": "AWS::Serverless::Function", "a": "b", "Properties": {}}), + ("ServerlessFunctions::${FunctionName}", {"Type": "AWS::Serverless::Function", "a": "b", "Properties": {}}), ] actual = [(id, resource.to_dict()) for id, resource in template.iterate({type})] diff --git a/tests/translator/input/intrinsic_for_each_resource.yaml b/tests/translator/input/intrinsic_for_each_resource.yaml new file mode 100644 index 0000000000..36553ec34b --- /dev/null +++ b/tests/translator/input/intrinsic_for_each_resource.yaml @@ -0,0 +1,68 @@ +Mappings: + TemplateLinksPolicies: + Template1Link1: + template: Template1 + principal: + type: "User" + id: "user1" + resource: + type: "Resource" + id: "resource1" + Template1Link2: + template: Template2 + principal: + type: "User" + id: "user2" + resource: + type: "Resource" + id: "resource2" + +Parameters: + Environment: + Type: String + Project: + Type: String + +Resources: + PolicyStore: + Type: AWS::VerifiedPermissions::PolicyStore + Properties: + Description: !Sub "AVP Policy store for ${Project}-${Environment}" + Schema: + CedarJson: + Fn::ToJsonString: + Fn::Transform: + Name: AWS::Include + Parameters: + Location: policy-store-schema.json + ValidationSettings: + Mode: STRICT + + Template1: + Type: AWS::VerifiedPermissions::PolicyTemplate + Properties: + PolicyStoreId: !Ref PolicyStore + Description: "AVP Template." + Statement: > + permit( + principal in ?principal, + action == Action::"action", + resource == ?resource + ); + + 'Fn::ForEach::TemplateLinked': + - TemplateKey + - {"Fn::FindInMap": ["TemplateLinksPolicies"]} + - '${TemplateKey}': + Type: AWS::VerifiedPermissions::Policy + Properties: + PolicyStoreId: !Ref PolicyStore + Definition: + TemplateLinked: + PolicyTemplateId: {"Fn::GetAtt": [{"Fn::FindInMap": ["TemplateLinksPolicies", "${TemplateKey}", "template"]}, "PolicyTemplateId"]} + Principal: + EntityType: {"Fn::FindInMap": ["TemplateLinksPolicies", '${TemplateKey}', "principal", "type"]} + EntityId: {"Fn::FindInMap": ["TemplateLinksPolicies", '${TemplateKey}', "principal", "id"]} + Resource: + EntityType: {"Fn::FindInMap": ["TemplateLinksPolicies", "${TemplateKey}", "resource", "type"]} + EntityId: {"Fn::FindInMap": ["TemplateLinksPolicies", "${TemplateKey}", "resource", "id"]} diff --git a/tests/translator/output/aws-cn/intrinsic_for_each_resource.json b/tests/translator/output/aws-cn/intrinsic_for_each_resource.json new file mode 100644 index 0000000000..699d748986 --- /dev/null +++ b/tests/translator/output/aws-cn/intrinsic_for_each_resource.json @@ -0,0 +1,141 @@ +{ + "Mappings": { + "TemplateLinksPolicies": { + "Template1Link1": { + "template": "Template1", + "principal": { + "type": "User", + "id": "user1" + }, + "resource": { + "type": "Resource", + "id": "resource1" + } + }, + "Template1Link2": { + "template": "Template2", + "principal": { + "type": "User", + "id": "user2" + }, + "resource": { + "type": "Resource", + "id": "resource2" + } + } + } + }, + "Parameters": { + "Environment": { + "Type": "String" + }, + "Project": { + "Type": "String" + } + }, + "Resources": { + "PolicyStore": { + "Type": "AWS::VerifiedPermissions::PolicyStore", + "Properties": { + "Description": { + "Fn::Sub": "AVP Policy store for ${Project}-${Environment}" + }, + "Schema": { + "CedarJson": { + "Fn::ToJsonString": { + "Fn::Transform": { + "Name": "AWS::Include", + "Parameters": { + "Location": "policy-store-schema.json" + } + } + } + } + }, + "ValidationSettings": { + "Mode": "STRICT" + } + } + }, + "Template1": { + "Type": "AWS::VerifiedPermissions::PolicyTemplate", + "Properties": { + "PolicyStoreId": { + "Ref": "PolicyStore" + }, + "Description": "AVP Template.", + "Statement": "permit(\n principal in ?principal,\n action == Action::\"action\",\n resource == ?resource\n );\n" + } + }, + "Fn::ForEach::TemplateLinked": [ + "TemplateKey", + { + "Fn::FindInMap": [ + "TemplateLinksPolicies" + ] + }, + { + "${TemplateKey}": { + "Type": "AWS::VerifiedPermissions::Policy", + "Properties": { + "PolicyStoreId": { + "Ref": "PolicyStore" + }, + "Definition": { + "TemplateLinked": { + "PolicyTemplateId": { + "Fn::GetAtt": [ + { + "Fn::FindInMap": [ + "TemplateLinksPolicies", + "${TemplateKey}", + "template" + ] + }, + "PolicyTemplateId" + ] + }, + "Principal": { + "EntityType": { + "Fn::FindInMap": [ + "TemplateLinksPolicies", + "${TemplateKey}", + "principal", + "type" + ] + }, + "EntityId": { + "Fn::FindInMap": [ + "TemplateLinksPolicies", + "${TemplateKey}", + "principal", + "id" + ] + } + }, + "Resource": { + "EntityType": { + "Fn::FindInMap": [ + "TemplateLinksPolicies", + "${TemplateKey}", + "resource", + "type" + ] + }, + "EntityId": { + "Fn::FindInMap": [ + "TemplateLinksPolicies", + "${TemplateKey}", + "resource", + "id" + ] + } + } + } + } + } + } + } + ] + } +} diff --git a/tests/translator/output/aws-us-gov/intrinsic_for_each_resource.json b/tests/translator/output/aws-us-gov/intrinsic_for_each_resource.json new file mode 100644 index 0000000000..699d748986 --- /dev/null +++ b/tests/translator/output/aws-us-gov/intrinsic_for_each_resource.json @@ -0,0 +1,141 @@ +{ + "Mappings": { + "TemplateLinksPolicies": { + "Template1Link1": { + "template": "Template1", + "principal": { + "type": "User", + "id": "user1" + }, + "resource": { + "type": "Resource", + "id": "resource1" + } + }, + "Template1Link2": { + "template": "Template2", + "principal": { + "type": "User", + "id": "user2" + }, + "resource": { + "type": "Resource", + "id": "resource2" + } + } + } + }, + "Parameters": { + "Environment": { + "Type": "String" + }, + "Project": { + "Type": "String" + } + }, + "Resources": { + "PolicyStore": { + "Type": "AWS::VerifiedPermissions::PolicyStore", + "Properties": { + "Description": { + "Fn::Sub": "AVP Policy store for ${Project}-${Environment}" + }, + "Schema": { + "CedarJson": { + "Fn::ToJsonString": { + "Fn::Transform": { + "Name": "AWS::Include", + "Parameters": { + "Location": "policy-store-schema.json" + } + } + } + } + }, + "ValidationSettings": { + "Mode": "STRICT" + } + } + }, + "Template1": { + "Type": "AWS::VerifiedPermissions::PolicyTemplate", + "Properties": { + "PolicyStoreId": { + "Ref": "PolicyStore" + }, + "Description": "AVP Template.", + "Statement": "permit(\n principal in ?principal,\n action == Action::\"action\",\n resource == ?resource\n );\n" + } + }, + "Fn::ForEach::TemplateLinked": [ + "TemplateKey", + { + "Fn::FindInMap": [ + "TemplateLinksPolicies" + ] + }, + { + "${TemplateKey}": { + "Type": "AWS::VerifiedPermissions::Policy", + "Properties": { + "PolicyStoreId": { + "Ref": "PolicyStore" + }, + "Definition": { + "TemplateLinked": { + "PolicyTemplateId": { + "Fn::GetAtt": [ + { + "Fn::FindInMap": [ + "TemplateLinksPolicies", + "${TemplateKey}", + "template" + ] + }, + "PolicyTemplateId" + ] + }, + "Principal": { + "EntityType": { + "Fn::FindInMap": [ + "TemplateLinksPolicies", + "${TemplateKey}", + "principal", + "type" + ] + }, + "EntityId": { + "Fn::FindInMap": [ + "TemplateLinksPolicies", + "${TemplateKey}", + "principal", + "id" + ] + } + }, + "Resource": { + "EntityType": { + "Fn::FindInMap": [ + "TemplateLinksPolicies", + "${TemplateKey}", + "resource", + "type" + ] + }, + "EntityId": { + "Fn::FindInMap": [ + "TemplateLinksPolicies", + "${TemplateKey}", + "resource", + "id" + ] + } + } + } + } + } + } + } + ] + } +} diff --git a/tests/translator/output/intrinsic_for_each_resource.json b/tests/translator/output/intrinsic_for_each_resource.json new file mode 100644 index 0000000000..699d748986 --- /dev/null +++ b/tests/translator/output/intrinsic_for_each_resource.json @@ -0,0 +1,141 @@ +{ + "Mappings": { + "TemplateLinksPolicies": { + "Template1Link1": { + "template": "Template1", + "principal": { + "type": "User", + "id": "user1" + }, + "resource": { + "type": "Resource", + "id": "resource1" + } + }, + "Template1Link2": { + "template": "Template2", + "principal": { + "type": "User", + "id": "user2" + }, + "resource": { + "type": "Resource", + "id": "resource2" + } + } + } + }, + "Parameters": { + "Environment": { + "Type": "String" + }, + "Project": { + "Type": "String" + } + }, + "Resources": { + "PolicyStore": { + "Type": "AWS::VerifiedPermissions::PolicyStore", + "Properties": { + "Description": { + "Fn::Sub": "AVP Policy store for ${Project}-${Environment}" + }, + "Schema": { + "CedarJson": { + "Fn::ToJsonString": { + "Fn::Transform": { + "Name": "AWS::Include", + "Parameters": { + "Location": "policy-store-schema.json" + } + } + } + } + }, + "ValidationSettings": { + "Mode": "STRICT" + } + } + }, + "Template1": { + "Type": "AWS::VerifiedPermissions::PolicyTemplate", + "Properties": { + "PolicyStoreId": { + "Ref": "PolicyStore" + }, + "Description": "AVP Template.", + "Statement": "permit(\n principal in ?principal,\n action == Action::\"action\",\n resource == ?resource\n );\n" + } + }, + "Fn::ForEach::TemplateLinked": [ + "TemplateKey", + { + "Fn::FindInMap": [ + "TemplateLinksPolicies" + ] + }, + { + "${TemplateKey}": { + "Type": "AWS::VerifiedPermissions::Policy", + "Properties": { + "PolicyStoreId": { + "Ref": "PolicyStore" + }, + "Definition": { + "TemplateLinked": { + "PolicyTemplateId": { + "Fn::GetAtt": [ + { + "Fn::FindInMap": [ + "TemplateLinksPolicies", + "${TemplateKey}", + "template" + ] + }, + "PolicyTemplateId" + ] + }, + "Principal": { + "EntityType": { + "Fn::FindInMap": [ + "TemplateLinksPolicies", + "${TemplateKey}", + "principal", + "type" + ] + }, + "EntityId": { + "Fn::FindInMap": [ + "TemplateLinksPolicies", + "${TemplateKey}", + "principal", + "id" + ] + } + }, + "Resource": { + "EntityType": { + "Fn::FindInMap": [ + "TemplateLinksPolicies", + "${TemplateKey}", + "resource", + "type" + ] + }, + "EntityId": { + "Fn::FindInMap": [ + "TemplateLinksPolicies", + "${TemplateKey}", + "resource", + "id" + ] + } + } + } + } + } + } + } + ] + } +} diff --git a/tests/utils/test_utils.py b/tests/utils/test_utils.py index 88b2f712d9..24a606252a 100644 --- a/tests/utils/test_utils.py +++ b/tests/utils/test_utils.py @@ -7,6 +7,8 @@ dict_deep_get, dict_deep_set, insert_unique, + namespace_prefix, + safe_dict, ) @@ -76,3 +78,101 @@ def test_remove_none_values(self): d = {"a": None, "b": None, "c": None} self.assertEqual(remove_none_values(d), {}) + + def test_namespace_prefix(self): + self.assertEqual(namespace_prefix("a", "b"), "a::b") + + self.assertEqual(namespace_prefix("a", ""), "a") + self.assertEqual(namespace_prefix("a", None), "a") + self.assertEqual(namespace_prefix("", "b"), "b") + self.assertEqual(namespace_prefix(None, "b"), "b") + + self.assertEqual(namespace_prefix("", ""), "") + self.assertEqual(namespace_prefix("", None), "") + self.assertEqual(namespace_prefix(None, ""), "") + self.assertEqual(namespace_prefix(None, None), "") + + def test_safe_dict_no_change(self): + d = {"resource_id_1": {"Type": "custom_type"}, "resource_id_2": {"Type": "other_type"}} + self.assertEqual(safe_dict(d), d) + + def test_safe_dict_with_missing_loop_entry(self): + d = {"Fn::ForEach::unique_loop_id": ["LoopKey", {"Type": "custom_type"}]} + self.assertEqual(safe_dict(d), d) + + def test_safe_dict_with_extra_loop_entry(self): + d = {"Fn::ForEach::unique_loop_id": ["LoopKey", ["LoopKey1", "LoopKey2"], {"custom_type_${LoopKey}": {"Type": "custom_type", "Properties": {"property": "value"}}}, "extra_loop_entry"]} + self.assertEqual(safe_dict(d), d) + + def test_safe_dict_with_loop_single_resource(self): + loop_id = "unique_loop_id" + d = {f"Fn::ForEach::{loop_id}": ["LoopKey", ["LoopKey1", "LoopKey2"], {"custom_type_${LoopKey}": {"Type": "custom_type", "Properties": {"property": "value"}}}]} + self.assertEqual(safe_dict(d), {f"{loop_id}::{k}": v for k,v in d[f"Fn::ForEach::{loop_id}"][2].items()}) + + def test_safe_dict_with_loop_multiple_resources(self): + loop_id = "unique_loop_id" + d = { + f"Fn::ForEach::{loop_id}": ["LoopKey", ["LoopKey1", "LoopKey2"], { + "custom_type_${LoopKey}": {"Type": "custom_type", "Properties": {"property": "value"}}, + "other_type_${LoopKey}": {"Type": "other_type", "Properties": {"property": "value"}} + }] + } + + self.assertEqual(safe_dict(d), {f"{loop_id}::{k}": v for k,v in d[f"Fn::ForEach::{loop_id}"][2].items()}) + + def test_safe_dict_with_loop_and_regular_resources(self): + loop_id = "unique_loop_id" + d = { + "resource_id_1": {"Type": "custom_type"}, + f"Fn::ForEach::{loop_id}": ["LoopKey", ["LoopKey1", "LoopKey2"], { + "custom_type_${LoopKey}": {"Type": "custom_type", "Properties": {"property": "value"}}, + "other_type_${LoopKey}": {"Type": "other_type", "Properties": {"property": "value"}} + }], + "resource_id_2": {"Type": "other_type"} + } + + regular_resources_expected_dict = {k: d[k] for k in ["resource_id_1", "resource_id_2"]} + loop_expected_dir = {f"{loop_id}::{k}": v for k,v in d[f"Fn::ForEach::{loop_id}"][2].items()} + self.assertEqual(safe_dict(d), (regular_resources_expected_dict | loop_expected_dir)) + + def test_safe_dict_with_multiple_loops(self): + first_loop_id = "first_loop_id" + second_loop_id = "second_loop_id" + d = { + f"Fn::ForEach::{first_loop_id}": ["LoopKey", ["LoopKey10", "LoopKey11"], { + "custom_type_${LoopKey}": {"Type": "custom_type", "Properties": {"property": "value"}}, + "other_type_${LoopKey}": {"Type": "other_type", "Properties": {"property": "value"}} + }], + f"Fn::ForEach::{second_loop_id}": ["LoopKey", ["LoopKey20", "LoopKey21"], { + "custom_type_${LoopKey}": {"Type": "custom_type", "Properties": {"property": "value"}}, + "other_type_${LoopKey}": {"Type": "other_type", "Properties": {"property": "value"}} + }] + } + + safe_d = safe_dict(d) + self.assertEqual(len(safe_d), 4) + + first_expected_dict = {f"{first_loop_id}::{k}": v for k,v in d[f"Fn::ForEach::{first_loop_id}"][2].items()} + second_expected_dict = {f"{second_loop_id}::{k}": v for k,v in d[f"Fn::ForEach::{second_loop_id}"][2].items()} + self.assertEqual(safe_d, (first_expected_dict | second_expected_dict)) + + def test_safe_dict_with_embedded_loops(self): + outer_loop_id = "outer_loop_id" + inner_loop_id = "inner_loop_id" + d = { + f"Fn::ForEach::{outer_loop_id}": ["LoopKey", ["LoopKey10", "LoopKey11"], { + "custom_type_${LoopKey}": {"Type": "custom_type", "Properties": {"property": "value"}}, + "other_type_${LoopKey}": {"Type": "other_type", "Properties": {"property": "value"}}, + f"Fn::ForEach::{inner_loop_id}": ["LoopKey", ["LoopKey20", "LoopKey21"], { + "custom_type_${LoopKey}": {"Type": "custom_type", "Properties": {"property": "value"}}, + "other_type_${LoopKey}": {"Type": "other_type", "Properties": {"property": "value"}} + }] + }], + } + + safe_d = safe_dict(d) + self.assertEqual(len(safe_d), 4) + + outer_loop_expected_dict = {"{}::{}".format(outer_loop_id, k): v for k,v in d[f"Fn::ForEach::{outer_loop_id}"][2].items() if not k.startswith("Fn::ForEach::")} + inner_loop_expected_dict = {"{}::{}::{}".format(outer_loop_id, inner_loop_id, k.removeprefix('Fn::ForEach::')): v for k,v in d[f"Fn::ForEach::{outer_loop_id}"][2][f"Fn::ForEach::{inner_loop_id}"][2].items()} + self.assertEqual(safe_d, (outer_loop_expected_dict | inner_loop_expected_dict))