diff --git a/docs/syntax/compose_x/vpc.rst b/docs/syntax/compose_x/vpc.rst index 1550d008a..10767a74c 100644 --- a/docs/syntax/compose_x/vpc.rst +++ b/docs/syntax/compose_x/vpc.rst @@ -96,23 +96,30 @@ policy name. Lookup ====== +You can Lookup the VPC ID (Subnets must use tags) via either + +* Identifier: vpc-abcd1234 +* Arn: arn:aws:ec2:eu-west-1:123456789012:vpc/vpc-0b33b2fb87c205260 +* Tags (recommended) + +See the ``uses_cases/vpc`` for examples. + .. code-block:: yaml x-vpc: Lookup: VpcId: Tags: - - key: value + key: value PublicSubnets: Tags: - - vpc::usage: public + vpc::usage: public AppSubnets: Tags: - - vpc::usage: application + vpc::usage: application StorageSubnets: Tags: - - vpc::usage: storage0 - + vpc::usage: storage .. warning:: diff --git a/ecs_composex/vpc/vpc_aws.py b/ecs_composex/vpc/vpc_aws.py index 9957ae10b..b54bc529e 100644 --- a/ecs_composex/vpc/vpc_aws.py +++ b/ecs_composex/vpc/vpc_aws.py @@ -1,9 +1,9 @@ # SPDX-License-Identifier: MPL-2.0 # Copyright 2020-2025 John Mille - +import botocore.client from boto3.session import Session -from compose_x_common.aws.arns import ARNS_PER_TAGGINGAPI_TYPE -from compose_x_common.compose_x_common import keyisset +from compose_x_common.aws.arns import ARNS_PER_CFN_TYPE, ARNS_PER_TAGGINGAPI_TYPE +from compose_x_common.compose_x_common import keyisset, set_else_none from ecs_composex.common.aws import find_aws_resource_arn_from_tags_api from ecs_composex.common.logging import LOG @@ -82,6 +82,55 @@ def validate_subnets_belong_with_vpc( ) +def lookup_vpc_id(vpc_id_details: dict, lookup_session: Session) -> str: + """ + Function to find the VPC either by ID, Arn or Tags. Arn takes priority, then ID, then Tags + """ + vpc_id = set_else_none("Identifier", vpc_id_details) + vpc_arn = set_else_none("Arn", vpc_id_details) + vpc_tags = set_else_none(TAGS_KEY, vpc_id_details) + arn_from_arn = True if vpc_arn and not vpc_id else False + + if vpc_arn: + vpc_re = ARNS_PER_CFN_TYPE["AWS::EC2::VPC"] + if not vpc_re.match(vpc_arn): + raise ValueError(f"{vpc_arn} is not a valid VPC ARN") + vpc_id = vpc_re.match(vpc_arn).group("id") + + if vpc_id: + cloud_control_client = lookup_session.client("cloudcontrol") + try: + cloud_control_client.get_resource( + TypeName="AWS::EC2::VPC", + Identifier=vpc_id, + ) + except botocore.client.ClientError as error: + LOG.exception(error) + raise ValueError(f"{vpc_id} is not a valid VPC ID") + if arn_from_arn: + return vpc_arn + else: + ec2_client = lookup_session.client("ec2") + sts_client = lookup_session.client("sts") + account_id = sts_client.get_caller_identity()["Account"] + return ( + f"arn:aws:ec2:{ec2_client.meta.region_name}:{account_id}:vpc/{vpc_id}" + ) + + elif vpc_tags: + return find_aws_resource_arn_from_tags_api( + vpc_id_details, + lookup_session, + "ec2:vpc", + allow_multi=False, + ) + raise LookupError( + "Failed to find VPC with given details: {}".format( + vpc_id or vpc_arn or vpc_tags + ) + ) + + def lookup_x_vpc_settings(vpc_resource): """ Method to set VPC settings from x-vpc @@ -103,11 +152,9 @@ def lookup_x_vpc_settings(vpc_resource): APP_SUBNETS.title, STORAGE_SUBNETS.title, ] - vpc_arn = find_aws_resource_arn_from_tags_api( + vpc_arn = lookup_vpc_id( vpc_resource.lookup[VPC_ID.title], vpc_resource.lookup_session, - vpc_type, - allow_multi=False, ) vpc_re = ARNS_PER_TAGGINGAPI_TYPE[vpc_type] vpc_settings = { diff --git a/tests/features/features/vpc.feature b/tests/features/features/vpc.feature index 34e292cd8..5797de851 100644 --- a/tests/features/features/vpc.feature +++ b/tests/features/features/vpc.feature @@ -8,6 +8,19 @@ Feature: ecs_composex.vpc Then I render the docker-compose to composex to validate And I render all files to verify execution Examples: - | file_path | override_file | - | use-cases/blog.features.yml | use-cases/vpc/new_vpc.yml | - | use-cases/blog.features.yml | use-cases/vpc/new_with_flowlogs.yml | + | file_path | override_file | + | use-cases/blog.features.yml | use-cases/vpc/new_vpc.yml | + | use-cases/blog.features.yml | use-cases/vpc/new_with_flowlogs.yml | + | use-cases/blog.features.yml | use-cases/vpc/no_nats_no_endpoints.yaml | + + Scenario Outline: VPC Lookup + Given With + And With + And I use defined files as input to define execution settings + Then I render the docker-compose to composex to validate + And I render all files to verify execution + Examples: + | file_path | override_file | + | use-cases/blog.features.yml | use-cases/vpc/lookup_vpc_via_tags.yaml | + | use-cases/blog.features.yml | use-cases/vpc/lookup_vpc_via_arn.yaml | + | use-cases/blog.features.yml | use-cases/vpc/lookup_vpc_via_id.yaml | diff --git a/use-cases/vpc/lookup_vpc.yml b/use-cases/vpc/lookup_vpc.yml deleted file mode 100644 index 60e1e5a09..000000000 --- a/use-cases/vpc/lookup_vpc.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- -# Blog applications - -version: '3.8' - -x-vpc: - Lookup: - VpcId: - Tags: - - Name: ${VPC_NAME} - AppSubnets: - Tags: - - vpc::usage: application - - vpc::primary: "true" - StorageSubnets: - Tags: - - vpc::usage: storage - PublicSubnets: - Tags: - - vpc::usage: public - Custom01: - Tags: - - vpc::usage: application - - vpc::internal: "true" diff --git a/use-cases/vpc/lookup_vpc_via_arn.yaml b/use-cases/vpc/lookup_vpc_via_arn.yaml new file mode 100644 index 000000000..778b9e0a3 --- /dev/null +++ b/use-cases/vpc/lookup_vpc_via_arn.yaml @@ -0,0 +1,18 @@ +--- +# Blog applications + +version: '3.8' + +x-vpc: + Lookup: + VpcId: + Arn: arn:aws:ec2:eu-west-1:373709687836:vpc/vpc-0b33b2fb87c205260 + AppSubnets: + Tags: + vpc::usage: application + StorageSubnets: + Tags: + vpc::usage: storage + PublicSubnets: + Tags: + vpc::usage: public diff --git a/use-cases/vpc/lookup_vpc_via_id.yaml b/use-cases/vpc/lookup_vpc_via_id.yaml new file mode 100644 index 000000000..28bd8ac5b --- /dev/null +++ b/use-cases/vpc/lookup_vpc_via_id.yaml @@ -0,0 +1,18 @@ +--- +# Blog applications + +version: '3.8' + +x-vpc: + Lookup: + VpcId: + Identifier: vpc-0b33b2fb87c205260 + AppSubnets: + Tags: + vpc::usage: application + StorageSubnets: + Tags: + vpc::usage: storage + PublicSubnets: + Tags: + vpc::usage: public diff --git a/use-cases/vpc/lookup_vpc_via_tags.yaml b/use-cases/vpc/lookup_vpc_via_tags.yaml new file mode 100644 index 000000000..08c87bb2d --- /dev/null +++ b/use-cases/vpc/lookup_vpc_via_tags.yaml @@ -0,0 +1,19 @@ +--- +# Blog applications + +version: '3.8' + +x-vpc: + Lookup: + VpcId: + Tags: + Name: e2e-lookup-vpc + AppSubnets: + Tags: + vpc::usage: application + StorageSubnets: + Tags: + vpc::usage: storage + PublicSubnets: + Tags: + vpc::usage: public diff --git a/use-cases/vpc/no_nats_no_endpoints.yaml b/use-cases/vpc/no_nats_no_endpoints.yaml new file mode 100644 index 000000000..271541064 --- /dev/null +++ b/use-cases/vpc/no_nats_no_endpoints.yaml @@ -0,0 +1,5 @@ +x-vpc: + Properties: + VpcCidr: 192.168.220.0/24 # A simple CIDR with plenty of room for the deployment. + DisableNat: True # Although the Public, App and Storage subnets are created, no NAT nor route is created. + Endpoints: {} # Set to {} to disable creating the default VPC endpoints Compose-X use. We won't be needing them.