diff --git a/Dockerfile b/Dockerfile index e361c0ac4..81d5956f0 100755 --- a/Dockerfile +++ b/Dockerfile @@ -14,9 +14,9 @@ ARG USE_DEVOPS=YES ARG USERNAME=cd3user ARG USER_GID=$USER_UID # Whether to download Provider as part of image creation -ARG DOWNLOAD_PROVIDER=YES +ARG DOWNLOAD_PROVIDER=NO # TF Provider version -ARG TF_OCI_PROVIDER=6.30.0 +ARG TF_OCI_PROVIDER=7.8.0 ARG TF_NULL_PROVIDER=3.2.3 RUN microdnf install -y sudo && \ @@ -31,6 +31,7 @@ RUN microdnf install -y sudo && \ microdnf install -y wget && \ microdnf install -y unzip && \ microdnf install -y graphviz && \ + ln -sf /usr/bin/python3 /usr/bin/python && \ echo 'alias vi="vim"' >> /etc/bashrc USER $USERNAME diff --git a/README.md b/README.md index 62cedce22..068a81a4a 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@
- [What's New](https://github.com/oracle-devrel/cd3-automation-toolkit/releases/tag/v2025.1.2)  • [Excel Templates](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/excel-templates/)  • [CD3 Docs](https://oracle-devrel.github.io/cd3-automation-toolkit/) •  [Watch & Learn](https://www.youtube.com/playlist?list=PLPIzp-E1msrbJ3WawXVhzimQnLw5iafcp)  • [Blogs & Tutorials](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/tutorials/)  • [Livelabs](https://apexapps.oracle.com/pls/apex/f?p=133:180:112501098061930::::wid:3724)  • [Slack Channel](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/queries) + [What's New](https://github.com/oracle-devrel/cd3-automation-toolkit/releases/tag/v2025.1.3)  • [Excel Templates](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/excel-templates/)  • [CD3 Docs](https://oracle-devrel.github.io/cd3-automation-toolkit/) •  [Watch & Learn](https://www.youtube.com/playlist?list=PLPIzp-E1msrbJ3WawXVhzimQnLw5iafcp)  • [Blogs & Tutorials](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/tutorials/)  • [Livelabs](https://apexapps.oracle.com/pls/apex/f?p=133:180:112501098061930::::wid:3724)  • [Slack Channel](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/queries)
diff --git a/cd3_automation_toolkit/Database/create_terraform_mysql_configuration.py b/cd3_automation_toolkit/Database/create_terraform_mysql_configuration.py index 6489c9cef..3ca3504bb 100644 --- a/cd3_automation_toolkit/Database/create_terraform_mysql_configuration.py +++ b/cd3_automation_toolkit/Database/create_terraform_mysql_configuration.py @@ -65,6 +65,9 @@ def make_config_value(config): for i in df.index: # Get values from row region = str(df.loc[i, 'Region']).strip() + if (region in commonTools.endNames): + break + region = region.strip().lower() compartment_name = str(df.loc[i, 'Compartment Name']).strip() display_name = str(df.loc[i, 'Display Name']).strip() description = str(df.loc[i, 'Description']).strip() diff --git a/cd3_automation_toolkit/Database/create_terraform_mysql_db.py b/cd3_automation_toolkit/Database/create_terraform_mysql_db.py index 08117989e..35cd3fced 100644 --- a/cd3_automation_toolkit/Database/create_terraform_mysql_db.py +++ b/cd3_automation_toolkit/Database/create_terraform_mysql_db.py @@ -50,7 +50,7 @@ def create_terraform_mysql_db(inputfile, outdir, service_dir, prefix, ct): # Iterate over rows for i in df.index: - region = str(df.loc[i, 'Region']).strip().lower() + region = str(df.loc[i, 'Region']).strip() if (region in commonTools.endNames): break diff --git a/cd3_automation_toolkit/Database/templates/mysql-template b/cd3_automation_toolkit/Database/templates/mysql-template index 4c8b1128b..e2beab49c 100644 --- a/cd3_automation_toolkit/Database/templates/mysql-template +++ b/cd3_automation_toolkit/Database/templates/mysql-template @@ -25,10 +25,6 @@ mysql_db_system = { mysql_db_system_fault_domain = "{{ fault_domain }}" configuration_compartment_id = "{{ configuration_compartment_id }}" configuration_id = "{{ configuration_id }}" - {% if depends_on_mysql_configuration %} - # Add explicit depends_on to ensure configuration is created first - depends_on = ["module.mysql_configurations[\"{{ configuration_id }}\"]"] - {% endif %} mysql_shape_name = "{{ shape }}" vcn_names = "{{ vcn_names }}" subnet_id = "{{ subnet_id }}" diff --git a/cd3_automation_toolkit/DeveloperServices/OKE/templates/nodepool-template b/cd3_automation_toolkit/DeveloperServices/OKE/templates/nodepool-template index 2a73c27e6..9fa4bbbad 100644 --- a/cd3_automation_toolkit/DeveloperServices/OKE/templates/nodepool-template +++ b/cd3_automation_toolkit/DeveloperServices/OKE/templates/nodepool-template @@ -39,6 +39,12 @@ nodepools = { {% if boot_volume_size_in_gbs %} boot_volume_size_in_gbs = {{ boot_volume_size_in_gbs }} {% endif %} + {% if is_pv_encryption_in_transit_enabled %} + is_pv_encryption_in_transit_enabled = {{ is_pv_encryption_in_transit_enabled }} + {% endif %} + {% if init_script_path %} + init_script_path = "{{ init_script_path }}" + {% endif %} {% if oke_labels and oke_labels != 'nan' and defined_tags != '' and oke_labels != [['nan']] %} {% if oke_labels[0] %} initial_node_labels = { diff --git a/cd3_automation_toolkit/Identity/Users/export_users_nonGreenField.py b/cd3_automation_toolkit/Identity/Users/export_users_nonGreenField.py index 22062a0ac..df9319079 100644 --- a/cd3_automation_toolkit/Identity/Users/export_users_nonGreenField.py +++ b/cd3_automation_toolkit/Identity/Users/export_users_nonGreenField.py @@ -112,11 +112,14 @@ def export_users(inputfile, outdir, service_dir, config, signer, ct,export_domai domain_name = domain_key.split("@")[1] domain_client = oci.identity_domains.IdentityDomainsClient(config=config, signer=signer,retry_strategy=oci.retry.DEFAULT_RETRY_STRATEGY, service_endpoint=idcs_endpoint) - list_users_response = domain_client.list_users() # change this to pagination once api supports - users = list_users_response.data.resources - while list_users_response.has_next_page: - list_users_response = domain_client.list_users(page=list_users_response.next_page) - users.extend(list_users_response.data.resources) + users = [] + next_page = None + while True: + response = domain_client.list_users(page=next_page) + users.extend(response.data.resources) + if not response.next_page or len(users) == response.data.total_results: + break + next_page = response.next_page index = 0 for user in users: diff --git a/cd3_automation_toolkit/Identity/export_identity_nonGreenField.py b/cd3_automation_toolkit/Identity/export_identity_nonGreenField.py index 39a49d4ff..be2266c59 100644 --- a/cd3_automation_toolkit/Identity/export_identity_nonGreenField.py +++ b/cd3_automation_toolkit/Identity/export_identity_nonGreenField.py @@ -359,13 +359,18 @@ def process_group(grp_info, members_list,membership_id_list, domain_name, is_dyn domain_name = domain_key.split("@")[1] domain_client = oci.identity_domains.IdentityDomainsClient(config=config, signer=signer,retry_strategy=oci.retry.DEFAULT_RETRY_STRATEGY, service_endpoint=idcs_endpoint) - list_groups_response = domain_client.list_groups(attributes=['members'], attribute_sets=['all']) - groups = list_groups_response.data.resources - page_done = [] - while list_groups_response.has_next_page and list_groups_response.next_page not in page_done: - page_done.append(list_groups_response.next_page) - list_groups_response = domain_client.list_groups(attributes=['members'], attribute_sets=['all'],page=list_groups_response.next_page) - groups.extend(list_groups_response.data.resources) + groups = [] + next_page = None + while True: + response = domain_client.list_groups( + attributes=['members'], + attribute_sets=['all'], + page=next_page + ) + groups.extend(response.data.resources) + if not response.next_page or len(groups) == response.data.total_results: + break + next_page = response.next_page for grp_info in groups: if grp_info.display_name in ["Domain_Administrators", "All Domain Users", "Administrators"]: @@ -374,16 +379,18 @@ def process_group(grp_info, members_list,membership_id_list, domain_name, is_dyn members_list = [section.name for section in grp_info.members if section and section.name] if grp_info.members else [] importCommands, values_for_column_groups = process_group(grp_info, members_list,[], domain_name, is_dynamic=False, importCommands=importCommands, values_for_column_groups=values_for_column_groups) - dyngroups_response = domain_client.list_dynamic_resource_groups(attributes=['matching_rule'], - attribute_sets=['all'] - ) - dyngroups = dyngroups_response.data.resources - while dyngroups_response.has_next_page: - dyngroups_response = domain_client.list_dynamic_resource_groups(attributes=['matching_rule'], - attribute_sets=['all'], - page=dyngroups_response.next_page - ) - dyngroups.extend(dyngroups_response.data.resources) + dyngroups = [] + next_page = None + while True: + response = domain_client.list_dynamic_resource_groups( + attributes=['matching_rule'], + attribute_sets=['all'], + page=next_page + ) + dyngroups.extend(response.data.resources) + if not response.next_page or len(dyngroups) == response.data.total_results: + break + next_page = response.next_page for dg in dyngroups: total_g += 1 diff --git a/cd3_automation_toolkit/Network/BaseNetwork/exportRoutetable.py b/cd3_automation_toolkit/Network/BaseNetwork/exportRoutetable.py index 72f3900b5..4e1e3b9f2 100644 --- a/cd3_automation_toolkit/Network/BaseNetwork/exportRoutetable.py +++ b/cd3_automation_toolkit/Network/BaseNetwork/exportRoutetable.py @@ -209,7 +209,11 @@ def insert_values(routetable,values_for_column,region,comp_name,name,routerule,e elif (routerule != None and col_header == 'Route Destination Object'): network_entity_id = routerule.network_entity_id - network_entity_name = get_network_entity_name(config, signer, network_entity_id,export_tags) + try: + network_entity_name = get_network_entity_name(config, signer, network_entity_id,export_tags) + except Exception as e: + print("\nCheck route rules for Route Table: '"+routetable.display_name+"' and Re-Try") + exit(1) values_for_column[col_header].append(network_entity_name) if ('internetgateway' in network_entity_id): if (routerule.destination not in values_for_vcninfo['igw_destinations']): @@ -240,7 +244,11 @@ def insert_values_drg(routetable,import_drg_route_distribution_name,values_for_c elif (routerule != None and col_header == 'Next Hop Attachment'): next_hop_attachment_id=routerule.next_hop_drg_attachment_id - network_entity_name = get_network_entity_name(config, signer, next_hop_attachment_id,export_tags) + try: + network_entity_name = get_network_entity_name(config, signer, next_hop_attachment_id,export_tags) + except Exception as e: + print("\nCheck route rules for DRG Route Table: '"+routetable.display_name+"' and Re-Try") + exit(1) values_for_column_drg[col_header].append(network_entity_name) else: diff --git a/cd3_automation_toolkit/Network/BaseNetwork/export_network_nonGreenField.py b/cd3_automation_toolkit/Network/BaseNetwork/export_network_nonGreenField.py index ae7a17311..49fdf86be 100644 --- a/cd3_automation_toolkit/Network/BaseNetwork/export_network_nonGreenField.py +++ b/cd3_automation_toolkit/Network/BaseNetwork/export_network_nonGreenField.py @@ -9,6 +9,7 @@ from .exportSeclist import export_seclist from .exportNSG import export_nsg import subprocess as sp +from pathlib import Path sys.path.append(os.getcwd() + "/..") from commonTools import * @@ -348,6 +349,9 @@ def get_drg_rt_name(drg_rpc_attachment_list, source_rpc_id, rpc_source_client, d def get_rpc_resources(source_region, SOURCE_RPC_LIST, dest_rpc_dict, rpc_source_client, ct, values_for_column, ntk_compartment_name, outdir, drg_info, drg_attachment_info, state_rpc): + # Get path to OCI_Regions file relative to current rpc script + oci_regions_path = Path(__file__).resolve().parents[2] / "OCI_Regions" + # Variables dest_rpc_drg_name = "" src_drg_rt_name = "" @@ -456,7 +460,8 @@ def get_comp_details(comp_data): # Fetch Dest DRG RT name, id if dest_drg_rpc_attachment_list.data: dest_drg_rt_name, dest_drg_rt_id = get_drg_rt_name(dest_drg_rpc_attachment_list, - dest_rpc_id, client,dest_rpc.drg_id) + dest_rpc_id, client, + dest_rpc.drg_id) if dest_drg_rt_name is not None: # Fetch source DRG import route distribution id, name @@ -467,15 +472,24 @@ def get_comp_details(comp_data): dest_import_rt_info = client.get_drg_route_distribution( drg_route_distribution_id=dest_drg_rt_import_dist_id) dest_drg_rt_dist_info = dest_import_rt_info - dest_drg_rt_import_dist_name = getattr(dest_import_rt_info.data, "display_name") + dest_drg_rt_import_dist_name = getattr(dest_import_rt_info.data, + "display_name") dest_import_rt_statements = client.list_drg_route_distribution_statements( drg_route_distribution_id=dest_drg_rt_import_dist_id) - tf_resource = f'module.rpcs[\\"{rpc_tf_name}\\"].oci_core_remote_peering_connection.{source_region.lower()}_{region.lower()}_requester_rpc[\\"region\\"]' + source_region_for_tf = next( + line.split(':')[1].strip().replace("-", "_") for line in open(oci_regions_path) if + line.startswith(f"{source_region.lower()}:")) + + region_for_tf = next( + line.split(':')[1].strip().replace("-", "_") for line in open(oci_regions_path) if + line.startswith(f"{region.lower()}:")) + + tf_resource = f'module.rpcs[\\"{rpc_tf_name}\\"].oci_core_remote_peering_connection.{source_region_for_tf}_{region_for_tf}_requester_rpc[\\"region\\"]' if tf_resource not in state_rpc["resources"]: importCommands_rpc["global"].write( f'\n{tf_or_tofu} import "{tf_resource}" {str(source_rpc_id)}') - tf_resource = f'module.rpcs[\\"{rpc_tf_name}\\"].oci_core_remote_peering_connection.{source_region.lower()}_{region.lower()}_accepter_rpc[\\"region\\"]' + tf_resource = f'module.rpcs[\\"{rpc_tf_name}\\"].oci_core_remote_peering_connection.{source_region_for_tf}_{region_for_tf}_accepter_rpc[\\"region\\"]' if tf_resource not in state_rpc["resources"]: importCommands_rpc["global"].write( f'\n{tf_or_tofu} import "{tf_resource}" {str(dest_rpc_id)}') @@ -589,6 +603,7 @@ def get_comp_details(comp_data): rpc_safe_file["global"].close() + def export_major_objects(inputfile, outdir, service_dir, config, signer, ct, export_compartments=[], export_regions=[], export_tags=[]): global sheet_dict_vcns diff --git a/cd3_automation_toolkit/Network/BaseNetwork/templates/major-objects-vcns-template b/cd3_automation_toolkit/Network/BaseNetwork/templates/major-objects-vcns-template index 78382e9d9..cc99e8415 100644 --- a/cd3_automation_toolkit/Network/BaseNetwork/templates/major-objects-vcns-template +++ b/cd3_automation_toolkit/Network/BaseNetwork/templates/major-objects-vcns-template @@ -40,7 +40,7 @@ vcns = { {% endif %} {% if ipv6private_cidr_blocks and ipv6private_cidr_blocks != [] %} - ipv6private_cidr_blocks = [ {{ ipv6private_cidr_blocks }} ] + ipv6private_cidr_blocks = [ "{{ ipv6private_cidr_blocks }}" ] {% endif %} {% if is_oracle_gua_allocation_enabled and is_oracle_gua_allocation_enabled != "" %} diff --git a/cd3_automation_toolkit/Network/DNS/create_dns_rrsets.py b/cd3_automation_toolkit/Network/DNS/create_dns_rrsets.py index 1bd56ce02..5fb822620 100644 --- a/cd3_automation_toolkit/Network/DNS/create_dns_rrsets.py +++ b/cd3_automation_toolkit/Network/DNS/create_dns_rrsets.py @@ -79,7 +79,8 @@ def create_terraform_dns_rrsets(inputfile, outdir, service_dir, prefix, ct): zone_name = str(df["Zone"][i]).strip() domain = str(df["Domain"][i]).strip() rtype = str(df["RType"][i]).strip() - if 'nan' in [view_name,zone_name,domain,rtype]: + values = [domain,rtype] + if not all(v == 'nan' for v in values) and 'nan' in values: print(f'Required parameters for record creation are missing. Skipping record creation for row : {i+3}') continue rrset_tf_name = str(view_name + "_" + zone_name+ "_" + domain+ "_" + rtype).replace(".", "_") diff --git a/cd3_automation_toolkit/Network/DNS/export_dns_views_zones_records.py b/cd3_automation_toolkit/Network/DNS/export_dns_views_zones_records.py index f5a0a0f32..961dff9d7 100644 --- a/cd3_automation_toolkit/Network/DNS/export_dns_views_zones_records.py +++ b/cd3_automation_toolkit/Network/DNS/export_dns_views_zones_records.py @@ -19,7 +19,7 @@ def get_rrset(zone_data,dns_client,record_default): zone_records = oci.pagination.list_call_get_all_results(dns_client.get_zone_records,zone_data.id).data for zone_record in zone_records.items: - if record_default == 'n' and zone_record.is_protected == True: + if (record_default == 'n' and zone_record.is_protected == True) or zone_record.rtype in ["SOA","NS"]: continue tmpdict = {} domain = zone_record.domain @@ -75,7 +75,7 @@ def print_data(region, ntk_compartment_name, rrset, zone_data, view_data, values values_for_column = commonTools.export_tags(view_data, col_header, values_for_column) -def print_empty_view(region, ntk_compartment_name, view_data, values_for_column): +def print_empty_view(region, ntk_compartment_name, view_data, values_for_column,zone_name=""): for col_header in values_for_column: if col_header == 'Region': values_for_column[col_header].append(region) @@ -85,7 +85,7 @@ def print_empty_view(region, ntk_compartment_name, view_data, values_for_column) values_for_column[col_header].append(view_data.display_name) elif col_header == 'Zone': - values_for_column[col_header].append("") + values_for_column[col_header].append(zone_name) elif col_header == 'Domain': values_for_column[col_header].append("") elif col_header == 'RType': @@ -214,12 +214,15 @@ def export_dns_views_zones_rrsets(inputfile, outdir, service_dir, config, signer if rrsets: for rrset in rrsets.values(): print_data(region, ntk_compartment_name, rrset, zone_data, view_data, values_for_column,state) - tf_resource = f'module.dns-zones[\\"{zone_tf_name}\\"].oci_dns_zone.zone' - if tf_resource not in state["resources"]: - importCommands[region.lower()] += f'\n{tf_or_tofu} import "{tf_resource}" {str(zone_data.id)}' else: - print_empty_view(region, ntk_compartment_name, view_data, values_for_column) + print_empty_view(region, ntk_compartment_name, view_data, values_for_column,zone_name=zone_data.name) + + tf_resource = f'module.dns-zones[\\"{zone_tf_name}\\"].oci_dns_zone.zone' + if tf_resource not in state["resources"]: + importCommands[ + region.lower()] += f'\n{tf_or_tofu} import "{tf_resource}" {str(zone_data.id)}' + else: print_empty_view(region, ntk_compartment_name, view_data, values_for_column) if print_zone==False: diff --git a/cd3_automation_toolkit/Network/Global/create_rpc_resources.py b/cd3_automation_toolkit/Network/Global/create_rpc_resources.py index 5dbd83592..23f1ed6d7 100755 --- a/cd3_automation_toolkit/Network/Global/create_rpc_resources.py +++ b/cd3_automation_toolkit/Network/Global/create_rpc_resources.py @@ -20,13 +20,16 @@ # Setting current working dir. owd = os.getcwd() +# Get path to OCI_Regions file relative to current script +oci_regions_path = Path(__file__).resolve().parents[2] / "OCI_Regions" -def find_subscribed_regions(inputfile, outdir, service_dir, prefix, config,signer,auth_mechanism): + +def find_subscribed_regions(inputfile, outdir, service_dir, prefix, config, signer, auth_mechanism): subs_region_list = [] new_subs_region_list = [] subs_region_pairs = [] - idc = IdentityClient(config=config, retry_strategy=oci.retry.DEFAULT_RETRY_STRATEGY,signer=signer) + idc = IdentityClient(config=config, retry_strategy=oci.retry.DEFAULT_RETRY_STRATEGY, signer=signer) regionsubscriptions = idc.list_region_subscriptions(tenancy_id=config['tenancy']) for reg in regionsubscriptions.data: @@ -35,10 +38,14 @@ def find_subscribed_regions(inputfile, outdir, service_dir, prefix, config,signe region_name = getattr(reg, 'region_name') subs_region_list.append(region_name) - for item in subs_region_list: - new_subs_region_list.append(item.split("-")[1]) + # for item in subs_region_list: + # new_subs_region_list.append(item.split("-")[1]) + # + # for item in list(itertools.permutations(new_subs_region_list, 2)): + # subs_region_pairs.append(item[0] + "##" + item[1]) - for item in list(itertools.permutations(new_subs_region_list, 2)): + new_subs_region_list = subs_region_list.copy() + for item in itertools.permutations(new_subs_region_list, 2): subs_region_pairs.append(item[0] + "##" + item[1]) # Load the template file @@ -64,9 +71,11 @@ def find_subscribed_regions(inputfile, outdir, service_dir, prefix, config,signe with open("rpc.tf", "r+") as provider_file: provider_file_data = provider_file.read().rstrip() if auth_mechanism == 'instance_principal': - provider_file_data = provider_file_data.replace("provider \"oci\" {", "provider \"oci\" {\nauth = \"InstancePrincipal\"") + provider_file_data = provider_file_data.replace("provider \"oci\" {", + "provider \"oci\" {\nauth = \"InstancePrincipal\"") if auth_mechanism == 'session_token': - provider_file_data = provider_file_data.replace("provider \"oci\" {", "provider \"oci\" {\nauth = \"SecurityToken\"\nconfig_file_profile = \"DEFAULT\"") + provider_file_data = provider_file_data.replace("provider \"oci\" {", + "provider \"oci\" {\nauth = \"SecurityToken\"\nconfig_file_profile = \"DEFAULT\"") f = open("rpc.tf", "w+") f.write(provider_file_data) @@ -99,11 +108,11 @@ def find_subscribed_regions(inputfile, outdir, service_dir, prefix, config,signe # Execution of the code begins here -def create_rpc_resource(inputfile, outdir, service_dir, prefix, auth_mechanism, config_file,ct, non_gf_tenancy): +def create_rpc_resource(inputfile, outdir, service_dir, prefix, auth_mechanism, config_file, ct, non_gf_tenancy): # Call pre-req func rpc_safe_file = {} config, signer = ct.authenticate(auth_mechanism, config_file) - find_subscribed_regions(inputfile, outdir, service_dir, prefix, config,signer,auth_mechanism) + find_subscribed_regions(inputfile, outdir, service_dir, prefix, config, signer, auth_mechanism) os.chdir(owd) @@ -138,9 +147,8 @@ def create_rpc_resource(inputfile, outdir, service_dir, prefix, auth_mechanism, for eachregion in ct.all_regions: tfStr["global"] = '' - match_list = [] - for i in df.index: + for i in range(len(df) - 1): if str(df.loc[i, 'Attached To']).lower().startswith("rpc"): region = str(df.loc[i, 'Region']) region = region.strip().lower() @@ -215,7 +223,7 @@ def create_rpc_resource(inputfile, outdir, service_dir, prefix, auth_mechanism, accepter_compartment_name = df.loc[i + 1, 'Compartment Name'] accepter_compartment_name = str(accepter_compartment_name) accepter_compartment_name = commonTools.check_tf_variable(accepter_compartment_name) - tempdict = {'rpc_tf_name': display_tf_name, 'rpc_name':columnvalue, + tempdict = {'rpc_tf_name': display_tf_name, 'rpc_name': columnvalue, 'accepter_rpc_display_name': accepter_rpc_display_name, 'accepter_compartment_name': accepter_compartment_name} @@ -228,12 +236,19 @@ def create_rpc_resource(inputfile, outdir, service_dir, prefix, auth_mechanism, if columnname == 'Attached To': accepter_compartment_var_name = columnvalue.strip().split("::") accepter_region = accepter_compartment_var_name[1] + accepter_region = next( + line.split(':')[1].strip() for line in open(oci_regions_path) if + line.startswith(f"{accepter_region}:")) accepter_drg_name = accepter_compartment_var_name[2] tempdict = {'accepter_region': accepter_region.lower(), 'accepter_drg_name': accepter_drg_name} if columnname == 'Region': requester_region = columnvalue.strip().lower() + requester_region = next( + line.split(':')[1].strip() for line in open(oci_regions_path) if + line.startswith(f"{requester_region}:")) + tempdict = {'requester_region': requester_region} if columnname == 'DRG Name': @@ -293,4 +308,3 @@ def create_rpc_resource(inputfile, outdir, service_dir, prefix, auth_mechanism, print(outfile + " has been created inside Global dir") oname.write(tfStr["global"]) oname.close() - diff --git a/cd3_automation_toolkit/Network/Global/templates/rpc-module/rpc-provider-terraform-template b/cd3_automation_toolkit/Network/Global/templates/rpc-module/rpc-provider-terraform-template index f3050e7be..dd31d0fd9 100755 --- a/cd3_automation_toolkit/Network/Global/templates/rpc-module/rpc-provider-terraform-template +++ b/cd3_automation_toolkit/Network/Global/templates/rpc-module/rpc-provider-terraform-template @@ -12,11 +12,10 @@ terraform { source = "oracle/oci" configuration_aliases = [ {% for region in subscribed_regions %} - {% set region_keys = region.split('-') %} - oci.{{region_keys[1]}}, + oci.{{ region.replace('-', '_') }}{{ "," if not loop.last }} {% endfor %} - ] } } } + diff --git a/cd3_automation_toolkit/Network/Global/templates/rpc-module/rpc-source-destination-terraform-template b/cd3_automation_toolkit/Network/Global/templates/rpc-module/rpc-source-destination-terraform-template index 8d4e9934b..c0896babb 100755 --- a/cd3_automation_toolkit/Network/Global/templates/rpc-module/rpc-source-destination-terraform-template +++ b/cd3_automation_toolkit/Network/Global/templates/rpc-module/rpc-source-destination-terraform-template @@ -1,166 +1,167 @@ # Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. -# -####################### -# Resource Block - RPC -# Create RPC -####################### - data "oci_identity_regions" "all_regions" {} - - locals { - region_map = { for region in data.oci_identity_regions.all_regions.regions : region.key => region.name } - } +data "oci_identity_regions" "all_regions" {} + +locals { + region_map = { for region in data.oci_identity_regions.all_regions.regions : region.key => region.name } +} {% for region in subscribed_regions %} {% set region_keys = region.split('##') %} - ########################################### - # Data Block - Network - {{region_keys[0]}} to {{region_keys[1]}} - ########################################### - - data "oci_core_drgs" "{{region_keys[0]}}_{{region_keys[1]}}_requester_drg" { - for_each = var.requester_region["region"] == "{{region_keys[0]}}" && var.accepter_region["region"] == "{{region_keys[1]}}" ? var.requester_region : {} - #Required - compartment_id = var.requester_compartment_id - filter { - name = "display_name" - values = [var.requester_drg_name] - } - provider = oci.{{region_keys[0]}} - } - - data "oci_core_drg_route_tables" "{{region_keys[0]}}_{{region_keys[1]}}_requester_drg_route_tables" { - for_each = var.requester_region["region"] == "{{region_keys[0]}}" && var.accepter_region["region"] == "{{region_keys[1]}}" ? var.requester_region : {} - #Required - drg_id = data.oci_core_drgs.{{region_keys[0]}}_{{region_keys[1]}}_requester_drg[each.key].drgs[0].id - - provider = oci.{{region_keys[0]}} - } - - locals { - {{region_keys[0]}}_{{region_keys[1]}}_requester_drg_rt_id = var.requester_region["region"] == "{{region_keys[0]}}" && var.accepter_region["region"] == "{{region_keys[1]}}" ? [for k,v in data.oci_core_drg_route_tables.{{region_keys[0]}}_{{region_keys[1]}}_requester_drg_route_tables["region"].drg_route_tables : v.id if v.display_name == var.requester_drg_rt_name ] : null - {{region_keys[0]}}_{{region_keys[1]}}_peer_region_name = var.requester_region["region"] == "{{region_keys[0]}}" && var.accepter_region["region"] == "{{region_keys[1]}}" ? length(split(".",oci_core_remote_peering_connection.{{region_keys[0]}}_{{region_keys[1]}}_requester_rpc["region"].drg_id)[3]) > 3 ? split(".",oci_core_remote_peering_connection.{{region_keys[0]}}_{{region_keys[1]}}_requester_rpc["region"].drg_id)[3] : lookup(local.region_map,upper(split(".",oci_core_remote_peering_connection.{{region_keys[0]}}_{{region_keys[1]}}_requester_rpc["region"].drg_id)[3]),"NOT FOUND") : null - } - - ########################################### - # Resource Block - Network - # Create Requester Remote Peering Connection - ########################################### - - resource "oci_core_remote_peering_connection" "{{region_keys[0]}}_{{region_keys[1]}}_requester_rpc" { - for_each = var.requester_region["region"] == "{{region_keys[0]}}" && var.accepter_region["region"] == "{{region_keys[1]}}" ? var.requester_region : {} - - #Required - compartment_id = var.requester_compartment_id - drg_id = data.oci_core_drgs.{{region_keys[0]}}_{{region_keys[1]}}_requester_drg[each.key].drgs[0].id - - #Optional - display_name = var.display_name - peer_id = var.requester_peer_id - peer_region_name = null - defined_tags = var.defined_tags - freeform_tags = var.freeform_tags - - lifecycle { - ignore_changes = [defined_tags["Oracle-Tags.CreatedOn"], - defined_tags["Oracle-Tags.CreatedBy"]] - } - - provider = oci.{{region_keys[0]}} - } - - - #################################################### - # Resource Block - Network - # Create Requester Dynamic Routing Gateway Attachment - #################################################### - resource "oci_core_drg_attachment_management" "{{region_keys[0]}}_{{region_keys[1]}}_requester_drg_attachment_management" { - for_each = var.requester_region["region"] == "{{region_keys[0]}}" && var.accepter_region["region"] == "{{region_keys[1]}}" ? var.requester_region : {} - - #Required - attachment_type = var.attachment_type - compartment_id = var.requester_compartment_id - network_id = oci_core_remote_peering_connection.{{region_keys[0]}}_{{region_keys[1]}}_requester_rpc[each.key].id - drg_id = data.oci_core_drgs.{{region_keys[0]}}_{{region_keys[1]}}_requester_drg[each.key].drgs[0].id - - #Optional - #display_name = var.display_name - drg_route_table_id = element(local.{{region_keys[0]}}_{{region_keys[1]}}_requester_drg_rt_id,0) - - provider = oci.{{region_keys[0]}} - } - - - ###################### - # Data Block - Network - ###################### - data "oci_core_drgs" "{{region_keys[0]}}_{{region_keys[1]}}_accepter_drg" { - for_each = var.requester_region["region"] == "{{region_keys[0]}}" && var.accepter_region["region"] == "{{region_keys[1]}}" ? var.accepter_region : {} - #Required - compartment_id = var.accepter_compartment_id - filter { - name = "display_name" - values = [var.accepter_drg_name] - } - provider = oci.{{region_keys[1]}} - } - - data "oci_core_drg_route_tables" "{{region_keys[0]}}_{{region_keys[1]}}_accepter_drg_route_tables" { - for_each = var.requester_region["region"] == "{{region_keys[0]}}" && var.accepter_region["region"] == "{{region_keys[1]}}" ? var.accepter_region : {} - #Required - drg_id = data.oci_core_drgs.{{region_keys[0]}}_{{region_keys[1]}}_accepter_drg[each.key].drgs[0].id - - provider = oci.{{region_keys[1]}} - } - - locals { - {{region_keys[0]}}_{{region_keys[1]}}_accepter_drg_rt_id = var.requester_region["region"] == "{{region_keys[0]}}" && var.accepter_region["region"] == "{{region_keys[1]}}" ? [for k,v in data.oci_core_drg_route_tables.{{region_keys[0]}}_{{region_keys[1]}}_accepter_drg_route_tables["region"].drg_route_tables : v.id if v.display_name == var.accepter_drg_rt_name ] : null - } - - ########################################### - # Resource Block - Network - # Create Accepter Remote Peering Connection - ########################################### - resource "oci_core_remote_peering_connection" "{{region_keys[0]}}_{{region_keys[1]}}_accepter_rpc" { - for_each = var.requester_region["region"] == "{{region_keys[0]}}" && var.accepter_region["region"] == "{{region_keys[1]}}" ? var.accepter_region : {} - - compartment_id = var.accepter_compartment_id - drg_id = data.oci_core_drgs.{{region_keys[0]}}_{{region_keys[1]}}_accepter_drg[each.key].drgs[0].id - - #Optional - display_name = var.accepter_rpc_display_name - peer_id = oci_core_remote_peering_connection.{{region_keys[0]}}_{{region_keys[1]}}_requester_rpc["region"].id - peer_region_name = local.{{region_keys[0]}}_{{region_keys[1]}}_peer_region_name - defined_tags = var.defined_tags - freeform_tags = var.freeform_tags - - lifecycle { - ignore_changes = [defined_tags["Oracle-Tags.CreatedOn"], - defined_tags["Oracle-Tags.CreatedBy"]] - } - - provider = oci.{{region_keys[1]}} - } - - #################################################### - # Resource Block - Network - # Create Accepter Dynamic Routing Gateway Attachment - #################################################### - resource "oci_core_drg_attachment_management" "{{region_keys[0]}}_{{region_keys[1]}}_accepter_drg_attachment_management" { - for_each = var.requester_region["region"] == "{{region_keys[0]}}" && var.accepter_region["region"] == "{{region_keys[1]}}" ? var.accepter_region : {} - - #Required - attachment_type = var.attachment_type - compartment_id = var.accepter_compartment_id - network_id = oci_core_remote_peering_connection.{{region_keys[0]}}_{{region_keys[1]}}_accepter_rpc[each.key].id - drg_id = data.oci_core_drgs.{{region_keys[0]}}_{{region_keys[1]}}_accepter_drg[each.key].drgs[0].id - - #Optional - #display_name = var.display_name - drg_route_table_id = element(local.{{region_keys[0]}}_{{region_keys[1]}}_accepter_drg_rt_id,0) - - provider = oci.{{region_keys[1]}} - } -#################################################END################################################################# + {% set requester_region_key = region_keys[0] %} + {% set accepter_region_key = region_keys[1] %} + + {# Make terraform-safe keys by replacing '-' with '_' #} + {% set requester_key_safe = requester_region_key.replace('-', '_') %} + {% set accepter_key_safe = accepter_region_key.replace('-', '_') %} + + {# Use original region names as provider aliases #} + {% set requester_alias = requester_key_safe %} + {% set accepter_alias = accepter_key_safe %} + + + + ########################################### + # Data Block - Network - {{requester_region_key}} to {{accepter_region_key}} + ########################################### + + data "oci_core_drgs" "{{requester_key_safe}}_{{accepter_key_safe}}_requester_drg" { + for_each = var.requester_region["region"] == "{{requester_region_key}}" && var.accepter_region["region"] == "{{accepter_region_key}}" ? var.requester_region : {} + compartment_id = var.requester_compartment_id + + filter { + name = "display_name" + values = [var.requester_drg_name] + } + + provider = oci.{{ requester_alias }} + } + + data "oci_core_drg_route_tables" "{{requester_key_safe}}_{{accepter_key_safe}}_requester_drg_route_tables" { + for_each = var.requester_region["region"] == "{{requester_region_key}}" && var.accepter_region["region"] == "{{accepter_region_key}}" ? var.requester_region : {} + drg_id = data.oci_core_drgs.{{requester_key_safe}}_{{accepter_key_safe}}_requester_drg[each.key].drgs[0].id + + provider = oci.{{ requester_alias }} + } + + locals { + {{requester_key_safe}}_{{accepter_key_safe}}_requester_drg_rt_id = var.requester_region["region"] == "{{requester_region_key}}" && var.accepter_region["region"] == "{{accepter_region_key}}" ? [for k,v in data.oci_core_drg_route_tables.{{requester_key_safe}}_{{accepter_key_safe}}_requester_drg_route_tables["region"].drg_route_tables : v.id if v.display_name == var.requester_drg_rt_name ] : null + + {{requester_key_safe}}_{{accepter_key_safe}}_peer_region_name = var.requester_region["region"] == "{{requester_region_key}}" && var.accepter_region["region"] == "{{accepter_region_key}}" ? length(split(".",oci_core_remote_peering_connection.{{requester_key_safe}}_{{accepter_key_safe}}_requester_rpc["region"].drg_id)[3]) > 3 ? split(".",oci_core_remote_peering_connection.{{requester_key_safe}}_{{accepter_key_safe}}_requester_rpc["region"].drg_id)[3] : lookup(local.region_map,upper(split(".",oci_core_remote_peering_connection.{{requester_key_safe}}_{{accepter_key_safe}}_requester_rpc["region"].drg_id)[3]),"NOT FOUND") : null + } + + ########################################### + # Resource Block - Network + # Create Requester Remote Peering Connection + ########################################### + resource "oci_core_remote_peering_connection" "{{requester_key_safe}}_{{accepter_key_safe}}_requester_rpc" { + for_each = var.requester_region["region"] == "{{requester_region_key}}" && var.accepter_region["region"] == "{{accepter_region_key}}" ? var.requester_region : {} + + compartment_id = var.requester_compartment_id + drg_id = data.oci_core_drgs.{{requester_key_safe}}_{{accepter_key_safe}}_requester_drg[each.key].drgs[0].id + + display_name = var.display_name + peer_id = var.requester_peer_id + peer_region_name = null + defined_tags = var.defined_tags + freeform_tags = var.freeform_tags + + lifecycle { + ignore_changes = [ + defined_tags["Oracle-Tags.CreatedOn"], + defined_tags["Oracle-Tags.CreatedBy"] + ] + } + + provider = oci.{{ requester_alias }} + } + + #################################################### + # Resource Block - Network + # Create Requester Dynamic Routing Gateway Attachment + #################################################### + resource "oci_core_drg_attachment_management" "{{requester_key_safe}}_{{accepter_key_safe}}_requester_drg_attachment_management" { + for_each = var.requester_region["region"] == "{{requester_region_key}}" && var.accepter_region["region"] == "{{accepter_region_key}}" ? var.requester_region : {} + + attachment_type = var.attachment_type + compartment_id = var.requester_compartment_id + network_id = oci_core_remote_peering_connection.{{requester_key_safe}}_{{accepter_key_safe}}_requester_rpc[each.key].id + drg_id = data.oci_core_drgs.{{requester_key_safe}}_{{accepter_key_safe}}_requester_drg[each.key].drgs[0].id + + drg_route_table_id = element(local.{{requester_key_safe}}_{{accepter_key_safe}}_requester_drg_rt_id,0) + + provider = oci.{{ requester_alias }} + } + + ###################### + # Data Block - Network + ###################### + data "oci_core_drgs" "{{requester_key_safe}}_{{accepter_key_safe}}_accepter_drg" { + for_each = var.requester_region["region"] == "{{requester_region_key}}" && var.accepter_region["region"] == "{{accepter_region_key}}" ? var.accepter_region : {} + + compartment_id = var.accepter_compartment_id + filter { + name = "display_name" + values = [var.accepter_drg_name] + } + + provider = oci.{{ accepter_alias }} + } + + data "oci_core_drg_route_tables" "{{requester_key_safe}}_{{accepter_key_safe}}_accepter_drg_route_tables" { + for_each = var.requester_region["region"] == "{{requester_region_key}}" && var.accepter_region["region"] == "{{accepter_region_key}}" ? var.accepter_region : {} + drg_id = data.oci_core_drgs.{{requester_key_safe}}_{{accepter_key_safe}}_accepter_drg[each.key].drgs[0].id + + provider = oci.{{ accepter_alias }} + } + + locals { + {{requester_key_safe}}_{{accepter_key_safe}}_accepter_drg_rt_id = var.requester_region["region"] == "{{requester_region_key}}" && var.accepter_region["region"] == "{{accepter_region_key}}" ? [for k,v in data.oci_core_drg_route_tables.{{requester_key_safe}}_{{accepter_key_safe}}_accepter_drg_route_tables["region"].drg_route_tables : v.id if v.display_name == var.accepter_drg_rt_name ] : null + } + + ########################################### + # Resource Block - Network + # Create Accepter Remote Peering Connection + ########################################### + resource "oci_core_remote_peering_connection" "{{requester_key_safe}}_{{accepter_key_safe}}_accepter_rpc" { + for_each = var.requester_region["region"] == "{{requester_region_key}}" && var.accepter_region["region"] == "{{accepter_region_key}}" ? var.accepter_region : {} + + compartment_id = var.accepter_compartment_id + drg_id = data.oci_core_drgs.{{requester_key_safe}}_{{accepter_key_safe}}_accepter_drg[each.key].drgs[0].id + + display_name = var.accepter_rpc_display_name + peer_id = oci_core_remote_peering_connection.{{requester_key_safe}}_{{accepter_key_safe}}_requester_rpc["region"].id + peer_region_name = local.{{requester_key_safe}}_{{accepter_key_safe}}_peer_region_name + defined_tags = var.defined_tags + freeform_tags = var.freeform_tags + + lifecycle { + ignore_changes = [ + defined_tags["Oracle-Tags.CreatedOn"], + defined_tags["Oracle-Tags.CreatedBy"] + ] + } + + provider = oci.{{ accepter_alias }} + } + + #################################################### + # Resource Block - Network + # Create Accepter Dynamic Routing Gateway Attachment + #################################################### + resource "oci_core_drg_attachment_management" "{{requester_key_safe}}_{{accepter_key_safe}}_accepter_drg_attachment_management" { + for_each = var.requester_region["region"] == "{{requester_region_key}}" && var.accepter_region["region"] == "{{accepter_region_key}}" ? var.accepter_region : {} + + attachment_type = var.attachment_type + compartment_id = var.accepter_compartment_id + network_id = oci_core_remote_peering_connection.{{requester_key_safe}}_{{accepter_key_safe}}_accepter_rpc[each.key].id + drg_id = data.oci_core_drgs.{{requester_key_safe}}_{{accepter_key_safe}}_accepter_drg[each.key].drgs[0].id + + drg_route_table_id = element(local.{{requester_key_safe}}_{{accepter_key_safe}}_accepter_drg_rt_id,0) + + provider = oci.{{ accepter_alias }} + } + {% endfor %} \ No newline at end of file diff --git a/cd3_automation_toolkit/Network/Global/templates/rpc-root-terraform-template b/cd3_automation_toolkit/Network/Global/templates/rpc-root-terraform-template index 85d160cfd..5b2004ec2 100755 --- a/cd3_automation_toolkit/Network/Global/templates/rpc-root-terraform-template +++ b/cd3_automation_toolkit/Network/Global/templates/rpc-root-terraform-template @@ -23,15 +23,15 @@ module "rpcs" { accepter_drg_name = each.value.accepter_drg_name accepter_drg_rt_name = each.value.accepter_drg_rt_name != "null" ? each.value.accepter_drg_rt_name : "Autogenerated Drg Route Table for RPC, VC, and IPSec attachments" accepter_rpc_display_name = each.value.accepter_rpc_display_name - defined_tags = each.value.defined_tags - freeform_tags = each.value.freeform_tags + defined_tags = each.value.defined_tags + freeform_tags = each.value.freeform_tags providers = { - {% for region in subscribed_regions %} - {% set region_keys = region.split('-') %} - oci.{{region_keys[1]}} = oci.{{region_keys[1]}} - {% endfor %} - } + {% for region in subscribed_regions %} + {% set alias = region.replace('-', '_') %} + oci.{{ alias }} = oci.{{ alias }} + {% endfor %} + } } ########################### @@ -40,15 +40,15 @@ module "rpcs" { {% for region in subscribed_regions %} - {% set region_keys = region.split('-') %} - - provider "oci" { - tenancy_ocid = var.tenancy_ocid - user_ocid = var.user_ocid - fingerprint = var.fingerprint - private_key_path = var.private_key_path - region = "{{ region }}" - ignore_defined_tags = ["Oracle-Tags.CreatedBy", "Oracle-Tags.CreatedOn"] - alias = "{{ region_keys[1] }}" + {% set alias = region.replace('-', '_') %} + +provider "oci" { + tenancy_ocid = var.tenancy_ocid + user_ocid = var.user_ocid + fingerprint = var.fingerprint + private_key_path = var.private_key_path + region = "{{ region }}" + ignore_defined_tags = ["Oracle-Tags.CreatedBy", "Oracle-Tags.CreatedOn"] + alias = "{{ alias }}" } {% endfor %} diff --git a/cd3_automation_toolkit/Release-Notes b/cd3_automation_toolkit/Release-Notes index 19cdd4e54..1dcaf8ea1 100644 --- a/cd3_automation_toolkit/Release-Notes +++ b/cd3_automation_toolkit/Release-Notes @@ -1,3 +1,11 @@ +------------------------------------- +CD3 Automation Toolkit Tag v2025.1.3 +Aug 1st, 2025 +------------------------------------- +1. DNS terraform performance tuning. +2. SDK support for new regions. +3. Bug fixes related to RPC, empty group creation, duplicate entries of identity domain users/groups during export, OKE, MySQL DB etc. + ------------------------------------- CD3 Automation Toolkit Tag v2025.1.2 May 14th, 2025 diff --git a/cd3_automation_toolkit/cd3Validator.py b/cd3_automation_toolkit/cd3Validator.py index 7c75a6b0c..33100ca6d 100644 --- a/cd3_automation_toolkit/cd3Validator.py +++ b/cd3_automation_toolkit/cd3Validator.py @@ -316,7 +316,7 @@ def validate_subnets(filename, comp_ids, vcnobj): for j in dfsub.keys(): if (str(dfsub[j][i]).strip() == "NaN" or str(dfsub[j][i]).strip() == "nan" or str(dfsub[j][i]).strip() == ""): # only dhcp_option_name, route table name, seclist_names and dns_label columns can be empty - if j in labels or commonTools.check_column_headers(j) in commonTools.tagColumns: + if j in labels or commonTools.check_column_headers(j) in commonTools.tagColumns or "ipv6" in j.lower(): pass else: if j == "Type(private|public)" and subnet_or_vlan.lower() == "vlan": @@ -460,7 +460,7 @@ def validate_vcns(filename, comp_ids, vcnobj):# config): # ,vcn_cidrs,vcn_compa # Check for null values and display appropriate message for j in dfv.keys(): if (str(dfv[j][i]).strip() == "NaN" or str(dfv[j][i]).strip() == "nan" or str(dfv[j][i]).strip() == ""): - if j == 'DNS Label' or commonTools.check_column_headers(j) in commonTools.tagColumns: + if j == 'DNS Label' or commonTools.check_column_headers(j) in commonTools.tagColumns or "ipv6" in j.lower(): continue else: log(f'ROW {count+2} : Empty value at column "{j}".') @@ -1780,7 +1780,7 @@ def validate_kms(filename,comp_ids): pass else: try: - vault_comp_name = commonTools.check_tf_variable(vault_comp_name) + vault_comp_name = commonTools.check_tf_variable(vault_compartment_name) comp_id = comp_ids[vault_compartment_name] except KeyError: log(f'ROW {i+3} : Compartment {vault_compartment_name} does not exist in OCI.') diff --git a/cd3_automation_toolkit/commonTools.py b/cd3_automation_toolkit/commonTools.py index 60bc09bdf..8f35569b4 100644 --- a/cd3_automation_toolkit/commonTools.py +++ b/cd3_automation_toolkit/commonTools.py @@ -68,7 +68,7 @@ def __init__(self): self.identity_domain_enabled = False self.reg_filter = None #Should be None but changed to "null" to do a quick fix for ct.get_compartment_map - self.comp_filter = "null" + self.comp_filter = None self.tag_filter = None self.vizoci_comp_filter = None self.default_dns = None @@ -143,6 +143,7 @@ def __init__(self): #os.chdir(dir) # Get Export filters def get_export_filters(self,export_filters): + self.comp_filter = "null" for i in export_filters: i = i.replace(" ", "") i = i.replace("\"", "") @@ -657,7 +658,7 @@ def check_multivalues_columnvalue(columnvalue, columnname, tempdict): multivalues = columnvalue.split("::") multivalues = [str(part).strip() for part in multivalues if part] tempdict = {columnname: multivalues} - elif columnname != 'Compartment Name': + elif columnname != 'Compartment Name' and "ipv6" not in columnname.lower(): columnname = commonTools.check_column_headers(columnname) multivalues = columnvalue.split("::") multivalues = [str(part).strip() for part in multivalues ]#if part] diff --git a/cd3_automation_toolkit/setUpOCI.py b/cd3_automation_toolkit/setUpOCI.py index 3dd24429e..8025eabc9 100644 --- a/cd3_automation_toolkit/setUpOCI.py +++ b/cd3_automation_toolkit/setUpOCI.py @@ -2082,6 +2082,8 @@ def create_firewall(inputfile, outdir, service_dir, prefix, ct,sub_options=[]): global import_scripts updated_paths = [] import_scripts = [] +# Opt-in to IMDS lookup +oci.regions.enable_instance_metadata_service() exec_start_time = datetime.datetime.now() parser = argparse.ArgumentParser(description='Sets Up OCI via TF') parser.add_argument('propsfile', help="Full Path of properties file containing input variables. eg setUpOCI.properties") diff --git a/cd3_automation_toolkit/shell_script.sh b/cd3_automation_toolkit/shell_script.sh index 01aaa645d..1ef4f8fbd 100644 --- a/cd3_automation_toolkit/shell_script.sh +++ b/cd3_automation_toolkit/shell_script.sh @@ -13,7 +13,7 @@ sudo dnf install python-pip -y #sudo ln -s /usr/bin/pip3 /usr/bin/pip # Install required Python packages -pip install --user oci-cli==3.51.2 +pip install --user oci-cli==3.62.2 pip install --user pycryptodomex==3.10.1 pip install --user regex==2022.10.31 pip install --user numpy==1.26.4 @@ -30,6 +30,7 @@ pip install --user Jinja2==3.1.2 pip install --user simplejson==3.18.3 pip install --user GitPython==3.1.40 pip install --user PyYAML==6.0.1 +pip install --user ansible==8.7.0 # Add Python3 site-packages to PYTHONPATH echo "export PYTHONPATH=\${PYTHONPATH}:/root/.local/lib/python3.9/site-packages/:/cd3user/.local/lib/python3.9/site-packages/" >> /cd3user/.bashrc diff --git a/cd3_automation_toolkit/user-scripts/createTenancyConfig.py b/cd3_automation_toolkit/user-scripts/createTenancyConfig.py index 1c55600fa..b26b83590 100644 --- a/cd3_automation_toolkit/user-scripts/createTenancyConfig.py +++ b/cd3_automation_toolkit/user-scripts/createTenancyConfig.py @@ -417,10 +417,10 @@ def create_bucket(config, signer): if tenancy == "" or tenancy == "\n": print("Tenancy ID cannot be left empty...Exiting !!") exit(1) - if ("ocid1.tenancy.oc1" in tenancy): - cloud_domain=".oraclecloud.com" - else: - cloud_domain=".oraclegovcloud.com" + # if ("ocid1.tenancy.oc1" in tenancy): + # cloud_domain=".oraclecloud.com" + # else: + # cloud_domain=".oraclegovcloud.com" auth_mechanism = config.get('Default', 'auth_mechanism').strip().lower() @@ -628,6 +628,9 @@ def create_bucket(config, signer): ## Authenticate ct = commonTools() config, signer = ct.authenticate(auth_mechanism, config_file_path) +_realm = config['tenancy'].split(".")[2] +cloud_domain = "."+oci.regions.REALMS[_realm] + ## Fetch OCI_regions cd3service = cd3Services() cd3service.fetch_regions(config, signer) diff --git a/cd3_automation_toolkit/user-scripts/terraform/dns.tf b/cd3_automation_toolkit/user-scripts/terraform/dns.tf index d295f8bc5..3f4a91092 100644 --- a/cd3_automation_toolkit/user-scripts/terraform/dns.tf +++ b/cd3_automation_toolkit/user-scripts/terraform/dns.tf @@ -1,87 +1,94 @@ -# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. # #################### ### DNS-Resolver ### #################### +locals { + resolver_vcns = { + for item_key, item_value in var.resolvers : "${item_value.vcn_name}" => (length(regexall("ocid1.compartment.oc*", item_value.network_compartment_id)) > 0 ? item_value.network_compartment_id : var.compartment_ocids[item_value.network_compartment_id])... + } + resolver_vcns_distinct = { for k, v in local.resolver_vcns : k => distinct(v)[0] } +} - -data "oci_core_vcn_dns_resolver_association" "resolver_vcn_dns_resolver_association" { - for_each = var.resolvers != null ? var.resolvers : {} - vcn_id = data.oci_core_vcns.resolver_oci_vcns[each.key].virtual_networks.*.id[0] +data "oci_core_vcns" "dns_oci_vcns" { + for_each = local.resolver_vcns_distinct + compartment_id = each.value != null ? (length(regexall("ocid1.compartment.oc*", each.value)) > 0 ? each.value : var.compartment_ocids[each.value]) : null + display_name = each.key } -data "oci_core_vcns" "resolver_oci_vcns" { - # depends_on = [module.vcns] # Uncomment to create resolver and vcn together - for_each = var.resolvers != null ? var.resolvers : {} - compartment_id = each.value.network_compartment_id != null ? (length(regexall("ocid1.compartment.oc*", each.value.network_compartment_id)) > 0 ? each.value.network_compartment_id : var.compartment_ocids[each.value.network_compartment_id]) : var.compartment_ocids[each.value.network_compartment_id] - display_name = each.value.vcn_name +data "oci_core_vcn_dns_resolver_association" "resolver_vcn_dns_resolver_association" { + for_each = local.resolver_vcns_distinct + vcn_id = data.oci_core_vcns.dns_oci_vcns[each.key].virtual_networks.*.id[0] } ### Data for Subnet ### locals { - subnets = flatten([ + endpoint_subnets = distinct(flatten([ for resolver_key, res in var.resolvers : [ for e_key, endpoint in res.endpoint_names : { vcn_name = res.vcn_name - network_compartment_id = res.network_compartment_id + network_compartment_id = length(regexall("ocid1.compartment.oc*", res.network_compartment_id)) > 0 ? res.network_compartment_id : var.compartment_ocids[res.network_compartment_id] subnet_name = endpoint.subnet_name - #subnet_name = trimprefix("${endpoint.subnet_name}", "${res.vcn_name}_") - resolver_key = resolver_key - endpoint_name = endpoint.name } ] - ]) + ])) } - -data "oci_core_subnets" "resolver_oci_subnets" { - # depends_on = [module.subnets] # Uncomment to create resolver and subnets together - for_each = { for sn in local.subnets : "${sn.endpoint_name}_${sn.subnet_name}" => sn } - compartment_id = length(regexall("ocid1.compartment.oc*", each.value.network_compartment_id)) > 0 ? each.value.network_compartment_id : var.compartment_ocids[each.value.network_compartment_id] - display_name = each.value.subnet_name - vcn_id = data.oci_core_vcns.resolver_oci_vcns[each.value.resolver_key].virtual_networks.*.id[0] +data "oci_core_subnets" "dns_oci_subnets" { + for_each = { for item in local.endpoint_subnets : item.subnet_name => item if length(regexall("ocid1.subnet.oc*", item.subnet_name)) == 0 } + compartment_id = each.value.network_compartment_id != null ? (length(regexall("ocid1.compartment.oc*", each.value.network_compartment_id)) > 0 ? each.value.network_compartment_id : var.compartment_ocids[each.value.network_compartment_id]) : null + display_name = each.key + vcn_id = data.oci_core_vcns.dns_oci_vcns[each.value.vcn_name].virtual_networks.*.id[0] } + ### Data for NSGs### locals { - nsgs = flatten([ + nsgs = distinct(flatten([ for resolver_key, res in var.resolvers : [ for e_key, endpoint in res.endpoint_names : [ for nsg in endpoint.nsg_ids : { vcn_name = res.vcn_name - network_compartment_id = res.network_compartment_id + network_compartment_id = length(regexall("ocid1.compartment.oc*", res.network_compartment_id)) > 0 ? res.network_compartment_id : var.compartment_ocids[res.network_compartment_id] nsg_name = nsg - resolver_key = resolver_key - endpoint_name = endpoint.name } ] ] - ]) + ])) } -data "oci_core_network_security_groups" "resolver_network_security_groups" { - for_each = { for nsg in local.nsgs : "${nsg.endpoint_name}_${nsg.nsg_name}" => nsg } + +data "oci_core_network_security_groups" "endpoint_nsgs" { + for_each = { for nsg in local.nsgs : nsg.nsg_name => nsg if length(regexall("ocid1.networksecuritygroup.oc*", nsg.nsg_name)) == 0 } compartment_id = length(regexall("ocid1.compartment.oc*", each.value.network_compartment_id)) > 0 ? each.value.network_compartment_id : var.compartment_ocids[each.value.network_compartment_id] display_name = each.value.nsg_name - vcn_id = data.oci_core_vcns.resolver_oci_vcns[each.value.resolver_key].virtual_networks.*.id[0] + vcn_id = data.oci_core_vcns.dns_oci_vcns[each.value.vcn_name].virtual_networks.*.id[0] } + ### Data for Views ### locals { - resolver_views = flatten([ + resolver_views = distinct(flatten([ for resolver_key, res in var.resolvers : [ for view_key, view in res.views : { - resolver_key = resolver_key - view_key = view_key view_name = view.view_id - view_compartment = view.view_compartment_id + view_compartment = length(regexall("ocid1.compartment.oc*", view.view_compartment_id)) > 0 ? view.view_compartment_id : var.compartment_ocids[view.view_compartment_id] } ] - ]) + ])) + zone_views = distinct([for k, v in var.zones : { + view_name = v.view_id + view_compartment = length(regexall("ocid1.compartment.oc*", v.view_compartment_id)) > 0 ? v.view_compartment_id : var.compartment_ocids[v.view_compartment_id] + }]) + rrset_views = distinct([for k, v in var.rrsets : { + view_name = v.view_id + view_compartment = length(regexall("ocid1.compartment.oc*", v.view_compartment_id)) > 0 ? v.view_compartment_id : var.compartment_ocids[v.view_compartment_id] + }]) + all_views = distinct(concat(local.resolver_views, local.zone_views, local.rrset_views)) } -data "oci_dns_views" "resolver_views_data" { +data "oci_dns_views" "all_views_data" { #Required - for_each = { for rv in local.resolver_views : "${rv.view_key}" => rv } + for_each = { for rv in local.all_views : "${rv.view_name}" => rv if length(regexall("ocid1.dnsview.oc*", rv.view_name)) == 0 } compartment_id = length(regexall("ocid1.compartment.oc*", each.value.view_compartment)) > 0 ? each.value.view_compartment : var.compartment_ocids[each.value.view_compartment] scope = "PRIVATE" #Optional @@ -89,17 +96,18 @@ data "oci_dns_views" "resolver_views_data" { state = "ACTIVE" } + ### Module ### module "dns-resolvers" { - source = "./modules/network/dns/dns_resolver" - # depends_on = [module.nsgs] # Uncomment to create NSG and DNS Resolvers together + source = "./modules/network/dns/dns_resolver" for_each = var.resolvers != null ? var.resolvers : {} target_resolver_id = data.oci_core_vcn_dns_resolver_association.resolver_vcn_dns_resolver_association[each.key].*.dns_resolver_id[0] resolver_scope = "PRIVATE" resolver_display_name = each.value.display_name != null ? each.value.display_name : null views = each.value.views != null ? { for v_key, view in each.value.views : v_key => { - view_id = length(regexall("ocid1.dnsview.oc*", view.view_id)) > 0 ? view.view_id : try(data.oci_dns_views.resolver_views_data["${v_key}"].views.*.id[0], module.dns-views[view.view_id]["dns_view_id"]) + view_id = length(regexall("ocid1.dnsview.oc*", view.view_id)) > 0 ? view.view_id : try(data.oci_dns_views.all_views_data[view.view_id].views.*.id[0], module.dns-views[view.view_id].views.*.id[0]) + #view_id = length(regexall("ocid1.dnsview.oc*", view.view_id)) > 0 ? view.view_id : merge(data.oci_dns_views.all_views_data[view.view_id], module.dns-views[view.view_id]).views.*.id[0] } } : null @@ -112,14 +120,14 @@ module "dns-resolvers" { listening = endpoint.is_listening name = endpoint.name #resolver_id = oci_dns_resolver.test_resolver.id - subnet_id = length(regexall("ocid1.subnet.oc*", endpoint.subnet_name)) > 0 ? endpoint.subnet_name : data.oci_core_subnets.resolver_oci_subnets["${endpoint.name}_${endpoint.subnet_name}"].subnets.*.id[0] + subnet_id = length(regexall("ocid1.subnet.oc*", endpoint.subnet_name)) > 0 ? endpoint.subnet_name : data.oci_core_subnets.dns_oci_subnets[endpoint.subnet_name].subnets.*.id[0] scope = "PRIVATE" #Optional endpoint_type = "VNIC" forwarding_address = endpoint.forwarding_address listening_address = endpoint.listening_address - nsg_ids = endpoint.nsg_ids != null ? flatten(tolist([for nsg in endpoint.nsg_ids : (length(regexall("ocid1.networksecuritygroup.oc*", nsg)) > 0 ? [nsg] : data.oci_core_network_security_groups.resolver_network_security_groups["${endpoint.name}_${nsg}"].network_security_groups[*].id)])) : null + nsg_ids = endpoint.nsg_ids != null ? flatten(tolist([for nsg in endpoint.nsg_ids : (length(regexall("ocid1.networksecuritygroup.oc*", nsg)) > 0 ? [nsg] : data.oci_core_network_security_groups.endpoint_nsgs[nsg].network_security_groups[*].id)])) : null } } : null @@ -129,75 +137,57 @@ module "dns-resolvers" { ################## ### DNS-RRsets ### ################## -data "oci_dns_views" "rrset_views_data" { - #Required - for_each = var.rrsets - compartment_id = each.value.view_compartment_id != null ? (length(regexall("ocid1.compartment.oc*", each.value.view_compartment_id)) > 0 ? each.value.view_compartment_id : var.compartment_ocids[each.value.view_compartment_id]) : null - scope = "PRIVATE" - #Optional - display_name = each.value.view_id - state = "ACTIVE" + +locals { + + rrset_zones = { + for item_key, item_value in var.rrsets : "${item_value.view_id}_${item_value.zone_id}" => { "comp" = "${item_value.compartment_id}", "zone_name" = "${item_value.zone_id}", "view" = "${item_value.view_id}" }... + } + rrset_zone_distinct = { for k, v in local.rrset_zones : k => distinct(v)[0] } } + data "oci_dns_zones" "rrset_zones_data" { - for_each = { for k, v in var.rrsets : k => v if try(data.oci_dns_views.rrset_views_data[k].views.*.id[0], 0) != 0 } - compartment_id = each.value.compartment_id != null ? (length(regexall("ocid1.compartment.oc*", each.value.compartment_id)) > 0 ? each.value.compartment_id : var.compartment_ocids[each.value.compartment_id]) : null + for_each = { for z, z_data in local.rrset_zone_distinct : z => z_data if length(regexall("ocid1.dnszone.oc*", z_data.zone_name)) == 0 } + compartment_id = length(regexall("ocid1.compartment.oc*", each.value.comp)) > 0 ? each.value.comp : var.compartment_ocids[each.value.comp] #Optional - name = each.value.zone_id + name = each.value.zone_name scope = "PRIVATE" state = "ACTIVE" - view_id = length(regexall("ocid1.dnsview.oc*", each.value.view_id)) > 0 ? each.value.view_id : try(data.oci_dns_views.rrset_views_data[each.key].views.*.id[0], module.dns-views[each.value.view_id]["dns_view_id"]) + view_id = length(regexall("ocid1.dnsview.oc*", each.value.view)) > 0 ? each.value.view : try(data.oci_dns_views.all_views_data[each.value.view].views.*.id[0], module.dns-views[each.value.view]["views"].*.id[0]) } module "dns-rrsets" { source = "./modules/network/dns/rrset" for_each = var.rrsets != null ? var.rrsets : {} - depends_on = [module.dns-views, module.dns-zones] - rrset_zone = try(data.oci_dns_zones.rrset_zones_data[each.key].zones.*.id[0], module.dns-zones[join("_", [each.value.view_id, replace(each.value.zone_id, ".", "_")])]["dns_zone_id"]) - #rrset_view_id = each.value.view_id != "" ? (length(regexall("ocid1.dnsview.oc*", each.value.view_id)) > 0 ? each.value.view_id : data.oci_dns_views.rrset_views_data[each.key].views.*.id[0]) : null - rrset_view_id = length(regexall("ocid1.dnsview.oc*", each.value.view_id)) > 0 ? each.value.view_id : try(data.oci_dns_views.rrset_views_data[each.key].views.*.id[0], module.dns-views[each.value.view_id]["dns_view_id"]) + #depends_on = [module.dns-views, module.dns-zones] + rrset_zone = length(regexall("ocid1.dnszone.oc*", each.value.zone_id)) > 0 ? each.value.zone_id : try(data.oci_dns_zones.rrset_zones_data["${each.value.view_id}_${each.value.zone_id}"].zones.*.id[0],module.dns-zones[join("_", [each.value.view_id, replace(each.value.zone_id, ".", "_")])].zones.*.id[0]) + rrset_view_id = length(regexall("ocid1.dnsview.oc*", each.value.view_id)) > 0 ? each.value.view_id : try(data.oci_dns_views.all_views_data[each.value.view_id].views.*.id[0], module.dns-views[each.value.view_id].views.*.id[0]) rrset_domain = each.value.domain rrset_rtype = each.value.rtype rrset_ttl = each.value.ttl - #rrset_compartment_id = each.value.compartment_id != null ? (length(regexall("ocid1.compartment.oc*", each.value.compartment_id)) > 0 ? each.value.compartment_id : var.compartment_ocids[each.value.compartment_id]) : null - rrset_rdata = each.value.rdata - rrset_scope = "PRIVATE" + rrset_rdata = each.value.rdata + rrset_scope = "PRIVATE" } -#output "zone_id" { -#value = { for key, value in var.rrsets: key => module.dns-zones[join("_", [value.view_id,replace(value.zone_id, ".", "_")])]["dns_zone_id"]} -#} -# -#output "zone_data" { -#value = { for key, value in var.rrsets: key => data.oci_dns_zones.rrset_zones_data[key].zones.*.id[0]} -#} ################# ### DNS-Zones ### ################# -data "oci_dns_views" "zone_views_data" { - #Required - for_each = { for k, v in var.zones : k => v if v.view_id != null } - compartment_id = length(regexall("ocid1.compartment.oc*", each.value.view_compartment_id)) > 0 ? each.value.view_compartment_id : var.compartment_ocids[each.value.view_compartment_id] - scope = "PRIVATE" - display_name = each.value.view_id - state = "ACTIVE" -} module "dns-zones" { source = "./modules/network/dns/zone" - depends_on = [module.dns-views] + #depends_on = [module.dns-views] for_each = { for k, v in var.zones : k => v if var.zones != null } zone_compartment_id = length(regexall("ocid1.compartment.oc*", each.value.compartment_id)) > 0 ? each.value.compartment_id : var.compartment_ocids[each.value.compartment_id] zone_name = each.value.display_name zone_type = "PRIMARY" zone_defined_tags = try(each.value.defined_tags, null) zone_freeform_tags = try(each.value.freeform_tags, null) - #external_masters = each.value.external_masters != null ? each.value.external_masters : {} - zone_scope = "PRIVATE" - view_id = length(regexall("ocid1.dnsview.oc*", each.value.view_id)) > 0 ? each.value.view_id : try(data.oci_dns_views.zone_views_data[each.key].views.*.id[0], module.dns-views[each.value.view_id]["dns_view_id"]) + zone_scope = "PRIVATE" + view_id = length(regexall("ocid1.dnsview.oc*", each.value.view_id)) > 0 ? each.value.view_id : try(data.oci_dns_views.all_views_data[each.value.view_id].views.*.id[0], module.dns-views[each.value.view_id]["views"].*.id[0]) } ################# @@ -213,4 +203,4 @@ module "dns-views" { view_defined_tags = try(each.value.defined_tags, null) view_freeform_tags = try(each.value.freeform_tags, null) -} \ No newline at end of file +} diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-group/main.tf b/cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-group/main.tf index 01ec84508..0d90045ef 100644 --- a/cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-group/main.tf +++ b/cd3_automation_toolkit/user-scripts/terraform/modules/identity/iam-group/main.tf @@ -7,7 +7,7 @@ ############################ locals { user_ids = { - for user in data.oci_identity_users.users.users : + for user in coalesce(data.oci_identity_users.users.users, []) : user.name => user.id } } diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/dns/view/outputs.tf b/cd3_automation_toolkit/user-scripts/terraform/modules/network/dns/view/outputs.tf index e8e76377a..4d6606613 100644 --- a/cd3_automation_toolkit/user-scripts/terraform/modules/network/dns/view/outputs.tf +++ b/cd3_automation_toolkit/user-scripts/terraform/modules/network/dns/view/outputs.tf @@ -5,6 +5,6 @@ # Outputs Block - DNS view # ############################ -output "dns_view_id" { - value = oci_dns_view.view.id +output "views" { + value = [for item in [oci_dns_view.view.id]:{"id" = "${item}"}] } \ No newline at end of file diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/network/dns/zone/outputs.tf b/cd3_automation_toolkit/user-scripts/terraform/modules/network/dns/zone/outputs.tf index 551e20e8f..0f1e882cc 100644 --- a/cd3_automation_toolkit/user-scripts/terraform/modules/network/dns/zone/outputs.tf +++ b/cd3_automation_toolkit/user-scripts/terraform/modules/network/dns/zone/outputs.tf @@ -10,4 +10,7 @@ output "dns_zone_id" { } output "dns_zone_name" { value = oci_dns_zone.zone.name +} +output "zones" { + value = [for item in [oci_dns_zone.zone.id]:{"id" = "${item}"}] } \ No newline at end of file diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/oke/cluster/main.tf b/cd3_automation_toolkit/user-scripts/terraform/modules/oke/cluster/main.tf index 9f0ead2d8..382b07c55 100644 --- a/cd3_automation_toolkit/user-scripts/terraform/modules/oke/cluster/main.tf +++ b/cd3_automation_toolkit/user-scripts/terraform/modules/oke/cluster/main.tf @@ -22,17 +22,15 @@ resource "oci_containerengine_cluster" "cluster" { subnet_id = var.endpoint_subnet_id } - image_policy_config { - #Optional - is_policy_enabled = var.is_policy_enabled - dynamic "key_details" { - for_each = var.policy_kms_key_id != null ? [1] : [] - content{ - #Optional - kms_key_id = var.policy_kms_key_id - } - } + dynamic "image_policy_config" { + for_each = var.policy_kms_key_id != null ? [1] : [] + content { + is_policy_enabled = true + key_details { + kms_key_id = var.policy_kms_key_id } + } + } options { add_ons { diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/oke/nodepool/main.tf b/cd3_automation_toolkit/user-scripts/terraform/modules/oke/nodepool/main.tf index 2bd4762e9..34008390c 100644 --- a/cd3_automation_toolkit/user-scripts/terraform/modules/oke/nodepool/main.tf +++ b/cd3_automation_toolkit/user-scripts/terraform/modules/oke/nodepool/main.tf @@ -50,9 +50,10 @@ resource "oci_containerengine_node_pool" "nodepool" { } - # node_metadata = { + node_metadata = { # user_data = var.cloudinit_nodepool_common == "" && lookup(var.cloudinit_nodepool, each.key, null) == null ? data.cloudinit_config.worker.rendered : lookup(var.cloudinit_nodepool, each.key, null) != null ? filebase64(lookup(var.cloudinit_nodepool, each.key, null)) : filebase64(var.cloudinit_nodepool_common) - # } + user_data = var.init_script_path != null? base64encode(file(var.init_script_path)): null + } node_shape_config { ocpus = var.ocpus diff --git a/cd3_automation_toolkit/user-scripts/terraform/modules/oke/nodepool/variables.tf b/cd3_automation_toolkit/user-scripts/terraform/modules/oke/nodepool/variables.tf index 65aff7d4f..a68eec5aa 100644 --- a/cd3_automation_toolkit/user-scripts/terraform/modules/oke/nodepool/variables.tf +++ b/cd3_automation_toolkit/user-scripts/terraform/modules/oke/nodepool/variables.tf @@ -138,6 +138,11 @@ variable "source_type" { default = null } +variable "init_script_path" { + type = string + default = null + } + variable "boot_volume_size_in_gbs" { type = number description = "The boot volume size for nodes in nodepool" diff --git a/cd3_automation_toolkit/user-scripts/terraform/mysql-dbsystem.tf b/cd3_automation_toolkit/user-scripts/terraform/mysql-dbsystem.tf index b8dcba931..0cb28c870 100644 --- a/cd3_automation_toolkit/user-scripts/terraform/mysql-dbsystem.tf +++ b/cd3_automation_toolkit/user-scripts/terraform/mysql-dbsystem.tf @@ -5,15 +5,13 @@ # Module Block - MySQL Database # Create MySQL DB Systems ############################################ - data "oci_mysql_mysql_configurations" "mysql_configurations" { - # depends_on = [module.mysql-configuration] + depends_on = [module.mysql_configuration] for_each = var.mysql_db_system != null ? var.mysql_db_system : {} compartment_id = each.value.configuration_compartment_id != null ? (length(regexall("ocid1.compartment.oc*", each.value.configuration_compartment_id)) > 0 ? each.value.configuration_compartment_id : var.compartment_ocids[each.value.configuration_compartment_id]) : var.compartment_ocids[each.value.configurations_compartment_id] display_name = each.value.configuration_id state = "ACTIVE" } - data "oci_core_subnets" "oci_mysql_subnets" { # depends_on = [module.subnets] # Uncomment to create Network and MySQL together for_each = var.mysql_db_system != null ? var.mysql_db_system : {} @@ -21,24 +19,28 @@ data "oci_core_subnets" "oci_mysql_subnets" { display_name = each.value.subnet_id vcn_id = data.oci_core_vcns.oci_mysql_vcns[each.key].virtual_networks.*.id[0] } - data "oci_core_vcns" "oci_mysql_vcns" { # depends_on = [module.vcns] # Uncomment to create Network and MySQL together for_each = var.mysql_db_system != null ? var.mysql_db_system : {} compartment_id = each.value.network_compartment_id != null ? (length(regexall("ocid1.compartment.oc*", each.value.network_compartment_id)) > 0 ? each.value.network_compartment_id : var.compartment_ocids[each.value.network_compartment_id]) : var.compartment_ocids[each.value.network_compartment_id] display_name = each.value.vcn_names } - - module "mysql_db_system" { - source = "./modules/database/mysql-dbsystem" for_each = var.mysql_db_system != null ? var.mysql_db_system : {} - + # Add explicit depends_on for mysql_configuration + depends_on = [module.mysql_configuration] compartment_id = each.value.compartment_id != null ? (length(regexall("ocid1.compartment.oc*", each.value.compartment_id)) > 0 ? each.value.compartment_id : var.compartment_ocids[each.value.compartment_id]) : null network_compartment_id = each.value.network_compartment_id != null ? (length(regexall("ocid1.compartment.oc*", each.value.network_compartment_id)) > 0 ? each.value.network_compartment_id : var.compartment_ocids[each.value.network_compartment_id]) : null configuration_compartment_id = each.value.configuration_compartment_id != null ? (length(regexall("ocid1.compartment.oc*", each.value.configuration_compartment_id)) > 0 ? each.value.configuration_compartment_id : var.compartment_ocids[each.value.configuration_compartment_id]) : var.compartment_ocids[each.value.compartment_id] - configuration_id = length(regexall("ocid1.mysqlconfiguration.*", each.value.configuration_id)) > 0 ? each.value.configuration_id : data.oci_mysql_mysql_configurations.mysql_configurations[each.key].configurations[0].id + + # Modified configuration_id handling to avoid data source lookup failures + configuration_id = length(regexall("ocid1.mysqlconfiguration.*", each.value.configuration_id)) > 0 ? each.value.configuration_id : ( + contains(keys(var.mysql_configuration), each.value.configuration_id) ? + module.mysql_configuration[each.value.configuration_id].db_system_configuration_id : + try(data.oci_mysql_mysql_configurations.mysql_configurations[each.key].configurations[0].id, null) + ) + display_name = each.value.mysql_db_system_display_name shape_name = each.value.mysql_shape_name admin_username = each.value.mysql_db_system_admin_username @@ -69,23 +71,18 @@ module "mysql_db_system" { defined_tags = each.value.defined_tags != null ? each.value.defined_tags : null freeform_tags = each.value.freeform_tags != null ? each.value.freeform_tags : null } - ############################################ # Module Block - MySQL Database # Create MySQL Configurations ############################################ - data "oci_mysql_shapes" "mysql_shapes" { for_each = var.mysql_configuration != null ? var.mysql_configuration : {} compartment_id = each.value.compartment_id != null ? (length(regexall("ocid1.compartment.oc*", each.value.compartment_id)) > 0 ? each.value.compartment_id : var.compartment_ocids[each.value.compartment_id]) : var.compartment_ocids[each.value.compartment_id] name = each.value.mysql_configuration_shape_name } - module "mysql_configuration" { - source = "./modules/database/mysql-configuration" for_each = var.mysql_configuration != null ? var.mysql_configuration : {} - compartment_id = each.value.compartment_id != null ? (length(regexall("ocid1.compartment.oc*", each.value.compartment_id)) > 0 ? each.value.compartment_id : var.compartment_ocids[each.value.compartment_id]) : null mysql_configuration_shape_name = each.value.mysql_configuration_shape_name != null ? (length(regexall("(VM\\.Standard\\.(E[234]\\.[12468]|E[34]\\.(16|24|32|48|64))|MySQL\\.(VM\\.Standard\\.(E[34]\\.[12468]|E[34]\\.(16|24|32|48|64)\\.(8|16|32|64|128|256|384|512|768|1024)GB)|HeatWave\\.(BM\\.Standard(\\.E3)?|VM\\.Standard(\\.E3)?)|VM\\.Optimized3\\.[12468]\\.((8|16|32|64|128|256|384|512|768|1024)GB)|[12468]|16|32|48|64|256))", each.value.mysql_configuration_shape_name)) > 0 ? each.value.mysql_configuration_shape_name : data.oci_mysql_shapes.mysql_shapes[each.key].shapes.*.name[0]) : null defined_tags = each.value.defined_tags @@ -165,5 +162,4 @@ module "mysql_configuration" { mysql_configuration_variables_tmp_table_size = each.value.mysql_configuration_variables_tmp_table_size mysql_configuration_variables_transaction_isolation = each.value.mysql_configuration_variables_transaction_isolation mysql_configuration_variables_wait_timeout = each.value.mysql_configuration_variables_wait_timeout - -} \ No newline at end of file +} diff --git a/cd3_automation_toolkit/user-scripts/terraform/oke.tf b/cd3_automation_toolkit/user-scripts/terraform/oke.tf index ba077dae7..3f3808e9e 100644 --- a/cd3_automation_toolkit/user-scripts/terraform/oke.tf +++ b/cd3_automation_toolkit/user-scripts/terraform/oke.tf @@ -118,6 +118,7 @@ module "nodepools" { size = each.value.size is_pv_encryption_in_transit_enabled = each.value.is_pv_encryption_in_transit_enabled cni_type = each.value.cni_type + init_script_path = each.value.init_script_path max_pods_per_node = each.value.max_pods_per_node pod_nsg_ids = each.value.pod_nsg_ids pod_subnet_ids = each.value.pod_subnet_ids != null ? (length(regexall("ocid1.subnet.oc*", each.value.pod_subnet_ids)) > 0 ? each.value.pod_subnet_ids : data.oci_core_subnets.oci_subnets_pod[each.key].subnets.*.id[0]) : null diff --git a/cd3_automation_toolkit/user-scripts/terraform/provider.tf b/cd3_automation_toolkit/user-scripts/terraform/provider.tf index acc0e95fb..34c1879f0 100644 --- a/cd3_automation_toolkit/user-scripts/terraform/provider.tf +++ b/cd3_automation_toolkit/user-scripts/terraform/provider.tf @@ -19,7 +19,7 @@ terraform { required_providers { oci = { source = "oracle/oci" - version = "6.30.0" + version = "7.8.0" } } } diff --git a/cd3_automation_toolkit/user-scripts/terraform/variables_example.tf b/cd3_automation_toolkit/user-scripts/terraform/variables_example.tf index a43208680..f788d1086 100644 --- a/cd3_automation_toolkit/user-scripts/terraform/variables_example.tf +++ b/cd3_automation_toolkit/user-scripts/terraform/variables_example.tf @@ -1813,6 +1813,7 @@ variable "nodepools" { source_type = string boot_volume_size_in_gbs = optional(number) ssh_public_key = optional(string) + init_script_path = optional(string) nodepool_kms_key_id = optional(string) node_defined_tags = optional(map(any)) node_freeform_tags = optional(map(any)) diff --git a/jenkins_install/plugins.txt b/jenkins_install/plugins.txt index 174d70f5a..63af28d06 100644 --- a/jenkins_install/plugins.txt +++ b/jenkins_install/plugins.txt @@ -19,6 +19,7 @@ uno-choice:latest file-parameters:latest scriptler:latest ansicolor:latest +#pipeline-graph-view:243.vc9e11fec486a_ pipeline-graph-view:latest javax-mail-api:latest jdk-tool:latest