Merge pull request #146 from oracle-devrel/develop

Automation Toolkit Release v2025.1.3

Author: bhlohumi

Dockerfile

@@ -14,9 +14,9 @@ ARG USE_DEVOPS=YES
 ARG USERNAME=cd3user
 ARG USER_GID=$USER_UID
 # Whether to download Provider as part of image creation
-ARG DOWNLOAD_PROVIDER=YES
+ARG DOWNLOAD_PROVIDER=NO
 # TF Provider version
-ARG TF_OCI_PROVIDER=6.30.0
+ARG TF_OCI_PROVIDER=7.8.0
 ARG TF_NULL_PROVIDER=3.2.3
 
 RUN microdnf install -y sudo && \
@@ -31,6 +31,7 @@ RUN microdnf install -y sudo && \
     microdnf install -y wget && \
     microdnf install -y unzip && \
     microdnf install -y graphviz && \
+    ln -sf /usr/bin/python3 /usr/bin/python && \
     echo 'alias vi="vim"' >> /etc/bashrc
 
 USER $USERNAME

README.md

@@ -7,7 +7,7 @@
 
 <br>
 
-  [What's New](https://github.com/oracle-devrel/cd3-automation-toolkit/releases/tag/v2025.1.2) &nbsp;•&nbsp;[Excel Templates](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/excel-templates/) &nbsp;•&nbsp;[CD3 Docs](https://oracle-devrel.github.io/cd3-automation-toolkit/)&nbsp;•&nbsp; [Watch & Learn](https://www.youtube.com/playlist?list=PLPIzp-E1msrbJ3WawXVhzimQnLw5iafcp) &nbsp;•&nbsp;[Blogs & Tutorials](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/tutorials/) &nbsp;•&nbsp;[Livelabs](https://apexapps.oracle.com/pls/apex/f?p=133:180:112501098061930::::wid:3724) &nbsp;•&nbsp;[Slack Channel](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/queries) 
+  [What's New](https://github.com/oracle-devrel/cd3-automation-toolkit/releases/tag/v2025.1.3) &nbsp;•&nbsp;[Excel Templates](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/excel-templates/) &nbsp;•&nbsp;[CD3 Docs](https://oracle-devrel.github.io/cd3-automation-toolkit/)&nbsp;•&nbsp; [Watch & Learn](https://www.youtube.com/playlist?list=PLPIzp-E1msrbJ3WawXVhzimQnLw5iafcp) &nbsp;•&nbsp;[Blogs & Tutorials](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/tutorials/) &nbsp;•&nbsp;[Livelabs](https://apexapps.oracle.com/pls/apex/f?p=133:180:112501098061930::::wid:3724) &nbsp;•&nbsp;[Slack Channel](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/queries) 
 
 <br>
 

cd3_automation_toolkit/Database/create_terraform_mysql_configuration.py

@@ -65,6 +65,9 @@ def make_config_value(config):
     for i in df.index:
         # Get values from row
         region = str(df.loc[i, 'Region']).strip()
+        if (region in commonTools.endNames):
+            break
+        region = region.strip().lower()
         compartment_name = str(df.loc[i, 'Compartment Name']).strip()
         display_name = str(df.loc[i, 'Display Name']).strip()
         description = str(df.loc[i, 'Description']).strip()

cd3_automation_toolkit/Database/create_terraform_mysql_db.py

@@ -50,7 +50,7 @@ def create_terraform_mysql_db(inputfile, outdir, service_dir, prefix, ct):
 
     # Iterate over rows
     for i in df.index:
-        region = str(df.loc[i, 'Region']).strip().lower()
+        region = str(df.loc[i, 'Region']).strip()
 
         if (region in commonTools.endNames):
             break

cd3_automation_toolkit/Database/templates/mysql-template

@@ -25,10 +25,6 @@ mysql_db_system = {
             mysql_db_system_fault_domain                 = "{{ fault_domain }}"
             configuration_compartment_id = "{{ configuration_compartment_id }}"
             configuration_id = "{{ configuration_id }}"
-            {% if depends_on_mysql_configuration %}
-            # Add explicit depends_on to ensure configuration is created first
-            depends_on = ["module.mysql_configurations[\"{{ configuration_id }}\"]"]
-            {% endif %}
             mysql_shape_name                   = "{{ shape }}"
             vcn_names                     = "{{ vcn_names }}"
             subnet_id                    = "{{ subnet_id }}"

cd3_automation_toolkit/DeveloperServices/OKE/templates/nodepool-template

@@ -39,6 +39,12 @@ nodepools = {
         {% if boot_volume_size_in_gbs %}
         boot_volume_size_in_gbs = {{ boot_volume_size_in_gbs }}
         {% endif %}
+        {% if is_pv_encryption_in_transit_enabled %}
+        is_pv_encryption_in_transit_enabled = {{ is_pv_encryption_in_transit_enabled }}
+        {% endif %}
+        {% if init_script_path %}
+        init_script_path = "{{ init_script_path }}"
+        {% endif %}
         {% if oke_labels and oke_labels != 'nan' and defined_tags != '' and oke_labels != [['nan']] %}
         {% if oke_labels[0] %}
         initial_node_labels = {

cd3_automation_toolkit/Identity/Users/export_users_nonGreenField.py

@@ -112,11 +112,14 @@ def export_users(inputfile, outdir, service_dir, config, signer, ct,export_domai
             domain_name = domain_key.split("@")[1]
             domain_client = oci.identity_domains.IdentityDomainsClient(config=config, signer=signer,retry_strategy=oci.retry.DEFAULT_RETRY_STRATEGY,
                                                                        service_endpoint=idcs_endpoint)
-            list_users_response = domain_client.list_users() # change this to pagination once api supports
-            users = list_users_response.data.resources
-            while list_users_response.has_next_page:
-                list_users_response = domain_client.list_users(page=list_users_response.next_page)
-                users.extend(list_users_response.data.resources)
+            users = []
+            next_page = None
+            while True:
+                response = domain_client.list_users(page=next_page)
+                users.extend(response.data.resources)
+                if not response.next_page or len(users) == response.data.total_results:
+                    break
+                next_page = response.next_page
 
             index = 0
             for user in users:

cd3_automation_toolkit/Identity/export_identity_nonGreenField.py

@@ -359,13 +359,18 @@ def process_group(grp_info, members_list,membership_id_list, domain_name, is_dyn
                 domain_name = domain_key.split("@")[1]
                 domain_client = oci.identity_domains.IdentityDomainsClient(config=config, signer=signer,retry_strategy=oci.retry.DEFAULT_RETRY_STRATEGY,
                                                                            service_endpoint=idcs_endpoint)
-                list_groups_response = domain_client.list_groups(attributes=['members'], attribute_sets=['all'])
-                groups = list_groups_response.data.resources
-                page_done = []
-                while list_groups_response.has_next_page and list_groups_response.next_page not in page_done:
-                    page_done.append(list_groups_response.next_page)
-                    list_groups_response = domain_client.list_groups(attributes=['members'], attribute_sets=['all'],page=list_groups_response.next_page)
-                    groups.extend(list_groups_response.data.resources)
+                groups = []
+                next_page = None
+                while True:
+                    response = domain_client.list_groups(
+                        attributes=['members'],
+                        attribute_sets=['all'],
+                        page=next_page
+                    )
+                    groups.extend(response.data.resources)
+                    if not response.next_page or len(groups) == response.data.total_results:
+                        break
+                    next_page = response.next_page
 
                 for grp_info in groups:
                     if grp_info.display_name in ["Domain_Administrators", "All Domain Users", "Administrators"]:
@@ -374,16 +379,18 @@ def process_group(grp_info, members_list,membership_id_list, domain_name, is_dyn
                     members_list = [section.name for section in grp_info.members if section and section.name] if grp_info.members else []
                     importCommands, values_for_column_groups = process_group(grp_info, members_list,[], domain_name, is_dynamic=False, importCommands=importCommands, values_for_column_groups=values_for_column_groups)
 
-                dyngroups_response = domain_client.list_dynamic_resource_groups(attributes=['matching_rule'],
-                                                                                attribute_sets=['all']
-                                                                                )
-                dyngroups = dyngroups_response.data.resources
-                while dyngroups_response.has_next_page:
-                    dyngroups_response = domain_client.list_dynamic_resource_groups(attributes=['matching_rule'],
-                                                                                    attribute_sets=['all'],
-                                                                                    page=dyngroups_response.next_page
-                                                                                    )
-                    dyngroups.extend(dyngroups_response.data.resources)
+                dyngroups = []
+                next_page = None
+                while True:
+                    response = domain_client.list_dynamic_resource_groups(
+                        attributes=['matching_rule'],
+                        attribute_sets=['all'],
+                        page=next_page
+                    )
+                    dyngroups.extend(response.data.resources)
+                    if not response.next_page or len(dyngroups) == response.data.total_results:
+                        break
+                    next_page = response.next_page
 
                 for dg in dyngroups:
                     total_g += 1

cd3_automation_toolkit/Network/BaseNetwork/exportRoutetable.py

@@ -209,7 +209,11 @@ def insert_values(routetable,values_for_column,region,comp_name,name,routerule,e
 
         elif (routerule != None and col_header == 'Route Destination Object'):
             network_entity_id = routerule.network_entity_id
-            network_entity_name = get_network_entity_name(config, signer, network_entity_id,export_tags)
+            try:
+                network_entity_name = get_network_entity_name(config, signer, network_entity_id,export_tags)
+            except Exception as e:
+                print("\nCheck route rules for Route Table: '"+routetable.display_name+"' and Re-Try")
+                exit(1)
             values_for_column[col_header].append(network_entity_name)
             if ('internetgateway' in network_entity_id):
                 if (routerule.destination not in values_for_vcninfo['igw_destinations']):
@@ -240,7 +244,11 @@ def insert_values_drg(routetable,import_drg_route_distribution_name,values_for_c
 
         elif (routerule != None and col_header == 'Next Hop Attachment'):
             next_hop_attachment_id=routerule.next_hop_drg_attachment_id
-            network_entity_name = get_network_entity_name(config, signer, next_hop_attachment_id,export_tags)
+            try:
+                network_entity_name = get_network_entity_name(config, signer, next_hop_attachment_id,export_tags)
+            except Exception as e:
+                print("\nCheck route rules for DRG Route Table: '"+routetable.display_name+"' and Re-Try")
+                exit(1)
             values_for_column_drg[col_header].append(network_entity_name)
 
         else:

cd3_automation_toolkit/Network/BaseNetwork/export_network_nonGreenField.py

@@ -9,6 +9,7 @@
 from .exportSeclist import export_seclist
 from .exportNSG import export_nsg
 import subprocess as sp
+from pathlib import Path
 
 sys.path.append(os.getcwd() + "/..")
 from commonTools import *
@@ -348,6 +349,9 @@ def get_drg_rt_name(drg_rpc_attachment_list, source_rpc_id, rpc_source_client, d
 
 def get_rpc_resources(source_region, SOURCE_RPC_LIST, dest_rpc_dict, rpc_source_client, ct, values_for_column,
                       ntk_compartment_name, outdir, drg_info, drg_attachment_info, state_rpc):
+    # Get path to OCI_Regions file relative to current rpc script
+    oci_regions_path = Path(__file__).resolve().parents[2] / "OCI_Regions"
+
     # Variables
     dest_rpc_drg_name = ""
     src_drg_rt_name = ""
@@ -456,7 +460,8 @@ def get_comp_details(comp_data):
                                 # Fetch Dest DRG RT name, id
                                 if dest_drg_rpc_attachment_list.data:
                                     dest_drg_rt_name, dest_drg_rt_id = get_drg_rt_name(dest_drg_rpc_attachment_list,
-                                                                                       dest_rpc_id, client,dest_rpc.drg_id)
+                                                                                       dest_rpc_id, client,
+                                                                                       dest_rpc.drg_id)
 
                                     if dest_drg_rt_name is not None:
                                         # Fetch source DRG import route distribution id, name
@@ -467,15 +472,24 @@ def get_comp_details(comp_data):
                                             dest_import_rt_info = client.get_drg_route_distribution(
                                                 drg_route_distribution_id=dest_drg_rt_import_dist_id)
                                             dest_drg_rt_dist_info = dest_import_rt_info
-                                            dest_drg_rt_import_dist_name = getattr(dest_import_rt_info.data, "display_name")
+                                            dest_drg_rt_import_dist_name = getattr(dest_import_rt_info.data,
+                                                                                   "display_name")
                                             dest_import_rt_statements = client.list_drg_route_distribution_statements(
                                                 drg_route_distribution_id=dest_drg_rt_import_dist_id)
 
-                                tf_resource = f'module.rpcs[\\"{rpc_tf_name}\\"].oci_core_remote_peering_connection.{source_region.lower()}_{region.lower()}_requester_rpc[\\"region\\"]'
+                                source_region_for_tf = next(
+                                    line.split(':')[1].strip().replace("-", "_") for line in open(oci_regions_path) if
+                                    line.startswith(f"{source_region.lower()}:"))
+
+                                region_for_tf = next(
+                                    line.split(':')[1].strip().replace("-", "_") for line in open(oci_regions_path) if
+                                    line.startswith(f"{region.lower()}:"))
+
+                                tf_resource = f'module.rpcs[\\"{rpc_tf_name}\\"].oci_core_remote_peering_connection.{source_region_for_tf}_{region_for_tf}_requester_rpc[\\"region\\"]'
                                 if tf_resource not in state_rpc["resources"]:
                                     importCommands_rpc["global"].write(
                                         f'\n{tf_or_tofu} import "{tf_resource}" {str(source_rpc_id)}')
-                                tf_resource = f'module.rpcs[\\"{rpc_tf_name}\\"].oci_core_remote_peering_connection.{source_region.lower()}_{region.lower()}_accepter_rpc[\\"region\\"]'
+                                tf_resource = f'module.rpcs[\\"{rpc_tf_name}\\"].oci_core_remote_peering_connection.{source_region_for_tf}_{region_for_tf}_accepter_rpc[\\"region\\"]'
                                 if tf_resource not in state_rpc["resources"]:
                                     importCommands_rpc["global"].write(
                                         f'\n{tf_or_tofu} import "{tf_resource}" {str(dest_rpc_id)}')
@@ -589,6 +603,7 @@ def get_comp_details(comp_data):
     rpc_safe_file["global"].close()
 
 
+
 def export_major_objects(inputfile, outdir, service_dir, config, signer, ct, export_compartments=[], export_regions=[],
                          export_tags=[]):
     global sheet_dict_vcns