Merge pull request #123 from oracle-devrel/develop

Automation Toolkit Release v2024.4.0

Author: bhlohumi

Dockerfile

@@ -20,6 +20,7 @@ RUN microdnf install -y sudo && \
 USER $USERNAME
 WORKDIR /cd3user/oci_tools/
 COPY cd3_automation_toolkit cd3_automation_toolkit/
+COPY othertools othertools/
 
 WORKDIR /cd3user/
 

OCIWorkVMStack/scripts/installToolkit.sh

@@ -75,7 +75,7 @@ user_id=$(echo "$metadata" | jq -r '.metadata.current_user_ocid')
 cust_name=$(echo "$metadata" | jq -r '.metadata.tenancy_name')
 tenancy_id=$(echo "$metadata" | jq -r '.metadata.tenancy_ocid')
 config_region=$(echo "$metadata" | jq -r '.metadata.config_region')
-sudo sed -c -i "s/customer_name=.*/customer_name=$cust_name/" $tenancyconfig_properties
+sudo sed -c -i "s/prefix=.*/prefix=$cust_name/" $tenancyconfig_properties
 sudo sed -c -i "s/tenancy_ocid=.*/tenancy_ocid=$tenancy_id/" $tenancyconfig_properties
 sudo sed -c -i "s/region=.*/region=$config_region/" $tenancyconfig_properties
 sudo sed -c -i "s/user_ocid=.*/user_ocid=$user_id/" $tenancyconfig_properties

README.md

@@ -7,12 +7,12 @@
 
 <br>
 
-  [What's New](https://github.com/oracle-devrel/cd3-automation-toolkit/releases/tag/v2024.3.2) &nbsp;•&nbsp;[Excel Templates](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/excel-templates/) &nbsp;•&nbsp;[CD3 Docs](https://oracle-devrel.github.io/cd3-automation-toolkit/)&nbsp;•&nbsp; [Watch & Learn](https://www.youtube.com/playlist?list=PLPIzp-E1msrbJ3WawXVhzimQnLw5iafcp) &nbsp;•&nbsp;[Blogs & Tutorials](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/tutorials/) &nbsp;•&nbsp;[Livelabs](https://apexapps.oracle.com/pls/apex/f?p=133:180:112501098061930::::wid:3724) &nbsp;•&nbsp;[Slack Channel](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/queries)
+  [What's New](https://github.com/oracle-devrel/cd3-automation-toolkit/releases/tag/v2024.4.0) &nbsp;•&nbsp;[Excel Templates](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/excel-templates/) &nbsp;•&nbsp;[CD3 Docs](https://oracle-devrel.github.io/cd3-automation-toolkit/)&nbsp;•&nbsp; [Watch & Learn](https://www.youtube.com/playlist?list=PLPIzp-E1msrbJ3WawXVhzimQnLw5iafcp) &nbsp;•&nbsp;[Blogs & Tutorials](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/tutorials/) &nbsp;•&nbsp;[Livelabs](https://apexapps.oracle.com/pls/apex/f?p=133:180:112501098061930::::wid:3724) &nbsp;•&nbsp;[Slack Channel](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/queries)
 
 <br>
 
 
-CD3 stands for **Cloud Deployment Design Deliverable**. The CD3 Automation toolkit enables you to effortlessly Build, Export and Manage OCI (Oracle Cloud Infrastruture) resources by converting Excel templates to fully functional Terraform modules within minutes ⚡️⚡️ .
+CD3 stands for **Cloud Deployment Design Deliverable**. The CD3 Automation toolkit enables you to effortlessly Build, Export and Manage OCI (Oracle Cloud Infrastructure) resources by converting Excel templates to fully functional Terraform modules within minutes ⚡️⚡️ .
 
 Additionally, the toolkit also supports seamless resource management using OCI DevOps GIT service and Jenkins Pipelines.
 

cd3_automation_toolkit/Compute/create_terraform_dedicatedhosts.py

@@ -130,9 +130,6 @@ def create_terraform_dedicatedhosts(inputfile, outdir, service_dir,prefix, ct):
             tfStr[reg] = template.render(count=0, region=reg).replace(src, tfStr[reg] + "\n" + src)
             tfStr[reg] = "".join([s for s in tfStr[reg].strip().splitlines(True) if s.strip("\r\n").strip()])
 
-            resource = sheetName.lower()
-            commonTools.backup_file(reg_out_dir + "/", resource, auto_tfvars_filename)
-
             # Write to TF file
             outfile = reg_out_dir + "/" + auto_tfvars_filename
             tfStr[reg] = "".join([s for s in tfStr[reg].strip().splitlines(True) if s.strip("\r\n").strip()])

cd3_automation_toolkit/Compute/create_terraform_instances.py

@@ -53,7 +53,7 @@ def create_terraform_instances(inputfile, outdir, service_dir, prefix, ct):
         tfStr[eachregion] = ''
         boot_policy_tfStr[eachregion] = ''
 
-    subnets = parseSubnets(filename)
+    #subnets = parseSubnets(filename)
 
     for i in df.index:
         region = str(df.loc[i, 'Region'])
@@ -93,9 +93,9 @@ def create_terraform_instances(inputfile, outdir, service_dir, prefix, ct):
                 df.loc[i, 'Shape']).lower() == 'nan' or str(df.loc[i, 'Compartment Name']).lower() == 'nan' or str(
             df.loc[i, 'Pub Address']).lower() == 'nan' or str(
             df.loc[i, 'Availability Domain(AD1|AD2|AD3)']).lower() == 'nan' or str(
-            df.loc[i, 'Subnet Name']).lower() == 'nan' or str(df.loc[i, 'Source Details']).lower() == 'nan'):
+            df.loc[i, 'Network Details']).lower() == 'nan' or str(df.loc[i, 'Source Details']).lower() == 'nan'):
             print(
-                "\nOne/All of the Column/Columns from Region, Shape, Compartment Name, Availability Domain, Display Name, Pub Address, Source Details and Subnet Name is empty in Instances sheet of CD3..exiting...Please check.")
+                "\nOne/All of the Column/Columns from Region, Shape, Compartment Name, Availability Domain, Display Name, Pub Address, Source Details and Network Details is empty in Instances sheet of CD3..exiting...Please check.")
             exit(1)
 
         # Perform the plugin match
@@ -137,25 +137,30 @@ def create_terraform_instances(inputfile, outdir, service_dir, prefix, ct):
                     columnvalue = columnvalue.strip()
                     tempdict = {'shape': [columnvalue]}
 
-            if columnname == "Subnet Name":
-                subnet_tf_name = columnvalue.strip()
-                if ("ocid1.subnet.oc1" in subnet_tf_name):
-                    network_compartment_id = ""
+            subnet_id = ''
+            network_compartment_id = ''
+            vcn_name = ''
+            if columnname == "Network Details":
+                columnvalue = columnvalue.strip()
+                if ("ocid1.subnet.oc" in columnvalue):
+                    network_compartment_id = "root"
                     vcn_name = ""
-                    subnet_id = subnet_tf_name
-                else:
-                    try:
-                        key = region, subnet_tf_name
-                        network_compartment_id = subnets.vcn_subnet_map[key][0]
-                        vcn_name = subnets.vcn_subnet_map[key][1]
-                        subnet_id = subnets.vcn_subnet_map[key][2]
-                    except Exception as e:
-                        print("Invalid Subnet Name specified for row " + str(
-                            i + 3) + ". It Doesnt exist in Subnets sheet. Exiting!!!")
+                    subnet_id = columnvalue
+                elif columnvalue.lower() != 'nan' and columnvalue.lower() != '':
+                    if len(columnvalue.split("@")) == 2:
+                        network_compartment_id = commonTools.check_tf_variable(columnvalue.split("@")[0].strip())
+                        vcn_subnet_name = columnvalue.split("@")[1].strip()
+                    else:
+                        network_compartment_id = commonTools.check_tf_variable(
+                            str(df.loc[i, 'Compartment Name']).strip())
+                        vcn_subnet_name = columnvalue
+                    if ("::" not in vcn_subnet_name):
+                        print("Invalid Network Details format specified for row " + str(i + 3) + ". Exiting!!!")
                         exit(1)
-
-                tempdict = {'network_compartment_id': commonTools.check_tf_variable(network_compartment_id),
-                            'vcn_name': vcn_name,
+                    else:
+                        vcn_name = vcn_subnet_name.split("::")[0].strip()
+                        subnet_id = vcn_subnet_name.split("::")[1].strip()
+                tempdict = {'network_compartment_id': network_compartment_id, 'vcn_name': vcn_name,
                             'subnet_id': subnet_id}
 
             if columnname == 'Display Name':
@@ -255,9 +260,6 @@ def create_terraform_instances(inputfile, outdir, service_dir, prefix, ct):
             tfStr[reg] = template.render(count=0, region=reg).replace(src, tfStr[reg] + "\n" + src)
             tfStr[reg] = "".join([s for s in tfStr[reg].strip().splitlines(True) if s.strip("\r\n").strip()])
 
-            resource = sheetName.lower()
-            commonTools.backup_file(reg_out_dir + "/", resource, auto_tfvars_filename)
-
             # Write to TF file
             outfile = reg_out_dir + "/" + auto_tfvars_filename
             tfStr[reg] = "".join([s for s in tfStr[reg].strip().splitlines(True) if s.strip("\r\n").strip()])

cd3_automation_toolkit/Compute/export_dedicatedvmhosts_nonGreenField.py

@@ -10,6 +10,7 @@
 
 import oci
 import os
+import subprocess as sp
 
 from oci.config import DEFAULT_LOCATION
 from commonTools import *
@@ -18,9 +19,11 @@
 oci_obj_names = {}
 
 
-def print_dedicatedvmhosts(region, dedicatedvmhost, values_for_column, ntk_compartment_name):
+def print_dedicatedvmhosts(region, dedicatedvmhost, values_for_column, ntk_compartment_name,state):
     dedicatedvmhost_tf_name = commonTools.check_tf_variable(dedicatedvmhost.display_name)
-    importCommands[region.lower()].write("\nterraform import \"module.dedicated-hosts[\\\"" +dedicatedvmhost_tf_name+ "\\\"].oci_core_dedicated_vm_host.dedicated_vm_host\" " + str(dedicatedvmhost.id))
+    tf_resource = f'module.dedicated-hosts[\\"{dedicatedvmhost_tf_name}\\"].oci_core_dedicated_vm_host.dedicated_vm_host'
+    if tf_resource not in state["resources"]:
+        importCommands[region.lower()] += f'\n{tf_or_tofu} import "{tf_resource}" {str(dedicatedvmhost.id)}'
 
     for col_header in values_for_column:
         if col_header == 'Region':
@@ -50,7 +53,10 @@ def export_dedicatedvmhosts(inputfile, outdir, service_dir, config, signer, ct,
     global importCommands
     global cd3file
     global reg
-    global values_for_column
+    global values_for_column,tf_or_tofu
+
+    tf_or_tofu = ct.tf_or_tofu
+    tf_state_list = [tf_or_tofu, "state", "list"]
 
 
     cd3file = inputfile
@@ -69,39 +75,48 @@ def export_dedicatedvmhosts(inputfile, outdir, service_dir, config, signer, ct,
     print("\nCD3 excel file should not be opened during export process!!!")
     print("Tabs- DedicatedVMHosts  will be overwritten during export process!!!\n")
 
+    # Fetch DVH Details
+    print("\nFetching details of Dedicated VM Hosts...")
+
     # Create backups
-    resource = 'tf_import_' + sheetName.lower()
-    file_name = 'tf_import_commands_' + sheetName.lower() + '_nonGF.sh'
+    resource = 'import_' + sheetName.lower()
+    file_name = 'import_commands_' + sheetName.lower() + '.sh'
+
     for reg in export_regions:
         script_file = f'{outdir}/{reg}/{service_dir}/'+file_name
         if (os.path.exists(script_file)):
             commonTools.backup_file(outdir + "/" + reg+"/"+service_dir, resource, file_name)
-        importCommands[reg] = open(script_file, "w")
-        importCommands[reg].write("#!/bin/bash")
-        importCommands[reg].write("\n")
-        importCommands[reg].write("terraform init")
-
-    # Fetch Block Volume Details
-    print("\nFetching details of Dedicated VM Hosts...")
+        importCommands[reg] = ''
 
-    for reg in export_regions:
-        importCommands[reg].write("\n\n######### Writing import for Dedicated VM Hosts #########\n\n")
         config.__setitem__("region", ct.region_dict[reg])
+        state = {'path': f'{outdir}/{reg}/{service_dir}', 'resources': []}
+        try:
+            byteOutput = sp.check_output(tf_state_list, cwd=state["path"], stderr=sp.DEVNULL)
+            output = byteOutput.decode('UTF-8').rstrip()
+            for item in output.split('\n'):
+                state["resources"].append(item.replace("\"", "\\\""))
+        except Exception as e:
+            pass
         region = reg.capitalize()
 
         compute_client = oci.core.ComputeClient(config=config,retry_strategy=oci.retry.DEFAULT_RETRY_STRATEGY, signer=signer)
 
         for ntk_compartment_name in export_compartments:
             dedicatedvmhosts = oci.pagination.list_call_get_all_results(compute_client.list_dedicated_vm_hosts,compartment_id=ct.ntk_compartment_ids[ntk_compartment_name], lifecycle_state="ACTIVE")
+
             for dedicatedvmhost in dedicatedvmhosts.data:
                 dedicatedvmhost=compute_client.get_dedicated_vm_host(dedicatedvmhost.id).data
-                print_dedicatedvmhosts(region, dedicatedvmhost,values_for_column, ntk_compartment_name)
+                print_dedicatedvmhosts(region, dedicatedvmhost,values_for_column, ntk_compartment_name,state)
 
-    # write data into file
+    # writing data
     for reg in export_regions:
-        script_file = f'{outdir}/{reg}/{service_dir}/'+file_name
-        with open(script_file, 'a') as importCommands[reg]:
-            importCommands[reg].write('\n\nterraform plan\n')
+        script_file = f'{outdir}/{reg}/{service_dir}/' + file_name
+
+        init_commands = f'\n######### Writing import for Dedicated VM Hosts #########\n\n#!/bin/bash\n{tf_or_tofu} init'
+        if importCommands[reg] != "":
+            importCommands[reg] += f'\n{tf_or_tofu} plan\n'
+            with open(script_file, 'a') as importCommandsfile:
+                importCommandsfile.write(init_commands + importCommands[reg])
 
     commonTools.write_to_cd3(values_for_column, cd3file, "DedicatedVMHosts")