Automation Toolkit Release v2025.1.0

Author: xs2suruchi

Dockerfile

@@ -16,8 +16,8 @@ ARG USER_GID=$USER_UID
 # Whether to download Provider as part of image creation
 ARG DOWNLOAD_PROVIDER=YES
 # TF Provider version
-ARG TF_OCI_PROVIDER=6.15.0
-ARG TF_NULL_PROVIDER=3.2.1
+ARG TF_OCI_PROVIDER=6.30.0
+ARG TF_NULL_PROVIDER=3.2.3
 
 RUN microdnf install -y sudo && \
     groupadd --gid $USER_GID $USERNAME && \

cd3_automation_toolkit/Database/export_exa_vmclusters_nonGreenField.py

@@ -81,7 +81,7 @@ def print_exa_vmcluster(region, vnc_client,exa_infra, exa_vmcluster, key_name,va
             values_for_column[col_header].append(key_name)
         elif col_header == 'Client Network Details':
             values_for_column[col_header].append(client_network)
-        elif col_header == 'Backup Network Detailse':
+        elif col_header == 'Backup Network Details':
             values_for_column[col_header].append(backup_network)
         elif (col_header == "NSGs"):
             values_for_column[col_header].append(nsg_names)

cd3_automation_toolkit/Identity/Users/export_users_nonGreenField.py

@@ -143,12 +143,12 @@ def export_users(inputfile, outdir, service_dir, config, signer, ct,export_domai
                     display_name = user_info.display_name
                     email = None
                     recovery_email = None
-
-                    for email_info in user_info.emails:
-                        if email_info.primary:
-                            email = email_info.value
-                        elif email_info.type == "recovery":
-                            recovery_email = email_info.value
+                    if hasattr(user_info.emails, "email_info"):
+                        for email_info in user_info.emails:
+                            if email_info.primary:
+                                email = email_info.value
+                            elif email_info.type == "recovery":
+                                recovery_email = email_info.value
 
 
                     tf_name = commonTools.check_tf_variable(username)

cd3_automation_toolkit/Identity/export_identity_nonGreenField.py

@@ -361,7 +361,9 @@ def process_group(grp_info, members_list,membership_id_list, domain_name, is_dyn
                                                                            service_endpoint=idcs_endpoint)
                 list_groups_response = domain_client.list_groups(attributes=['members'], attribute_sets=['all'])
                 groups = list_groups_response.data.resources
-                while list_groups_response.has_next_page:
+                page_done = []
+                while list_groups_response.has_next_page and list_groups_response.next_page not in page_done:
+                    page_done.append(list_groups_response.next_page)
                     list_groups_response = domain_client.list_groups(attributes=['members'], attribute_sets=['all'],page=list_groups_response.next_page)
                     groups.extend(list_groups_response.data.resources)
 

cd3_automation_toolkit/Release-Notes

@@ -2,12 +2,12 @@
 CD3 Automation Toolkit Tag v2025.1.0
 Apr 4th, 2024
 -------------------------------------
-1. Introduced new service support - MySQL Database.
+1. Introduced new service - MySQL Database.
 2. Included Tunnel Inspection for OCI Network Firewall.
-3. Introduced filter for tags while export of resources.
-4. Moved compartment filter before showing xport options to make it common for all services.
+3. Introduced filtering of resources using tags during export workflow.
+4. Moved compartment filter before showing export options to make it common for all services when using toolkit with CLI.
 5. Running Create/Modify Network through Jenkins will automatically call export of rules after successful terraform apply.
-   This was a manua process earlier.
+   This was a manual process earlier.
 
 
 

cd3_automation_toolkit/Storage/BlockVolume/export_blockvolumes_nonGreenField.py

@@ -139,7 +139,7 @@ def print_blockvolumes(region, BVOLS, bvol, compute, ct, values_for_column, ntk_
             source_ocids[commonTools.check_tf_variable(blockvols.display_name.strip())] = tmp_key
         autotune_type = ''
         max_vpus_per_gb = ''
-        if len(blockvols.autotune_policies) == 0:
+        if blockvols.autotune_policies == None:
             autotune_type = ''
             max_vpus_per_gb = ''
         elif len(blockvols.autotune_policies) == 1:

cd3_automation_toolkit/user-scripts/tenancyconfig.properties

@@ -85,6 +85,7 @@ oci_devops_git_repo_name=
 # Customer Secret Key will be created for this user for S3 credentials of the bucket.
 # When left empty, it will be fetched from $(user_ocid) for $(auth_mechanism) as api_key.
 # Format: <domainName>/<userName>@<tenancyName> eg oracleidentitycloudservice/[email protected]@ocitenant
+# Users in Custom Domain are not supported as of now.
 oci_devops_git_user=
 
 # When left empty, same key file from $(key_path) used for $(auth_mechanism) as api_key will be copied to

jenkins_install/scriptler/scripts/ValidateParams.groovy

@@ -10,7 +10,7 @@ def validate_params(Workflow,MainOptions,SubOptions,SubChildOptions,AdditionalFi
     "DNS Management":["Add/Modify/Delete DNS Views/Zones/Records", "Add/Modify/Delete DNS Resolvers"],
     "Compute":["Add/Modify/Delete Dedicated VM Hosts", "Add/Modify/Delete Instances/Boot Backup Policy"],
     "Storage":["Add/Modify/Delete Block Volumes/Block Backup Policy", "Add/Modify/Delete File Systems", "Add/Modify/Delete Object Storage Buckets"],
-    "Database":["Add/Modify/Delete Virtual Machine or Bare Metal DB Systems", "Add/Modify/Delete EXA Infra and EXA VM Clusters", "Add/Modify/Delete ADBs"],
+    "Database":["Add/Modify/Delete Virtual Machine or Bare Metal DB Systems", "Add/Modify/Delete EXA Infra and EXA VM Clusters", "Add/Modify/Delete ADBs", "Add/Modify/Delete MySQL DBs"],
     "Load Balancers":["Add/Modify/Delete Load Balancers", "Add/Modify/Delete Network Load Balancers"],
     "Management Services":["Add/Modify/Delete Notifications", "Add/Modify/Delete Events", "Add/Modify/Delete Alarms", "Add/Modify/Delete ServiceConnectors"],
     "Developer Services":["Upload current terraform files/state to Resource Manager", "Add/Modify/Delete OKE Cluster and Nodepools"],
@@ -28,7 +28,7 @@ def validate_params(Workflow,MainOptions,SubOptions,SubChildOptions,AdditionalFi
     "Export DNS Management":["Export DNS Views/Zones/Records", "Export DNS Resolvers"],
     "Export Compute":["Export Dedicated VM Hosts", "Export Instances (excludes instances launched by OKE)"],
     "Export Storage":["Export Block Volumes/Block Backup Policy", "Export File Systems", "Export Object Storage Buckets"],
-    "Export Databases":["Export Virtual Machine or Bare Metal DB Systems", "Export EXA Infra and EXA VMClusters", "Export ADBs"],
+    "Export Databases":["Export Virtual Machine or Bare Metal DB Systems", "Export EXA Infra and EXA VMClusters", "Export ADBs", "Export MySQL DBs"],
     "Export Load Balancers":["Export Load Balancers", "Export Network Load Balancers"],
     "Export Management Services":["Export Notifications", "Export Events", "Export Alarms", "Export Service Connectors"],
     "Export Developer Services":["Export OKE cluster and Nodepools"],

jenkins_install/setUpOCI.groovy

@@ -5,12 +5,31 @@ def exportNetworkRules(stage_name) {
     return {
         stage("${stage_name}") {
                 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
-                sh """
+                labelledShell( label: 'Executing setUpOCI python script', script: """
                     cd /cd3user/oci_tools/cd3_automation_toolkit
                     python setUpOCI.py --devops True --main_options "Network" --sub_options "Security Rules,Route Rules,DRG Route Rules" --sub_child_options "Export Security Rules (From OCI into SecRulesinOCI sheet),Add/Modify/Delete Security Rules (Reads SecRulesinOCI sheet),Export Route Rules (From OCI into RouteRulesinOCI sheet),Add/Modify/Delete Route Rules (Reads RouteRulesinOCI sheet),Export DRG Route Rules (From OCI into DRGRouteRulesinOCI sheet),Add/Modify/Delete DRG Route Rules (Reads DRGRouteRulesinOCI sheet)" --add_filter "comp_filter=,[],@," ${env.prop_file}
-                """
-                file_path = sh(script: "grep '^cd3file' ${env.prop_file}| cut -d'=' -f2", returnStdout: true).trim()
-                file_name = sh(script:"echo '${file_path}'| rev|cut -d '/' -f1 | rev", returnStdout: true).trim()
+                """)
+                script {
+                    git_status = labelledShell( label: 'Check git status', script: 'cd ${prefix_dir}/terraform_files; git status --porcelain | wc -l', returnStdout: true).trim()
+                    // Check if anything to commit
+                    if ("${git_status}" > 0) {
+                        labelledShell( label: 'Performing git commit to develop', script: '''
+                            set +x
+                            cd ${prefix_dir}/terraform_files
+                            echo "-----start timestamp-----"
+                            time_stamp="$(date +%m-%d-%Y-%H-%M-%S)"
+                            commit_msg="commit for setUpOCI build ${BUILD_NUMBER}"
+                            git add -A .
+                            git commit -m "${commit_msg}"
+                            git push origin develop
+                         ''')
+                    }else {
+                        echo 'Nothing to commit. Skipping further stages.'
+                    }
+                }
+
+                file_path = sh(script: "set +x; grep '^cd3file' ${env.prop_file}| cut -d'=' -f2", returnStdout: true).trim()
+                file_name = sh(script:"set +x; echo '${file_path}'| rev|cut -d '/' -f1 | rev", returnStdout: true).trim()
                 sh """
                 set +x
                 cp '${file_path}' '${WORKSPACE}/${file_name}'
@@ -40,9 +59,7 @@ def generateStage(job) {
             //println(job_exec_details.getFullProjectName())
             //println((job_exec_details.getFullProjectName()).split("/")[3])
             if (!["ABORTED","FAILURE"].contains(job_exec_details.getResult()) && ["apply","network"].contains((job_exec_details.getFullProjectName()).split("/")[3])) {
-                println("first condition passed")
                 if ( SubOptions.contains('Create Network') || SubOptions.contains('Modify Network') )  {
-                    println("Calling export rules")
                     def stage_name = "Export Network Rules"
                     parallel([stage_name : exportNetworkRules(stage_name)])
                 }