Merge pull request #67 from oracle-devrel/develop

Develop

Author: xs2suruchi

README.md

@@ -58,6 +58,7 @@
 
    <li> <a href = "/cd3_automation_toolkit/documentation/user_guide/RestructuringOutDirectory.md">Grouping of generated Terraform files</a></li>
    <li> <a href = "/cd3_automation_toolkit/documentation/user_guide/learn_more/ResourceManagerUpload.md">OCI Resource Manager Upload</a></li>
+   <li> <a href = "/cd3_automation_toolkit/documentation/user_guide/learn_more/OPAForCompliance.md">OPA For Compliance with Terraform</a></li>
    <li> <a href = "/cd3_automation_toolkit/documentation/user_guide/learn_more/CISFeatures.md">Additional CIS Compliance Features</a></li>
    <li> <a href = "/cd3_automation_toolkit/documentation/user_guide/learn_more/SupportForCD3Validator.md">CD3 Validator Features</a></li>
 

cd3_automation_toolkit/Compute/create_terraform_instances.py

@@ -11,6 +11,7 @@
 
 import sys
 import os
+import fnmatch
 from oci.config import DEFAULT_LOCATION
 from pathlib import Path
 
@@ -100,6 +101,9 @@ def create_terraform_instances(inputfile, outdir, service_dir, prefix, config):
                 "\nOne/All of the Column/Columns from Region, Shape, Compartment Name, Availability Domain, Display Name, Pub Address, Source Details and Subnet Name is empty in Instances sheet of CD3..exiting...Please check.")
             exit(1)
 
+        # Perform the plugin match
+        plugin_match = None
+        plugin_column = fnmatch.filter(df.columns.tolist(), 'Plugin*')
         for columnname in dfcolumns:
 
             # Column value
@@ -115,6 +119,20 @@ def create_terraform_instances(inputfile, outdir, service_dir, prefix, config):
             if columnname.lower() in commonTools.tagColumns:
                 tempdict = commonTools.split_tag_values(columnname, columnvalue, tempdict)
 
+            if columnname in plugin_column:
+                columnvalue = columnvalue.strip()
+                if columnvalue != "":
+                    plugin_match = True
+                    tempdict = {'plugin_match': plugin_match}
+
+            if columnname == 'Network Type':
+                network_type = columnvalue.strip()
+                tempdict = {'network_type': network_type}
+
+            if columnname == 'Platform Config Type':
+                network_type = columnvalue.strip()
+                tempdict = {'platform_config_type': network_type}
+
             if columnname == 'Shape':
                 if ".Flex" not in columnvalue and ".Micro" not in columnvalue:
                     columnvalue = columnvalue.strip()
@@ -156,9 +174,9 @@ def create_terraform_instances(inputfile, outdir, service_dir, prefix, config):
                     if ("@" in columnvalue):
                         remote_execute = columnvalue.strip().split("@")
                         tempdict = {'remote_execute': remote_execute[1],
-                                'bastion_ip': remote_execute[0]}
+                                    'bastion_ip': remote_execute[0]}
                     else:
-                        tempdict = {'remote_execute': columnvalue.strip() }
+                        tempdict = {'remote_execute': columnvalue.strip()}
 
             if columnname == 'Custom Policy Compartment Name':
                 if columnvalue != "":
@@ -232,4 +250,3 @@ def create_terraform_instances(inputfile, outdir, service_dir, prefix, config):
             print(outfile + " for instances and boot volume backup policy has been created for region " + reg)
             oname.write(tfStr[reg])
             oname.close()
-

cd3_automation_toolkit/Compute/export_instances_nonGreenField.py

@@ -7,22 +7,17 @@
 import sys
 import oci
 import os
-from oci.config import DEFAULT_LOCATION
-from pathlib import Path
 sys.path.append(os.getcwd() + "/..")
 from commonTools import *
-from jinja2 import Environment, FileSystemLoader
 
 
 def adding_columns_values(region, ad, fd, vs, publicip, privateip, os_dname, shape, key_name, c_name,
-                          bkp_policy_name, nsgs, d_host, instance_data, values_for_column_instances,  bdet,
-                          cpcn,shape_config,vnic_info,vnic_defined_tags,vnic_freeform_tags,launch_options):
-    # print("CPCN=",cpcn)
+                          bkp_policy_name, nsgs, d_host, instance_data, values_for_column_instances, bdet,
+                          cpcn, shape_config, vnic_info, vnic_defined_tags, vnic_freeform_tags, launch_options,avail_config,ins_options,
+                          platform_config, plugin_config):
     for col_header in values_for_column_instances.keys():
         if (col_header == "Region"):
             values_for_column_instances[col_header].append(region)
-        #elif (col_header == "Hostname"):
-        #    values_for_column_instances[col_header].append(hostname)
         elif (col_header == "Availability Domain(AD1|AD2|AD3)"):
             values_for_column_instances[col_header].append(ad)
         elif (col_header == "Fault Domain"):
@@ -51,7 +46,16 @@ def adding_columns_values(region, ad, fd, vs, publicip, privateip, os_dname, sha
             values_for_column_instances[col_header].append(vnic_freeform_tags)
         elif (col_header == "VNIC Display Name"):
             values_for_column_instances[col_header].append(vnic_info.display_name)
-        elif (col_header == "Dedicated VM Host"):
+
+        elif(col_header.lower().startswith("plugin")):
+            col_temp = col_header.lower().replace("plugin","")
+            col_temp=col_temp.strip()
+            col_temp=col_temp.strip("_")
+            col_temp=col_temp.replace(" ","_")
+            col_temp = col_temp.replace("-", "_")
+            values_for_column_instances[col_header].append(plugin_config.get(col_temp))
+
+        elif(col_header == "Dedicated VM Host"):
             if (d_host == None):
                 values_for_column_instances[col_header].append('')
             else:
@@ -62,12 +66,12 @@ def adding_columns_values(region, ad, fd, vs, publicip, privateip, os_dname, sha
             values_for_column_instances = commonTools.export_tags(instance_data, col_header,
                                                                   values_for_column_instances)
         else:
-            oci_objs = [instance_data, bdet,shape_config,vnic_info,d_host,launch_options]
+            oci_objs = [instance_data, bdet, shape_config, vnic_info, d_host, launch_options, avail_config, ins_options, platform_config]
             values_for_column_instances = commonTools.export_extra_columns(oci_objs, col_header, sheet_dict_instances,
                                                                            values_for_column_instances)
 
 
-def find_vnic(ins_id, config,compartment_id):
+def find_vnic(ins_id, config, compartment_id):
     compute = oci.core.ComputeClient(config, retry_strategy=oci.retry.DEFAULT_RETRY_STRATEGY)
     #for comp in all_compartments:
     net = oci.pagination.list_call_get_all_results(compute.list_vnic_attachments, compartment_id=compartment_id,
@@ -76,8 +80,7 @@ def find_vnic(ins_id, config,compartment_id):
         return net
 
 
-
-def __get_instances_info(compartment_name, compartment_id, reg_name, config,display_names, ad_names,ct):
+def __get_instances_info(compartment_name, compartment_id, reg_name, config, display_names, ad_names, ct):
     config.__setitem__("region", ct.region_dict[reg_name])
     compute = oci.core.ComputeClient(config, retry_strategy=oci.retry.DEFAULT_RETRY_STRATEGY)
     network = oci.core.VirtualNetworkClient(config, retry_strategy=oci.retry.DEFAULT_RETRY_STRATEGY)
@@ -244,9 +247,21 @@ def __get_instances_info(compartment_name, compartment_id, reg_name, config,disp
                 os_dname = "bootVolume::" + os_dname
 
             launch_options = ins.launch_options
+            avail_config = ins.availability_config
+            ins_options = ins.instance_options
+            platform_data = ins.platform_config
+            plugins_config = getattr(ins.agent_config, 'plugins_config')
+            plugin_config = {}
+            if plugins_config is not None:
+                for item in plugins_config:
+                    plugin_config[getattr(item, 'name').lower().replace(" ","_").replace("-","_")] = getattr(item, 'desired_state')
+
             adding_columns_values(reg_name.title(), AD_name, ins_fd, vs, publicip, privateip, os_dname,
-                                      ins_shape, key_name, compartment_name, bkp_policy_name, nsg_names, dedicated_host,
-                                      ins, values_for_column_instances, bdet, cpcn,shape_config, vnic_info, vnic_defined_tags, vnic_freeform_tags, launch_options)
+                                  ins_shape, key_name, compartment_name, bkp_policy_name, nsg_names, dedicated_host,
+                                  ins, values_for_column_instances, bdet, cpcn, shape_config, vnic_info,
+                                  vnic_defined_tags, vnic_freeform_tags, launch_options,avail_config,ins_options,
+                                  platform_data, plugin_config)
+
 
 # Execution of the code begins here
 def export_instances(inputfile, outdir, service_dir,config,ct, export_compartments=[], export_regions=[],display_names=[],ad_names=[]):

cd3_automation_toolkit/Compute/templates/instances-template

@@ -87,9 +87,78 @@ instances = {
             create_is_pv_encryption_in_transit_enabled = {{ create_is_pv_encryption_in_transit_enabled }}
             {% endif %}
 
-            {% if update_is_pv_encryption_in_transit_enabled %}
-            update_is_pv_encryption_in_transit_enabled = {{ update_is_pv_encryption_in_transit_enabled }}
+            {% if plugin_match %}
+            plugins_details = {
+                    {% if plugin_bastion %}
+                        Bastion = "{{ plugin_bastion.upper() }}",
+                    {% endif %}
+                    {% if plugin_management_agent %}
+                        "Management Agent" =  "{{ plugin_management_agent.upper() }}",
+                    {% endif %}
+                    {% if plugin_vulnerability_scanning %}
+                        "Vulnerability Scanning" = "{{ plugin_vulnerability_scanning.upper() }}"
+                    {% endif %}
+                    {% if plugin_oracle_java_management_service %}
+                        "Oracle Java Management Service" = "{{ plugin_oracle_java_management_service.upper() }}"
+                    {% endif %}
+                    {% if plugin_os_management_service_agent %}
+                        "OS Management Service Agent"  = "{{ plugin_os_management_service_agent.upper() }}"
+                    {% endif %}
+                    {% if plugin_custom_logs_monitoring %}
+                        "Custom Logs Monitoring" = "{{ plugin_custom_logs_monitoring.upper() }}"
+                    {% endif %}
+                    {% if plugin_compute_rdma_gpu_monitoring %}
+                        "Compute RDMA GPU Monitoring" = "{{ plugin_compute_rdma_gpu_monitoring.upper() }}"
+                    {% endif %}
+                    {% if plugin_compute_instance_run_command %}
+                        "Compute Instance Run Command" = "{{ plugin_compute_instance_run_command.upper() }}"
+                    {% endif %}
+                    {% if plugin_compute_instance_monitoring %}
+                        "Compute Instance Monitoring" = "{{ plugin_compute_instance_monitoring.upper() }}"
+                    {% endif %}
+                    {% if plugin_compute_hpc_rdma_authentication %}
+                        "Compute HPC RDMA Authentication" = "{{ plugin_compute_hpc_rdma_authentication.upper() }}"
+                    {% endif %}
+                    {% if plugin_block_volume_management %}
+                        "Block Volume Management" = "{{ plugin_block_volume_management.upper() }}"
+                    {% endif %}
+                    {% if plugin_compute_hpc_rdma_auto_configuration %}
+                        "Compute HPC RDMA Auto-Configuration" = "{{ plugin_compute_hpc_rdma_auto_configuration.upper() }}"
+                    {% endif %}
+            }
+            {% endif %}
 
+            {% if network_type or update_is_pv_encryption_in_transit_enabled %}
+            launch_options = [{
+              {% if firmware %}
+                    firmware = "{{ firmware }}",
+              {% endif %}
+              {% if is_consistent_volume_naming_enabled %}
+                    is_consistent_volume_naming_enabled = "{{is_consistent_volume_naming_enabled}}",
+              {% endif %}
+              {% if update_is_pv_encryption_in_transit_enabled %}
+                    is_pv_encryption_in_transit_enabled = {{update_is_pv_encryption_in_transit_enabled}},
+              {% endif %}
+                    network_type = "{{network_type}}"
+            }]
+            {% endif %}
+
+            {% if platform_config_type %}
+            platform_config = [{
+                    config_type = "{{platform_config_type}}"
+              {% if is_secure_boot_enabled %}
+                    is_secure_boot_enabled = {{ is_secure_boot_enabled }},
+              {% endif %}
+              {% if is_measured_boot_enabled %}
+                    is_measured_boot_enabled = {{ is_measured_boot_enabled }},
+              {% endif %}
+              {% if is_trusted_platform_module_enabled %}
+                    is_trusted_platform_module_enabled = {{ is_trusted_platform_module_enabled }},
+              {% endif %}
+              {% if numa_nodes_per_socket %}
+                    numa_nodes_per_socket = "{{ numa_nodes_per_socket }}"
+              {% endif %}
+            }]
             {% endif %}
 
             {% if ssh_key_var_name != '' %}
@@ -105,7 +174,7 @@ instances = {
             {% endif %}
 
           {#  ##All optional params required for TF root module is below ##  #}
-          {#  {% if extended_metadata %}
+            {% if extended_metadata %}
             extended_metadata = "{{ extended_metadata }}"
             {% endif %}
 
@@ -133,10 +202,6 @@ instances = {
             is_monitoring_disabled = {{ is_monitoring_disabled }}
             {% endif %}
 
-            {% if plugins_details %}
-            plugins_details = "{{ plugins_details }}"
-            {% endif %}
-
             {% if is_live_migration_preferred %}
             is_live_migration_preferred = {{ is_live_migration_preferred }}
             {% endif %}
@@ -153,26 +218,6 @@ instances = {
             boot_volume_type = "{{ boot_volume_type }}"
             {% endif %}
 
-            {% if firmware %}
-            firmware = "{{ firmware }}"
-            {% endif %}
-
-            {% if is_consistent_volume_naming_enabled %}
-            is_consistent_volume_naming_enabled = {{ is_consistent_volume_naming_enabled }}
-            {% endif %}
-
-            {% if network_type %}
-            network_type = "{{ network_type }}"
-            {% endif %}
-
-            {% if remote_data_volume_type %}
-            remote_data_volume_type = "{{ remote_data_volume_type }}"
-            {% endif %}
-
-            {% if platform_config %}
-            platform_config = "{{ platform_config }}"
-            {% endif %}
-
             {% if ipxe_script %}
             ipxe_script = "{{ ipxe_script }}"
             {% endif %}

cd3_automation_toolkit/Database/create_terraform_exa_vmclusters.py

@@ -80,6 +80,7 @@ def create_terraform_exa_vmclusters(inputfile, outdir, service_dir, prefix, conf
         tempStr = {}
         tempdict = {}
         nsg_id = ''
+        db_server_id = ''
         backup_nsg_id = ''
 
         # Check if values are entered for mandatory fields
@@ -188,6 +189,25 @@ def create_terraform_exa_vmclusters(inputfile, outdir, service_dir, prefix, conf
                             c += 1
                 columnvalue = nsg_id
 
+            if columnname == 'DB Servers':
+                if columnvalue != '':
+                    db_servers = str(columnvalue).strip().split(",")
+                    if len(db_servers) == 1:
+                        for db_server in db_servers:
+                            db_server_id = "\"" + db_server.strip() + "\""
+
+                    elif len(db_servers) >= 2:
+                        c = 1
+                        for db_server in db_servers:
+                            data = "\"" + db_server.strip() + "\""
+
+                            if c == len(db_servers):
+                                db_server_id = db_server_id + data
+                            else:
+                                db_server_id = db_server_id + data + ","
+                            c += 1
+                columnvalue = db_server_id
+
 
             if columnname == 'Backup Network NSGs':
                 if columnvalue != '':

cd3_automation_toolkit/Database/export_exa_vmclusters_nonGreenField.py

@@ -20,7 +20,7 @@
 oci_obj_names = {}
 
 
-def print_exa_vmcluster(region, vnc_client,exa_infra, exa_vmcluster, key_name,values_for_column, ntk_compartment_name):
+def print_exa_vmcluster(region, vnc_client,exa_infra, exa_vmcluster, key_name,values_for_column, ntk_compartment_name, db_servers):
     exa_infra_tf_name = commonTools.check_tf_variable(exa_infra.display_name)
     exa_vmcluster_tf_name = commonTools.check_tf_variable(exa_vmcluster.display_name)
 
@@ -54,6 +54,7 @@ def print_exa_vmcluster(region, vnc_client,exa_infra, exa_vmcluster, key_name,va
 
     maintenance_window = exa_infra.maintenance_window
 
+
     importCommands[region.lower()].write("\nterraform import \"module.exa-vmclusters[\\\"" + exa_vmcluster_tf_name + "\\\"].oci_database_cloud_vm_cluster.exa_vmcluster\" " + str(exa_vmcluster.id))
 
     for col_header in values_for_column:
@@ -73,6 +74,8 @@ def print_exa_vmcluster(region, vnc_client,exa_infra, exa_vmcluster, key_name,va
             values_for_column[col_header].append(nsg_names)
         elif (col_header == "Backup Network NSGs"):
             values_for_column[col_header].append(backup_nsg_names)
+        elif (col_header == "DB Servers"):
+            values_for_column[col_header].append(db_servers)
         elif col_header.lower() in commonTools.tagColumns:
             values_for_column = commonTools.export_tags(exa_vmcluster, col_header, values_for_column)
         else:
@@ -163,7 +166,14 @@ def export_exa_vmclusters(inputfile, _outdir, service_dir, _config, ct, export_c
                         db_ssh_keys = json.dumps(db_ssh_keys)
                         db[key_name] = db_ssh_keys
 
-                        print_exa_vmcluster(region, vnc_client, exa_infra,exa_vmcluster,key_name, values_for_column, ntk_compartment_name_again)
+                        db_serverids = exa_vmcluster.db_servers
+                        db_servers = ''
+                        if (db_serverids is not None and len(db_serverids)):
+                            for db_server in db_serverids:
+                                db_server_name = db_client.get_db_server(exa_infra.id, db_server).data.display_name
+                                db_servers = db_server_name +","+db_servers
+
+                        print_exa_vmcluster(region, vnc_client, exa_infra,exa_vmcluster,key_name, values_for_column, ntk_compartment_name_again,db_servers)
 
         file = f'{outdir}/{reg}/{service_dir}/variables_{reg}.tf'
         # Read variables file data

cd3_automation_toolkit/Database/templates/exa-vmclusters-template

@@ -76,6 +76,30 @@ exa_vmclusters = {
             data_storage_percentage  = null
             {% endif %}
 
+            {% if db_node_storage_size_in_gbs %}
+            db_node_storage_size_in_gbs = {{ db_node_storage_size_in_gbs }}
+            {% else %}
+            db_node_storage_size_in_gbs  = null
+            {% endif %}
+
+            {% if memory_size_in_gbs %}
+            memory_size_in_gbs       = {{ memory_size_in_gbs }}
+            {% else %}
+            memory_size_in_gbs  = null
+            {% endif %}
+
+            {% if data_storage_size_in_tbs %}
+            data_storage_size_in_tbs = {{ data_storage_size_in_tbs }}
+            {% else %}
+            data_storage_size_in_tbs  = null
+            {% endif %}
+
+            {% if db_servers %}
+            db_servers               = [{{ db_servers }}]
+            {% else %}
+            db_servers               = []
+            {% endif %}
+
             {% if nsgs %}
             nsg_ids                  = [{{ nsgs }}]
             {% else %}