Automation Toolkit Release v2024.2.2

bhlohumi · bhlohumi · commit 69037cd026ec · 2024-04-24T21:41:57.000+05:30
diff --git a/cd3_automation_toolkit/ManagementServices/EventsAndNotifications/export_events_notifications_nonGreenField.py b/cd3_automation_toolkit/ManagementServices/EventsAndNotifications/export_events_notifications_nonGreenField.py
@@ -118,15 +118,16 @@ def print_events(values_for_column_events, region, ntk_compartment_name, event,
                    data = str(condition["data"])
                else:
                    data = "{}"
-               for val in condition["eventType"]:
-                 if "oraclecloud" in val:
-                   service = val.split("com.oraclecloud.")[1]
-                 elif "oracle" in val:
-                   service = val.split("com.oracle.")[1]
-                 event_prod = service.split('.', 1)[0]
-                 event_res = service.split('.', 1)[1]
-                 if ( action_name != "" ):
-                     events_rows(values_for_column_events, region, ntk_compartment_name, event_name, event_desc, action_type, action_is_enabled, action_description, event_prod, event_res,data,  event_is_enabled, action_name, event, event_info)
+               if "eventType" in condition:
+                   for val in condition["eventType"]:
+                     if "oraclecloud" in val:
+                       service = val.split("com.oraclecloud.")[1]
+                     elif "oracle" in val:
+                       service = val.split("com.oracle.")[1]
+                     event_prod = service.split('.', 1)[0]
+                     event_res = service.split('.', 1)[1]
+                     if ( action_name != "" ):
+                         events_rows(values_for_column_events, region, ntk_compartment_name, event_name, event_desc, action_type, action_is_enabled, action_description, event_prod, event_res,data,  event_is_enabled, action_name, event, event_info)
           if ( i > 0 and action_name != ""):
              events_rows(values_for_column_events, region, ntk_compartment_name, event_name, event_desc, action_type, action_is_enabled, action_description, event_prod, event_res,data,  event_is_enabled, action_name, event, event_info)
           i = i + 1
diff --git a/cd3_automation_toolkit/ManagementServices/Monitoring/create_terraform_alarms.py b/cd3_automation_toolkit/ManagementServices/Monitoring/create_terraform_alarms.py
@@ -161,7 +161,7 @@ def create_terraform_alarms(inputfile, outdir, service_dir, prefix, ct):
 
 
         # Write all info to TF string
-        tfStr[region]=tfStr[region][:-1] +alarms_template.render(tempStr)
+        tfStr[region]=tfStr[region][:-2] +alarms_template.render(tempStr)
 
     # Write to output
     for reg in ct.all_regions:
diff --git a/cd3_automation_toolkit/ManagementServices/Monitoring/templates/alarms-template b/cd3_automation_toolkit/ManagementServices/Monitoring/templates/alarms-template
@@ -22,7 +22,13 @@ alarms = {
         query = "{{ query }}"
         severity = "{{ severity }}"
         {% if body and body != "" %}
+        {% if '\n' not in body %}
         body = "{{ body }}"
+        {% else %}
+        body = <<-EOF
+        {{ body }}
+        EOF
+        {% endif %}
         {% endif %}
 
         {% if message_format and message_format != "" %}
diff --git a/cd3_automation_toolkit/cd3Validator.py b/cd3_automation_toolkit/cd3Validator.py
@@ -1397,9 +1397,10 @@ def validate_buckets(filename, comp_ids):
                     log(f'ROW {i + 3} : Value of "Auto Tiering" can be only either "Enabled" or "Disabled".')
                     buckets_invalid_check = True
 
+            # Check for the Object Versioning column
             if columnname == 'Object Versioning':
-                if columnvalue.lower() not in ['enabled','disabled']:
-                    log(f'ROW {i + 3} : Value of "Object Versioning" can be only either "Enabled" or "Disabled".')
+                if columnvalue.lower() not in ['enabled', 'disabled']:
+                    log(f'ROW {i + 3} : Value of "Object Versioning" can only be "Enabled" or "Disabled".')
                     buckets_invalid_check = True
 
             if columnname == 'Emit Object Events':
@@ -1438,69 +1439,74 @@ def validate_buckets(filename, comp_ids):
             #Check for the retention policy details
             if columnname == 'Retention Rules':
                 rule_values = columnvalue.split("\n")
-                retention_rules = []
-                for rule in rule_values:
-                    rule_components = rule.split("::")
-                    if len(rule_components) >= 1:
-                        retention_rule_display_name = rule_components[0]
-                        time_unit = None
-                        time_amount = None
-                        time_rule_locked = None
-
-                        if len(rule_components) >= 2:
-                            if rule_components[1].lower() == 'indefinite':
-                                time_amount = None
-                            else:
-                                time_amount = rule_components[1]
-                                if not time_amount.isdigit():
-                                    log(f'ROW {i + 3} : "time_amount" of retention rule is not in valid format. It should be an "integer" or "indefinite".')
-                                    buckets_invalid_check = True
-                                    continue
+                if rule_values and str(dfbuckets.loc[i, 'Object Versioning']).strip().lower() == 'enabled':
+                    log(f'ROW {i + 3} : Retention policy cannot be created when Object Versioning is enabled.')
+                    buckets_invalid_check = True
+
+                elif rule_values and str(dfbuckets.loc[i, 'Object Versioning']).strip().lower() == 'disabled':
+                    retention_rules = []
+                    for rule in rule_values:
+                        rule_components = rule.split("::")
+                        if len(rule_components) >= 1:
+                            retention_rule_display_name = rule_components[0]
+                            time_unit = None
+                            time_amount = None
+                            time_rule_locked = None
+
+                            if len(rule_components) >= 2:
+                                if rule_components[1].lower() == 'indefinite':
+                                    time_amount = None
                                 else:
-                                    time_amount = int(time_amount)
+                                    time_amount = rule_components[1]
+                                    if not time_amount.isdigit():
+                                        log(f'ROW {i + 3} : "time_amount" of retention rule is not in valid format. It should be an "integer" or "indefinite".')
+                                        buckets_invalid_check = True
+                                        continue
+                                    else:
+                                        time_amount = int(time_amount)
 
-                        if len(rule_components) >= 3:
-                            time_unit = rule_components[2].upper()
-                            if time_unit not in ('DAYS', 'YEARS'):
-                                log(f'ROW {i + 3} : "time_unit" of retention rule is not in valid format. It should be either DAYS or YEARS.')
-                                buckets_invalid_check = True
-                            else:
-                                # If time_unit is valid, set the flag to True for processing time_rule_locked
-                                process_time_rule_locked = True
-
-                        if len(rule_components) == 4 and process_time_rule_locked:
-                            time_rule_locked = rule_components[3]
-                            if time_rule_locked.endswith(".000Z"):
-                                time_rule_locked = time_rule_locked[:-5] + "Z"
-                            elif not re.match(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z|\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z",time_rule_locked):
-                                # Convert from "dd-mm-yyyy" to "YYYY-MM-DDThh:mm:ssZ" format
-                                if re.match(r"\d{2}-\d{2}-\d{4}", time_rule_locked):
-                                    try:
-                                        datetime_obj = datetime.datetime.strptime(time_rule_locked, "%d-%m-%Y")
-                                        time_rule_locked = datetime_obj.strftime("%Y-%m-%dT%H:%M:%SZ")
-                                    except ValueError:
+                            if len(rule_components) >= 3:
+                                time_unit = rule_components[2].upper()
+                                if time_unit not in ('DAYS', 'YEARS'):
+                                    log(f'ROW {i + 3} : "time_unit" of retention rule is not in valid format. It should be either DAYS or YEARS.')
+                                    buckets_invalid_check = True
+                                else:
+                                    # If time_unit is valid, set the flag to True for processing time_rule_locked
+                                    process_time_rule_locked = True
+
+                            if len(rule_components) == 4 and process_time_rule_locked:
+                                time_rule_locked = rule_components[3]
+                                if time_rule_locked.endswith(".000Z"):
+                                    time_rule_locked = time_rule_locked[:-5] + "Z"
+                                elif not re.match(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z|\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z",time_rule_locked):
+                                    # Convert from "dd-mm-yyyy" to "YYYY-MM-DDThh:mm:ssZ" format
+                                    if re.match(r"\d{2}-\d{2}-\d{4}", time_rule_locked):
+                                        try:
+                                            datetime_obj = datetime.datetime.strptime(time_rule_locked, "%d-%m-%Y")
+                                            time_rule_locked = datetime_obj.strftime("%Y-%m-%dT%H:%M:%SZ")
+                                        except ValueError:
+                                            log(f'ROW {i + 3} : "time_rule_locked" of retention rule is not in valid format. It should be in the format "dd-mm-yyyy".')
+                                            buckets_invalid_check = True
+                                            continue
+                                    else:
                                         log(f'ROW {i + 3} : "time_rule_locked" of retention rule is not in valid format. It should be in the format "dd-mm-yyyy".')
                                         buckets_invalid_check = True
                                         continue
-                                else:
-                                    log(f'ROW {i + 3} : "time_rule_locked" of retention rule is not in valid format. It should be in the format "dd-mm-yyyy".')
+                                # Parse the time_rule_locked into a datetime object
+                                try:
+                                    time_rule_locked_datetime = datetime.datetime.strptime(time_rule_locked, "%Y-%m-%dT%H:%M:%SZ")
+                                except ValueError:
+                                    log(f'ROW {i + 3} : "time_rule_locked" of retention rule is not in valid format. It should be in the format "YYYY-MM-DDThh:mm:ssZ".')
                                     buckets_invalid_check = True
                                     continue
-                            # Parse the time_rule_locked into a datetime object
-                            try:
-                                time_rule_locked_datetime = datetime.datetime.strptime(time_rule_locked, "%Y-%m-%dT%H:%M:%SZ")
-                            except ValueError:
-                                log(f'ROW {i + 3} : "time_rule_locked" of retention rule is not in valid format. It should be in the format "YYYY-MM-DDThh:mm:ssZ".')
-                                buckets_invalid_check = True
-                                continue
 
-                            # Calculate the difference between current time and time_rule_locked
-                            time_difference = time_rule_locked_datetime - current_time
+                                # Calculate the difference between current time and time_rule_locked
+                                time_difference = time_rule_locked_datetime - current_time
 
-                            # Check if the difference is less than 14 days
-                            if time_difference.days < 14:
-                                log(f'ROW {i + 3} : "time_rule_locked" of retention rule must be more than 14 days from the current time.')
-                                buckets_invalid_check = True
+                                # Check if the difference is less than 14 days
+                                if time_difference.days < 14:
+                                    log(f'ROW {i + 3} : "time_rule_locked" of retention rule must be more than 14 days from the current time.')
+                                    buckets_invalid_check = True
 
             # Check for the Lifecycle Policy Details
             if lifecycle_input == True:
@@ -1545,6 +1551,7 @@ def validate_buckets(filename, comp_ids):
                             if time_unit not in ['days','years']:
                                 log(f'ROW {i + 3} : Invalid time amount. "Lifecycle Rule Period" must be "DAYS" or "YEARS".')
                                 buckets_invalid_check = True
+
                         else:
                             log(f'ROW {i + 3} : Invalid format for  "Lifecycle Rule Period" ')
                             buckets_invalid_check = True
diff --git a/cd3_automation_toolkit/setUpOCI.py b/cd3_automation_toolkit/setUpOCI.py
@@ -1300,7 +1300,6 @@ def start_cis_scan(outdir, prefix, config_file):
         if user_input!='':
             cmd = "{}".format(user_input)
     split = str.split(cmd)
-
     dirname = prefix + "_cis_report"
     resource = "cis_report"
     out_rep = outdir + '/'+ dirname
diff --git a/cd3_automation_toolkit/user-scripts/createTenancyConfig.py b/cd3_automation_toolkit/user-scripts/createTenancyConfig.py
@@ -148,10 +148,10 @@ def create_devops_resources(config,signer):
 def update_devops_config(prefix,git_config_file, repo_ssh_url,files_in_repo,dir_values,devops_user,devops_user_key,devops_dir,ct):
     # create git config file
     file = open(git_config_file, "w")
-    file.write("Host devops.scmservice.*.oci.oraclecloud.com\n "
-                "StrictHostKeyChecking no\n "
-                "User "+str(devops_user)+"\n "
-                "IdentityFile "+str(devops_user_key)+"\n")
+    file.write("Host devops.scmservice.*.oci"+cloud_domain+"\n "
+               "StrictHostKeyChecking no\n "
+               "User " + str(devops_user) + "\n "
+                                            "IdentityFile " + str(devops_user_key) + "\n")
 
     file.close()
 
@@ -205,6 +205,8 @@ def update_devops_config(prefix,git_config_file, repo_ssh_url,files_in_repo,dir_
         cfg = yaml.dump(cfg, stream=yaml_file, default_flow_style=False, sort_keys=False)
     # Clean repo config if exists and initiate git repo
     subprocess.run(['git', 'init'], cwd=devops_dir,stdout=DEVNULL)
+    subprocess.run(['git', 'config', '--global', 'init.defaultBranch', "main"], cwd=devops_dir)
+    subprocess.run(['git', 'config', '--global', 'safe.directory', devops_dir], cwd=devops_dir)
     f = open(devops_dir + ".gitignore", "w")
     git_ignore_file_data = ".DS_Store\n*tfstate*\n*terraform*\ntfplan.out\ntfplan.json\n*backup*\ntf_import_commands*\n*cis_report*\n*showoci_report*\n*.safe\n*stacks.zip\n*cd3Validator*"
     f.write(git_ignore_file_data)
@@ -255,7 +257,6 @@ def update_devops_config(prefix,git_config_file, repo_ssh_url,files_in_repo,dir_
 
     subprocess.run(['git', 'config','--global','user.email',devops_user], cwd=devops_dir)
     subprocess.run(['git', 'config', '--global', 'user.name', devops_user], cwd=devops_dir)
-    subprocess.run(['git', 'config', '--global', 'init.defaultBranch', "main"], cwd=devops_dir)
     commit_id='None'
     try:
         subprocess.run(['git', 'commit', '-m','Initial commit from createTenancyConfig.py'], cwd=devops_dir,stdout=DEVNULL)
@@ -349,11 +350,17 @@ def create_bucket(config, signer):
     user=''
     _key_path=''
     fingerprint=''
+    cloud_domain=''
 
     tenancy = config.get('Default', 'tenancy_ocid').strip()
     if tenancy == "" or tenancy == "\n":
         print("Tenancy ID cannot be left empty...Exiting !!")
         exit(1)
+    if ("ocid1.tenancy.oc1" in tenancy):
+        cloud_domain=".oraclecloud.com"
+    else:
+        cloud_domain=".oraclegovcloud.com"
+
 
     auth_mechanism = config.get('Default', 'auth_mechanism').strip().lower()
     if auth_mechanism == "" or auth_mechanism == "\n" or (auth_mechanism!='api_key' and auth_mechanism!='session_token' and auth_mechanism!='instance_principal'):
@@ -673,7 +680,7 @@ def create_bucket(config, signer):
         elif line.__contains__("region   = "):
             global_backend_file_data += "    region   = \"" + bucket_region + "\"\n"
         elif line.__contains__("endpoint = "):
-            global_backend_file_data += "    endpoint = \"https://" + namespace + ".compat.objectstorage." + bucket_region + ".oraclecloud.com\"\n"
+            global_backend_file_data += "    endpoint = \"https://" + namespace + ".compat.objectstorage." + bucket_region + cloud_domain + "\"\n"
         elif line.__contains__("shared_credentials_file     = "):
             global_backend_file_data += "    shared_credentials_file     = \"" + s3_credential_file_path + "\"\n"
         else:
diff --git a/cd3_automation_toolkit/user-scripts/tenancyconfig.properties b/cd3_automation_toolkit/user-scripts/tenancyconfig.properties
@@ -74,15 +74,14 @@ use_oci_devops_git=no
 # in ${region}.
 oci_devops_git_repo_name=
 
-# User Details to perform GIT operations in OCI Devops GIT Repo; Mandatory when using $(auth_mechanism) as instance_principal
+# User Details to perform GIT operations in OCI Devops GIT Repo and Remote Terraform State Management; Mandatory when using $(auth_mechanism) as instance_principal
 # or session_token
-# Format: <domainName>/<userName>@<tenancyName> eg oracleidentitycloudservice/devopsuser@oracle.com@ocitenant
+# Customer Secret Key will be created for this user for S3 credentials of the bucket.
 # When left empty, it will be fetched from $(user_ocid) for $(auth_mechanism) as api_key.
-# Customer Secret Key will also be configured for this user for S3 credentials of the bucket when $(auth_mechanism) is
-# instance_principal or session_token
+# Format: <domainName>/<userName>@<tenancyName> eg oracleidentitycloudservice/devopsuser@oracle.com@ocitenant
 oci_devops_git_user=
+
 # When left empty, same key file from $(key_path) used for $(auth_mechanism) as api_key will be copied to
 # /cd3user/tenancies/<customer_name>/ and used for GIT Operations.
-# Make sure the api key file permissions are rw(600) for cd3user
 oci_devops_git_key=
 
